mojiahui/WebApp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

create

Browse Source
This commit is contained in:
Yao
2024-12-20 17:49:45 +08:00
parent 86b0363ce1
commit 654d456c7d
7011 changed files with 1705926 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

556
Pods/abseil/absl/container/fixed_array.h generated Normal file
View File

@@ -0,0 +1,556 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// -----------------------------------------------------------------------------
// File: fixed_array.h
// -----------------------------------------------------------------------------
//
// A `FixedArray<T>` represents a non-resizable array of `T` where the length of
// the array can be determined at run-time. It is a good replacement for
// non-standard and deprecated uses of `alloca()` and variable length arrays
// within the GCC extension. (See
// https://gcc.gnu.org/onlinedocs/gcc/Variable-Length.html).
//
// `FixedArray` allocates small arrays inline, keeping performance fast by
// avoiding heap operations. It also helps reduce the chances of
// accidentally overflowing your stack if large input is passed to
// your function.
#ifndef ABSL_CONTAINER_FIXED_ARRAY_H_
#define ABSL_CONTAINER_FIXED_ARRAY_H_
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <initializer_list>
#include <iterator>
#include <limits>
#include <memory>
#include <new>
#include <type_traits>
#include "absl/algorithm/algorithm.h"
#include "absl/base/config.h"
#include "absl/base/dynamic_annotations.h"
#include "absl/base/internal/throw_delegate.h"
#include "absl/base/macros.h"
#include "absl/base/optimization.h"
#include "absl/base/port.h"
#include "absl/container/internal/compressed_tuple.h"
#include "absl/memory/memory.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
constexpr static auto kFixedArrayUseDefault = static_cast<size_t>(-1);
// -----------------------------------------------------------------------------
// FixedArray
// -----------------------------------------------------------------------------
//
// A `FixedArray` provides a run-time fixed-size array, allocating a small array
// inline for efficiency.
//
// Most users should not specify the `N` template parameter and let `FixedArray`
// automatically determine the number of elements to store inline based on
// `sizeof(T)`. If `N` is specified, the `FixedArray` implementation will use
// inline storage for arrays with a length <= `N`.
//
// Note that a `FixedArray` constructed with a `size_type` argument will
// default-initialize its values by leaving trivially constructible types
// uninitialized (e.g. int, int[4], double), and others default-constructed.
// This matches the behavior of c-style arrays and `std::array`, but not
// `std::vector`.
template <typename T, size_t N = kFixedArrayUseDefault,
typename A = std::allocator<T>>
class FixedArray {
static_assert(!std::is_array<T>::value || std::extent<T>::value > 0,
"Arrays with unknown bounds cannot be used with FixedArray.");
static constexpr size_t kInlineBytesDefault = 256;
using AllocatorTraits = std::allocator_traits<A>;
// std::iterator_traits isn't guaranteed to be SFINAE-friendly until C++17,
// but this seems to be mostly pedantic.
template <typename Iterator>
using EnableIfForwardIterator = absl::enable_if_t<std::is_convertible<
typename std::iterator_traits<Iterator>::iterator_category,
std::forward_iterator_tag>::value>;
static constexpr bool NoexceptCopyable() {
return std::is_nothrow_copy_constructible<StorageElement>::value &&
absl::allocator_is_nothrow<allocator_type>::value;
}
static constexpr bool NoexceptMovable() {
return std::is_nothrow_move_constructible<StorageElement>::value &&
absl::allocator_is_nothrow<allocator_type>::value;
}
static constexpr bool DefaultConstructorIsNonTrivial() {
return !absl::is_trivially_default_constructible<StorageElement>::value;
}
public:
using allocator_type = typename AllocatorTraits::allocator_type;
using value_type = typename AllocatorTraits::value_type;
using pointer = typename AllocatorTraits::pointer;
using const_pointer = typename AllocatorTraits::const_pointer;
using reference = value_type&;
using const_reference = const value_type&;
using size_type = typename AllocatorTraits::size_type;
using difference_type = typename AllocatorTraits::difference_type;
using iterator = pointer;
using const_iterator = const_pointer;
using reverse_iterator = std::reverse_iterator<iterator>;
using const_reverse_iterator = std::reverse_iterator<const_iterator>;
static constexpr size_type inline_elements =
(N == kFixedArrayUseDefault ? kInlineBytesDefault / sizeof(value_type)
: static_cast<size_type>(N));
FixedArray(const FixedArray& other) noexcept(NoexceptCopyable())
: FixedArray(other,
AllocatorTraits::select_on_container_copy_construction(
other.storage_.alloc())) {}
FixedArray(const FixedArray& other,
const allocator_type& a) noexcept(NoexceptCopyable())
: FixedArray(other.begin(), other.end(), a) {}
FixedArray(FixedArray&& other) noexcept(NoexceptMovable())
: FixedArray(std::move(other), other.storage_.alloc()) {}
FixedArray(FixedArray&& other,
const allocator_type& a) noexcept(NoexceptMovable())
: FixedArray(std::make_move_iterator(other.begin()),
std::make_move_iterator(other.end()), a) {}
// Creates an array object that can store `n` elements.
// Note that trivially constructible elements will be uninitialized.
explicit FixedArray(size_type n, const allocator_type& a = allocator_type())
: storage_(n, a) {
if (DefaultConstructorIsNonTrivial()) {
memory_internal::ConstructRange(storage_.alloc(), storage_.begin(),
storage_.end());
}
}
// Creates an array initialized with `n` copies of `val`.
FixedArray(size_type n, const value_type& val,
const allocator_type& a = allocator_type())
: storage_(n, a) {
memory_internal::ConstructRange(storage_.alloc(), storage_.begin(),
storage_.end(), val);
}
// Creates an array initialized with the size and contents of `init_list`.
FixedArray(std::initializer_list<value_type> init_list,
const allocator_type& a = allocator_type())
: FixedArray(init_list.begin(), init_list.end(), a) {}
// Creates an array initialized with the elements from the input
// range. The array's size will always be `std::distance(first, last)`.
// REQUIRES: Iterator must be a forward_iterator or better.
template <typename Iterator, EnableIfForwardIterator<Iterator>* = nullptr>
FixedArray(Iterator first, Iterator last,
const allocator_type& a = allocator_type())
: storage_(std::distance(first, last), a) {
memory_internal::CopyRange(storage_.alloc(), storage_.begin(), first, last);
}
~FixedArray() noexcept {
for (auto* cur = storage_.begin(); cur != storage_.end(); ++cur) {
AllocatorTraits::destroy(storage_.alloc(), cur);
}
}
// Assignments are deleted because they break the invariant that the size of a
// `FixedArray` never changes.
void operator=(FixedArray&&) = delete;
void operator=(const FixedArray&) = delete;
// FixedArray::size()
//
// Returns the length of the fixed array.
size_type size() const { return storage_.size(); }
// FixedArray::max_size()
//
// Returns the largest possible value of `std::distance(begin(), end())` for a
// `FixedArray<T>`. This is equivalent to the most possible addressable bytes
// over the number of bytes taken by T.
constexpr size_type max_size() const {
return (std::numeric_limits<difference_type>::max)() / sizeof(value_type);
}
// FixedArray::empty()
//
// Returns whether or not the fixed array is empty.
bool empty() const { return size() == 0; }
// FixedArray::memsize()
//
// Returns the memory size of the fixed array in bytes.
size_t memsize() const { return size() * sizeof(value_type); }
// FixedArray::data()
//
// Returns a const T* pointer to elements of the `FixedArray`. This pointer
// can be used to access (but not modify) the contained elements.
const_pointer data() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return AsValueType(storage_.begin());
}
// Overload of FixedArray::data() to return a T* pointer to elements of the
// fixed array. This pointer can be used to access and modify the contained
// elements.
pointer data() ABSL_ATTRIBUTE_LIFETIME_BOUND {
return AsValueType(storage_.begin());
}
// FixedArray::operator[]
//
// Returns a reference the ith element of the fixed array.
// REQUIRES: 0 <= i < size()
reference operator[](size_type i) ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(i < size());
return data()[i];
}
// Overload of FixedArray::operator()[] to return a const reference to the
// ith element of the fixed array.
// REQUIRES: 0 <= i < size()
const_reference operator[](size_type i) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(i < size());
return data()[i];
}
// FixedArray::at
//
// Bounds-checked access. Returns a reference to the ith element of the fixed
// array, or throws std::out_of_range
reference at(size_type i) ABSL_ATTRIBUTE_LIFETIME_BOUND {
if (ABSL_PREDICT_FALSE(i >= size())) {
base_internal::ThrowStdOutOfRange("FixedArray::at failed bounds check");
}
return data()[i];
}
// Overload of FixedArray::at() to return a const reference to the ith element
// of the fixed array.
const_reference at(size_type i) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
if (ABSL_PREDICT_FALSE(i >= size())) {
base_internal::ThrowStdOutOfRange("FixedArray::at failed bounds check");
}
return data()[i];
}
// FixedArray::front()
//
// Returns a reference to the first element of the fixed array.
reference front() ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(!empty());
return data()[0];
}
// Overload of FixedArray::front() to return a reference to the first element
// of a fixed array of const values.
const_reference front() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(!empty());
return data()[0];
}
// FixedArray::back()
//
// Returns a reference to the last element of the fixed array.
reference back() ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(!empty());
return data()[size() - 1];
}
// Overload of FixedArray::back() to return a reference to the last element
// of a fixed array of const values.
const_reference back() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(!empty());
return data()[size() - 1];
}
// FixedArray::begin()
//
// Returns an iterator to the beginning of the fixed array.
iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND { return data(); }
// Overload of FixedArray::begin() to return a const iterator to the
// beginning of the fixed array.
const_iterator begin() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return data(); }
// FixedArray::cbegin()
//
// Returns a const iterator to the beginning of the fixed array.
const_iterator cbegin() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return begin();
}
// FixedArray::end()
//
// Returns an iterator to the end of the fixed array.
iterator end() ABSL_ATTRIBUTE_LIFETIME_BOUND { return data() + size(); }
// Overload of FixedArray::end() to return a const iterator to the end of the
// fixed array.
const_iterator end() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return data() + size();
}
// FixedArray::cend()
//
// Returns a const iterator to the end of the fixed array.
const_iterator cend() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return end(); }
// FixedArray::rbegin()
//
// Returns a reverse iterator from the end of the fixed array.
reverse_iterator rbegin() ABSL_ATTRIBUTE_LIFETIME_BOUND {
return reverse_iterator(end());
}
// Overload of FixedArray::rbegin() to return a const reverse iterator from
// the end of the fixed array.
const_reverse_iterator rbegin() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_reverse_iterator(end());
}
// FixedArray::crbegin()
//
// Returns a const reverse iterator from the end of the fixed array.
const_reverse_iterator crbegin() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return rbegin();
}
// FixedArray::rend()
//
// Returns a reverse iterator from the beginning of the fixed array.
reverse_iterator rend() ABSL_ATTRIBUTE_LIFETIME_BOUND {
return reverse_iterator(begin());
}
// Overload of FixedArray::rend() for returning a const reverse iterator
// from the beginning of the fixed array.
const_reverse_iterator rend() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_reverse_iterator(begin());
}
// FixedArray::crend()
//
// Returns a reverse iterator from the beginning of the fixed array.
const_reverse_iterator crend() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return rend();
}
// FixedArray::fill()
//
// Assigns the given `value` to all elements in the fixed array.
void fill(const value_type& val) { std::fill(begin(), end(), val); }
// Relational operators. Equality operators are elementwise using
// `operator==`, while order operators order FixedArrays lexicographically.
friend bool operator==(const FixedArray& lhs, const FixedArray& rhs) {
return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
}
friend bool operator!=(const FixedArray& lhs, const FixedArray& rhs) {
return !(lhs == rhs);
}
friend bool operator<(const FixedArray& lhs, const FixedArray& rhs) {
return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(),
rhs.end());
}
friend bool operator>(const FixedArray& lhs, const FixedArray& rhs) {
return rhs < lhs;
}
friend bool operator<=(const FixedArray& lhs, const FixedArray& rhs) {
return !(rhs < lhs);
}
friend bool operator>=(const FixedArray& lhs, const FixedArray& rhs) {
return !(lhs < rhs);
}
template <typename H>
friend H AbslHashValue(H h, const FixedArray& v) {
return H::combine(H::combine_contiguous(std::move(h), v.data(), v.size()),
v.size());
}
private:
// StorageElement
//
// For FixedArrays with a C-style-array value_type, StorageElement is a POD
// wrapper struct called StorageElementWrapper that holds the value_type
// instance inside. This is needed for construction and destruction of the
// entire array regardless of how many dimensions it has. For all other cases,
// StorageElement is just an alias of value_type.
//
// Maintainer's Note: The simpler solution would be to simply wrap value_type
// in a struct whether it's an array or not. That causes some paranoid
// diagnostics to misfire, believing that 'data()' returns a pointer to a
// single element, rather than the packed array that it really is.
// e.g.:
//
// FixedArray<char> buf(1);
// sprintf(buf.data(), "foo");
//
// error: call to int __builtin___sprintf_chk(etc...)
// will always overflow destination buffer [-Werror]
//
template <typename OuterT, typename InnerT = absl::remove_extent_t<OuterT>,
size_t InnerN = std::extent<OuterT>::value>
struct StorageElementWrapper {
InnerT array[InnerN];
};
using StorageElement =
absl::conditional_t<std::is_array<value_type>::value,
StorageElementWrapper<value_type>, value_type>;
static pointer AsValueType(pointer ptr) { return ptr; }
static pointer AsValueType(StorageElementWrapper<value_type>* ptr) {
return std::addressof(ptr->array);
}
static_assert(sizeof(StorageElement) == sizeof(value_type), "");
static_assert(alignof(StorageElement) == alignof(value_type), "");
class NonEmptyInlinedStorage {
public:
StorageElement* data() { return reinterpret_cast<StorageElement*>(buff_); }
void AnnotateConstruct(size_type n);
void AnnotateDestruct(size_type n);
#ifdef ABSL_HAVE_ADDRESS_SANITIZER
void* RedzoneBegin() { return &redzone_begin_; }
void* RedzoneEnd() { return &redzone_end_ + 1; }
#endif // ABSL_HAVE_ADDRESS_SANITIZER
private:
ABSL_ADDRESS_SANITIZER_REDZONE(redzone_begin_);
alignas(StorageElement) char buff_[sizeof(StorageElement[inline_elements])];
ABSL_ADDRESS_SANITIZER_REDZONE(redzone_end_);
};
class EmptyInlinedStorage {
public:
StorageElement* data() { return nullptr; }
void AnnotateConstruct(size_type) {}
void AnnotateDestruct(size_type) {}
};
using InlinedStorage =
absl::conditional_t<inline_elements == 0, EmptyInlinedStorage,
NonEmptyInlinedStorage>;
// Storage
//
// An instance of Storage manages the inline and out-of-line memory for
// instances of FixedArray. This guarantees that even when construction of
// individual elements fails in the FixedArray constructor body, the
// destructor for Storage will still be called and out-of-line memory will be
// properly deallocated.
//
class Storage : public InlinedStorage {
public:
Storage(size_type n, const allocator_type& a)
: size_alloc_(n, a), data_(InitializeData()) {}
~Storage() noexcept {
if (UsingInlinedStorage(size())) {
InlinedStorage::AnnotateDestruct(size());
} else {
AllocatorTraits::deallocate(alloc(), AsValueType(begin()), size());
}
}
size_type size() const { return size_alloc_.template get<0>(); }
StorageElement* begin() const { return data_; }
StorageElement* end() const { return begin() + size(); }
allocator_type& alloc() { return size_alloc_.template get<1>(); }
const allocator_type& alloc() const {
return size_alloc_.template get<1>();
}
private:
static bool UsingInlinedStorage(size_type n) {
return n <= inline_elements;
}
#ifdef ABSL_HAVE_ADDRESS_SANITIZER
ABSL_ATTRIBUTE_NOINLINE
#endif // ABSL_HAVE_ADDRESS_SANITIZER
StorageElement* InitializeData() {
if (UsingInlinedStorage(size())) {
InlinedStorage::AnnotateConstruct(size());
return InlinedStorage::data();
} else {
return reinterpret_cast<StorageElement*>(
AllocatorTraits::allocate(alloc(), size()));
}
}
// `CompressedTuple` takes advantage of EBCO for stateless `allocator_type`s
container_internal::CompressedTuple<size_type, allocator_type> size_alloc_;
StorageElement* data_;
};
Storage storage_;
};
#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL
template <typename T, size_t N, typename A>
constexpr size_t FixedArray<T, N, A>::kInlineBytesDefault;
template <typename T, size_t N, typename A>
constexpr typename FixedArray<T, N, A>::size_type
FixedArray<T, N, A>::inline_elements;
#endif
template <typename T, size_t N, typename A>
void FixedArray<T, N, A>::NonEmptyInlinedStorage::AnnotateConstruct(
typename FixedArray<T, N, A>::size_type n) {
#ifdef ABSL_HAVE_ADDRESS_SANITIZER
if (!n) return;
ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(data(), RedzoneEnd(), RedzoneEnd(),
data() + n);
ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(RedzoneBegin(), data(), data(),
RedzoneBegin());
#endif // ABSL_HAVE_ADDRESS_SANITIZER
static_cast<void>(n); // Mark used when not in asan mode
}
template <typename T, size_t N, typename A>
void FixedArray<T, N, A>::NonEmptyInlinedStorage::AnnotateDestruct(
typename FixedArray<T, N, A>::size_type n) {
#ifdef ABSL_HAVE_ADDRESS_SANITIZER
if (!n) return;
ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(data(), RedzoneEnd(), data() + n,
RedzoneEnd());
ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(RedzoneBegin(), data(), RedzoneBegin(),
data());
#endif // ABSL_HAVE_ADDRESS_SANITIZER
static_cast<void>(n); // Mark used when not in asan mode
}
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_FIXED_ARRAY_H_

617
Pods/abseil/absl/container/flat_hash_map.h generated Normal file
View File

@@ -0,0 +1,617 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// -----------------------------------------------------------------------------
// File: flat_hash_map.h
// -----------------------------------------------------------------------------
//
// An `absl::flat_hash_map<K, V>` is an unordered associative container of
// unique keys and associated values designed to be a more efficient replacement
// for `std::unordered_map`. Like `unordered_map`, search, insertion, and
// deletion of map elements can be done as an `O(1)` operation. However,
// `flat_hash_map` (and other unordered associative containers known as the
// collection of Abseil "Swiss tables") contain other optimizations that result
// in both memory and computation advantages.
//
// In most cases, your default choice for a hash map should be a map of type
// `flat_hash_map`.
#ifndef ABSL_CONTAINER_FLAT_HASH_MAP_H_
#define ABSL_CONTAINER_FLAT_HASH_MAP_H_
#include <cstddef>
#include <new>
#include <type_traits>
#include <utility>
#include "absl/algorithm/container.h"
#include "absl/base/macros.h"
#include "absl/container/internal/container_memory.h"
#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export
#include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export
#include "absl/memory/memory.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
template <class K, class V>
struct FlatHashMapPolicy;
} // namespace container_internal
// -----------------------------------------------------------------------------
// absl::flat_hash_map
// -----------------------------------------------------------------------------
//
// An `absl::flat_hash_map<K, V>` is an unordered associative container which
// has been optimized for both speed and memory footprint in most common use
// cases. Its interface is similar to that of `std::unordered_map<K, V>` with
// the following notable differences:
//
// * Requires keys that are CopyConstructible
// * Requires values that are MoveConstructible
// * Supports heterogeneous lookup, through `find()`, `operator[]()` and
// `insert()`, provided that the map is provided a compatible heterogeneous
// hashing function and equality operator.
// * Invalidates any references and pointers to elements within the table after
// `rehash()` and when the table is moved.
// * Contains a `capacity()` member function indicating the number of element
// slots (open, deleted, and empty) within the hash map.
// * Returns `void` from the `erase(iterator)` overload.
//
// By default, `flat_hash_map` uses the `absl::Hash` hashing framework.
// All fundamental and Abseil types that support the `absl::Hash` framework have
// a compatible equality operator for comparing insertions into `flat_hash_map`.
// If your type is not yet supported by the `absl::Hash` framework, see
// absl/hash/hash.h for information on extending Abseil hashing to user-defined
// types.
//
// Using `absl::flat_hash_map` at interface boundaries in dynamically loaded
// libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may
// be randomized across dynamically loaded libraries.
//
// NOTE: A `flat_hash_map` stores its value types directly inside its
// implementation array to avoid memory indirection. Because a `flat_hash_map`
// is designed to move data when rehashed, map values will not retain pointer
// stability. If you require pointer stability, or if your values are large,
// consider using `absl::flat_hash_map<Key, std::unique_ptr<Value>>` instead.
// If your types are not moveable or you require pointer stability for keys,
// consider `absl::node_hash_map`.
//
// Example:
//
// // Create a flat hash map of three strings (that map to strings)
// absl::flat_hash_map<std::string, std::string> ducks =
// {{"a", "huey"}, {"b", "dewey"}, {"c", "louie"}};
//
// // Insert a new element into the flat hash map
// ducks.insert({"d", "donald"});
//
// // Force a rehash of the flat hash map
// ducks.rehash(0);
//
// // Find the element with the key "b"
// std::string search_key = "b";
// auto result = ducks.find(search_key);
// if (result != ducks.end()) {
// std::cout << "Result: " << result->second << std::endl;
// }
template <class K, class V,
class Hash = absl::container_internal::hash_default_hash<K>,
class Eq = absl::container_internal::hash_default_eq<K>,
class Allocator = std::allocator<std::pair<const K, V>>>
class flat_hash_map : public absl::container_internal::raw_hash_map<
absl::container_internal::FlatHashMapPolicy<K, V>,
Hash, Eq, Allocator> {
using Base = typename flat_hash_map::raw_hash_map;
public:
// Constructors and Assignment Operators
//
// A flat_hash_map supports the same overload set as `std::unordered_map`
// for construction and assignment:
//
// * Default constructor
//
// // No allocation for the table's elements is made.
// absl::flat_hash_map<int, std::string> map1;
//
// * Initializer List constructor
//
// absl::flat_hash_map<int, std::string> map2 =
// {{1, "huey"}, {2, "dewey"}, {3, "louie"},};
//
// * Copy constructor
//
// absl::flat_hash_map<int, std::string> map3(map2);
//
// * Copy assignment operator
//
// // Hash functor and Comparator are copied as well
// absl::flat_hash_map<int, std::string> map4;
// map4 = map3;
//
// * Move constructor
//
// // Move is guaranteed efficient
// absl::flat_hash_map<int, std::string> map5(std::move(map4));
//
// * Move assignment operator
//
// // May be efficient if allocators are compatible
// absl::flat_hash_map<int, std::string> map6;
// map6 = std::move(map5);
//
// * Range constructor
//
// std::vector<std::pair<int, std::string>> v = {{1, "a"}, {2, "b"}};
// absl::flat_hash_map<int, std::string> map7(v.begin(), v.end());
flat_hash_map() {}
using Base::Base;
// flat_hash_map::begin()
//
// Returns an iterator to the beginning of the `flat_hash_map`.
using Base::begin;
// flat_hash_map::cbegin()
//
// Returns a const iterator to the beginning of the `flat_hash_map`.
using Base::cbegin;
// flat_hash_map::cend()
//
// Returns a const iterator to the end of the `flat_hash_map`.
using Base::cend;
// flat_hash_map::end()
//
// Returns an iterator to the end of the `flat_hash_map`.
using Base::end;
// flat_hash_map::capacity()
//
// Returns the number of element slots (assigned, deleted, and empty)
// available within the `flat_hash_map`.
//
// NOTE: this member function is particular to `absl::flat_hash_map` and is
// not provided in the `std::unordered_map` API.
using Base::capacity;
// flat_hash_map::empty()
//
// Returns whether or not the `flat_hash_map` is empty.
using Base::empty;
// flat_hash_map::max_size()
//
// Returns the largest theoretical possible number of elements within a
// `flat_hash_map` under current memory constraints. This value can be thought
// of the largest value of `std::distance(begin(), end())` for a
// `flat_hash_map<K, V>`.
using Base::max_size;
// flat_hash_map::size()
//
// Returns the number of elements currently within the `flat_hash_map`.
using Base::size;
// flat_hash_map::clear()
//
// Removes all elements from the `flat_hash_map`. Invalidates any references,
// pointers, or iterators referring to contained elements.
//
// NOTE: this operation may shrink the underlying buffer. To avoid shrinking
// the underlying buffer call `erase(begin(), end())`.
using Base::clear;
// flat_hash_map::erase()
//
// Erases elements within the `flat_hash_map`. Erasing does not trigger a
// rehash. Overloads are listed below.
//
// void erase(const_iterator pos):
//
// Erases the element at `position` of the `flat_hash_map`, returning
// `void`.
//
// NOTE: returning `void` in this case is different than that of STL
// containers in general and `std::unordered_map` in particular (which
// return an iterator to the element following the erased element). If that
// iterator is needed, simply post increment the iterator:
//
// map.erase(it++);
//
// iterator erase(const_iterator first, const_iterator last):
//
// Erases the elements in the open interval [`first`, `last`), returning an
// iterator pointing to `last`. The special case of calling
// `erase(begin(), end())` resets the reserved growth such that if
// `reserve(N)` has previously been called and there has been no intervening
// call to `clear()`, then after calling `erase(begin(), end())`, it is safe
// to assume that inserting N elements will not cause a rehash.
//
// size_type erase(const key_type& key):
//
// Erases the element with the matching key, if it exists, returning the
// number of elements erased (0 or 1).
using Base::erase;
// flat_hash_map::insert()
//
// Inserts an element of the specified value into the `flat_hash_map`,
// returning an iterator pointing to the newly inserted element, provided that
// an element with the given key does not already exist. If rehashing occurs
// due to the insertion, all iterators are invalidated. Overloads are listed
// below.
//
// std::pair<iterator,bool> insert(const init_type& value):
//
// Inserts a value into the `flat_hash_map`. Returns a pair consisting of an
// iterator to the inserted element (or to the element that prevented the
// insertion) and a bool denoting whether the insertion took place.
//
// std::pair<iterator,bool> insert(T&& value):
// std::pair<iterator,bool> insert(init_type&& value):
//
// Inserts a moveable value into the `flat_hash_map`. Returns a pair
// consisting of an iterator to the inserted element (or to the element that
// prevented the insertion) and a bool denoting whether the insertion took
// place.
//
// iterator insert(const_iterator hint, const init_type& value):
// iterator insert(const_iterator hint, T&& value):
// iterator insert(const_iterator hint, init_type&& value);
//
// Inserts a value, using the position of `hint` as a non-binding suggestion
// for where to begin the insertion search. Returns an iterator to the
// inserted element, or to the existing element that prevented the
// insertion.
//
// void insert(InputIterator first, InputIterator last):
//
// Inserts a range of values [`first`, `last`).
//
// NOTE: Although the STL does not specify which element may be inserted if
// multiple keys compare equivalently, for `flat_hash_map` we guarantee the
// first match is inserted.
//
// void insert(std::initializer_list<init_type> ilist):
//
// Inserts the elements within the initializer list `ilist`.
//
// NOTE: Although the STL does not specify which element may be inserted if
// multiple keys compare equivalently within the initializer list, for
// `flat_hash_map` we guarantee the first match is inserted.
using Base::insert;
// flat_hash_map::insert_or_assign()
//
// Inserts an element of the specified value into the `flat_hash_map` provided
// that a value with the given key does not already exist, or replaces it with
// the element value if a key for that value already exists, returning an
// iterator pointing to the newly inserted element. If rehashing occurs due
// to the insertion, all existing iterators are invalidated. Overloads are
// listed below.
//
// pair<iterator, bool> insert_or_assign(const init_type& k, T&& obj):
// pair<iterator, bool> insert_or_assign(init_type&& k, T&& obj):
//
// Inserts/Assigns (or moves) the element of the specified key into the
// `flat_hash_map`.
//
// iterator insert_or_assign(const_iterator hint,
// const init_type& k, T&& obj):
// iterator insert_or_assign(const_iterator hint, init_type&& k, T&& obj):
//
// Inserts/Assigns (or moves) the element of the specified key into the
// `flat_hash_map` using the position of `hint` as a non-binding suggestion
// for where to begin the insertion search.
using Base::insert_or_assign;
// flat_hash_map::emplace()
//
// Inserts an element of the specified value by constructing it in-place
// within the `flat_hash_map`, provided that no element with the given key
// already exists.
//
// The element may be constructed even if there already is an element with the
// key in the container, in which case the newly constructed element will be
// destroyed immediately. Prefer `try_emplace()` unless your key is not
// copyable or moveable.
//
// If rehashing occurs due to the insertion, all iterators are invalidated.
using Base::emplace;
// flat_hash_map::emplace_hint()
//
// Inserts an element of the specified value by constructing it in-place
// within the `flat_hash_map`, using the position of `hint` as a non-binding
// suggestion for where to begin the insertion search, and only inserts
// provided that no element with the given key already exists.
//
// The element may be constructed even if there already is an element with the
// key in the container, in which case the newly constructed element will be
// destroyed immediately. Prefer `try_emplace()` unless your key is not
// copyable or moveable.
//
// If rehashing occurs due to the insertion, all iterators are invalidated.
using Base::emplace_hint;
// flat_hash_map::try_emplace()
//
// Inserts an element of the specified value by constructing it in-place
// within the `flat_hash_map`, provided that no element with the given key
// already exists. Unlike `emplace()`, if an element with the given key
// already exists, we guarantee that no element is constructed.
//
// If rehashing occurs due to the insertion, all iterators are invalidated.
// Overloads are listed below.
//
// pair<iterator, bool> try_emplace(const key_type& k, Args&&... args):
// pair<iterator, bool> try_emplace(key_type&& k, Args&&... args):
//
// Inserts (via copy or move) the element of the specified key into the
// `flat_hash_map`.
//
// iterator try_emplace(const_iterator hint,
// const key_type& k, Args&&... args):
// iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args):
//
// Inserts (via copy or move) the element of the specified key into the
// `flat_hash_map` using the position of `hint` as a non-binding suggestion
// for where to begin the insertion search.
//
// All `try_emplace()` overloads make the same guarantees regarding rvalue
// arguments as `std::unordered_map::try_emplace()`, namely that these
// functions will not move from rvalue arguments if insertions do not happen.
using Base::try_emplace;
// flat_hash_map::extract()
//
// Extracts the indicated element, erasing it in the process, and returns it
// as a C++17-compatible node handle. Overloads are listed below.
//
// node_type extract(const_iterator position):
//
// Extracts the key,value pair of the element at the indicated position and
// returns a node handle owning that extracted data.
//
// node_type extract(const key_type& x):
//
// Extracts the key,value pair of the element with a key matching the passed
// key value and returns a node handle owning that extracted data. If the
// `flat_hash_map` does not contain an element with a matching key, this
// function returns an empty node handle.
//
// NOTE: when compiled in an earlier version of C++ than C++17,
// `node_type::key()` returns a const reference to the key instead of a
// mutable reference. We cannot safely return a mutable reference without
// std::launder (which is not available before C++17).
using Base::extract;
// flat_hash_map::merge()
//
// Extracts elements from a given `source` flat hash map into this
// `flat_hash_map`. If the destination `flat_hash_map` already contains an
// element with an equivalent key, that element is not extracted.
using Base::merge;
// flat_hash_map::swap(flat_hash_map& other)
//
// Exchanges the contents of this `flat_hash_map` with those of the `other`
// flat hash map, avoiding invocation of any move, copy, or swap operations on
// individual elements.
//
// All iterators and references on the `flat_hash_map` remain valid, excepting
// for the past-the-end iterator, which is invalidated.
//
// `swap()` requires that the flat hash map's hashing and key equivalence
// functions be Swappable, and are exchanged using unqualified calls to
// non-member `swap()`. If the map's allocator has
// `std::allocator_traits<allocator_type>::propagate_on_container_swap::value`
// set to `true`, the allocators are also exchanged using an unqualified call
// to non-member `swap()`; otherwise, the allocators are not swapped.
using Base::swap;
// flat_hash_map::rehash(count)
//
// Rehashes the `flat_hash_map`, setting the number of slots to be at least
// the passed value. If the new number of slots increases the load factor more
// than the current maximum load factor
// (`count` < `size()` / `max_load_factor()`), then the new number of slots
// will be at least `size()` / `max_load_factor()`.
//
// To force a rehash, pass rehash(0).
//
// NOTE: unlike behavior in `std::unordered_map`, references are also
// invalidated upon a `rehash()`.
using Base::rehash;
// flat_hash_map::reserve(count)
//
// Sets the number of slots in the `flat_hash_map` to the number needed to
// accommodate at least `count` total elements without exceeding the current
// maximum load factor, and may rehash the container if needed.
using Base::reserve;
// flat_hash_map::at()
//
// Returns a reference to the mapped value of the element with key equivalent
// to the passed key.
using Base::at;
// flat_hash_map::contains()
//
// Determines whether an element with a key comparing equal to the given `key`
// exists within the `flat_hash_map`, returning `true` if so or `false`
// otherwise.
using Base::contains;
// flat_hash_map::count(const Key& key) const
//
// Returns the number of elements with a key comparing equal to the given
// `key` within the `flat_hash_map`. note that this function will return
// either `1` or `0` since duplicate keys are not allowed within a
// `flat_hash_map`.
using Base::count;
// flat_hash_map::equal_range()
//
// Returns a closed range [first, last], defined by a `std::pair` of two
// iterators, containing all elements with the passed key in the
// `flat_hash_map`.
using Base::equal_range;
// flat_hash_map::find()
//
// Finds an element with the passed `key` within the `flat_hash_map`.
using Base::find;
// flat_hash_map::operator[]()
//
// Returns a reference to the value mapped to the passed key within the
// `flat_hash_map`, performing an `insert()` if the key does not already
// exist.
//
// If an insertion occurs and results in a rehashing of the container, all
// iterators are invalidated. Otherwise iterators are not affected and
// references are not invalidated. Overloads are listed below.
//
// T& operator[](const Key& key):
//
// Inserts an init_type object constructed in-place if the element with the
// given key does not exist.
//
// T& operator[](Key&& key):
//
// Inserts an init_type object constructed in-place provided that an element
// with the given key does not exist.
using Base::operator[];
// flat_hash_map::bucket_count()
//
// Returns the number of "buckets" within the `flat_hash_map`. Note that
// because a flat hash map contains all elements within its internal storage,
// this value simply equals the current capacity of the `flat_hash_map`.
using Base::bucket_count;
// flat_hash_map::load_factor()
//
// Returns the current load factor of the `flat_hash_map` (the average number
// of slots occupied with a value within the hash map).
using Base::load_factor;
// flat_hash_map::max_load_factor()
//
// Manages the maximum load factor of the `flat_hash_map`. Overloads are
// listed below.
//
// float flat_hash_map::max_load_factor()
//
// Returns the current maximum load factor of the `flat_hash_map`.
//
// void flat_hash_map::max_load_factor(float ml)
//
// Sets the maximum load factor of the `flat_hash_map` to the passed value.
//
// NOTE: This overload is provided only for API compatibility with the STL;
// `flat_hash_map` will ignore any set load factor and manage its rehashing
// internally as an implementation detail.
using Base::max_load_factor;
// flat_hash_map::get_allocator()
//
// Returns the allocator function associated with this `flat_hash_map`.
using Base::get_allocator;
// flat_hash_map::hash_function()
//
// Returns the hashing function used to hash the keys within this
// `flat_hash_map`.
using Base::hash_function;
// flat_hash_map::key_eq()
//
// Returns the function used for comparing keys equality.
using Base::key_eq;
};
// erase_if(flat_hash_map<>, Pred)
//
// Erases all elements that satisfy the predicate `pred` from the container `c`.
// Returns the number of erased elements.
template <typename K, typename V, typename H, typename E, typename A,
typename Predicate>
typename flat_hash_map<K, V, H, E, A>::size_type erase_if(
flat_hash_map<K, V, H, E, A>& c, Predicate pred) {
return container_internal::EraseIf(pred, &c);
}
namespace container_internal {
template <class K, class V>
struct FlatHashMapPolicy {
using slot_policy = container_internal::map_slot_policy<K, V>;
using slot_type = typename slot_policy::slot_type;
using key_type = K;
using mapped_type = V;
using init_type = std::pair</*non const*/ key_type, mapped_type>;
template <class Allocator, class... Args>
static void construct(Allocator* alloc, slot_type* slot, Args&&... args) {
slot_policy::construct(alloc, slot, std::forward<Args>(args)...);
}
template <class Allocator>
static void destroy(Allocator* alloc, slot_type* slot) {
slot_policy::destroy(alloc, slot);
}
template <class Allocator>
static auto transfer(Allocator* alloc, slot_type* new_slot,
slot_type* old_slot) {
return slot_policy::transfer(alloc, new_slot, old_slot);
}
template <class F, class... Args>
static decltype(absl::container_internal::DecomposePair(
std::declval<F>(), std::declval<Args>()...))
apply(F&& f, Args&&... args) {
return absl::container_internal::DecomposePair(std::forward<F>(f),
std::forward<Args>(args)...);
}
static size_t space_used(const slot_type*) { return 0; }
static std::pair<const K, V>& element(slot_type* slot) { return slot->value; }
static V& value(std::pair<const K, V>* kv) { return kv->second; }
static const V& value(const std::pair<const K, V>* kv) { return kv->second; }
};
} // namespace container_internal
namespace container_algorithm_internal {
// Specialization of trait in absl/algorithm/container.h
template <class Key, class T, class Hash, class KeyEqual, class Allocator>
struct IsUnorderedContainer<
absl::flat_hash_map<Key, T, Hash, KeyEqual, Allocator>> : std::true_type {};
} // namespace container_algorithm_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_FLAT_HASH_MAP_H_

507
Pods/abseil/absl/container/flat_hash_set.h generated Normal file
View File

@@ -0,0 +1,507 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// -----------------------------------------------------------------------------
// File: flat_hash_set.h
// -----------------------------------------------------------------------------
//
// An `absl::flat_hash_set<T>` is an unordered associative container designed to
// be a more efficient replacement for `std::unordered_set`. Like
// `unordered_set`, search, insertion, and deletion of set elements can be done
// as an `O(1)` operation. However, `flat_hash_set` (and other unordered
// associative containers known as the collection of Abseil "Swiss tables")
// contain other optimizations that result in both memory and computation
// advantages.
//
// In most cases, your default choice for a hash set should be a set of type
// `flat_hash_set`.
#ifndef ABSL_CONTAINER_FLAT_HASH_SET_H_
#define ABSL_CONTAINER_FLAT_HASH_SET_H_
#include <type_traits>
#include <utility>
#include "absl/algorithm/container.h"
#include "absl/base/macros.h"
#include "absl/container/internal/container_memory.h"
#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export
#include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export
#include "absl/memory/memory.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
template <typename T>
struct FlatHashSetPolicy;
} // namespace container_internal
// -----------------------------------------------------------------------------
// absl::flat_hash_set
// -----------------------------------------------------------------------------
//
// An `absl::flat_hash_set<T>` is an unordered associative container which has
// been optimized for both speed and memory footprint in most common use cases.
// Its interface is similar to that of `std::unordered_set<T>` with the
// following notable differences:
//
// * Requires keys that are CopyConstructible
// * Supports heterogeneous lookup, through `find()` and `insert()`, provided
// that the set is provided a compatible heterogeneous hashing function and
// equality operator.
// * Invalidates any references and pointers to elements within the table after
// `rehash()` and when the table is moved.
// * Contains a `capacity()` member function indicating the number of element
// slots (open, deleted, and empty) within the hash set.
// * Returns `void` from the `erase(iterator)` overload.
//
// By default, `flat_hash_set` uses the `absl::Hash` hashing framework. All
// fundamental and Abseil types that support the `absl::Hash` framework have a
// compatible equality operator for comparing insertions into `flat_hash_set`.
// If your type is not yet supported by the `absl::Hash` framework, see
// absl/hash/hash.h for information on extending Abseil hashing to user-defined
// types.
//
// Using `absl::flat_hash_set` at interface boundaries in dynamically loaded
// libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may
// be randomized across dynamically loaded libraries.
//
// NOTE: A `flat_hash_set` stores its keys directly inside its implementation
// array to avoid memory indirection. Because a `flat_hash_set` is designed to
// move data when rehashed, set keys will not retain pointer stability. If you
// require pointer stability, consider using
// `absl::flat_hash_set<std::unique_ptr<T>>`. If your type is not moveable and
// you require pointer stability, consider `absl::node_hash_set` instead.
//
// Example:
//
// // Create a flat hash set of three strings
// absl::flat_hash_set<std::string> ducks =
// {"huey", "dewey", "louie"};
//
// // Insert a new element into the flat hash set
// ducks.insert("donald");
//
// // Force a rehash of the flat hash set
// ducks.rehash(0);
//
// // See if "dewey" is present
// if (ducks.contains("dewey")) {
// std::cout << "We found dewey!" << std::endl;
// }
template <class T, class Hash = absl::container_internal::hash_default_hash<T>,
class Eq = absl::container_internal::hash_default_eq<T>,
class Allocator = std::allocator<T>>
class flat_hash_set
: public absl::container_internal::raw_hash_set<
absl::container_internal::FlatHashSetPolicy<T>, Hash, Eq, Allocator> {
using Base = typename flat_hash_set::raw_hash_set;
public:
// Constructors and Assignment Operators
//
// A flat_hash_set supports the same overload set as `std::unordered_set`
// for construction and assignment:
//
// * Default constructor
//
// // No allocation for the table's elements is made.
// absl::flat_hash_set<std::string> set1;
//
// * Initializer List constructor
//
// absl::flat_hash_set<std::string> set2 =
// {{"huey"}, {"dewey"}, {"louie"},};
//
// * Copy constructor
//
// absl::flat_hash_set<std::string> set3(set2);
//
// * Copy assignment operator
//
// // Hash functor and Comparator are copied as well
// absl::flat_hash_set<std::string> set4;
// set4 = set3;
//
// * Move constructor
//
// // Move is guaranteed efficient
// absl::flat_hash_set<std::string> set5(std::move(set4));
//
// * Move assignment operator
//
// // May be efficient if allocators are compatible
// absl::flat_hash_set<std::string> set6;
// set6 = std::move(set5);
//
// * Range constructor
//
// std::vector<std::string> v = {"a", "b"};
// absl::flat_hash_set<std::string> set7(v.begin(), v.end());
flat_hash_set() {}
using Base::Base;
// flat_hash_set::begin()
//
// Returns an iterator to the beginning of the `flat_hash_set`.
using Base::begin;
// flat_hash_set::cbegin()
//
// Returns a const iterator to the beginning of the `flat_hash_set`.
using Base::cbegin;
// flat_hash_set::cend()
//
// Returns a const iterator to the end of the `flat_hash_set`.
using Base::cend;
// flat_hash_set::end()
//
// Returns an iterator to the end of the `flat_hash_set`.
using Base::end;
// flat_hash_set::capacity()
//
// Returns the number of element slots (assigned, deleted, and empty)
// available within the `flat_hash_set`.
//
// NOTE: this member function is particular to `absl::flat_hash_set` and is
// not provided in the `std::unordered_set` API.
using Base::capacity;
// flat_hash_set::empty()
//
// Returns whether or not the `flat_hash_set` is empty.
using Base::empty;
// flat_hash_set::max_size()
//
// Returns the largest theoretical possible number of elements within a
// `flat_hash_set` under current memory constraints. This value can be thought
// of the largest value of `std::distance(begin(), end())` for a
// `flat_hash_set<T>`.
using Base::max_size;
// flat_hash_set::size()
//
// Returns the number of elements currently within the `flat_hash_set`.
using Base::size;
// flat_hash_set::clear()
//
// Removes all elements from the `flat_hash_set`. Invalidates any references,
// pointers, or iterators referring to contained elements.
//
// NOTE: this operation may shrink the underlying buffer. To avoid shrinking
// the underlying buffer call `erase(begin(), end())`.
using Base::clear;
// flat_hash_set::erase()
//
// Erases elements within the `flat_hash_set`. Erasing does not trigger a
// rehash. Overloads are listed below.
//
// void erase(const_iterator pos):
//
// Erases the element at `position` of the `flat_hash_set`, returning
// `void`.
//
// NOTE: returning `void` in this case is different than that of STL
// containers in general and `std::unordered_set` in particular (which
// return an iterator to the element following the erased element). If that
// iterator is needed, simply post increment the iterator:
//
// set.erase(it++);
//
// iterator erase(const_iterator first, const_iterator last):
//
// Erases the elements in the open interval [`first`, `last`), returning an
// iterator pointing to `last`. The special case of calling
// `erase(begin(), end())` resets the reserved growth such that if
// `reserve(N)` has previously been called and there has been no intervening
// call to `clear()`, then after calling `erase(begin(), end())`, it is safe
// to assume that inserting N elements will not cause a rehash.
//
// size_type erase(const key_type& key):
//
// Erases the element with the matching key, if it exists, returning the
// number of elements erased (0 or 1).
using Base::erase;
// flat_hash_set::insert()
//
// Inserts an element of the specified value into the `flat_hash_set`,
// returning an iterator pointing to the newly inserted element, provided that
// an element with the given key does not already exist. If rehashing occurs
// due to the insertion, all iterators are invalidated. Overloads are listed
// below.
//
// std::pair<iterator,bool> insert(const T& value):
//
// Inserts a value into the `flat_hash_set`. Returns a pair consisting of an
// iterator to the inserted element (or to the element that prevented the
// insertion) and a bool denoting whether the insertion took place.
//
// std::pair<iterator,bool> insert(T&& value):
//
// Inserts a moveable value into the `flat_hash_set`. Returns a pair
// consisting of an iterator to the inserted element (or to the element that
// prevented the insertion) and a bool denoting whether the insertion took
// place.
//
// iterator insert(const_iterator hint, const T& value):
// iterator insert(const_iterator hint, T&& value):
//
// Inserts a value, using the position of `hint` as a non-binding suggestion
// for where to begin the insertion search. Returns an iterator to the
// inserted element, or to the existing element that prevented the
// insertion.
//
// void insert(InputIterator first, InputIterator last):
//
// Inserts a range of values [`first`, `last`).
//
// NOTE: Although the STL does not specify which element may be inserted if
// multiple keys compare equivalently, for `flat_hash_set` we guarantee the
// first match is inserted.
//
// void insert(std::initializer_list<T> ilist):
//
// Inserts the elements within the initializer list `ilist`.
//
// NOTE: Although the STL does not specify which element may be inserted if
// multiple keys compare equivalently within the initializer list, for
// `flat_hash_set` we guarantee the first match is inserted.
using Base::insert;
// flat_hash_set::emplace()
//
// Inserts an element of the specified value by constructing it in-place
// within the `flat_hash_set`, provided that no element with the given key
// already exists.
//
// The element may be constructed even if there already is an element with the
// key in the container, in which case the newly constructed element will be
// destroyed immediately.
//
// If rehashing occurs due to the insertion, all iterators are invalidated.
using Base::emplace;
// flat_hash_set::emplace_hint()
//
// Inserts an element of the specified value by constructing it in-place
// within the `flat_hash_set`, using the position of `hint` as a non-binding
// suggestion for where to begin the insertion search, and only inserts
// provided that no element with the given key already exists.
//
// The element may be constructed even if there already is an element with the
// key in the container, in which case the newly constructed element will be
// destroyed immediately.
//
// If rehashing occurs due to the insertion, all iterators are invalidated.
using Base::emplace_hint;
// flat_hash_set::extract()
//
// Extracts the indicated element, erasing it in the process, and returns it
// as a C++17-compatible node handle. Overloads are listed below.
//
// node_type extract(const_iterator position):
//
// Extracts the element at the indicated position and returns a node handle
// owning that extracted data.
//
// node_type extract(const key_type& x):
//
// Extracts the element with the key matching the passed key value and
// returns a node handle owning that extracted data. If the `flat_hash_set`
// does not contain an element with a matching key, this function returns an
// empty node handle.
using Base::extract;
// flat_hash_set::merge()
//
// Extracts elements from a given `source` flat hash set into this
// `flat_hash_set`. If the destination `flat_hash_set` already contains an
// element with an equivalent key, that element is not extracted.
using Base::merge;
// flat_hash_set::swap(flat_hash_set& other)
//
// Exchanges the contents of this `flat_hash_set` with those of the `other`
// flat hash set, avoiding invocation of any move, copy, or swap operations on
// individual elements.
//
// All iterators and references on the `flat_hash_set` remain valid, excepting
// for the past-the-end iterator, which is invalidated.
//
// `swap()` requires that the flat hash set's hashing and key equivalence
// functions be Swappable, and are exchanged using unqualified calls to
// non-member `swap()`. If the set's allocator has
// `std::allocator_traits<allocator_type>::propagate_on_container_swap::value`
// set to `true`, the allocators are also exchanged using an unqualified call
// to non-member `swap()`; otherwise, the allocators are not swapped.
using Base::swap;
// flat_hash_set::rehash(count)
//
// Rehashes the `flat_hash_set`, setting the number of slots to be at least
// the passed value. If the new number of slots increases the load factor more
// than the current maximum load factor
// (`count` < `size()` / `max_load_factor()`), then the new number of slots
// will be at least `size()` / `max_load_factor()`.
//
// To force a rehash, pass rehash(0).
//
// NOTE: unlike behavior in `std::unordered_set`, references are also
// invalidated upon a `rehash()`.
using Base::rehash;
// flat_hash_set::reserve(count)
//
// Sets the number of slots in the `flat_hash_set` to the number needed to
// accommodate at least `count` total elements without exceeding the current
// maximum load factor, and may rehash the container if needed.
using Base::reserve;
// flat_hash_set::contains()
//
// Determines whether an element comparing equal to the given `key` exists
// within the `flat_hash_set`, returning `true` if so or `false` otherwise.
using Base::contains;
// flat_hash_set::count(const Key& key) const
//
// Returns the number of elements comparing equal to the given `key` within
// the `flat_hash_set`. note that this function will return either `1` or `0`
// since duplicate elements are not allowed within a `flat_hash_set`.
using Base::count;
// flat_hash_set::equal_range()
//
// Returns a closed range [first, last], defined by a `std::pair` of two
// iterators, containing all elements with the passed key in the
// `flat_hash_set`.
using Base::equal_range;
// flat_hash_set::find()
//
// Finds an element with the passed `key` within the `flat_hash_set`.
using Base::find;
// flat_hash_set::bucket_count()
//
// Returns the number of "buckets" within the `flat_hash_set`. Note that
// because a flat hash set contains all elements within its internal storage,
// this value simply equals the current capacity of the `flat_hash_set`.
using Base::bucket_count;
// flat_hash_set::load_factor()
//
// Returns the current load factor of the `flat_hash_set` (the average number
// of slots occupied with a value within the hash set).
using Base::load_factor;
// flat_hash_set::max_load_factor()
//
// Manages the maximum load factor of the `flat_hash_set`. Overloads are
// listed below.
//
// float flat_hash_set::max_load_factor()
//
// Returns the current maximum load factor of the `flat_hash_set`.
//
// void flat_hash_set::max_load_factor(float ml)
//
// Sets the maximum load factor of the `flat_hash_set` to the passed value.
//
// NOTE: This overload is provided only for API compatibility with the STL;
// `flat_hash_set` will ignore any set load factor and manage its rehashing
// internally as an implementation detail.
using Base::max_load_factor;
// flat_hash_set::get_allocator()
//
// Returns the allocator function associated with this `flat_hash_set`.
using Base::get_allocator;
// flat_hash_set::hash_function()
//
// Returns the hashing function used to hash the keys within this
// `flat_hash_set`.
using Base::hash_function;
// flat_hash_set::key_eq()
//
// Returns the function used for comparing keys equality.
using Base::key_eq;
};
// erase_if(flat_hash_set<>, Pred)
//
// Erases all elements that satisfy the predicate `pred` from the container `c`.
// Returns the number of erased elements.
template <typename T, typename H, typename E, typename A, typename Predicate>
typename flat_hash_set<T, H, E, A>::size_type erase_if(
flat_hash_set<T, H, E, A>& c, Predicate pred) {
return container_internal::EraseIf(pred, &c);
}
namespace container_internal {
template <class T>
struct FlatHashSetPolicy {
using slot_type = T;
using key_type = T;
using init_type = T;
using constant_iterators = std::true_type;
template <class Allocator, class... Args>
static void construct(Allocator* alloc, slot_type* slot, Args&&... args) {
absl::allocator_traits<Allocator>::construct(*alloc, slot,
std::forward<Args>(args)...);
}
template <class Allocator>
static void destroy(Allocator* alloc, slot_type* slot) {
absl::allocator_traits<Allocator>::destroy(*alloc, slot);
}
static T& element(slot_type* slot) { return *slot; }
template <class F, class... Args>
static decltype(absl::container_internal::DecomposeValue(
std::declval<F>(), std::declval<Args>()...))
apply(F&& f, Args&&... args) {
return absl::container_internal::DecomposeValue(
std::forward<F>(f), std::forward<Args>(args)...);
}
static size_t space_used(const T*) { return 0; }
};
} // namespace container_internal
namespace container_algorithm_internal {
// Specialization of trait in absl/algorithm/container.h
template <class Key, class Hash, class KeyEqual, class Allocator>
struct IsUnorderedContainer<absl::flat_hash_set<Key, Hash, KeyEqual, Allocator>>
: std::true_type {};
} // namespace container_algorithm_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_FLAT_HASH_SET_H_

1002
Pods/abseil/absl/container/inlined_vector.h generated Normal file
View File

@@ -0,0 +1,1002 @@
// Copyright 2019 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// -----------------------------------------------------------------------------
// File: inlined_vector.h
// -----------------------------------------------------------------------------
//
// This header file contains the declaration and definition of an "inlined
// vector" which behaves in an equivalent fashion to a `std::vector`, except
// that storage for small sequences of the vector are provided inline without
// requiring any heap allocation.
//
// An `absl::InlinedVector<T, N>` specifies the default capacity `N` as one of
// its template parameters. Instances where `size() <= N` hold contained
// elements in inline space. Typically `N` is very small so that sequences that
// are expected to be short do not require allocations.
//
// An `absl::InlinedVector` does not usually require a specific allocator. If
// the inlined vector grows beyond its initial constraints, it will need to
// allocate (as any normal `std::vector` would). This is usually performed with
// the default allocator (defined as `std::allocator<T>`). Optionally, a custom
// allocator type may be specified as `A` in `absl::InlinedVector<T, N, A>`.
#ifndef ABSL_CONTAINER_INLINED_VECTOR_H_
#define ABSL_CONTAINER_INLINED_VECTOR_H_
#include <algorithm>
#include <cstddef>
#include <cstdlib>
#include <cstring>
#include <initializer_list>
#include <iterator>
#include <memory>
#include <type_traits>
#include <utility>
#include "absl/algorithm/algorithm.h"
#include "absl/base/internal/throw_delegate.h"
#include "absl/base/macros.h"
#include "absl/base/optimization.h"
#include "absl/base/port.h"
#include "absl/container/internal/inlined_vector.h"
#include "absl/memory/memory.h"
#include "absl/meta/type_traits.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
// -----------------------------------------------------------------------------
// InlinedVector
// -----------------------------------------------------------------------------
//
// An `absl::InlinedVector` is designed to be a drop-in replacement for
// `std::vector` for use cases where the vector's size is sufficiently small
// that it can be inlined. If the inlined vector does grow beyond its estimated
// capacity, it will trigger an initial allocation on the heap, and will behave
// as a `std::vector`. The API of the `absl::InlinedVector` within this file is
// designed to cover the same API footprint as covered by `std::vector`.
template <typename T, size_t N, typename A = std::allocator<T>>
class InlinedVector {
static_assert(N > 0, "`absl::InlinedVector` requires an inlined capacity.");
using Storage = inlined_vector_internal::Storage<T, N, A>;
template <typename TheA>
using AllocatorTraits = inlined_vector_internal::AllocatorTraits<TheA>;
template <typename TheA>
using MoveIterator = inlined_vector_internal::MoveIterator<TheA>;
template <typename TheA>
using IsMoveAssignOk = inlined_vector_internal::IsMoveAssignOk<TheA>;
template <typename TheA, typename Iterator>
using IteratorValueAdapter =
inlined_vector_internal::IteratorValueAdapter<TheA, Iterator>;
template <typename TheA>
using CopyValueAdapter = inlined_vector_internal::CopyValueAdapter<TheA>;
template <typename TheA>
using DefaultValueAdapter =
inlined_vector_internal::DefaultValueAdapter<TheA>;
template <typename Iterator>
using EnableIfAtLeastForwardIterator = absl::enable_if_t<
inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value, int>;
template <typename Iterator>
using DisableIfAtLeastForwardIterator = absl::enable_if_t<
!inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value, int>;
using MemcpyPolicy = typename Storage::MemcpyPolicy;
using ElementwiseAssignPolicy = typename Storage::ElementwiseAssignPolicy;
using ElementwiseConstructPolicy =
typename Storage::ElementwiseConstructPolicy;
using MoveAssignmentPolicy = typename Storage::MoveAssignmentPolicy;
public:
using allocator_type = A;
using value_type = inlined_vector_internal::ValueType<A>;
using pointer = inlined_vector_internal::Pointer<A>;
using const_pointer = inlined_vector_internal::ConstPointer<A>;
using size_type = inlined_vector_internal::SizeType<A>;
using difference_type = inlined_vector_internal::DifferenceType<A>;
using reference = inlined_vector_internal::Reference<A>;
using const_reference = inlined_vector_internal::ConstReference<A>;
using iterator = inlined_vector_internal::Iterator<A>;
using const_iterator = inlined_vector_internal::ConstIterator<A>;
using reverse_iterator = inlined_vector_internal::ReverseIterator<A>;
using const_reverse_iterator =
inlined_vector_internal::ConstReverseIterator<A>;
// ---------------------------------------------------------------------------
// InlinedVector Constructors and Destructor
// ---------------------------------------------------------------------------
// Creates an empty inlined vector with a value-initialized allocator.
InlinedVector() noexcept(noexcept(allocator_type())) : storage_() {}
// Creates an empty inlined vector with a copy of `allocator`.
explicit InlinedVector(const allocator_type& allocator) noexcept
: storage_(allocator) {}
// Creates an inlined vector with `n` copies of `value_type()`.
explicit InlinedVector(size_type n,
const allocator_type& allocator = allocator_type())
: storage_(allocator) {
storage_.Initialize(DefaultValueAdapter<A>(), n);
}
// Creates an inlined vector with `n` copies of `v`.
InlinedVector(size_type n, const_reference v,
const allocator_type& allocator = allocator_type())
: storage_(allocator) {
storage_.Initialize(CopyValueAdapter<A>(std::addressof(v)), n);
}
// Creates an inlined vector with copies of the elements of `list`.
InlinedVector(std::initializer_list<value_type> list,
const allocator_type& allocator = allocator_type())
: InlinedVector(list.begin(), list.end(), allocator) {}
// Creates an inlined vector with elements constructed from the provided
// forward iterator range [`first`, `last`).
//
// NOTE: the `enable_if` prevents ambiguous interpretation between a call to
// this constructor with two integral arguments and a call to the above
// `InlinedVector(size_type, const_reference)` constructor.
template <typename ForwardIterator,
EnableIfAtLeastForwardIterator<ForwardIterator> = 0>
InlinedVector(ForwardIterator first, ForwardIterator last,
const allocator_type& allocator = allocator_type())
: storage_(allocator) {
storage_.Initialize(IteratorValueAdapter<A, ForwardIterator>(first),
static_cast<size_t>(std::distance(first, last)));
}
// Creates an inlined vector with elements constructed from the provided input
// iterator range [`first`, `last`).
template <typename InputIterator,
DisableIfAtLeastForwardIterator<InputIterator> = 0>
InlinedVector(InputIterator first, InputIterator last,
const allocator_type& allocator = allocator_type())
: storage_(allocator) {
std::copy(first, last, std::back_inserter(*this));
}
// Creates an inlined vector by copying the contents of `other` using
// `other`'s allocator.
InlinedVector(const InlinedVector& other)
: InlinedVector(other, other.storage_.GetAllocator()) {}
// Creates an inlined vector by copying the contents of `other` using the
// provided `allocator`.
InlinedVector(const InlinedVector& other, const allocator_type& allocator)
: storage_(allocator) {
// Fast path: if the other vector is empty, there's nothing for us to do.
if (other.empty()) {
return;
}
// Fast path: if the value type is trivially copy constructible, we know the
// allocator doesn't do anything fancy, and there is nothing on the heap
// then we know it is legal for us to simply memcpy the other vector's
// inlined bytes to form our copy of its elements.
if (absl::is_trivially_copy_constructible<value_type>::value &&
std::is_same<A, std::allocator<value_type>>::value &&
!other.storage_.GetIsAllocated()) {
storage_.MemcpyFrom(other.storage_);
return;
}
storage_.InitFrom(other.storage_);
}
// Creates an inlined vector by moving in the contents of `other` without
// allocating. If `other` contains allocated memory, the newly-created inlined
// vector will take ownership of that memory. However, if `other` does not
// contain allocated memory, the newly-created inlined vector will perform
// element-wise move construction of the contents of `other`.
//
// NOTE: since no allocation is performed for the inlined vector in either
// case, the `noexcept(...)` specification depends on whether moving the
// underlying objects can throw. It is assumed assumed that...
// a) move constructors should only throw due to allocation failure.
// b) if `value_type`'s move constructor allocates, it uses the same
// allocation function as the inlined vector's allocator.
// Thus, the move constructor is non-throwing if the allocator is non-throwing
// or `value_type`'s move constructor is specified as `noexcept`.
InlinedVector(InlinedVector&& other) noexcept(
absl::allocator_is_nothrow<allocator_type>::value ||
std::is_nothrow_move_constructible<value_type>::value)
: storage_(other.storage_.GetAllocator()) {
// Fast path: if the value type can be trivially relocated (i.e. moved from
// and destroyed), and we know the allocator doesn't do anything fancy, then
// it's safe for us to simply adopt the contents of the storage for `other`
// and remove its own reference to them. It's as if we had individually
// move-constructed each value and then destroyed the original.
if (absl::is_trivially_relocatable<value_type>::value &&
std::is_same<A, std::allocator<value_type>>::value) {
storage_.MemcpyFrom(other.storage_);
other.storage_.SetInlinedSize(0);
return;
}
// Fast path: if the other vector is on the heap, we can simply take over
// its allocation.
if (other.storage_.GetIsAllocated()) {
storage_.SetAllocation({other.storage_.GetAllocatedData(),
other.storage_.GetAllocatedCapacity()});
storage_.SetAllocatedSize(other.storage_.GetSize());
other.storage_.SetInlinedSize(0);
return;
}
// Otherwise we must move each element individually.
IteratorValueAdapter<A, MoveIterator<A>> other_values(
MoveIterator<A>(other.storage_.GetInlinedData()));
inlined_vector_internal::ConstructElements<A>(
storage_.GetAllocator(), storage_.GetInlinedData(), other_values,
other.storage_.GetSize());
storage_.SetInlinedSize(other.storage_.GetSize());
}
// Creates an inlined vector by moving in the contents of `other` with a copy
// of `allocator`.
//
// NOTE: if `other`'s allocator is not equal to `allocator`, even if `other`
// contains allocated memory, this move constructor will still allocate. Since
// allocation is performed, this constructor can only be `noexcept` if the
// specified allocator is also `noexcept`.
InlinedVector(
InlinedVector&& other,
const allocator_type&
allocator) noexcept(absl::allocator_is_nothrow<allocator_type>::value)
: storage_(allocator) {
// Fast path: if the value type can be trivially relocated (i.e. moved from
// and destroyed), and we know the allocator doesn't do anything fancy, then
// it's safe for us to simply adopt the contents of the storage for `other`
// and remove its own reference to them. It's as if we had individually
// move-constructed each value and then destroyed the original.
if (absl::is_trivially_relocatable<value_type>::value &&
std::is_same<A, std::allocator<value_type>>::value) {
storage_.MemcpyFrom(other.storage_);
other.storage_.SetInlinedSize(0);
return;
}
// Fast path: if the other vector is on the heap and shared the same
// allocator, we can simply take over its allocation.
if ((storage_.GetAllocator() == other.storage_.GetAllocator()) &&
other.storage_.GetIsAllocated()) {
storage_.SetAllocation({other.storage_.GetAllocatedData(),
other.storage_.GetAllocatedCapacity()});
storage_.SetAllocatedSize(other.storage_.GetSize());
other.storage_.SetInlinedSize(0);
return;
}
// Otherwise we must move each element individually.
storage_.Initialize(
IteratorValueAdapter<A, MoveIterator<A>>(MoveIterator<A>(other.data())),
other.size());
}
~InlinedVector() {}
// ---------------------------------------------------------------------------
// InlinedVector Member Accessors
// ---------------------------------------------------------------------------
// `InlinedVector::empty()`
//
// Returns whether the inlined vector contains no elements.
bool empty() const noexcept { return !size(); }
// `InlinedVector::size()`
//
// Returns the number of elements in the inlined vector.
size_type size() const noexcept { return storage_.GetSize(); }
// `InlinedVector::max_size()`
//
// Returns the maximum number of elements the inlined vector can hold.
size_type max_size() const noexcept {
// One bit of the size storage is used to indicate whether the inlined
// vector contains allocated memory. As a result, the maximum size that the
// inlined vector can express is the minimum of the limit of how many
// objects we can allocate and std::numeric_limits<size_type>::max() / 2.
return (std::min)(AllocatorTraits<A>::max_size(storage_.GetAllocator()),
(std::numeric_limits<size_type>::max)() / 2);
}
// `InlinedVector::capacity()`
//
// Returns the number of elements that could be stored in the inlined vector
// without requiring a reallocation.
//
// NOTE: for most inlined vectors, `capacity()` should be equal to the
// template parameter `N`. For inlined vectors which exceed this capacity,
// they will no longer be inlined and `capacity()` will equal the capactity of
// the allocated memory.
size_type capacity() const noexcept {
return storage_.GetIsAllocated() ? storage_.GetAllocatedCapacity()
: storage_.GetInlinedCapacity();
}
// `InlinedVector::data()`
//
// Returns a `pointer` to the elements of the inlined vector. This pointer
// can be used to access and modify the contained elements.
//
// NOTE: only elements within [`data()`, `data() + size()`) are valid.
pointer data() noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return storage_.GetIsAllocated() ? storage_.GetAllocatedData()
: storage_.GetInlinedData();
}
// Overload of `InlinedVector::data()` that returns a `const_pointer` to the
// elements of the inlined vector. This pointer can be used to access but not
// modify the contained elements.
//
// NOTE: only elements within [`data()`, `data() + size()`) are valid.
const_pointer data() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return storage_.GetIsAllocated() ? storage_.GetAllocatedData()
: storage_.GetInlinedData();
}
// `InlinedVector::operator[](...)`
//
// Returns a `reference` to the `i`th element of the inlined vector.
reference operator[](size_type i) ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(i < size());
return data()[i];
}
// Overload of `InlinedVector::operator[](...)` that returns a
// `const_reference` to the `i`th element of the inlined vector.
const_reference operator[](size_type i) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(i < size());
return data()[i];
}
// `InlinedVector::at(...)`
//
// Returns a `reference` to the `i`th element of the inlined vector.
//
// NOTE: if `i` is not within the required range of `InlinedVector::at(...)`,
// in both debug and non-debug builds, `std::out_of_range` will be thrown.
reference at(size_type i) ABSL_ATTRIBUTE_LIFETIME_BOUND {
if (ABSL_PREDICT_FALSE(i >= size())) {
base_internal::ThrowStdOutOfRange(
"`InlinedVector::at(size_type)` failed bounds check");
}
return data()[i];
}
// Overload of `InlinedVector::at(...)` that returns a `const_reference` to
// the `i`th element of the inlined vector.
//
// NOTE: if `i` is not within the required range of `InlinedVector::at(...)`,
// in both debug and non-debug builds, `std::out_of_range` will be thrown.
const_reference at(size_type i) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
if (ABSL_PREDICT_FALSE(i >= size())) {
base_internal::ThrowStdOutOfRange(
"`InlinedVector::at(size_type) const` failed bounds check");
}
return data()[i];
}
// `InlinedVector::front()`
//
// Returns a `reference` to the first element of the inlined vector.
reference front() ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(!empty());
return data()[0];
}
// Overload of `InlinedVector::front()` that returns a `const_reference` to
// the first element of the inlined vector.
const_reference front() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(!empty());
return data()[0];
}
// `InlinedVector::back()`
//
// Returns a `reference` to the last element of the inlined vector.
reference back() ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(!empty());
return data()[size() - 1];
}
// Overload of `InlinedVector::back()` that returns a `const_reference` to the
// last element of the inlined vector.
const_reference back() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(!empty());
return data()[size() - 1];
}
// `InlinedVector::begin()`
//
// Returns an `iterator` to the beginning of the inlined vector.
iterator begin() noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND { return data(); }
// Overload of `InlinedVector::begin()` that returns a `const_iterator` to
// the beginning of the inlined vector.
const_iterator begin() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return data();
}
// `InlinedVector::end()`
//
// Returns an `iterator` to the end of the inlined vector.
iterator end() noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return data() + size();
}
// Overload of `InlinedVector::end()` that returns a `const_iterator` to the
// end of the inlined vector.
const_iterator end() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return data() + size();
}
// `InlinedVector::cbegin()`
//
// Returns a `const_iterator` to the beginning of the inlined vector.
const_iterator cbegin() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return begin();
}
// `InlinedVector::cend()`
//
// Returns a `const_iterator` to the end of the inlined vector.
const_iterator cend() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return end();
}
// `InlinedVector::rbegin()`
//
// Returns a `reverse_iterator` from the end of the inlined vector.
reverse_iterator rbegin() noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return reverse_iterator(end());
}
// Overload of `InlinedVector::rbegin()` that returns a
// `const_reverse_iterator` from the end of the inlined vector.
const_reverse_iterator rbegin() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_reverse_iterator(end());
}
// `InlinedVector::rend()`
//
// Returns a `reverse_iterator` from the beginning of the inlined vector.
reverse_iterator rend() noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return reverse_iterator(begin());
}
// Overload of `InlinedVector::rend()` that returns a `const_reverse_iterator`
// from the beginning of the inlined vector.
const_reverse_iterator rend() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_reverse_iterator(begin());
}
// `InlinedVector::crbegin()`
//
// Returns a `const_reverse_iterator` from the end of the inlined vector.
const_reverse_iterator crbegin() const noexcept
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return rbegin();
}
// `InlinedVector::crend()`
//
// Returns a `const_reverse_iterator` from the beginning of the inlined
// vector.
const_reverse_iterator crend() const noexcept ABSL_ATTRIBUTE_LIFETIME_BOUND {
return rend();
}
// `InlinedVector::get_allocator()`
//
// Returns a copy of the inlined vector's allocator.
allocator_type get_allocator() const { return storage_.GetAllocator(); }
// ---------------------------------------------------------------------------
// InlinedVector Member Mutators
// ---------------------------------------------------------------------------
// `InlinedVector::operator=(...)`
//
// Replaces the elements of the inlined vector with copies of the elements of
// `list`.
InlinedVector& operator=(std::initializer_list<value_type> list) {
assign(list.begin(), list.end());
return *this;
}
// Overload of `InlinedVector::operator=(...)` that replaces the elements of
// the inlined vector with copies of the elements of `other`.
InlinedVector& operator=(const InlinedVector& other) {
if (ABSL_PREDICT_TRUE(this != std::addressof(other))) {
const_pointer other_data = other.data();
assign(other_data, other_data + other.size());
}
return *this;
}
// Overload of `InlinedVector::operator=(...)` that moves the elements of
// `other` into the inlined vector.
//
// NOTE: as a result of calling this overload, `other` is left in a valid but
// unspecified state.
InlinedVector& operator=(InlinedVector&& other) {
if (ABSL_PREDICT_TRUE(this != std::addressof(other))) {
MoveAssignment(MoveAssignmentPolicy{}, std::move(other));
}
return *this;
}
// `InlinedVector::assign(...)`
//
// Replaces the contents of the inlined vector with `n` copies of `v`.
void assign(size_type n, const_reference v) {
storage_.Assign(CopyValueAdapter<A>(std::addressof(v)), n);
}
// Overload of `InlinedVector::assign(...)` that replaces the contents of the
// inlined vector with copies of the elements of `list`.
void assign(std::initializer_list<value_type> list) {
assign(list.begin(), list.end());
}
// Overload of `InlinedVector::assign(...)` to replace the contents of the
// inlined vector with the range [`first`, `last`).
//
// NOTE: this overload is for iterators that are "forward" category or better.
template <typename ForwardIterator,
EnableIfAtLeastForwardIterator<ForwardIterator> = 0>
void assign(ForwardIterator first, ForwardIterator last) {
storage_.Assign(IteratorValueAdapter<A, ForwardIterator>(first),
static_cast<size_t>(std::distance(first, last)));
}
// Overload of `InlinedVector::assign(...)` to replace the contents of the
// inlined vector with the range [`first`, `last`).
//
// NOTE: this overload is for iterators that are "input" category.
template <typename InputIterator,
DisableIfAtLeastForwardIterator<InputIterator> = 0>
void assign(InputIterator first, InputIterator last) {
size_type i = 0;
for (; i < size() && first != last; ++i, static_cast<void>(++first)) {
data()[i] = *first;
}
erase(data() + i, data() + size());
std::copy(first, last, std::back_inserter(*this));
}
// `InlinedVector::resize(...)`
//
// Resizes the inlined vector to contain `n` elements.
//
// NOTE: If `n` is smaller than `size()`, extra elements are destroyed. If `n`
// is larger than `size()`, new elements are value-initialized.
void resize(size_type n) {
ABSL_HARDENING_ASSERT(n <= max_size());
storage_.Resize(DefaultValueAdapter<A>(), n);
}
// Overload of `InlinedVector::resize(...)` that resizes the inlined vector to
// contain `n` elements.
//
// NOTE: if `n` is smaller than `size()`, extra elements are destroyed. If `n`
// is larger than `size()`, new elements are copied-constructed from `v`.
void resize(size_type n, const_reference v) {
ABSL_HARDENING_ASSERT(n <= max_size());
storage_.Resize(CopyValueAdapter<A>(std::addressof(v)), n);
}
// `InlinedVector::insert(...)`
//
// Inserts a copy of `v` at `pos`, returning an `iterator` to the newly
// inserted element.
iterator insert(const_iterator pos,
const_reference v) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return emplace(pos, v);
}
// Overload of `InlinedVector::insert(...)` that inserts `v` at `pos` using
// move semantics, returning an `iterator` to the newly inserted element.
iterator insert(const_iterator pos,
value_type&& v) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return emplace(pos, std::move(v));
}
// Overload of `InlinedVector::insert(...)` that inserts `n` contiguous copies
// of `v` starting at `pos`, returning an `iterator` pointing to the first of
// the newly inserted elements.
iterator insert(const_iterator pos, size_type n,
const_reference v) ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(pos >= begin());
ABSL_HARDENING_ASSERT(pos <= end());
if (ABSL_PREDICT_TRUE(n != 0)) {
value_type dealias = v;
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102329#c2
// It appears that GCC thinks that since `pos` is a const pointer and may
// point to uninitialized memory at this point, a warning should be
// issued. But `pos` is actually only used to compute an array index to
// write to.
#if !defined(__clang__) && defined(__GNUC__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
return storage_.Insert(pos, CopyValueAdapter<A>(std::addressof(dealias)),
n);
#if !defined(__clang__) && defined(__GNUC__)
#pragma GCC diagnostic pop
#endif
} else {
return const_cast<iterator>(pos);
}
}
// Overload of `InlinedVector::insert(...)` that inserts copies of the
// elements of `list` starting at `pos`, returning an `iterator` pointing to
// the first of the newly inserted elements.
iterator insert(const_iterator pos, std::initializer_list<value_type> list)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert(pos, list.begin(), list.end());
}
// Overload of `InlinedVector::insert(...)` that inserts the range [`first`,
// `last`) starting at `pos`, returning an `iterator` pointing to the first
// of the newly inserted elements.
//
// NOTE: this overload is for iterators that are "forward" category or better.
template <typename ForwardIterator,
EnableIfAtLeastForwardIterator<ForwardIterator> = 0>
iterator insert(const_iterator pos, ForwardIterator first,
ForwardIterator last) ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(pos >= begin());
ABSL_HARDENING_ASSERT(pos <= end());
if (ABSL_PREDICT_TRUE(first != last)) {
return storage_.Insert(
pos, IteratorValueAdapter<A, ForwardIterator>(first),
static_cast<size_type>(std::distance(first, last)));
} else {
return const_cast<iterator>(pos);
}
}
// Overload of `InlinedVector::insert(...)` that inserts the range [`first`,
// `last`) starting at `pos`, returning an `iterator` pointing to the first
// of the newly inserted elements.
//
// NOTE: this overload is for iterators that are "input" category.
template <typename InputIterator,
DisableIfAtLeastForwardIterator<InputIterator> = 0>
iterator insert(const_iterator pos, InputIterator first,
InputIterator last) ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(pos >= begin());
ABSL_HARDENING_ASSERT(pos <= end());
size_type index = static_cast<size_type>(std::distance(cbegin(), pos));
for (size_type i = index; first != last; ++i, static_cast<void>(++first)) {
insert(data() + i, *first);
}
return iterator(data() + index);
}
// `InlinedVector::emplace(...)`
//
// Constructs and inserts an element using `args...` in the inlined vector at
// `pos`, returning an `iterator` pointing to the newly emplaced element.
template <typename... Args>
iterator emplace(const_iterator pos,
Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(pos >= begin());
ABSL_HARDENING_ASSERT(pos <= end());
value_type dealias(std::forward<Args>(args)...);
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102329#c2
// It appears that GCC thinks that since `pos` is a const pointer and may
// point to uninitialized memory at this point, a warning should be
// issued. But `pos` is actually only used to compute an array index to
// write to.
#if !defined(__clang__) && defined(__GNUC__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
return storage_.Insert(pos,
IteratorValueAdapter<A, MoveIterator<A>>(
MoveIterator<A>(std::addressof(dealias))),
1);
#if !defined(__clang__) && defined(__GNUC__)
#pragma GCC diagnostic pop
#endif
}
// `InlinedVector::emplace_back(...)`
//
// Constructs and inserts an element using `args...` in the inlined vector at
// `end()`, returning a `reference` to the newly emplaced element.
template <typename... Args>
reference emplace_back(Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return storage_.EmplaceBack(std::forward<Args>(args)...);
}
// `InlinedVector::push_back(...)`
//
// Inserts a copy of `v` in the inlined vector at `end()`.
void push_back(const_reference v) { static_cast<void>(emplace_back(v)); }
// Overload of `InlinedVector::push_back(...)` for inserting `v` at `end()`
// using move semantics.
void push_back(value_type&& v) {
static_cast<void>(emplace_back(std::move(v)));
}
// `InlinedVector::pop_back()`
//
// Destroys the element at `back()`, reducing the size by `1`.
void pop_back() noexcept {
ABSL_HARDENING_ASSERT(!empty());
AllocatorTraits<A>::destroy(storage_.GetAllocator(), data() + (size() - 1));
storage_.SubtractSize(1);
}
// `InlinedVector::erase(...)`
//
// Erases the element at `pos`, returning an `iterator` pointing to where the
// erased element was located.
//
// NOTE: may return `end()`, which is not dereferenceable.
iterator erase(const_iterator pos) ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(pos >= begin());
ABSL_HARDENING_ASSERT(pos < end());
return storage_.Erase(pos, pos + 1);
}
// Overload of `InlinedVector::erase(...)` that erases every element in the
// range [`from`, `to`), returning an `iterator` pointing to where the first
// erased element was located.
//
// NOTE: may return `end()`, which is not dereferenceable.
iterator erase(const_iterator from,
const_iterator to) ABSL_ATTRIBUTE_LIFETIME_BOUND {
ABSL_HARDENING_ASSERT(from >= begin());
ABSL_HARDENING_ASSERT(from <= to);
ABSL_HARDENING_ASSERT(to <= end());
if (ABSL_PREDICT_TRUE(from != to)) {
return storage_.Erase(from, to);
} else {
return const_cast<iterator>(from);
}
}
// `InlinedVector::clear()`
//
// Destroys all elements in the inlined vector, setting the size to `0` and
// deallocating any held memory.
void clear() noexcept {
inlined_vector_internal::DestroyAdapter<A>::DestroyElements(
storage_.GetAllocator(), data(), size());
storage_.DeallocateIfAllocated();
storage_.SetInlinedSize(0);
}
// `InlinedVector::reserve(...)`
//
// Ensures that there is enough room for at least `n` elements.
void reserve(size_type n) { storage_.Reserve(n); }
// `InlinedVector::shrink_to_fit()`
//
// Attempts to reduce memory usage by moving elements to (or keeping elements
// in) the smallest available buffer sufficient for containing `size()`
// elements.
//
// If `size()` is sufficiently small, the elements will be moved into (or kept
// in) the inlined space.
void shrink_to_fit() {
if (storage_.GetIsAllocated()) {
storage_.ShrinkToFit();
}
}
// `InlinedVector::swap(...)`
//
// Swaps the contents of the inlined vector with `other`.
void swap(InlinedVector& other) {
if (ABSL_PREDICT_TRUE(this != std::addressof(other))) {
storage_.Swap(std::addressof(other.storage_));
}
}
private:
template <typename H, typename TheT, size_t TheN, typename TheA>
friend H AbslHashValue(H h, const absl::InlinedVector<TheT, TheN, TheA>& a);
void MoveAssignment(MemcpyPolicy, InlinedVector&& other) {
// Assumption check: we shouldn't be told to use memcpy to implement move
// assignment unless we have trivially destructible elements and an
// allocator that does nothing fancy.
static_assert(absl::is_trivially_destructible<value_type>::value, "");
static_assert(std::is_same<A, std::allocator<value_type>>::value, "");
// Throw away our existing heap allocation, if any. There is no need to
// destroy the existing elements one by one because we know they are
// trivially destructible.
storage_.DeallocateIfAllocated();
// Adopt the other vector's inline elements or heap allocation.
storage_.MemcpyFrom(other.storage_);
other.storage_.SetInlinedSize(0);
}
// Destroy our existing elements, if any, and adopt the heap-allocated
// elements of the other vector.
//
// REQUIRES: other.storage_.GetIsAllocated()
void DestroyExistingAndAdopt(InlinedVector&& other) {
ABSL_HARDENING_ASSERT(other.storage_.GetIsAllocated());
inlined_vector_internal::DestroyAdapter<A>::DestroyElements(
storage_.GetAllocator(), data(), size());
storage_.DeallocateIfAllocated();
storage_.MemcpyFrom(other.storage_);
other.storage_.SetInlinedSize(0);
}
void MoveAssignment(ElementwiseAssignPolicy, InlinedVector&& other) {
// Fast path: if the other vector is on the heap then we don't worry about
// actually move-assigning each element. Instead we only throw away our own
// existing elements and adopt the heap allocation of the other vector.
if (other.storage_.GetIsAllocated()) {
DestroyExistingAndAdopt(std::move(other));
return;
}
storage_.Assign(IteratorValueAdapter<A, MoveIterator<A>>(
MoveIterator<A>(other.storage_.GetInlinedData())),
other.size());
}
void MoveAssignment(ElementwiseConstructPolicy, InlinedVector&& other) {
// Fast path: if the other vector is on the heap then we don't worry about
// actually move-assigning each element. Instead we only throw away our own
// existing elements and adopt the heap allocation of the other vector.
if (other.storage_.GetIsAllocated()) {
DestroyExistingAndAdopt(std::move(other));
return;
}
inlined_vector_internal::DestroyAdapter<A>::DestroyElements(
storage_.GetAllocator(), data(), size());
storage_.DeallocateIfAllocated();
IteratorValueAdapter<A, MoveIterator<A>> other_values(
MoveIterator<A>(other.storage_.GetInlinedData()));
inlined_vector_internal::ConstructElements<A>(
storage_.GetAllocator(), storage_.GetInlinedData(), other_values,
other.storage_.GetSize());
storage_.SetInlinedSize(other.storage_.GetSize());
}
Storage storage_;
};
// -----------------------------------------------------------------------------
// InlinedVector Non-Member Functions
// -----------------------------------------------------------------------------
// `swap(...)`
//
// Swaps the contents of two inlined vectors.
template <typename T, size_t N, typename A>
void swap(absl::InlinedVector<T, N, A>& a,
absl::InlinedVector<T, N, A>& b) noexcept(noexcept(a.swap(b))) {
a.swap(b);
}
// `operator==(...)`
//
// Tests for value-equality of two inlined vectors.
template <typename T, size_t N, typename A>
bool operator==(const absl::InlinedVector<T, N, A>& a,
const absl::InlinedVector<T, N, A>& b) {
auto a_data = a.data();
auto b_data = b.data();
return std::equal(a_data, a_data + a.size(), b_data, b_data + b.size());
}
// `operator!=(...)`
//
// Tests for value-inequality of two inlined vectors.
template <typename T, size_t N, typename A>
bool operator!=(const absl::InlinedVector<T, N, A>& a,
const absl::InlinedVector<T, N, A>& b) {
return !(a == b);
}
// `operator<(...)`
//
// Tests whether the value of an inlined vector is less than the value of
// another inlined vector using a lexicographical comparison algorithm.
template <typename T, size_t N, typename A>
bool operator<(const absl::InlinedVector<T, N, A>& a,
const absl::InlinedVector<T, N, A>& b) {
auto a_data = a.data();
auto b_data = b.data();
return std::lexicographical_compare(a_data, a_data + a.size(), b_data,
b_data + b.size());
}
// `operator>(...)`
//
// Tests whether the value of an inlined vector is greater than the value of
// another inlined vector using a lexicographical comparison algorithm.
template <typename T, size_t N, typename A>
bool operator>(const absl::InlinedVector<T, N, A>& a,
const absl::InlinedVector<T, N, A>& b) {
return b < a;
}
// `operator<=(...)`
//
// Tests whether the value of an inlined vector is less than or equal to the
// value of another inlined vector using a lexicographical comparison algorithm.
template <typename T, size_t N, typename A>
bool operator<=(const absl::InlinedVector<T, N, A>& a,
const absl::InlinedVector<T, N, A>& b) {
return !(b < a);
}
// `operator>=(...)`
//
// Tests whether the value of an inlined vector is greater than or equal to the
// value of another inlined vector using a lexicographical comparison algorithm.
template <typename T, size_t N, typename A>
bool operator>=(const absl::InlinedVector<T, N, A>& a,
const absl::InlinedVector<T, N, A>& b) {
return !(a < b);
}
// `AbslHashValue(...)`
//
// Provides `absl::Hash` support for `absl::InlinedVector`. It is uncommon to
// call this directly.
template <typename H, typename T, size_t N, typename A>
H AbslHashValue(H h, const absl::InlinedVector<T, N, A>& a) {
auto size = a.size();
return H::combine(H::combine_contiguous(std::move(h), a.data(), size), size);
}
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INLINED_VECTOR_H_

207
Pods/abseil/absl/container/internal/common.h generated Normal file
View File

@@ -0,0 +1,207 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef ABSL_CONTAINER_INTERNAL_COMMON_H_
#define ABSL_CONTAINER_INTERNAL_COMMON_H_
#include <cassert>
#include <type_traits>
#include "absl/meta/type_traits.h"
#include "absl/types/optional.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
template <class, class = void>
struct IsTransparent : std::false_type {};
template <class T>
struct IsTransparent<T, absl::void_t<typename T::is_transparent>>
: std::true_type {};
template <bool is_transparent>
struct KeyArg {
// Transparent. Forward `K`.
template <typename K, typename key_type>
using type = K;
};
template <>
struct KeyArg<false> {
// Not transparent. Always use `key_type`.
template <typename K, typename key_type>
using type = key_type;
};
// The node_handle concept from C++17.
// We specialize node_handle for sets and maps. node_handle_base holds the
// common API of both.
template <typename PolicyTraits, typename Alloc>
class node_handle_base {
protected:
using slot_type = typename PolicyTraits::slot_type;
public:
using allocator_type = Alloc;
constexpr node_handle_base() = default;
node_handle_base(node_handle_base&& other) noexcept {
*this = std::move(other);
}
~node_handle_base() { destroy(); }
node_handle_base& operator=(node_handle_base&& other) noexcept {
destroy();
if (!other.empty()) {
alloc_ = other.alloc_;
PolicyTraits::transfer(alloc(), slot(), other.slot());
other.reset();
}
return *this;
}
bool empty() const noexcept { return !alloc_; }
explicit operator bool() const noexcept { return !empty(); }
allocator_type get_allocator() const { return *alloc_; }
protected:
friend struct CommonAccess;
struct transfer_tag_t {};
node_handle_base(transfer_tag_t, const allocator_type& a, slot_type* s)
: alloc_(a) {
PolicyTraits::transfer(alloc(), slot(), s);
}
struct construct_tag_t {};
template <typename... Args>
node_handle_base(construct_tag_t, const allocator_type& a, Args&&... args)
: alloc_(a) {
PolicyTraits::construct(alloc(), slot(), std::forward<Args>(args)...);
}
void destroy() {
if (!empty()) {
PolicyTraits::destroy(alloc(), slot());
reset();
}
}
void reset() {
assert(alloc_.has_value());
alloc_ = absl::nullopt;
}
slot_type* slot() const {
assert(!empty());
return reinterpret_cast<slot_type*>(std::addressof(slot_space_));
}
allocator_type* alloc() { return std::addressof(*alloc_); }
private:
absl::optional<allocator_type> alloc_ = {};
alignas(slot_type) mutable unsigned char slot_space_[sizeof(slot_type)] = {};
};
// For sets.
template <typename Policy, typename PolicyTraits, typename Alloc,
typename = void>
class node_handle : public node_handle_base<PolicyTraits, Alloc> {
using Base = node_handle_base<PolicyTraits, Alloc>;
public:
using value_type = typename PolicyTraits::value_type;
constexpr node_handle() {}
value_type& value() const { return PolicyTraits::element(this->slot()); }
private:
friend struct CommonAccess;
using Base::Base;
};
// For maps.
template <typename Policy, typename PolicyTraits, typename Alloc>
class node_handle<Policy, PolicyTraits, Alloc,
absl::void_t<typename Policy::mapped_type>>
: public node_handle_base<PolicyTraits, Alloc> {
using Base = node_handle_base<PolicyTraits, Alloc>;
using slot_type = typename PolicyTraits::slot_type;
public:
using key_type = typename Policy::key_type;
using mapped_type = typename Policy::mapped_type;
constexpr node_handle() {}
// When C++17 is available, we can use std::launder to provide mutable
// access to the key. Otherwise, we provide const access.
auto key() const
-> decltype(PolicyTraits::mutable_key(std::declval<slot_type*>())) {
return PolicyTraits::mutable_key(this->slot());
}
mapped_type& mapped() const {
return PolicyTraits::value(&PolicyTraits::element(this->slot()));
}
private:
friend struct CommonAccess;
using Base::Base;
};
// Provide access to non-public node-handle functions.
struct CommonAccess {
template <typename Node>
static auto GetSlot(const Node& node) -> decltype(node.slot()) {
return node.slot();
}
template <typename Node>
static void Destroy(Node* node) {
node->destroy();
}
template <typename Node>
static void Reset(Node* node) {
node->reset();
}
template <typename T, typename... Args>
static T Transfer(Args&&... args) {
return T(typename T::transfer_tag_t{}, std::forward<Args>(args)...);
}
template <typename T, typename... Args>
static T Construct(Args&&... args) {
return T(typename T::construct_tag_t{}, std::forward<Args>(args)...);
}
};
// Implement the insert_return_type<> concept of C++17.
template <class Iterator, class NodeType>
struct InsertReturnType {
Iterator position;
bool inserted;
NodeType node;
};
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_COMMON_H_

134
Pods/abseil/absl/container/internal/common_policy_traits.h generated Normal file
View File

@@ -0,0 +1,134 @@
// Copyright 2022 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef ABSL_CONTAINER_INTERNAL_COMMON_POLICY_TRAITS_H_
#define ABSL_CONTAINER_INTERNAL_COMMON_POLICY_TRAITS_H_
#include <cstddef>
#include <cstring>
#include <memory>
#include <new>
#include <type_traits>
#include <utility>
#include "absl/meta/type_traits.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
// Defines how slots are initialized/destroyed/moved.
template <class Policy, class = void>
struct common_policy_traits {
// The actual object stored in the container.
using slot_type = typename Policy::slot_type;
using reference = decltype(Policy::element(std::declval<slot_type*>()));
using value_type = typename std::remove_reference<reference>::type;
// PRECONDITION: `slot` is UNINITIALIZED
// POSTCONDITION: `slot` is INITIALIZED
template <class Alloc, class... Args>
static void construct(Alloc* alloc, slot_type* slot, Args&&... args) {
Policy::construct(alloc, slot, std::forward<Args>(args)...);
}
// PRECONDITION: `slot` is INITIALIZED
// POSTCONDITION: `slot` is UNINITIALIZED
template <class Alloc>
static void destroy(Alloc* alloc, slot_type* slot) {
Policy::destroy(alloc, slot);
}
// Transfers the `old_slot` to `new_slot`. Any memory allocated by the
// allocator inside `old_slot` to `new_slot` can be transferred.
//
// OPTIONAL: defaults to:
//
// clone(new_slot, std::move(*old_slot));
// destroy(old_slot);
//
// PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED
// POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is
// UNINITIALIZED
template <class Alloc>
static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) {
transfer_impl(alloc, new_slot, old_slot, Rank0{});
}
// PRECONDITION: `slot` is INITIALIZED
// POSTCONDITION: `slot` is INITIALIZED
// Note: we use remove_const_t so that the two overloads have different args
// in the case of sets with explicitly const value_types.
template <class P = Policy>
static auto element(absl::remove_const_t<slot_type>* slot)
-> decltype(P::element(slot)) {
return P::element(slot);
}
template <class P = Policy>
static auto element(const slot_type* slot) -> decltype(P::element(slot)) {
return P::element(slot);
}
static constexpr bool transfer_uses_memcpy() {
return std::is_same<decltype(transfer_impl<std::allocator<char>>(
nullptr, nullptr, nullptr, Rank0{})),
std::true_type>::value;
}
private:
// To rank the overloads below for overload resolution. Rank0 is preferred.
struct Rank2 {};
struct Rank1 : Rank2 {};
struct Rank0 : Rank1 {};
// Use auto -> decltype as an enabler.
// P::transfer returns std::true_type if transfer uses memcpy (e.g. in
// node_slot_policy).
template <class Alloc, class P = Policy>
static auto transfer_impl(Alloc* alloc, slot_type* new_slot,
slot_type* old_slot, Rank0)
-> decltype(P::transfer(alloc, new_slot, old_slot)) {
return P::transfer(alloc, new_slot, old_slot);
}
#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606
// This overload returns true_type for the trait below.
// The conditional_t is to make the enabler type dependent.
template <class Alloc,
typename = std::enable_if_t<absl::is_trivially_relocatable<
std::conditional_t<false, Alloc, value_type>>::value>>
static std::true_type transfer_impl(Alloc*, slot_type* new_slot,
slot_type* old_slot, Rank1) {
// TODO(b/247130232): remove casts after fixing warnings.
// TODO(b/251814870): remove casts after fixing warnings.
std::memcpy(
static_cast<void*>(std::launder(
const_cast<std::remove_const_t<value_type>*>(&element(new_slot)))),
static_cast<const void*>(&element(old_slot)), sizeof(value_type));
return {};
}
#endif
template <class Alloc>
static void transfer_impl(Alloc* alloc, slot_type* new_slot,
slot_type* old_slot, Rank2) {
construct(alloc, new_slot, std::move(element(old_slot)));
destroy(alloc, old_slot);
}
};
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_COMMON_POLICY_TRAITS_H_

272
Pods/abseil/absl/container/internal/compressed_tuple.h generated Normal file
View File

@@ -0,0 +1,272 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Helper class to perform the Empty Base Optimization.
// Ts can contain classes and non-classes, empty or not. For the ones that
// are empty classes, we perform the optimization. If all types in Ts are empty
// classes, then CompressedTuple<Ts...> is itself an empty class.
//
// To access the members, use member get<N>() function.
//
// Eg:
// absl::container_internal::CompressedTuple<int, T1, T2, T3> value(7, t1, t2,
// t3);
// assert(value.get<0>() == 7);
// T1& t1 = value.get<1>();
// const T2& t2 = value.get<2>();
// ...
//
// https://en.cppreference.com/w/cpp/language/ebo
#ifndef ABSL_CONTAINER_INTERNAL_COMPRESSED_TUPLE_H_
#define ABSL_CONTAINER_INTERNAL_COMPRESSED_TUPLE_H_
#include <initializer_list>
#include <tuple>
#include <type_traits>
#include <utility>
#include "absl/utility/utility.h"
#if defined(_MSC_VER) && !defined(__NVCC__)
// We need to mark these classes with this declspec to ensure that
// CompressedTuple happens.
#define ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC __declspec(empty_bases)
#else
#define ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC
#endif
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
template <typename... Ts>
class CompressedTuple;
namespace internal_compressed_tuple {
template <typename D, size_t I>
struct Elem;
template <typename... B, size_t I>
struct Elem<CompressedTuple<B...>, I>
: std::tuple_element<I, std::tuple<B...>> {};
template <typename D, size_t I>
using ElemT = typename Elem<D, I>::type;
// We can't use EBCO on other CompressedTuples because that would mean that we
// derive from multiple Storage<> instantiations with the same I parameter,
// and potentially from multiple identical Storage<> instantiations. So anytime
// we use type inheritance rather than encapsulation, we mark
// CompressedTupleImpl, to make this easy to detect.
struct uses_inheritance {};
template <typename T>
constexpr bool ShouldUseBase() {
return std::is_class<T>::value && std::is_empty<T>::value &&
!std::is_final<T>::value &&
!std::is_base_of<uses_inheritance, T>::value;
}
// The storage class provides two specializations:
// - For empty classes, it stores T as a base class.
// - For everything else, it stores T as a member.
template <typename T, size_t I, bool UseBase = ShouldUseBase<T>()>
struct Storage {
T value;
constexpr Storage() = default;
template <typename V>
explicit constexpr Storage(absl::in_place_t, V&& v)
: value(absl::forward<V>(v)) {}
constexpr const T& get() const& { return value; }
T& get() & { return value; }
constexpr const T&& get() const&& { return absl::move(*this).value; }
T&& get() && { return std::move(*this).value; }
};
template <typename T, size_t I>
struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage<T, I, true> : T {
constexpr Storage() = default;
template <typename V>
explicit constexpr Storage(absl::in_place_t, V&& v)
: T(absl::forward<V>(v)) {}
constexpr const T& get() const& { return *this; }
T& get() & { return *this; }
constexpr const T&& get() const&& { return absl::move(*this); }
T&& get() && { return std::move(*this); }
};
template <typename D, typename I, bool ShouldAnyUseBase>
struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl;
template <typename... Ts, size_t... I, bool ShouldAnyUseBase>
struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl<
CompressedTuple<Ts...>, absl::index_sequence<I...>, ShouldAnyUseBase>
// We use the dummy identity function through std::integral_constant to
// convince MSVC of accepting and expanding I in that context. Without it
// you would get:
// error C3548: 'I': parameter pack cannot be used in this context
: uses_inheritance,
Storage<Ts, std::integral_constant<size_t, I>::value>... {
constexpr CompressedTupleImpl() = default;
template <typename... Vs>
explicit constexpr CompressedTupleImpl(absl::in_place_t, Vs&&... args)
: Storage<Ts, I>(absl::in_place, absl::forward<Vs>(args))... {}
friend CompressedTuple<Ts...>;
};
template <typename... Ts, size_t... I>
struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl<
CompressedTuple<Ts...>, absl::index_sequence<I...>, false>
// We use the dummy identity function as above...
: Storage<Ts, std::integral_constant<size_t, I>::value, false>... {
constexpr CompressedTupleImpl() = default;
template <typename... Vs>
explicit constexpr CompressedTupleImpl(absl::in_place_t, Vs&&... args)
: Storage<Ts, I, false>(absl::in_place, absl::forward<Vs>(args))... {}
friend CompressedTuple<Ts...>;
};
std::false_type Or(std::initializer_list<std::false_type>);
std::true_type Or(std::initializer_list<bool>);
// MSVC requires this to be done separately rather than within the declaration
// of CompressedTuple below.
template <typename... Ts>
constexpr bool ShouldAnyUseBase() {
return decltype(
Or({std::integral_constant<bool, ShouldUseBase<Ts>()>()...})){};
}
template <typename T, typename V>
using TupleElementMoveConstructible =
typename std::conditional<std::is_reference<T>::value,
std::is_convertible<V, T>,
std::is_constructible<T, V&&>>::type;
template <bool SizeMatches, class T, class... Vs>
struct TupleMoveConstructible : std::false_type {};
template <class... Ts, class... Vs>
struct TupleMoveConstructible<true, CompressedTuple<Ts...>, Vs...>
: std::integral_constant<
bool, absl::conjunction<
TupleElementMoveConstructible<Ts, Vs&&>...>::value> {};
template <typename T>
struct compressed_tuple_size;
template <typename... Es>
struct compressed_tuple_size<CompressedTuple<Es...>>
: public std::integral_constant<std::size_t, sizeof...(Es)> {};
template <class T, class... Vs>
struct TupleItemsMoveConstructible
: std::integral_constant<
bool, TupleMoveConstructible<compressed_tuple_size<T>::value ==
sizeof...(Vs),
T, Vs...>::value> {};
} // namespace internal_compressed_tuple
// Helper class to perform the Empty Base Class Optimization.
// Ts can contain classes and non-classes, empty or not. For the ones that
// are empty classes, we perform the CompressedTuple. If all types in Ts are
// empty classes, then CompressedTuple<Ts...> is itself an empty class. (This
// does not apply when one or more of those empty classes is itself an empty
// CompressedTuple.)
//
// To access the members, use member .get<N>() function.
//
// Eg:
// absl::container_internal::CompressedTuple<int, T1, T2, T3> value(7, t1, t2,
// t3);
// assert(value.get<0>() == 7);
// T1& t1 = value.get<1>();
// const T2& t2 = value.get<2>();
// ...
//
// https://en.cppreference.com/w/cpp/language/ebo
template <typename... Ts>
class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple
: private internal_compressed_tuple::CompressedTupleImpl<
CompressedTuple<Ts...>, absl::index_sequence_for<Ts...>,
internal_compressed_tuple::ShouldAnyUseBase<Ts...>()> {
private:
template <int I>
using ElemT = internal_compressed_tuple::ElemT<CompressedTuple, I>;
template <int I>
using StorageT = internal_compressed_tuple::Storage<ElemT<I>, I>;
public:
// There seems to be a bug in MSVC dealing in which using '=default' here will
// cause the compiler to ignore the body of other constructors. The work-
// around is to explicitly implement the default constructor.
#if defined(_MSC_VER)
constexpr CompressedTuple() : CompressedTuple::CompressedTupleImpl() {}
#else
constexpr CompressedTuple() = default;
#endif
explicit constexpr CompressedTuple(const Ts&... base)
: CompressedTuple::CompressedTupleImpl(absl::in_place, base...) {}
template <typename First, typename... Vs,
absl::enable_if_t<
absl::conjunction<
// Ensure we are not hiding default copy/move constructors.
absl::negation<std::is_same<void(CompressedTuple),
void(absl::decay_t<First>)>>,
internal_compressed_tuple::TupleItemsMoveConstructible<
CompressedTuple<Ts...>, First, Vs...>>::value,
bool> = true>
explicit constexpr CompressedTuple(First&& first, Vs&&... base)
: CompressedTuple::CompressedTupleImpl(absl::in_place,
absl::forward<First>(first),
absl::forward<Vs>(base)...) {}
template <int I>
ElemT<I>& get() & {
return StorageT<I>::get();
}
template <int I>
constexpr const ElemT<I>& get() const& {
return StorageT<I>::get();
}
template <int I>
ElemT<I>&& get() && {
return std::move(*this).StorageT<I>::get();
}
template <int I>
constexpr const ElemT<I>&& get() const&& {
return absl::move(*this).StorageT<I>::get();
}
};
// Explicit specialization for a zero-element tuple
// (needed to avoid ambiguous overloads for the default constructor).
template <>
class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple<> {};
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#undef ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC
#endif // ABSL_CONTAINER_INTERNAL_COMPRESSED_TUPLE_H_

458
Pods/abseil/absl/container/internal/container_memory.h generated Normal file
View File

@@ -0,0 +1,458 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef ABSL_CONTAINER_INTERNAL_CONTAINER_MEMORY_H_
#define ABSL_CONTAINER_INTERNAL_CONTAINER_MEMORY_H_
#include <cassert>
#include <cstddef>
#include <cstring>
#include <memory>
#include <new>
#include <tuple>
#include <type_traits>
#include <utility>
#include "absl/base/config.h"
#include "absl/memory/memory.h"
#include "absl/meta/type_traits.h"
#include "absl/utility/utility.h"
#ifdef ABSL_HAVE_ADDRESS_SANITIZER
#include <sanitizer/asan_interface.h>
#endif
#ifdef ABSL_HAVE_MEMORY_SANITIZER
#include <sanitizer/msan_interface.h>
#endif
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
template <size_t Alignment>
struct alignas(Alignment) AlignedType {};
// Allocates at least n bytes aligned to the specified alignment.
// Alignment must be a power of 2. It must be positive.
//
// Note that many allocators don't honor alignment requirements above certain
// threshold (usually either alignof(std::max_align_t) or alignof(void*)).
// Allocate() doesn't apply alignment corrections. If the underlying allocator
// returns insufficiently alignment pointer, that's what you are going to get.
template <size_t Alignment, class Alloc>
void* Allocate(Alloc* alloc, size_t n) {
static_assert(Alignment > 0, "");
assert(n && "n must be positive");
using M = AlignedType<Alignment>;
using A = typename absl::allocator_traits<Alloc>::template rebind_alloc<M>;
using AT = typename absl::allocator_traits<Alloc>::template rebind_traits<M>;
// On macOS, "mem_alloc" is a #define with one argument defined in
// rpc/types.h, so we can't name the variable "mem_alloc" and initialize it
// with the "foo(bar)" syntax.
A my_mem_alloc(*alloc);
void* p = AT::allocate(my_mem_alloc, (n + sizeof(M) - 1) / sizeof(M));
assert(reinterpret_cast<uintptr_t>(p) % Alignment == 0 &&
"allocator does not respect alignment");
return p;
}
// The pointer must have been previously obtained by calling
// Allocate<Alignment>(alloc, n).
template <size_t Alignment, class Alloc>
void Deallocate(Alloc* alloc, void* p, size_t n) {
static_assert(Alignment > 0, "");
assert(n && "n must be positive");
using M = AlignedType<Alignment>;
using A = typename absl::allocator_traits<Alloc>::template rebind_alloc<M>;
using AT = typename absl::allocator_traits<Alloc>::template rebind_traits<M>;
// On macOS, "mem_alloc" is a #define with one argument defined in
// rpc/types.h, so we can't name the variable "mem_alloc" and initialize it
// with the "foo(bar)" syntax.
A my_mem_alloc(*alloc);
AT::deallocate(my_mem_alloc, static_cast<M*>(p),
(n + sizeof(M) - 1) / sizeof(M));
}
namespace memory_internal {
// Constructs T into uninitialized storage pointed by `ptr` using the args
// specified in the tuple.
template <class Alloc, class T, class Tuple, size_t... I>
void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t,
absl::index_sequence<I...>) {
absl::allocator_traits<Alloc>::construct(
*alloc, ptr, std::get<I>(std::forward<Tuple>(t))...);
}
template <class T, class F>
struct WithConstructedImplF {
template <class... Args>
decltype(std::declval<F>()(std::declval<T>())) operator()(
Args&&... args) const {
return std::forward<F>(f)(T(std::forward<Args>(args)...));
}
F&& f;
};
template <class T, class Tuple, size_t... Is, class F>
decltype(std::declval<F>()(std::declval<T>())) WithConstructedImpl(
Tuple&& t, absl::index_sequence<Is...>, F&& f) {
return WithConstructedImplF<T, F>{std::forward<F>(f)}(
std::get<Is>(std::forward<Tuple>(t))...);
}
template <class T, size_t... Is>
auto TupleRefImpl(T&& t, absl::index_sequence<Is...>)
-> decltype(std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...)) {
return std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...);
}
// Returns a tuple of references to the elements of the input tuple. T must be a
// tuple.
template <class T>
auto TupleRef(T&& t) -> decltype(TupleRefImpl(
std::forward<T>(t),
absl::make_index_sequence<
std::tuple_size<typename std::decay<T>::type>::value>())) {
return TupleRefImpl(
std::forward<T>(t),
absl::make_index_sequence<
std::tuple_size<typename std::decay<T>::type>::value>());
}
template <class F, class K, class V>
decltype(std::declval<F>()(std::declval<const K&>(), std::piecewise_construct,
std::declval<std::tuple<K>>(), std::declval<V>()))
DecomposePairImpl(F&& f, std::pair<std::tuple<K>, V> p) {
const auto& key = std::get<0>(p.first);
return std::forward<F>(f)(key, std::piecewise_construct, std::move(p.first),
std::move(p.second));
}
} // namespace memory_internal
// Constructs T into uninitialized storage pointed by `ptr` using the args
// specified in the tuple.
template <class Alloc, class T, class Tuple>
void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) {
memory_internal::ConstructFromTupleImpl(
alloc, ptr, std::forward<Tuple>(t),
absl::make_index_sequence<
std::tuple_size<typename std::decay<Tuple>::type>::value>());
}
// Constructs T using the args specified in the tuple and calls F with the
// constructed value.
template <class T, class Tuple, class F>
decltype(std::declval<F>()(std::declval<T>())) WithConstructed(Tuple&& t,
F&& f) {
return memory_internal::WithConstructedImpl<T>(
std::forward<Tuple>(t),
absl::make_index_sequence<
std::tuple_size<typename std::decay<Tuple>::type>::value>(),
std::forward<F>(f));
}
// Given arguments of an std::pair's constructor, PairArgs() returns a pair of
// tuples with references to the passed arguments. The tuples contain
// constructor arguments for the first and the second elements of the pair.
//
// The following two snippets are equivalent.
//
// 1. std::pair<F, S> p(args...);
//
// 2. auto a = PairArgs(args...);
// std::pair<F, S> p(std::piecewise_construct,
// std::move(a.first), std::move(a.second));
inline std::pair<std::tuple<>, std::tuple<>> PairArgs() { return {}; }
template <class F, class S>
std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(F&& f, S&& s) {
return {std::piecewise_construct, std::forward_as_tuple(std::forward<F>(f)),
std::forward_as_tuple(std::forward<S>(s))};
}
template <class F, class S>
std::pair<std::tuple<const F&>, std::tuple<const S&>> PairArgs(
const std::pair<F, S>& p) {
return PairArgs(p.first, p.second);
}
template <class F, class S>
std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(std::pair<F, S>&& p) {
return PairArgs(std::forward<F>(p.first), std::forward<S>(p.second));
}
template <class F, class S>
auto PairArgs(std::piecewise_construct_t, F&& f, S&& s)
-> decltype(std::make_pair(memory_internal::TupleRef(std::forward<F>(f)),
memory_internal::TupleRef(std::forward<S>(s)))) {
return std::make_pair(memory_internal::TupleRef(std::forward<F>(f)),
memory_internal::TupleRef(std::forward<S>(s)));
}
// A helper function for implementing apply() in map policies.
template <class F, class... Args>
auto DecomposePair(F&& f, Args&&... args)
-> decltype(memory_internal::DecomposePairImpl(
std::forward<F>(f), PairArgs(std::forward<Args>(args)...))) {
return memory_internal::DecomposePairImpl(
std::forward<F>(f), PairArgs(std::forward<Args>(args)...));
}
// A helper function for implementing apply() in set policies.
template <class F, class Arg>
decltype(std::declval<F>()(std::declval<const Arg&>(), std::declval<Arg>()))
DecomposeValue(F&& f, Arg&& arg) {
const auto& key = arg;
return std::forward<F>(f)(key, std::forward<Arg>(arg));
}
// Helper functions for asan and msan.
inline void SanitizerPoisonMemoryRegion(const void* m, size_t s) {
#ifdef ABSL_HAVE_ADDRESS_SANITIZER
ASAN_POISON_MEMORY_REGION(m, s);
#endif
#ifdef ABSL_HAVE_MEMORY_SANITIZER
__msan_poison(m, s);
#endif
(void)m;
(void)s;
}
inline void SanitizerUnpoisonMemoryRegion(const void* m, size_t s) {
#ifdef ABSL_HAVE_ADDRESS_SANITIZER
ASAN_UNPOISON_MEMORY_REGION(m, s);
#endif
#ifdef ABSL_HAVE_MEMORY_SANITIZER
__msan_unpoison(m, s);
#endif
(void)m;
(void)s;
}
template <typename T>
inline void SanitizerPoisonObject(const T* object) {
SanitizerPoisonMemoryRegion(object, sizeof(T));
}
template <typename T>
inline void SanitizerUnpoisonObject(const T* object) {
SanitizerUnpoisonMemoryRegion(object, sizeof(T));
}
namespace memory_internal {
// If Pair is a standard-layout type, OffsetOf<Pair>::kFirst and
// OffsetOf<Pair>::kSecond are equivalent to offsetof(Pair, first) and
// offsetof(Pair, second) respectively. Otherwise they are -1.
//
// The purpose of OffsetOf is to avoid calling offsetof() on non-standard-layout
// type, which is non-portable.
template <class Pair, class = std::true_type>
struct OffsetOf {
static constexpr size_t kFirst = static_cast<size_t>(-1);
static constexpr size_t kSecond = static_cast<size_t>(-1);
};
template <class Pair>
struct OffsetOf<Pair, typename std::is_standard_layout<Pair>::type> {
static constexpr size_t kFirst = offsetof(Pair, first);
static constexpr size_t kSecond = offsetof(Pair, second);
};
template <class K, class V>
struct IsLayoutCompatible {
private:
struct Pair {
K first;
V second;
};
// Is P layout-compatible with Pair?
template <class P>
static constexpr bool LayoutCompatible() {
return std::is_standard_layout<P>() && sizeof(P) == sizeof(Pair) &&
alignof(P) == alignof(Pair) &&
memory_internal::OffsetOf<P>::kFirst ==
memory_internal::OffsetOf<Pair>::kFirst &&
memory_internal::OffsetOf<P>::kSecond ==
memory_internal::OffsetOf<Pair>::kSecond;
}
public:
// Whether pair<const K, V> and pair<K, V> are layout-compatible. If they are,
// then it is safe to store them in a union and read from either.
static constexpr bool value = std::is_standard_layout<K>() &&
std::is_standard_layout<Pair>() &&
memory_internal::OffsetOf<Pair>::kFirst == 0 &&
LayoutCompatible<std::pair<K, V>>() &&
LayoutCompatible<std::pair<const K, V>>();
};
} // namespace memory_internal
// The internal storage type for key-value containers like flat_hash_map.
//
// It is convenient for the value_type of a flat_hash_map<K, V> to be
// pair<const K, V>; the "const K" prevents accidental modification of the key
// when dealing with the reference returned from find() and similar methods.
// However, this creates other problems; we want to be able to emplace(K, V)
// efficiently with move operations, and similarly be able to move a
// pair<K, V> in insert().
//
// The solution is this union, which aliases the const and non-const versions
// of the pair. This also allows flat_hash_map<const K, V> to work, even though
// that has the same efficiency issues with move in emplace() and insert() -
// but people do it anyway.
//
// If kMutableKeys is false, only the value member can be accessed.
//
// If kMutableKeys is true, key can be accessed through all slots while value
// and mutable_value must be accessed only via INITIALIZED slots. Slots are
// created and destroyed via mutable_value so that the key can be moved later.
//
// Accessing one of the union fields while the other is active is safe as
// long as they are layout-compatible, which is guaranteed by the definition of
// kMutableKeys. For C++11, the relevant section of the standard is
// https://timsong-cpp.github.io/cppwp/n3337/class.mem#19 (9.2.19)
template <class K, class V>
union map_slot_type {
map_slot_type() {}
~map_slot_type() = delete;
using value_type = std::pair<const K, V>;
using mutable_value_type =
std::pair<absl::remove_const_t<K>, absl::remove_const_t<V>>;
value_type value;
mutable_value_type mutable_value;
absl::remove_const_t<K> key;
};
template <class K, class V>
struct map_slot_policy {
using slot_type = map_slot_type<K, V>;
using value_type = std::pair<const K, V>;
using mutable_value_type =
std::pair<absl::remove_const_t<K>, absl::remove_const_t<V>>;
private:
static void emplace(slot_type* slot) {
// The construction of union doesn't do anything at runtime but it allows us
// to access its members without violating aliasing rules.
new (slot) slot_type;
}
// If pair<const K, V> and pair<K, V> are layout-compatible, we can accept one
// or the other via slot_type. We are also free to access the key via
// slot_type::key in this case.
using kMutableKeys = memory_internal::IsLayoutCompatible<K, V>;
public:
static value_type& element(slot_type* slot) { return slot->value; }
static const value_type& element(const slot_type* slot) {
return slot->value;
}
// When C++17 is available, we can use std::launder to provide mutable
// access to the key for use in node handle.
#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606
static K& mutable_key(slot_type* slot) {
// Still check for kMutableKeys so that we can avoid calling std::launder
// unless necessary because it can interfere with optimizations.
return kMutableKeys::value ? slot->key
: *std::launder(const_cast<K*>(
std::addressof(slot->value.first)));
}
#else // !(defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606)
static const K& mutable_key(slot_type* slot) { return key(slot); }
#endif
static const K& key(const slot_type* slot) {
return kMutableKeys::value ? slot->key : slot->value.first;
}
template <class Allocator, class... Args>
static void construct(Allocator* alloc, slot_type* slot, Args&&... args) {
emplace(slot);
if (kMutableKeys::value) {
absl::allocator_traits<Allocator>::construct(*alloc, &slot->mutable_value,
std::forward<Args>(args)...);
} else {
absl::allocator_traits<Allocator>::construct(*alloc, &slot->value,
std::forward<Args>(args)...);
}
}
// Construct this slot by moving from another slot.
template <class Allocator>
static void construct(Allocator* alloc, slot_type* slot, slot_type* other) {
emplace(slot);
if (kMutableKeys::value) {
absl::allocator_traits<Allocator>::construct(
*alloc, &slot->mutable_value, std::move(other->mutable_value));
} else {
absl::allocator_traits<Allocator>::construct(*alloc, &slot->value,
std::move(other->value));
}
}
// Construct this slot by copying from another slot.
template <class Allocator>
static void construct(Allocator* alloc, slot_type* slot,
const slot_type* other) {
emplace(slot);
absl::allocator_traits<Allocator>::construct(*alloc, &slot->value,
other->value);
}
template <class Allocator>
static void destroy(Allocator* alloc, slot_type* slot) {
if (kMutableKeys::value) {
absl::allocator_traits<Allocator>::destroy(*alloc, &slot->mutable_value);
} else {
absl::allocator_traits<Allocator>::destroy(*alloc, &slot->value);
}
}
template <class Allocator>
static auto transfer(Allocator* alloc, slot_type* new_slot,
slot_type* old_slot) {
auto is_relocatable =
typename absl::is_trivially_relocatable<value_type>::type();
emplace(new_slot);
#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606
if (is_relocatable) {
// TODO(b/247130232,b/251814870): remove casts after fixing warnings.
std::memcpy(static_cast<void*>(std::launder(&new_slot->value)),
static_cast<const void*>(&old_slot->value),
sizeof(value_type));
return is_relocatable;
}
#endif
if (kMutableKeys::value) {
absl::allocator_traits<Allocator>::construct(
*alloc, &new_slot->mutable_value, std::move(old_slot->mutable_value));
} else {
absl::allocator_traits<Allocator>::construct(*alloc, &new_slot->value,
std::move(old_slot->value));
}
destroy(alloc, old_slot);
return is_relocatable;
}
};
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_CONTAINER_MEMORY_H_

209
Pods/abseil/absl/container/internal/hash_function_defaults.h generated Normal file
View File

@@ -0,0 +1,209 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Define the default Hash and Eq functions for SwissTable containers.
//
// std::hash<T> and std::equal_to<T> are not appropriate hash and equal
// functions for SwissTable containers. There are two reasons for this.
//
// SwissTable containers are power of 2 sized containers:
//
// This means they use the lower bits of the hash value to find the slot for
// each entry. The typical hash function for integral types is the identity.
// This is a very weak hash function for SwissTable and any power of 2 sized
// hashtable implementation which will lead to excessive collisions. For
// SwissTable we use murmur3 style mixing to reduce collisions to a minimum.
//
// SwissTable containers support heterogeneous lookup:
//
// In order to make heterogeneous lookup work, hash and equal functions must be
// polymorphic. At the same time they have to satisfy the same requirements the
// C++ standard imposes on hash functions and equality operators. That is:
//
// if hash_default_eq<T>(a, b) returns true for any a and b of type T, then
// hash_default_hash<T>(a) must equal hash_default_hash<T>(b)
//
// For SwissTable containers this requirement is relaxed to allow a and b of
// any and possibly different types. Note that like the standard the hash and
// equal functions are still bound to T. This is important because some type U
// can be hashed by/tested for equality differently depending on T. A notable
// example is `const char*`. `const char*` is treated as a c-style string when
// the hash function is hash<std::string> but as a pointer when the hash
// function is hash<void*>.
//
#ifndef ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_
#define ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_
#include <stdint.h>
#include <cstddef>
#include <memory>
#include <string>
#include <type_traits>
#include "absl/base/config.h"
#include "absl/hash/hash.h"
#include "absl/strings/cord.h"
#include "absl/strings/string_view.h"
#ifdef ABSL_HAVE_STD_STRING_VIEW
#include <string_view>
#endif
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
// The hash of an object of type T is computed by using absl::Hash.
template <class T, class E = void>
struct HashEq {
using Hash = absl::Hash<T>;
using Eq = std::equal_to<T>;
};
struct StringHash {
using is_transparent = void;
size_t operator()(absl::string_view v) const {
return absl::Hash<absl::string_view>{}(v);
}
size_t operator()(const absl::Cord& v) const {
return absl::Hash<absl::Cord>{}(v);
}
};
struct StringEq {
using is_transparent = void;
bool operator()(absl::string_view lhs, absl::string_view rhs) const {
return lhs == rhs;
}
bool operator()(const absl::Cord& lhs, const absl::Cord& rhs) const {
return lhs == rhs;
}
bool operator()(const absl::Cord& lhs, absl::string_view rhs) const {
return lhs == rhs;
}
bool operator()(absl::string_view lhs, const absl::Cord& rhs) const {
return lhs == rhs;
}
};
// Supports heterogeneous lookup for string-like elements.
struct StringHashEq {
using Hash = StringHash;
using Eq = StringEq;
};
template <>
struct HashEq<std::string> : StringHashEq {};
template <>
struct HashEq<absl::string_view> : StringHashEq {};
template <>
struct HashEq<absl::Cord> : StringHashEq {};
#ifdef ABSL_HAVE_STD_STRING_VIEW
template <typename TChar>
struct BasicStringHash {
using is_transparent = void;
size_t operator()(std::basic_string_view<TChar> v) const {
return absl::Hash<std::basic_string_view<TChar>>{}(v);
}
};
template <typename TChar>
struct BasicStringEq {
using is_transparent = void;
bool operator()(std::basic_string_view<TChar> lhs,
std::basic_string_view<TChar> rhs) const {
return lhs == rhs;
}
};
// Supports heterogeneous lookup for w/u16/u32 string + string_view + char*.
template <typename TChar>
struct BasicStringHashEq {
using Hash = BasicStringHash<TChar>;
using Eq = BasicStringEq<TChar>;
};
template <>
struct HashEq<std::wstring> : BasicStringHashEq<wchar_t> {};
template <>
struct HashEq<std::wstring_view> : BasicStringHashEq<wchar_t> {};
template <>
struct HashEq<std::u16string> : BasicStringHashEq<char16_t> {};
template <>
struct HashEq<std::u16string_view> : BasicStringHashEq<char16_t> {};
template <>
struct HashEq<std::u32string> : BasicStringHashEq<char32_t> {};
template <>
struct HashEq<std::u32string_view> : BasicStringHashEq<char32_t> {};
#endif // ABSL_HAVE_STD_STRING_VIEW
// Supports heterogeneous lookup for pointers and smart pointers.
template <class T>
struct HashEq<T*> {
struct Hash {
using is_transparent = void;
template <class U>
size_t operator()(const U& ptr) const {
return absl::Hash<const T*>{}(HashEq::ToPtr(ptr));
}
};
struct Eq {
using is_transparent = void;
template <class A, class B>
bool operator()(const A& a, const B& b) const {
return HashEq::ToPtr(a) == HashEq::ToPtr(b);
}
};
private:
static const T* ToPtr(const T* ptr) { return ptr; }
template <class U, class D>
static const T* ToPtr(const std::unique_ptr<U, D>& ptr) {
return ptr.get();
}
template <class U>
static const T* ToPtr(const std::shared_ptr<U>& ptr) {
return ptr.get();
}
};
template <class T, class D>
struct HashEq<std::unique_ptr<T, D>> : HashEq<T*> {};
template <class T>
struct HashEq<std::shared_ptr<T>> : HashEq<T*> {};
// This header's visibility is restricted. If you need to access the default
// hasher please use the container's ::hasher alias instead.
//
// Example: typename Hash = typename absl::flat_hash_map<K, V>::hasher
template <class T>
using hash_default_hash = typename container_internal::HashEq<T>::Hash;
// This header's visibility is restricted. If you need to access the default
// key equal please use the container's ::key_equal alias instead.
//
// Example: typename Eq = typename absl::flat_hash_map<K, V, Hash>::key_equal
template <class T>
using hash_default_eq = typename container_internal::HashEq<T>::Eq;
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_

157
Pods/abseil/absl/container/internal/hash_policy_traits.h generated Normal file
View File

@@ -0,0 +1,157 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef ABSL_CONTAINER_INTERNAL_HASH_POLICY_TRAITS_H_
#define ABSL_CONTAINER_INTERNAL_HASH_POLICY_TRAITS_H_
#include <cstddef>
#include <memory>
#include <new>
#include <type_traits>
#include <utility>
#include "absl/container/internal/common_policy_traits.h"
#include "absl/meta/type_traits.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
// Defines how slots are initialized/destroyed/moved.
template <class Policy, class = void>
struct hash_policy_traits : common_policy_traits<Policy> {
// The type of the keys stored in the hashtable.
using key_type = typename Policy::key_type;
private:
struct ReturnKey {
// When C++17 is available, we can use std::launder to provide mutable
// access to the key for use in node handle.
#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606
template <class Key,
absl::enable_if_t<std::is_lvalue_reference<Key>::value, int> = 0>
static key_type& Impl(Key&& k, int) {
return *std::launder(
const_cast<key_type*>(std::addressof(std::forward<Key>(k))));
}
#endif
template <class Key>
static Key Impl(Key&& k, char) {
return std::forward<Key>(k);
}
// When Key=T&, we forward the lvalue reference.
// When Key=T, we return by value to avoid a dangling reference.
// eg, for string_hash_map.
template <class Key, class... Args>
auto operator()(Key&& k, const Args&...) const
-> decltype(Impl(std::forward<Key>(k), 0)) {
return Impl(std::forward<Key>(k), 0);
}
};
template <class P = Policy, class = void>
struct ConstantIteratorsImpl : std::false_type {};
template <class P>
struct ConstantIteratorsImpl<P, absl::void_t<typename P::constant_iterators>>
: P::constant_iterators {};
public:
// The actual object stored in the hash table.
using slot_type = typename Policy::slot_type;
// The argument type for insertions into the hashtable. This is different
// from value_type for increased performance. See initializer_list constructor
// and insert() member functions for more details.
using init_type = typename Policy::init_type;
using reference = decltype(Policy::element(std::declval<slot_type*>()));
using pointer = typename std::remove_reference<reference>::type*;
using value_type = typename std::remove_reference<reference>::type;
// Policies can set this variable to tell raw_hash_set that all iterators
// should be constant, even `iterator`. This is useful for set-like
// containers.
// Defaults to false if not provided by the policy.
using constant_iterators = ConstantIteratorsImpl<>;
// Returns the amount of memory owned by `slot`, exclusive of `sizeof(*slot)`.
//
// If `slot` is nullptr, returns the constant amount of memory owned by any
// full slot or -1 if slots own variable amounts of memory.
//
// PRECONDITION: `slot` is INITIALIZED or nullptr
template <class P = Policy>
static size_t space_used(const slot_type* slot) {
return P::space_used(slot);
}
// Provides generalized access to the key for elements, both for elements in
// the table and for elements that have not yet been inserted (or even
// constructed). We would like an API that allows us to say: `key(args...)`
// but we cannot do that for all cases, so we use this more general API that
// can be used for many things, including the following:
//
// - Given an element in a table, get its key.
// - Given an element initializer, get its key.
// - Given `emplace()` arguments, get the element key.
//
// Implementations of this must adhere to a very strict technical
// specification around aliasing and consuming arguments:
//
// Let `value_type` be the result type of `element()` without ref- and
// cv-qualifiers. The first argument is a functor, the rest are constructor
// arguments for `value_type`. Returns `std::forward<F>(f)(k, xs...)`, where
// `k` is the element key, and `xs...` are the new constructor arguments for
// `value_type`. It's allowed for `k` to alias `xs...`, and for both to alias
// `ts...`. The key won't be touched once `xs...` are used to construct an
// element; `ts...` won't be touched at all, which allows `apply()` to consume
// any rvalues among them.
//
// If `value_type` is constructible from `Ts&&...`, `Policy::apply()` must not
// trigger a hard compile error unless it originates from `f`. In other words,
// `Policy::apply()` must be SFINAE-friendly. If `value_type` is not
// constructible from `Ts&&...`, either SFINAE or a hard compile error is OK.
//
// If `Ts...` is `[cv] value_type[&]` or `[cv] init_type[&]`,
// `Policy::apply()` must work. A compile error is not allowed, SFINAE or not.
template <class F, class... Ts, class P = Policy>
static auto apply(F&& f, Ts&&... ts)
-> decltype(P::apply(std::forward<F>(f), std::forward<Ts>(ts)...)) {
return P::apply(std::forward<F>(f), std::forward<Ts>(ts)...);
}
// Returns the "key" portion of the slot.
// Used for node handle manipulation.
template <class P = Policy>
static auto mutable_key(slot_type* slot)
-> decltype(P::apply(ReturnKey(), hash_policy_traits::element(slot))) {
return P::apply(ReturnKey(), hash_policy_traits::element(slot));
}
// Returns the "value" (as opposed to the "key") portion of the element. Used
// by maps to implement `operator[]`, `at()` and `insert_or_assign()`.
template <class T, class P = Policy>
static auto value(T* elem) -> decltype(P::value(elem)) {
return P::value(elem);
}
};
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_HASH_POLICY_TRAITS_H_

85
Pods/abseil/absl/container/internal/hashtable_debug_hooks.h generated Normal file
View File

@@ -0,0 +1,85 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Provides the internal API for hashtable_debug.h.
#ifndef ABSL_CONTAINER_INTERNAL_HASHTABLE_DEBUG_HOOKS_H_
#define ABSL_CONTAINER_INTERNAL_HASHTABLE_DEBUG_HOOKS_H_
#include <cstddef>
#include <algorithm>
#include <type_traits>
#include <vector>
#include "absl/base/config.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
namespace hashtable_debug_internal {
// If it is a map, call get<0>().
using std::get;
template <typename T, typename = typename T::mapped_type>
auto GetKey(const typename T::value_type& pair, int) -> decltype(get<0>(pair)) {
return get<0>(pair);
}
// If it is not a map, return the value directly.
template <typename T>
const typename T::key_type& GetKey(const typename T::key_type& key, char) {
return key;
}
// Containers should specialize this to provide debug information for that
// container.
template <class Container, typename Enabler = void>
struct HashtableDebugAccess {
// Returns the number of probes required to find `key` in `c`. The "number of
// probes" is a concept that can vary by container. Implementations should
// return 0 when `key` was found in the minimum number of operations and
// should increment the result for each non-trivial operation required to find
// `key`.
//
// The default implementation uses the bucket api from the standard and thus
// works for `std::unordered_*` containers.
static size_t GetNumProbes(const Container& c,
const typename Container::key_type& key) {
if (!c.bucket_count()) return {};
size_t num_probes = 0;
size_t bucket = c.bucket(key);
for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) {
if (it == e) return num_probes;
if (c.key_eq()(key, GetKey<Container>(*it, 0))) return num_probes;
}
}
// Returns the number of bytes requested from the allocator by the container
// and not freed.
//
// static size_t AllocatedByteSize(const Container& c);
// Returns a tight lower bound for AllocatedByteSize(c) where `c` is of type
// `Container` and `c.size()` is equal to `num_elements`.
//
// static size_t LowerBoundAllocatedByteSize(size_t num_elements);
};
} // namespace hashtable_debug_internal
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_HASHTABLE_DEBUG_HOOKS_H_

285
Pods/abseil/absl/container/internal/hashtablez_sampler.cc generated Normal file
View File

@@ -0,0 +1,285 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/container/internal/hashtablez_sampler.h"
#include <algorithm>
#include <atomic>
#include <cassert>
#include <cmath>
#include <functional>
#include <limits>
#include "absl/base/attributes.h"
#include "absl/base/config.h"
#include "absl/base/internal/raw_logging.h"
#include "absl/debugging/stacktrace.h"
#include "absl/memory/memory.h"
#include "absl/profiling/internal/exponential_biased.h"
#include "absl/profiling/internal/sample_recorder.h"
#include "absl/synchronization/mutex.h"
#include "absl/time/clock.h"
#include "absl/utility/utility.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL
constexpr int HashtablezInfo::kMaxStackDepth;
#endif
namespace {
ABSL_CONST_INIT std::atomic<bool> g_hashtablez_enabled{
false
};
ABSL_CONST_INIT std::atomic<int32_t> g_hashtablez_sample_parameter{1 << 10};
std::atomic<HashtablezConfigListener> g_hashtablez_config_listener{nullptr};
#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
ABSL_PER_THREAD_TLS_KEYWORD absl::profiling_internal::ExponentialBiased
g_exponential_biased_generator;
#endif
void TriggerHashtablezConfigListener() {
auto* listener = g_hashtablez_config_listener.load(std::memory_order_acquire);
if (listener != nullptr) listener();
}
} // namespace
#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample = {0, 0};
#endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
HashtablezSampler& GlobalHashtablezSampler() {
static auto* sampler = new HashtablezSampler();
return *sampler;
}
HashtablezInfo::HashtablezInfo() = default;
HashtablezInfo::~HashtablezInfo() = default;
void HashtablezInfo::PrepareForSampling(int64_t stride,
size_t inline_element_size_value) {
capacity.store(0, std::memory_order_relaxed);
size.store(0, std::memory_order_relaxed);
num_erases.store(0, std::memory_order_relaxed);
num_rehashes.store(0, std::memory_order_relaxed);
max_probe_length.store(0, std::memory_order_relaxed);
total_probe_length.store(0, std::memory_order_relaxed);
hashes_bitwise_or.store(0, std::memory_order_relaxed);
hashes_bitwise_and.store(~size_t{}, std::memory_order_relaxed);
hashes_bitwise_xor.store(0, std::memory_order_relaxed);
max_reserve.store(0, std::memory_order_relaxed);
create_time = absl::Now();
weight = stride;
// The inliner makes hardcoded skip_count difficult (especially when combined
// with LTO). We use the ability to exclude stacks by regex when encoding
// instead.
depth = absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth,
/* skip_count= */ 0);
inline_element_size = inline_element_size_value;
}
static bool ShouldForceSampling() {
enum ForceState {
kDontForce,
kForce,
kUninitialized
};
ABSL_CONST_INIT static std::atomic<ForceState> global_state{
kUninitialized};
ForceState state = global_state.load(std::memory_order_relaxed);
if (ABSL_PREDICT_TRUE(state == kDontForce)) return false;
if (state == kUninitialized) {
state = ABSL_INTERNAL_C_SYMBOL(AbslContainerInternalSampleEverything)()
? kForce
: kDontForce;
global_state.store(state, std::memory_order_relaxed);
}
return state == kForce;
}
HashtablezInfo* SampleSlow(SamplingState& next_sample,
size_t inline_element_size) {
if (ABSL_PREDICT_FALSE(ShouldForceSampling())) {
next_sample.next_sample = 1;
const int64_t old_stride = exchange(next_sample.sample_stride, 1);
HashtablezInfo* result =
GlobalHashtablezSampler().Register(old_stride, inline_element_size);
return result;
}
#if !defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
next_sample = {
std::numeric_limits<int64_t>::max(),
std::numeric_limits<int64_t>::max(),
};
return nullptr;
#else
bool first = next_sample.next_sample < 0;
const int64_t next_stride = g_exponential_biased_generator.GetStride(
g_hashtablez_sample_parameter.load(std::memory_order_relaxed));
next_sample.next_sample = next_stride;
const int64_t old_stride = exchange(next_sample.sample_stride, next_stride);
// Small values of interval are equivalent to just sampling next time.
ABSL_ASSERT(next_stride >= 1);
// g_hashtablez_enabled can be dynamically flipped, we need to set a threshold
// low enough that we will start sampling in a reasonable time, so we just use
// the default sampling rate.
if (!g_hashtablez_enabled.load(std::memory_order_relaxed)) return nullptr;
// We will only be negative on our first count, so we should just retry in
// that case.
if (first) {
if (ABSL_PREDICT_TRUE(--next_sample.next_sample > 0)) return nullptr;
return SampleSlow(next_sample, inline_element_size);
}
return GlobalHashtablezSampler().Register(old_stride, inline_element_size);
#endif
}
void UnsampleSlow(HashtablezInfo* info) {
GlobalHashtablezSampler().Unregister(info);
}
void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length) {
#ifdef ABSL_INTERNAL_HAVE_SSE2
total_probe_length /= 16;
#else
total_probe_length /= 8;
#endif
info->total_probe_length.store(total_probe_length, std::memory_order_relaxed);
info->num_erases.store(0, std::memory_order_relaxed);
// There is only one concurrent writer, so `load` then `store` is sufficient
// instead of using `fetch_add`.
info->num_rehashes.store(
1 + info->num_rehashes.load(std::memory_order_relaxed),
std::memory_order_relaxed);
}
void RecordReservationSlow(HashtablezInfo* info, size_t target_capacity) {
info->max_reserve.store(
(std::max)(info->max_reserve.load(std::memory_order_relaxed),
target_capacity),
std::memory_order_relaxed);
}
void RecordClearedReservationSlow(HashtablezInfo* info) {
info->max_reserve.store(0, std::memory_order_relaxed);
}
void RecordStorageChangedSlow(HashtablezInfo* info, size_t size,
size_t capacity) {
info->size.store(size, std::memory_order_relaxed);
info->capacity.store(capacity, std::memory_order_relaxed);
if (size == 0) {
// This is a clear, reset the total/num_erases too.
info->total_probe_length.store(0, std::memory_order_relaxed);
info->num_erases.store(0, std::memory_order_relaxed);
}
}
void RecordInsertSlow(HashtablezInfo* info, size_t hash,
size_t distance_from_desired) {
// SwissTables probe in groups of 16, so scale this to count items probes and
// not offset from desired.
size_t probe_length = distance_from_desired;
#ifdef ABSL_INTERNAL_HAVE_SSE2
probe_length /= 16;
#else
probe_length /= 8;
#endif
info->hashes_bitwise_and.fetch_and(hash, std::memory_order_relaxed);
info->hashes_bitwise_or.fetch_or(hash, std::memory_order_relaxed);
info->hashes_bitwise_xor.fetch_xor(hash, std::memory_order_relaxed);
info->max_probe_length.store(
std::max(info->max_probe_length.load(std::memory_order_relaxed),
probe_length),
std::memory_order_relaxed);
info->total_probe_length.fetch_add(probe_length, std::memory_order_relaxed);
info->size.fetch_add(1, std::memory_order_relaxed);
}
void RecordEraseSlow(HashtablezInfo* info) {
info->size.fetch_sub(1, std::memory_order_relaxed);
// There is only one concurrent writer, so `load` then `store` is sufficient
// instead of using `fetch_add`.
info->num_erases.store(1 + info->num_erases.load(std::memory_order_relaxed),
std::memory_order_relaxed);
}
void SetHashtablezConfigListener(HashtablezConfigListener l) {
g_hashtablez_config_listener.store(l, std::memory_order_release);
}
bool IsHashtablezEnabled() {
return g_hashtablez_enabled.load(std::memory_order_acquire);
}
void SetHashtablezEnabled(bool enabled) {
SetHashtablezEnabledInternal(enabled);
TriggerHashtablezConfigListener();
}
void SetHashtablezEnabledInternal(bool enabled) {
g_hashtablez_enabled.store(enabled, std::memory_order_release);
}
int32_t GetHashtablezSampleParameter() {
return g_hashtablez_sample_parameter.load(std::memory_order_acquire);
}
void SetHashtablezSampleParameter(int32_t rate) {
SetHashtablezSampleParameterInternal(rate);
TriggerHashtablezConfigListener();
}
void SetHashtablezSampleParameterInternal(int32_t rate) {
if (rate > 0) {
g_hashtablez_sample_parameter.store(rate, std::memory_order_release);
} else {
ABSL_RAW_LOG(ERROR, "Invalid hashtablez sample rate: %lld",
static_cast<long long>(rate)); // NOLINT(runtime/int)
}
}
size_t GetHashtablezMaxSamples() {
return GlobalHashtablezSampler().GetMaxSamples();
}
void SetHashtablezMaxSamples(size_t max) {
SetHashtablezMaxSamplesInternal(max);
TriggerHashtablezConfigListener();
}
void SetHashtablezMaxSamplesInternal(size_t max) {
if (max > 0) {
GlobalHashtablezSampler().SetMaxSamples(max);
} else {
ABSL_RAW_LOG(ERROR, "Invalid hashtablez max samples: 0");
}
}
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl

257
Pods/abseil/absl/container/internal/hashtablez_sampler.h generated Normal file
View File

@@ -0,0 +1,257 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// -----------------------------------------------------------------------------
// File: hashtablez_sampler.h
// -----------------------------------------------------------------------------
//
// This header file defines the API for a low level library to sample hashtables
// and collect runtime statistics about them.
//
// `HashtablezSampler` controls the lifecycle of `HashtablezInfo` objects which
// store information about a single sample.
//
// `Record*` methods store information into samples.
// `Sample()` and `Unsample()` make use of a single global sampler with
// properties controlled by the flags hashtablez_enabled,
// hashtablez_sample_rate, and hashtablez_max_samples.
//
// WARNING
//
// Using this sampling API may cause sampled Swiss tables to use the global
// allocator (operator `new`) in addition to any custom allocator. If you
// are using a table in an unusual circumstance where allocation or calling a
// linux syscall is unacceptable, this could interfere.
//
// This utility is internal-only. Use at your own risk.
#ifndef ABSL_CONTAINER_INTERNAL_HASHTABLEZ_SAMPLER_H_
#define ABSL_CONTAINER_INTERNAL_HASHTABLEZ_SAMPLER_H_
#include <atomic>
#include <functional>
#include <memory>
#include <vector>
#include "absl/base/config.h"
#include "absl/base/internal/per_thread_tls.h"
#include "absl/base/optimization.h"
#include "absl/profiling/internal/sample_recorder.h"
#include "absl/synchronization/mutex.h"
#include "absl/utility/utility.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
// Stores information about a sampled hashtable. All mutations to this *must*
// be made through `Record*` functions below. All reads from this *must* only
// occur in the callback to `HashtablezSampler::Iterate`.
struct HashtablezInfo : public profiling_internal::Sample<HashtablezInfo> {
// Constructs the object but does not fill in any fields.
HashtablezInfo();
~HashtablezInfo();
HashtablezInfo(const HashtablezInfo&) = delete;
HashtablezInfo& operator=(const HashtablezInfo&) = delete;
// Puts the object into a clean state, fills in the logically `const` members,
// blocking for any readers that are currently sampling the object.
void PrepareForSampling(int64_t stride, size_t inline_element_size_value)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu);
// These fields are mutated by the various Record* APIs and need to be
// thread-safe.
std::atomic<size_t> capacity;
std::atomic<size_t> size;
std::atomic<size_t> num_erases;
std::atomic<size_t> num_rehashes;
std::atomic<size_t> max_probe_length;
std::atomic<size_t> total_probe_length;
std::atomic<size_t> hashes_bitwise_or;
std::atomic<size_t> hashes_bitwise_and;
std::atomic<size_t> hashes_bitwise_xor;
std::atomic<size_t> max_reserve;
// All of the fields below are set by `PrepareForSampling`, they must not be
// mutated in `Record*` functions. They are logically `const` in that sense.
// These are guarded by init_mu, but that is not externalized to clients,
// which can read them only during `SampleRecorder::Iterate` which will hold
// the lock.
static constexpr int kMaxStackDepth = 64;
absl::Time create_time;
int32_t depth;
void* stack[kMaxStackDepth];
size_t inline_element_size; // How big is the slot?
};
void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length);
void RecordReservationSlow(HashtablezInfo* info, size_t target_capacity);
void RecordClearedReservationSlow(HashtablezInfo* info);
void RecordStorageChangedSlow(HashtablezInfo* info, size_t size,
size_t capacity);
void RecordInsertSlow(HashtablezInfo* info, size_t hash,
size_t distance_from_desired);
void RecordEraseSlow(HashtablezInfo* info);
struct SamplingState {
int64_t next_sample;
// When we make a sampling decision, we record that distance so we can weight
// each sample.
int64_t sample_stride;
};
HashtablezInfo* SampleSlow(SamplingState& next_sample,
size_t inline_element_size);
void UnsampleSlow(HashtablezInfo* info);
#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
#error ABSL_INTERNAL_HASHTABLEZ_SAMPLE cannot be directly set
#endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
class HashtablezInfoHandle {
public:
explicit HashtablezInfoHandle() : info_(nullptr) {}
explicit HashtablezInfoHandle(HashtablezInfo* info) : info_(info) {}
// We do not have a destructor. Caller is responsible for calling Unregister
// before destroying the handle.
void Unregister() {
if (ABSL_PREDICT_TRUE(info_ == nullptr)) return;
UnsampleSlow(info_);
}
inline bool IsSampled() const { return ABSL_PREDICT_FALSE(info_ != nullptr); }
inline void RecordStorageChanged(size_t size, size_t capacity) {
if (ABSL_PREDICT_TRUE(info_ == nullptr)) return;
RecordStorageChangedSlow(info_, size, capacity);
}
inline void RecordRehash(size_t total_probe_length) {
if (ABSL_PREDICT_TRUE(info_ == nullptr)) return;
RecordRehashSlow(info_, total_probe_length);
}
inline void RecordReservation(size_t target_capacity) {
if (ABSL_PREDICT_TRUE(info_ == nullptr)) return;
RecordReservationSlow(info_, target_capacity);
}
inline void RecordClearedReservation() {
if (ABSL_PREDICT_TRUE(info_ == nullptr)) return;
RecordClearedReservationSlow(info_);
}
inline void RecordInsert(size_t hash, size_t distance_from_desired) {
if (ABSL_PREDICT_TRUE(info_ == nullptr)) return;
RecordInsertSlow(info_, hash, distance_from_desired);
}
inline void RecordErase() {
if (ABSL_PREDICT_TRUE(info_ == nullptr)) return;
RecordEraseSlow(info_);
}
friend inline void swap(HashtablezInfoHandle& lhs,
HashtablezInfoHandle& rhs) {
std::swap(lhs.info_, rhs.info_);
}
private:
friend class HashtablezInfoHandlePeer;
HashtablezInfo* info_;
};
#else
// Ensure that when Hashtablez is turned off at compile time, HashtablezInfo can
// be removed by the linker, in order to reduce the binary size.
class HashtablezInfoHandle {
public:
explicit HashtablezInfoHandle() = default;
explicit HashtablezInfoHandle(std::nullptr_t) {}
inline void Unregister() {}
inline bool IsSampled() const { return false; }
inline void RecordStorageChanged(size_t /*size*/, size_t /*capacity*/) {}
inline void RecordRehash(size_t /*total_probe_length*/) {}
inline void RecordReservation(size_t /*target_capacity*/) {}
inline void RecordClearedReservation() {}
inline void RecordInsert(size_t /*hash*/, size_t /*distance_from_desired*/) {}
inline void RecordErase() {}
friend inline void swap(HashtablezInfoHandle& /*lhs*/,
HashtablezInfoHandle& /*rhs*/) {}
};
#endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
extern ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample;
#endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
// Returns an RAII sampling handle that manages registration and unregistation
// with the global sampler.
inline HashtablezInfoHandle Sample(
size_t inline_element_size ABSL_ATTRIBUTE_UNUSED) {
#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
if (ABSL_PREDICT_TRUE(--global_next_sample.next_sample > 0)) {
return HashtablezInfoHandle(nullptr);
}
return HashtablezInfoHandle(
SampleSlow(global_next_sample, inline_element_size));
#else
return HashtablezInfoHandle(nullptr);
#endif // !ABSL_PER_THREAD_TLS
}
using HashtablezSampler =
::absl::profiling_internal::SampleRecorder<HashtablezInfo>;
// Returns a global Sampler.
HashtablezSampler& GlobalHashtablezSampler();
using HashtablezConfigListener = void (*)();
void SetHashtablezConfigListener(HashtablezConfigListener l);
// Enables or disables sampling for Swiss tables.
bool IsHashtablezEnabled();
void SetHashtablezEnabled(bool enabled);
void SetHashtablezEnabledInternal(bool enabled);
// Sets the rate at which Swiss tables will be sampled.
int32_t GetHashtablezSampleParameter();
void SetHashtablezSampleParameter(int32_t rate);
void SetHashtablezSampleParameterInternal(int32_t rate);
// Sets a soft max for the number of samples that will be kept.
size_t GetHashtablezMaxSamples();
void SetHashtablezMaxSamples(size_t max);
void SetHashtablezMaxSamplesInternal(size_t max);
// Configuration override.
// This allows process-wide sampling without depending on order of
// initialization of static storage duration objects.
// The definition of this constant is weak, which allows us to inject a
// different value for it at link time.
extern "C" bool ABSL_INTERNAL_C_SYMBOL(AbslContainerInternalSampleEverything)();
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_HASHTABLEZ_SAMPLER_H_

31
Pods/abseil/absl/container/internal/hashtablez_sampler_force_weak_definition.cc generated Normal file
View File

@@ -0,0 +1,31 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/container/internal/hashtablez_sampler.h"
#include "absl/base/attributes.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
// See hashtablez_sampler.h for details.
extern "C" ABSL_ATTRIBUTE_WEAK bool ABSL_INTERNAL_C_SYMBOL(
AbslContainerInternalSampleEverything)() {
return false;
}
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl

1101
Pods/abseil/absl/container/internal/inlined_vector.h generated Normal file
View File

@@ -0,0 +1,1101 @@
// Copyright 2019 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_H_
#define ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_H_
#include <algorithm>
#include <cstddef>
#include <cstring>
#include <iterator>
#include <limits>
#include <memory>
#include <new>
#include <type_traits>
#include <utility>
#include "absl/base/attributes.h"
#include "absl/base/config.h"
#include "absl/base/macros.h"
#include "absl/container/internal/compressed_tuple.h"
#include "absl/memory/memory.h"
#include "absl/meta/type_traits.h"
#include "absl/types/span.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace inlined_vector_internal {
// GCC does not deal very well with the below code
#if !defined(__clang__) && defined(__GNUC__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Warray-bounds"
#endif
template <typename A>
using AllocatorTraits = std::allocator_traits<A>;
template <typename A>
using ValueType = typename AllocatorTraits<A>::value_type;
template <typename A>
using SizeType = typename AllocatorTraits<A>::size_type;
template <typename A>
using Pointer = typename AllocatorTraits<A>::pointer;
template <typename A>
using ConstPointer = typename AllocatorTraits<A>::const_pointer;
template <typename A>
using SizeType = typename AllocatorTraits<A>::size_type;
template <typename A>
using DifferenceType = typename AllocatorTraits<A>::difference_type;
template <typename A>
using Reference = ValueType<A>&;
template <typename A>
using ConstReference = const ValueType<A>&;
template <typename A>
using Iterator = Pointer<A>;
template <typename A>
using ConstIterator = ConstPointer<A>;
template <typename A>
using ReverseIterator = typename std::reverse_iterator<Iterator<A>>;
template <typename A>
using ConstReverseIterator = typename std::reverse_iterator<ConstIterator<A>>;
template <typename A>
using MoveIterator = typename std::move_iterator<Iterator<A>>;
template <typename Iterator>
using IsAtLeastForwardIterator = std::is_convertible<
typename std::iterator_traits<Iterator>::iterator_category,
std::forward_iterator_tag>;
template <typename A>
using IsMoveAssignOk = std::is_move_assignable<ValueType<A>>;
template <typename A>
using IsSwapOk = absl::type_traits_internal::IsSwappable<ValueType<A>>;
template <typename T>
struct TypeIdentity {
using type = T;
};
// Used for function arguments in template functions to prevent ADL by forcing
// callers to explicitly specify the template parameter.
template <typename T>
using NoTypeDeduction = typename TypeIdentity<T>::type;
template <typename A, bool IsTriviallyDestructible =
absl::is_trivially_destructible<ValueType<A>>::value>
struct DestroyAdapter;
template <typename A>
struct DestroyAdapter<A, /* IsTriviallyDestructible */ false> {
static void DestroyElements(A& allocator, Pointer<A> destroy_first,
SizeType<A> destroy_size) {
for (SizeType<A> i = destroy_size; i != 0;) {
--i;
AllocatorTraits<A>::destroy(allocator, destroy_first + i);
}
}
};
template <typename A>
struct DestroyAdapter<A, /* IsTriviallyDestructible */ true> {
static void DestroyElements(A& allocator, Pointer<A> destroy_first,
SizeType<A> destroy_size) {
static_cast<void>(allocator);
static_cast<void>(destroy_first);
static_cast<void>(destroy_size);
}
};
template <typename A>
struct Allocation {
Pointer<A> data = nullptr;
SizeType<A> capacity = 0;
};
template <typename A,
bool IsOverAligned =
(alignof(ValueType<A>) > ABSL_INTERNAL_DEFAULT_NEW_ALIGNMENT)>
struct MallocAdapter {
static Allocation<A> Allocate(A& allocator, SizeType<A> requested_capacity) {
return {AllocatorTraits<A>::allocate(allocator, requested_capacity),
requested_capacity};
}
static void Deallocate(A& allocator, Pointer<A> pointer,
SizeType<A> capacity) {
AllocatorTraits<A>::deallocate(allocator, pointer, capacity);
}
};
template <typename A, typename ValueAdapter>
void ConstructElements(NoTypeDeduction<A>& allocator,
Pointer<A> construct_first, ValueAdapter& values,
SizeType<A> construct_size) {
for (SizeType<A> i = 0; i < construct_size; ++i) {
ABSL_INTERNAL_TRY { values.ConstructNext(allocator, construct_first + i); }
ABSL_INTERNAL_CATCH_ANY {
DestroyAdapter<A>::DestroyElements(allocator, construct_first, i);
ABSL_INTERNAL_RETHROW;
}
}
}
template <typename A, typename ValueAdapter>
void AssignElements(Pointer<A> assign_first, ValueAdapter& values,
SizeType<A> assign_size) {
for (SizeType<A> i = 0; i < assign_size; ++i) {
values.AssignNext(assign_first + i);
}
}
template <typename A>
struct StorageView {
Pointer<A> data;
SizeType<A> size;
SizeType<A> capacity;
};
template <typename A, typename Iterator>
class IteratorValueAdapter {
public:
explicit IteratorValueAdapter(const Iterator& it) : it_(it) {}
void ConstructNext(A& allocator, Pointer<A> construct_at) {
AllocatorTraits<A>::construct(allocator, construct_at, *it_);
++it_;
}
void AssignNext(Pointer<A> assign_at) {
*assign_at = *it_;
++it_;
}
private:
Iterator it_;
};
template <typename A>
class CopyValueAdapter {
public:
explicit CopyValueAdapter(ConstPointer<A> p) : ptr_(p) {}
void ConstructNext(A& allocator, Pointer<A> construct_at) {
AllocatorTraits<A>::construct(allocator, construct_at, *ptr_);
}
void AssignNext(Pointer<A> assign_at) { *assign_at = *ptr_; }
private:
ConstPointer<A> ptr_;
};
template <typename A>
class DefaultValueAdapter {
public:
explicit DefaultValueAdapter() {}
void ConstructNext(A& allocator, Pointer<A> construct_at) {
AllocatorTraits<A>::construct(allocator, construct_at);
}
void AssignNext(Pointer<A> assign_at) { *assign_at = ValueType<A>(); }
};
template <typename A>
class AllocationTransaction {
public:
explicit AllocationTransaction(A& allocator)
: allocator_data_(allocator, nullptr), capacity_(0) {}
~AllocationTransaction() {
if (DidAllocate()) {
MallocAdapter<A>::Deallocate(GetAllocator(), GetData(), GetCapacity());
}
}
AllocationTransaction(const AllocationTransaction&) = delete;
void operator=(const AllocationTransaction&) = delete;
A& GetAllocator() { return allocator_data_.template get<0>(); }
Pointer<A>& GetData() { return allocator_data_.template get<1>(); }
SizeType<A>& GetCapacity() { return capacity_; }
bool DidAllocate() { return GetData() != nullptr; }
Pointer<A> Allocate(SizeType<A> requested_capacity) {
Allocation<A> result =
MallocAdapter<A>::Allocate(GetAllocator(), requested_capacity);
GetData() = result.data;
GetCapacity() = result.capacity;
return result.data;
}
ABSL_MUST_USE_RESULT Allocation<A> Release() && {
Allocation<A> result = {GetData(), GetCapacity()};
Reset();
return result;
}
private:
void Reset() {
GetData() = nullptr;
GetCapacity() = 0;
}
container_internal::CompressedTuple<A, Pointer<A>> allocator_data_;
SizeType<A> capacity_;
};
template <typename A>
class ConstructionTransaction {
public:
explicit ConstructionTransaction(A& allocator)
: allocator_data_(allocator, nullptr), size_(0) {}
~ConstructionTransaction() {
if (DidConstruct()) {
DestroyAdapter<A>::DestroyElements(GetAllocator(), GetData(), GetSize());
}
}
ConstructionTransaction(const ConstructionTransaction&) = delete;
void operator=(const ConstructionTransaction&) = delete;
A& GetAllocator() { return allocator_data_.template get<0>(); }
Pointer<A>& GetData() { return allocator_data_.template get<1>(); }
SizeType<A>& GetSize() { return size_; }
bool DidConstruct() { return GetData() != nullptr; }
template <typename ValueAdapter>
void Construct(Pointer<A> data, ValueAdapter& values, SizeType<A> size) {
ConstructElements<A>(GetAllocator(), data, values, size);
GetData() = data;
GetSize() = size;
}
void Commit() && {
GetData() = nullptr;
GetSize() = 0;
}
private:
container_internal::CompressedTuple<A, Pointer<A>> allocator_data_;
SizeType<A> size_;
};
template <typename T, size_t N, typename A>
class Storage {
public:
struct MemcpyPolicy {};
struct ElementwiseAssignPolicy {};
struct ElementwiseSwapPolicy {};
struct ElementwiseConstructPolicy {};
using MoveAssignmentPolicy = absl::conditional_t<
// Fast path: if the value type can be trivially move assigned and
// destroyed, and we know the allocator doesn't do anything fancy, then
// it's safe for us to simply adopt the contents of the storage for
// `other` and remove its own reference to them. It's as if we had
// individually move-assigned each value and then destroyed the original.
absl::conjunction<absl::is_trivially_move_assignable<ValueType<A>>,
absl::is_trivially_destructible<ValueType<A>>,
std::is_same<A, std::allocator<ValueType<A>>>>::value,
MemcpyPolicy,
// Otherwise we use move assignment if possible. If not, we simulate
// move assignment using move construction.
//
// Note that this is in contrast to e.g. std::vector and std::optional,
// which are themselves not move-assignable when their contained type is
// not.
absl::conditional_t<IsMoveAssignOk<A>::value, ElementwiseAssignPolicy,
ElementwiseConstructPolicy>>;
// The policy to be used specifically when swapping inlined elements.
using SwapInlinedElementsPolicy = absl::conditional_t<
// Fast path: if the value type can be trivially move constructed/assigned
// and destroyed, and we know the allocator doesn't do anything fancy,
// then it's safe for us to simply swap the bytes in the inline storage.
// It's as if we had move-constructed a temporary vector, move-assigned
// one to the other, then move-assigned the first from the temporary.
absl::conjunction<absl::is_trivially_move_constructible<ValueType<A>>,
absl::is_trivially_move_assignable<ValueType<A>>,
absl::is_trivially_destructible<ValueType<A>>,
std::is_same<A, std::allocator<ValueType<A>>>>::value,
MemcpyPolicy,
absl::conditional_t<IsSwapOk<A>::value, ElementwiseSwapPolicy,
ElementwiseConstructPolicy>>;
static SizeType<A> NextCapacity(SizeType<A> current_capacity) {
return current_capacity * 2;
}
static SizeType<A> ComputeCapacity(SizeType<A> current_capacity,
SizeType<A> requested_capacity) {
return (std::max)(NextCapacity(current_capacity), requested_capacity);
}
// ---------------------------------------------------------------------------
// Storage Constructors and Destructor
// ---------------------------------------------------------------------------
Storage() : metadata_(A(), /* size and is_allocated */ 0u) {}
explicit Storage(const A& allocator)
: metadata_(allocator, /* size and is_allocated */ 0u) {}
~Storage() {
// Fast path: if we are empty and not allocated, there's nothing to do.
if (GetSizeAndIsAllocated() == 0) {
return;
}
// Fast path: if no destructors need to be run and we know the allocator
// doesn't do anything fancy, then all we need to do is deallocate (and
// maybe not even that).
if (absl::is_trivially_destructible<ValueType<A>>::value &&
std::is_same<A, std::allocator<ValueType<A>>>::value) {
DeallocateIfAllocated();
return;
}
DestroyContents();
}
// ---------------------------------------------------------------------------
// Storage Member Accessors
// ---------------------------------------------------------------------------
SizeType<A>& GetSizeAndIsAllocated() { return metadata_.template get<1>(); }
const SizeType<A>& GetSizeAndIsAllocated() const {
return metadata_.template get<1>();
}
SizeType<A> GetSize() const { return GetSizeAndIsAllocated() >> 1; }
bool GetIsAllocated() const { return GetSizeAndIsAllocated() & 1; }
Pointer<A> GetAllocatedData() {
// GCC 12 has a false-positive -Wmaybe-uninitialized warning here.
#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
return data_.allocated.allocated_data;
#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0)
#pragma GCC diagnostic pop
#endif
}
ConstPointer<A> GetAllocatedData() const {
return data_.allocated.allocated_data;
}
// ABSL_ATTRIBUTE_NO_SANITIZE_CFI is used because the memory pointed to may be
// uninitialized, a common pattern in allocate()+construct() APIs.
// https://clang.llvm.org/docs/ControlFlowIntegrity.html#bad-cast-checking
// NOTE: When this was written, LLVM documentation did not explicitly
// mention that casting `char*` and using `reinterpret_cast` qualifies
// as a bad cast.
ABSL_ATTRIBUTE_NO_SANITIZE_CFI Pointer<A> GetInlinedData() {
return reinterpret_cast<Pointer<A>>(data_.inlined.inlined_data);
}
ABSL_ATTRIBUTE_NO_SANITIZE_CFI ConstPointer<A> GetInlinedData() const {
return reinterpret_cast<ConstPointer<A>>(data_.inlined.inlined_data);
}
SizeType<A> GetAllocatedCapacity() const {
return data_.allocated.allocated_capacity;
}
SizeType<A> GetInlinedCapacity() const {
return static_cast<SizeType<A>>(kOptimalInlinedSize);
}
StorageView<A> MakeStorageView() {
return GetIsAllocated() ? StorageView<A>{GetAllocatedData(), GetSize(),
GetAllocatedCapacity()}
: StorageView<A>{GetInlinedData(), GetSize(),
GetInlinedCapacity()};
}
A& GetAllocator() { return metadata_.template get<0>(); }
const A& GetAllocator() const { return metadata_.template get<0>(); }
// ---------------------------------------------------------------------------
// Storage Member Mutators
// ---------------------------------------------------------------------------
ABSL_ATTRIBUTE_NOINLINE void InitFrom(const Storage& other);
template <typename ValueAdapter>
void Initialize(ValueAdapter values, SizeType<A> new_size);
template <typename ValueAdapter>
void Assign(ValueAdapter values, SizeType<A> new_size);
template <typename ValueAdapter>
void Resize(ValueAdapter values, SizeType<A> new_size);
template <typename ValueAdapter>
Iterator<A> Insert(ConstIterator<A> pos, ValueAdapter values,
SizeType<A> insert_count);
template <typename... Args>
Reference<A> EmplaceBack(Args&&... args);
Iterator<A> Erase(ConstIterator<A> from, ConstIterator<A> to);
void Reserve(SizeType<A> requested_capacity);
void ShrinkToFit();
void Swap(Storage* other_storage_ptr);
void SetIsAllocated() {
GetSizeAndIsAllocated() |= static_cast<SizeType<A>>(1);
}
void UnsetIsAllocated() {
GetSizeAndIsAllocated() &= ((std::numeric_limits<SizeType<A>>::max)() - 1);
}
void SetSize(SizeType<A> size) {
GetSizeAndIsAllocated() =
(size << 1) | static_cast<SizeType<A>>(GetIsAllocated());
}
void SetAllocatedSize(SizeType<A> size) {
GetSizeAndIsAllocated() = (size << 1) | static_cast<SizeType<A>>(1);
}
void SetInlinedSize(SizeType<A> size) {
GetSizeAndIsAllocated() = size << static_cast<SizeType<A>>(1);
}
void AddSize(SizeType<A> count) {
GetSizeAndIsAllocated() += count << static_cast<SizeType<A>>(1);
}
void SubtractSize(SizeType<A> count) {
ABSL_HARDENING_ASSERT(count <= GetSize());
GetSizeAndIsAllocated() -= count << static_cast<SizeType<A>>(1);
}
void SetAllocation(Allocation<A> allocation) {
data_.allocated.allocated_data = allocation.data;
data_.allocated.allocated_capacity = allocation.capacity;
}
void MemcpyFrom(const Storage& other_storage) {
// Assumption check: it doesn't make sense to memcpy inlined elements unless
// we know the allocator doesn't do anything fancy, and one of the following
// holds:
//
// * The elements are trivially relocatable.
//
// * It's possible to trivially assign the elements and then destroy the
// source.
//
// * It's possible to trivially copy construct/assign the elements.
//
{
using V = ValueType<A>;
ABSL_HARDENING_ASSERT(
other_storage.GetIsAllocated() ||
(std::is_same<A, std::allocator<V>>::value &&
(
// First case above
absl::is_trivially_relocatable<V>::value ||
// Second case above
(absl::is_trivially_move_assignable<V>::value &&
absl::is_trivially_destructible<V>::value) ||
// Third case above
(absl::is_trivially_copy_constructible<V>::value ||
absl::is_trivially_copy_assignable<V>::value))));
}
GetSizeAndIsAllocated() = other_storage.GetSizeAndIsAllocated();
data_ = other_storage.data_;
}
void DeallocateIfAllocated() {
if (GetIsAllocated()) {
MallocAdapter<A>::Deallocate(GetAllocator(), GetAllocatedData(),
GetAllocatedCapacity());
}
}
private:
ABSL_ATTRIBUTE_NOINLINE void DestroyContents();
using Metadata = container_internal::CompressedTuple<A, SizeType<A>>;
struct Allocated {
Pointer<A> allocated_data;
SizeType<A> allocated_capacity;
};
// `kOptimalInlinedSize` is an automatically adjusted inlined capacity of the
// `InlinedVector`. Sometimes, it is possible to increase the capacity (from
// the user requested `N`) without increasing the size of the `InlinedVector`.
static constexpr size_t kOptimalInlinedSize =
(std::max)(N, sizeof(Allocated) / sizeof(ValueType<A>));
struct Inlined {
alignas(ValueType<A>) char inlined_data[sizeof(
ValueType<A>[kOptimalInlinedSize])];
};
union Data {
Allocated allocated;
Inlined inlined;
};
void SwapN(ElementwiseSwapPolicy, Storage* other, SizeType<A> n);
void SwapN(ElementwiseConstructPolicy, Storage* other, SizeType<A> n);
void SwapInlinedElements(MemcpyPolicy, Storage* other);
template <typename NotMemcpyPolicy>
void SwapInlinedElements(NotMemcpyPolicy, Storage* other);
template <typename... Args>
ABSL_ATTRIBUTE_NOINLINE Reference<A> EmplaceBackSlow(Args&&... args);
Metadata metadata_;
Data data_;
};
template <typename T, size_t N, typename A>
void Storage<T, N, A>::DestroyContents() {
Pointer<A> data = GetIsAllocated() ? GetAllocatedData() : GetInlinedData();
DestroyAdapter<A>::DestroyElements(GetAllocator(), data, GetSize());
DeallocateIfAllocated();
}
template <typename T, size_t N, typename A>
void Storage<T, N, A>::InitFrom(const Storage& other) {
const SizeType<A> n = other.GetSize();
ABSL_HARDENING_ASSERT(n > 0); // Empty sources handled handled in caller.
ConstPointer<A> src;
Pointer<A> dst;
if (!other.GetIsAllocated()) {
dst = GetInlinedData();
src = other.GetInlinedData();
} else {
// Because this is only called from the `InlinedVector` constructors, it's
// safe to take on the allocation with size `0`. If `ConstructElements(...)`
// throws, deallocation will be automatically handled by `~Storage()`.
SizeType<A> requested_capacity = ComputeCapacity(GetInlinedCapacity(), n);
Allocation<A> allocation =
MallocAdapter<A>::Allocate(GetAllocator(), requested_capacity);
SetAllocation(allocation);
dst = allocation.data;
src = other.GetAllocatedData();
}
// Fast path: if the value type is trivially copy constructible and we know
// the allocator doesn't do anything fancy, then we know it is legal for us to
// simply memcpy the other vector's elements.
if (absl::is_trivially_copy_constructible<ValueType<A>>::value &&
std::is_same<A, std::allocator<ValueType<A>>>::value) {
std::memcpy(reinterpret_cast<char*>(dst),
reinterpret_cast<const char*>(src), n * sizeof(ValueType<A>));
} else {
auto values = IteratorValueAdapter<A, ConstPointer<A>>(src);
ConstructElements<A>(GetAllocator(), dst, values, n);
}
GetSizeAndIsAllocated() = other.GetSizeAndIsAllocated();
}
template <typename T, size_t N, typename A>
template <typename ValueAdapter>
auto Storage<T, N, A>::Initialize(ValueAdapter values, SizeType<A> new_size)
-> void {
// Only callable from constructors!
ABSL_HARDENING_ASSERT(!GetIsAllocated());
ABSL_HARDENING_ASSERT(GetSize() == 0);
Pointer<A> construct_data;
if (new_size > GetInlinedCapacity()) {
// Because this is only called from the `InlinedVector` constructors, it's
// safe to take on the allocation with size `0`. If `ConstructElements(...)`
// throws, deallocation will be automatically handled by `~Storage()`.
SizeType<A> requested_capacity =
ComputeCapacity(GetInlinedCapacity(), new_size);
Allocation<A> allocation =
MallocAdapter<A>::Allocate(GetAllocator(), requested_capacity);
construct_data = allocation.data;
SetAllocation(allocation);
SetIsAllocated();
} else {
construct_data = GetInlinedData();
}
ConstructElements<A>(GetAllocator(), construct_data, values, new_size);
// Since the initial size was guaranteed to be `0` and the allocated bit is
// already correct for either case, *adding* `new_size` gives us the correct
// result faster than setting it directly.
AddSize(new_size);
}
template <typename T, size_t N, typename A>
template <typename ValueAdapter>
auto Storage<T, N, A>::Assign(ValueAdapter values, SizeType<A> new_size)
-> void {
StorageView<A> storage_view = MakeStorageView();
AllocationTransaction<A> allocation_tx(GetAllocator());
absl::Span<ValueType<A>> assign_loop;
absl::Span<ValueType<A>> construct_loop;
absl::Span<ValueType<A>> destroy_loop;
if (new_size > storage_view.capacity) {
SizeType<A> requested_capacity =
ComputeCapacity(storage_view.capacity, new_size);
construct_loop = {allocation_tx.Allocate(requested_capacity), new_size};
destroy_loop = {storage_view.data, storage_view.size};
} else if (new_size > storage_view.size) {
assign_loop = {storage_view.data, storage_view.size};
construct_loop = {storage_view.data + storage_view.size,
new_size - storage_view.size};
} else {
assign_loop = {storage_view.data, new_size};
destroy_loop = {storage_view.data + new_size, storage_view.size - new_size};
}
AssignElements<A>(assign_loop.data(), values, assign_loop.size());
ConstructElements<A>(GetAllocator(), construct_loop.data(), values,
construct_loop.size());
DestroyAdapter<A>::DestroyElements(GetAllocator(), destroy_loop.data(),
destroy_loop.size());
if (allocation_tx.DidAllocate()) {
DeallocateIfAllocated();
SetAllocation(std::move(allocation_tx).Release());
SetIsAllocated();
}
SetSize(new_size);
}
template <typename T, size_t N, typename A>
template <typename ValueAdapter>
auto Storage<T, N, A>::Resize(ValueAdapter values, SizeType<A> new_size)
-> void {
StorageView<A> storage_view = MakeStorageView();
Pointer<A> const base = storage_view.data;
const SizeType<A> size = storage_view.size;
A& alloc = GetAllocator();
if (new_size <= size) {
// Destroy extra old elements.
DestroyAdapter<A>::DestroyElements(alloc, base + new_size, size - new_size);
} else if (new_size <= storage_view.capacity) {
// Construct new elements in place.
ConstructElements<A>(alloc, base + size, values, new_size - size);
} else {
// Steps:
// a. Allocate new backing store.
// b. Construct new elements in new backing store.
// c. Move existing elements from old backing store to new backing store.
// d. Destroy all elements in old backing store.
// Use transactional wrappers for the first two steps so we can roll
// back if necessary due to exceptions.
AllocationTransaction<A> allocation_tx(alloc);
SizeType<A> requested_capacity =
ComputeCapacity(storage_view.capacity, new_size);
Pointer<A> new_data = allocation_tx.Allocate(requested_capacity);
ConstructionTransaction<A> construction_tx(alloc);
construction_tx.Construct(new_data + size, values, new_size - size);
IteratorValueAdapter<A, MoveIterator<A>> move_values(
(MoveIterator<A>(base)));
ConstructElements<A>(alloc, new_data, move_values, size);
DestroyAdapter<A>::DestroyElements(alloc, base, size);
std::move(construction_tx).Commit();
DeallocateIfAllocated();
SetAllocation(std::move(allocation_tx).Release());
SetIsAllocated();
}
SetSize(new_size);
}
template <typename T, size_t N, typename A>
template <typename ValueAdapter>
auto Storage<T, N, A>::Insert(ConstIterator<A> pos, ValueAdapter values,
SizeType<A> insert_count) -> Iterator<A> {
StorageView<A> storage_view = MakeStorageView();
auto insert_index = static_cast<SizeType<A>>(
std::distance(ConstIterator<A>(storage_view.data), pos));
SizeType<A> insert_end_index = insert_index + insert_count;
SizeType<A> new_size = storage_view.size + insert_count;
if (new_size > storage_view.capacity) {
AllocationTransaction<A> allocation_tx(GetAllocator());
ConstructionTransaction<A> construction_tx(GetAllocator());
ConstructionTransaction<A> move_construction_tx(GetAllocator());
IteratorValueAdapter<A, MoveIterator<A>> move_values(
MoveIterator<A>(storage_view.data));
SizeType<A> requested_capacity =
ComputeCapacity(storage_view.capacity, new_size);
Pointer<A> new_data = allocation_tx.Allocate(requested_capacity);
construction_tx.Construct(new_data + insert_index, values, insert_count);
move_construction_tx.Construct(new_data, move_values, insert_index);
ConstructElements<A>(GetAllocator(), new_data + insert_end_index,
move_values, storage_view.size - insert_index);
DestroyAdapter<A>::DestroyElements(GetAllocator(), storage_view.data,
storage_view.size);
std::move(construction_tx).Commit();
std::move(move_construction_tx).Commit();
DeallocateIfAllocated();
SetAllocation(std::move(allocation_tx).Release());
SetAllocatedSize(new_size);
return Iterator<A>(new_data + insert_index);
} else {
SizeType<A> move_construction_destination_index =
(std::max)(insert_end_index, storage_view.size);
ConstructionTransaction<A> move_construction_tx(GetAllocator());
IteratorValueAdapter<A, MoveIterator<A>> move_construction_values(
MoveIterator<A>(storage_view.data +
(move_construction_destination_index - insert_count)));
absl::Span<ValueType<A>> move_construction = {
storage_view.data + move_construction_destination_index,
new_size - move_construction_destination_index};
Pointer<A> move_assignment_values = storage_view.data + insert_index;
absl::Span<ValueType<A>> move_assignment = {
storage_view.data + insert_end_index,
move_construction_destination_index - insert_end_index};
absl::Span<ValueType<A>> insert_assignment = {move_assignment_values,
move_construction.size()};
absl::Span<ValueType<A>> insert_construction = {
insert_assignment.data() + insert_assignment.size(),
insert_count - insert_assignment.size()};
move_construction_tx.Construct(move_construction.data(),
move_construction_values,
move_construction.size());
for (Pointer<A>
destination = move_assignment.data() + move_assignment.size(),
last_destination = move_assignment.data(),
source = move_assignment_values + move_assignment.size();
;) {
--destination;
--source;
if (destination < last_destination) break;
*destination = std::move(*source);
}
AssignElements<A>(insert_assignment.data(), values,
insert_assignment.size());
ConstructElements<A>(GetAllocator(), insert_construction.data(), values,
insert_construction.size());
std::move(move_construction_tx).Commit();
AddSize(insert_count);
return Iterator<A>(storage_view.data + insert_index);
}
}
template <typename T, size_t N, typename A>
template <typename... Args>
auto Storage<T, N, A>::EmplaceBack(Args&&... args) -> Reference<A> {
StorageView<A> storage_view = MakeStorageView();
const SizeType<A> n = storage_view.size;
if (ABSL_PREDICT_TRUE(n != storage_view.capacity)) {
// Fast path; new element fits.
Pointer<A> last_ptr = storage_view.data + n;
AllocatorTraits<A>::construct(GetAllocator(), last_ptr,
std::forward<Args>(args)...);
AddSize(1);
return *last_ptr;
}
// TODO(b/173712035): Annotate with musttail attribute to prevent regression.
return EmplaceBackSlow(std::forward<Args>(args)...);
}
template <typename T, size_t N, typename A>
template <typename... Args>
auto Storage<T, N, A>::EmplaceBackSlow(Args&&... args) -> Reference<A> {
StorageView<A> storage_view = MakeStorageView();
AllocationTransaction<A> allocation_tx(GetAllocator());
IteratorValueAdapter<A, MoveIterator<A>> move_values(
MoveIterator<A>(storage_view.data));
SizeType<A> requested_capacity = NextCapacity(storage_view.capacity);
Pointer<A> construct_data = allocation_tx.Allocate(requested_capacity);
Pointer<A> last_ptr = construct_data + storage_view.size;
// Construct new element.
AllocatorTraits<A>::construct(GetAllocator(), last_ptr,
std::forward<Args>(args)...);
// Move elements from old backing store to new backing store.
ABSL_INTERNAL_TRY {
ConstructElements<A>(GetAllocator(), allocation_tx.GetData(), move_values,
storage_view.size);
}
ABSL_INTERNAL_CATCH_ANY {
AllocatorTraits<A>::destroy(GetAllocator(), last_ptr);
ABSL_INTERNAL_RETHROW;
}
// Destroy elements in old backing store.
DestroyAdapter<A>::DestroyElements(GetAllocator(), storage_view.data,
storage_view.size);
DeallocateIfAllocated();
SetAllocation(std::move(allocation_tx).Release());
SetIsAllocated();
AddSize(1);
return *last_ptr;
}
template <typename T, size_t N, typename A>
auto Storage<T, N, A>::Erase(ConstIterator<A> from, ConstIterator<A> to)
-> Iterator<A> {
StorageView<A> storage_view = MakeStorageView();
auto erase_size = static_cast<SizeType<A>>(std::distance(from, to));
auto erase_index = static_cast<SizeType<A>>(
std::distance(ConstIterator<A>(storage_view.data), from));
SizeType<A> erase_end_index = erase_index + erase_size;
IteratorValueAdapter<A, MoveIterator<A>> move_values(
MoveIterator<A>(storage_view.data + erase_end_index));
AssignElements<A>(storage_view.data + erase_index, move_values,
storage_view.size - erase_end_index);
DestroyAdapter<A>::DestroyElements(
GetAllocator(), storage_view.data + (storage_view.size - erase_size),
erase_size);
SubtractSize(erase_size);
return Iterator<A>(storage_view.data + erase_index);
}
template <typename T, size_t N, typename A>
auto Storage<T, N, A>::Reserve(SizeType<A> requested_capacity) -> void {
StorageView<A> storage_view = MakeStorageView();
if (ABSL_PREDICT_FALSE(requested_capacity <= storage_view.capacity)) return;
AllocationTransaction<A> allocation_tx(GetAllocator());
IteratorValueAdapter<A, MoveIterator<A>> move_values(
MoveIterator<A>(storage_view.data));
SizeType<A> new_requested_capacity =
ComputeCapacity(storage_view.capacity, requested_capacity);
Pointer<A> new_data = allocation_tx.Allocate(new_requested_capacity);
ConstructElements<A>(GetAllocator(), new_data, move_values,
storage_view.size);
DestroyAdapter<A>::DestroyElements(GetAllocator(), storage_view.data,
storage_view.size);
DeallocateIfAllocated();
SetAllocation(std::move(allocation_tx).Release());
SetIsAllocated();
}
template <typename T, size_t N, typename A>
auto Storage<T, N, A>::ShrinkToFit() -> void {
// May only be called on allocated instances!
ABSL_HARDENING_ASSERT(GetIsAllocated());
StorageView<A> storage_view{GetAllocatedData(), GetSize(),
GetAllocatedCapacity()};
if (ABSL_PREDICT_FALSE(storage_view.size == storage_view.capacity)) return;
AllocationTransaction<A> allocation_tx(GetAllocator());
IteratorValueAdapter<A, MoveIterator<A>> move_values(
MoveIterator<A>(storage_view.data));
Pointer<A> construct_data;
if (storage_view.size > GetInlinedCapacity()) {
SizeType<A> requested_capacity = storage_view.size;
construct_data = allocation_tx.Allocate(requested_capacity);
if (allocation_tx.GetCapacity() >= storage_view.capacity) {
// Already using the smallest available heap allocation.
return;
}
} else {
construct_data = GetInlinedData();
}
ABSL_INTERNAL_TRY {
ConstructElements<A>(GetAllocator(), construct_data, move_values,
storage_view.size);
}
ABSL_INTERNAL_CATCH_ANY {
SetAllocation({storage_view.data, storage_view.capacity});
ABSL_INTERNAL_RETHROW;
}
DestroyAdapter<A>::DestroyElements(GetAllocator(), storage_view.data,
storage_view.size);
MallocAdapter<A>::Deallocate(GetAllocator(), storage_view.data,
storage_view.capacity);
if (allocation_tx.DidAllocate()) {
SetAllocation(std::move(allocation_tx).Release());
} else {
UnsetIsAllocated();
}
}
template <typename T, size_t N, typename A>
auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void {
using std::swap;
ABSL_HARDENING_ASSERT(this != other_storage_ptr);
if (GetIsAllocated() && other_storage_ptr->GetIsAllocated()) {
swap(data_.allocated, other_storage_ptr->data_.allocated);
} else if (!GetIsAllocated() && !other_storage_ptr->GetIsAllocated()) {
SwapInlinedElements(SwapInlinedElementsPolicy{}, other_storage_ptr);
} else {
Storage* allocated_ptr = this;
Storage* inlined_ptr = other_storage_ptr;
if (!allocated_ptr->GetIsAllocated()) swap(allocated_ptr, inlined_ptr);
StorageView<A> allocated_storage_view{
allocated_ptr->GetAllocatedData(), allocated_ptr->GetSize(),
allocated_ptr->GetAllocatedCapacity()};
IteratorValueAdapter<A, MoveIterator<A>> move_values(
MoveIterator<A>(inlined_ptr->GetInlinedData()));
ABSL_INTERNAL_TRY {
ConstructElements<A>(inlined_ptr->GetAllocator(),
allocated_ptr->GetInlinedData(), move_values,
inlined_ptr->GetSize());
}
ABSL_INTERNAL_CATCH_ANY {
allocated_ptr->SetAllocation(Allocation<A>{
allocated_storage_view.data, allocated_storage_view.capacity});
ABSL_INTERNAL_RETHROW;
}
DestroyAdapter<A>::DestroyElements(inlined_ptr->GetAllocator(),
inlined_ptr->GetInlinedData(),
inlined_ptr->GetSize());
inlined_ptr->SetAllocation(Allocation<A>{allocated_storage_view.data,
allocated_storage_view.capacity});
}
swap(GetSizeAndIsAllocated(), other_storage_ptr->GetSizeAndIsAllocated());
swap(GetAllocator(), other_storage_ptr->GetAllocator());
}
template <typename T, size_t N, typename A>
void Storage<T, N, A>::SwapN(ElementwiseSwapPolicy, Storage* other,
SizeType<A> n) {
std::swap_ranges(GetInlinedData(), GetInlinedData() + n,
other->GetInlinedData());
}
template <typename T, size_t N, typename A>
void Storage<T, N, A>::SwapN(ElementwiseConstructPolicy, Storage* other,
SizeType<A> n) {
Pointer<A> a = GetInlinedData();
Pointer<A> b = other->GetInlinedData();
// see note on allocators in `SwapInlinedElements`.
A& allocator_a = GetAllocator();
A& allocator_b = other->GetAllocator();
for (SizeType<A> i = 0; i < n; ++i, ++a, ++b) {
ValueType<A> tmp(std::move(*a));
AllocatorTraits<A>::destroy(allocator_a, a);
AllocatorTraits<A>::construct(allocator_b, a, std::move(*b));
AllocatorTraits<A>::destroy(allocator_b, b);
AllocatorTraits<A>::construct(allocator_a, b, std::move(tmp));
}
}
template <typename T, size_t N, typename A>
void Storage<T, N, A>::SwapInlinedElements(MemcpyPolicy, Storage* other) {
Data tmp = data_;
data_ = other->data_;
other->data_ = tmp;
}
template <typename T, size_t N, typename A>
template <typename NotMemcpyPolicy>
void Storage<T, N, A>::SwapInlinedElements(NotMemcpyPolicy policy,
Storage* other) {
// Note: `destroy` needs to use pre-swap allocator while `construct` -
// post-swap allocator. Allocators will be swapped later on outside of
// `SwapInlinedElements`.
Storage* small_ptr = this;
Storage* large_ptr = other;
if (small_ptr->GetSize() > large_ptr->GetSize()) {
std::swap(small_ptr, large_ptr);
}
auto small_size = small_ptr->GetSize();
auto diff = large_ptr->GetSize() - small_size;
SwapN(policy, other, small_size);
IteratorValueAdapter<A, MoveIterator<A>> move_values(
MoveIterator<A>(large_ptr->GetInlinedData() + small_size));
ConstructElements<A>(large_ptr->GetAllocator(),
small_ptr->GetInlinedData() + small_size, move_values,
diff);
DestroyAdapter<A>::DestroyElements(large_ptr->GetAllocator(),
large_ptr->GetInlinedData() + small_size,
diff);
}
// End ignore "array-bounds"
#if !defined(__clang__) && defined(__GNUC__)
#pragma GCC diagnostic pop
#endif
} // namespace inlined_vector_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_H_

728
Pods/abseil/absl/container/internal/layout.h generated Normal file
View File

@@ -0,0 +1,728 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// MOTIVATION AND TUTORIAL
//
// If you want to put in a single heap allocation N doubles followed by M ints,
// it's easy if N and M are known at compile time.
//
// struct S {
// double a[N];
// int b[M];
// };
//
// S* p = new S;
//
// But what if N and M are known only in run time? Class template Layout to the
// rescue! It's a portable generalization of the technique known as struct hack.
//
// // This object will tell us everything we need to know about the memory
// // layout of double[N] followed by int[M]. It's structurally identical to
// // size_t[2] that stores N and M. It's very cheap to create.
// const Layout<double, int> layout(N, M);
//
// // Allocate enough memory for both arrays. `AllocSize()` tells us how much
// // memory is needed. We are free to use any allocation function we want as
// // long as it returns aligned memory.
// std::unique_ptr<unsigned char[]> p(new unsigned char[layout.AllocSize()]);
//
// // Obtain the pointer to the array of doubles.
// // Equivalent to `reinterpret_cast<double*>(p.get())`.
// //
// // We could have written layout.Pointer<0>(p) instead. If all the types are
// // unique you can use either form, but if some types are repeated you must
// // use the index form.
// double* a = layout.Pointer<double>(p.get());
//
// // Obtain the pointer to the array of ints.
// // Equivalent to `reinterpret_cast<int*>(p.get() + N * 8)`.
// int* b = layout.Pointer<int>(p);
//
// If we are unable to specify sizes of all fields, we can pass as many sizes as
// we can to `Partial()`. In return, it'll allow us to access the fields whose
// locations and sizes can be computed from the provided information.
// `Partial()` comes in handy when the array sizes are embedded into the
// allocation.
//
// // size_t[0] containing N, size_t[1] containing M, double[N], int[M].
// using L = Layout<size_t, size_t, double, int>;
//
// unsigned char* Allocate(size_t n, size_t m) {
// const L layout(1, 1, n, m);
// unsigned char* p = new unsigned char[layout.AllocSize()];
// *layout.Pointer<0>(p) = n;
// *layout.Pointer<1>(p) = m;
// return p;
// }
//
// void Use(unsigned char* p) {
// // First, extract N and M.
// // Specify that the first array has only one element. Using `prefix` we
// // can access the first two arrays but not more.
// constexpr auto prefix = L::Partial(1);
// size_t n = *prefix.Pointer<0>(p);
// size_t m = *prefix.Pointer<1>(p);
//
// // Now we can get pointers to the payload.
// const L layout(1, 1, n, m);
// double* a = layout.Pointer<double>(p);
// int* b = layout.Pointer<int>(p);
// }
//
// The layout we used above combines fixed-size with dynamically-sized fields.
// This is quite common. Layout is optimized for this use case and generates
// optimal code. All computations that can be performed at compile time are
// indeed performed at compile time.
//
// Efficiency tip: The order of fields matters. In `Layout<T1, ..., TN>` try to
// ensure that `alignof(T1) >= ... >= alignof(TN)`. This way you'll have no
// padding in between arrays.
//
// You can manually override the alignment of an array by wrapping the type in
// `Aligned<T, N>`. `Layout<..., Aligned<T, N>, ...>` has exactly the same API
// and behavior as `Layout<..., T, ...>` except that the first element of the
// array of `T` is aligned to `N` (the rest of the elements follow without
// padding). `N` cannot be less than `alignof(T)`.
//
// `AllocSize()` and `Pointer()` are the most basic methods for dealing with
// memory layouts. Check out the reference or code below to discover more.
//
// EXAMPLE
//
// // Immutable move-only string with sizeof equal to sizeof(void*). The
// // string size and the characters are kept in the same heap allocation.
// class CompactString {
// public:
// CompactString(const char* s = "") {
// const size_t size = strlen(s);
// // size_t[1] followed by char[size + 1].
// const L layout(1, size + 1);
// p_.reset(new unsigned char[layout.AllocSize()]);
// // If running under ASAN, mark the padding bytes, if any, to catch
// // memory errors.
// layout.PoisonPadding(p_.get());
// // Store the size in the allocation.
// *layout.Pointer<size_t>(p_.get()) = size;
// // Store the characters in the allocation.
// memcpy(layout.Pointer<char>(p_.get()), s, size + 1);
// }
//
// size_t size() const {
// // Equivalent to reinterpret_cast<size_t&>(*p).
// return *L::Partial().Pointer<size_t>(p_.get());
// }
//
// const char* c_str() const {
// // Equivalent to reinterpret_cast<char*>(p.get() + sizeof(size_t)).
// // The argument in Partial(1) specifies that we have size_t[1] in front
// // of the characters.
// return L::Partial(1).Pointer<char>(p_.get());
// }
//
// private:
// // Our heap allocation contains a size_t followed by an array of chars.
// using L = Layout<size_t, char>;
// std::unique_ptr<unsigned char[]> p_;
// };
//
// int main() {
// CompactString s = "hello";
// assert(s.size() == 5);
// assert(strcmp(s.c_str(), "hello") == 0);
// }
//
// DOCUMENTATION
//
// The interface exported by this file consists of:
// - class `Layout<>` and its public members.
// - The public members of class `internal_layout::LayoutImpl<>`. That class
// isn't intended to be used directly, and its name and template parameter
// list are internal implementation details, but the class itself provides
// most of the functionality in this file. See comments on its members for
// detailed documentation.
//
// `Layout<T1,... Tn>::Partial(count1,..., countm)` (where `m` <= `n`) returns a
// `LayoutImpl<>` object. `Layout<T1,..., Tn> layout(count1,..., countn)`
// creates a `Layout` object, which exposes the same functionality by inheriting
// from `LayoutImpl<>`.
#ifndef ABSL_CONTAINER_INTERNAL_LAYOUT_H_
#define ABSL_CONTAINER_INTERNAL_LAYOUT_H_
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include <ostream>
#include <string>
#include <tuple>
#include <type_traits>
#include <typeinfo>
#include <utility>
#include "absl/base/config.h"
#include "absl/debugging/internal/demangle.h"
#include "absl/meta/type_traits.h"
#include "absl/strings/str_cat.h"
#include "absl/types/span.h"
#include "absl/utility/utility.h"
#ifdef ABSL_HAVE_ADDRESS_SANITIZER
#include <sanitizer/asan_interface.h>
#endif
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
// A type wrapper that instructs `Layout` to use the specific alignment for the
// array. `Layout<..., Aligned<T, N>, ...>` has exactly the same API
// and behavior as `Layout<..., T, ...>` except that the first element of the
// array of `T` is aligned to `N` (the rest of the elements follow without
// padding).
//
// Requires: `N >= alignof(T)` and `N` is a power of 2.
template <class T, size_t N>
struct Aligned;
namespace internal_layout {
template <class T>
struct NotAligned {};
template <class T, size_t N>
struct NotAligned<const Aligned<T, N>> {
static_assert(sizeof(T) == 0, "Aligned<T, N> cannot be const-qualified");
};
template <size_t>
using IntToSize = size_t;
template <class>
using TypeToSize = size_t;
template <class T>
struct Type : NotAligned<T> {
using type = T;
};
template <class T, size_t N>
struct Type<Aligned<T, N>> {
using type = T;
};
template <class T>
struct SizeOf : NotAligned<T>, std::integral_constant<size_t, sizeof(T)> {};
template <class T, size_t N>
struct SizeOf<Aligned<T, N>> : std::integral_constant<size_t, sizeof(T)> {};
// Note: workaround for https://gcc.gnu.org/PR88115
template <class T>
struct AlignOf : NotAligned<T> {
static constexpr size_t value = alignof(T);
};
template <class T, size_t N>
struct AlignOf<Aligned<T, N>> {
static_assert(N % alignof(T) == 0,
"Custom alignment can't be lower than the type's alignment");
static constexpr size_t value = N;
};
// Does `Ts...` contain `T`?
template <class T, class... Ts>
using Contains = absl::disjunction<std::is_same<T, Ts>...>;
template <class From, class To>
using CopyConst =
typename std::conditional<std::is_const<From>::value, const To, To>::type;
// Note: We're not qualifying this with absl:: because it doesn't compile under
// MSVC.
template <class T>
using SliceType = Span<T>;
// This namespace contains no types. It prevents functions defined in it from
// being found by ADL.
namespace adl_barrier {
template <class Needle, class... Ts>
constexpr size_t Find(Needle, Needle, Ts...) {
static_assert(!Contains<Needle, Ts...>(), "Duplicate element type");
return 0;
}
template <class Needle, class T, class... Ts>
constexpr size_t Find(Needle, T, Ts...) {
return adl_barrier::Find(Needle(), Ts()...) + 1;
}
constexpr bool IsPow2(size_t n) { return !(n & (n - 1)); }
// Returns `q * m` for the smallest `q` such that `q * m >= n`.
// Requires: `m` is a power of two. It's enforced by IsLegalElementType below.
constexpr size_t Align(size_t n, size_t m) { return (n + m - 1) & ~(m - 1); }
constexpr size_t Min(size_t a, size_t b) { return b < a ? b : a; }
constexpr size_t Max(size_t a) { return a; }
template <class... Ts>
constexpr size_t Max(size_t a, size_t b, Ts... rest) {
return adl_barrier::Max(b < a ? a : b, rest...);
}
template <class T>
std::string TypeName() {
std::string out;
#if ABSL_INTERNAL_HAS_RTTI
absl::StrAppend(&out, "<",
absl::debugging_internal::DemangleString(typeid(T).name()),
">");
#endif
return out;
}
} // namespace adl_barrier
template <bool C>
using EnableIf = typename std::enable_if<C, int>::type;
// Can `T` be a template argument of `Layout`?
template <class T>
using IsLegalElementType = std::integral_constant<
bool, !std::is_reference<T>::value && !std::is_volatile<T>::value &&
!std::is_reference<typename Type<T>::type>::value &&
!std::is_volatile<typename Type<T>::type>::value &&
adl_barrier::IsPow2(AlignOf<T>::value)>;
template <class Elements, class SizeSeq, class OffsetSeq>
class LayoutImpl;
// Public base class of `Layout` and the result type of `Layout::Partial()`.
//
// `Elements...` contains all template arguments of `Layout` that created this
// instance.
//
// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is the number of arguments
// passed to `Layout::Partial()` or `Layout::Layout()`.
//
// `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is
// `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we
// can compute offsets).
template <class... Elements, size_t... SizeSeq, size_t... OffsetSeq>
class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>,
absl::index_sequence<OffsetSeq...>> {
private:
static_assert(sizeof...(Elements) > 0, "At least one field is required");
static_assert(absl::conjunction<IsLegalElementType<Elements>...>::value,
"Invalid element type (see IsLegalElementType)");
enum {
NumTypes = sizeof...(Elements),
NumSizes = sizeof...(SizeSeq),
NumOffsets = sizeof...(OffsetSeq),
};
// These are guaranteed by `Layout`.
static_assert(NumOffsets == adl_barrier::Min(NumTypes, NumSizes + 1),
"Internal error");
static_assert(NumTypes > 0, "Internal error");
// Returns the index of `T` in `Elements...`. Results in a compilation error
// if `Elements...` doesn't contain exactly one instance of `T`.
template <class T>
static constexpr size_t ElementIndex() {
static_assert(Contains<Type<T>, Type<typename Type<Elements>::type>...>(),
"Type not found");
return adl_barrier::Find(Type<T>(),
Type<typename Type<Elements>::type>()...);
}
template <size_t N>
using ElementAlignment =
AlignOf<typename std::tuple_element<N, std::tuple<Elements...>>::type>;
public:
// Element types of all arrays packed in a tuple.
using ElementTypes = std::tuple<typename Type<Elements>::type...>;
// Element type of the Nth array.
template <size_t N>
using ElementType = typename std::tuple_element<N, ElementTypes>::type;
constexpr explicit LayoutImpl(IntToSize<SizeSeq>... sizes)
: size_{sizes...} {}
// Alignment of the layout, equal to the strictest alignment of all elements.
// All pointers passed to the methods of layout must be aligned to this value.
static constexpr size_t Alignment() {
return adl_barrier::Max(AlignOf<Elements>::value...);
}
// Offset in bytes of the Nth array.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// assert(x.Offset<0>() == 0); // The ints starts from 0.
// assert(x.Offset<1>() == 16); // The doubles starts from 16.
//
// Requires: `N <= NumSizes && N < sizeof...(Ts)`.
template <size_t N, EnableIf<N == 0> = 0>
constexpr size_t Offset() const {
return 0;
}
template <size_t N, EnableIf<N != 0> = 0>
constexpr size_t Offset() const {
static_assert(N < NumOffsets, "Index out of bounds");
return adl_barrier::Align(
Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * size_[N - 1],
ElementAlignment<N>::value);
}
// Offset in bytes of the array with the specified element type. There must
// be exactly one such array and its zero-based index must be at most
// `NumSizes`.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// assert(x.Offset<int>() == 0); // The ints starts from 0.
// assert(x.Offset<double>() == 16); // The doubles starts from 16.
template <class T>
constexpr size_t Offset() const {
return Offset<ElementIndex<T>()>();
}
// Offsets in bytes of all arrays for which the offsets are known.
constexpr std::array<size_t, NumOffsets> Offsets() const {
return {{Offset<OffsetSeq>()...}};
}
// The number of elements in the Nth array. This is the Nth argument of
// `Layout::Partial()` or `Layout::Layout()` (zero-based).
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// assert(x.Size<0>() == 3);
// assert(x.Size<1>() == 4);
//
// Requires: `N < NumSizes`.
template <size_t N>
constexpr size_t Size() const {
static_assert(N < NumSizes, "Index out of bounds");
return size_[N];
}
// The number of elements in the array with the specified element type.
// There must be exactly one such array and its zero-based index must be
// at most `NumSizes`.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// assert(x.Size<int>() == 3);
// assert(x.Size<double>() == 4);
template <class T>
constexpr size_t Size() const {
return Size<ElementIndex<T>()>();
}
// The number of elements of all arrays for which they are known.
constexpr std::array<size_t, NumSizes> Sizes() const {
return {{Size<SizeSeq>()...}};
}
// Pointer to the beginning of the Nth array.
//
// `Char` must be `[const] [signed|unsigned] char`.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// unsigned char* p = new unsigned char[x.AllocSize()];
// int* ints = x.Pointer<0>(p);
// double* doubles = x.Pointer<1>(p);
//
// Requires: `N <= NumSizes && N < sizeof...(Ts)`.
// Requires: `p` is aligned to `Alignment()`.
template <size_t N, class Char>
CopyConst<Char, ElementType<N>>* Pointer(Char* p) const {
using C = typename std::remove_const<Char>::type;
static_assert(
std::is_same<C, char>() || std::is_same<C, unsigned char>() ||
std::is_same<C, signed char>(),
"The argument must be a pointer to [const] [signed|unsigned] char");
constexpr size_t alignment = Alignment();
(void)alignment;
assert(reinterpret_cast<uintptr_t>(p) % alignment == 0);
return reinterpret_cast<CopyConst<Char, ElementType<N>>*>(p + Offset<N>());
}
// Pointer to the beginning of the array with the specified element type.
// There must be exactly one such array and its zero-based index must be at
// most `NumSizes`.
//
// `Char` must be `[const] [signed|unsigned] char`.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// unsigned char* p = new unsigned char[x.AllocSize()];
// int* ints = x.Pointer<int>(p);
// double* doubles = x.Pointer<double>(p);
//
// Requires: `p` is aligned to `Alignment()`.
template <class T, class Char>
CopyConst<Char, T>* Pointer(Char* p) const {
return Pointer<ElementIndex<T>()>(p);
}
// Pointers to all arrays for which pointers are known.
//
// `Char` must be `[const] [signed|unsigned] char`.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// unsigned char* p = new unsigned char[x.AllocSize()];
//
// int* ints;
// double* doubles;
// std::tie(ints, doubles) = x.Pointers(p);
//
// Requires: `p` is aligned to `Alignment()`.
//
// Note: We're not using ElementType alias here because it does not compile
// under MSVC.
template <class Char>
std::tuple<CopyConst<
Char, typename std::tuple_element<OffsetSeq, ElementTypes>::type>*...>
Pointers(Char* p) const {
return std::tuple<CopyConst<Char, ElementType<OffsetSeq>>*...>(
Pointer<OffsetSeq>(p)...);
}
// The Nth array.
//
// `Char` must be `[const] [signed|unsigned] char`.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// unsigned char* p = new unsigned char[x.AllocSize()];
// Span<int> ints = x.Slice<0>(p);
// Span<double> doubles = x.Slice<1>(p);
//
// Requires: `N < NumSizes`.
// Requires: `p` is aligned to `Alignment()`.
template <size_t N, class Char>
SliceType<CopyConst<Char, ElementType<N>>> Slice(Char* p) const {
return SliceType<CopyConst<Char, ElementType<N>>>(Pointer<N>(p), Size<N>());
}
// The array with the specified element type. There must be exactly one
// such array and its zero-based index must be less than `NumSizes`.
//
// `Char` must be `[const] [signed|unsigned] char`.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// unsigned char* p = new unsigned char[x.AllocSize()];
// Span<int> ints = x.Slice<int>(p);
// Span<double> doubles = x.Slice<double>(p);
//
// Requires: `p` is aligned to `Alignment()`.
template <class T, class Char>
SliceType<CopyConst<Char, T>> Slice(Char* p) const {
return Slice<ElementIndex<T>()>(p);
}
// All arrays with known sizes.
//
// `Char` must be `[const] [signed|unsigned] char`.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// unsigned char* p = new unsigned char[x.AllocSize()];
//
// Span<int> ints;
// Span<double> doubles;
// std::tie(ints, doubles) = x.Slices(p);
//
// Requires: `p` is aligned to `Alignment()`.
//
// Note: We're not using ElementType alias here because it does not compile
// under MSVC.
template <class Char>
std::tuple<SliceType<CopyConst<
Char, typename std::tuple_element<SizeSeq, ElementTypes>::type>>...>
Slices(Char* p) const {
// Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed
// in 6.1).
(void)p;
return std::tuple<SliceType<CopyConst<Char, ElementType<SizeSeq>>>...>(
Slice<SizeSeq>(p)...);
}
// The size of the allocation that fits all arrays.
//
// // int[3], 4 bytes of padding, double[4].
// Layout<int, double> x(3, 4);
// unsigned char* p = new unsigned char[x.AllocSize()]; // 48 bytes
//
// Requires: `NumSizes == sizeof...(Ts)`.
constexpr size_t AllocSize() const {
static_assert(NumTypes == NumSizes, "You must specify sizes of all fields");
return Offset<NumTypes - 1>() +
SizeOf<ElementType<NumTypes - 1>>::value * size_[NumTypes - 1];
}
// If built with --config=asan, poisons padding bytes (if any) in the
// allocation. The pointer must point to a memory block at least
// `AllocSize()` bytes in length.
//
// `Char` must be `[const] [signed|unsigned] char`.
//
// Requires: `p` is aligned to `Alignment()`.
template <class Char, size_t N = NumOffsets - 1, EnableIf<N == 0> = 0>
void PoisonPadding(const Char* p) const {
Pointer<0>(p); // verify the requirements on `Char` and `p`
}
template <class Char, size_t N = NumOffsets - 1, EnableIf<N != 0> = 0>
void PoisonPadding(const Char* p) const {
static_assert(N < NumOffsets, "Index out of bounds");
(void)p;
#ifdef ABSL_HAVE_ADDRESS_SANITIZER
PoisonPadding<Char, N - 1>(p);
// The `if` is an optimization. It doesn't affect the observable behaviour.
if (ElementAlignment<N - 1>::value % ElementAlignment<N>::value) {
size_t start =
Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * size_[N - 1];
ASAN_POISON_MEMORY_REGION(p + start, Offset<N>() - start);
}
#endif
}
// Human-readable description of the memory layout. Useful for debugging.
// Slow.
//
// // char[5], 3 bytes of padding, int[3], 4 bytes of padding, followed
// // by an unknown number of doubles.
// auto x = Layout<char, int, double>::Partial(5, 3);
// assert(x.DebugString() ==
// "@0<char>(1)[5]; @8<int>(4)[3]; @24<double>(8)");
//
// Each field is in the following format: @offset<type>(sizeof)[size] (<type>
// may be missing depending on the target platform). For example,
// @8<int>(4)[3] means that at offset 8 we have an array of ints, where each
// int is 4 bytes, and we have 3 of those ints. The size of the last field may
// be missing (as in the example above). Only fields with known offsets are
// described. Type names may differ across platforms: one compiler might
// produce "unsigned*" where another produces "unsigned int *".
std::string DebugString() const {
const auto offsets = Offsets();
const size_t sizes[] = {SizeOf<ElementType<OffsetSeq>>::value...};
const std::string types[] = {
adl_barrier::TypeName<ElementType<OffsetSeq>>()...};
std::string res = absl::StrCat("@0", types[0], "(", sizes[0], ")");
for (size_t i = 0; i != NumOffsets - 1; ++i) {
absl::StrAppend(&res, "[", size_[i], "]; @", offsets[i + 1], types[i + 1],
"(", sizes[i + 1], ")");
}
// NumSizes is a constant that may be zero. Some compilers cannot see that
// inside the if statement "size_[NumSizes - 1]" must be valid.
int last = static_cast<int>(NumSizes) - 1;
if (NumTypes == NumSizes && last >= 0) {
absl::StrAppend(&res, "[", size_[last], "]");
}
return res;
}
private:
// Arguments of `Layout::Partial()` or `Layout::Layout()`.
size_t size_[NumSizes > 0 ? NumSizes : 1];
};
template <size_t NumSizes, class... Ts>
using LayoutType = LayoutImpl<
std::tuple<Ts...>, absl::make_index_sequence<NumSizes>,
absl::make_index_sequence<adl_barrier::Min(sizeof...(Ts), NumSizes + 1)>>;
} // namespace internal_layout
// Descriptor of arrays of various types and sizes laid out in memory one after
// another. See the top of the file for documentation.
//
// Check out the public API of internal_layout::LayoutImpl above. The type is
// internal to the library but its methods are public, and they are inherited
// by `Layout`.
template <class... Ts>
class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...> {
public:
static_assert(sizeof...(Ts) > 0, "At least one field is required");
static_assert(
absl::conjunction<internal_layout::IsLegalElementType<Ts>...>::value,
"Invalid element type (see IsLegalElementType)");
// The result type of `Partial()` with `NumSizes` arguments.
template <size_t NumSizes>
using PartialType = internal_layout::LayoutType<NumSizes, Ts...>;
// `Layout` knows the element types of the arrays we want to lay out in
// memory but not the number of elements in each array.
// `Partial(size1, ..., sizeN)` allows us to specify the latter. The
// resulting immutable object can be used to obtain pointers to the
// individual arrays.
//
// It's allowed to pass fewer array sizes than the number of arrays. E.g.,
// if all you need is to the offset of the second array, you only need to
// pass one argument -- the number of elements in the first array.
//
// // int[3] followed by 4 bytes of padding and an unknown number of
// // doubles.
// auto x = Layout<int, double>::Partial(3);
// // doubles start at byte 16.
// assert(x.Offset<1>() == 16);
//
// If you know the number of elements in all arrays, you can still call
// `Partial()` but it's more convenient to use the constructor of `Layout`.
//
// Layout<int, double> x(3, 5);
//
// Note: The sizes of the arrays must be specified in number of elements,
// not in bytes.
//
// Requires: `sizeof...(Sizes) <= sizeof...(Ts)`.
// Requires: all arguments are convertible to `size_t`.
template <class... Sizes>
static constexpr PartialType<sizeof...(Sizes)> Partial(Sizes&&... sizes) {
static_assert(sizeof...(Sizes) <= sizeof...(Ts), "");
return PartialType<sizeof...(Sizes)>(absl::forward<Sizes>(sizes)...);
}
// Creates a layout with the sizes of all arrays specified. If you know
// only the sizes of the first N arrays (where N can be zero), you can use
// `Partial()` defined above. The constructor is essentially equivalent to
// calling `Partial()` and passing in all array sizes; the constructor is
// provided as a convenient abbreviation.
//
// Note: The sizes of the arrays must be specified in number of elements,
// not in bytes.
constexpr explicit Layout(internal_layout::TypeToSize<Ts>... sizes)
: internal_layout::LayoutType<sizeof...(Ts), Ts...>(sizes...) {}
};
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_LAYOUT_H_

224
Pods/abseil/absl/container/internal/raw_hash_map.h generated Normal file
View File

@@ -0,0 +1,224 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef ABSL_CONTAINER_INTERNAL_RAW_HASH_MAP_H_
#define ABSL_CONTAINER_INTERNAL_RAW_HASH_MAP_H_
#include <tuple>
#include <type_traits>
#include <utility>
#include "absl/base/attributes.h"
#include "absl/base/config.h"
#include "absl/base/internal/throw_delegate.h"
#include "absl/container/internal/container_memory.h"
#include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
template <class Policy, class Hash, class Eq, class Alloc>
class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc> {
// P is Policy. It's passed as a template argument to support maps that have
// incomplete types as values, as in unordered_map<K, IncompleteType>.
// MappedReference<> may be a non-reference type.
template <class P>
using MappedReference = decltype(P::value(
std::addressof(std::declval<typename raw_hash_map::reference>())));
// MappedConstReference<> may be a non-reference type.
template <class P>
using MappedConstReference = decltype(P::value(
std::addressof(std::declval<typename raw_hash_map::const_reference>())));
using KeyArgImpl =
KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>;
public:
using key_type = typename Policy::key_type;
using mapped_type = typename Policy::mapped_type;
template <class K>
using key_arg = typename KeyArgImpl::template type<K, key_type>;
static_assert(!std::is_reference<key_type>::value, "");
// TODO(b/187807849): Evaluate whether to support reference mapped_type and
// remove this assertion if/when it is supported.
static_assert(!std::is_reference<mapped_type>::value, "");
using iterator = typename raw_hash_map::raw_hash_set::iterator;
using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator;
raw_hash_map() {}
using raw_hash_map::raw_hash_set::raw_hash_set;
// The last two template parameters ensure that both arguments are rvalues
// (lvalue arguments are handled by the overloads below). This is necessary
// for supporting bitfield arguments.
//
// union { int n : 1; };
// flat_hash_map<int, int> m;
// m.insert_or_assign(n, n);
template <class K = key_type, class V = mapped_type, K* = nullptr,
V* = nullptr>
std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v));
}
template <class K = key_type, class V = mapped_type, K* = nullptr>
std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert_or_assign_impl(std::forward<K>(k), v);
}
template <class K = key_type, class V = mapped_type, V* = nullptr>
std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert_or_assign_impl(k, std::forward<V>(v));
}
template <class K = key_type, class V = mapped_type>
std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert_or_assign_impl(k, v);
}
template <class K = key_type, class V = mapped_type, K* = nullptr,
V* = nullptr>
iterator insert_or_assign(const_iterator, key_arg<K>&& k,
V&& v) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first;
}
template <class K = key_type, class V = mapped_type, K* = nullptr>
iterator insert_or_assign(const_iterator, key_arg<K>&& k,
const V& v) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert_or_assign(std::forward<K>(k), v).first;
}
template <class K = key_type, class V = mapped_type, V* = nullptr>
iterator insert_or_assign(const_iterator, const key_arg<K>& k,
V&& v) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert_or_assign(k, std::forward<V>(v)).first;
}
template <class K = key_type, class V = mapped_type>
iterator insert_or_assign(const_iterator, const key_arg<K>& k,
const V& v) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert_or_assign(k, v).first;
}
// All `try_emplace()` overloads make the same guarantees regarding rvalue
// arguments as `std::unordered_map::try_emplace()`, namely that these
// functions will not move from rvalue arguments if insertions do not happen.
template <class K = key_type, class... Args,
typename std::enable_if<
!std::is_convertible<K, const_iterator>::value, int>::type = 0,
K* = nullptr>
std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...);
}
template <class K = key_type, class... Args,
typename std::enable_if<
!std::is_convertible<K, const_iterator>::value, int>::type = 0>
std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return try_emplace_impl(k, std::forward<Args>(args)...);
}
template <class K = key_type, class... Args, K* = nullptr>
iterator try_emplace(const_iterator, key_arg<K>&& k,
Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first;
}
template <class K = key_type, class... Args>
iterator try_emplace(const_iterator, const key_arg<K>& k,
Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return try_emplace(k, std::forward<Args>(args)...).first;
}
template <class K = key_type, class P = Policy>
MappedReference<P> at(const key_arg<K>& key) ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto it = this->find(key);
if (it == this->end()) {
base_internal::ThrowStdOutOfRange(
"absl::container_internal::raw_hash_map<>::at");
}
return Policy::value(&*it);
}
template <class K = key_type, class P = Policy>
MappedConstReference<P> at(const key_arg<K>& key) const
ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto it = this->find(key);
if (it == this->end()) {
base_internal::ThrowStdOutOfRange(
"absl::container_internal::raw_hash_map<>::at");
}
return Policy::value(&*it);
}
template <class K = key_type, class P = Policy, K* = nullptr>
MappedReference<P> operator[](key_arg<K>&& key)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
// It is safe to use unchecked_deref here because try_emplace
// will always return an iterator pointing to a valid item in the table,
// since it inserts if nothing is found for the given key.
return Policy::value(
&this->unchecked_deref(try_emplace(std::forward<K>(key)).first));
}
template <class K = key_type, class P = Policy>
MappedReference<P> operator[](const key_arg<K>& key)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
// It is safe to use unchecked_deref here because try_emplace
// will always return an iterator pointing to a valid item in the table,
// since it inserts if nothing is found for the given key.
return Policy::value(&this->unchecked_deref(try_emplace(key).first));
}
private:
template <class K, class V>
std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto res = this->find_or_prepare_insert(k);
if (res.second)
this->emplace_at(res.first, std::forward<K>(k), std::forward<V>(v));
else
Policy::value(&*this->iterator_at(res.first)) = std::forward<V>(v);
return {this->iterator_at(res.first), res.second};
}
template <class K = key_type, class... Args>
std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto res = this->find_or_prepare_insert(k);
if (res.second)
this->emplace_at(res.first, std::piecewise_construct,
std::forward_as_tuple(std::forward<K>(k)),
std::forward_as_tuple(std::forward<Args>(args)...));
return {this->iterator_at(res.first), res.second};
}
};
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_MAP_H_

380
Pods/abseil/absl/container/internal/raw_hash_set.cc generated Normal file
View File

@@ -0,0 +1,380 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/container/internal/raw_hash_set.h"
#include <atomic>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include "absl/base/attributes.h"
#include "absl/base/config.h"
#include "absl/base/dynamic_annotations.h"
#include "absl/container/internal/container_memory.h"
#include "absl/hash/hash.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
// We have space for `growth_left` before a single block of control bytes. A
// single block of empty control bytes for tables without any slots allocated.
// This enables removing a branch in the hot path of find(). In order to ensure
// that the control bytes are aligned to 16, we have 16 bytes before the control
// bytes even though growth_left only needs 8.
constexpr ctrl_t ZeroCtrlT() { return static_cast<ctrl_t>(0); }
alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = {
ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(),
ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(),
ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(),
ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(),
ctrl_t::kSentinel, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty};
#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL
constexpr size_t Group::kWidth;
#endif
namespace {
// Returns "random" seed.
inline size_t RandomSeed() {
#ifdef ABSL_HAVE_THREAD_LOCAL
static thread_local size_t counter = 0;
// On Linux kernels >= 5.4 the MSAN runtime has a false-positive when
// accessing thread local storage data from loaded libraries
// (https://github.com/google/sanitizers/issues/1265), for this reason counter
// needs to be annotated as initialized.
ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(&counter, sizeof(size_t));
size_t value = ++counter;
#else // ABSL_HAVE_THREAD_LOCAL
static std::atomic<size_t> counter(0);
size_t value = counter.fetch_add(1, std::memory_order_relaxed);
#endif // ABSL_HAVE_THREAD_LOCAL
return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter));
}
bool ShouldRehashForBugDetection(const ctrl_t* ctrl, size_t capacity) {
// Note: we can't use the abseil-random library because abseil-random
// depends on swisstable. We want to return true with probability
// `min(1, RehashProbabilityConstant() / capacity())`. In order to do this,
// we probe based on a random hash and see if the offset is less than
// RehashProbabilityConstant().
return probe(ctrl, capacity, absl::HashOf(RandomSeed())).offset() <
RehashProbabilityConstant();
}
} // namespace
GenerationType* EmptyGeneration() {
if (SwisstableGenerationsEnabled()) {
constexpr size_t kNumEmptyGenerations = 1024;
static constexpr GenerationType kEmptyGenerations[kNumEmptyGenerations]{};
return const_cast<GenerationType*>(
&kEmptyGenerations[RandomSeed() % kNumEmptyGenerations]);
}
return nullptr;
}
bool CommonFieldsGenerationInfoEnabled::
should_rehash_for_bug_detection_on_insert(const ctrl_t* ctrl,
size_t capacity) const {
if (reserved_growth_ == kReservedGrowthJustRanOut) return true;
if (reserved_growth_ > 0) return false;
return ShouldRehashForBugDetection(ctrl, capacity);
}
bool CommonFieldsGenerationInfoEnabled::should_rehash_for_bug_detection_on_move(
const ctrl_t* ctrl, size_t capacity) const {
return ShouldRehashForBugDetection(ctrl, capacity);
}
bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl) {
// To avoid problems with weak hashes and single bit tests, we use % 13.
// TODO(kfm,sbenza): revisit after we do unconditional mixing
return (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6;
}
void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) {
assert(ctrl[capacity] == ctrl_t::kSentinel);
assert(IsValidCapacity(capacity));
for (ctrl_t* pos = ctrl; pos < ctrl + capacity; pos += Group::kWidth) {
Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos);
}
// Copy the cloned ctrl bytes.
std::memcpy(ctrl + capacity + 1, ctrl, NumClonedBytes());
ctrl[capacity] = ctrl_t::kSentinel;
}
// Extern template instantiation for inline function.
template FindInfo find_first_non_full(const CommonFields&, size_t);
FindInfo find_first_non_full_outofline(const CommonFields& common,
size_t hash) {
return find_first_non_full(common, hash);
}
// Returns the address of the slot just after slot assuming each slot has the
// specified size.
static inline void* NextSlot(void* slot, size_t slot_size) {
return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot) + slot_size);
}
// Returns the address of the slot just before slot assuming each slot has the
// specified size.
static inline void* PrevSlot(void* slot, size_t slot_size) {
return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot) - slot_size);
}
void DropDeletesWithoutResize(CommonFields& common,
const PolicyFunctions& policy, void* tmp_space) {
void* set = &common;
void* slot_array = common.slot_array();
const size_t capacity = common.capacity();
assert(IsValidCapacity(capacity));
assert(!is_small(capacity));
// Algorithm:
// - mark all DELETED slots as EMPTY
// - mark all FULL slots as DELETED
// - for each slot marked as DELETED
// hash = Hash(element)
// target = find_first_non_full(hash)
// if target is in the same group
// mark slot as FULL
// else if target is EMPTY
// transfer element to target
// mark slot as EMPTY
// mark target as FULL
// else if target is DELETED
// swap current element with target element
// mark target as FULL
// repeat procedure for current slot with moved from element (target)
ctrl_t* ctrl = common.control();
ConvertDeletedToEmptyAndFullToDeleted(ctrl, capacity);
auto hasher = policy.hash_slot;
auto transfer = policy.transfer;
const size_t slot_size = policy.slot_size;
size_t total_probe_length = 0;
void* slot_ptr = SlotAddress(slot_array, 0, slot_size);
for (size_t i = 0; i != capacity;
++i, slot_ptr = NextSlot(slot_ptr, slot_size)) {
assert(slot_ptr == SlotAddress(slot_array, i, slot_size));
if (!IsDeleted(ctrl[i])) continue;
const size_t hash = (*hasher)(set, slot_ptr);
const FindInfo target = find_first_non_full(common, hash);
const size_t new_i = target.offset;
total_probe_length += target.probe_length;
// Verify if the old and new i fall within the same group wrt the hash.
// If they do, we don't need to move the object as it falls already in the
// best probe we can.
const size_t probe_offset = probe(common, hash).offset();
const auto probe_index = [probe_offset, capacity](size_t pos) {
return ((pos - probe_offset) & capacity) / Group::kWidth;
};
// Element doesn't move.
if (ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) {
SetCtrl(common, i, H2(hash), slot_size);
continue;
}
void* new_slot_ptr = SlotAddress(slot_array, new_i, slot_size);
if (IsEmpty(ctrl[new_i])) {
// Transfer element to the empty spot.
// SetCtrl poisons/unpoisons the slots so we have to call it at the
// right time.
SetCtrl(common, new_i, H2(hash), slot_size);
(*transfer)(set, new_slot_ptr, slot_ptr);
SetCtrl(common, i, ctrl_t::kEmpty, slot_size);
} else {
assert(IsDeleted(ctrl[new_i]));
SetCtrl(common, new_i, H2(hash), slot_size);
// Until we are done rehashing, DELETED marks previously FULL slots.
// Swap i and new_i elements.
(*transfer)(set, tmp_space, new_slot_ptr);
(*transfer)(set, new_slot_ptr, slot_ptr);
(*transfer)(set, slot_ptr, tmp_space);
// repeat the processing of the ith slot
--i;
slot_ptr = PrevSlot(slot_ptr, slot_size);
}
}
ResetGrowthLeft(common);
common.infoz().RecordRehash(total_probe_length);
}
static bool WasNeverFull(CommonFields& c, size_t index) {
if (is_single_group(c.capacity())) {
return true;
}
const size_t index_before = (index - Group::kWidth) & c.capacity();
const auto empty_after = Group(c.control() + index).MaskEmpty();
const auto empty_before = Group(c.control() + index_before).MaskEmpty();
// We count how many consecutive non empties we have to the right and to the
// left of `it`. If the sum is >= kWidth then there is at least one probe
// window that might have seen a full group.
return empty_before && empty_after &&
static_cast<size_t>(empty_after.TrailingZeros()) +
empty_before.LeadingZeros() <
Group::kWidth;
}
void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) {
assert(IsFull(c.control()[index]) && "erasing a dangling iterator");
c.decrement_size();
c.infoz().RecordErase();
if (WasNeverFull(c, index)) {
SetCtrl(c, index, ctrl_t::kEmpty, slot_size);
c.set_growth_left(c.growth_left() + 1);
return;
}
SetCtrl(c, index, ctrl_t::kDeleted, slot_size);
}
void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy,
bool reuse) {
c.set_size(0);
if (reuse) {
ResetCtrl(c, policy.slot_size);
ResetGrowthLeft(c);
c.infoz().RecordStorageChanged(0, c.capacity());
} else {
// We need to record infoz before calling dealloc, which will unregister
// infoz.
c.infoz().RecordClearedReservation();
c.infoz().RecordStorageChanged(0, 0);
(*policy.dealloc)(c, policy);
c.set_control(EmptyGroup());
c.set_generation_ptr(EmptyGeneration());
c.set_slots(nullptr);
c.set_capacity(0);
}
}
void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes(
ctrl_t* new_ctrl, size_t new_capacity) const {
assert(is_single_group(new_capacity));
constexpr size_t kHalfWidth = Group::kWidth / 2;
assert(old_capacity_ < kHalfWidth);
const size_t half_old_capacity = old_capacity_ / 2;
// NOTE: operations are done with compile time known size = kHalfWidth.
// Compiler optimizes that into single ASM operation.
// Copy second half of bytes to the beginning.
// We potentially copy more bytes in order to have compile time known size.
// Mirrored bytes from the old_ctrl_ will also be copied.
// In case of old_capacity_ == 3, we will copy 1st element twice.
// Examples:
// old_ctrl = 0S0EEEEEEE...
// new_ctrl = S0EEEEEEEE...
//
// old_ctrl = 01S01EEEEE...
// new_ctrl = 1S01EEEEEE...
//
// old_ctrl = 0123456S0123456EE...
// new_ctrl = 456S0123?????????...
std::memcpy(new_ctrl, old_ctrl_ + half_old_capacity + 1, kHalfWidth);
// Clean up copied kSentinel from old_ctrl.
new_ctrl[half_old_capacity] = ctrl_t::kEmpty;
// Clean up damaged or uninitialized bytes.
// Clean bytes after the intended size of the copy.
// Example:
// new_ctrl = 1E01EEEEEEE????
// *new_ctrl= 1E0EEEEEEEE????
// position /
std::memset(new_ctrl + old_capacity_ + 1, static_cast<int8_t>(ctrl_t::kEmpty),
kHalfWidth);
// Clean non-mirrored bytes that are not initialized.
// For small old_capacity that may be inside of mirrored bytes zone.
// Examples:
// new_ctrl = 1E0EEEEEEEE??????????....
// *new_ctrl= 1E0EEEEEEEEEEEEE?????....
// position /
//
// new_ctrl = 456E0123???????????...
// *new_ctrl= 456E0123EEEEEEEE???...
// position /
std::memset(new_ctrl + kHalfWidth, static_cast<int8_t>(ctrl_t::kEmpty),
kHalfWidth);
// Clean last mirrored bytes that are not initialized
// and will not be overwritten by mirroring.
// Examples:
// new_ctrl = 1E0EEEEEEEEEEEEE????????
// *new_ctrl= 1E0EEEEEEEEEEEEEEEEEEEEE
// position S /
//
// new_ctrl = 456E0123EEEEEEEE???????????????
// *new_ctrl= 456E0123EEEEEEEE???????EEEEEEEE
// position S /
std::memset(new_ctrl + new_capacity + kHalfWidth,
static_cast<int8_t>(ctrl_t::kEmpty), kHalfWidth);
// Create mirrored bytes. old_capacity_ < kHalfWidth
// Example:
// new_ctrl = 456E0123EEEEEEEE???????EEEEEEEE
// *new_ctrl= 456E0123EEEEEEEE456E0123EEEEEEE
// position S/
ctrl_t g[kHalfWidth];
std::memcpy(g, new_ctrl, kHalfWidth);
std::memcpy(new_ctrl + new_capacity + 1, g, kHalfWidth);
// Finally set sentinel to its place.
new_ctrl[new_capacity] = ctrl_t::kSentinel;
}
void HashSetResizeHelper::GrowIntoSingleGroupShuffleTransferableSlots(
void* old_slots, void* new_slots, size_t slot_size) const {
assert(old_capacity_ > 0);
const size_t half_old_capacity = old_capacity_ / 2;
SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_);
std::memcpy(new_slots,
SlotAddress(old_slots, half_old_capacity + 1, slot_size),
slot_size * half_old_capacity);
std::memcpy(SlotAddress(new_slots, half_old_capacity + 1, slot_size),
old_slots, slot_size * (half_old_capacity + 1));
}
void HashSetResizeHelper::GrowSizeIntoSingleGroupTransferable(
CommonFields& c, void* old_slots, size_t slot_size) {
assert(old_capacity_ < Group::kWidth / 2);
assert(is_single_group(c.capacity()));
assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()));
GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity());
GrowIntoSingleGroupShuffleTransferableSlots(old_slots, c.slot_array(),
slot_size);
// We poison since GrowIntoSingleGroupShuffleTransferableSlots
// may leave empty slots unpoisoned.
PoisonSingleGroupEmptySlots(c, slot_size);
}
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl

3325
Pods/abseil/absl/container/internal/raw_hash_set.h generated Normal file
View File

@@ -0,0 +1,3325 @@
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An open-addressing
// hashtable with quadratic probing.
//
// This is a low level hashtable on top of which different interfaces can be
// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc.
//
// The table interface is similar to that of std::unordered_set. Notable
// differences are that most member functions support heterogeneous keys when
// BOTH the hash and eq functions are marked as transparent. They do so by
// providing a typedef called `is_transparent`.
//
// When heterogeneous lookup is enabled, functions that take key_type act as if
// they have an overload set like:
//
// iterator find(const key_type& key);
// template <class K>
// iterator find(const K& key);
//
// size_type erase(const key_type& key);
// template <class K>
// size_type erase(const K& key);
//
// std::pair<iterator, iterator> equal_range(const key_type& key);
// template <class K>
// std::pair<iterator, iterator> equal_range(const K& key);
//
// When heterogeneous lookup is disabled, only the explicit `key_type` overloads
// exist.
//
// find() also supports passing the hash explicitly:
//
// iterator find(const key_type& key, size_t hash);
// template <class U>
// iterator find(const U& key, size_t hash);
//
// In addition the pointer to element and iterator stability guarantees are
// weaker: all iterators and pointers are invalidated after a new element is
// inserted.
//
// IMPLEMENTATION DETAILS
//
// # Table Layout
//
// A raw_hash_set's backing array consists of control bytes followed by slots
// that may or may not contain objects.
//
// The layout of the backing array, for `capacity` slots, is thus, as a
// pseudo-struct:
//
// struct BackingArray {
// // Sampling handler. This field isn't present when the sampling is
// // disabled or this allocation hasn't been selected for sampling.
// HashtablezInfoHandle infoz_;
// // The number of elements we can insert before growing the capacity.
// size_t growth_left;
// // Control bytes for the "real" slots.
// ctrl_t ctrl[capacity];
// // Always `ctrl_t::kSentinel`. This is used by iterators to find when to
// // stop and serves no other purpose.
// ctrl_t sentinel;
// // A copy of the first `kWidth - 1` elements of `ctrl`. This is used so
// // that if a probe sequence picks a value near the end of `ctrl`,
// // `Group` will have valid control bytes to look at.
// ctrl_t clones[kWidth - 1];
// // The actual slot data.
// slot_type slots[capacity];
// };
//
// The length of this array is computed by `AllocSize()` below.
//
// Control bytes (`ctrl_t`) are bytes (collected into groups of a
// platform-specific size) that define the state of the corresponding slot in
// the slot array. Group manipulation is tightly optimized to be as efficient
// as possible: SSE and friends on x86, clever bit operations on other arches.
//
// Group 1 Group 2 Group 3
// +---------------+---------------+---------------+
// | | | | | | | | | | | | | | | | | | | | | | | | |
// +---------------+---------------+---------------+
//
// Each control byte is either a special value for empty slots, deleted slots
// (sometimes called *tombstones*), and a special end-of-table marker used by
// iterators, or, if occupied, seven bits (H2) from the hash of the value in the
// corresponding slot.
//
// Storing control bytes in a separate array also has beneficial cache effects,
// since more logical slots will fit into a cache line.
//
// # Hashing
//
// We compute two separate hashes, `H1` and `H2`, from the hash of an object.
// `H1(hash(x))` is an index into `slots`, and essentially the starting point
// for the probe sequence. `H2(hash(x))` is a 7-bit value used to filter out
// objects that cannot possibly be the one we are looking for.
//
// # Table operations.
//
// The key operations are `insert`, `find`, and `erase`.
//
// Since `insert` and `erase` are implemented in terms of `find`, we describe
// `find` first. To `find` a value `x`, we compute `hash(x)`. From
// `H1(hash(x))` and the capacity, we construct a `probe_seq` that visits every
// group of slots in some interesting order.
//
// We now walk through these indices. At each index, we select the entire group
// starting with that index and extract potential candidates: occupied slots
// with a control byte equal to `H2(hash(x))`. If we find an empty slot in the
// group, we stop and return an error. Each candidate slot `y` is compared with
// `x`; if `x == y`, we are done and return `&y`; otherwise we continue to the
// next probe index. Tombstones effectively behave like full slots that never
// match the value we're looking for.
//
// The `H2` bits ensure when we compare a slot to an object with `==`, we are
// likely to have actually found the object. That is, the chance is low that
// `==` is called and returns `false`. Thus, when we search for an object, we
// are unlikely to call `==` many times. This likelyhood can be analyzed as
// follows (assuming that H2 is a random enough hash function).
//
// Let's assume that there are `k` "wrong" objects that must be examined in a
// probe sequence. For example, when doing a `find` on an object that is in the
// table, `k` is the number of objects between the start of the probe sequence
// and the final found object (not including the final found object). The
// expected number of objects with an H2 match is then `k/128`. Measurements
// and analysis indicate that even at high load factors, `k` is less than 32,
// meaning that the number of "false positive" comparisons we must perform is
// less than 1/8 per `find`.
// `insert` is implemented in terms of `unchecked_insert`, which inserts a
// value presumed to not be in the table (violating this requirement will cause
// the table to behave erratically). Given `x` and its hash `hash(x)`, to insert
// it, we construct a `probe_seq` once again, and use it to find the first
// group with an unoccupied (empty *or* deleted) slot. We place `x` into the
// first such slot in the group and mark it as full with `x`'s H2.
//
// To `insert`, we compose `unchecked_insert` with `find`. We compute `h(x)` and
// perform a `find` to see if it's already present; if it is, we're done. If
// it's not, we may decide the table is getting overcrowded (i.e. the load
// factor is greater than 7/8 for big tables; `is_small()` tables use a max load
// factor of 1); in this case, we allocate a bigger array, `unchecked_insert`
// each element of the table into the new array (we know that no insertion here
// will insert an already-present value), and discard the old backing array. At
// this point, we may `unchecked_insert` the value `x`.
//
// Below, `unchecked_insert` is partly implemented by `prepare_insert`, which
// presents a viable, initialized slot pointee to the caller.
//
// `erase` is implemented in terms of `erase_at`, which takes an index to a
// slot. Given an offset, we simply create a tombstone and destroy its contents.
// If we can prove that the slot would not appear in a probe sequence, we can
// make the slot as empty, instead. We can prove this by observing that if a
// group has any empty slots, it has never been full (assuming we never create
// an empty slot in a group with no empties, which this heuristic guarantees we
// never do) and find would stop at this group anyways (since it does not probe
// beyond groups with empties).
//
// `erase` is `erase_at` composed with `find`: if we
// have a value `x`, we can perform a `find`, and then `erase_at` the resulting
// slot.
//
// To iterate, we simply traverse the array, skipping empty and deleted slots
// and stopping when we hit a `kSentinel`.
#ifndef ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_
#define ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <initializer_list>
#include <iterator>
#include <limits>
#include <memory>
#include <tuple>
#include <type_traits>
#include <utility>
#include "absl/base/attributes.h"
#include "absl/base/config.h"
#include "absl/base/internal/endian.h"
#include "absl/base/internal/raw_logging.h"
#include "absl/base/macros.h"
#include "absl/base/optimization.h"
#include "absl/base/options.h"
#include "absl/base/port.h"
#include "absl/base/prefetch.h"
#include "absl/container/internal/common.h" // IWYU pragma: export // for node_handle
#include "absl/container/internal/compressed_tuple.h"
#include "absl/container/internal/container_memory.h"
#include "absl/container/internal/hash_policy_traits.h"
#include "absl/container/internal/hashtable_debug_hooks.h"
#include "absl/container/internal/hashtablez_sampler.h"
#include "absl/memory/memory.h"
#include "absl/meta/type_traits.h"
#include "absl/numeric/bits.h"
#include "absl/utility/utility.h"
#ifdef ABSL_INTERNAL_HAVE_SSE2
#include <emmintrin.h>
#endif
#ifdef ABSL_INTERNAL_HAVE_SSSE3
#include <tmmintrin.h>
#endif
#ifdef _MSC_VER
#include <intrin.h>
#endif
#ifdef ABSL_INTERNAL_HAVE_ARM_NEON
#include <arm_neon.h>
#endif
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace container_internal {
#ifdef ABSL_SWISSTABLE_ENABLE_GENERATIONS
#error ABSL_SWISSTABLE_ENABLE_GENERATIONS cannot be directly set
#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \
defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \
defined(ABSL_HAVE_MEMORY_SANITIZER)
// When compiled in sanitizer mode, we add generation integers to the backing
// array and iterators. In the backing array, we store the generation between
// the control bytes and the slots. When iterators are dereferenced, we assert
// that the container has not been mutated in a way that could cause iterator
// invalidation since the iterator was initialized.
#define ABSL_SWISSTABLE_ENABLE_GENERATIONS
#endif
// We use uint8_t so we don't need to worry about padding.
using GenerationType = uint8_t;
// A sentinel value for empty generations. Using 0 makes it easy to constexpr
// initialize an array of this value.
constexpr GenerationType SentinelEmptyGeneration() { return 0; }
constexpr GenerationType NextGeneration(GenerationType generation) {
return ++generation == SentinelEmptyGeneration() ? ++generation : generation;
}
#ifdef ABSL_SWISSTABLE_ENABLE_GENERATIONS
constexpr bool SwisstableGenerationsEnabled() { return true; }
constexpr size_t NumGenerationBytes() { return sizeof(GenerationType); }
#else
constexpr bool SwisstableGenerationsEnabled() { return false; }
constexpr size_t NumGenerationBytes() { return 0; }
#endif
template <typename AllocType>
void SwapAlloc(AllocType& lhs, AllocType& rhs,
std::true_type /* propagate_on_container_swap */) {
using std::swap;
swap(lhs, rhs);
}
template <typename AllocType>
void SwapAlloc(AllocType& lhs, AllocType& rhs,
std::false_type /* propagate_on_container_swap */) {
(void)lhs;
(void)rhs;
assert(lhs == rhs &&
"It's UB to call swap with unequal non-propagating allocators.");
}
template <typename AllocType>
void CopyAlloc(AllocType& lhs, AllocType& rhs,
std::true_type /* propagate_alloc */) {
lhs = rhs;
}
template <typename AllocType>
void CopyAlloc(AllocType&, AllocType&, std::false_type /* propagate_alloc */) {}
// The state for a probe sequence.
//
// Currently, the sequence is a triangular progression of the form
//
// p(i) := Width * (i^2 + i)/2 + hash (mod mask + 1)
//
// The use of `Width` ensures that each probe step does not overlap groups;
// the sequence effectively outputs the addresses of *groups* (although not
// necessarily aligned to any boundary). The `Group` machinery allows us
// to check an entire group with minimal branching.
//
// Wrapping around at `mask + 1` is important, but not for the obvious reason.
// As described above, the first few entries of the control byte array
// are mirrored at the end of the array, which `Group` will find and use
// for selecting candidates. However, when those candidates' slots are
// actually inspected, there are no corresponding slots for the cloned bytes,
// so we need to make sure we've treated those offsets as "wrapping around".
//
// It turns out that this probe sequence visits every group exactly once if the
// number of groups is a power of two, since (i^2+i)/2 is a bijection in
// Z/(2^m). See https://en.wikipedia.org/wiki/Quadratic_probing
template <size_t Width>
class probe_seq {
public:
// Creates a new probe sequence using `hash` as the initial value of the
// sequence and `mask` (usually the capacity of the table) as the mask to
// apply to each value in the progression.
probe_seq(size_t hash, size_t mask) {
assert(((mask + 1) & mask) == 0 && "not a mask");
mask_ = mask;
offset_ = hash & mask_;
}
// The offset within the table, i.e., the value `p(i)` above.
size_t offset() const { return offset_; }
size_t offset(size_t i) const { return (offset_ + i) & mask_; }
void next() {
index_ += Width;
offset_ += index_;
offset_ &= mask_;
}
// 0-based probe index, a multiple of `Width`.
size_t index() const { return index_; }
private:
size_t mask_;
size_t offset_;
size_t index_ = 0;
};
template <class ContainerKey, class Hash, class Eq>
struct RequireUsableKey {
template <class PassedKey, class... Args>
std::pair<
decltype(std::declval<const Hash&>()(std::declval<const PassedKey&>())),
decltype(std::declval<const Eq&>()(std::declval<const ContainerKey&>(),
std::declval<const PassedKey&>()))>*
operator()(const PassedKey&, const Args&...) const;
};
template <class E, class Policy, class Hash, class Eq, class... Ts>
struct IsDecomposable : std::false_type {};
template <class Policy, class Hash, class Eq, class... Ts>
struct IsDecomposable<
absl::void_t<decltype(Policy::apply(
RequireUsableKey<typename Policy::key_type, Hash, Eq>(),
std::declval<Ts>()...))>,
Policy, Hash, Eq, Ts...> : std::true_type {};
// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it.
template <class T>
constexpr bool IsNoThrowSwappable(std::true_type = {} /* is_swappable */) {
using std::swap;
return noexcept(swap(std::declval<T&>(), std::declval<T&>()));
}
template <class T>
constexpr bool IsNoThrowSwappable(std::false_type /* is_swappable */) {
return false;
}
template <typename T>
uint32_t TrailingZeros(T x) {
ABSL_ASSUME(x != 0);
return static_cast<uint32_t>(countr_zero(x));
}
// An abstract bitmask, such as that emitted by a SIMD instruction.
//
// Specifically, this type implements a simple bitset whose representation is
// controlled by `SignificantBits` and `Shift`. `SignificantBits` is the number
// of abstract bits in the bitset, while `Shift` is the log-base-two of the
// width of an abstract bit in the representation.
// This mask provides operations for any number of real bits set in an abstract
// bit. To add iteration on top of that, implementation must guarantee no more
// than the most significant real bit is set in a set abstract bit.
template <class T, int SignificantBits, int Shift = 0>
class NonIterableBitMask {
public:
explicit NonIterableBitMask(T mask) : mask_(mask) {}
explicit operator bool() const { return this->mask_ != 0; }
// Returns the index of the lowest *abstract* bit set in `self`.
uint32_t LowestBitSet() const {
return container_internal::TrailingZeros(mask_) >> Shift;
}
// Returns the index of the highest *abstract* bit set in `self`.
uint32_t HighestBitSet() const {
return static_cast<uint32_t>((bit_width(mask_) - 1) >> Shift);
}
// Returns the number of trailing zero *abstract* bits.
uint32_t TrailingZeros() const {
return container_internal::TrailingZeros(mask_) >> Shift;
}
// Returns the number of leading zero *abstract* bits.
uint32_t LeadingZeros() const {
constexpr int total_significant_bits = SignificantBits << Shift;
constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits;
return static_cast<uint32_t>(
countl_zero(static_cast<T>(mask_ << extra_bits))) >>
Shift;
}
T mask_;
};
// Mask that can be iterable
//
// For example, when `SignificantBits` is 16 and `Shift` is zero, this is just
// an ordinary 16-bit bitset occupying the low 16 bits of `mask`. When
// `SignificantBits` is 8 and `Shift` is 3, abstract bits are represented as
// the bytes `0x00` and `0x80`, and it occupies all 64 bits of the bitmask.
//
// For example:
// for (int i : BitMask<uint32_t, 16>(0b101)) -> yields 0, 2
// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3
template <class T, int SignificantBits, int Shift = 0>
class BitMask : public NonIterableBitMask<T, SignificantBits, Shift> {
using Base = NonIterableBitMask<T, SignificantBits, Shift>;
static_assert(std::is_unsigned<T>::value, "");
static_assert(Shift == 0 || Shift == 3, "");
public:
explicit BitMask(T mask) : Base(mask) {}
// BitMask is an iterator over the indices of its abstract bits.
using value_type = int;
using iterator = BitMask;
using const_iterator = BitMask;
BitMask& operator++() {
if (Shift == 3) {
constexpr uint64_t msbs = 0x8080808080808080ULL;
this->mask_ &= msbs;
}
this->mask_ &= (this->mask_ - 1);
return *this;
}
uint32_t operator*() const { return Base::LowestBitSet(); }
BitMask begin() const { return *this; }
BitMask end() const { return BitMask(0); }
private:
friend bool operator==(const BitMask& a, const BitMask& b) {
return a.mask_ == b.mask_;
}
friend bool operator!=(const BitMask& a, const BitMask& b) {
return a.mask_ != b.mask_;
}
};
using h2_t = uint8_t;
// The values here are selected for maximum performance. See the static asserts
// below for details.
// A `ctrl_t` is a single control byte, which can have one of four
// states: empty, deleted, full (which has an associated seven-bit h2_t value)
// and the sentinel. They have the following bit patterns:
//
// empty: 1 0 0 0 0 0 0 0
// deleted: 1 1 1 1 1 1 1 0
// full: 0 h h h h h h h // h represents the hash bits.
// sentinel: 1 1 1 1 1 1 1 1
//
// These values are specifically tuned for SSE-flavored SIMD.
// The static_asserts below detail the source of these choices.
//
// We use an enum class so that when strict aliasing is enabled, the compiler
// knows ctrl_t doesn't alias other types.
enum class ctrl_t : int8_t {
kEmpty = -128, // 0b10000000
kDeleted = -2, // 0b11111110
kSentinel = -1, // 0b11111111
};
static_assert(
(static_cast<int8_t>(ctrl_t::kEmpty) &
static_cast<int8_t>(ctrl_t::kDeleted) &
static_cast<int8_t>(ctrl_t::kSentinel) & 0x80) != 0,
"Special markers need to have the MSB to make checking for them efficient");
static_assert(
ctrl_t::kEmpty < ctrl_t::kSentinel && ctrl_t::kDeleted < ctrl_t::kSentinel,
"ctrl_t::kEmpty and ctrl_t::kDeleted must be smaller than "
"ctrl_t::kSentinel to make the SIMD test of IsEmptyOrDeleted() efficient");
static_assert(
ctrl_t::kSentinel == static_cast<ctrl_t>(-1),
"ctrl_t::kSentinel must be -1 to elide loading it from memory into SIMD "
"registers (pcmpeqd xmm, xmm)");
static_assert(ctrl_t::kEmpty == static_cast<ctrl_t>(-128),
"ctrl_t::kEmpty must be -128 to make the SIMD check for its "
"existence efficient (psignb xmm, xmm)");
static_assert(
(~static_cast<int8_t>(ctrl_t::kEmpty) &
~static_cast<int8_t>(ctrl_t::kDeleted) &
static_cast<int8_t>(ctrl_t::kSentinel) & 0x7F) != 0,
"ctrl_t::kEmpty and ctrl_t::kDeleted must share an unset bit that is not "
"shared by ctrl_t::kSentinel to make the scalar test for "
"MaskEmptyOrDeleted() efficient");
static_assert(ctrl_t::kDeleted == static_cast<ctrl_t>(-2),
"ctrl_t::kDeleted must be -2 to make the implementation of "
"ConvertSpecialToEmptyAndFullToDeleted efficient");
// See definition comment for why this is size 32.
ABSL_DLL extern const ctrl_t kEmptyGroup[32];
// Returns a pointer to a control byte group that can be used by empty tables.
inline ctrl_t* EmptyGroup() {
// Const must be cast away here; no uses of this function will actually write
// to it, because it is only used for empty tables.
return const_cast<ctrl_t*>(kEmptyGroup + 16);
}
// Returns a pointer to a generation to use for an empty hashtable.
GenerationType* EmptyGeneration();
// Returns whether `generation` is a generation for an empty hashtable that
// could be returned by EmptyGeneration().
inline bool IsEmptyGeneration(const GenerationType* generation) {
return *generation == SentinelEmptyGeneration();
}
// Mixes a randomly generated per-process seed with `hash` and `ctrl` to
// randomize insertion order within groups.
bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl);
// Returns a per-table, hash salt, which changes on resize. This gets mixed into
// H1 to randomize iteration order per-table.
//
// The seed consists of the ctrl_ pointer, which adds enough entropy to ensure
// non-determinism of iteration order in most cases.
inline size_t PerTableSalt(const ctrl_t* ctrl) {
// The low bits of the pointer have little or no entropy because of
// alignment. We shift the pointer to try to use higher entropy bits. A
// good number seems to be 12 bits, because that aligns with page size.
return reinterpret_cast<uintptr_t>(ctrl) >> 12;
}
// Extracts the H1 portion of a hash: 57 bits mixed with a per-table salt.
inline size_t H1(size_t hash, const ctrl_t* ctrl) {
return (hash >> 7) ^ PerTableSalt(ctrl);
}
// Extracts the H2 portion of a hash: the 7 bits not used for H1.
//
// These are used as an occupied control byte.
inline h2_t H2(size_t hash) { return hash & 0x7F; }
// Helpers for checking the state of a control byte.
inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; }
inline bool IsFull(ctrl_t c) { return c >= static_cast<ctrl_t>(0); }
inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; }
inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; }
#ifdef ABSL_INTERNAL_HAVE_SSE2
// Quick reference guide for intrinsics used below:
//
// * __m128i: An XMM (128-bit) word.
//
// * _mm_setzero_si128: Returns a zero vector.
// * _mm_set1_epi8: Returns a vector with the same i8 in each lane.
//
// * _mm_subs_epi8: Saturating-subtracts two i8 vectors.
// * _mm_and_si128: Ands two i128s together.
// * _mm_or_si128: Ors two i128s together.
// * _mm_andnot_si128: And-nots two i128s together.
//
// * _mm_cmpeq_epi8: Component-wise compares two i8 vectors for equality,
// filling each lane with 0x00 or 0xff.
// * _mm_cmpgt_epi8: Same as above, but using > rather than ==.
//
// * _mm_loadu_si128: Performs an unaligned load of an i128.
// * _mm_storeu_si128: Performs an unaligned store of an i128.
//
// * _mm_sign_epi8: Retains, negates, or zeroes each i8 lane of the first
// argument if the corresponding lane of the second
// argument is positive, negative, or zero, respectively.
// * _mm_movemask_epi8: Selects the sign bit out of each i8 lane and produces a
// bitmask consisting of those bits.
// * _mm_shuffle_epi8: Selects i8s from the first argument, using the low
// four bits of each i8 lane in the second argument as
// indices.
// https://github.com/abseil/abseil-cpp/issues/209
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853
// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char
// Work around this by using the portable implementation of Group
// when using -funsigned-char under GCC.
inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) {
#if defined(__GNUC__) && !defined(__clang__)
if (std::is_unsigned<char>::value) {
const __m128i mask = _mm_set1_epi8(0x80);
const __m128i diff = _mm_subs_epi8(b, a);
return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask);
}
#endif
return _mm_cmpgt_epi8(a, b);
}
struct GroupSse2Impl {
static constexpr size_t kWidth = 16; // the number of slots per group
explicit GroupSse2Impl(const ctrl_t* pos) {
ctrl = _mm_loadu_si128(reinterpret_cast<const __m128i*>(pos));
}
// Returns a bitmask representing the positions of slots that match hash.
BitMask<uint16_t, kWidth> Match(h2_t hash) const {
auto match = _mm_set1_epi8(static_cast<char>(hash));
BitMask<uint16_t, kWidth> result = BitMask<uint16_t, kWidth>(0);
result = BitMask<uint16_t, kWidth>(
static_cast<uint16_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))));
return result;
}
// Returns a bitmask representing the positions of empty slots.
NonIterableBitMask<uint16_t, kWidth> MaskEmpty() const {
#ifdef ABSL_INTERNAL_HAVE_SSSE3
// This only works because ctrl_t::kEmpty is -128.
return NonIterableBitMask<uint16_t, kWidth>(
static_cast<uint16_t>(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))));
#else
auto match = _mm_set1_epi8(static_cast<char>(ctrl_t::kEmpty));
return NonIterableBitMask<uint16_t, kWidth>(
static_cast<uint16_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))));
#endif
}
// Returns a bitmask representing the positions of full slots.
// Note: for `is_small()` tables group may contain the "same" slot twice:
// original and mirrored.
BitMask<uint16_t, kWidth> MaskFull() const {
return BitMask<uint16_t, kWidth>(
static_cast<uint16_t>(_mm_movemask_epi8(ctrl) ^ 0xffff));
}
// Returns a bitmask representing the positions of empty or deleted slots.
NonIterableBitMask<uint16_t, kWidth> MaskEmptyOrDeleted() const {
auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel));
return NonIterableBitMask<uint16_t, kWidth>(static_cast<uint16_t>(
_mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))));
}
// Returns the number of trailing empty or deleted elements in the group.
uint32_t CountLeadingEmptyOrDeleted() const {
auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel));
return TrailingZeros(static_cast<uint32_t>(
_mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1));
}
void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
auto msbs = _mm_set1_epi8(static_cast<char>(-128));
auto x126 = _mm_set1_epi8(126);
#ifdef ABSL_INTERNAL_HAVE_SSSE3
auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs);
#else
auto zero = _mm_setzero_si128();
auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl);
auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126));
#endif
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res);
}
__m128i ctrl;
};
#endif // ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2
#if defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN)
struct GroupAArch64Impl {
static constexpr size_t kWidth = 8;
explicit GroupAArch64Impl(const ctrl_t* pos) {
ctrl = vld1_u8(reinterpret_cast<const uint8_t*>(pos));
}
BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const {
uint8x8_t dup = vdup_n_u8(hash);
auto mask = vceq_u8(ctrl, dup);
return BitMask<uint64_t, kWidth, 3>(
vget_lane_u64(vreinterpret_u64_u8(mask), 0));
}
NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const {
uint64_t mask =
vget_lane_u64(vreinterpret_u64_u8(vceq_s8(
vdup_n_s8(static_cast<int8_t>(ctrl_t::kEmpty)),
vreinterpret_s8_u8(ctrl))),
0);
return NonIterableBitMask<uint64_t, kWidth, 3>(mask);
}
// Returns a bitmask representing the positions of full slots.
// Note: for `is_small()` tables group may contain the "same" slot twice:
// original and mirrored.
BitMask<uint64_t, kWidth, 3> MaskFull() const {
uint64_t mask = vget_lane_u64(
vreinterpret_u64_u8(vcge_s8(vreinterpret_s8_u8(ctrl),
vdup_n_s8(static_cast<int8_t>(0)))),
0);
return BitMask<uint64_t, kWidth, 3>(mask);
}
NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const {
uint64_t mask =
vget_lane_u64(vreinterpret_u64_u8(vcgt_s8(
vdup_n_s8(static_cast<int8_t>(ctrl_t::kSentinel)),
vreinterpret_s8_u8(ctrl))),
0);
return NonIterableBitMask<uint64_t, kWidth, 3>(mask);
}
uint32_t CountLeadingEmptyOrDeleted() const {
uint64_t mask =
vget_lane_u64(vreinterpret_u64_u8(vcle_s8(
vdup_n_s8(static_cast<int8_t>(ctrl_t::kSentinel)),
vreinterpret_s8_u8(ctrl))),
0);
// Similar to MaskEmptyorDeleted() but we invert the logic to invert the
// produced bitfield. We then count number of trailing zeros.
// Clang and GCC optimize countr_zero to rbit+clz without any check for 0,
// so we should be fine.
return static_cast<uint32_t>(countr_zero(mask)) >> 3;
}
void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0);
constexpr uint64_t msbs = 0x8080808080808080ULL;
constexpr uint64_t slsbs = 0x0202020202020202ULL;
constexpr uint64_t midbs = 0x7e7e7e7e7e7e7e7eULL;
auto x = slsbs & (mask >> 6);
auto res = (x + midbs) | msbs;
little_endian::Store64(dst, res);
}
uint8x8_t ctrl;
};
#endif // ABSL_INTERNAL_HAVE_ARM_NEON && ABSL_IS_LITTLE_ENDIAN
struct GroupPortableImpl {
static constexpr size_t kWidth = 8;
explicit GroupPortableImpl(const ctrl_t* pos)
: ctrl(little_endian::Load64(pos)) {}
BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const {
// For the technique, see:
// http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord
// (Determine if a word has a byte equal to n).
//
// Caveat: there are false positives but:
// - they only occur if there is a real match
// - they never occur on ctrl_t::kEmpty, ctrl_t::kDeleted, ctrl_t::kSentinel
// - they will be handled gracefully by subsequent checks in code
//
// Example:
// v = 0x1716151413121110
// hash = 0x12
// retval = (v - lsbs) & ~v & msbs = 0x0000000080800000
constexpr uint64_t msbs = 0x8080808080808080ULL;
constexpr uint64_t lsbs = 0x0101010101010101ULL;
auto x = ctrl ^ (lsbs * hash);
return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs);
}
NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const {
constexpr uint64_t msbs = 0x8080808080808080ULL;
return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 6)) &
msbs);
}
// Returns a bitmask representing the positions of full slots.
// Note: for `is_small()` tables group may contain the "same" slot twice:
// original and mirrored.
BitMask<uint64_t, kWidth, 3> MaskFull() const {
constexpr uint64_t msbs = 0x8080808080808080ULL;
return BitMask<uint64_t, kWidth, 3>((ctrl ^ msbs) & msbs);
}
NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const {
constexpr uint64_t msbs = 0x8080808080808080ULL;
return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 7)) &
msbs);
}
uint32_t CountLeadingEmptyOrDeleted() const {
// ctrl | ~(ctrl >> 7) will have the lowest bit set to zero for kEmpty and
// kDeleted. We lower all other bits and count number of trailing zeros.
constexpr uint64_t bits = 0x0101010101010101ULL;
return static_cast<uint32_t>(countr_zero((ctrl | ~(ctrl >> 7)) & bits) >>
3);
}
void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
constexpr uint64_t msbs = 0x8080808080808080ULL;
constexpr uint64_t lsbs = 0x0101010101010101ULL;
auto x = ctrl & msbs;
auto res = (~x + (x >> 7)) & ~lsbs;
little_endian::Store64(dst, res);
}
uint64_t ctrl;
};
#ifdef ABSL_INTERNAL_HAVE_SSE2
using Group = GroupSse2Impl;
using GroupEmptyOrDeleted = GroupSse2Impl;
#elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN)
using Group = GroupAArch64Impl;
// For Aarch64, we use the portable implementation for counting and masking
// empty or deleted group elements. This is to avoid the latency of moving
// between data GPRs and Neon registers when it does not provide a benefit.
// Using Neon is profitable when we call Match(), but is not when we don't,
// which is the case when we do *EmptyOrDeleted operations. It is difficult to
// make a similar approach beneficial on other architectures such as x86 since
// they have much lower GPR <-> vector register transfer latency and 16-wide
// Groups.
using GroupEmptyOrDeleted = GroupPortableImpl;
#else
using Group = GroupPortableImpl;
using GroupEmptyOrDeleted = GroupPortableImpl;
#endif
// When there is an insertion with no reserved growth, we rehash with
// probability `min(1, RehashProbabilityConstant() / capacity())`. Using a
// constant divided by capacity ensures that inserting N elements is still O(N)
// in the average case. Using the constant 16 means that we expect to rehash ~8
// times more often than when generations are disabled. We are adding expected
// rehash_probability * #insertions/capacity_growth = 16/capacity * ((7/8 -
// 7/16) * capacity)/capacity_growth = ~7 extra rehashes per capacity growth.
inline size_t RehashProbabilityConstant() { return 16; }
class CommonFieldsGenerationInfoEnabled {
// A sentinel value for reserved_growth_ indicating that we just ran out of
// reserved growth on the last insertion. When reserve is called and then
// insertions take place, reserved_growth_'s state machine is N, ..., 1,
// kReservedGrowthJustRanOut, 0.
static constexpr size_t kReservedGrowthJustRanOut =
(std::numeric_limits<size_t>::max)();
public:
CommonFieldsGenerationInfoEnabled() = default;
CommonFieldsGenerationInfoEnabled(CommonFieldsGenerationInfoEnabled&& that)
: reserved_growth_(that.reserved_growth_),
reservation_size_(that.reservation_size_),
generation_(that.generation_) {
that.reserved_growth_ = 0;
that.reservation_size_ = 0;
that.generation_ = EmptyGeneration();
}
CommonFieldsGenerationInfoEnabled& operator=(
CommonFieldsGenerationInfoEnabled&&) = default;
// Whether we should rehash on insert in order to detect bugs of using invalid
// references. We rehash on the first insertion after reserved_growth_ reaches
// 0 after a call to reserve. We also do a rehash with low probability
// whenever reserved_growth_ is zero.
bool should_rehash_for_bug_detection_on_insert(const ctrl_t* ctrl,
size_t capacity) const;
// Similar to above, except that we don't depend on reserved_growth_.
bool should_rehash_for_bug_detection_on_move(const ctrl_t* ctrl,
size_t capacity) const;
void maybe_increment_generation_on_insert() {
if (reserved_growth_ == kReservedGrowthJustRanOut) reserved_growth_ = 0;
if (reserved_growth_ > 0) {
if (--reserved_growth_ == 0) reserved_growth_ = kReservedGrowthJustRanOut;
} else {
increment_generation();
}
}
void increment_generation() { *generation_ = NextGeneration(*generation_); }
void reset_reserved_growth(size_t reservation, size_t size) {
reserved_growth_ = reservation - size;
}
size_t reserved_growth() const { return reserved_growth_; }
void set_reserved_growth(size_t r) { reserved_growth_ = r; }
size_t reservation_size() const { return reservation_size_; }
void set_reservation_size(size_t r) { reservation_size_ = r; }
GenerationType generation() const { return *generation_; }
void set_generation(GenerationType g) { *generation_ = g; }
GenerationType* generation_ptr() const { return generation_; }
void set_generation_ptr(GenerationType* g) { generation_ = g; }
private:
// The number of insertions remaining that are guaranteed to not rehash due to
// a prior call to reserve. Note: we store reserved growth in addition to
// reservation size because calls to erase() decrease size_ but don't decrease
// reserved growth.
size_t reserved_growth_ = 0;
// The maximum argument to reserve() since the container was cleared. We need
// to keep track of this, in addition to reserved growth, because we reset
// reserved growth to this when erase(begin(), end()) is called.
size_t reservation_size_ = 0;
// Pointer to the generation counter, which is used to validate iterators and
// is stored in the backing array between the control bytes and the slots.
// Note that we can't store the generation inside the container itself and
// keep a pointer to the container in the iterators because iterators must
// remain valid when the container is moved.
// Note: we could derive this pointer from the control pointer, but it makes
// the code more complicated, and there's a benefit in having the sizes of
// raw_hash_set in sanitizer mode and non-sanitizer mode a bit more different,
// which is that tests are less likely to rely on the size remaining the same.
GenerationType* generation_ = EmptyGeneration();
};
class CommonFieldsGenerationInfoDisabled {
public:
CommonFieldsGenerationInfoDisabled() = default;
CommonFieldsGenerationInfoDisabled(CommonFieldsGenerationInfoDisabled&&) =
default;
CommonFieldsGenerationInfoDisabled& operator=(
CommonFieldsGenerationInfoDisabled&&) = default;
bool should_rehash_for_bug_detection_on_insert(const ctrl_t*, size_t) const {
return false;
}
bool should_rehash_for_bug_detection_on_move(const ctrl_t*, size_t) const {
return false;
}
void maybe_increment_generation_on_insert() {}
void increment_generation() {}
void reset_reserved_growth(size_t, size_t) {}
size_t reserved_growth() const { return 0; }
void set_reserved_growth(size_t) {}
size_t reservation_size() const { return 0; }
void set_reservation_size(size_t) {}
GenerationType generation() const { return 0; }
void set_generation(GenerationType) {}
GenerationType* generation_ptr() const { return nullptr; }
void set_generation_ptr(GenerationType*) {}
};
class HashSetIteratorGenerationInfoEnabled {
public:
HashSetIteratorGenerationInfoEnabled() = default;
explicit HashSetIteratorGenerationInfoEnabled(
const GenerationType* generation_ptr)
: generation_ptr_(generation_ptr), generation_(*generation_ptr) {}
GenerationType generation() const { return generation_; }
void reset_generation() { generation_ = *generation_ptr_; }
const GenerationType* generation_ptr() const { return generation_ptr_; }
void set_generation_ptr(const GenerationType* ptr) { generation_ptr_ = ptr; }
private:
const GenerationType* generation_ptr_ = EmptyGeneration();
GenerationType generation_ = *generation_ptr_;
};
class HashSetIteratorGenerationInfoDisabled {
public:
HashSetIteratorGenerationInfoDisabled() = default;
explicit HashSetIteratorGenerationInfoDisabled(const GenerationType*) {}
GenerationType generation() const { return 0; }
void reset_generation() {}
const GenerationType* generation_ptr() const { return nullptr; }
void set_generation_ptr(const GenerationType*) {}
};
#ifdef ABSL_SWISSTABLE_ENABLE_GENERATIONS
using CommonFieldsGenerationInfo = CommonFieldsGenerationInfoEnabled;
using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoEnabled;
#else
using CommonFieldsGenerationInfo = CommonFieldsGenerationInfoDisabled;
using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled;
#endif
// Returns whether `n` is a valid capacity (i.e., number of slots).
//
// A valid capacity is a non-zero integer `2^m - 1`.
inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; }
// Computes the offset from the start of the backing allocation of control.
// infoz and growth_left are stored at the beginning of the backing array.
inline size_t ControlOffset(bool has_infoz) {
return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(size_t);
}
// Returns the number of "cloned control bytes".
//
// This is the number of control bytes that are present both at the beginning
// of the control byte array and at the end, such that we can create a
// `Group::kWidth`-width probe window starting from any control byte.
constexpr size_t NumClonedBytes() { return Group::kWidth - 1; }
// Given the capacity of a table, computes the offset (from the start of the
// backing allocation) of the generation counter (if it exists).
inline size_t GenerationOffset(size_t capacity, bool has_infoz) {
assert(IsValidCapacity(capacity));
const size_t num_control_bytes = capacity + 1 + NumClonedBytes();
return ControlOffset(has_infoz) + num_control_bytes;
}
// Given the capacity of a table, computes the offset (from the start of the
// backing allocation) at which the slots begin.
inline size_t SlotOffset(size_t capacity, size_t slot_align, bool has_infoz) {
assert(IsValidCapacity(capacity));
return (GenerationOffset(capacity, has_infoz) + NumGenerationBytes() +
slot_align - 1) &
(~slot_align + 1);
}
// Given the capacity of a table, computes the total size of the backing
// array.
inline size_t AllocSize(size_t capacity, size_t slot_size, size_t slot_align,
bool has_infoz) {
return SlotOffset(capacity, slot_align, has_infoz) + capacity * slot_size;
}
// CommonFields hold the fields in raw_hash_set that do not depend
// on template parameters. This allows us to conveniently pass all
// of this state to helper functions as a single argument.
class CommonFields : public CommonFieldsGenerationInfo {
public:
CommonFields() = default;
// Not copyable
CommonFields(const CommonFields&) = delete;
CommonFields& operator=(const CommonFields&) = delete;
// Movable
CommonFields(CommonFields&& that) = default;
CommonFields& operator=(CommonFields&&) = default;
ctrl_t* control() const { return control_; }
void set_control(ctrl_t* c) { control_ = c; }
void* backing_array_start() const {
// growth_left (and maybe infoz) is stored before control bytes.
assert(reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0);
return control() - ControlOffset(has_infoz());
}
// Note: we can't use slots() because Qt defines "slots" as a macro.
void* slot_array() const { return slots_; }
void set_slots(void* s) { slots_ = s; }
// The number of filled slots.
size_t size() const { return size_ >> HasInfozShift(); }
void set_size(size_t s) {
size_ = (s << HasInfozShift()) | (size_ & HasInfozMask());
}
void increment_size() {
assert(size() < capacity());
size_ += size_t{1} << HasInfozShift();
}
void decrement_size() {
assert(size() > 0);
size_ -= size_t{1} << HasInfozShift();
}
// The total number of available slots.
size_t capacity() const { return capacity_; }
void set_capacity(size_t c) {
assert(c == 0 || IsValidCapacity(c));
capacity_ = c;
}
// The number of slots we can still fill without needing to rehash.
// This is stored in the heap allocation before the control bytes.
size_t growth_left() const {
const size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1;
assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0);
return *gl_ptr;
}
void set_growth_left(size_t gl) {
size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1;
assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0);
*gl_ptr = gl;
}
bool has_infoz() const {
return ABSL_PREDICT_FALSE((size_ & HasInfozMask()) != 0);
}
void set_has_infoz(bool has_infoz) {
size_ = (size() << HasInfozShift()) | static_cast<size_t>(has_infoz);
}
HashtablezInfoHandle infoz() {
return has_infoz()
? *reinterpret_cast<HashtablezInfoHandle*>(backing_array_start())
: HashtablezInfoHandle();
}
void set_infoz(HashtablezInfoHandle infoz) {
assert(has_infoz());
*reinterpret_cast<HashtablezInfoHandle*>(backing_array_start()) = infoz;
}
bool should_rehash_for_bug_detection_on_insert() const {
return CommonFieldsGenerationInfo::
should_rehash_for_bug_detection_on_insert(control(), capacity());
}
bool should_rehash_for_bug_detection_on_move() const {
return CommonFieldsGenerationInfo::
should_rehash_for_bug_detection_on_move(control(), capacity());
}
void maybe_increment_generation_on_move() {
if (capacity() == 0) return;
increment_generation();
}
void reset_reserved_growth(size_t reservation) {
CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size());
}
// The size of the backing array allocation.
size_t alloc_size(size_t slot_size, size_t slot_align) const {
return AllocSize(capacity(), slot_size, slot_align, has_infoz());
}
// Returns the number of control bytes set to kDeleted. For testing only.
size_t TombstonesCount() const {
return static_cast<size_t>(
std::count(control(), control() + capacity(), ctrl_t::kDeleted));
}
private:
// We store the has_infoz bit in the lowest bit of size_.
static constexpr size_t HasInfozShift() { return 1; }
static constexpr size_t HasInfozMask() {
return (size_t{1} << HasInfozShift()) - 1;
}
// TODO(b/182800944): Investigate removing some of these fields:
// - control/slots can be derived from each other
// The control bytes (and, also, a pointer near to the base of the backing
// array).
//
// This contains `capacity + 1 + NumClonedBytes()` entries, even
// when the table is empty (hence EmptyGroup).
//
// Note that growth_left is stored immediately before this pointer.
ctrl_t* control_ = EmptyGroup();
// The beginning of the slots, located at `SlotOffset()` bytes after
// `control`. May be null for empty tables.
void* slots_ = nullptr;
// The number of slots in the backing array. This is always 2^N-1 for an
// integer N. NOTE: we tried experimenting with compressing the capacity and
// storing it together with size_: (a) using 6 bits to store the corresponding
// power (N in 2^N-1), and (b) storing 2^N as the most significant bit of
// size_ and storing size in the low bits. Both of these experiments were
// regressions, presumably because we need capacity to do find operations.
size_t capacity_ = 0;
// The size and also has one bit that stores whether we have infoz.
size_t size_ = 0;
};
template <class Policy, class Hash, class Eq, class Alloc>
class raw_hash_set;
// Returns the next valid capacity after `n`.
inline size_t NextCapacity(size_t n) {
assert(IsValidCapacity(n) || n == 0);
return n * 2 + 1;
}
// Applies the following mapping to every byte in the control array:
// * kDeleted -> kEmpty
// * kEmpty -> kEmpty
// * _ -> kDeleted
// PRECONDITION:
// IsValidCapacity(capacity)
// ctrl[capacity] == ctrl_t::kSentinel
// ctrl[i] != ctrl_t::kSentinel for all i < capacity
void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity);
// Converts `n` into the next valid capacity, per `IsValidCapacity`.
inline size_t NormalizeCapacity(size_t n) {
return n ? ~size_t{} >> countl_zero(n) : 1;
}
// General notes on capacity/growth methods below:
// - We use 7/8th as maximum load factor. For 16-wide groups, that gives an
// average of two empty slots per group.
// - For (capacity+1) >= Group::kWidth, growth is 7/8*capacity.
// - For (capacity+1) < Group::kWidth, growth == capacity. In this case, we
// never need to probe (the whole table fits in one group) so we don't need a
// load factor less than 1.
// Given `capacity`, applies the load factor; i.e., it returns the maximum
// number of values we should put into the table before a resizing rehash.
inline size_t CapacityToGrowth(size_t capacity) {
assert(IsValidCapacity(capacity));
// `capacity*7/8`
if (Group::kWidth == 8 && capacity == 7) {
// x-x/8 does not work when x==7.
return 6;
}
return capacity - capacity / 8;
}
// Given `growth`, "unapplies" the load factor to find how large the capacity
// should be to stay within the load factor.
//
// This might not be a valid capacity and `NormalizeCapacity()` should be
// called on this.
inline size_t GrowthToLowerboundCapacity(size_t growth) {
// `growth*8/7`
if (Group::kWidth == 8 && growth == 7) {
// x+(x-1)/7 does not work when x==7.
return 8;
}
return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7);
}
template <class InputIter>
size_t SelectBucketCountForIterRange(InputIter first, InputIter last,
size_t bucket_count) {
if (bucket_count != 0) {
return bucket_count;
}
using InputIterCategory =
typename std::iterator_traits<InputIter>::iterator_category;
if (std::is_base_of<std::random_access_iterator_tag,
InputIterCategory>::value) {
return GrowthToLowerboundCapacity(
static_cast<size_t>(std::distance(first, last)));
}
return 0;
}
constexpr bool SwisstableDebugEnabled() {
#if defined(ABSL_SWISSTABLE_ENABLE_GENERATIONS) || \
ABSL_OPTION_HARDENED == 1 || !defined(NDEBUG)
return true;
#else
return false;
#endif
}
inline void AssertIsFull(const ctrl_t* ctrl, GenerationType generation,
const GenerationType* generation_ptr,
const char* operation) {
if (!SwisstableDebugEnabled()) return;
// `SwisstableDebugEnabled()` is also true for release builds with hardening
// enabled. To minimize their impact in those builds:
// - use `ABSL_PREDICT_FALSE()` to provide a compiler hint for code layout
// - use `ABSL_RAW_LOG()` with a format string to reduce code size and improve
// the chances that the hot paths will be inlined.
if (ABSL_PREDICT_FALSE(ctrl == nullptr)) {
ABSL_RAW_LOG(FATAL, "%s called on end() iterator.", operation);
}
if (ABSL_PREDICT_FALSE(ctrl == EmptyGroup())) {
ABSL_RAW_LOG(FATAL, "%s called on default-constructed iterator.",
operation);
}
if (SwisstableGenerationsEnabled()) {
if (ABSL_PREDICT_FALSE(generation != *generation_ptr)) {
ABSL_RAW_LOG(FATAL,
"%s called on invalid iterator. The table could have "
"rehashed or moved since this iterator was initialized.",
operation);
}
if (ABSL_PREDICT_FALSE(!IsFull(*ctrl))) {
ABSL_RAW_LOG(
FATAL,
"%s called on invalid iterator. The element was likely erased.",
operation);
}
} else {
if (ABSL_PREDICT_FALSE(!IsFull(*ctrl))) {
ABSL_RAW_LOG(
FATAL,
"%s called on invalid iterator. The element might have been erased "
"or the table might have rehashed. Consider running with "
"--config=asan to diagnose rehashing issues.",
operation);
}
}
}
// Note that for comparisons, null/end iterators are valid.
inline void AssertIsValidForComparison(const ctrl_t* ctrl,
GenerationType generation,
const GenerationType* generation_ptr) {
if (!SwisstableDebugEnabled()) return;
const bool ctrl_is_valid_for_comparison =
ctrl == nullptr || ctrl == EmptyGroup() || IsFull(*ctrl);
if (SwisstableGenerationsEnabled()) {
if (ABSL_PREDICT_FALSE(generation != *generation_ptr)) {
ABSL_RAW_LOG(FATAL,
"Invalid iterator comparison. The table could have rehashed "
"or moved since this iterator was initialized.");
}
if (ABSL_PREDICT_FALSE(!ctrl_is_valid_for_comparison)) {
ABSL_RAW_LOG(
FATAL, "Invalid iterator comparison. The element was likely erased.");
}
} else {
ABSL_HARDENING_ASSERT(
ctrl_is_valid_for_comparison &&
"Invalid iterator comparison. The element might have been erased or "
"the table might have rehashed. Consider running with --config=asan to "
"diagnose rehashing issues.");
}
}
// If the two iterators come from the same container, then their pointers will
// interleave such that ctrl_a <= ctrl_b < slot_a <= slot_b or vice/versa.
// Note: we take slots by reference so that it's not UB if they're uninitialized
// as long as we don't read them (when ctrl is null).
inline bool AreItersFromSameContainer(const ctrl_t* ctrl_a,
const ctrl_t* ctrl_b,
const void* const& slot_a,
const void* const& slot_b) {
// If either control byte is null, then we can't tell.
if (ctrl_a == nullptr || ctrl_b == nullptr) return true;
const void* low_slot = slot_a;
const void* hi_slot = slot_b;
if (ctrl_a > ctrl_b) {
std::swap(ctrl_a, ctrl_b);
std::swap(low_slot, hi_slot);
}
return ctrl_b < low_slot && low_slot <= hi_slot;
}
// Asserts that two iterators come from the same container.
// Note: we take slots by reference so that it's not UB if they're uninitialized
// as long as we don't read them (when ctrl is null).
inline void AssertSameContainer(const ctrl_t* ctrl_a, const ctrl_t* ctrl_b,
const void* const& slot_a,
const void* const& slot_b,
const GenerationType* generation_ptr_a,
const GenerationType* generation_ptr_b) {
if (!SwisstableDebugEnabled()) return;
// `SwisstableDebugEnabled()` is also true for release builds with hardening
// enabled. To minimize their impact in those builds:
// - use `ABSL_PREDICT_FALSE()` to provide a compiler hint for code layout
// - use `ABSL_RAW_LOG()` with a format string to reduce code size and improve
// the chances that the hot paths will be inlined.
const bool a_is_default = ctrl_a == EmptyGroup();
const bool b_is_default = ctrl_b == EmptyGroup();
if (ABSL_PREDICT_FALSE(a_is_default != b_is_default)) {
ABSL_RAW_LOG(
FATAL,
"Invalid iterator comparison. Comparing default-constructed iterator "
"with non-default-constructed iterator.");
}
if (a_is_default && b_is_default) return;
if (SwisstableGenerationsEnabled()) {
if (ABSL_PREDICT_TRUE(generation_ptr_a == generation_ptr_b)) return;
const bool a_is_empty = IsEmptyGeneration(generation_ptr_a);
const bool b_is_empty = IsEmptyGeneration(generation_ptr_b);
if (a_is_empty != b_is_empty) {
ABSL_RAW_LOG(FATAL,
"Invalid iterator comparison. Comparing iterator from a "
"non-empty hashtable with an iterator from an empty "
"hashtable.");
}
if (a_is_empty && b_is_empty) {
ABSL_RAW_LOG(FATAL,
"Invalid iterator comparison. Comparing iterators from "
"different empty hashtables.");
}
const bool a_is_end = ctrl_a == nullptr;
const bool b_is_end = ctrl_b == nullptr;
if (a_is_end || b_is_end) {
ABSL_RAW_LOG(FATAL,
"Invalid iterator comparison. Comparing iterator with an "
"end() iterator from a different hashtable.");
}
ABSL_RAW_LOG(FATAL,
"Invalid iterator comparison. Comparing non-end() iterators "
"from different hashtables.");
} else {
ABSL_HARDENING_ASSERT(
AreItersFromSameContainer(ctrl_a, ctrl_b, slot_a, slot_b) &&
"Invalid iterator comparison. The iterators may be from different "
"containers or the container might have rehashed or moved. Consider "
"running with --config=asan to diagnose issues.");
}
}
struct FindInfo {
size_t offset;
size_t probe_length;
};
// Whether a table is "small". A small table fits entirely into a probing
// group, i.e., has a capacity < `Group::kWidth`.
//
// In small mode we are able to use the whole capacity. The extra control
// bytes give us at least one "empty" control byte to stop the iteration.
// This is important to make 1 a valid capacity.
//
// In small mode only the first `capacity` control bytes after the sentinel
// are valid. The rest contain dummy ctrl_t::kEmpty values that do not
// represent a real slot. This is important to take into account on
// `find_first_non_full()`, where we never try
// `ShouldInsertBackwards()` for small tables.
inline bool is_small(size_t capacity) { return capacity < Group::kWidth - 1; }
// Whether a table fits entirely into a probing group.
// Arbitrary order of elements in such tables is correct.
inline bool is_single_group(size_t capacity) {
return capacity <= Group::kWidth;
}
// Begins a probing operation on `common.control`, using `hash`.
inline probe_seq<Group::kWidth> probe(const ctrl_t* ctrl, const size_t capacity,
size_t hash) {
return probe_seq<Group::kWidth>(H1(hash, ctrl), capacity);
}
inline probe_seq<Group::kWidth> probe(const CommonFields& common, size_t hash) {
return probe(common.control(), common.capacity(), hash);
}
// Probes an array of control bits using a probe sequence derived from `hash`,
// and returns the offset corresponding to the first deleted or empty slot.
//
// Behavior when the entire table is full is undefined.
//
// NOTE: this function must work with tables having both empty and deleted
// slots in the same group. Such tables appear during `erase()`.
template <typename = void>
inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) {
auto seq = probe(common, hash);
const ctrl_t* ctrl = common.control();
while (true) {
GroupEmptyOrDeleted g{ctrl + seq.offset()};
auto mask = g.MaskEmptyOrDeleted();
if (mask) {
#if !defined(NDEBUG)
// We want to add entropy even when ASLR is not enabled.
// In debug build we will randomly insert in either the front or back of
// the group.
// TODO(kfm,sbenza): revisit after we do unconditional mixing
if (!is_small(common.capacity()) && ShouldInsertBackwards(hash, ctrl)) {
return {seq.offset(mask.HighestBitSet()), seq.index()};
}
#endif
return {seq.offset(mask.LowestBitSet()), seq.index()};
}
seq.next();
assert(seq.index() <= common.capacity() && "full table!");
}
}
// Extern template for inline function keep possibility of inlining.
// When compiler decided to not inline, no symbols will be added to the
// corresponding translation unit.
extern template FindInfo find_first_non_full(const CommonFields&, size_t);
// Non-inlined version of find_first_non_full for use in less
// performance critical routines.
FindInfo find_first_non_full_outofline(const CommonFields&, size_t);
inline void ResetGrowthLeft(CommonFields& common) {
common.set_growth_left(CapacityToGrowth(common.capacity()) - common.size());
}
// Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire
// array as marked as empty.
inline void ResetCtrl(CommonFields& common, size_t slot_size) {
const size_t capacity = common.capacity();
ctrl_t* ctrl = common.control();
std::memset(ctrl, static_cast<int8_t>(ctrl_t::kEmpty),
capacity + 1 + NumClonedBytes());
ctrl[capacity] = ctrl_t::kSentinel;
SanitizerPoisonMemoryRegion(common.slot_array(), slot_size * capacity);
}
// Sets `ctrl[i]` to `h`.
//
// Unlike setting it directly, this function will perform bounds checks and
// mirror the value to the cloned tail if necessary.
inline void SetCtrl(const CommonFields& common, size_t i, ctrl_t h,
size_t slot_size) {
const size_t capacity = common.capacity();
assert(i < capacity);
auto* slot_i = static_cast<const char*>(common.slot_array()) + i * slot_size;
if (IsFull(h)) {
SanitizerUnpoisonMemoryRegion(slot_i, slot_size);
} else {
SanitizerPoisonMemoryRegion(slot_i, slot_size);
}
ctrl_t* ctrl = common.control();
ctrl[i] = h;
ctrl[((i - NumClonedBytes()) & capacity) + (NumClonedBytes() & capacity)] = h;
}
// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`.
inline void SetCtrl(const CommonFields& common, size_t i, h2_t h,
size_t slot_size) {
SetCtrl(common, i, static_cast<ctrl_t>(h), slot_size);
}
// growth_left (which is a size_t) is stored with the backing array.
constexpr size_t BackingArrayAlignment(size_t align_of_slot) {
return (std::max)(align_of_slot, alignof(size_t));
}
// Returns the address of the ith slot in slots where each slot occupies
// slot_size.
inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) {
return reinterpret_cast<void*>(reinterpret_cast<char*>(slot_array) +
(slot * slot_size));
}
// Helper class to perform resize of the hash set.
//
// It contains special optimizations for small group resizes.
// See GrowIntoSingleGroupShuffleControlBytes for details.
class HashSetResizeHelper {
public:
explicit HashSetResizeHelper(CommonFields& c)
: old_ctrl_(c.control()),
old_capacity_(c.capacity()),
had_infoz_(c.has_infoz()) {}
// Optimized for small groups version of `find_first_non_full` applicable
// only right after calling `raw_hash_set::resize`.
// It has implicit assumption that `resize` will call
// `GrowSizeIntoSingleGroup*` in case `IsGrowingIntoSingleGroupApplicable`.
// Falls back to `find_first_non_full` in case of big groups, so it is
// safe to use after `rehash_and_grow_if_necessary`.
static FindInfo FindFirstNonFullAfterResize(const CommonFields& c,
size_t old_capacity,
size_t hash) {
if (!IsGrowingIntoSingleGroupApplicable(old_capacity, c.capacity())) {
return find_first_non_full(c, hash);
}
// Find a location for the new element non-deterministically.
// Note that any position is correct.
// It will located at `half_old_capacity` or one of the other
// empty slots with approximately 50% probability each.
size_t offset = probe(c, hash).offset();
// Note that we intentionally use unsigned int underflow.
if (offset - (old_capacity + 1) >= old_capacity) {
// Offset fall on kSentinel or into the mostly occupied first half.
offset = old_capacity / 2;
}
assert(IsEmpty(c.control()[offset]));
return FindInfo{offset, 0};
}
ctrl_t* old_ctrl() const { return old_ctrl_; }
size_t old_capacity() const { return old_capacity_; }
// Allocates a backing array for the hashtable.
// Reads `capacity` and updates all other fields based on the result of
// the allocation.
//
// It also may do the folowing actions:
// 1. initialize control bytes
// 2. initialize slots
// 3. deallocate old slots.
//
// We are bundling a lot of functionality
// in one ABSL_ATTRIBUTE_NOINLINE function in order to minimize binary code
// duplication in raw_hash_set<>::resize.
//
// `c.capacity()` must be nonzero.
// POSTCONDITIONS:
// 1. CommonFields is initialized.
//
// if IsGrowingIntoSingleGroupApplicable && TransferUsesMemcpy
// Both control bytes and slots are fully initialized.
// old_slots are deallocated.
// infoz.RecordRehash is called.
//
// if IsGrowingIntoSingleGroupApplicable && !TransferUsesMemcpy
// Control bytes are fully initialized.
// infoz.RecordRehash is called.
// GrowSizeIntoSingleGroup must be called to finish slots initialization.
//
// if !IsGrowingIntoSingleGroupApplicable
// Control bytes are initialized to empty table via ResetCtrl.
// raw_hash_set<>::resize must insert elements regularly.
// infoz.RecordRehash is called if old_capacity == 0.
//
// Returns IsGrowingIntoSingleGroupApplicable result to avoid recomputation.
template <typename Alloc, size_t SizeOfSlot, bool TransferUsesMemcpy,
size_t AlignOfSlot>
ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, void* old_slots,
Alloc alloc) {
assert(c.capacity());
// Folks with custom allocators often make unwarranted assumptions about the
// behavior of their classes vis-a-vis trivial destructability and what
// calls they will or won't make. Avoid sampling for people with custom
// allocators to get us out of this mess. This is not a hard guarantee but
// a workaround while we plan the exact guarantee we want to provide.
const size_t sample_size =
(std::is_same<Alloc, std::allocator<char>>::value &&
c.slot_array() == nullptr)
? SizeOfSlot
: 0;
HashtablezInfoHandle infoz =
sample_size > 0 ? Sample(sample_size) : c.infoz();
const bool has_infoz = infoz.IsSampled();
const size_t cap = c.capacity();
const size_t alloc_size =
AllocSize(cap, SizeOfSlot, AlignOfSlot, has_infoz);
char* mem = static_cast<char*>(
Allocate<BackingArrayAlignment(AlignOfSlot)>(&alloc, alloc_size));
const GenerationType old_generation = c.generation();
c.set_generation_ptr(reinterpret_cast<GenerationType*>(
mem + GenerationOffset(cap, has_infoz)));
c.set_generation(NextGeneration(old_generation));
c.set_control(reinterpret_cast<ctrl_t*>(mem + ControlOffset(has_infoz)));
c.set_slots(mem + SlotOffset(cap, AlignOfSlot, has_infoz));
ResetGrowthLeft(c);
const bool grow_single_group =
IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity());
if (old_capacity_ != 0 && grow_single_group) {
if (TransferUsesMemcpy) {
GrowSizeIntoSingleGroupTransferable(c, old_slots, SizeOfSlot);
DeallocateOld<AlignOfSlot>(alloc, SizeOfSlot, old_slots);
} else {
GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity());
}
} else {
ResetCtrl(c, SizeOfSlot);
}
c.set_has_infoz(has_infoz);
if (has_infoz) {
infoz.RecordStorageChanged(c.size(), cap);
if (grow_single_group || old_capacity_ == 0) {
infoz.RecordRehash(0);
}
c.set_infoz(infoz);
}
return grow_single_group;
}
// Relocates slots into new single group consistent with
// GrowIntoSingleGroupShuffleControlBytes.
//
// PRECONDITIONS:
// 1. GrowIntoSingleGroupShuffleControlBytes was already called.
template <class PolicyTraits, class Alloc>
void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref,
typename PolicyTraits::slot_type* old_slots) {
assert(old_capacity_ < Group::kWidth / 2);
assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()));
using slot_type = typename PolicyTraits::slot_type;
assert(is_single_group(c.capacity()));
auto* new_slots = reinterpret_cast<slot_type*>(c.slot_array());
size_t shuffle_bit = old_capacity_ / 2 + 1;
for (size_t i = 0; i < old_capacity_; ++i) {
if (IsFull(old_ctrl_[i])) {
size_t new_i = i ^ shuffle_bit;
SanitizerUnpoisonMemoryRegion(new_slots + new_i, sizeof(slot_type));
PolicyTraits::transfer(&alloc_ref, new_slots + new_i, old_slots + i);
}
}
PoisonSingleGroupEmptySlots(c, sizeof(slot_type));
}
// Deallocates old backing array.
template <size_t AlignOfSlot, class CharAlloc>
void DeallocateOld(CharAlloc alloc_ref, size_t slot_size, void* old_slots) {
SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_);
Deallocate<BackingArrayAlignment(AlignOfSlot)>(
&alloc_ref, old_ctrl_ - ControlOffset(had_infoz_),
AllocSize(old_capacity_, slot_size, AlignOfSlot, had_infoz_));
}
private:
// Returns true if `GrowSizeIntoSingleGroup` can be used for resizing.
static bool IsGrowingIntoSingleGroupApplicable(size_t old_capacity,
size_t new_capacity) {
// NOTE that `old_capacity < new_capacity` in order to have
// `old_capacity < Group::kWidth / 2` to make faster copies of 8 bytes.
return is_single_group(new_capacity) && old_capacity < new_capacity;
}
// Relocates control bytes and slots into new single group for
// transferable objects.
// Must be called only if IsGrowingIntoSingleGroupApplicable returned true.
void GrowSizeIntoSingleGroupTransferable(CommonFields& c, void* old_slots,
size_t slot_size);
// Shuffle control bits deterministically to the next capacity.
// Returns offset for newly added element with given hash.
//
// PRECONDITIONs:
// 1. new_ctrl is allocated for new_capacity,
// but not initialized.
// 2. new_capacity is a single group.
//
// All elements are transferred into the first `old_capacity + 1` positions
// of the new_ctrl. Elements are rotated by `old_capacity_ / 2 + 1` positions
// in order to change an order and keep it non deterministic.
// Although rotation itself deterministic, position of the new added element
// will be based on `H1` and is not deterministic.
//
// Examples:
// S = kSentinel, E = kEmpty
//
// old_ctrl = SEEEEEEEE...
// new_ctrl = ESEEEEEEE...
//
// old_ctrl = 0SEEEEEEE...
// new_ctrl = E0ESE0EEE...
//
// old_ctrl = 012S012EEEEEEEEE...
// new_ctrl = 2E01EEES2E01EEE...
//
// old_ctrl = 0123456S0123456EEEEEEEEEEE...
// new_ctrl = 456E0123EEEEEES456E0123EEE...
void GrowIntoSingleGroupShuffleControlBytes(ctrl_t* new_ctrl,
size_t new_capacity) const;
// Shuffle trivially transferable slots in the way consistent with
// GrowIntoSingleGroupShuffleControlBytes.
//
// PRECONDITIONs:
// 1. old_capacity must be non-zero.
// 2. new_ctrl is fully initialized using
// GrowIntoSingleGroupShuffleControlBytes.
// 3. new_slots is allocated and *not* poisoned.
//
// POSTCONDITIONS:
// 1. new_slots are transferred from old_slots_ consistent with
// GrowIntoSingleGroupShuffleControlBytes.
// 2. Empty new_slots are *not* poisoned.
void GrowIntoSingleGroupShuffleTransferableSlots(void* old_slots,
void* new_slots,
size_t slot_size) const;
// Poison empty slots that were transferred using the deterministic algorithm
// described above.
// PRECONDITIONs:
// 1. new_ctrl is fully initialized using
// GrowIntoSingleGroupShuffleControlBytes.
// 2. new_slots is fully initialized consistent with
// GrowIntoSingleGroupShuffleControlBytes.
void PoisonSingleGroupEmptySlots(CommonFields& c, size_t slot_size) const {
// poison non full items
for (size_t i = 0; i < c.capacity(); ++i) {
if (!IsFull(c.control()[i])) {
SanitizerPoisonMemoryRegion(SlotAddress(c.slot_array(), i, slot_size),
slot_size);
}
}
}
ctrl_t* old_ctrl_;
size_t old_capacity_;
bool had_infoz_;
};
// PolicyFunctions bundles together some information for a particular
// raw_hash_set<T, ...> instantiation. This information is passed to
// type-erased functions that want to do small amounts of type-specific
// work.
struct PolicyFunctions {
size_t slot_size;
// Returns the hash of the pointed-to slot.
size_t (*hash_slot)(void* set, void* slot);
// Transfer the contents of src_slot to dst_slot.
void (*transfer)(void* set, void* dst_slot, void* src_slot);
// Deallocate the backing store from common.
void (*dealloc)(CommonFields& common, const PolicyFunctions& policy);
};
// ClearBackingArray clears the backing array, either modifying it in place,
// or creating a new one based on the value of "reuse".
// REQUIRES: c.capacity > 0
void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy,
bool reuse);
// Type-erased version of raw_hash_set::erase_meta_only.
void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size);
// Function to place in PolicyFunctions::dealloc for raw_hash_sets
// that are using std::allocator. This allows us to share the same
// function body for raw_hash_set instantiations that have the
// same slot alignment.
template <size_t AlignOfSlot>
ABSL_ATTRIBUTE_NOINLINE void DeallocateStandard(CommonFields& common,
const PolicyFunctions& policy) {
// Unpoison before returning the memory to the allocator.
SanitizerUnpoisonMemoryRegion(common.slot_array(),
policy.slot_size * common.capacity());
std::allocator<char> alloc;
common.infoz().Unregister();
Deallocate<BackingArrayAlignment(AlignOfSlot)>(
&alloc, common.backing_array_start(),
common.alloc_size(policy.slot_size, AlignOfSlot));
}
// For trivially relocatable types we use memcpy directly. This allows us to
// share the same function body for raw_hash_set instantiations that have the
// same slot size as long as they are relocatable.
template <size_t SizeOfSlot>
ABSL_ATTRIBUTE_NOINLINE void TransferRelocatable(void*, void* dst, void* src) {
memcpy(dst, src, SizeOfSlot);
}
// Type-erased version of raw_hash_set::drop_deletes_without_resize.
void DropDeletesWithoutResize(CommonFields& common,
const PolicyFunctions& policy, void* tmp_space);
// A SwissTable.
//
// Policy: a policy defines how to perform different operations on
// the slots of the hashtable (see hash_policy_traits.h for the full interface
// of policy).
//
// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The
// functor should accept a key and return size_t as hash. For best performance
// it is important that the hash function provides high entropy across all bits
// of the hash.
//
// Eq: a (possibly polymorphic) functor that compares two keys for equality. It
// should accept two (of possibly different type) keys and return a bool: true
// if they are equal, false if they are not. If two keys compare equal, then
// their hash values as defined by Hash MUST be equal.
//
// Allocator: an Allocator
// [https://en.cppreference.com/w/cpp/named_req/Allocator] with which
// the storage of the hashtable will be allocated and the elements will be
// constructed and destroyed.
template <class Policy, class Hash, class Eq, class Alloc>
class raw_hash_set {
using PolicyTraits = hash_policy_traits<Policy>;
using KeyArgImpl =
KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>;
public:
using init_type = typename PolicyTraits::init_type;
using key_type = typename PolicyTraits::key_type;
// TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user
// code fixes!
using slot_type = typename PolicyTraits::slot_type;
using allocator_type = Alloc;
using size_type = size_t;
using difference_type = ptrdiff_t;
using hasher = Hash;
using key_equal = Eq;
using policy_type = Policy;
using value_type = typename PolicyTraits::value_type;
using reference = value_type&;
using const_reference = const value_type&;
using pointer = typename absl::allocator_traits<
allocator_type>::template rebind_traits<value_type>::pointer;
using const_pointer = typename absl::allocator_traits<
allocator_type>::template rebind_traits<value_type>::const_pointer;
// Alias used for heterogeneous lookup functions.
// `key_arg<K>` evaluates to `K` when the functors are transparent and to
// `key_type` otherwise. It permits template argument deduction on `K` for the
// transparent case.
template <class K>
using key_arg = typename KeyArgImpl::template type<K, key_type>;
private:
// Give an early error when key_type is not hashable/eq.
auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k));
auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k));
using AllocTraits = absl::allocator_traits<allocator_type>;
using SlotAlloc = typename absl::allocator_traits<
allocator_type>::template rebind_alloc<slot_type>;
// People are often sloppy with the exact type of their allocator (sometimes
// it has an extra const or is missing the pair, but rebinds made it work
// anyway).
using CharAlloc =
typename absl::allocator_traits<Alloc>::template rebind_alloc<char>;
using SlotAllocTraits = typename absl::allocator_traits<
allocator_type>::template rebind_traits<slot_type>;
static_assert(std::is_lvalue_reference<reference>::value,
"Policy::element() must return a reference");
template <typename T>
struct SameAsElementReference
: std::is_same<typename std::remove_cv<
typename std::remove_reference<reference>::type>::type,
typename std::remove_cv<
typename std::remove_reference<T>::type>::type> {};
// An enabler for insert(T&&): T must be convertible to init_type or be the
// same as [cv] value_type [ref].
// Note: we separate SameAsElementReference into its own type to avoid using
// reference unless we need to. MSVC doesn't seem to like it in some
// cases.
template <class T>
using RequiresInsertable = typename std::enable_if<
absl::disjunction<std::is_convertible<T, init_type>,
SameAsElementReference<T>>::value,
int>::type;
// RequiresNotInit is a workaround for gcc prior to 7.1.
// See https://godbolt.org/g/Y4xsUh.
template <class T>
using RequiresNotInit =
typename std::enable_if<!std::is_same<T, init_type>::value, int>::type;
template <class... Ts>
using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>;
public:
static_assert(std::is_same<pointer, value_type*>::value,
"Allocators with custom pointer types are not supported");
static_assert(std::is_same<const_pointer, const value_type*>::value,
"Allocators with custom pointer types are not supported");
class iterator : private HashSetIteratorGenerationInfo {
friend class raw_hash_set;
public:
using iterator_category = std::forward_iterator_tag;
using value_type = typename raw_hash_set::value_type;
using reference =
absl::conditional_t<PolicyTraits::constant_iterators::value,
const value_type&, value_type&>;
using pointer = absl::remove_reference_t<reference>*;
using difference_type = typename raw_hash_set::difference_type;
iterator() {}
// PRECONDITION: not an end() iterator.
reference operator*() const {
AssertIsFull(ctrl_, generation(), generation_ptr(), "operator*()");
return unchecked_deref();
}
// PRECONDITION: not an end() iterator.
pointer operator->() const {
AssertIsFull(ctrl_, generation(), generation_ptr(), "operator->");
return &operator*();
}
// PRECONDITION: not an end() iterator.
iterator& operator++() {
AssertIsFull(ctrl_, generation(), generation_ptr(), "operator++");
++ctrl_;
++slot_;
skip_empty_or_deleted();
return *this;
}
// PRECONDITION: not an end() iterator.
iterator operator++(int) {
auto tmp = *this;
++*this;
return tmp;
}
friend bool operator==(const iterator& a, const iterator& b) {
AssertIsValidForComparison(a.ctrl_, a.generation(), a.generation_ptr());
AssertIsValidForComparison(b.ctrl_, b.generation(), b.generation_ptr());
AssertSameContainer(a.ctrl_, b.ctrl_, a.slot_, b.slot_,
a.generation_ptr(), b.generation_ptr());
return a.ctrl_ == b.ctrl_;
}
friend bool operator!=(const iterator& a, const iterator& b) {
return !(a == b);
}
private:
iterator(ctrl_t* ctrl, slot_type* slot,
const GenerationType* generation_ptr)
: HashSetIteratorGenerationInfo(generation_ptr),
ctrl_(ctrl),
slot_(slot) {
// This assumption helps the compiler know that any non-end iterator is
// not equal to any end iterator.
ABSL_ASSUME(ctrl != nullptr);
}
// For end() iterators.
explicit iterator(const GenerationType* generation_ptr)
: HashSetIteratorGenerationInfo(generation_ptr), ctrl_(nullptr) {}
// Fixes up `ctrl_` to point to a full by advancing it and `slot_` until
// they reach one.
//
// If a sentinel is reached, we null `ctrl_` out instead.
void skip_empty_or_deleted() {
while (IsEmptyOrDeleted(*ctrl_)) {
uint32_t shift =
GroupEmptyOrDeleted{ctrl_}.CountLeadingEmptyOrDeleted();
ctrl_ += shift;
slot_ += shift;
}
if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr;
}
ctrl_t* control() const { return ctrl_; }
slot_type* slot() const { return slot_; }
// We use EmptyGroup() for default-constructed iterators so that they can
// be distinguished from end iterators, which have nullptr ctrl_.
ctrl_t* ctrl_ = EmptyGroup();
// To avoid uninitialized member warnings, put slot_ in an anonymous union.
// The member is not initialized on singleton and end iterators.
union {
slot_type* slot_;
};
// An equality check which skips ABSL Hardening iterator invalidation
// checks.
// Should be used when the lifetimes of the iterators are well-enough
// understood to prove that they cannot be invalid.
bool unchecked_equals(const iterator& b) { return ctrl_ == b.control(); }
// Dereferences the iterator without ABSL Hardening iterator invalidation
// checks.
reference unchecked_deref() const { return PolicyTraits::element(slot_); }
};
class const_iterator {
friend class raw_hash_set;
template <class Container, typename Enabler>
friend struct absl::container_internal::hashtable_debug_internal::
HashtableDebugAccess;
public:
using iterator_category = typename iterator::iterator_category;
using value_type = typename raw_hash_set::value_type;
using reference = typename raw_hash_set::const_reference;
using pointer = typename raw_hash_set::const_pointer;
using difference_type = typename raw_hash_set::difference_type;
const_iterator() = default;
// Implicit construction from iterator.
const_iterator(iterator i) : inner_(std::move(i)) {} // NOLINT
reference operator*() const { return *inner_; }
pointer operator->() const { return inner_.operator->(); }
const_iterator& operator++() {
++inner_;
return *this;
}
const_iterator operator++(int) { return inner_++; }
friend bool operator==(const const_iterator& a, const const_iterator& b) {
return a.inner_ == b.inner_;
}
friend bool operator!=(const const_iterator& a, const const_iterator& b) {
return !(a == b);
}
private:
const_iterator(const ctrl_t* ctrl, const slot_type* slot,
const GenerationType* gen)
: inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot), gen) {
}
ctrl_t* control() const { return inner_.control(); }
slot_type* slot() const { return inner_.slot(); }
iterator inner_;
bool unchecked_equals(const const_iterator& b) {
return inner_.unchecked_equals(b.inner_);
}
};
using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>;
using insert_return_type = InsertReturnType<iterator, node_type>;
// Note: can't use `= default` due to non-default noexcept (causes
// problems for some compilers). NOLINTNEXTLINE
raw_hash_set() noexcept(
std::is_nothrow_default_constructible<hasher>::value &&
std::is_nothrow_default_constructible<key_equal>::value &&
std::is_nothrow_default_constructible<allocator_type>::value) {}
ABSL_ATTRIBUTE_NOINLINE explicit raw_hash_set(
size_t bucket_count, const hasher& hash = hasher(),
const key_equal& eq = key_equal(),
const allocator_type& alloc = allocator_type())
: settings_(CommonFields{}, hash, eq, alloc) {
if (bucket_count) {
resize(NormalizeCapacity(bucket_count));
}
}
raw_hash_set(size_t bucket_count, const hasher& hash,
const allocator_type& alloc)
: raw_hash_set(bucket_count, hash, key_equal(), alloc) {}
raw_hash_set(size_t bucket_count, const allocator_type& alloc)
: raw_hash_set(bucket_count, hasher(), key_equal(), alloc) {}
explicit raw_hash_set(const allocator_type& alloc)
: raw_hash_set(0, hasher(), key_equal(), alloc) {}
template <class InputIter>
raw_hash_set(InputIter first, InputIter last, size_t bucket_count = 0,
const hasher& hash = hasher(), const key_equal& eq = key_equal(),
const allocator_type& alloc = allocator_type())
: raw_hash_set(SelectBucketCountForIterRange(first, last, bucket_count),
hash, eq, alloc) {
insert(first, last);
}
template <class InputIter>
raw_hash_set(InputIter first, InputIter last, size_t bucket_count,
const hasher& hash, const allocator_type& alloc)
: raw_hash_set(first, last, bucket_count, hash, key_equal(), alloc) {}
template <class InputIter>
raw_hash_set(InputIter first, InputIter last, size_t bucket_count,
const allocator_type& alloc)
: raw_hash_set(first, last, bucket_count, hasher(), key_equal(), alloc) {}
template <class InputIter>
raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc)
: raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {}
// Instead of accepting std::initializer_list<value_type> as the first
// argument like std::unordered_set<value_type> does, we have two overloads
// that accept std::initializer_list<T> and std::initializer_list<init_type>.
// This is advantageous for performance.
//
// // Turns {"abc", "def"} into std::initializer_list<std::string>, then
// // copies the strings into the set.
// std::unordered_set<std::string> s = {"abc", "def"};
//
// // Turns {"abc", "def"} into std::initializer_list<const char*>, then
// // copies the strings into the set.
// absl::flat_hash_set<std::string> s = {"abc", "def"};
//
// The same trick is used in insert().
//
// The enabler is necessary to prevent this constructor from triggering where
// the copy constructor is meant to be called.
//
// absl::flat_hash_set<int> a, b{a};
//
// RequiresNotInit<T> is a workaround for gcc prior to 7.1.
template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
raw_hash_set(std::initializer_list<T> init, size_t bucket_count = 0,
const hasher& hash = hasher(), const key_equal& eq = key_equal(),
const allocator_type& alloc = allocator_type())
: raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {}
raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count = 0,
const hasher& hash = hasher(), const key_equal& eq = key_equal(),
const allocator_type& alloc = allocator_type())
: raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {}
template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
raw_hash_set(std::initializer_list<T> init, size_t bucket_count,
const hasher& hash, const allocator_type& alloc)
: raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {}
raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count,
const hasher& hash, const allocator_type& alloc)
: raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {}
template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
raw_hash_set(std::initializer_list<T> init, size_t bucket_count,
const allocator_type& alloc)
: raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {}
raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count,
const allocator_type& alloc)
: raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {}
template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0>
raw_hash_set(std::initializer_list<T> init, const allocator_type& alloc)
: raw_hash_set(init, 0, hasher(), key_equal(), alloc) {}
raw_hash_set(std::initializer_list<init_type> init,
const allocator_type& alloc)
: raw_hash_set(init, 0, hasher(), key_equal(), alloc) {}
raw_hash_set(const raw_hash_set& that)
: raw_hash_set(that, AllocTraits::select_on_container_copy_construction(
that.alloc_ref())) {}
raw_hash_set(const raw_hash_set& that, const allocator_type& a)
: raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) {
const size_t size = that.size();
if (size == 0) return;
reserve(size);
// Because the table is guaranteed to be empty, we can do something faster
// than a full `insert`.
for (const auto& v : that) {
const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v);
auto target = find_first_non_full_outofline(common(), hash);
SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type));
emplace_at(target.offset, v);
common().maybe_increment_generation_on_insert();
infoz().RecordInsert(hash, target.probe_length);
}
common().set_size(size);
set_growth_left(growth_left() - size);
}
ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept(
std::is_nothrow_copy_constructible<hasher>::value &&
std::is_nothrow_copy_constructible<key_equal>::value &&
std::is_nothrow_copy_constructible<allocator_type>::value)
: // Hash, equality and allocator are copied instead of moved because
// `that` must be left valid. If Hash is std::function<Key>, moving it
// would create a nullptr functor that cannot be called.
// TODO(b/296061262): move instead of copying hash/eq/alloc.
// Note: we avoid using exchange for better generated code.
settings_(std::move(that.common()), that.hash_ref(), that.eq_ref(),
that.alloc_ref()) {
that.common() = CommonFields{};
maybe_increment_generation_or_rehash_on_move();
}
raw_hash_set(raw_hash_set&& that, const allocator_type& a)
: settings_(CommonFields{}, that.hash_ref(), that.eq_ref(), a) {
if (a == that.alloc_ref()) {
std::swap(common(), that.common());
maybe_increment_generation_or_rehash_on_move();
} else {
move_elements_allocs_unequal(std::move(that));
}
}
raw_hash_set& operator=(const raw_hash_set& that) {
if (ABSL_PREDICT_FALSE(this == &that)) return *this;
constexpr bool propagate_alloc =
AllocTraits::propagate_on_container_copy_assignment::value;
// TODO(ezb): maybe avoid allocating a new backing array if this->capacity()
// is an exact match for that.size(). If this->capacity() is too big, then
// it would make iteration very slow to reuse the allocation. Maybe we can
// do the same heuristic as clear() and reuse if it's small enough.
raw_hash_set tmp(that, propagate_alloc ? that.alloc_ref() : alloc_ref());
// NOLINTNEXTLINE: not returning *this for performance.
return assign_impl<propagate_alloc>(std::move(tmp));
}
raw_hash_set& operator=(raw_hash_set&& that) noexcept(
absl::allocator_traits<allocator_type>::is_always_equal::value &&
std::is_nothrow_move_assignable<hasher>::value &&
std::is_nothrow_move_assignable<key_equal>::value) {
// TODO(sbenza): We should only use the operations from the noexcept clause
// to make sure we actually adhere to that contract.
// NOLINTNEXTLINE: not returning *this for performance.
return move_assign(
std::move(that),
typename AllocTraits::propagate_on_container_move_assignment());
}
~raw_hash_set() { destructor_impl(); }
iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto it = iterator_at(0);
it.skip_empty_or_deleted();
return it;
}
iterator end() ABSL_ATTRIBUTE_LIFETIME_BOUND {
return iterator(common().generation_ptr());
}
const_iterator begin() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_cast<raw_hash_set*>(this)->begin();
}
const_iterator end() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return iterator(common().generation_ptr());
}
const_iterator cbegin() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return begin();
}
const_iterator cend() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return end(); }
bool empty() const { return !size(); }
size_t size() const { return common().size(); }
size_t capacity() const { return common().capacity(); }
size_t max_size() const { return (std::numeric_limits<size_t>::max)(); }
ABSL_ATTRIBUTE_REINITIALIZES void clear() {
// Iterating over this container is O(bucket_count()). When bucket_count()
// is much greater than size(), iteration becomes prohibitively expensive.
// For clear() it is more important to reuse the allocated array when the
// container is small because allocation takes comparatively long time
// compared to destruction of the elements of the container. So we pick the
// largest bucket_count() threshold for which iteration is still fast and
// past that we simply deallocate the array.
const size_t cap = capacity();
if (cap == 0) {
// Already guaranteed to be empty; so nothing to do.
} else {
destroy_slots();
ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128);
}
common().set_reserved_growth(0);
common().set_reservation_size(0);
}
// This overload kicks in when the argument is an rvalue of insertable and
// decomposable type other than init_type.
//
// flat_hash_map<std::string, int> m;
// m.insert(std::make_pair("abc", 42));
// TODO(cheshire): A type alias T2 is introduced as a workaround for the nvcc
// bug.
template <class T, RequiresInsertable<T> = 0, class T2 = T,
typename std::enable_if<IsDecomposable<T2>::value, int>::type = 0,
T* = nullptr>
std::pair<iterator, bool> insert(T&& value) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return emplace(std::forward<T>(value));
}
// This overload kicks in when the argument is a bitfield or an lvalue of
// insertable and decomposable type.
//
// union { int n : 1; };
// flat_hash_set<int> s;
// s.insert(n);
//
// flat_hash_set<std::string> s;
// const char* p = "hello";
// s.insert(p);
//
template <
class T, RequiresInsertable<const T&> = 0,
typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0>
std::pair<iterator, bool> insert(const T& value)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return emplace(value);
}
// This overload kicks in when the argument is an rvalue of init_type. Its
// purpose is to handle brace-init-list arguments.
//
// flat_hash_map<std::string, int> s;
// s.insert({"abc", 42});
std::pair<iterator, bool> insert(init_type&& value)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return emplace(std::move(value));
}
// TODO(cheshire): A type alias T2 is introduced as a workaround for the nvcc
// bug.
template <class T, RequiresInsertable<T> = 0, class T2 = T,
typename std::enable_if<IsDecomposable<T2>::value, int>::type = 0,
T* = nullptr>
iterator insert(const_iterator, T&& value) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert(std::forward<T>(value)).first;
}
template <
class T, RequiresInsertable<const T&> = 0,
typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0>
iterator insert(const_iterator,
const T& value) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert(value).first;
}
iterator insert(const_iterator,
init_type&& value) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return insert(std::move(value)).first;
}
template <class InputIt>
void insert(InputIt first, InputIt last) {
for (; first != last; ++first) emplace(*first);
}
template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0>
void insert(std::initializer_list<T> ilist) {
insert(ilist.begin(), ilist.end());
}
void insert(std::initializer_list<init_type> ilist) {
insert(ilist.begin(), ilist.end());
}
insert_return_type insert(node_type&& node) ABSL_ATTRIBUTE_LIFETIME_BOUND {
if (!node) return {end(), false, node_type()};
const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node));
auto res = PolicyTraits::apply(
InsertSlot<false>{*this, std::move(*CommonAccess::GetSlot(node))},
elem);
if (res.second) {
CommonAccess::Reset(&node);
return {res.first, true, node_type()};
} else {
return {res.first, false, std::move(node)};
}
}
iterator insert(const_iterator,
node_type&& node) ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto res = insert(std::move(node));
node = std::move(res.node);
return res.position;
}
// This overload kicks in if we can deduce the key from args. This enables us
// to avoid constructing value_type if an entry with the same key already
// exists.
//
// For example:
//
// flat_hash_map<std::string, std::string> m = {{"abc", "def"}};
// // Creates no std::string copies and makes no heap allocations.
// m.emplace("abc", "xyz");
template <class... Args, typename std::enable_if<
IsDecomposable<Args...>::value, int>::type = 0>
std::pair<iterator, bool> emplace(Args&&... args)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return PolicyTraits::apply(EmplaceDecomposable{*this},
std::forward<Args>(args)...);
}
// This overload kicks in if we cannot deduce the key from args. It constructs
// value_type unconditionally and then either moves it into the table or
// destroys.
template <class... Args, typename std::enable_if<
!IsDecomposable<Args...>::value, int>::type = 0>
std::pair<iterator, bool> emplace(Args&&... args)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
alignas(slot_type) unsigned char raw[sizeof(slot_type)];
slot_type* slot = reinterpret_cast<slot_type*>(&raw);
construct(slot, std::forward<Args>(args)...);
const auto& elem = PolicyTraits::element(slot);
return PolicyTraits::apply(InsertSlot<true>{*this, std::move(*slot)}, elem);
}
template <class... Args>
iterator emplace_hint(const_iterator,
Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return emplace(std::forward<Args>(args)...).first;
}
// Extension API: support for lazy emplace.
//
// Looks up key in the table. If found, returns the iterator to the element.
// Otherwise calls `f` with one argument of type `raw_hash_set::constructor`,
// and returns an iterator to the new element.
//
// `f` must abide by several restrictions:
// - it MUST call `raw_hash_set::constructor` with arguments as if a
// `raw_hash_set::value_type` is constructed,
// - it MUST NOT access the container before the call to
// `raw_hash_set::constructor`, and
// - it MUST NOT erase the lazily emplaced element.
// Doing any of these is undefined behavior.
//
// For example:
//
// std::unordered_set<ArenaString> s;
// // Makes ArenaStr even if "abc" is in the map.
// s.insert(ArenaString(&arena, "abc"));
//
// flat_hash_set<ArenaStr> s;
// // Makes ArenaStr only if "abc" is not in the map.
// s.lazy_emplace("abc", [&](const constructor& ctor) {
// ctor(&arena, "abc");
// });
//
// WARNING: This API is currently experimental. If there is a way to implement
// the same thing with the rest of the API, prefer that.
class constructor {
friend class raw_hash_set;
public:
template <class... Args>
void operator()(Args&&... args) const {
assert(*slot_);
PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...);
*slot_ = nullptr;
}
private:
constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {}
allocator_type* alloc_;
slot_type** slot_;
};
template <class K = key_type, class F>
iterator lazy_emplace(const key_arg<K>& key,
F&& f) ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto res = find_or_prepare_insert(key);
if (res.second) {
slot_type* slot = slot_array() + res.first;
std::forward<F>(f)(constructor(&alloc_ref(), &slot));
assert(!slot);
}
return iterator_at(res.first);
}
// Extension API: support for heterogeneous keys.
//
// std::unordered_set<std::string> s;
// // Turns "abc" into std::string.
// s.erase("abc");
//
// flat_hash_set<std::string> s;
// // Uses "abc" directly without copying it into std::string.
// s.erase("abc");
template <class K = key_type>
size_type erase(const key_arg<K>& key) {
auto it = find(key);
if (it == end()) return 0;
erase(it);
return 1;
}
// Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`,
// this method returns void to reduce algorithmic complexity to O(1). The
// iterator is invalidated, so any increment should be done before calling
// erase. In order to erase while iterating across a map, use the following
// idiom (which also works for standard containers):
//
// for (auto it = m.begin(), end = m.end(); it != end;) {
// // `erase()` will invalidate `it`, so advance `it` first.
// auto copy_it = it++;
// if (<pred>) {
// m.erase(copy_it);
// }
// }
void erase(const_iterator cit) { erase(cit.inner_); }
// This overload is necessary because otherwise erase<K>(const K&) would be
// a better match if non-const iterator is passed as an argument.
void erase(iterator it) {
AssertIsFull(it.control(), it.generation(), it.generation_ptr(), "erase()");
destroy(it.slot());
erase_meta_only(it);
}
iterator erase(const_iterator first,
const_iterator last) ABSL_ATTRIBUTE_LIFETIME_BOUND {
// We check for empty first because ClearBackingArray requires that
// capacity() > 0 as a precondition.
if (empty()) return end();
if (first == begin() && last == end()) {
// TODO(ezb): we access control bytes in destroy_slots so it could make
// sense to combine destroy_slots and ClearBackingArray to avoid cache
// misses when the table is large. Note that we also do this in clear().
destroy_slots();
ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true);
common().set_reserved_growth(common().reservation_size());
return end();
}
while (first != last) {
erase(first++);
}
return last.inner_;
}
// Moves elements from `src` into `this`.
// If the element already exists in `this`, it is left unmodified in `src`.
template <typename H, typename E>
void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT
assert(this != &src);
for (auto it = src.begin(), e = src.end(); it != e;) {
auto next = std::next(it);
if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot())},
PolicyTraits::element(it.slot()))
.second) {
src.erase_meta_only(it);
}
it = next;
}
}
template <typename H, typename E>
void merge(raw_hash_set<Policy, H, E, Alloc>&& src) {
merge(src);
}
node_type extract(const_iterator position) {
AssertIsFull(position.control(), position.inner_.generation(),
position.inner_.generation_ptr(), "extract()");
auto node = CommonAccess::Transfer<node_type>(alloc_ref(), position.slot());
erase_meta_only(position);
return node;
}
template <
class K = key_type,
typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0>
node_type extract(const key_arg<K>& key) {
auto it = find(key);
return it == end() ? node_type() : extract(const_iterator{it});
}
void swap(raw_hash_set& that) noexcept(
IsNoThrowSwappable<hasher>() && IsNoThrowSwappable<key_equal>() &&
IsNoThrowSwappable<allocator_type>(
typename AllocTraits::propagate_on_container_swap{})) {
using std::swap;
swap(common(), that.common());
swap(hash_ref(), that.hash_ref());
swap(eq_ref(), that.eq_ref());
SwapAlloc(alloc_ref(), that.alloc_ref(),
typename AllocTraits::propagate_on_container_swap{});
}
void rehash(size_t n) {
if (n == 0 && capacity() == 0) return;
if (n == 0 && size() == 0) {
ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false);
return;
}
// bitor is a faster way of doing `max` here. We will round up to the next
// power-of-2-minus-1, so bitor is good enough.
auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size()));
// n == 0 unconditionally rehashes as per the standard.
if (n == 0 || m > capacity()) {
resize(m);
// This is after resize, to ensure that we have completed the allocation
// and have potentially sampled the hashtable.
infoz().RecordReservation(n);
}
}
void reserve(size_t n) {
if (n > size() + growth_left()) {
size_t m = GrowthToLowerboundCapacity(n);
resize(NormalizeCapacity(m));
// This is after resize, to ensure that we have completed the allocation
// and have potentially sampled the hashtable.
infoz().RecordReservation(n);
}
common().reset_reserved_growth(n);
common().set_reservation_size(n);
}
// Extension API: support for heterogeneous keys.
//
// std::unordered_set<std::string> s;
// // Turns "abc" into std::string.
// s.count("abc");
//
// ch_set<std::string> s;
// // Uses "abc" directly without copying it into std::string.
// s.count("abc");
template <class K = key_type>
size_t count(const key_arg<K>& key) const {
return find(key) == end() ? 0 : 1;
}
// Issues CPU prefetch instructions for the memory needed to find or insert
// a key. Like all lookup functions, this support heterogeneous keys.
//
// NOTE: This is a very low level operation and should not be used without
// specific benchmarks indicating its importance.
template <class K = key_type>
void prefetch(const key_arg<K>& key) const {
(void)key;
// Avoid probing if we won't be able to prefetch the addresses received.
#ifdef ABSL_HAVE_PREFETCH
prefetch_heap_block();
auto seq = probe(common(), hash_ref()(key));
PrefetchToLocalCache(control() + seq.offset());
PrefetchToLocalCache(slot_array() + seq.offset());
#endif // ABSL_HAVE_PREFETCH
}
// The API of find() has two extensions.
//
// 1. The hash can be passed by the user. It must be equal to the hash of the
// key.
//
// 2. The type of the key argument doesn't have to be key_type. This is so
// called heterogeneous key support.
template <class K = key_type>
iterator find(const key_arg<K>& key,
size_t hash) ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto seq = probe(common(), hash);
slot_type* slot_ptr = slot_array();
const ctrl_t* ctrl = control();
while (true) {
Group g{ctrl + seq.offset()};
for (uint32_t i : g.Match(H2(hash))) {
if (ABSL_PREDICT_TRUE(PolicyTraits::apply(
EqualElement<K>{key, eq_ref()},
PolicyTraits::element(slot_ptr + seq.offset(i)))))
return iterator_at(seq.offset(i));
}
if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end();
seq.next();
assert(seq.index() <= capacity() && "full table!");
}
}
template <class K = key_type>
iterator find(const key_arg<K>& key) ABSL_ATTRIBUTE_LIFETIME_BOUND {
prefetch_heap_block();
return find(key, hash_ref()(key));
}
template <class K = key_type>
const_iterator find(const key_arg<K>& key,
size_t hash) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return const_cast<raw_hash_set*>(this)->find(key, hash);
}
template <class K = key_type>
const_iterator find(const key_arg<K>& key) const
ABSL_ATTRIBUTE_LIFETIME_BOUND {
prefetch_heap_block();
return find(key, hash_ref()(key));
}
template <class K = key_type>
bool contains(const key_arg<K>& key) const {
// Here neither the iterator returned by `find()` nor `end()` can be invalid
// outside of potential thread-safety issues.
// `find()`'s return value is constructed, used, and then destructed
// all in this context.
return !find(key).unchecked_equals(end());
}
template <class K = key_type>
std::pair<iterator, iterator> equal_range(const key_arg<K>& key)
ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto it = find(key);
if (it != end()) return {it, std::next(it)};
return {it, it};
}
template <class K = key_type>
std::pair<const_iterator, const_iterator> equal_range(
const key_arg<K>& key) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
auto it = find(key);
if (it != end()) return {it, std::next(it)};
return {it, it};
}
size_t bucket_count() const { return capacity(); }
float load_factor() const {
return capacity() ? static_cast<double>(size()) / capacity() : 0.0;
}
float max_load_factor() const { return 1.0f; }
void max_load_factor(float) {
// Does nothing.
}
hasher hash_function() const { return hash_ref(); }
key_equal key_eq() const { return eq_ref(); }
allocator_type get_allocator() const { return alloc_ref(); }
friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) {
if (a.size() != b.size()) return false;
const raw_hash_set* outer = &a;
const raw_hash_set* inner = &b;
if (outer->capacity() > inner->capacity()) std::swap(outer, inner);
for (const value_type& elem : *outer) {
auto it = PolicyTraits::apply(FindElement{*inner}, elem);
if (it == inner->end() || !(*it == elem)) return false;
}
return true;
}
friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) {
return !(a == b);
}
template <typename H>
friend typename std::enable_if<H::template is_hashable<value_type>::value,
H>::type
AbslHashValue(H h, const raw_hash_set& s) {
return H::combine(H::combine_unordered(std::move(h), s.begin(), s.end()),
s.size());
}
friend void swap(raw_hash_set& a,
raw_hash_set& b) noexcept(noexcept(a.swap(b))) {
a.swap(b);
}
private:
template <class Container, typename Enabler>
friend struct absl::container_internal::hashtable_debug_internal::
HashtableDebugAccess;
struct FindElement {
template <class K, class... Args>
const_iterator operator()(const K& key, Args&&...) const {
return s.find(key);
}
const raw_hash_set& s;
};
struct HashElement {
template <class K, class... Args>
size_t operator()(const K& key, Args&&...) const {
return h(key);
}
const hasher& h;
};
template <class K1>
struct EqualElement {
template <class K2, class... Args>
bool operator()(const K2& lhs, Args&&...) const {
return eq(lhs, rhs);
}
const K1& rhs;
const key_equal& eq;
};
struct EmplaceDecomposable {
template <class K, class... Args>
std::pair<iterator, bool> operator()(const K& key, Args&&... args) const {
auto res = s.find_or_prepare_insert(key);
if (res.second) {
s.emplace_at(res.first, std::forward<Args>(args)...);
}
return {s.iterator_at(res.first), res.second};
}
raw_hash_set& s;
};
template <bool do_destroy>
struct InsertSlot {
template <class K, class... Args>
std::pair<iterator, bool> operator()(const K& key, Args&&...) && {
auto res = s.find_or_prepare_insert(key);
if (res.second) {
s.transfer(s.slot_array() + res.first, &slot);
} else if (do_destroy) {
s.destroy(&slot);
}
return {s.iterator_at(res.first), res.second};
}
raw_hash_set& s;
// Constructed slot. Either moved into place or destroyed.
slot_type&& slot;
};
// TODO(b/303305702): re-enable reentrant validation.
template <typename... Args>
inline void construct(slot_type* slot, Args&&... args) {
PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...);
}
inline void destroy(slot_type* slot) {
PolicyTraits::destroy(&alloc_ref(), slot);
}
inline void transfer(slot_type* to, slot_type* from) {
PolicyTraits::transfer(&alloc_ref(), to, from);
}
inline void destroy_slots() {
const size_t cap = capacity();
const ctrl_t* ctrl = control();
slot_type* slot = slot_array();
for (size_t i = 0; i != cap; ++i) {
if (IsFull(ctrl[i])) {
destroy(slot + i);
}
}
}
inline void dealloc() {
assert(capacity() != 0);
// Unpoison before returning the memory to the allocator.
SanitizerUnpoisonMemoryRegion(slot_array(), sizeof(slot_type) * capacity());
infoz().Unregister();
Deallocate<BackingArrayAlignment(alignof(slot_type))>(
&alloc_ref(), common().backing_array_start(),
common().alloc_size(sizeof(slot_type), alignof(slot_type)));
}
inline void destructor_impl() {
if (capacity() == 0) return;
destroy_slots();
dealloc();
}
// Erases, but does not destroy, the value pointed to by `it`.
//
// This merely updates the pertinent control byte. This can be used in
// conjunction with Policy::transfer to move the object to another place.
void erase_meta_only(const_iterator it) {
EraseMetaOnly(common(), static_cast<size_t>(it.control() - control()),
sizeof(slot_type));
}
// Resizes table to the new capacity and move all elements to the new
// positions accordingly.
//
// Note that for better performance instead of
// find_first_non_full(common(), hash),
// HashSetResizeHelper::FindFirstNonFullAfterResize(
// common(), old_capacity, hash)
// can be called right after `resize`.
ABSL_ATTRIBUTE_NOINLINE void resize(size_t new_capacity) {
assert(IsValidCapacity(new_capacity));
HashSetResizeHelper resize_helper(common());
auto* old_slots = slot_array();
common().set_capacity(new_capacity);
// Note that `InitializeSlots` does different number initialization steps
// depending on the values of `transfer_uses_memcpy` and capacities.
// Refer to the comment in `InitializeSlots` for more details.
const bool grow_single_group =
resize_helper.InitializeSlots<CharAlloc, sizeof(slot_type),
PolicyTraits::transfer_uses_memcpy(),
alignof(slot_type)>(
common(), const_cast<std::remove_const_t<slot_type>*>(old_slots),
CharAlloc(alloc_ref()));
if (resize_helper.old_capacity() == 0) {
// InitializeSlots did all the work including infoz().RecordRehash().
return;
}
if (grow_single_group) {
if (PolicyTraits::transfer_uses_memcpy()) {
// InitializeSlots did all the work.
return;
}
// We want GrowSizeIntoSingleGroup to be called here in order to make
// InitializeSlots not depend on PolicyTraits.
resize_helper.GrowSizeIntoSingleGroup<PolicyTraits>(common(), alloc_ref(),
old_slots);
} else {
// InitializeSlots prepares control bytes to correspond to empty table.
auto* new_slots = slot_array();
size_t total_probe_length = 0;
for (size_t i = 0; i != resize_helper.old_capacity(); ++i) {
if (IsFull(resize_helper.old_ctrl()[i])) {
size_t hash = PolicyTraits::apply(
HashElement{hash_ref()}, PolicyTraits::element(old_slots + i));
auto target = find_first_non_full(common(), hash);
size_t new_i = target.offset;
total_probe_length += target.probe_length;
SetCtrl(common(), new_i, H2(hash), sizeof(slot_type));
transfer(new_slots + new_i, old_slots + i);
}
}
infoz().RecordRehash(total_probe_length);
}
resize_helper.DeallocateOld<alignof(slot_type)>(
CharAlloc(alloc_ref()), sizeof(slot_type),
const_cast<std::remove_const_t<slot_type>*>(old_slots));
}
// Prunes control bytes to remove as many tombstones as possible.
//
// See the comment on `rehash_and_grow_if_necessary()`.
inline void drop_deletes_without_resize() {
// Stack-allocate space for swapping elements.
alignas(slot_type) unsigned char tmp[sizeof(slot_type)];
DropDeletesWithoutResize(common(), GetPolicyFunctions(), tmp);
}
// Called whenever the table *might* need to conditionally grow.
//
// This function is an optimization opportunity to perform a rehash even when
// growth is unnecessary, because vacating tombstones is beneficial for
// performance in the long-run.
void rehash_and_grow_if_necessary() {
const size_t cap = capacity();
if (cap > Group::kWidth &&
// Do these calculations in 64-bit to avoid overflow.
size() * uint64_t{32} <= cap * uint64_t{25}) {
// Squash DELETED without growing if there is enough capacity.
//
// Rehash in place if the current size is <= 25/32 of capacity.
// Rationale for such a high factor: 1) drop_deletes_without_resize() is
// faster than resize, and 2) it takes quite a bit of work to add
// tombstones. In the worst case, seems to take approximately 4
// insert/erase pairs to create a single tombstone and so if we are
// rehashing because of tombstones, we can afford to rehash-in-place as
// long as we are reclaiming at least 1/8 the capacity without doing more
// than 2X the work. (Where "work" is defined to be size() for rehashing
// or rehashing in place, and 1 for an insert or erase.) But rehashing in
// place is faster per operation than inserting or even doubling the size
// of the table, so we actually afford to reclaim even less space from a
// resize-in-place. The decision is to rehash in place if we can reclaim
// at about 1/8th of the usable capacity (specifically 3/28 of the
// capacity) which means that the total cost of rehashing will be a small
// fraction of the total work.
//
// Here is output of an experiment using the BM_CacheInSteadyState
// benchmark running the old case (where we rehash-in-place only if we can
// reclaim at least 7/16*capacity) vs. this code (which rehashes in place
// if we can recover 3/32*capacity).
//
// Note that although in the worst-case number of rehashes jumped up from
// 15 to 190, but the number of operations per second is almost the same.
//
// Abridged output of running BM_CacheInSteadyState benchmark from
// raw_hash_set_benchmark. N is the number of insert/erase operations.
//
// | OLD (recover >= 7/16 | NEW (recover >= 3/32)
// size | N/s LoadFactor NRehashes | N/s LoadFactor NRehashes
// 448 | 145284 0.44 18 | 140118 0.44 19
// 493 | 152546 0.24 11 | 151417 0.48 28
// 538 | 151439 0.26 11 | 151152 0.53 38
// 583 | 151765 0.28 11 | 150572 0.57 50
// 628 | 150241 0.31 11 | 150853 0.61 66
// 672 | 149602 0.33 12 | 150110 0.66 90
// 717 | 149998 0.35 12 | 149531 0.70 129
// 762 | 149836 0.37 13 | 148559 0.74 190
// 807 | 149736 0.39 14 | 151107 0.39 14
// 852 | 150204 0.42 15 | 151019 0.42 15
drop_deletes_without_resize();
} else {
// Otherwise grow the container.
resize(NextCapacity(cap));
}
}
void maybe_increment_generation_or_rehash_on_move() {
common().maybe_increment_generation_on_move();
if (!empty() && common().should_rehash_for_bug_detection_on_move()) {
resize(capacity());
}
}
template<bool propagate_alloc>
raw_hash_set& assign_impl(raw_hash_set&& that) {
// We don't bother checking for this/that aliasing. We just need to avoid
// breaking the invariants in that case.
destructor_impl();
common() = std::move(that.common());
// TODO(b/296061262): move instead of copying hash/eq/alloc.
hash_ref() = that.hash_ref();
eq_ref() = that.eq_ref();
CopyAlloc(alloc_ref(), that.alloc_ref(),
std::integral_constant<bool, propagate_alloc>());
that.common() = CommonFields{};
maybe_increment_generation_or_rehash_on_move();
return *this;
}
raw_hash_set& move_elements_allocs_unequal(raw_hash_set&& that) {
const size_t size = that.size();
if (size == 0) return *this;
reserve(size);
for (iterator it = that.begin(); it != that.end(); ++it) {
insert(std::move(PolicyTraits::element(it.slot())));
that.destroy(it.slot());
}
that.dealloc();
that.common() = CommonFields{};
maybe_increment_generation_or_rehash_on_move();
return *this;
}
raw_hash_set& move_assign(raw_hash_set&& that,
std::true_type /*propagate_alloc*/) {
return assign_impl<true>(std::move(that));
}
raw_hash_set& move_assign(raw_hash_set&& that,
std::false_type /*propagate_alloc*/) {
if (alloc_ref() == that.alloc_ref()) {
return assign_impl<false>(std::move(that));
}
// Aliasing can't happen here because allocs would compare equal above.
assert(this != &that);
destructor_impl();
// We can't take over that's memory so we need to move each element.
// While moving elements, this should have that's hash/eq so copy hash/eq
// before moving elements.
// TODO(b/296061262): move instead of copying hash/eq.
hash_ref() = that.hash_ref();
eq_ref() = that.eq_ref();
return move_elements_allocs_unequal(std::move(that));
}
protected:
// Attempts to find `key` in the table; if it isn't found, returns a slot that
// the value can be inserted into, with the control byte already set to
// `key`'s H2.
template <class K>
std::pair<size_t, bool> find_or_prepare_insert(const K& key) {
prefetch_heap_block();
auto hash = hash_ref()(key);
auto seq = probe(common(), hash);
const ctrl_t* ctrl = control();
while (true) {
Group g{ctrl + seq.offset()};
for (uint32_t i : g.Match(H2(hash))) {
if (ABSL_PREDICT_TRUE(PolicyTraits::apply(
EqualElement<K>{key, eq_ref()},
PolicyTraits::element(slot_array() + seq.offset(i)))))
return {seq.offset(i), false};
}
if (ABSL_PREDICT_TRUE(g.MaskEmpty())) break;
seq.next();
assert(seq.index() <= capacity() && "full table!");
}
return {prepare_insert(hash), true};
}
// Given the hash of a value not currently in the table, finds the next
// viable slot index to insert it at.
//
// REQUIRES: At least one non-full slot available.
size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE {
const bool rehash_for_bug_detection =
common().should_rehash_for_bug_detection_on_insert();
if (rehash_for_bug_detection) {
// Move to a different heap allocation in order to detect bugs.
const size_t cap = capacity();
resize(growth_left() > 0 ? cap : NextCapacity(cap));
}
auto target = find_first_non_full(common(), hash);
if (!rehash_for_bug_detection &&
ABSL_PREDICT_FALSE(growth_left() == 0 &&
!IsDeleted(control()[target.offset]))) {
size_t old_capacity = capacity();
rehash_and_grow_if_necessary();
// NOTE: It is safe to use `FindFirstNonFullAfterResize`.
// `FindFirstNonFullAfterResize` must be called right after resize.
// `rehash_and_grow_if_necessary` may *not* call `resize`
// and perform `drop_deletes_without_resize` instead. But this
// could happen only on big tables.
// For big tables `FindFirstNonFullAfterResize` will always
// fallback to normal `find_first_non_full`, so it is safe to use it.
target = HashSetResizeHelper::FindFirstNonFullAfterResize(
common(), old_capacity, hash);
}
common().increment_size();
set_growth_left(growth_left() - IsEmpty(control()[target.offset]));
SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type));
common().maybe_increment_generation_on_insert();
infoz().RecordInsert(hash, target.probe_length);
return target.offset;
}
// Constructs the value in the space pointed by the iterator. This only works
// after an unsuccessful find_or_prepare_insert() and before any other
// modifications happen in the raw_hash_set.
//
// PRECONDITION: i is an index returned from find_or_prepare_insert(k), where
// k is the key decomposed from `forward<Args>(args)...`, and the bool
// returned by find_or_prepare_insert(k) was true.
// POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...).
template <class... Args>
void emplace_at(size_t i, Args&&... args) {
construct(slot_array() + i, std::forward<Args>(args)...);
assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) ==
iterator_at(i) &&
"constructed value does not match the lookup key");
}
iterator iterator_at(size_t i) ABSL_ATTRIBUTE_LIFETIME_BOUND {
return {control() + i, slot_array() + i, common().generation_ptr()};
}
const_iterator iterator_at(size_t i) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return {control() + i, slot_array() + i, common().generation_ptr()};
}
reference unchecked_deref(iterator it) { return it.unchecked_deref(); }
private:
friend struct RawHashSetTestOnlyAccess;
// The number of slots we can still fill without needing to rehash.
//
// This is stored separately due to tombstones: we do not include tombstones
// in the growth capacity, because we'd like to rehash when the table is
// otherwise filled with tombstones: otherwise, probe sequences might get
// unacceptably long without triggering a rehash. Callers can also force a
// rehash via the standard `rehash(0)`, which will recompute this value as a
// side-effect.
//
// See `CapacityToGrowth()`.
size_t growth_left() const { return common().growth_left(); }
void set_growth_left(size_t gl) { return common().set_growth_left(gl); }
// Prefetch the heap-allocated memory region to resolve potential TLB and
// cache misses. This is intended to overlap with execution of calculating the
// hash for a key.
void prefetch_heap_block() const {
#if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__)
__builtin_prefetch(control(), 0, 1);
#endif
}
CommonFields& common() { return settings_.template get<0>(); }
const CommonFields& common() const { return settings_.template get<0>(); }
ctrl_t* control() const { return common().control(); }
slot_type* slot_array() const {
return static_cast<slot_type*>(common().slot_array());
}
HashtablezInfoHandle infoz() { return common().infoz(); }
hasher& hash_ref() { return settings_.template get<1>(); }
const hasher& hash_ref() const { return settings_.template get<1>(); }
key_equal& eq_ref() { return settings_.template get<2>(); }
const key_equal& eq_ref() const { return settings_.template get<2>(); }
allocator_type& alloc_ref() { return settings_.template get<3>(); }
const allocator_type& alloc_ref() const {
return settings_.template get<3>();
}
// Make type-specific functions for this type's PolicyFunctions struct.
static size_t hash_slot_fn(void* set, void* slot) {
auto* h = static_cast<raw_hash_set*>(set);
return PolicyTraits::apply(
HashElement{h->hash_ref()},
PolicyTraits::element(static_cast<slot_type*>(slot)));
}
static void transfer_slot_fn(void* set, void* dst, void* src) {
auto* h = static_cast<raw_hash_set*>(set);
h->transfer(static_cast<slot_type*>(dst), static_cast<slot_type*>(src));
}
// Note: dealloc_fn will only be used if we have a non-standard allocator.
static void dealloc_fn(CommonFields& common, const PolicyFunctions&) {
auto* set = reinterpret_cast<raw_hash_set*>(&common);
// Unpoison before returning the memory to the allocator.
SanitizerUnpoisonMemoryRegion(common.slot_array(),
sizeof(slot_type) * common.capacity());
common.infoz().Unregister();
Deallocate<BackingArrayAlignment(alignof(slot_type))>(
&set->alloc_ref(), common.backing_array_start(),
common.alloc_size(sizeof(slot_type), alignof(slot_type)));
}
static const PolicyFunctions& GetPolicyFunctions() {
static constexpr PolicyFunctions value = {
sizeof(slot_type),
&raw_hash_set::hash_slot_fn,
PolicyTraits::transfer_uses_memcpy()
? TransferRelocatable<sizeof(slot_type)>
: &raw_hash_set::transfer_slot_fn,
(std::is_same<SlotAlloc, std::allocator<slot_type>>::value
? &DeallocateStandard<alignof(slot_type)>
: &raw_hash_set::dealloc_fn),
};
return value;
}
// Bundle together CommonFields plus other objects which might be empty.
// CompressedTuple will ensure that sizeof is not affected by any of the empty
// fields that occur after CommonFields.
absl::container_internal::CompressedTuple<CommonFields, hasher, key_equal,
allocator_type>
settings_{CommonFields{}, hasher{}, key_equal{}, allocator_type{}};
};
// Erases all elements that satisfy the predicate `pred` from the container `c`.
template <typename P, typename H, typename E, typename A, typename Predicate>
typename raw_hash_set<P, H, E, A>::size_type EraseIf(
Predicate& pred, raw_hash_set<P, H, E, A>* c) {
const auto initial_size = c->size();
for (auto it = c->begin(), last = c->end(); it != last;) {
if (pred(*it)) {
c->erase(it++);
} else {
++it;
}
}
return initial_size - c->size();
}
namespace hashtable_debug_internal {
template <typename Set>
struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> {
using Traits = typename Set::PolicyTraits;
using Slot = typename Traits::slot_type;
static size_t GetNumProbes(const Set& set,
const typename Set::key_type& key) {
size_t num_probes = 0;
size_t hash = set.hash_ref()(key);
auto seq = probe(set.common(), hash);
const ctrl_t* ctrl = set.control();
while (true) {
container_internal::Group g{ctrl + seq.offset()};
for (uint32_t i : g.Match(container_internal::H2(hash))) {
if (Traits::apply(
typename Set::template EqualElement<typename Set::key_type>{
key, set.eq_ref()},
Traits::element(set.slot_array() + seq.offset(i))))
return num_probes;
++num_probes;
}
if (g.MaskEmpty()) return num_probes;
seq.next();
++num_probes;
}
}
static size_t AllocatedByteSize(const Set& c) {
size_t capacity = c.capacity();
if (capacity == 0) return 0;
size_t m = c.common().alloc_size(sizeof(Slot), alignof(Slot));
size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr));
if (per_slot != ~size_t{}) {
m += per_slot * c.size();
} else {
for (auto it = c.begin(); it != c.end(); ++it) {
m += Traits::space_used(it.slot());
}
}
return m;
}
};
} // namespace hashtable_debug_internal
} // namespace container_internal
ABSL_NAMESPACE_END
} // namespace absl
#undef ABSL_SWISSTABLE_ENABLE_GENERATIONS
#endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_