OpenE2K
/
gcc
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

libstdc++: Optimize operations on small size hashtable [PR 68303] 

When hasher is identified as slow and the number of elements is limited in the
container use a brute-force loop on those elements to look for a given key using
the key_equal functor. For the moment the default threshold to consider the
container as small is 20.

libstdc++-v3/ChangeLog:

	PR libstdc++/68303
	* include/bits/hashtable_policy.h
	(_Hashtable_hash_traits<_Hash>): New.
	(_Hash_code_base<>::_M_hash_code(const _Hash_node_value<>&)): New.
	(_Hashtable_base<>::_M_key_equals): New.
	(_Hashtable_base<>::_M_equals): Use latter.
	(_Hashtable_base<>::_M_key_equals_tr): New.
	(_Hashtable_base<>::_M_equals_tr): Use latter.
	* include/bits/hashtable.h
	(_Hashtable<>::__small_size_threshold()): New, use _Hashtable_hash_traits.
	(_Hashtable<>::find): Loop through elements to look for key if size is lower
	than __small_size_threshold().
	(_Hashtable<>::_M_emplace(true_type, _Args&&...)): Likewise.
	(_Hashtable<>::_M_insert_unique(_Kt&&, _Args&&, const _NodeGenerator&)): Likewise.
	(_Hashtable<>::_M_compute_hash_code(const_iterator, const key_type&)): New.
	(_Hashtable<>::_M_emplace(const_iterator, false_type, _Args&&...)): Use latter.
	(_Hashtable<>::_M_find_before_node(const key_type&)): New.
	(_Hashtable<>::_M_erase(true_type, const key_type&)): Use latter.
	(_Hashtable<>::_M_erase(false_type, const key_type&)): Likewise.
	* src/c++11/hashtable_c++0x.cc: Include <bits/functional_hash.h>.
	* testsuite/util/testsuite_performance.h
	(report_performance): Use 9 width to display memory.
	* testsuite/performance/23_containers/insert_erase/unordered_small_size.cc:
	New performance test case.
This commit is contained in:
François Dumont 2020-01-20 08:17:09 +01:00
parent 194f712f8b
commit e3ef832a9e
5 changed files with 336 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
libstdc++-v3/include/bits/hashtable.h
View File

@ -419,6 +419,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
_M_uses_single_bucket() const
{ return _M_uses_single_bucket(_M_buckets); }
static constexpr size_t
__small_size_threshold() noexcept
{
return
__detail::_Hashtable_hash_traits<_Hash>::__small_size_threshold();
}
__hashtable_alloc&
_M_base_alloc() { return *this; }
@ -788,6 +795,9 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
_M_bucket_index(__hash_code __c) const
{ return __hash_code_base::_M_bucket_index(__c, _M_bucket_count); }
__node_base_ptr
_M_find_before_node(const key_type&);
// Find and insert helper functions and types
// Find the node before the one matching the criteria.
__node_base_ptr
@ -831,6 +841,9 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
__node_base_ptr
_M_get_previous_node(size_type __bkt, __node_ptr __n);
pair<const_iterator, __hash_code>
_M_compute_hash_code(const_iterator __hint, const key_type& __k) const;
// Insert node __n with hash code __code, in bucket __bkt if no
// rehash (assumes no element with same key already present).
// Takes ownership of __n if insertion succeeds, throws otherwise.
@ -1126,7 +1139,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
void _M_rehash(size_type __bkt_count, const __rehash_state& __state);
};
// Definitions of class template _Hashtable's out-of-line member functions.
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
@ -1628,6 +1640,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
find(const key_type& __k)
-> iterator
{
if (size() <= __small_size_threshold())
{
for (auto __it = begin(); __it != end(); ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
return __it;
return end();
}
__hash_code __code = this->_M_hash_code(__k);
std::size_t __bkt = _M_bucket_index(__code);
return iterator(_M_find_node(__bkt, __k, __code));
@ -1643,6 +1663,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
find(const key_type& __k) const
-> const_iterator
{
if (size() <= __small_size_threshold())
{
for (auto __it = begin(); __it != end(); ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
return __it;
return end();
}
__hash_code __code = this->_M_hash_code(__k);
std::size_t __bkt = _M_bucket_index(__code);
return const_iterator(_M_find_node(__bkt, __k, __code));
@ -1855,6 +1883,35 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
}
#endif
// Find the node before the one whose key compares equal to k.
// Return nullptr if no node is found.
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_find_before_node(const key_type& __k)
-> __node_base_ptr
{
__node_base_ptr __prev_p = &_M_before_begin;
if (!__prev_p->_M_nxt)
return nullptr;
for (__node_ptr __p = static_cast<__node_ptr>(__prev_p->_M_nxt);
__p != nullptr;
__p = __p->_M_next())
{
if (this->_M_key_equals(__k, *__p))
return __prev_p;
__prev_p = __p;
}
return nullptr;
}
// Find the node before the one whose key compares equal to k in the bucket
// bkt. Return nullptr if no node is found.
template<typename _Key, typename _Value, typename _Alloc,
@ -2003,11 +2060,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
// First build the node to get access to the hash code
_Scoped_node __node { this, std::forward<_Args>(__args)... };
const key_type& __k = _ExtractKey{}(__node._M_node->_M_v());
if (size() <= __small_size_threshold())
{
for (auto __it = begin(); __it != end(); ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
// There is already an equivalent node, no insertion
return { __it, false };
}
__hash_code __code = this->_M_hash_code(__k);
size_type __bkt = _M_bucket_index(__code);
if (__node_ptr __p = _M_find_node(__bkt, __k, __code))
// There is already an equivalent node, no insertion
return std::make_pair(iterator(__p), false);
if (size() > __small_size_threshold())
if (__node_ptr __p = _M_find_node(__bkt, __k, __code))
// There is already an equivalent node, no insertion
return { iterator(__p), false };
// Insert the node
auto __pos = _M_insert_unique_node(__bkt, __code, __node._M_node);
@ -2031,13 +2097,41 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
_Scoped_node __node { this, std::forward<_Args>(__args)... };
const key_type& __k = _ExtractKey{}(__node._M_node->_M_v());
__hash_code __code = this->_M_hash_code(__k);
auto __res = this->_M_compute_hash_code(__hint, __k);
auto __pos
= _M_insert_multi_node(__hint._M_cur, __code, __node._M_node);
= _M_insert_multi_node(__res.first._M_cur, __res.second,
__node._M_node);
__node._M_node = nullptr;
return __pos;
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
typename _RehashPolicy, typename _Traits>
auto
_Hashtable<_Key, _Value, _Alloc, _ExtractKey, _Equal,
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits>::
_M_compute_hash_code(const_iterator __hint, const key_type& __k) const
-> pair<const_iterator, __hash_code>
{
if (size() <= __small_size_threshold())
{
if (__hint != cend())
{
for (auto __it = __hint; __it != cend(); ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
return { __it, this->_M_hash_code(*__it._M_cur) };
}
for (auto __it = cbegin(); __it != __hint; ++__it)
if (this->_M_key_equals(__k, *__it._M_cur))
return { __it, this->_M_hash_code(*__it._M_cur) };
}
return { __hint, this->_M_hash_code(__k) };
}
template<typename _Key, typename _Value, typename _Alloc,
typename _ExtractKey, typename _Equal,
typename _Hash, typename _RangeHash, typename _Unused,
@ -2136,11 +2230,17 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
const _NodeGenerator& __node_gen)
-> pair<iterator, bool>
{
if (size() <= __small_size_threshold())
for (auto __it = begin(); __it != end(); ++__it)
if (this->_M_key_equals_tr(__k, *__it._M_cur))
return { __it, false };
__hash_code __code = this->_M_hash_code_tr(__k);
size_type __bkt = _M_bucket_index(__code);
if (__node_ptr __node = _M_find_node_tr(__bkt, __k, __code))
return { iterator(__node), false };
if (size() > __small_size_threshold())
if (__node_ptr __node = _M_find_node_tr(__bkt, __k, __code))
return { iterator(__node), false };
_Scoped_node __node {
__node_builder_t::_S_build(std::forward<_Kt>(__k),
@ -2172,11 +2272,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
_Scoped_node __node{ __node_gen(std::forward<_Arg>(__v)), this };
// Second compute the hash code so that we don't rehash if it throws.
__hash_code __code
= this->_M_hash_code(_ExtractKey{}(__node._M_node->_M_v()));
auto __res = this->_M_compute_hash_code(
__hint, _ExtractKey{}(__node._M_node->_M_v()));
auto __pos
= _M_insert_multi_node(__hint._M_cur, __code, __node._M_node);
= _M_insert_multi_node(__res.first._M_cur, __res.second,
__node._M_node);
__node._M_node = nullptr;
return __pos;
}
@ -2239,16 +2340,33 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
_M_erase(true_type /* __uks */, const key_type& __k)
-> size_type
{
__hash_code __code = this->_M_hash_code(__k);
std::size_t __bkt = _M_bucket_index(__code);
__node_base_ptr __prev_n;
__node_ptr __n;
std::size_t __bkt;
if (size() <= __small_size_threshold())
{
__prev_n = _M_find_before_node(__k);
if (!__prev_n)
return 0;
// Look for the node before the first matching node.
__node_base_ptr __prev_n = _M_find_before_node(__bkt, __k, __code);
if (!__prev_n)
return 0;
// We found a matching node, erase it.
__n = static_cast<__node_ptr>(__prev_n->_M_nxt);
__bkt = _M_bucket_index(*__n);
}
else
{
__hash_code __code = this->_M_hash_code(__k);
__bkt = _M_bucket_index(__code);
// Look for the node before the first matching node.
__prev_n = _M_find_before_node(__bkt, __k, __code);
if (!__prev_n)
return 0;
// We found a matching node, erase it.
__n = static_cast<__node_ptr>(__prev_n->_M_nxt);
}
// We found a matching node, erase it.
__node_ptr __n = static_cast<__node_ptr>(__prev_n->_M_nxt);
_M_erase(__bkt, __prev_n, __n);
return 1;
}
@ -2263,13 +2381,31 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
_M_erase(false_type /* __uks */, const key_type& __k)
-> size_type
{
__hash_code __code = this->_M_hash_code(__k);
std::size_t __bkt = _M_bucket_index(__code);
std::size_t __bkt;
__node_base_ptr __prev_n;
__node_ptr __n;
if (size() <= __small_size_threshold())
{
__prev_n = _M_find_before_node(__k);
if (!__prev_n)
return 0;
// Look for the node before the first matching node.
__node_base_ptr __prev_n = _M_find_before_node(__bkt, __k, __code);
if (!__prev_n)
return 0;
// We found a matching node, erase it.
__n = static_cast<__node_ptr>(__prev_n->_M_nxt);
__bkt = _M_bucket_index(*__n);
}
else
{
__hash_code __code = this->_M_hash_code(__k);
__bkt = _M_bucket_index(__code);
// Look for the node before the first matching node.
__prev_n = _M_find_before_node(__bkt, __k, __code);
if (!__prev_n)
return 0;
__n = static_cast<__node_ptr>(__prev_n->_M_nxt);
}
// _GLIBCXX_RESOLVE_LIB_DEFECTS
// 526. Is it undefined if a function in the standard changes
@ -2277,7 +2413,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
// We use one loop to find all matching nodes and another to deallocate
// them so that the key stays valid during the first loop. It might be
// invalidated indirectly when destroying nodes.
__node_ptr __n = static_cast<__node_ptr>(__prev_n->_M_nxt);
__node_ptr __n_last = __n->_M_next();
while (__n_last && this->_M_node_equals(*__n, *__n_last))
__n_last = __n_last->_M_next();

59
libstdc++-v3/include/bits/hashtable_policy.h
View File

@ -246,6 +246,20 @@ namespace __detail
using __unique_keys = __bool_constant<_Unique_keys>;
};
/**
* struct _Hashtable_hash_traits
*
* Important traits for hash tables depending on associated hasher.
*
*/
template<typename _Hash>
struct _Hashtable_hash_traits
{
static constexpr std::size_t
__small_size_threshold() noexcept
{ return std::__is_fast_hash<_Hash>::value ? 0 : 20; }
};
/**
* struct _Hash_node_base
*
@ -1105,10 +1119,12 @@ namespace __detail
_Hash, _RangeHash, _Unused, _RehashPolicy, _Traits,
true_type /* Has load factor */>
{
private:
using __hashtable = _Hashtable<_Key, _Value, _Alloc, _ExtractKey,
_Equal, _Hash, _RangeHash, _Unused,
_RehashPolicy, _Traits>;
public:
float
max_load_factor() const noexcept
{
@ -1263,6 +1279,14 @@ namespace __detail
const _Hash_node_value<_Value, __cache_hash_code>& __n) const
{ return _M_hash_code(_ExtractKey{}(__n._M_v())); }
__hash_code
_M_hash_code(const _Hash_node_value<_Value, false>& __n) const
{ return _M_hash_code(_ExtractKey{}(__n._M_v())); }
__hash_code
_M_hash_code(const _Hash_node_value<_Value, true>& __n) const
{ return __n._M_hash_code; }
std::size_t
_M_bucket_index(__hash_code __c, std::size_t __bkt_count) const
{ return _RangeHash{}(__c, __bkt_count); }
@ -1641,27 +1665,40 @@ namespace __detail
{ }
bool
_M_equals(const _Key& __k, __hash_code __c,
const _Hash_node_value<_Value, __hash_cached::value>& __n) const
_M_key_equals(const _Key& __k,
const _Hash_node_value<_Value,
__hash_cached::value>& __n) const
{
static_assert(__is_invocable<const _Equal&, const _Key&, const _Key&>{},
"key equality predicate must be invocable with two arguments of "
"key type");
return _S_equals(__c, __n) && _M_eq()(__k, _ExtractKey{}(__n._M_v()));
return _M_eq()(__k, _ExtractKey{}(__n._M_v()));
}
template<typename _Kt>
bool
_M_key_equals_tr(const _Kt& __k,
const _Hash_node_value<_Value,
__hash_cached::value>& __n) const
{
static_assert(
__is_invocable<const _Equal&, const _Kt&, const _Key&>{},
"key equality predicate must be invocable with two arguments of "
"key type");
return _M_eq()(__k, _ExtractKey{}(__n._M_v()));
}
bool
_M_equals(const _Key& __k, __hash_code __c,
const _Hash_node_value<_Value, __hash_cached::value>& __n) const
{ return _S_equals(__c, __n) && _M_key_equals(__k, __n); }
template<typename _Kt>
bool
_M_equals_tr(const _Kt& __k, __hash_code __c,
const _Hash_node_value<_Value,
__hash_cached::value>& __n) const
{
static_assert(
__is_invocable<const _Equal&, const _Kt&, const _Key&>{},
"key equality predicate must be invocable with two arguments of "
"key type");
return _S_equals(__c, __n) && _M_eq()(__k, _ExtractKey{}(__n._M_v()));
}
{ return _S_equals(__c, __n) && _M_key_equals_tr(__k, __n); }
bool
_M_node_equals(
@ -1669,7 +1706,7 @@ namespace __detail
const _Hash_node_value<_Value, __hash_cached::value>& __rhn) const
{
return _S_node_equals(__lhn, __rhn)
&& _M_eq()(_ExtractKey{}(__lhn._M_v()), _ExtractKey{}(__rhn._M_v()));
&& _M_key_equals(_ExtractKey{}(__lhn._M_v()), __rhn);
}
void

1
libstdc++-v3/src/c++11/hashtable_c++0x.cc
View File

@ -30,6 +30,7 @@
#include <tuple>
#include <ext/aligned_buffer.h>
#include <ext/alloc_traits.h>
#include <bits/functional_hash.h>
#include <bits/hashtable_policy.h>
namespace std _GLIBCXX_VISIBILITY(default)

125
libstdc++-v3/testsuite/performance/23_containers/insert_erase/unordered_small_size.cc Normal file
View File

@ -0,0 +1,125 @@
// { dg-do run { target c++11 } }
// Copyright (C) 2021 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library. This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along
// with this library; see the file COPYING3. If not see
// <http://www.gnu.org/licenses/>.
#include <string>
#include <sstream>
#include <vector>
#include <unordered_set>
#include <testsuite_performance.h>
namespace
{
const int nb_elements = 20;
const int nb_insts = 150000;
template<typename _ElemType>
void bench(const char* desc, const std::vector<_ElemType>& elems)
{
using namespace __gnu_test;
time_counter time;
resource_counter resource;
std::vector<std::unordered_set<_ElemType>> insts(nb_insts);
for (int j = 0; j != nb_insts; ++j)
insts.emplace_back();
start_counters(time, resource);
for (auto& us : insts)
for (int i = 0; i != nb_elements; ++i)
us.insert(elems[i]);
stop_counters(time, resource);
std::ostringstream ostr;
ostr << desc << " 1st insert";
report_performance(__FILE__, ostr.str().c_str(), time, resource);
start_counters(time, resource);
for (auto& us : insts)
for (int i = nb_elements - 1; i >= 0; --i)
{
auto it = us.find(elems[i]);
if (it != us.end())
us.erase(it);
}
stop_counters(time, resource);
ostr.str("");
ostr << desc << " find/erase";
report_performance(__FILE__, ostr.str().c_str(), time, resource);
start_counters(time, resource);
for (auto& us : insts)
{
us.insert(elems[0]);
for (int i = nb_elements - 1; i >= 0; --i)
us.insert(elems[i]);
}
stop_counters(time, resource);
ostr.str("");
ostr << desc << " 2nd insert";
report_performance(__FILE__, ostr.str().c_str(), time, resource);
start_counters(time, resource);
for (auto& us : insts)
for (int j = nb_elements - 1; j >= 0; --j)
us.erase(elems[j]);
stop_counters(time, resource);
ostr.str("");
ostr << desc << " erase key ";
report_performance(__FILE__, ostr.str().c_str(), time, resource);
}
}
int main()
{
{
std::vector<int> elems;
elems.reserve(nb_elements);
for (int i = 0; i != nb_elements; ++i)
elems.push_back(i);
bench("std::unordered_set<int>: ", elems);
}
{
std::vector<std::string> elems;
{
elems.reserve(nb_elements);
for (int i = 0; i != nb_elements; ++i)
{
std::ostringstream ostr;
ostr << "string #" << i << ' ' << std::string(1000, 'a' + i);
elems.push_back(ostr.str());
}
}
bench("std::unordered_set<string>: ", elems);
}
return 0;
}

2
libstdc++-v3/testsuite/util/testsuite_performance.h
View File

@ -239,7 +239,7 @@ namespace __gnu_test
out << std::setw(4) << t.real_time() << "r" << space;
out << std::setw(4) << t.user_time() << "u" << space;
out << std::setw(4) << t.system_time() << "s" << space;
out << std::setw(8) << r.allocated_memory() << "mem" << space;
out << std::setw(9) << r.allocated_memory() << "mem" << space;
out << std::setw(4) << r.hard_page_fault() << "pf" << space;
out << std::endl;