File 1041-erts-Refactor-internal-term-hashing.patch of Package erlang
From ef0d257501d7a9687e0a3e1be1f37b1ac4334b7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Tue, 16 May 2023 15:30:02 +0200
Subject: [PATCH] erts: Refactor internal term hashing
Improve performance and hash quality by using MurmurHash3 instead
of our hand-rolled Jenkins96 derivative, and reduce the chances of
needing collision nodes in maps by making the hash width follow the
word size, allowing an extra 8 levels of collision before that
happens on 64-bit platforms.
---
erts/emulator/beam/beam_common.c | 8 +-
erts/emulator/beam/beam_common.h | 2 +-
erts/emulator/beam/emu/map_instrs.tab | 2 +-
erts/emulator/beam/emu/ops.tab | 4 +-
erts/emulator/beam/erl_bif_info.c | 2 +-
erts/emulator/beam/erl_bif_persistent.c | 2 +-
erts/emulator/beam/erl_bits.c | 14 +-
erts/emulator/beam/erl_bits.h | 4 +-
erts/emulator/beam/erl_db_hash.c | 2 +-
erts/emulator/beam/erl_map.c | 229 +++--
erts/emulator/beam/erl_map.h | 41 +-
erts/emulator/beam/erl_nif.c | 2 +-
erts/emulator/beam/erl_proc_sig_queue.c | 2 +-
erts/emulator/beam/erl_process_dict.c | 18 +-
erts/emulator/beam/erl_process_dict.h | 7 +-
erts/emulator/beam/erl_term_hashing.c | 794 +++++++++++-------
erts/emulator/beam/erl_term_hashing.h | 30 +-
erts/emulator/beam/erl_trace.c | 2 +-
erts/emulator/beam/global.h | 6 +-
.../beam/jit/arm/beam_asm_global.hpp.pl | 2 +-
erts/emulator/beam/jit/arm/instr_map.cpp | 89 +-
erts/emulator/beam/jit/arm/ops.tab | 4 +-
erts/emulator/beam/jit/beam_jit_common.cpp | 2 +-
.../beam/jit/x86/beam_asm_global.hpp.pl | 2 +-
erts/emulator/beam/jit/x86/instr_map.cpp | 116 ++-
erts/emulator/beam/jit/x86/ops.tab | 4 +-
erts/emulator/test/map_SUITE.erl | 230 ++++-
erts/emulator/test/nif_SUITE.erl | 2 +-
erts/emulator/test/persistent_term_SUITE.erl | 248 +++---
29 files changed, 1163 insertions(+), 707 deletions(-)
diff --git a/erts/emulator/beam/beam_common.c b/erts/emulator/beam/beam_common.c
index 6eb97fc864..0cd6a5a5db 100644
--- a/erts/emulator/beam/beam_common.c
+++ b/erts/emulator/beam/beam_common.c
@@ -1877,7 +1877,7 @@ is_function2(Eterm Term, Uint arity)
Eterm get_map_element(Eterm map, Eterm key)
{
- Uint32 hx;
+ erts_ihash_t hx;
const Eterm *vs;
if (is_flatmap(map)) {
flatmap_t *mp;
@@ -1910,7 +1910,7 @@ Eterm get_map_element(Eterm map, Eterm key)
return vs ? *vs : THE_NON_VALUE;
}
-Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx)
+Eterm get_map_element_hash(Eterm map, Eterm key, erts_ihash_t hx)
{
const Eterm *vs;
@@ -2082,7 +2082,7 @@ erts_gc_update_map_assoc(Process* p, Eterm* reg, Uint live,
map = reg[live];
if (is_not_flatmap(map)) {
- Uint32 hx;
+ erts_ihash_t hx;
Eterm val;
ASSERT(is_hashmap(map));
@@ -2325,7 +2325,7 @@ erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live,
map = reg[live];
if (is_not_flatmap(map)) {
- Uint32 hx;
+ erts_ihash_t hx;
Eterm val;
/* apparently the compiler does not emit is_map instructions,
diff --git a/erts/emulator/beam/beam_common.h b/erts/emulator/beam/beam_common.h
index 0349d488ac..ee7a41c9b8 100644
--- a/erts/emulator/beam/beam_common.h
+++ b/erts/emulator/beam/beam_common.h
@@ -268,7 +268,7 @@ Eterm erts_gc_update_map_assoc(Process* p, Eterm* reg, Uint live,
Eterm erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live,
Uint n, const Eterm* data);
Eterm get_map_element(Eterm map, Eterm key);
-Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx);
+Eterm get_map_element_hash(Eterm map, Eterm key, erts_ihash_t hx);
int raw_raise(Eterm stacktrace, Eterm exc_class, Eterm value, Process *c_p);
void erts_sanitize_freason(Process* c_p, Eterm exc);
Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc);
diff --git a/erts/emulator/beam/emu/map_instrs.tab b/erts/emulator/beam/emu/map_instrs.tab
index 9cd64662f8..9afc8a2226 100644
--- a/erts/emulator/beam/emu/map_instrs.tab
+++ b/erts/emulator/beam/emu/map_instrs.tab
@@ -104,7 +104,7 @@ i_get_map_elements(Fail, Src, N) {
$FAIL($Fail);
} else {
const Eterm *v;
- Uint32 hx;
+ erts_ihash_t hx;
ASSERT(is_hashmap(map));
while(n--) {
hx = fs[2];
diff --git a/erts/emulator/beam/emu/ops.tab b/erts/emulator/beam/emu/ops.tab
index 7c4447d145..0615f32382 100644
--- a/erts/emulator/beam/emu/ops.tab
+++ b/erts/emulator/beam/emu/ops.tab
@@ -1013,7 +1013,7 @@ bif1 Fail=f Bif S1 Dst => i_bif1 S1 Fail Bif Dst
bif2 p Bif S1 S2 Dst => i_bif2_body S2 S1 Bif Dst
bif2 Fail=f Bif S1 S2 Dst => i_bif2 S2 S1 Fail Bif Dst
-i_get_hash c I d
+i_get_hash c W d
i_get s d
self xy
@@ -1573,7 +1573,7 @@ i_get_map_elements f? s I *
i_get_map_element_hash Fail Src=c Key Hash Dst =>
move Src x | i_get_map_element_hash Fail x Key Hash Dst
-i_get_map_element_hash f? xy c I xy
+i_get_map_element_hash f? xy c W xy
i_get_map_element Fail Src=c Key Dst =>
move Src x | i_get_map_element Fail x Key Dst
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 4ba368ec7c..07265c1402 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4527,7 +4527,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups));
}
else if (ERTS_IS_ATOM_STR("internal_hash", tp[1])) {
- Uint hash = (Uint) make_internal_hash(tp[2], 0);
+ Uint hash = (Uint) erts_internal_hash(tp[2]);
Uint hsz = 0;
Eterm* hp;
erts_bld_uint(NULL, &hsz, hash);
diff --git a/erts/emulator/beam/erl_bif_persistent.c b/erts/emulator/beam/erl_bif_persistent.c
index 08e15f55d6..e811e96e49 100644
--- a/erts/emulator/beam/erl_bif_persistent.c
+++ b/erts/emulator/beam/erl_bif_persistent.c
@@ -971,8 +971,8 @@ cleanup_trap_data(Binary *bp)
static Uint
lookup(HashTable* hash_table, Eterm key, Eterm *bucket)
{
+ erts_ihash_t idx = erts_internal_hash(key);
Uint mask = hash_table->mask;
- Uint32 idx = make_internal_hash(key, 0);
Eterm term;
while (1) {
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 8b6698b6ae..13e5107127 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -2162,13 +2162,13 @@ erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t siz
void
-erts_copy_bits(byte* src, /* Base pointer to source. */
- size_t soffs, /* Bit offset for source relative to src. */
- int sdir, /* Direction: 1 (forward) or -1 (backward). */
- byte* dst, /* Base pointer to destination. */
- size_t doffs, /* Bit offset for destination relative to dst. */
- int ddir, /* Direction: 1 (forward) or -1 (backward). */
- size_t n) /* Number of bits to copy. */
+erts_copy_bits(const byte* src, /* Base pointer to source. */
+ size_t soffs, /* Bit offset for source relative to src. */
+ int sdir, /* Direction: 1 (forward) or -1 (backward). */
+ byte* dst, /* Base pointer to destination. */
+ size_t doffs, /* Bit offset for destination relative to dst. */
+ int ddir, /* Direction: 1 (forward) or -1 (backward). */
+ size_t n) /* Number of bits to copy. */
{
Uint lmask;
Uint rmask;
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 1d95536a68..e7cc6017a3 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -192,8 +192,8 @@ Eterm erts_bs_init_writable(Process* p, Eterm sz);
/*
* Common utilities.
*/
-void erts_copy_bits(byte* src, size_t soffs, int sdir,
- byte* dst, size_t doffs,int ddir, size_t n);
+void erts_copy_bits(const byte* src, size_t soffs, int sdir,
+ byte* dst, size_t doffs, int ddir, size_t n);
int erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t size);
/*
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index fa8a8c15ec..fd61747f17 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -276,7 +276,7 @@ static ERTS_INLINE int is_pseudo_deleted(HashDbTerm* p)
/* optimised version of make_hash (normal case? atomic key) */
#define MAKE_HASH(term) \
((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \
- make_internal_hash(term, 0)) & MAX_HASH_MASK)
+ erts_internal_hash(term)) & MAX_HASH_MASK)
# define GET_LOCK_MASK(NUMBER_OF_LOCKS) ((NUMBER_OF_LOCKS)-1)
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 337567406b..431fcf66e8 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -77,12 +77,14 @@
/* for hashmap_from_list/1 */
typedef struct {
- Uint32 hx;
- Uint32 skip;
+ erts_ihash_t hx;
+ Uint skip;
Uint i;
Eterm val;
} hxnode_t;
+/* Reverses the path element/slot order of `hash` */
+static ERTS_INLINE erts_ihash_t swizzle_map_hash(erts_ihash_t hash);
static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB);
static BIF_RETTYPE map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args);
@@ -92,7 +94,7 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_
static Export hashmap_merge_trap_export;
static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1);
static Uint hashmap_subtree_size(Eterm node);
-static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node, Eterm *value);
+static Eterm hashmap_delete(Process *p, erts_ihash_t hx, Eterm key, Eterm node, Eterm *value);
static Eterm flatmap_from_validated_list(Process *p, Eterm list, Eterm fill_value, Uint size);
static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys, ErtsAlcType_t temp_memory_allocator);
static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, ErtsAlcType_t temp_memory_allocator);
@@ -101,18 +103,12 @@ static Eterm hashmap_info(Process *p, Eterm node);
static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]);
static int hxnodecmp(const void* a, const void* b);
static int hxnodecmpkey(const void* a, const void* b);
-#define swizzle32(D,S) \
- do { \
- (D) = ((S) & 0x0000000f) << 28 | ((S) & 0x000000f0) << 20 \
- | ((S) & 0x00000f00) << 12 | ((S) & 0x0000f000) << 4 \
- | ((S) & 0x000f0000) >> 4 | ((S) & 0x00f00000) >> 12 \
- | ((S) & 0x0f000000) >> 20 | ((S) & 0xf0000000) >> 28; \
- } while(0)
#define cdepth(V1,V2) (hashmap_clz((V1) ^ (V2)) >> 2)
-#define maskval(V,L) (((V) >> ((7 - (L))*4)) & 0xf)
+#define maskval(V,L) (((V) >> (((HAMT_MAX_LEVEL - 1) - (L)) * 4)) & 0xF)
#define DBG_PRINT(X)
/*erts_printf X*/
#define HALLOC_EXTRA 200
+
/* *******************************
* ** Yielding C Fun (YCF) Note **
* *******************************
@@ -191,7 +187,7 @@ erts_map_size(Eterm map)
const Eterm *
erts_maps_get(Eterm key, Eterm map)
{
- Uint32 hx;
+ erts_ihash_t hx;
if (is_flatmap(map)) {
Eterm *ks, *vs;
flatmap_t *mp;
@@ -495,8 +491,8 @@ static Eterm hashmap_from_validated_list(Process *p,
Eterm res;
Eterm key;
Eterm value;
- Uint32 sw;
- Uint32 hx;
+ erts_ihash_t sw;
+ erts_ihash_t hx;
Uint ix = 0;
hxnode_t *hxns;
ErtsHeapFactory *factory;
@@ -528,7 +524,7 @@ static Eterm hashmap_from_validated_list(Process *p,
value = kv[2];
}
hx = hashmap_restore_hash(0,key);
- swizzle32(sw,hx);
+ sw = swizzle_map_hash(hx);
hxns[ix].hx = sw;
hxns[ix].val = CONS(hp, key, value); hp += 2;
hxns[ix].skip = 1; /* will be reassigned in from_array */
@@ -649,7 +645,7 @@ BIF_RETTYPE maps_from_keys_2(BIF_ALIST_2) {
Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n,
int reject_dupkeys) {
- Uint32 sw, hx;
+ erts_ihash_t sw, hx;
Uint ix;
hxnode_t *hxns;
Eterm res;
@@ -659,7 +655,7 @@ Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n,
for (ix = 0; ix < n; ix++) {
hx = hashmap_make_hash(*leafs);
- swizzle32(sw,hx);
+ sw = swizzle_map_hash(hx);
hxns[ix].hx = sw;
hxns[ix].val = make_list(leafs);
hxns[ix].skip = 1;
@@ -734,7 +730,7 @@ Eterm erts_map_from_ks_and_vs(ErtsHeapFactory *factory, Eterm *ks, Eterm *vs, Ui
Eterm erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory,
Eterm *ks, Eterm *vs, Uint n,
Eterm key, Eterm value) {
- Uint32 sw, hx;
+ erts_ihash_t sw, hx;
Uint i,sz;
hxnode_t *hxns;
Eterm *hp, res;
@@ -748,7 +744,7 @@ Eterm erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory,
for(i = 0; i < n; i++) {
hx = hashmap_make_hash(ks[i]);
- swizzle32(sw,hx);
+ sw = swizzle_map_hash(hx);
hxns[i].hx = sw;
hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2;
hxns[i].skip = 1; /* will be reassigned in from_array */
@@ -757,7 +753,7 @@ Eterm erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory,
if (key != THE_NON_VALUE) {
hx = hashmap_make_hash(key);
- swizzle32(sw,hx);
+ sw = swizzle_map_hash(hx);
hxns[i].hx = sw;
hxns[i].val = CONS(hp, key, value); hp += 2;
hxns[i].skip = 1;
@@ -839,23 +835,26 @@ static Eterm hashmap_from_unsorted_array(ErtsHeapFactory* factory,
}
if (cx > 1) {
- /* recursive decompose array */
- res = hashmap_from_sorted_unique_array(factory, hxns, cx,
+ /* recursive decompose array */
+ res = hashmap_from_sorted_unique_array(factory, hxns, cx,
temp_memory_allocator);
} else {
- Eterm *hp;
+ Eterm slot;
+ Eterm *hp;
- /* we only have one item, either because n was 1 or
- * because we hade multiples of the same key.
- *
- * hash value has been swizzled, need to drag it down to get the
- * correct slot. */
+ /* We only have one item, either because n was 1 or because we have
+ * multiples of the same key.
+ *
+ * As the hash value has been swizzled, we need to drag it down to get
+ * the correct slot. */
+ slot = hxns[0].hx >> ((HAMT_MAX_LEVEL - 1) * 4);
+ ASSERT(slot < 16);
- hp = erts_produce_heap(factory, HAMT_HEAD_BITMAP_SZ(1), 0);
- hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << ((hxns[0].hx >> 0x1c) & 0xf));
- hp[1] = 1;
- hp[2] = hxns[0].val;
- res = make_hashmap(hp);
+ hp = erts_produce_heap(factory, HAMT_HEAD_BITMAP_SZ(1), 0);
+ hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << slot);
+ hp[1] = 1;
+ hp[2] = hxns[0].val;
+ res = make_hashmap(hp);
}
return res;
@@ -943,9 +942,9 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns
Uint dc;
Uint slot;
Uint elems;
- Uint32 v;
- Uint32 vp;
- Uint32 vn;
+ erts_ihash_t v;
+ erts_ihash_t vp;
+ erts_ihash_t vn;
Uint32 hdr;
Uint bp;
Uint sz;
@@ -978,7 +977,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns
if (n == 1) {
res = hxns[0].val;
v = hxns[0].hx;
- for (d = 7; d > 0; d--) {
+ for (d = HAMT_MAX_LEVEL-1; d > 0; d--) {
slot = maskval(v,d);
hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << slot);
@@ -1015,7 +1014,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns
res = hxns[ix].val;
if (hxns[ix].skip > 1) {
- dc = 7;
+ dc = HAMT_MAX_LEVEL - 1;
/* build collision nodes */
while (dc > d) {
hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
@@ -1045,7 +1044,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns
if (hxns[ix].skip > 1) {
int wat = (d > dn) ? d : dn;
- dc = 7;
+ dc = HAMT_MAX_LEVEL - 1;
/* build collision nodes */
while (dc > wat) {
hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
@@ -1114,7 +1113,7 @@ static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns
res = hxns[ix].val;
if (hxns[ix].skip > 1) {
- dc = 7;
+ dc = HAMT_MAX_LEVEL - 1;
/* build collision nodes */
while (dc > dn) {
hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA);
@@ -1400,7 +1399,7 @@ static Eterm flatmap_merge(Process *p, Eterm map1, Eterm map2) {
/* Reshape map to a hashmap if the map exceeds the limit */
if (n > MAP_SMALL_MAP_LIMIT) {
- Uint32 hx,sw;
+ erts_ihash_t hx,sw;
Uint i;
Eterm res;
hxnode_t *hxns;
@@ -1415,7 +1414,7 @@ static Eterm flatmap_merge(Process *p, Eterm map1, Eterm map2) {
for (i = 0; i < n; i++) {
hx = hashmap_make_hash(ks[i]);
- swizzle32(sw,hx);
+ sw = swizzle_map_hash(hx);
hxns[i].hx = sw;
hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2;
hxns[i].skip = 1;
@@ -1440,7 +1439,7 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
flatmap_t *mp;
Uint n, i;
hxnode_t *hxns;
- Uint32 sw, hx;
+ erts_ihash_t sw, hx;
ErtsHeapFactory factory;
/* convert flat to tree */
@@ -1461,7 +1460,7 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args)
for (i = 0; i < n; i++) {
hx = hashmap_make_hash(ks[i]);
- swizzle32(sw,hx);
+ sw = swizzle_map_hash(hx);
hxns[i].hx = sw;
hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2;
hxns[i].skip = 1;
@@ -1574,7 +1573,7 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm map_A, Eterm map_B,
PSTACK_DECLARE(s, 4);
HashmapMergeContext local_ctx;
struct HashmapMergePStackType* sp;
- Uint32 hx;
+ erts_ihash_t hx;
Eterm res = THE_NON_VALUE;
Eterm hdrA, hdrB;
Eterm *hp, *nhp;
@@ -1875,29 +1874,35 @@ static Uint hashmap_subtree_size(Eterm node) {
return size;
}
-
-static int hash_cmp(Uint32 ha, Uint32 hb)
+static int hash_cmp(erts_ihash_t ha, erts_ihash_t hb)
{
- int i;
- for (i=0; i<8; i++) {
- int cmp = (int)(ha & 0xF) - (int)(hb & 0xF);
- if (cmp)
- return cmp;
- ha >>= 4;
- hb >>= 4;
+ for (int i = 0; i < HAMT_MAX_LEVEL; i++) {
+ int cmp = (int)(ha & 0xF) - (int)(hb & 0xF);
+
+ if (cmp) {
+ return cmp;
+ }
+
+ ha >>= 4;
+ hb >>= 4;
}
+
return 0;
}
int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp)
{
if (ap && bp) {
- Uint32 ha, hb;
- ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0);
+ erts_ihash_t ha, hb;
+
+ ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0);
+
ha = hashmap_make_hash(CAR(ap));
hb = hashmap_make_hash(CAR(bp));
+
return hash_cmp(ha, hb);
}
+
ASSERT(ap || bp);
return ap ? -1 : 1;
}
@@ -1949,7 +1954,7 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) {
*/
int erts_maps_take(Process *p, Eterm key, Eterm map,
Eterm *res, Eterm *value) {
- Uint32 hx;
+ erts_ihash_t hx;
Eterm ret;
if (is_flatmap(map)) {
Sint n;
@@ -2041,7 +2046,7 @@ found_key:
}
int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) {
- Uint32 hx;
+ erts_ihash_t hx;
if (is_flatmap(map)) {
Sint n,i;
Eterm* hp,*shp;
@@ -2110,7 +2115,7 @@ found_key:
}
Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
- Uint32 hx;
+ erts_ihash_t hx;
Eterm res;
if (is_flatmap(map)) {
Sint n,i;
@@ -2427,7 +2432,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) {
}
const Eterm *
-erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
+erts_hashmap_get(erts_ihash_t hx, Eterm key, Eterm node)
{
Eterm *ptr, hdr;
Uint ix, lvl = 0;
@@ -2469,6 +2474,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
} while (!is_arity_value(hdr));
/* collision node */
+ ASSERT(lvl == HAMT_MAX_LEVEL);
ix = arityval(hdr);
ASSERT(ix > 1);
do {
@@ -2479,7 +2485,7 @@ erts_hashmap_get(Uint32 hx, Eterm key, Eterm node)
return NULL;
}
-Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
+Eterm erts_hashmap_insert(Process *p, erts_ihash_t hx, Eterm key, Eterm value,
Eterm map, int is_update) {
Uint size, upsz;
Eterm *hp, res = THE_NON_VALUE;
@@ -2508,13 +2514,14 @@ Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
}
-int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm value, Eterm node, Uint *sz,
+int erts_hashmap_insert_down(erts_ihash_t hx, Eterm key, Eterm value, Eterm node, Uint *sz,
Uint *update_size, ErtsEStack *sp, int is_update) {
Eterm *ptr;
Eterm hdr, ckey;
- Uint32 ix, cix, bp, hval, chx;
+ Uint32 ix, cix, bp, hval;
Uint slot, lvl = 0, clvl;
Uint size = 0, n = 0;
+ erts_ihash_t chx;
*update_size = 1;
@@ -2881,7 +2888,7 @@ static Eterm hashmap_values(Process* p, Eterm node) {
}
#endif /* INCLUDE_YCF_TRANSFORMED_ONLY_FUNCTIONS */
-static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key,
+static Eterm hashmap_delete(Process *p, erts_ihash_t hx, Eterm key,
Eterm map, Eterm *value) {
Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL;
Eterm *ptr;
@@ -2961,6 +2968,7 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key,
goto not_found;
default: /* collision node */
ERTS_ASSERT(is_arity_value(hdr));
+ ASSERT(lvl == HAMT_MAX_LEVEL);
n = arityval(hdr);
ASSERT(n >= 2);
for (slot = 0; slot < n; slot++) {
@@ -3521,6 +3529,7 @@ static Eterm hashmap_info(Process *p, Eterm node) {
break;
default: /* collision node */
ERTS_ASSERT(is_arity_value(hdr));
+ ASSERT(clvl == HAMT_MAX_LEVEL);
ncollision++;
sz = arityval(hdr);
ASSERT(sz >= 2);
@@ -4025,30 +4034,86 @@ BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) {
}
}
-/* implementation of builtin emulations */
+/* Implementation of builtin emulations */
+
+#if defined(ARCH_64) && (ERTS_AT_LEAST_GCC_VSN__(5, 1, 0) || \
+ __has_builtin(__builtin_bswap64))
+# define hashmap_byte_swap(N) __builtin_bswap64((Uint64)(N))
+#elif defined(ARCH_32) && (ERTS_AT_LEAST_GCC_VSN__(5, 1, 0) || \
+ __has_builtin(__builtin_bswap32))
+# define hashmap_byte_swap(N) __builtin_bswap32((Uint32)(N))
+#elif defined(_MSC_VER) && _MSC_VER >= 1900
+/* UCRT intrinsics are spread throughout the ordinary C headers, strangely
+ * enough. */
+# include <stdlib.h>
+
+# if defined(ARCH_64)
+# define hashmap_byte_swap(N) _byteswap_uint64((Uint64)(N))
+# elif defined(ARCH_32)
+# define hashmap_byte_swap(N) _byteswap_ulong((Uint32)(N))
+# endif
+#else
+/* No byte-swap intrinsic available. Fall back to C and hope that the compiler
+ * turns it into something efficient. */
+static ERTS_INLINE erts_ihash_t hashmap_byte_swap(erts_ihash_t hash) {
+ erts_ihash_t result = 0;
+
+ for (int i = 0; i < sizeof(hash); i++) {
+ ERTS_CT_ASSERT(CHAR_BIT == 8);
+ result |= (((hash) >> i * 8) & 0xFF) << ((sizeof(hash) - i - 1) * 8);
+ }
+
+ return result;
+}
+#endif
+
+static ERTS_INLINE erts_ihash_t swizzle_map_hash(erts_ihash_t hash) {
+ const erts_ihash_t mask = (erts_ihash_t)0xF0F0F0F0F0F0F0F0ull;
+ erts_ihash_t result;
+
+ /* ABCDEFGH -> GHEFCDAB */
+ result = hashmap_byte_swap(hash);
+
+ /* GHEFCDAB -> HGFEDCBA */
+ return ((result & mask)) >> 4 | ((result & (mask >> 4)) << 4);
+}
-#if !ERTS_AT_LEAST_GCC_VSN__(3, 4, 0)
/* Count leading zeros emulation */
-Uint32 hashmap_clz(Uint32 x) {
- Uint32 y;
+#ifndef hashmap_clz
+erts_ihash_t hashmap_clz(erts_ihash_t x) {
+ erts_ihash_t y;
+
+#if defined(ARCH_64)
+ int n = 64;
+
+ y = x >> 32; if (y != 0) { n = n - 32; x = y; }
+#elif defined(ARCH_32)
int n = 32;
- y = x >>16; if (y != 0) {n = n -16; x = y;}
- y = x >> 8; if (y != 0) {n = n - 8; x = y;}
- y = x >> 4; if (y != 0) {n = n - 4; x = y;}
- y = x >> 2; if (y != 0) {n = n - 2; x = y;}
- y = x >> 1; if (y != 0) return n - 2;
+#endif
+
+ y = x >> 16; if (y != 0) { n = n - 16; x = y; }
+ y = x >> 8; if (y != 0) { n = n - 8; x = y; }
+ y = x >> 4; if (y != 0) { n = n - 4; x = y; }
+ y = x >> 2; if (y != 0) { n = n - 2; x = y; }
+ y = x >> 1; if (y != 0) { return n - 2; }
+
return n - x;
}
-
-const Uint32 SK5 = 0x55555555, SK3 = 0x33333333;
-const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF;
+#endif
/* CTPOP emulation */
-Uint32 hashmap_bitcount(Uint32 x) {
- x -= ((x >> 1 ) & SK5);
- x = (x & SK3 ) + ((x >> 2 ) & SK3 );
- x = (x & SKF0) + ((x >> 4 ) & SKF0);
- x += x >> 8;
- return (x + (x >> 16)) & 0x3F;
+#ifndef hashmap_bitcount
+erts_ihash_t hashmap_bitcount(erts_ihash_t x) {
+ const erts_ihash_t SK55 = (erts_ihash_t)0x5555555555555555ull;
+ const erts_ihash_t SK33 = (erts_ihash_t)0x3333333333333333ull;
+ const erts_ihash_t SK0F = (erts_ihash_t)0x0F0F0F0F0F0F0F0Full;
+ const erts_ihash_t SK01 = (erts_ihash_t)0x0101010101010101ull;
+
+ x -= ((x >> 1) & SK55);
+ x = (x & SK33) + ((x >> 2) & SK33);
+ x = ((x + (x >> 4)) & SK0F);
+ x *= SK01;
+
+ return x >> (sizeof(erts_ihash_t) - 1) * CHAR_BIT;
}
#endif
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index f046ae2093..27415a639e 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -23,14 +23,31 @@
#define __ERL_MAP_H__
#include "sys.h"
+#include "erl_term_hashing.h"
/* intrinsic wrappers */
-#if ERTS_AT_LEAST_GCC_VSN__(3, 4, 0)
-#define hashmap_clz(x) ((Uint32) __builtin_clz((unsigned int)(x)))
-#define hashmap_bitcount(x) ((Uint32) __builtin_popcount((unsigned int) (x)))
+#if ERTS_AT_LEAST_GCC_VSN__(3, 4, 0) || __has_builtin(__builtin_clz)
+# if defined(ARCH_64)
+# define hashmap_clz(x) \
+ ((erts_ihash_t)__builtin_clzl((erts_ihash_t)(x)))
+# elif defined(ARCH_32)
+# define hashmap_clz(x) \
+ ((erts_ihash_t)__builtin_clz((erts_ihash_t)(x)))
+# endif
#else
-Uint32 hashmap_clz(Uint32 x);
-Uint32 hashmap_bitcount(Uint32 x);
+erts_ihash_t hashmap_clz(erts_ihash_t x);
+#endif
+
+#if ERTS_AT_LEAST_GCC_VSN__(3, 4, 0) || __has_builtin(__builtin_popcount)
+# if defined(ARCH_64)
+# define hashmap_bitcount(x) \
+ ((erts_ihash_t)__builtin_popcountl((erts_ihash_t)(x)))
+# elif defined(ARCH_32)
+# define hashmap_bitcount(x) \
+ ((erts_ihash_t)__builtin_popcount((erts_ihash_t)(x)))
+# endif
+#else
+erts_ihash_t hashmap_bitcount(erts_ihash_t x);
#endif
/* MAP */
@@ -56,10 +73,10 @@ typedef struct flatmap_s {
/* the head-node is a bitmap or array with an untagged size */
#define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size)
-#define hashmap_make_hash(Key) make_map_hash(Key)
+#define hashmap_make_hash(Key) erts_map_hash(Key)
#define hashmap_restore_hash(Lvl, Key) \
- (ASSERT(Lvl < 8), \
+ (ASSERT(Lvl < HAMT_MAX_LEVEL), \
hashmap_make_hash(Key) >> (4*(Lvl)))
#define hashmap_shift_hash(Hx, Lvl, Key) \
@@ -85,9 +102,9 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *re
int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);
int erts_maps_take(Process *p, Eterm key, Eterm map, Eterm *res, Eterm *value);
-Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
+Eterm erts_hashmap_insert(Process *p, erts_ihash_t hx, Eterm key, Eterm value,
Eterm node, int is_update);
-int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm value, Eterm node, Uint *sz,
+int erts_hashmap_insert_down(erts_ihash_t hx, Eterm key, Eterm value, Eterm node, Uint *sz,
Uint *upsz, struct ErtsEStack_ *sp, int is_update);
Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
Uint upsz, struct ErtsEStack_ *sp);
@@ -110,7 +127,7 @@ Eterm erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory,
const Eterm *erts_maps_get(Eterm key, Eterm map);
-const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
+const Eterm *erts_hashmap_get(erts_ihash_t hx, Eterm key, Eterm map);
Sint erts_map_size(Eterm map);
@@ -191,9 +208,9 @@ typedef struct hashmap_head_s {
#define HAMT_SUBTAG_HEAD_BITMAP ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG)
#define HAMT_SUBTAG_HEAD_FLATMAP ((MAP_HEADER_TAG_FLATMAP_HEAD << _HEADER_ARITY_OFFS) | MAP_SUBTAG)
-#define hashmap_index(hash) (((Uint32)hash) & 0xf)
+#define hashmap_index(hash) ((hash) & 0xf)
-#define HAMT_MAX_LEVEL 8
+#define HAMT_MAX_LEVEL ((sizeof(erts_ihash_t) * CHAR_BIT) / 4)
/* hashmap heap size:
[one cons cell + one list term in parent node] per key
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index a9661dc780..119138eaab 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1535,7 +1535,7 @@ ErlNifUInt64 enif_hash(ErlNifHash type, Eterm term, ErlNifUInt64 salt)
{
switch (type) {
case ERL_NIF_INTERNAL_HASH:
- return make_internal_hash(term, (Uint32) salt);
+ return erts_internal_salted_hash(term, (erts_ihash_t)salt);
case ERL_NIF_PHASH2:
/* It appears that make_hash2 doesn't always react to seasoning
* as well as it should. Therefore, let's make it ignore the salt
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index ee1d345fb3..d6fc9bcfd4 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -8686,7 +8686,7 @@ erts_proc_sig_queue_try_enqueue_to_buffer(Eterm from,
/* Use the sender id to hash to an outer signal queue buffer. This
* guarantees that all signals from the same process are ordered in
* send order. */
- slot = make_internal_hash(from, 0) %
+ slot = erts_internal_hash(from) %
ERTS_PROC_SIG_INQ_BUFFERED_NR_OF_BUFFERS;
buffer = &buffers->slots[slot];
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 4ffaf35edc..b2027fe207 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -53,11 +53,11 @@
/* Hash utility macros */
#define HASH_RANGE(PDict) ((PDict)->usedSlots)
-#define MAKE_HASH(Term) \
- ((is_small(Term)) ? (Uint32) unsigned_val(Term) : \
- ((is_atom(Term)) ? \
- (Uint32) atom_val(Term) : \
- make_internal_hash(Term, 0)))
+#define MAKE_HASH(Term) \
+ ((is_small(Term)) ? (erts_ihash_t) unsigned_val(Term) : \
+ ((is_atom(Term)) ? \
+ (erts_ihash_t) atom_val(Term) : \
+ erts_internal_hash(Term)))
#define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm))
@@ -103,7 +103,7 @@ static void grow(Process *p);
static void array_shrink(ProcDict **ppd, unsigned int need);
static void ensure_array_size(ProcDict**, unsigned int size);
-static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx);
+static unsigned int pd_hash_value_to_ix(ProcDict *pdict, erts_ihash_t hx);
static unsigned int next_array_size(unsigned int need);
/*
@@ -441,12 +441,12 @@ static void pd_hash_erase_all(Process *p)
}
}
-Uint32 erts_pd_make_hx(Eterm key)
+erts_ihash_t erts_pd_make_hx(Eterm key)
{
return MAKE_HASH(key);
}
-Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id)
+Eterm erts_pd_hash_get_with_hx(Process *p, erts_ihash_t hx, Eterm id)
{
unsigned int hval;
ProcDict *pd = p->dictionary;
@@ -1003,7 +1003,7 @@ static void ensure_array_size(ProcDict **ppdict, unsigned int size)
** Basic utilities
*/
-static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx)
+static unsigned int pd_hash_value_to_ix(ProcDict *pdict, erts_ihash_t hx)
{
Uint high;
diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h
index 3ff2354f91..a79de56faf 100644
--- a/erts/emulator/beam/erl_process_dict.h
+++ b/erts/emulator/beam/erl_process_dict.h
@@ -20,7 +20,9 @@
#ifndef _ERL_PROCESS_DICT_H
#define _ERL_PROCESS_DICT_H
+
#include "sys.h"
+#include "erl_term_hashing.h"
typedef struct proc_dict {
unsigned int sizeMask;
@@ -43,7 +45,8 @@ void erts_deep_dictionary_dump(fmtfn_t to, void *to_arg,
Eterm erts_dictionary_copy(ErtsHeapFactory *hfact, ProcDict *pd, Uint reserve_size);
Eterm erts_pd_hash_get(struct process *p, Eterm id);
-Uint32 erts_pd_make_hx(Eterm key);
-Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id);
+
+erts_ihash_t erts_pd_make_hx(Eterm key);
+Eterm erts_pd_hash_get_with_hx(Process *p, erts_ihash_t hx, Eterm id);
#endif
diff --git a/erts/emulator/beam/erl_term_hashing.c b/erts/emulator/beam/erl_term_hashing.c
index 848757c2f2..e13c5e9773 100644
--- a/erts/emulator/beam/erl_term_hashing.c
+++ b/erts/emulator/beam/erl_term_hashing.c
@@ -32,14 +32,6 @@
#include "erl_binary.h"
#include "erl_bits.h"
-#ifdef ERL_INTERNAL_HASH_CRC32C
-# if defined(__x86_64__)
-# include <immintrin.h>
-# elif defined(__aarch64__)
-# include <arm_acle.h>
-# endif
-#endif
-
/* *\
* *
\* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -1532,6 +1524,16 @@ make_hash2_helper(Eterm term_param, const int can_trap, Eterm* state_mref_write_
#undef TRAP_LOCATION_NO_CTX
}
+#undef HASH_MAP_TAIL
+#undef HASH_MAP_PAIR
+
+#undef UINT32_HASH_2
+#undef UINT32_HASH
+#undef SINT32_HASH
+
+#undef HCONST
+#undef MIX
+
Uint32
make_hash2(Eterm term)
{
@@ -1559,124 +1561,256 @@ trapping_make_hash2(Eterm term, Eterm* state_mref_write_back, Process* p)
* hashmap implementation that now uses collision nodes at the bottom of
* the HAMT when all hash bits are exhausted.
*
- */
+ * The underlying hash primitive is the public-domain `MurmurHash3` by Austin
+ * Appleby, which has been modified to work incrementally over our terms rather
+ * than plain byte arrays. It provides a decent 128-bit hash with good
+ * performance on most hardware, only narrowly losing to variants that use
+ * specialized instructions (e.g. SHA3 or AES) that are much harder to
+ * maintain.
+ *
+ * Note that we only implement the 64-bit variant of MurmurHash and skip the
+ * 32-bit optimized version, as the difference in performance appears to be
+ * modest on the most popular 32-bit platform (ARM). It should not be terribly
+ * difficult to adapt this for both versions if that becomes a problem. */
+
+enum {
+ IHASH_TYPE_IMMEDIATE = 1,
+ IHASH_TYPE_ARRAY_ELEMENT,
+ IHASH_TYPE_CAR,
+ IHASH_TYPE_CDR,
+ IHASH_TYPE_STRING,
+ IHASH_TYPE_TUPLE,
+ IHASH_TYPE_FLATMAP,
+ IHASH_TYPE_HASHMAP_HEAD_ARRAY,
+ IHASH_TYPE_HASHMAP_HEAD_BITMAP,
+ IHASH_TYPE_HASHMAP_NODE,
+ IHASH_TYPE_BINARY,
+ IHASH_TYPE_LOCAL_FUN,
+ IHASH_TYPE_EXTERNAL_FUN,
+ IHASH_TYPE_NEG_BIGNUM,
+ IHASH_TYPE_POS_BIGNUM,
+ IHASH_TYPE_LOCAL_REF,
+ IHASH_TYPE_EXTERNAL_REF,
+ IHASH_TYPE_EXTERNAL_PID,
+ IHASH_TYPE_EXTERNAL_PORT,
+ IHASH_TYPE_FLOAT
+};
+
+#define IHASH_CAR_MARKER (_make_header(1,_TAG_HEADER_REF))
+#define IHASH_CDR_MARKER (_make_header(2,_TAG_HEADER_REF))
+
+#define ROTL64(x, y) (x << y) | (x >> (64 - y));
+
+static const Uint64 IHASH_C1 = 0x87C37B91114253D5ull;
+static const Uint64 IHASH_C2 = 0x4CF5AD432745937Full;
+
+#define IHASH_MIX_ALPHA(Expr) \
+ do { \
+ Uint64 expr = (Uint64)(Expr); \
+ expr *= IHASH_C1; \
+ expr = ROTL64(expr, 31); \
+ expr *= IHASH_C2; \
+ hash_alpha ^= expr; \
+ hash_alpha = ROTL64(hash_alpha, 27) \
+ hash_alpha += hash_beta; \
+ hash_alpha = hash_alpha * 5 + 0x52DCE729ull; \
+ hash_ticks += 1; \
+ } while(0)
+
+#define IHASH_MIX_ALPHA_2F32(Expr1, Expr2) \
+ IHASH_MIX_ALPHA((Uint64)(Expr1) | ((Uint64)(Expr2) << 32))
+
+#define IHASH_MIX_BETA(Expr) \
+ do { \
+ Uint64 expr = (Uint64)(Expr); \
+ expr *= IHASH_C2; \
+ expr = ROTL64(expr, 33); \
+ expr *= IHASH_C1; \
+ hash_beta ^= expr; \
+ hash_beta = ROTL64(hash_beta, 31); \
+ hash_beta += hash_alpha; \
+ hash_beta = hash_beta * 5 + 0x38495AB5ull; \
+ hash_ticks += 1; \
+ } while(0)
+
+#define IHASH_MIX_BETA_2F32(Expr1, Expr2) \
+ IHASH_MIX_BETA((Uint64)(Expr1) | ((Uint64)(Expr2) << 32))
+
+#ifdef ARCH_64
+# define IHASH_MIX_IMMEDIATE(term) \
+ do { \
+ IHASH_MIX_ALPHA(IHASH_TYPE_IMMEDIATE); \
+ IHASH_MIX_BETA(term); \
+ } while(0)
+#else
+# define IHASH_MIX_IMMEDIATE(term) \
+ IHASH_MIX_ALPHA_2F32(IHASH_TYPE_IMMEDIATE, term);
+#endif
-/* Use a better mixing function if available. */
-#if defined(ERL_INTERNAL_HASH_CRC32C)
-# undef MIX
-# if defined(__x86_64__)
-# define MIX(a,b,c) \
- do { \
- Uint32 initial_hash = c; \
- c = __builtin_ia32_crc32si(c, a); \
- c = __builtin_ia32_crc32si(c + initial_hash, b); \
- } while(0)
-# elif defined(__aarch64__)
-# define MIX(a,b,c) \
- do { \
- Uint32 initial_hash = c; \
- c = __crc32cw(c, a); \
- c = __crc32cw(c + initial_hash, b); \
- } while(0)
-# else
-# error "No suitable CRC32 intrinsic available."
-# endif
+/* Pushes a term to the stack, optionally handling it up-front if it's an
+ * immediate to speed up `{atom(), immed()}` keys in maps. We hash the presence
+ * of non-immediates to ensure that terms with a different internal order hash
+ * differently.
+ *
+ * Take for example `{a,{},b,{}}` and `{{},a,{},b}`. This will be processed in
+ * the order `a,b,{},{}` in both cases as the non-immediates are deferred. If
+ * we don't hash the order of the terms, they will always hash equally. */
+#define IHASH_PUSH_TERM(stack, term) \
+ do { \
+ if (ERTS_LIKELY(is_immed(term))) { \
+ IHASH_MIX_IMMEDIATE(term); \
+ } else { \
+ IHASH_MIX_ALPHA(IHASH_TYPE_ARRAY_ELEMENT); \
+ ESTACK_PUSH(stack, (term)); \
+ } \
+ } while(0)
+
+/* Endian-agnostic 64-bit read. This helps the compiler generate optimized code
+ * in a hot loop where the data is unlikely to be properly aligned, saving us
+ * from having to wrangle that manually. */
+static ERTS_FORCE_INLINE
+Uint64 read_u64(const byte *data) {
+ Uint64 value = 0;
+
+ for (int i = 0; i < sizeof(Uint64); i++) {
+#ifdef WORDS_BIGENDIAN
+ value = ((Uint64)data[i]) | (value << CHAR_BIT);
+#else
+ value |= ((Uint64)data[i]) << (i * CHAR_BIT);
#endif
+ }
-#define CONST_HASH(AConst) \
- do { /* Lightweight mixing of constant (type info) */ \
- hash ^= AConst; \
- hash = (hash << 17) ^ (hash >> (32-17)); \
- } while (0)
+ return value;
+}
-/*
- * Start with salt, 32-bit prime number, to avoid getting same hash as phash2
- * which can cause bad hashing in distributed ETS tables for example.
- */
-#define INTERNAL_HASH_SALT 3432918353U
+static Uint64 ihash_mix64(Uint64 input)
+{
+ Uint64 hash = input;
+
+ hash ^= hash >> 33;
+ hash *= 0xFF51AFD7ED558CCDull;
+ hash ^= hash >> 33;
+ hash *= 0xC4CEB9FE1A85EC53ull;
+ hash ^= hash >> 33;
+
+ /* Inverse, if needed for testing. The constants are the modular inverse of
+ * the ones above (over 1 << 64).
+ *
+ * hash ^= hash >> 33;
+ * hash *= 0x9CB4B2F8129337DBull;
+ * hash ^= hash >> 33;
+ * hash *= 0x4F74430C22A54005ull;
+ * hash ^= hash >> 33; */
-Uint32
-make_internal_hash(Eterm term, Uint32 salt)
+ return hash;
+}
+
+static erts_ihash_t
+make_internal_hash(Eterm term, erts_ihash_t salt)
{
- Uint32 hash = salt ^ INTERNAL_HASH_SALT;
+ Uint64 hash_alpha, hash_beta;
+ Uint hash_ticks;
- /* Optimization. Simple cases before declaration of estack. */
- if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
- #if ERTS_SIZEOF_ETERM == 8
- UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST);
- #elif ERTS_SIZEOF_ETERM == 4
- UINT32_HASH(term, HCONST);
- #else
- # error "No you don't"
- #endif
- return hash;
- }
- {
- Eterm tmp;
DECLARE_ESTACK(s);
+ hash_alpha = (Uint64)salt;
+ hash_beta = (Uint64)salt;
+ hash_ticks = 0;
+
for (;;) {
switch (primary_tag(term)) {
case TAG_PRIMARY_LIST:
{
- int c = 0;
- Uint32 sh = 0;
- Eterm* ptr = list_val(term);
- while (is_byte(*ptr)) {
- /* Optimization for strings. */
- sh = (sh << 8) + unsigned_val(*ptr);
- if (c == 3) {
- UINT32_HASH(sh, HCONST_4);
- c = sh = 0;
- } else {
- c++;
- }
- term = CDR(ptr);
- if (is_not_list(term))
+ const Eterm *cell;
+ UWord value = 0;
+ int bytes = 0;
+
+ /* Optimization for strings. */
+ while (is_list(term)) {
+ cell = list_val(term);
+
+ if (!is_byte(CAR(cell))) {
break;
- ptr = list_val(term);
+ }
+
+ value = (value << 8) | unsigned_val(CAR(cell));
+ bytes++;
+
+ if ((bytes % 4) == 0) {
+ IHASH_MIX_ALPHA_2F32(IHASH_TYPE_STRING | (bytes << 8),
+ value);
+ value = 0;
+ bytes = 0;
+ }
+
+ term = CDR(cell);
+ }
+
+ if (bytes > 0) {
+ IHASH_MIX_ALPHA_2F32(IHASH_TYPE_STRING | (bytes << 8), value);
}
- if (c > 0)
- UINT32_HASH_2(sh, (Uint32)c, HCONST_22);
if (is_list(term)) {
- tmp = CDR(ptr);
- CONST_HASH(HCONST_17); /* Hash CAR in cons cell */
- ESTACK_PUSH(s, tmp);
- if (is_not_list(tmp)) {
- ESTACK_PUSH(s, HASH_CDR);
+ Eterm head, tail;
+
+ cell = list_val(term);
+ head = CAR(cell);
+ tail = CDR(cell);
+
+ if (is_immed(head)) {
+ IHASH_MIX_ALPHA_2F32(IHASH_TYPE_IMMEDIATE, IHASH_TYPE_CAR);
+ IHASH_MIX_BETA(head);
+
+ if (is_not_list(tail)) {
+ IHASH_MIX_ALPHA(IHASH_TYPE_CDR);
+ }
+
+ term = tail;
+ } else {
+ ESTACK_PUSH(s, tail);
+ if (is_not_list(tail)) {
+ ESTACK_PUSH(s, IHASH_CDR_MARKER);
+ }
+
+ IHASH_MIX_ALPHA(IHASH_TYPE_CAR);
+ term = head;
}
- term = CAR(ptr);
}
+
+ continue;
}
break;
case TAG_PRIMARY_BOXED:
{
Eterm hdr = *boxed_val(term);
ASSERT(is_header(hdr));
+
switch (hdr & _TAG_HEADER_MASK) {
case ARITYVAL_SUBTAG:
{
- int i;
- int arity = header_arity(hdr);
- Eterm* elem = tuple_val(term);
- UINT32_HASH(arity, HCONST_9);
- if (arity == 0) /* Empty tuple */
- goto pop_next;
- for (i = arity; ; i--) {
- term = elem[i];
- if (i == 1)
- break;
- ESTACK_PUSH(s, term);
+ const Eterm *elements = &tuple_val(term)[0];
+ const int arity = header_arity(hdr);
+
+ IHASH_MIX_ALPHA(IHASH_TYPE_TUPLE);
+ IHASH_MIX_BETA(arity);
+
+ if (arity > 0) {
+ for (int i = 1; i < arity; i++) {
+ IHASH_PUSH_TERM(s, elements[i]);
+ }
+
+ term = elements[arity];
+ continue;
}
+
+ goto pop_next;
}
break;
case MAP_SUBTAG:
{
- Eterm* ptr = boxed_val(term) + 1;
+ const Eterm *elements = &boxed_val(term)[1];
Uint size;
- int i;
/*
* We rely on key-value iteration order being constant
@@ -1685,79 +1819,113 @@ make_internal_hash(Eterm term, Uint32 salt)
switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
case HAMT_SUBTAG_HEAD_FLATMAP:
{
- flatmap_t *mp = (flatmap_t *)flatmap_val(term);
- Eterm *ks = flatmap_get_keys(mp);
- Eterm *vs = flatmap_get_values(mp);
- size = flatmap_get_size(mp);
- UINT32_HASH(size, HCONST_16);
- if (size == 0)
- goto pop_next;
+ const flatmap_t *mp = (const flatmap_t *)flatmap_val(term);
+ const Eterm *ks = flatmap_get_keys(mp);
+ const Eterm *vs = flatmap_get_values(mp);
+ size = flatmap_get_size(mp);
+
+ IHASH_MIX_ALPHA(IHASH_TYPE_FLATMAP);
+ IHASH_MIX_BETA(size);
+
+ if (size > 0) {
+ for (int i = 0; i < size - 1; i++) {
+ IHASH_PUSH_TERM(s, vs[i]);
+ IHASH_PUSH_TERM(s, ks[i]);
+ }
- for (i = size - 1; i >= 0; i--) {
- ESTACK_PUSH(s, vs[i]);
- ESTACK_PUSH(s, ks[i]);
+ IHASH_PUSH_TERM(s, vs[size - 1]);
+ term = ks[size - 1];
+ continue;
}
+
goto pop_next;
}
case HAMT_SUBTAG_HEAD_ARRAY:
+ size = *elements++;
+
+ IHASH_MIX_ALPHA(IHASH_TYPE_HASHMAP_HEAD_ARRAY);
+ IHASH_MIX_BETA(size);
+
+ if (size == 0) {
+ goto pop_next;
+ }
+ break;
case HAMT_SUBTAG_HEAD_BITMAP:
- size = *ptr++;
- UINT32_HASH(size, HCONST_16);
- if (size == 0)
+ size = *elements++;
+
+ IHASH_MIX_ALPHA(IHASH_TYPE_HASHMAP_HEAD_BITMAP);
+ IHASH_MIX_BETA(size);
+
+ if (size == 0) {
goto pop_next;
+ }
+ break;
+ case HAMT_SUBTAG_NODE_BITMAP:
+ IHASH_MIX_ALPHA(IHASH_TYPE_HASHMAP_NODE);
+ break;
}
+
switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
case HAMT_SUBTAG_HEAD_ARRAY:
- i = 16;
+ size = 16;
break;
case HAMT_SUBTAG_HEAD_BITMAP:
case HAMT_SUBTAG_NODE_BITMAP:
- i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+ size = hashmap_bitcount(MAP_HEADER_VAL(hdr));
break;
default:
erts_exit(ERTS_ERROR_EXIT, "bad header");
}
- while (i) {
- if (is_list(*ptr)) {
- Eterm* cons = list_val(*ptr);
- ESTACK_PUSH(s, CDR(cons));
- ESTACK_PUSH(s, CAR(cons));
- }
- else {
- ASSERT(is_boxed(*ptr));
- /* no special treatment of collision nodes needed,
- hash them as the tuples they are */
- ESTACK_PUSH(s, *ptr);
+
+ for (int i = 0; i < size; i++) {
+ if (is_list(elements[i])) {
+ /* [Key | Value] */
+ const Eterm *cons = list_val(elements[i]);
+ IHASH_PUSH_TERM(s, CDR(cons));
+ IHASH_PUSH_TERM(s, CAR(cons));
+ } else {
+ /* Child or collision node. We don't need to treat the
+ * latter in any special way, and can hash them as the
+ * tuples they are. */
+ ASSERT(is_boxed(elements[i]));
+ ESTACK_PUSH(s, elements[i]);
}
- i--; ptr++;
}
+
goto pop_next;
}
break;
case FUN_SUBTAG:
{
- ErlFunThing* funp = (ErlFunThing *) fun_val(term);
+ const ErlFunThing *funp = (const ErlFunThing*)fun_val(term);
if (is_local_fun(funp)) {
- ErlFunEntry* fe = funp->entry.fun;
+ const ErlFunEntry *fe = funp->entry.fun;
Uint num_free = funp->num_free;
- UINT32_HASH_2(num_free, fe->module, HCONST_20);
- UINT32_HASH_2(fe->index, fe->old_uniq, HCONST_21);
- if (num_free == 0) {
- goto pop_next;
- } else {
- Eterm* bptr = funp->env + num_free - 1;
- while (num_free-- > 1) {
- term = *bptr--;
- ESTACK_PUSH(s, term);
+
+ IHASH_MIX_ALPHA_2F32(IHASH_TYPE_LOCAL_FUN, num_free);
+ IHASH_MIX_BETA_2F32(fe->index, fe->old_uniq);
+
+ IHASH_MIX_ALPHA(IHASH_TYPE_IMMEDIATE);
+ IHASH_MIX_BETA(fe->module);
+
+ if (num_free > 0) {
+ for (int i = 0; i < num_free - 1; i++) {
+ IHASH_PUSH_TERM(s, funp->env[i]);
}
- term = *bptr;
+
+ term = funp->env[num_free - 1];
+ continue;
}
+
+ goto pop_next;
} else {
ASSERT(is_external_fun(funp) && funp->next == NULL);
/* Assumes Export entries never move */
- POINTER_HASH(funp->entry.exp, HCONST_14);
+ IHASH_MIX_ALPHA(IHASH_TYPE_EXTERNAL_FUN);
+ IHASH_MIX_BETA((UWord)funp->entry.exp);
+
goto pop_next;
}
}
@@ -1766,249 +1934,303 @@ make_internal_hash(Eterm term, Uint32 salt)
case HEAP_BINARY_SUBTAG:
case SUB_BINARY_SUBTAG:
{
- byte* bptr;
- Uint sz = binary_size(term);
- Uint32 con = HCONST_13 + hash;
- Uint bitoffs;
- Uint bitsize;
+ Uint bit_offset, bit_size, byte_size;
+ const byte *data;
+
+ ERTS_GET_BINARY_BYTES(term, data, bit_offset, bit_size);
+ byte_size = binary_size(term);
+
+ IHASH_MIX_ALPHA_2F32(IHASH_TYPE_BINARY, bit_size);
+ IHASH_MIX_BETA(byte_size);
+
+ if (byte_size > 0 || bit_size > 0) {
+ const byte *bytes = data;
+ Uint64 value;
+ Uint it;
+
+ if (ERTS_UNLIKELY(bit_offset != 0)) {
+ byte *tmp = (byte*)erts_alloc(ERTS_ALC_T_TMP,
+ byte_size + (bit_size != 0));
+ erts_copy_bits(data, bit_offset, 1, tmp, 0, 1,
+ byte_size * 8 + bit_size);
+ bytes = tmp;
+ }
- ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize);
- if (sz == 0 && bitsize == 0) {
- hash = con;
- } else {
- if (bitoffs == 0) {
- hash = block_hash(bptr, sz, con);
- if (bitsize > 0) {
- UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)),
- HCONST_15);
+ for (it = 0;
+ (it + sizeof(Uint64[2])) <= byte_size;
+ it += sizeof(Uint64[2])) {
+ IHASH_MIX_ALPHA(read_u64(&bytes[it]));
+ IHASH_MIX_BETA(read_u64(&bytes[it + sizeof(Uint64)]));
+ }
+
+ value = 0;
+ switch(byte_size % sizeof(Uint64[2]))
+ {
+ case 15: value ^= ((Uint64)bytes[it + 14]) << 0x30;
+ case 14: value ^= ((Uint64)bytes[it + 13]) << 0x28;
+ case 13: value ^= ((Uint64)bytes[it + 12]) << 0x20;
+ case 12: value ^= ((Uint64)bytes[it + 11]) << 0x18;
+ case 11: value ^= ((Uint64)bytes[it + 10]) << 0x10;
+ case 10: value ^= ((Uint64)bytes[it + 9]) << 0x08;
+ case 9: value ^= ((Uint64)bytes[it + 8]) << 0x00;
+ {
+ value *= IHASH_C2;
+ value = ROTL64(value, 33);
+ value *= IHASH_C1;
+ hash_beta ^= value;
+ value = 0;
+ /* !! FALL THROUGH !! */
}
- } else {
- byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP,
- sz + (bitsize != 0));
- erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize);
- hash = block_hash(buf, sz, con);
- if (bitsize > 0) {
- UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)),
- HCONST_15);
+ case 8: value ^= ((Uint64)bytes[it + 7]) << 0x38;
+ case 7: value ^= ((Uint64)bytes[it + 6]) << 0x30;
+ case 6: value ^= ((Uint64)bytes[it + 5]) << 0x28;
+ case 5: value ^= ((Uint64)bytes[it + 4]) << 0x20;
+ case 4: value ^= ((Uint64)bytes[it + 3]) << 0x18;
+ case 3: value ^= ((Uint64)bytes[it + 2]) << 0x10;
+ case 2: value ^= ((Uint64)bytes[it + 1]) << 0x08;
+ case 1: value ^= ((Uint64)bytes[it + 0]) << 0x00;
+ {
+ value *= IHASH_C1;
+ value = ROTL64(value, 31);
+ value *= IHASH_C2;
+ hash_alpha ^= value;
+ break;
}
- erts_free(ERTS_ALC_T_TMP, (void *) buf);
+ };
+
+ if (bit_size > 0) {
+ IHASH_MIX_ALPHA(bytes[byte_size] >> (8 - bit_size));
+ }
+
+ if (bytes != data) {
+ erts_free(ERTS_ALC_T_TMP, (void *)bytes);
}
}
+
goto pop_next;
}
break;
case POS_BIG_SUBTAG:
case NEG_BIG_SUBTAG:
{
- Eterm* ptr = big_val(term);
- Uint i = 0;
- Uint n = BIG_SIZE(ptr);
- Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11;
-#if D_EXP == 16
- do {
- Uint32 x, y;
- x = i < n ? BIG_DIGIT(ptr, i++) : 0;
- x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
- y = i < n ? BIG_DIGIT(ptr, i++) : 0;
- y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
- UINT32_HASH_2(x, y, con);
- } while (i < n);
-#elif D_EXP == 32
- do {
- Uint32 x, y;
- x = i < n ? BIG_DIGIT(ptr, i++) : 0;
- y = i < n ? BIG_DIGIT(ptr, i++) : 0;
- UINT32_HASH_2(x, y, con);
- } while (i < n);
-#elif D_EXP == 64
- do {
- Uint t;
- Uint32 x, y;
- ASSERT(i < n);
- t = BIG_DIGIT(ptr, i++);
- x = t & 0xffffffff;
- y = t >> 32;
- UINT32_HASH_2(x, y, con);
- } while (i < n);
-#else
-#error "unsupported D_EXP size"
-#endif
+ const Eterm *ptr = big_val(term);
+ int i, n;
+
+ /* `n` must fit in a signed int. */
+ ERTS_CT_ASSERT((1ull << 31) > (Uint64)BIG_ARITY_MAX);
+ n = BIG_SIZE(ptr);
+ ASSERT(n < BIG_ARITY_MAX);
+
+ IHASH_MIX_ALPHA_2F32((BIG_SIGN(ptr) ?
+ IHASH_TYPE_NEG_BIGNUM :
+ IHASH_TYPE_POS_BIGNUM),
+ n);
+
+ for (i = 0; (i + 2) <= n; i += 2) {
+ IHASH_MIX_ALPHA(BIG_DIGIT(ptr, i+0));
+ IHASH_MIX_BETA(BIG_DIGIT(ptr, i+1));
+ }
+
+ if (i < n) {
+ IHASH_MIX_BETA(BIG_DIGIT(ptr, i));
+ }
+
goto pop_next;
}
break;
case REF_SUBTAG: {
Uint32 *numbers = internal_ref_numbers(term);
ASSERT(internal_ref_no_numbers(term) >= 3);
- UINT32_HASH(numbers[0], HCONST_7);
- UINT32_HASH_2(numbers[1], numbers[2], HCONST_8);
+
+ IHASH_MIX_ALPHA_2F32(IHASH_TYPE_LOCAL_REF, numbers[0]);
+ IHASH_MIX_BETA_2F32(numbers[1], numbers[2]);
+
if (is_internal_pid_ref(term)) {
#ifdef ARCH_64
ASSERT(internal_ref_no_numbers(term) == 5);
- UINT32_HASH_2(numbers[3], numbers[4], HCONST_9);
+ IHASH_MIX_ALPHA_2F32(numbers[3], numbers[4]);
#else
ASSERT(internal_ref_no_numbers(term) == 4);
- UINT32_HASH(numbers[3], HCONST_9);
+ IHASH_MIX_ALPHA(numbers[3]);
#endif
}
+
goto pop_next;
}
case EXTERNAL_REF_SUBTAG:
{
- ExternalThing* thing = external_thing_ptr(term);
- Uint n = external_thing_ref_no_numbers(thing);
- Uint32 *numbers = external_thing_ref_numbers(thing);
+ const ExternalThing* thing = external_thing_ptr(term);
+ const Uint32 *numbers;
+ int i, n;
/* Can contain 0 to 5 32-bit numbers... */
+ n = external_thing_ref_no_numbers(thing);
+ numbers = external_thing_ref_numbers(thing);
+ ASSERT(n <= 5);
- /* See limitation #2 */
- switch (n) {
- case 5: {
- Uint32 num4 = numbers[4];
- if (0) {
- case 4:
- num4 = 0;
- /* Fall through... */
- }
- UINT32_HASH_2(numbers[3], num4, HCONST_9);
- /* Fall through... */
- }
- case 3: {
- Uint32 num2 = numbers[2];
- if (0) {
- case 2:
- num2 = 0;
- /* Fall through... */
- }
- UINT32_HASH_2(numbers[1], num2, HCONST_8);
- /* Fall through... */
+ IHASH_MIX_ALPHA_2F32(IHASH_TYPE_EXTERNAL_REF, n);
+
+ for (i = 0; (i + 2) <= n; i += 2) {
+ IHASH_MIX_BETA_2F32(numbers[i], numbers[i + 1]);
}
- case 1:
-#ifdef ARCH_64
- POINTER_HASH(thing->node, HCONST_7);
- UINT32_HASH(numbers[0], HCONST_7);
-#else
- UINT32_HASH_2(thing->node, numbers[0], HCONST_7);
-#endif
- break;
- case 0:
- POINTER_HASH(thing->node, HCONST_7);
- break;
- default:
- ASSERT(!"Invalid amount of external reference numbers");
- break;
+
+ if (i < n) {
+ IHASH_MIX_BETA(numbers[i]);
}
+
+ IHASH_MIX_ALPHA((UWord)thing->node);
goto pop_next;
}
case EXTERNAL_PID_SUBTAG: {
- ExternalThing* thing = external_thing_ptr(term);
+ const ExternalThing *thing = external_thing_ptr(term);
/* See limitation #2 */
- POINTER_HASH(thing->node, HCONST_5);
- UINT32_HASH_2(thing->data.pid.num, thing->data.pid.ser, HCONST_5);
+ IHASH_MIX_ALPHA(IHASH_TYPE_EXTERNAL_PID);
+ IHASH_MIX_BETA((UWord)thing->node);
+ IHASH_MIX_ALPHA_2F32(thing->data.pid.num, thing->data.pid.ser);
goto pop_next;
}
case EXTERNAL_PORT_SUBTAG: {
- ExternalThing* thing = external_thing_ptr(term);
+ const ExternalThing *thing = external_thing_ptr(term);
/* See limitation #2 */
- POINTER_HASH(thing->node, HCONST_6);
- UINT32_HASH_2(thing->data.ui32[0], thing->data.ui32[1], HCONST_6);
+ IHASH_MIX_ALPHA(IHASH_TYPE_EXTERNAL_PORT);
+ IHASH_MIX_BETA((UWord)thing->node);
+#ifdef ARCH_64
+ IHASH_MIX_ALPHA(thing->data.port.id);
+#else
+ IHASH_MIX_ALPHA_2F32(thing->data.port.low,
+ thing->data.port.high);
+#endif
goto pop_next;
}
case FLOAT_SUBTAG:
{
FloatDef ff;
+
GET_DOUBLE(term, ff);
+
if (ff.fd == 0.0f) {
/* ensure positive 0.0 */
ff.fd = erts_get_positive_zero_float();
}
- UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
+
+ IHASH_MIX_ALPHA(IHASH_TYPE_FLOAT);
+ IHASH_MIX_BETA_2F32(ff.fw[0], ff.fw[1]);
+
goto pop_next;
}
default:
- erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt);
+ erts_exit(ERTS_ERROR_EXIT,
+ "Invalid tag in make_internal_hash(0x%X, _, %i)\n",
+ term);
}
}
break;
case TAG_PRIMARY_IMMED1:
- #if ERTS_SIZEOF_ETERM == 8
- UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST);
- #else
- UINT32_HASH(term, HCONST);
- #endif
+ IHASH_MIX_IMMEDIATE(term);
goto pop_next;
default:
- erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt);
+ erts_exit(ERTS_ERROR_EXIT,
+ "Invalid tag in make_internal_hash(0x%X, _, %i)\n",
+ term);
pop_next:
if (ESTACK_ISEMPTY(s)) {
DESTROY_ESTACK(s);
- return hash;
+ hash_alpha ^= hash_ticks;
+ hash_beta ^= hash_ticks;
+
+ hash_alpha += hash_beta;
+ hash_beta += hash_alpha;
+
+ hash_alpha = ihash_mix64(hash_alpha);
+ hash_beta = ihash_mix64(hash_beta);
+
+ hash_alpha += hash_beta;
+ hash_beta += hash_alpha;
+
+ return (erts_ihash_t)(hash_alpha ^ hash_beta);
}
term = ESTACK_POP(s);
switch (term) {
- case HASH_CDR:
- CONST_HASH(HCONST_18); /* Hash CDR i cons cell */
- goto pop_next;
- default:
- break;
+ case IHASH_CAR_MARKER:
+ /* Hash CAR in cons cell */
+ IHASH_MIX_BETA(IHASH_TYPE_CAR);
+ term = ESTACK_POP(s);
+ continue;
+ case IHASH_CDR_MARKER:
+ /* Hash CDR in cons cell */
+ IHASH_MIX_BETA(IHASH_TYPE_CDR);
+ term = ESTACK_POP(s);
+ continue;
}
}
}
- }
-
}
#ifdef DBG_HASHMAP_COLLISION_BONANZA
-Uint32 erts_dbg_hashmap_collision_bonanza(Uint32 hash, Eterm key)
+erts_ihash_t erts_dbg_hashmap_collision_bonanza(erts_ihash_t hash, Eterm key)
{
-/*{
- static Uint32 hashvec[7] = {
- 0x02345678,
- 0x12345678,
- 0xe2345678,
- 0xf2345678,
- 0x12abcdef,
- 0x13abcdef,
- 0xcafebabe
- };
- hash = hashvec[hash % (sizeof(hashvec) / sizeof(hashvec[0]))];
- }*/
- const Uint32 bad_hash = (hash & 0x12482481) * 1442968193;
- const Uint32 bad_bits = hash % 67;
- if (bad_bits < 32) {
+ /* Keep only 8 bits to ensure a high collision rate (1/256). */
+ erts_ihash_t bad_hash = (hash & 0x12482481u);
+ erts_ihash_t bad_bits;
+
+ switch (sizeof(erts_ihash_t) * CHAR_BIT) {
+ case 64:
+ bad_hash *= UWORD_CONSTANT(11400714819323198485);
+ bad_hash ^= (bad_hash >> 31);
+ bad_bits = hash % 137;
+ break;
+ case 32:
+ bad_hash *= UWORD_CONSTANT(2654435769);
+ bad_hash ^= (bad_hash >> 15);
+ bad_bits = hash % 67;
+ break;
+ default:
+ ASSERT(!"Unknown sizeof(erts_ihash_t)");
+ }
+
+ (void)key;
+
+ if (bad_bits < (sizeof(erts_ihash_t) * CHAR_BIT)) {
/* Mix in a number of high good bits to get "randomly" close
- to the collision nodes */
- const Uint32 bad_mask = (1 << bad_bits) - 1;
- return (hash & ~bad_mask) | (bad_hash & bad_mask);
+ * to the collision nodes */
+ const erts_ihash_t bad_mask = (1 << bad_bits) - 1;
+ bad_hash = (hash & ~bad_mask) | (bad_hash & bad_mask);
}
+
return bad_hash;
}
#endif
-/* Term hash function for hashmaps */
-Uint32 make_map_hash(Eterm key) {
- Uint32 hash;
+erts_ihash_t erts_internal_salted_hash(Eterm term, erts_ihash_t salt) {
+ if (ERTS_LIKELY(is_immed(term))) {
+ /* Fast path for immediates. The vast majority of calls land here. */
+ return ihash_mix64(term + salt);
+ }
+
+ return make_internal_hash(term, salt);
+}
+
+erts_ihash_t erts_internal_hash(Eterm term) {
+ if (ERTS_LIKELY(is_immed(term))) {
+ return ihash_mix64(term);
+ }
- hash = make_internal_hash(key, 0);
+ return make_internal_hash(term, 0);
+}
+
+/* Term hash function for hashmaps, identical to erts_internal_hash except in
+ * certain debug configurations that weaken the hash. */
+erts_ihash_t erts_map_hash(Eterm key) {
+ erts_ihash_t hash = erts_internal_hash(key);
#ifdef DBG_HASHMAP_COLLISION_BONANZA
hash = erts_dbg_hashmap_collision_bonanza(hash, key);
#endif
+
return hash;
}
-
-#undef CONST_HASH
-#undef HASH_MAP_TAIL
-#undef HASH_MAP_PAIR
-#undef HASH_CDR
-
-#undef UINT32_HASH_2
-#undef UINT32_HASH
-#undef SINT32_HASH
-
-#undef HCONST
-#undef MIX
diff --git a/erts/emulator/beam/erl_term_hashing.h b/erts/emulator/beam/erl_term_hashing.h
index 8a898b7c52..dacd944d4d 100644
--- a/erts/emulator/beam/erl_term_hashing.h
+++ b/erts/emulator/beam/erl_term_hashing.h
@@ -24,11 +24,25 @@
#include "sys.h"
#include "erl_drv_nif.h"
-#if (defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)) || \
- (defined(__x86_64__) && defined(__SSE4_2__))
-# define ERL_INTERNAL_HASH_CRC32C
+/* Internal hash routines that can be changed at will. */
+
+typedef UWord erts_ihash_t;
+
+erts_ihash_t erts_internal_salted_hash(Eterm term, erts_ihash_t salt);
+erts_ihash_t erts_internal_hash(Eterm term);
+erts_ihash_t erts_map_hash(Eterm term);
+
+#ifdef DEBUG
+# define DBG_HASHMAP_COLLISION_BONANZA
+#endif
+
+#ifdef DBG_HASHMAP_COLLISION_BONANZA
+erts_ihash_t erts_dbg_hashmap_collision_bonanza(erts_ihash_t hash, Eterm key);
#endif
+/* Portable hash routines whose results should be bug-compatible across
+ * versions. */
+
typedef struct {
Uint32 a,b,c;
} ErtsBlockHashHelperCtx;
@@ -52,14 +66,7 @@ typedef struct {
Uint32 make_hash2(Eterm);
Uint32 trapping_make_hash2(Eterm, Eterm*, struct process*);
Uint32 make_hash(Eterm);
-Uint32 make_internal_hash(Eterm, Uint32 salt);
-#ifdef DEBUG
-# define DBG_HASHMAP_COLLISION_BONANZA
-#endif
-#ifdef DBG_HASHMAP_COLLISION_BONANZA
-Uint32 erts_dbg_hashmap_collision_bonanza(Uint32 hash, Eterm key);
-#endif
-Uint32 make_map_hash(Eterm key);
+
void erts_block_hash_init(ErtsBlockHashState *state,
const byte *ptr,
Uint len,
@@ -75,5 +82,4 @@ int erts_iov_block_hash(Uint32 *hashp,
Uint *sizep,
ErtsIovBlockHashState *state);
-
#endif
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 2dd6c99d4c..d689b653f5 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -3130,7 +3130,7 @@ static int tracer_cmp_fun(void* a, void* b)
static HashValue tracer_hash_fun(void* obj)
{
- return make_internal_hash(((ErtsTracerNif*)obj)->module, 0);
+ return erts_internal_hash(((ErtsTracerNif*)obj)->module);
}
static void *tracer_alloc_fun(void* tmpl)
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 3170cebe95..75db8fe792 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -734,8 +734,8 @@ do { \
typedef struct ErtsPStack_ {
byte* pstart;
- int offs; /* "stack pointer" as byte offset from pstart */
- int size; /* allocated size in bytes */
+ SWord offs; /* "stack pointer" as byte offset from pstart */
+ SWord size; /* allocated size in bytes */
ErtsAlcType_t alloc_type;
}ErtsPStack;
@@ -746,7 +746,7 @@ void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes);
#define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \
PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE]; \
ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \
- -(int)sizeof(PSTACK_TYPE), /* offs */ \
+ -(SWord)sizeof(PSTACK_TYPE), /* offs */ \
DEF_PSTACK_SIZE*sizeof(PSTACK_TYPE), /* size */ \
ERTS_ALC_T_ESTACK /* alloc_type */ \
}
diff --git a/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl b/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl
index b2478bd56d..59524b32c7 100644
--- a/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl
+++ b/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl
@@ -94,7 +94,6 @@ my @beam_global_funcs = qw(
i_bxor_body_shared
int_div_rem_body_shared
int_div_rem_guard_shared
- internal_hash_helper
is_in_range_shared
is_ge_lt_shared
minus_body_shared
@@ -194,6 +193,7 @@ $decl_emit_funcs
void emit_bif_element_helper(Label fail);
void emit_bif_tuple_size_helper(Label fail);
+ void emit_internal_hash_helper();
void emit_flatmap_get_element();
void emit_hashmap_get_element();
diff --git a/erts/emulator/beam/jit/arm/instr_map.cpp b/erts/emulator/beam/jit/arm/instr_map.cpp
index 8d7ad6f45f..daecdb80ef 100644
--- a/erts/emulator/beam/jit/arm/instr_map.cpp
+++ b/erts/emulator/beam/jit/arm/instr_map.cpp
@@ -29,58 +29,32 @@ extern "C"
#include "beam_common.h"
}
-static const Uint32 INTERNAL_HASH_SALT = 3432918353;
-static const Uint32 HCONST = 0x9E3779B9;
-
-/* ARG6 = lower 32
- * ARG7 = upper 32
- *
- * Helper function for calculating the internal hash of keys before looking
- * them up in a map.
+/* ARG2 = term
*
- * This is essentially just a manual expansion of the `UINT32_HASH_2` macro.
- * Whenever the internal hash algorithm is updated, this and all of its users
- * must follow suit.
+ * Helper for calculating the internal hash of keys before looking them up in a
+ * map. This is a manual expansion of `erts_internal_hash`, and all changes to
+ * that function must be mirrored here.
*
- * Result is returned in ARG3. All arguments are clobbered. */
+ * Result in ARG3. Clobbers TMP1. */
void BeamGlobalAssembler::emit_internal_hash_helper() {
- a64::Gp hash = ARG3.w(), lower = ARG6.w(), upper = ARG7.w(),
- constant = ARG8.w();
-
- mov_imm(hash, INTERNAL_HASH_SALT);
- mov_imm(constant, HCONST);
-
- a.add(lower, lower, constant);
- a.add(upper, upper, constant);
-
-#if defined(ERL_INTERNAL_HASH_CRC32C)
- a.crc32cw(lower, hash, lower);
- a.add(hash, hash, lower);
- a.crc32cw(hash, hash, upper);
-#else
- using rounds =
- std::initializer_list<std::tuple<a64::Gp, a64::Gp, a64::Gp, int>>;
- for (const auto &round : rounds{{lower, upper, hash, 13},
- {upper, hash, lower, -8},
- {hash, lower, upper, 13},
- {lower, upper, hash, 12},
- {upper, hash, lower, -16},
- {hash, lower, upper, 5},
- {lower, upper, hash, 3},
- {upper, hash, lower, -10},
- {hash, lower, upper, 15}}) {
- const auto &[r_a, r_b, r_c, shift] = round;
-
- a.sub(r_a, r_a, r_b);
- a.sub(r_a, r_a, r_c);
-
- if (shift > 0) {
- a.eor(r_a, r_a, r_c, arm::lsr(shift));
- } else {
- a.eor(r_a, r_a, r_c, arm::lsl(-shift));
- }
- }
-#endif
+ a64::Gp key = ARG2, key_hash = ARG3;
+
+ /* key_hash = key ^ (key >> 33); */
+ a.eor(key_hash, key, key, arm::lsr(33));
+
+ /* key_hash *= 0xFF51AFD7ED558CCDull */
+ mov_imm(TMP1, 0xFF51AFD7ED558CCDull);
+ a.mul(key_hash, key_hash, TMP1);
+
+ /* key_hash ^= key_hash >> 33; */
+ a.eor(key_hash, key_hash, key_hash, arm::lsr(33));
+
+ /* key_hash *= 0xC4CEB9FE1A85EC53ull */
+ mov_imm(TMP1, 0xC4CEB9FE1A85EC53ull);
+ a.mul(key_hash, key_hash, TMP1);
+
+ /* key_hash ^= key_hash >> 33; */
+ a.eor(key_hash, key_hash, key_hash, arm::lsr(33));
#ifdef DBG_HASHMAP_COLLISION_BONANZA
emit_enter_runtime_frame();
@@ -99,8 +73,6 @@ void BeamGlobalAssembler::emit_internal_hash_helper() {
emit_leave_runtime();
emit_leave_runtime_frame();
#endif
-
- a.ret(a64::x30);
}
/* ARG1 = untagged hash map root
@@ -171,7 +143,7 @@ void BeamGlobalAssembler::emit_hashmap_get_element() {
* word. */
a.ldr(header_val, arm::Mem(node).post(sizeof(Eterm)));
- /* After 8 nodes we've run out of the 32 bits we started with
+ /* After 8/16 nodes we've run out of the hash bits we've started with
* and we end up in a collision node. */
a.cmp(depth, imm(HAMT_MAX_LEVEL));
a.b_ne(node_loop);
@@ -375,15 +347,9 @@ void BeamGlobalAssembler::emit_i_get_map_element_shared() {
a.bind(hashmap);
{
- emit_enter_runtime_frame();
-
- /* Calculate the internal hash of ARG2 before diving into the HAMT. */
- a.mov(ARG6.w(), ARG2.w());
- a.lsr(ARG7, ARG2, imm(32));
- a.bl(labels[internal_hash_helper]);
-
- emit_leave_runtime_frame();
-
+ /* Calculate the internal hash of the key before diving into the
+ * HAMT. */
+ emit_internal_hash_helper();
emit_hashmap_get_element();
}
}
@@ -527,7 +493,6 @@ void BeamGlobalAssembler::emit_i_get_map_element_hash_shared() {
a.and_(TMP1, ARG4, imm(_HEADER_MAP_SUBTAG_MASK));
a.cmp(TMP1, imm(HAMT_SUBTAG_HEAD_FLATMAP));
a.b_ne(hashmap);
-
emit_flatmap_get_element();
a.bind(hashmap);
diff --git a/erts/emulator/beam/jit/arm/ops.tab b/erts/emulator/beam/jit/arm/ops.tab
index 6d27a18c2d..3989f5139b 100644
--- a/erts/emulator/beam/jit/arm/ops.tab
+++ b/erts/emulator/beam/jit/arm/ops.tab
@@ -786,7 +786,7 @@ nofail_bif2 S1=d S2 Bif Dst | is_ne_exact_bif(Bif) => bif_is_ne_exact S1 S2 Dst
nofail_bif2 S1 S2 Bif Dst | is_ge_bif(Bif) => bif_is_ge S1 S2 Dst
nofail_bif2 S1 S2 Bif Dst | is_lt_bif(Bif) => bif_is_lt S1 S2 Dst
-i_get_hash c I d
+i_get_hash c W d
i_get s d
self d
@@ -1283,7 +1283,7 @@ i_get_map_elements f s I *
i_get_map_element_hash Fail Src=c Key Hash Dst =>
move Src x | i_get_map_element_hash Fail x Key Hash Dst
-i_get_map_element_hash f S c I S
+i_get_map_element_hash f S c W S
i_get_map_element Fail Src=c Key Dst =>
move Src x | i_get_map_element Fail x Key Dst
diff --git a/erts/emulator/beam/jit/beam_jit_common.cpp b/erts/emulator/beam/jit/beam_jit_common.cpp
index 3200f75407..04ee598abc 100644
--- a/erts/emulator/beam/jit/beam_jit_common.cpp
+++ b/erts/emulator/beam/jit/beam_jit_common.cpp
@@ -744,8 +744,8 @@ Uint beam_jit_get_map_elements(Eterm map,
ASSERT(is_hashmap(map));
while (n--) {
+ erts_ihash_t hx;
const Eterm *v;
- Uint32 hx;
hx = fs[2];
ASSERT(hx == hashmap_make_hash(fs[0]));
diff --git a/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl b/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl
index af16da1eee..3c620462b3 100755
--- a/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl
+++ b/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl
@@ -88,7 +88,6 @@ my @beam_global_funcs = qw(
int_div_rem_guard_shared
is_in_range_shared
is_ge_lt_shared
- internal_hash_helper
minus_body_shared
minus_guard_shared
new_map_shared
@@ -184,6 +183,7 @@ $decl_emit_funcs
x86::Mem emit_i_length_common(Label fail, int state_size);
+ void emit_internal_hash_helper();
void emit_flatmap_get_element();
void emit_hashmap_get_element();
diff --git a/erts/emulator/beam/jit/x86/instr_map.cpp b/erts/emulator/beam/jit/x86/instr_map.cpp
index 5f89077ba6..110054881e 100644
--- a/erts/emulator/beam/jit/x86/instr_map.cpp
+++ b/erts/emulator/beam/jit/x86/instr_map.cpp
@@ -29,63 +29,55 @@ extern "C"
#include "beam_common.h"
}
-static const Uint32 INTERNAL_HASH_SALT = 3432918353;
-static const Uint32 HCONST = 0x9E3779B9;
-
-/*
- * ARG4 = lower 32
- * ARG5 = upper 32
+/* ARG2 = term
*
- * Helper function for calculating the internal hash of keys before looking
- * them up in a map.
+ * Helper for calculating the internal hash of keys before looking them up in a
+ * map. This is a manual expansion of `erts_internal_hash`, and all changes to
+ * that function must be mirrored here.
*
- * This is essentially just a manual expansion of the `UINT32_HASH_2` macro.
- * Whenever the internal hash algorithm is updated, this and all of its users
- * must follow suit.
- *
- * Result is returned in ARG3. */
+ * Result in ARG3. */
void BeamGlobalAssembler::emit_internal_hash_helper() {
- x86::Gp hash = ARG3d, lower = ARG4d, upper = ARG5d;
-
- a.mov(hash, imm(INTERNAL_HASH_SALT));
- a.add(lower, imm(HCONST));
- a.add(upper, imm(HCONST));
-
-#if defined(ERL_INTERNAL_HASH_CRC32C)
- a.mov(ARG6d, hash);
- a.crc32(hash, lower);
- a.add(hash, ARG6d);
- a.crc32(hash, upper);
-#else
- using rounds =
- std::initializer_list<std::tuple<x86::Gp, x86::Gp, x86::Gp, int>>;
- for (const auto &round : rounds{{lower, upper, hash, 13},
- {upper, hash, lower, -8},
- {hash, lower, upper, 13},
- {lower, upper, hash, 12},
- {upper, hash, lower, -16},
- {hash, lower, upper, 5},
- {lower, upper, hash, 3},
- {upper, hash, lower, -10},
- {hash, lower, upper, 15}}) {
- const auto &[r_a, r_b, r_c, shift] = round;
-
- a.sub(r_a, r_b);
- a.sub(r_a, r_c);
-
- /* We have no use for the type constant anymore, reuse its register for
- * the `a ^= r_c << shift` expression. */
- a.mov(ARG6d, r_c);
-
- if (shift > 0) {
- a.shr(ARG6d, imm(shift));
- } else {
- a.shl(ARG6d, imm(-shift));
- }
+ x86::Gp key = ARG2, key_hash = ARG3;
- a.xor_(r_a, ARG6d);
+ /* Unsigned multiplication instructions on x86 either use RDX as an
+ * implicit source or clobber it. Sigh. */
+ if (key == x86::rdx) {
+ a.mov(TMP_MEM1q, x86::rdx);
+ } else {
+ ASSERT(key_hash == x86::rdx);
+ }
+
+ /* key_hash = key ^ (key >> 33); */
+ a.mov(ARG4, ARG2);
+ a.shr(ARG4, imm(33));
+ a.mov(x86::rdx, ARG2);
+ a.xor_(x86::rdx, ARG4);
+
+ /* `RDX * ARG6` storing a 128 bit result in ARG4:RDX. We only want the
+ * lower 64 bits in RDX.
+ *
+ * key_hash *= 0xFF51AFD7ED558CCDull */
+ mov_imm(ARG6, 0xFF51AFD7ED558CCDull);
+ a.mulx(ARG4, x86::rdx, ARG6);
+
+ /* key_hash ^= key_hash >> 33; */
+ a.mov(ARG4, x86::rdx);
+ a.shr(ARG4, imm(33));
+ a.xor_(x86::rdx, ARG4);
+
+ /* key_hash *= 0xC4CEB9FE1A85EC53ull */
+ mov_imm(ARG6, 0xC4CEB9FE1A85EC53ull);
+ a.mulx(ARG4, x86::rdx, ARG6);
+
+ /* key_hash ^= key_hash >> 33; */
+ a.mov(ARG4, x86::rdx);
+ a.shr(ARG4, imm(33));
+ a.xor_(x86::rdx, ARG4);
+
+ if (key == x86::rdx) {
+ a.mov(key_hash, x86::rdx);
+ a.mov(key, TMP_MEM1q);
}
-#endif
#ifdef DBG_HASHMAP_COLLISION_BONANZA
a.mov(TMP_MEM1q, ARG1);
@@ -97,14 +89,12 @@ void BeamGlobalAssembler::emit_internal_hash_helper() {
runtime_call<2>(erts_dbg_hashmap_collision_bonanza);
emit_leave_runtime();
- a.mov(ARG3d, RETd);
+ a.mov(ARG3, RET);
a.mov(ARG1, TMP_MEM1q);
a.mov(ARG2, TMP_MEM2q);
a.mov(RET, TMP_MEM3q);
#endif
-
- a.ret();
}
/* ARG1 = hash map root, ARG2 = key, ARG3 = key hash, RETd = node header
@@ -113,7 +103,7 @@ void BeamGlobalAssembler::emit_internal_hash_helper() {
void BeamGlobalAssembler::emit_hashmap_get_element() {
Label node_loop = a.newLabel();
- x86::Gp node = ARG1, key = ARG2, key_hash = ARG3d, header_val = RETd,
+ x86::Gp node = ARG1, key = ARG2, key_hash = ARG3, header_val = RETd,
index = ARG4d, depth = ARG5d;
const int header_shift =
@@ -131,7 +121,7 @@ void BeamGlobalAssembler::emit_hashmap_get_element() {
/* Find out which child we should follow, and shift the hash for the
* next round. */
- a.mov(index, key_hash);
+ a.mov(index, key_hash.r32());
a.and_(index, imm(0xF));
a.shr(key_hash, imm(4));
a.inc(depth);
@@ -171,7 +161,7 @@ void BeamGlobalAssembler::emit_hashmap_get_element() {
/* Nope, we have to search another node. */
a.mov(header_val, emit_boxed_val(node, 0, sizeof(Uint32)));
- /* After 8 nodes we've run out of the 32 bits we started with
+ /* After 8/16 nodes we've run out of the hash bits we've started with
* and we end up in a collision node. */
a.test(depth, imm(HAMT_MAX_LEVEL - 1));
a.short_().jnz(node_loop);
@@ -390,13 +380,9 @@ void BeamGlobalAssembler::emit_i_get_map_element_shared() {
a.bind(hashmap);
{
- /* Calculate the internal hash of ARG2 before diving into the HAMT. */
- a.mov(ARG5, ARG2);
- a.shr(ARG5, imm(32));
- a.mov(ARG4d, ARG2d);
-
- a.call(labels[internal_hash_helper]);
-
+ /* Calculate the internal hash of the key before diving into the
+ * HAMT. */
+ emit_internal_hash_helper();
emit_hashmap_get_element();
}
}
diff --git a/erts/emulator/beam/jit/x86/ops.tab b/erts/emulator/beam/jit/x86/ops.tab
index ef57021d83..de7ac5961f 100644
--- a/erts/emulator/beam/jit/x86/ops.tab
+++ b/erts/emulator/beam/jit/x86/ops.tab
@@ -751,7 +751,7 @@ nofail_bif2 S1=d S2 Bif Dst | is_ne_exact_bif(Bif) => bif_is_ne_exact S1 S2 Dst
nofail_bif2 S1 S2 Bif Dst | is_ge_bif(Bif) => bif_is_ge S1 S2 Dst
nofail_bif2 S1 S2 Bif Dst | is_lt_bif(Bif) => bif_is_lt S1 S2 Dst
-i_get_hash c I d
+i_get_hash c W d
i_get s d
self d
@@ -1244,7 +1244,7 @@ i_get_map_elements f s I *
i_get_map_element_hash Fail Src=c Key Hash Dst =>
move Src x | i_get_map_element_hash Fail x Key Hash Dst
-i_get_map_element_hash f S c I S
+i_get_map_element_hash f S c W S
i_get_map_element Fail Src=c Key Dst =>
move Src x | i_get_map_element Fail x Key Dst
diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl
index 5d7546c1a4..ef4db545b4 100644
--- a/erts/emulator/test/map_SUITE.erl
+++ b/erts/emulator/test/map_SUITE.erl
@@ -97,6 +97,10 @@
%% Benchmarks
-export([benchmarks/1]).
+%% Helper for generating new colliding keys after the internal hashing
+%% algorithm changes.
+-export([find_colliding_keys/1]).
+
-include_lib("stdlib/include/ms_transform.hrl").
-include_lib("common_test/include/ct_event.hrl").
@@ -3559,45 +3563,213 @@ minor_gcs() ->
{minor_gcs, GCS} = lists:keyfind(minor_gcs, 1, Info),
GCS.
-%% Generate a map with N (or N+1) keys that has an abnormal heap demand.
-%% Done by finding keys that collide in the first 32-bit hash.
+%% Generate a map with N (or N+1) keys that have an abnormal heap demand. Done
+%% by finding keys that collide in the first 32 bits of the hash.
fatmap(N) ->
- %%erts_debug:set_internal_state(available_internal_state, true),
- Table = ets:new(void, [bag, private]),
-
- Seed0 = rand:seed_s(exsplus, {4711, 3141592, 2718281}),
- Seed1 = fatmap_populate(Table, Seed0, (1 bsl 16)),
- Keys = fatmap_generate(Table, Seed1, N, []),
- ets:delete(Table),
- maps:from_list([{K,K} || K <- Keys]).
+ Groups0 = colliding_keys(),
+ Groups = lists:nthtail(length(Groups0) - (N div 2), Groups0),
+ Keys = lists:append([[A, B] || [A, B | _Rest] <- Groups]),
+ maps:from_keys(Keys, []).
+
+colliding_keys() ->
+ %% Collide to 8 levels, anything more than this takes way too long to
+ %% generate.
+ Mask = 16#FFFFFFFF,
+
+ %% Collisions found by find_colliding_keys(Mask) below. When regenerating
+ %% keys, make sure to run it outside testing as it might time-trap
+ %% otherwise.
+ %%
+ %% io:format("Finding new colliding keys for mask ~p~n", [Mask]),
+ %% io:format("Colliding keys\n\t~p\n", [find_colliding_keys(Mask)]),
+ ByMethod = #{
+ %% 64-bit internal hash of `0`
+ 15677855740172624429 =>
+ [[-4294967296,-3502771103,1628104549],
+ [-2750312253,-2208396507,-2147483648,1926198452,3660971145],
+ [-2542330914,-1089175976,-1073741824,290495829],
+ [-2155350068,0],
+ [1073741824,2807978463,3625918826],
+ [-1032333168,-705082324,1541401419,1594347321,2147483648,
+ 2266580263,2823045213],
+ [-2465550512,3221225472],
+ [2854075383,651030299,-1581781966,-3419595364,-4294967295],
+ [3351133532,968011333,-2217176682,-4294967294],
+ [598547769,-1379599129,-4294967293],
+ [-649195724,-4294967292],
+ [2943767758,-645518858,-875893937,-1294474094,-4294967291],
+ [3255309205,-2208705073,-4294967290],
+ [2162086262,-3745041100,-4294967288],
+ [-36087602,-1146855151,-1687820340,-3221225471],
+ [4177844763,3846951687,3485974116,3175597814,590007752,
+ -3221225470],
+ [3264460518,1553643847,1183174568,-3221225469],
+ [-577423597,-3221225468,-3522984153],
+ [3855876603,3019389034,-1323003840,-2576022240,-3221225467],
+ [-471176452,-3221225466],
+ [-1122194611,-3221225465,-4210494386],
+ [3603262778,994932591,-1788155141,-1921175318,-3221225464],
+ [3836440544,-1003007187,-2147483647],
+ [-2051344765,-2147483646],
+ [3650711544,-2147483645,-2799381711,-3556915274],
+ [3489936963,1240642555,-2147483644,-3957745840],
+ [1085161678,-2052366093,-2147483643,-3483479006],
+ [1939744936,-2147483642,-3856508363],
+ [-566163246,-2060332302,-2147483641,-4230104575],
+ [1203359280,237551462,-1073741823],
+ [1727228961,-813544803,-1073741822,-1309725727,-1666872574,
+ -2203000992],
+ [3698637395,3362609925,876970478,-714241238,-1073741821],
+ [1765842640,-354951691,-566902540,-1073741820],
+ [3963091352,2371749084,591553116,-1073741819],
+ [-1073741817,-2715118400],
+ [-1073741816,-3224015310],
+ [2762405117,1,-2123671186],
+ [2470477117,2,-331878960,-2322233731],
+ [3815926349,2088957086,3],
+ [1968999576,870968367,4,-1268233288,-3048698020],
+ [979559827,5],
+ [946684365,753214037,6,-2648059890],
+ [3790852688,2964822264,2830450758,7,-3580232887],
+ [1073741825,-3356417243,-3706053980],
+ [1073741827,-2621798828],
+ [1073741828,-2347690873],
+ [2090309310,1073741830,-1375115411,-2016799213,-4267952630],
+ [1073741831,672032559],
+ [1073741832,-2577014530,-3065907606],
+ [3796535022,2351766515,2147483649,-2136894649],
+ [2280176922,2147483650],
+ [4198987324,3244673818,2147483651,270823276,-2880202587],
+ [3880317786,3256588678,2670024934,2147483652,-2327563310,
+ -3284218582,-3844717086],
+ [2178108296,2147483653,-3361345880],
+ [2954325696,2147483654,-1059451308,-1331847237],
+ [3189358149,2147483655,-1477948284,-1669797549,-3362853705,
+ -3928750615],
+ [2147483656,471953932,-355892383],
+ [3221225473,-3995083753,-4092880912],
+ [3221225474,-2207482759,-3373076062],
+ [3221225475,2400978919,2246389041,1052806668,-781893221,
+ -1811850779],
+ [3221225476,-245369539,-1842612521],
+ [3221225477,688232807],
+ [3221225478,209327542,-2793530395],
+ [3221225479,-2303080520,-4225327222],
+ [4216539003,3221225480]],
+
+ %% 32-bit internal hash of `0`
+ 416211501 =>
+ [[-55973163,-134217697],[43918753,-134217684],
+ [107875525,-134217667],[-30291033,-134217663],
+ [-40285269,-111848095],[35020004,-111848056],
+ [-44437601,-111848046],[103325476,-69901823,-111848030],
+ [126809757,-111848012],[-92672406,-111848005],
+ [-64199103,-111847990],[102238942,-111847982],
+ [62106519,-89478468],[-89478462,-128994853],
+ [-67899866,-89478412],[-45432484,-89478397],
+ [120764819,-89478387],[9085208,-89478382],
+ [10859155,-89478369],[45834467,-67108863],
+ [-67108857,-124327693],[104597114,-67108847],
+ [11918558,-67108783],[50986187,-67108760],
+ [113683827,64978564,-67108752],
+ [111972669,-67108751],[27085194,-44739227],
+ [46760231,-44739221],[101248827,-44739220],
+ [30692154,-44739176],[33768394,-44739117],
+ [-12083942,-44739116],[-22369572,-112420685],
+ [-22369568,-98812798],[-22369550,-78759395],
+ [47792095,-22369543],[9899495,-22369540],
+ [99744593,-22369511],[76325343,52],
+ [122425143,68],[21651445,74],
+ [129537216,119],[125,-110161190],
+ [80229747,22369626],[22369629,-55742042],
+ [128416574,22369631],[105267606,22369643],
+ [22369693,-2286278],[126622985,22369698],
+ [22369701,-13725583],[22369728,-22765683],
+ [22369731,-54786216],[22369740,-65637968],
+ [44739246,12048008],[44739259,-26636781],
+ [126966693,44739272],[44739274,-130215175],
+ [44739277,15051453],[44739292,17890441],
+ [44739301,-72627814],[106949249,44739322],
+ [44739323,-56882381],[67108879,-111259055],
+ [67108888,37627968],[67108894,-53291767],
+ [67108896,-127782577],[67108908,-1014167],
+ [82796148,67108959],[67108962,-71355523],
+ [67108984,-62077338,-77539719],[126106374,89478485],
+ [89478488,85703113],[132215738,89478495],
+ [89478515,-122049151],[89478518,-22611374],
+ [94050181,89478530],[89478547,42736340],
+ [89478553,86641584],[129419863,111848199],
+ [111848217,-32493354],[112586988,111848229]]
+ },
+
+ HashKey = internal_hash(0),
+ #{ HashKey := Keys } = ByMethod,
+
+ verify_colliding_keys(Keys, Mask).
+
+verify_colliding_keys([[K | Ks]=Group | Gs], Mask) ->
+ Hash = internal_hash(K) band Mask,
+ [Hash] = lists:usort([(internal_hash(Key) band Mask) || Key <- Ks]),
+ [Group | verify_colliding_keys(Gs, Mask)];
+verify_colliding_keys([], _Mask) ->
+ [].
+
+%% Use this function to (re)generate the list in colliding_keys/0. This takes
+%% several hours to run so you may want to run it overnight.
+find_colliding_keys(Mask) ->
+ NumScheds = erlang:system_info(schedulers_online),
+ %% Stay below the limit for smalls on 32-bit platforms to prevent the
+ %% search from taking forever due to bignums.
+ Start = -(1 bsl 27),
+ End = -Start,
+ Range = End - Start,
+ Step = Range div NumScheds,
+ timer:tc(fun() ->
+ ckf_spawn(NumScheds, NumScheds, Start, End, Step, Mask, [])
+ end).
+
+ckf_spawn(0, _NumScheds, _Start, _End, _Step, _Mask, Refs) ->
+ lists:append(ckf_await(Refs));
+ckf_spawn(N, NumScheds, Start, End, Step, Mask, Refs) ->
+ Keys = [Start + Z + (N - 1) * Step || Z <- lists:seq(1, 128)],
+ {_, Ref} = spawn_monitor(fun() ->
+ exit(ckf_finder(Start, End, Mask, Keys))
+ end),
+ ckf_spawn(N - 1, NumScheds, Start, End, Step, Mask, [Ref | Refs]).
+
+ckf_await([Ref | Refs]) ->
+ receive
+ {'DOWN', Ref, _, _, []} ->
+ %% Ignore empty slices.
+ ckf_await(Refs);
+ {'DOWN', Ref, _, _, Collisions} ->
+ [Collisions | ckf_await(Refs)]
+ end;
+ckf_await([]) ->
+ [].
-fatmap_populate(_, Seed, 0) -> Seed;
-fatmap_populate(Table, Seed, N) ->
- {I, NextSeed} = rand:uniform_s(1 bsl 48, Seed),
- Hash = internal_hash(I),
- ets:insert(Table, [{Hash, I}]),
- fatmap_populate(Table, NextSeed, N-1).
+ckf_finder(Start, End, Mask, Keys) ->
+ [ckf_finder_1(Start, End, Mask, Key) || Key <- Keys].
+ckf_finder_1(Start, End, Mask, Key) ->
+ true = Key >= Start, true = Key < End, %Assertion.
+ Target = internal_hash(Key) band Mask,
+ ckf_finder_2(Start, End, Mask, Target, []).
-fatmap_generate(_, _, N, Acc) when N =< 0 ->
+ckf_finder_2(Same, Same, _Mask, _Target, [_]) ->
+ %% Key collided with itself, ignore it.
+ [];
+ckf_finder_2(Same, Same, _Mask, _Target, Acc) ->
Acc;
-fatmap_generate(Table, Seed, N0, Acc0) ->
- {I, NextSeed} = rand:uniform_s(1 bsl 48, Seed),
- Hash = internal_hash(I),
- case ets:member(Table, Hash) of
- true ->
- NewKeys = [I | ets:lookup_element(Table, Hash, 2)],
- Acc1 = lists:usort(Acc0 ++ NewKeys),
- N1 = N0 - (length(Acc1) - length(Acc0)),
- fatmap_generate(Table, NextSeed, N1, Acc1);
- false ->
- fatmap_generate(Table, NextSeed, N0, Acc0)
+ckf_finder_2(Next, End, Mask, Target, Acc) ->
+ case (internal_hash(Next) band Mask) =:= Target of
+ true -> ckf_finder_2(Next + 1, End, Mask, Target, [Next | Acc]);
+ false -> ckf_finder_2(Next + 1, End, Mask, Target, Acc)
end.
internal_hash(Term) ->
erts_debug:get_internal_state({internal_hash, Term}).
-
%% map external_format (fannerl).
fannerl() ->
<<131,116,0,0,0,28,100,0,13,108,101,97,114,110,105,110,103,95,114,
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 359646ca5c..1fb9ca8933 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -3747,7 +3747,7 @@ test_bit_distribution_fitness(Integers, BitSize) ->
(FailureText =:= [] orelse ct:fail(FailureText)).
-nif_hash_result_bitsize(internal) -> 32;
+nif_hash_result_bitsize(internal) -> erlang:system_info(wordsize) * 8;
nif_hash_result_bitsize(phash2) -> 27.
unique(List) ->
diff --git a/erts/emulator/test/persistent_term_SUITE.erl b/erts/emulator/test/persistent_term_SUITE.erl
index 39dd2d3428..178eb295ee 100644
--- a/erts/emulator/test/persistent_term_SUITE.erl
+++ b/erts/emulator/test/persistent_term_SUITE.erl
@@ -40,10 +40,6 @@
%%
-export([test_init_restart_cmd/1]).
-%% Test writing helper
--export([find_colliding_keys/0]).
-
-
suite() ->
[{ct_hooks,[ts_install_cth]},
{timetrap,{minutes,10}}].
@@ -603,120 +599,144 @@ collisions_delete([], _) ->
ok.
colliding_keys() ->
- %% Collisions found by find_colliding_keys() below
- %% ct:timetrap({minutes, 60}),
- %% ct:pal("Colliding keys = ~p", [find_colliding_keys()]),
- Collisions =
- #{
- %% Collisions for Jenkins96 hashing.
- 1268203079 => [[77674392,148027],
- [103370644,950908],
- [106444046,870178],
- [22217246,735880],
- [18088843,694607],
- [63426007,612179],
- [117354942,906431],
- [121434305,94282311,816072],
- [118441466,93873772,783366],
- [124338174,1414801,123089],
- [20240282,17113486,923647],
- [126495528,61463488,164994],
- [125341723,5729072,445539],
- [127450932,80442669,348245],
- [123354692,85724182,14241288,180793],
- [99159367,65959274,61680971,289939],
- [107637580,104512101,62639807,181644],
- [139547511,51654420,2062545,151944],
- [88078274,73031465,53388204,428872],
- [141314238,75761379,55699508,861797],
- [88045216,59272943,21030492,180903]],
- %% Collisions for CRC32-C hashing.
- 1982459178 => [[-4294967296,654663773],
- [-3758096384,117792861],
- [-3221225472,1728405597],
- [-2684354560,1191534685],
- [-2147483648,2706162303],
- [-1610612736,2169291391],
- [-1073741824,3779904127],
- [-536870912,3243033215],
- [-3640303523,0],
- [-4177174435,536870912],
- [-2566561699,1073741824],
- [-3103432611,1610612736],
- [-1588804993,2147483648],
- [-2125675905,2684354560],
- [-515063169,3221225472],
- [-1051934081,3758096384]]
- },
-
- Key = internal_hash(2),
- ct:pal("internal_hash(2) = ~p", [Key]),
- #{ Key := L } = Collisions,
-
- %% Verify that the keys still collide (this will fail if the
- %% internal hash function has been changed).
- case erlang:system_info(wordsize) of
- 8 ->
- verify_colliding_keys(L);
- 4 ->
- %% Not guaranteed to collide on a 32-bit system.
- ok
- end,
-
- L.
-
-verify_colliding_keys([[K|Ks]|Gs]) ->
- Hash = internal_hash(K),
- [Hash] = lists:usort([internal_hash(Key) || Key <- Ks]),
- verify_colliding_keys(Gs);
-verify_colliding_keys([]) ->
- ok.
+ Mask = 16#FFFFFFFF,
+
+ %% Collisions found by find_colliding_keys(Mask) in `map_SUITE`.
+ ByMethod = #{
+ %% 64-bit internal hash of `0`
+ 15677855740172624429 =>
+ [[-4294967296,-3502771103,1628104549],
+ [-2750312253,-2208396507,-2147483648,1926198452,3660971145],
+ [-2542330914,-1089175976,-1073741824,290495829],
+ [-2155350068,0],
+ [1073741824,2807978463,3625918826],
+ [-1032333168,-705082324,1541401419,1594347321,2147483648,
+ 2266580263,2823045213],
+ [-2465550512,3221225472],
+ [2854075383,651030299,-1581781966,-3419595364,-4294967295],
+ [3351133532,968011333,-2217176682,-4294967294],
+ [598547769,-1379599129,-4294967293],
+ [-649195724,-4294967292],
+ [2943767758,-645518858,-875893937,-1294474094,-4294967291],
+ [3255309205,-2208705073,-4294967290],
+ [2162086262,-3745041100,-4294967288],
+ [-36087602,-1146855151,-1687820340,-3221225471],
+ [4177844763,3846951687,3485974116,3175597814,590007752,
+ -3221225470],
+ [3264460518,1553643847,1183174568,-3221225469],
+ [-577423597,-3221225468,-3522984153],
+ [3855876603,3019389034,-1323003840,-2576022240,-3221225467],
+ [-471176452,-3221225466],
+ [-1122194611,-3221225465,-4210494386],
+ [3603262778,994932591,-1788155141,-1921175318,-3221225464],
+ [3836440544,-1003007187,-2147483647],
+ [-2051344765,-2147483646],
+ [3650711544,-2147483645,-2799381711,-3556915274],
+ [3489936963,1240642555,-2147483644,-3957745840],
+ [1085161678,-2052366093,-2147483643,-3483479006],
+ [1939744936,-2147483642,-3856508363],
+ [-566163246,-2060332302,-2147483641,-4230104575],
+ [1203359280,237551462,-1073741823],
+ [1727228961,-813544803,-1073741822,-1309725727,-1666872574,
+ -2203000992],
+ [3698637395,3362609925,876970478,-714241238,-1073741821],
+ [1765842640,-354951691,-566902540,-1073741820],
+ [3963091352,2371749084,591553116,-1073741819],
+ [-1073741817,-2715118400],
+ [-1073741816,-3224015310],
+ [2762405117,1,-2123671186],
+ [2470477117,2,-331878960,-2322233731],
+ [3815926349,2088957086,3],
+ [1968999576,870968367,4,-1268233288,-3048698020],
+ [979559827,5],
+ [946684365,753214037,6,-2648059890],
+ [3790852688,2964822264,2830450758,7,-3580232887],
+ [1073741825,-3356417243,-3706053980],
+ [1073741827,-2621798828],
+ [1073741828,-2347690873],
+ [2090309310,1073741830,-1375115411,-2016799213,-4267952630],
+ [1073741831,672032559],
+ [1073741832,-2577014530,-3065907606],
+ [3796535022,2351766515,2147483649,-2136894649],
+ [2280176922,2147483650],
+ [4198987324,3244673818,2147483651,270823276,-2880202587],
+ [3880317786,3256588678,2670024934,2147483652,-2327563310,
+ -3284218582,-3844717086],
+ [2178108296,2147483653,-3361345880],
+ [2954325696,2147483654,-1059451308,-1331847237],
+ [3189358149,2147483655,-1477948284,-1669797549,-3362853705,
+ -3928750615],
+ [2147483656,471953932,-355892383],
+ [3221225473,-3995083753,-4092880912],
+ [3221225474,-2207482759,-3373076062],
+ [3221225475,2400978919,2246389041,1052806668,-781893221,
+ -1811850779],
+ [3221225476,-245369539,-1842612521],
+ [3221225477,688232807],
+ [3221225478,209327542,-2793530395],
+ [3221225479,-2303080520,-4225327222],
+ [4216539003,3221225480]],
+
+ %% 32-bit internal hash of `0`
+ 416211501 =>
+ [[-55973163,-134217697],[43918753,-134217684],
+ [107875525,-134217667],[-30291033,-134217663],
+ [-40285269,-111848095],[35020004,-111848056],
+ [-44437601,-111848046],[103325476,-69901823,-111848030],
+ [126809757,-111848012],[-92672406,-111848005],
+ [-64199103,-111847990],[102238942,-111847982],
+ [62106519,-89478468],[-89478462,-128994853],
+ [-67899866,-89478412],[-45432484,-89478397],
+ [120764819,-89478387],[9085208,-89478382],
+ [10859155,-89478369],[45834467,-67108863],
+ [-67108857,-124327693],[104597114,-67108847],
+ [11918558,-67108783],[50986187,-67108760],
+ [113683827,64978564,-67108752],
+ [111972669,-67108751],[27085194,-44739227],
+ [46760231,-44739221],[101248827,-44739220],
+ [30692154,-44739176],[33768394,-44739117],
+ [-12083942,-44739116],[-22369572,-112420685],
+ [-22369568,-98812798],[-22369550,-78759395],
+ [47792095,-22369543],[9899495,-22369540],
+ [99744593,-22369511],[76325343,52],
+ [122425143,68],[21651445,74],
+ [129537216,119],[125,-110161190],
+ [80229747,22369626],[22369629,-55742042],
+ [128416574,22369631],[105267606,22369643],
+ [22369693,-2286278],[126622985,22369698],
+ [22369701,-13725583],[22369728,-22765683],
+ [22369731,-54786216],[22369740,-65637968],
+ [44739246,12048008],[44739259,-26636781],
+ [126966693,44739272],[44739274,-130215175],
+ [44739277,15051453],[44739292,17890441],
+ [44739301,-72627814],[106949249,44739322],
+ [44739323,-56882381],[67108879,-111259055],
+ [67108888,37627968],[67108894,-53291767],
+ [67108896,-127782577],[67108908,-1014167],
+ [82796148,67108959],[67108962,-71355523],
+ [67108984,-62077338,-77539719],[126106374,89478485],
+ [89478488,85703113],[132215738,89478495],
+ [89478515,-122049151],[89478518,-22611374],
+ [94050181,89478530],[89478547,42736340],
+ [89478553,86641584],[129419863,111848199],
+ [111848217,-32493354],[112586988,111848229]]
+ },
+
+ HashKey = internal_hash(0),
+ #{ HashKey := Keys } = ByMethod,
+
+ verify_colliding_keys(Keys, Mask).
+
+verify_colliding_keys([[K | Ks]=Group | Gs], Mask) ->
+ Hash = internal_hash(K) band Mask,
+ [Hash] = lists:usort([(internal_hash(Key) band Mask) || Key <- Ks]),
+ [Group | verify_colliding_keys(Gs, Mask)];
+verify_colliding_keys([], _Mask) ->
+ [].
internal_hash(Term) ->
erts_debug:get_internal_state({internal_hash,Term}).
-%% Use this function to (re)generate the list in colliding_keys/0
-%%
-%% Grab a coffee, it will take a while.
-find_colliding_keys() ->
- erts_debug:set_internal_state(available_internal_state, true),
- NumScheds = erlang:system_info(schedulers_online),
- Start = -(1 bsl 32),
- End = -Start,
- Range = End - Start,
- Step = Range div NumScheds,
- timer:tc(fun() -> fck_spawn(NumScheds, NumScheds, Start, End, Step, []) end).
-
-fck_spawn(0, _NumScheds, _Start, _End, _Step, Refs) ->
- fck_await(Refs);
-fck_spawn(N, NumScheds, Start, End, Step, Refs) ->
- Key = Start + (N - 1) * Step,
- {_, Ref} = spawn_monitor(fun() -> exit(fck_finder(Start, End, Key)) end),
- fck_spawn(N - 1, NumScheds, Start, End, Step, [Ref | Refs]).
-
-fck_await([Ref | Refs]) ->
- receive
- {'DOWN', Ref, _, _, [_Initial]} ->
- %% Ignore slices where the initial value only collided with itself.
- fck_await(Refs);
- {'DOWN', Ref, _, _, Collisions} ->
- [Collisions | fck_await(Refs)]
- end;
-fck_await([]) ->
- [].
-
-fck_finder(Start, End, Key) ->
- true = Key >= Start, true = Key < End, %Assertion.
- fck_finder_1(Start, End, internal_hash(Key)).
-
-fck_finder_1(Same, Same, _Target) ->
- [];
-fck_finder_1(Next, End, Target) ->
- case internal_hash(Next) =:= Target of
- true -> [Next | fck_finder_1(Next + 1, End, Target)];
- false -> fck_finder_1(Next + 1, End, Target)
- end.
-
%% OTP-17700 Bug skipped refc++ of shared magic reference
shared_magic_ref(_Config) ->
Ref = atomics:new(10, []),
--
2.35.3
