File 1133-Always-combine-is_tuple-and-test_arity-instructions.patch of Package erlang
From befdce1cd7bf227fccf8f319b4a9f68d4febab26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sat, 9 Sep 2023 17:57:17 +0200
Subject: [PATCH 13/25] Always combine is_tuple and test_arity instructions
We used to only combine `is_tuple` and `test_arity` instructions
having the same failure labels.
---
erts/emulator/beam/jit/arm/instr_common.cpp | 28 +++++++++++++++++++
erts/emulator/beam/jit/arm/ops.tab | 5 ++++
erts/emulator/beam/jit/x86/instr_common.cpp | 31 +++++++++++++++++++++
erts/emulator/beam/jit/x86/ops.tab | 5 ++++
4 files changed, 69 insertions(+)
diff --git a/erts/emulator/beam/jit/arm/instr_common.cpp b/erts/emulator/beam/jit/arm/instr_common.cpp
index 117c17a098..15e5d54553 100644
--- a/erts/emulator/beam/jit/arm/instr_common.cpp
+++ b/erts/emulator/beam/jit/arm/instr_common.cpp
@@ -1418,6 +1418,34 @@ void BeamModuleAssembler::emit_i_is_tuple_of_arity(const ArgLabel &Fail,
a.b_ne(resolve_beam_label(Fail, disp1MB));
}
+/* Note: This instruction leaves the untagged pointer to the tuple in
+ * ARG1. */
+void BeamModuleAssembler::emit_i_is_tuple_of_arity_ff(const ArgLabel &NotTuple,
+ const ArgLabel &BadArity,
+ const ArgSource &Src,
+ const ArgWord &Arity) {
+ auto src = load_source(Src, ARG1);
+
+ emit_is_boxed(resolve_beam_label(NotTuple, dispUnknown), Src, src.reg);
+
+ emit_untag_ptr(ARG1, src.reg);
+
+ a.ldr(TMP1, arm::Mem(ARG1));
+
+ /* As an optimization for the `error | {ok, Value}` case, skip checking the
+ * header word when we know that the only possible boxed type is a tuple. */
+ if (masked_types<BeamTypeId::MaybeBoxed>(Src) == BeamTypeId::Tuple) {
+ comment("skipped header test since we know it's a tuple when boxed");
+ } else {
+ ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0);
+ a.tst(TMP1, imm(_TAG_HEADER_MASK));
+ a.b_ne(resolve_beam_label(NotTuple, disp1MB));
+ }
+
+ cmp_arg(TMP1, Arity);
+ a.b_ne(resolve_beam_label(BadArity, disp1MB));
+}
+
/* Note: This instruction leaves the untagged pointer to the tuple in
* ARG1. */
void BeamModuleAssembler::emit_i_test_arity(const ArgLabel &Fail,
diff --git a/erts/emulator/beam/jit/arm/ops.tab b/erts/emulator/beam/jit/arm/ops.tab
index 1f2c95612e..5e75154e81 100644
--- a/erts/emulator/beam/jit/arm/ops.tab
+++ b/erts/emulator/beam/jit/arm/ops.tab
@@ -207,6 +207,10 @@ is_tuple Fail=f Src | test_arity Fail2 Src2 Arity |
equal(Fail, Fail2) | equal(Src, Src) =>
i_is_tuple_of_arity Fail Src Arity | current_tuple Src
+is_tuple Fail1=f Src | test_arity Fail2 Src2 Arity |
+ equal(Src, Src) =>
+ i_is_tuple_of_arity_ff Fail1 Fail2 Src Arity | current_tuple Src
+
test_arity Fail Src Arity => i_test_arity Fail Src Arity | current_tuple Src
is_tuple NotTupleFail Src |
@@ -221,6 +225,7 @@ is_tagged_tuple Fail Tuple Arity Atom =>
is_tuple Fail=f Src => i_is_tuple Fail Src | current_tuple Src
i_is_tuple_of_arity f s A
+i_is_tuple_of_arity_ff f f s A
i_test_arity f s A
i_is_tagged_tuple f s A a
diff --git a/erts/emulator/beam/jit/x86/instr_common.cpp b/erts/emulator/beam/jit/x86/instr_common.cpp
index fa45b9e0cd..48b21e8bc6 100644
--- a/erts/emulator/beam/jit/x86/instr_common.cpp
+++ b/erts/emulator/beam/jit/x86/instr_common.cpp
@@ -1336,6 +1336,37 @@ void BeamModuleAssembler::emit_i_is_tuple_of_arity(const ArgLabel &Fail,
a.jne(resolve_beam_label(Fail));
}
+/* Note: This instruction leaves the pointer to the tuple in ARG2. */
+void BeamModuleAssembler::emit_i_is_tuple_of_arity_ff(const ArgLabel &NotTuple,
+ const ArgLabel &BadArity,
+ const ArgSource &Src,
+ const ArgWord &Arity) {
+ mov_arg(ARG2, Src);
+
+ if (masked_types<BeamTypeId::MaybeBoxed>(Src) == BeamTypeId::Tuple) {
+ /* Fast path for the `error | {ok, Value}` case. */
+ comment("simplified tuple test since the source is always a tuple "
+ "when boxed");
+ /* We must be careful to still leave the pointer to the tuple
+ * in ARG2. */
+ (void)emit_ptr_val(ARG2, ARG2);
+ emit_test_boxed(ARG2);
+ a.jne(resolve_beam_label(NotTuple));
+ ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL)));
+ a.cmp(emit_boxed_val(ARG2, 0, sizeof(Uint32)), imm(Arity.get()));
+ a.jne(resolve_beam_label(BadArity));
+ } else {
+ emit_is_boxed(resolve_beam_label(NotTuple), Src, ARG2);
+ (void)emit_ptr_val(ARG2, ARG2);
+ ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL)));
+ a.mov(RETd, emit_boxed_val(ARG2, 0, sizeof(Uint32)));
+ a.test(RETb, imm(_TAG_HEADER_MASK));
+ a.jne(resolve_beam_label(NotTuple));
+ a.cmp(RETd, imm(Arity.get()));
+ a.jne(resolve_beam_label(BadArity));
+ }
+}
+
/* Note: This instruction leaves the pointer to the tuple in ARG2. */
void BeamModuleAssembler::emit_i_test_arity(const ArgLabel &Fail,
const ArgSource &Src,
diff --git a/erts/emulator/beam/jit/x86/ops.tab b/erts/emulator/beam/jit/x86/ops.tab
index c0cdcfe22d..39b9f32ef3 100644
--- a/erts/emulator/beam/jit/x86/ops.tab
+++ b/erts/emulator/beam/jit/x86/ops.tab
@@ -215,6 +215,10 @@ is_tuple Fail=f Src | test_arity Fail2 Src2 Arity |
equal(Fail, Fail2) | equal(Src, Src2) =>
i_is_tuple_of_arity Fail Src Arity | current_tuple Src
+is_tuple Fail1=f Src | test_arity Fail2 Src2 Arity |
+ equal(Src, Src) =>
+ i_is_tuple_of_arity_ff Fail1 Fail2 Src Arity | current_tuple Src
+
test_arity Fail Src Arity => i_test_arity Fail Src Arity | current_tuple Src
is_tuple NotTupleFail Src |
@@ -229,6 +233,7 @@ is_tagged_tuple Fail Tuple Arity Atom =>
is_tuple Fail=f Src => i_is_tuple Fail Src | current_tuple Src
i_is_tuple_of_arity f s A
+i_is_tuple_of_arity_ff f f s A
i_test_arity f s A
i_is_tagged_tuple f s A a
--
2.35.3
