File 1123-JIT-Optimize-creation-of-binaries.patch of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 1123-JIT-Optimize-creation-of-binaries.patch of Package erlang (Revision 8f70100112790b2ecaef2042dd03930d)

Currently displaying revision 8f70100112790b2ecaef2042dd03930d , Show latest

From 6fc65ab81fc9a5c92109e450f8947bd106c53eb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= <bjorn@erlang.org>
Date: Sun, 15 May 2022 06:21:33 +0200
Subject: [PATCH 3/3] JIT: Optimize creation of binaries

In the JIT, improve the code generation for binary construction
for segments of fixed widths of no more than 64 bits and for
variable width segments that are a set to 0.

I have run the benchmark linked to from https://github.com/erlang/otp/issues/5639.

On my Intel iMac from 2017, the result without this commit was:

== Testing with 1 MB ==
    fun base64:encode/1: 1000 iterations in 16264 ms: 61 it/sec
    fun base64:decode/1: 1000 iterations in 18597 ms: 53 it/sec

With this commit:

== Testing with 1 MB ==
    fun base64:encode/1: 1000 iterations in 10955 ms: 91 it/sec
    fun base64:decode/1: 1000 iterations in 10629 ms: 94 it/sec

On my M1 MacBook Pro from 2020 the result before was:

== Testing with 1 MB ==
    fun base64:encode/1: 1000 iterations in 12299 ms: 81 it/sec
    fun base64:decode/1: 1000 iterations in 15147 ms: 66 it/sec

After:

== Testing with 1 MB ==
    fun base64:encode/1: 1000 iterations in 8550 ms: 116 it/sec
    fun base64:decode/1: 1000 iterations in 10066 ms: 99 it/sec
---
 erts/emulator/beam/jit/arm/beam_asm.hpp       |   6 +
 .../beam/jit/arm/beam_asm_global.hpp.pl       |   1 +
 erts/emulator/beam/jit/arm/generators.tab     |  22 +
 erts/emulator/beam/jit/arm/instr_bs.cpp       | 614 +++++++++++++++-
 erts/emulator/beam/jit/beam_jit_args.hpp      |   5 +
 erts/emulator/beam/jit/x86/beam_asm.hpp       |   9 +
 .../beam/jit/x86/beam_asm_global.hpp.pl       |   1 +
 erts/emulator/beam/jit/x86/generators.tab     |  22 +
 erts/emulator/beam/jit/x86/instr_bs.cpp       | 690 ++++++++++++++++--
 erts/emulator/test/bs_construct_SUITE.erl     | 348 ++++++++-
 10 files changed, 1618 insertions(+), 100 deletions(-)

diff --git a/erts/emulator/beam/jit/arm/beam_asm.hpp b/erts/emulator/beam/jit/arm/beam_asm.hpp
index eb3baeefb2..485f904860 100644
--- a/erts/emulator/beam/jit/arm/beam_asm.hpp
+++ b/erts/emulator/beam/jit/arm/beam_asm.hpp
@@ -1267,6 +1267,12 @@ protected:
     void emit_bs_get_utf16(const ArgRegister &Ctx,
                            const ArgLabel &Fail,
                            const ArgWord &Flags);
+    void update_bin_state(arm::Gp bin_base,
+                          arm::Gp bin_offset,
+                          Sint bit_offset,
+                          Sint size,
+                          arm::Gp size_reg);
+    void set_zero(Sint effectiveSize);
 
     void emit_raise_exception();
     void emit_raise_exception(const ErtsCodeMFA *exp);
diff --git a/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl b/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl
index 0099e6cc94..3b7b2c9fa1 100644
--- a/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl
+++ b/erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl
@@ -56,6 +56,7 @@ my @beam_global_funcs = qw(
     dispatch_save_calls_fun
     export_trampoline
     fconv_shared
+    get_sint64_shared
     handle_and_error
     handle_call_fun_error
     handle_element_error_shared
diff --git a/erts/emulator/beam/jit/arm/generators.tab b/erts/emulator/beam/jit/arm/generators.tab
index b6ec48c678..3df71180db 100644
--- a/erts/emulator/beam/jit/arm/generators.tab
+++ b/erts/emulator/beam/jit/arm/generators.tab
@@ -567,6 +567,28 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) {
         Flags.val = flags;
         $NativeEndian(Flags);
         op->a[i+fixed_args+3] = Flags;
+
+        /*
+         * Replace short string segments with integer segments.
+         * Integer segments can be combined with adjacent integer
+         * segments for better performance.
+         */
+        if (op->a[i+fixed_args+0].val == am_string) {
+            Sint num_chars = op->a[i+fixed_args+5].val;
+            if (num_chars <= 4) {
+                Sint index = op->a[i+fixed_args+4].val;
+                const byte* s = S->beam.strings.data + index;
+                Uint num = 0;
+                op->a[i+fixed_args+0].val = am_integer;
+                op->a[i+fixed_args+2].val = 8;
+                op->a[i+fixed_args+5].val = num_chars;
+                while (num_chars-- > 0) {
+                    num = num << 8 | *s++;
+                }
+                op->a[i+fixed_args+4].type = TAG_i;
+                op->a[i+fixed_args+4].val = num;
+            }
+        }
     }
 
     if (op->a[4].val == am_private_append && Alloc.val != 0) {
diff --git a/erts/emulator/beam/jit/arm/instr_bs.cpp b/erts/emulator/beam/jit/arm/instr_bs.cpp
index d18585121d..6ac63dd473 100644
--- a/erts/emulator/beam/jit/arm/instr_bs.cpp
+++ b/erts/emulator/beam/jit/arm/instr_bs.cpp
@@ -1429,10 +1429,49 @@ void BeamGlobalAssembler::emit_bs_bit_size_shared() {
     a.ret(a64::x30);
 }
 
+/*
+ * ARG1 = tagged bignum term
+ */
+void BeamGlobalAssembler::emit_get_sint64_shared() {
+    Label success = a.newLabel();
+    Label fail = a.newLabel();
+
+    emit_is_boxed(fail, ARG1);
+    arm::Gp boxed_ptr = emit_ptr_val(TMP3, ARG1);
+    a.ldr(TMP1, emit_boxed_val(boxed_ptr));
+    a.ldr(TMP2, emit_boxed_val(boxed_ptr, sizeof(Eterm)));
+    a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK));
+    a.cmp(TMP1, imm(POS_BIG_SUBTAG));
+    a.b_eq(success);
+
+    a.cmp(TMP1, imm(NEG_BIG_SUBTAG));
+    a.b_ne(fail);
+
+    a.neg(TMP2, TMP2);
+
+    a.bind(success);
+    {
+        a.mov(ARG1, TMP2);
+        /* Clear Z flag.
+         *
+         * TMP1 is known to be POS_BIG_SUBTAG or NEG_BIG_SUBTAG at this point.
+         */
+        ERTS_CT_ASSERT(POS_BIG_SUBTAG != 0 && NEG_BIG_SUBTAG != 0);
+        a.tst(TMP1, TMP1);
+        a.ret(a64::x30);
+    }
+
+    a.bind(fail);
+    {
+        a.tst(ZERO, ZERO);
+        a.ret(a64::x30);
+    }
+}
+
 struct BscSegment {
     BscSegment()
             : type(am_false), unit(1), flags(0), src(ArgNil()), size(ArgNil()),
-              error_info(0), effectiveSize(-1) {
+              error_info(0), effectiveSize(-1), action(action::DIRECT) {
     }
 
     Eterm type;
@@ -1443,8 +1482,209 @@ struct BscSegment {
 
     Uint error_info;
     Sint effectiveSize;
+
+    /* Here are sub actions for storing integer segments.
+     *
+     * We use the ACCUMULATE_FIRST and ACCUMULATE actions to shift the
+     * values of segments with known, small sizes (no more than 64 bits)
+     * into an accumulator register.
+     *
+     * When no more segments can be accumulated, the STORE action is
+     * used to store the value of the accumulator into the binary.
+     *
+     * The DIRECT action is used when it is not possible to use the
+     * accumulator (for unknown or too large sizes).
+     */
+    enum class action { DIRECT, ACCUMULATE_FIRST, ACCUMULATE, STORE } action;
 };
 
+static std::vector<BscSegment> bs_combine_segments(
+        const std::vector<BscSegment> segments) {
+    std::vector<BscSegment> segs;
+
+    for (auto seg : segments) {
+        switch (seg.type) {
+        case am_integer: {
+            if (!(0 < seg.effectiveSize && seg.effectiveSize <= 64)) {
+                /* Unknown or too large size. Handle using the default
+                 * DIRECT action. */
+                segs.push_back(seg);
+                continue;
+            }
+
+            if (seg.flags & BSF_LITTLE || segs.size() == 0 ||
+                segs.back().action == BscSegment::action::DIRECT) {
+                /* There are no previous compatible ACCUMULATE / STORE
+                 * actions. Create the first ones. */
+                seg.action = BscSegment::action::ACCUMULATE_FIRST;
+                segs.push_back(seg);
+                seg.action = BscSegment::action::STORE;
+                segs.push_back(seg);
+                continue;
+            }
+
+            auto prev = segs.back();
+            if (prev.flags & BSF_LITTLE) {
+                /* Little-endian segments cannot be combined with other
+                 * segments. Create new ACCUMULATE_FIRST / STORE actions. */
+                seg.action = BscSegment::action::ACCUMULATE_FIRST;
+                segs.push_back(seg);
+                seg.action = BscSegment::action::STORE;
+                segs.push_back(seg);
+                continue;
+            }
+
+            /* The current segment is compatible with the previous
+             * segment. Try combining them. */
+            if (prev.effectiveSize + seg.effectiveSize <= 64) {
+                /* The combined values of the segments fit in the
+                 * accumulator. Insert an ACCUMULATE action for the
+                 * current segment before the pre-existing STORE
+                 * action. */
+                segs.pop_back();
+                prev.effectiveSize += seg.effectiveSize;
+                seg.action = BscSegment::action::ACCUMULATE;
+                segs.push_back(seg);
+                segs.push_back(prev);
+            } else {
+                /* The size exceeds 64 bits. Can't combine. */
+                seg.action = BscSegment::action::ACCUMULATE_FIRST;
+                segs.push_back(seg);
+                seg.action = BscSegment::action::STORE;
+                segs.push_back(seg);
+            }
+            break;
+        }
+        default:
+            segs.push_back(seg);
+            break;
+        }
+    }
+    return segs;
+}
+
+void BeamModuleAssembler::update_bin_state(arm::Gp bin_base,
+                                           arm::Gp bin_offset,
+                                           Sint bit_offset,
+                                           Sint size,
+                                           arm::Gp size_reg) {
+    int cur_bin_offset = offsetof(ErtsSchedulerRegisters,
+                                  aux_regs.d.erl_bits_state.erts_current_bin_);
+    arm::Mem mem_bin_base = arm::Mem(scheduler_registers, cur_bin_offset);
+    arm::Mem mem_bin_offset =
+            arm::Mem(scheduler_registers, cur_bin_offset + sizeof(Eterm));
+
+    if (bit_offset % 8) {
+        /* The bit offset is unknown or not byte-aligned. */
+        ERTS_CT_ASSERT_FIELD_PAIR(struct erl_bits_state,
+                                  erts_current_bin_,
+                                  erts_bin_offset_);
+        a.ldp(bin_base, bin_offset, mem_bin_base);
+
+        if (size_reg.isValid()) {
+            a.add(TMP1, bin_offset, size_reg);
+        } else {
+            a.add(TMP1, bin_offset, imm(size));
+        }
+        a.str(TMP1, mem_bin_offset);
+
+        a.add(TMP1, bin_base, bin_offset, arm::lsr(3));
+    } else {
+        comment("optimized updating of binary construction state");
+        ASSERT(size >= 0);
+        ASSERT(bit_offset % 8 == 0);
+        a.ldr(TMP1, mem_bin_base);
+        mov_imm(TMP2, bit_offset + size);
+        a.str(TMP2, mem_bin_offset);
+        add(TMP1, TMP1, bit_offset >> 3);
+    }
+}
+
+/*
+ * The size of the segment is assumed to be in ARG3.
+ */
+void BeamModuleAssembler::set_zero(Sint effectiveSize) {
+    Label store_units = a.newLabel();
+    Label less_than_a_store_unit = a.newLabel();
+    Sint store_unit = 1;
+
+    update_bin_state(ARG1, ARG2, -1, -1, ARG3);
+
+    if (effectiveSize >= 256) {
+        /* Store four 64-bit words machine words when the size is
+         * known and at least 256 bits. */
+        store_unit = 4;
+        a.movi(a64::d31, 0);
+    } else if (effectiveSize >= 128) {
+        /* Store two 64-bit words machine words when the size is
+         * known and at least 128 bits. */
+        store_unit = 2;
+    }
+
+    if (effectiveSize < Sint(store_unit * 8 * sizeof(Eterm))) {
+        /* The size is either not known or smaller than a word. */
+        a.cmp(ARG3, imm(store_unit * 8 * sizeof(Eterm)));
+        a.b_lt(less_than_a_store_unit);
+    }
+
+    a.bind(store_units);
+    if (store_unit == 4) {
+        a.stp(a64::q31, a64::q31, arm::Mem(TMP1).post(sizeof(Eterm[4])));
+    } else if (store_unit == 2) {
+        a.stp(ZERO, ZERO, arm::Mem(TMP1).post(sizeof(Eterm[2])));
+    } else {
+        a.str(ZERO, arm::Mem(TMP1).post(sizeof(Eterm)));
+    }
+    a.sub(ARG3, ARG3, imm(store_unit * 8 * sizeof(Eterm)));
+
+    a.cmp(ARG3, imm(store_unit * 8 * sizeof(Eterm)));
+    a.b_ge(store_units);
+
+    a.bind(less_than_a_store_unit);
+    if (effectiveSize < 0) {
+        /* Unknown size. */
+        Label byte_loop = a.newLabel();
+        Label done = a.newLabel();
+
+        ASSERT(store_unit = 1);
+
+        a.cbz(ARG3, done);
+
+        a.bind(byte_loop);
+        a.strb(ZERO.w(), arm::Mem(TMP1).post(1));
+        a.subs(ARG3, ARG3, imm(8));
+        a.b_gt(byte_loop);
+
+        a.bind(done);
+    } else if (effectiveSize % (store_unit * 8 * sizeof(Eterm)) != 0) {
+        /* The size is known, and we know that there are less than
+         * 256 bits to initialize. */
+        if (store_unit == 4 && (effectiveSize & 255) >= 128) {
+            a.stp(ZERO, ZERO, arm::Mem(TMP1).post(16));
+        }
+
+        if ((effectiveSize & 127) >= 64) {
+            a.str(ZERO, arm::Mem(TMP1).post(8));
+        }
+
+        if ((effectiveSize & 63) >= 32) {
+            a.str(ZERO.w(), arm::Mem(TMP1).post(4));
+        }
+
+        if ((effectiveSize & 31) >= 16) {
+            a.strh(ZERO.w(), arm::Mem(TMP1).post(2));
+        }
+
+        if ((effectiveSize & 15) >= 8) {
+            a.strb(ZERO.w(), arm::Mem(TMP1).post(1));
+        }
+
+        if ((effectiveSize & 7) > 0) {
+            a.strb(ZERO.w(), arm::Mem(TMP1));
+        }
+    }
+}
+
 void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
                                                const ArgWord &Alloc,
                                                const ArgWord &Live0,
@@ -1453,9 +1693,11 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
     Uint num_bits = 0;
     std::size_t n = args.size();
     std::vector<BscSegment> segments;
-    Label error;
+    Label error; /* Intentionally uninitialized */
     ArgWord Live = Live0;
     arm::Gp sizeReg;
+    Sint allocated_size = -1;
+    bool need_error_handler = false;
 
     /*
      * Collect information about each segment and calculate sizes of
@@ -1500,6 +1742,23 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
          */
         seg.error_info = beam_jit_set_bsc_segment_op(bsc_segment, bsc_op);
 
+        /*
+         * Test whether we can omit the code for the error handler.
+         */
+        switch (seg.type) {
+        case am_integer:
+            if (!always_one_of(seg.src, BEAM_TYPE_INTEGER)) {
+                need_error_handler = true;
+            }
+            break;
+        case am_private_append:
+        case am_string:
+            break;
+        default:
+            need_error_handler = true;
+            break;
+        }
+
         /*
          * Attempt to calculate the effective size of this segment.
          * Give up is variable or invalid.
@@ -1520,14 +1779,15 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             /* At least one segment will need a dynamic size
              * calculation. */
             sizeReg = ARG8;
+            need_error_handler = true;
         }
 
         segments.insert(segments.end(), seg);
     }
 
-    if (Fail.get() != 0) {
+    if (need_error_handler && Fail.get() != 0) {
         error = resolve_beam_label(Fail, dispUnknown);
-    } else {
+    } else if (need_error_handler) {
         Label past_error = a.newLabel();
 
         a.b(past_error);
@@ -1550,6 +1810,8 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
         }
 
         a.bind(past_error);
+    } else {
+        comment("(cannot fail)");
     }
 
     /* We count the total number of bits in an unsigned integer. To
@@ -1773,6 +2035,38 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
         runtime_call<4>(erts_bs_private_append_checked);
         emit_leave_runtime(Live.get());
         /* There is no way the call can fail on a 64-bit architecture. */
+    } else if (!sizeReg.isValid() && num_bits % 8 == 0 &&
+               num_bits / 8 <= ERL_ONHEAP_BIN_LIMIT) {
+        Uint need;
+        Uint num_bytes = num_bits / 8;
+        int cur_bin_offset =
+                offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) +
+                offsetof(struct erl_bits_state, erts_current_bin_);
+        arm::Mem mem_bin_base = arm::Mem(scheduler_registers, cur_bin_offset);
+        arm::Mem mem_bin_offset =
+                arm::Mem(scheduler_registers, cur_bin_offset + sizeof(Eterm));
+        ERTS_CT_ASSERT_FIELD_PAIR(struct erl_bits_state,
+                                  erts_current_bin_,
+                                  erts_bin_offset_);
+
+        comment("allocate heap binary");
+        allocated_size = (num_bytes + 7) & (-8);
+
+        /* Ensure that there is sufficient room on the heap. */
+        need = heap_bin_size(num_bytes) + Alloc.get();
+        emit_gc_test(ArgWord(0), ArgWord(need), Live);
+
+        /* Create the heap binary. */
+        a.add(ARG1, HTOP, imm(TAG_PRIMARY_BOXED));
+        mov_imm(TMP1, header_heap_bin(num_bytes));
+        mov_imm(TMP2, num_bytes);
+        a.stp(TMP1, TMP2, arm::Mem(HTOP).post(sizeof(Eterm[2])));
+
+        /* Initialize the erl_bin_state struct. */
+        a.stp(HTOP, ZERO, mem_bin_base);
+
+        /* Update HTOP. */
+        a.add(HTOP, HTOP, imm(allocated_size));
     } else {
         comment("allocate binary");
         mov_arg(ARG5, Alloc);
@@ -1786,11 +2080,11 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             comment("(size in bits)");
             a.mov(ARG4, sizeReg);
             runtime_call<6>(beam_jit_bs_init_bits);
-        } else if (num_bits % 8 == 0) {
-            comment("(size in bytes)");
-            mov_imm(ARG4, num_bits / 8);
-            runtime_call<6>(beam_jit_bs_init);
         } else {
+            allocated_size = (num_bits + 7) / 8;
+            if (allocated_size <= ERL_ONHEAP_BIN_LIMIT) {
+                allocated_size = (allocated_size + 7) & (-8);
+            }
             mov_imm(ARG4, num_bits);
             runtime_call<6>(beam_jit_bs_init_bits);
         }
@@ -1799,11 +2093,16 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
     }
     a.str(ARG1, TMP_MEM1q);
 
+    segments = bs_combine_segments(segments);
+
+    Sint bit_offset = 0;
+
     /* Build each segment of the binary. */
     for (auto seg : segments) {
         switch (seg.type) {
         case am_append:
         case am_private_append:
+            bit_offset = -1;
             break;
         case am_binary: {
             Uint error_info;
@@ -1904,38 +2203,274 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             emit_branch_if_value(ARG1, resolve_label(error, dispUnknown));
             break;
         case am_integer:
-            comment("construct integer segment");
-            if (seg.effectiveSize >= 0) {
-                mov_imm(ARG3, seg.effectiveSize);
-            } else {
-                mov_arg(ARG3, seg.size);
-                a.asr(ARG3, ARG3, imm(_TAG_IMMED1_SIZE));
-                if (seg.unit != 1) {
-                    mov_imm(TMP1, seg.unit);
-                    a.mul(ARG3, ARG3, TMP1);
+            switch (seg.action) {
+            case BscSegment::action::ACCUMULATE_FIRST:
+            case BscSegment::action::ACCUMULATE: {
+                /* Shift an integer of known size (no more than 64 bits)
+                 * into a word-size accumulator. */
+                Label value_is_small = a.newLabel();
+                Label done = a.newLabel();
+
+                comment("accumulate value for integer segment");
+                auto src = load_source(seg.src, ARG1);
+                if (seg.effectiveSize < 64 &&
+                    seg.action == BscSegment::action::ACCUMULATE) {
+                    a.lsl(ARG8, ARG8, imm(seg.effectiveSize));
+                }
+
+                if (!always_small(seg.src)) {
+                    if (always_one_of(seg.src,
+                                      BEAM_TYPE_INTEGER |
+                                              BEAM_TYPE_MASK_ALWAYS_BOXED)) {
+                        comment("simplified small test since all other types "
+                                "are boxed");
+                        emit_is_boxed(value_is_small, seg.src, src.reg);
+                    } else {
+                        a.and_(TMP1, src.reg, imm(_TAG_IMMED1_MASK));
+                        a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
+                        a.b_eq(value_is_small);
+                    }
+
+                    /* The value is boxed. If it is a bignum, extract the
+                     * least significant 64 bits. */
+                    mov_var(ARG1, src);
+                    fragment_call(ga->get_get_sint64_shared());
+                    if (seg.effectiveSize == 64) {
+                        a.mov(ARG8, ARG1);
+                    } else {
+                        a.bfxil(ARG8,
+                                ARG1,
+                                arm::lsr(0),
+                                imm(seg.effectiveSize));
+                    }
+
+                    if (always_one_of(seg.src, BEAM_TYPE_INTEGER)) {
+                        a.b(done);
+                    } else {
+                        a.b_ne(done);
+
+                        /* Not a bignum. Signal error. */
+                        if (Fail.get() == 0) {
+                            mov_imm(ARG4,
+                                    beam_jit_update_bsc_reason_info(
+                                            seg.error_info,
+                                            BSC_REASON_BADARG,
+                                            BSC_INFO_TYPE,
+                                            BSC_VALUE_ARG1));
+                        }
+                        a.b(resolve_label(error, disp128MB));
+                    }
+                }
+
+                a.bind(value_is_small);
+                if (seg.effectiveSize == 64) {
+                    a.asr(ARG8, src.reg, imm(_TAG_IMMED1_SIZE));
+                } else if (seg.effectiveSize + _TAG_IMMED1_SIZE > 64) {
+                    a.asr(TMP1, src.reg, imm(_TAG_IMMED1_SIZE));
+                    a.bfxil(ARG8, TMP1, arm::lsr(0), imm(seg.effectiveSize));
+                } else {
+                    a.bfxil(ARG8,
+                            src.reg,
+                            arm::lsr(_TAG_IMMED1_SIZE),
+                            imm(seg.effectiveSize));
                 }
+
+                a.bind(done);
+                break;
             }
-            mov_arg(ARG2, seg.src);
-            mov_imm(ARG4, seg.flags);
-            load_erl_bits_state(ARG1);
+            case BscSegment::action::STORE: {
+                /* The accumulator is now full or the next segment is
+                 * not possible to accumulate, so it's time to store
+                 * the accumulator to the current position in the
+                 * binary. */
+                Label store = a.newLabel();
+                Label done = a.newLabel();
+
+                comment("construct integer segment from accumulator");
+
+                /* First we'll need to ensure that the value in the
+                 * accumulator is in little endian format. */
+                ASSERT(seg.effectiveSize >= 0);
+                if (seg.effectiveSize % 8) {
+                    if (seg.flags & BSF_LITTLE) {
+                        a.bfc(ARG8,
+                              arm::lsr(seg.effectiveSize),
+                              64 - seg.effectiveSize);
+                    } else {
+                        a.lsl(ARG8, ARG8, imm(64 - seg.effectiveSize));
+                        a.rev64(ARG8, ARG8);
+                    }
+                } else if ((seg.flags & BSF_LITTLE) == 0) {
+                    switch (seg.effectiveSize) {
+                    case 8:
+                        break;
+                    case 16:
+                        a.rev16(ARG8, ARG8);
+                        break;
+                    case 32:
+                        a.rev32(ARG8, ARG8);
+                        break;
+                    case 64:
+                        a.rev64(ARG8, ARG8);
+                        break;
+                    default:
+                        a.rev64(ARG8, ARG8);
+                        a.lsr(ARG8, ARG8, imm(64 - seg.effectiveSize));
+                    }
+                }
 
-            emit_enter_runtime(Live.get());
-            runtime_call<4>(erts_new_bs_put_integer);
-            emit_leave_runtime(Live.get());
+                arm::Gp bin_base = ARG2;
+                arm::Gp bin_offset = ARG3;
+                arm::Gp bin_data = ARG8;
 
-            if (exact_type(seg.src, BEAM_TYPE_INTEGER)) {
-                comment("skipped test for success because construction can't "
-                        "fail");
-            } else {
-                if (Fail.get() == 0) {
-                    mov_arg(ARG3, seg.src);
-                    mov_imm(ARG4,
-                            beam_jit_update_bsc_reason_info(seg.error_info,
-                                                            BSC_REASON_BADARG,
-                                                            BSC_INFO_TYPE,
-                                                            BSC_VALUE_ARG3));
+                update_bin_state(bin_base,
+                                 bin_offset,
+                                 bit_offset,
+                                 seg.effectiveSize,
+                                 arm::Gp());
+
+                if (bit_offset < 0) {
+                    /* Bit offset is unknown. Must test alignment. */
+                    a.tst(bin_offset, imm(7));
+                    a.b_eq(store);
+                }
+
+                if (bit_offset % 8 != 0) {
+                    /* Bit offset is tested or known to be unaligned. */
+                    a.str(bin_data, TMP_MEM2q); /* MEM1q is already in use. */
+                    lea(ARG1, TMP_MEM2q);
+                    mov_imm(ARG4, seg.effectiveSize);
+
+                    emit_enter_runtime(Live.get());
+                    runtime_call<4>(erts_copy_bits_restricted);
+                    emit_leave_runtime(Live.get());
+
+                    if (bit_offset < 0) {
+                        /* Need to jump around the store code. */
+                        a.b(done);
+                    }
+                }
+
+                a.bind(store);
+
+                if (bit_offset < 0 || bit_offset % 8 == 0) {
+                    /* Bit offset is tested or known to be
+                     * byte-aligned. Emit inline code to store the
+                     * value of the accumulator into the binary. */
+                    int num_bytes = (seg.effectiveSize + 7) / 8;
+
+                    /* If more than one instruction is required for
+                     * doing the store, test whether it would be safe
+                     * to do a single 32 or 64 bit store. */
+                    switch (num_bytes) {
+                    case 3:
+                        if (bit_offset >= 0 &&
+                            allocated_size * 8 - bit_offset >= 32) {
+                            comment("simplified complicated store");
+                            num_bytes = 4;
+                        }
+                        break;
+                    case 5:
+                    case 6:
+                    case 7:
+                        if (bit_offset >= 0 &&
+                            allocated_size * 8 - bit_offset >= 64) {
+                            comment("simplified complicated store");
+                            num_bytes = 8;
+                        }
+                        break;
+                    }
+
+                    do {
+                        switch (num_bytes) {
+                        case 1:
+                            a.strb(bin_data.w(), arm::Mem(TMP1));
+                            break;
+                        case 2:
+                            a.strh(bin_data.w(), arm::Mem(TMP1));
+                            break;
+                        case 3:
+                            a.strh(bin_data.w(), arm::Mem(TMP1));
+                            a.lsr(bin_data, bin_data, imm(16));
+                            a.strb(bin_data.w(), arm::Mem(TMP1, 2));
+                            break;
+                        case 4:
+                            a.str(bin_data.w(), arm::Mem(TMP1));
+                            break;
+                        case 5:
+                        case 6:
+                        case 7:
+                            a.str(bin_data.w(), arm::Mem(TMP1).post(4));
+                            a.lsr(bin_data, bin_data, imm(32));
+                            break;
+                        case 8:
+                            a.str(bin_data, arm::Mem(TMP1));
+                            num_bytes = 0;
+                            break;
+                        }
+                        num_bytes -= 4;
+                    } while (num_bytes > 0);
+                }
+
+                a.bind(done);
+                break;
+            }
+            case BscSegment::action::DIRECT:
+                /* This segment either has a size exceeding the maximum
+                 * accumulator size of 64 bits or has a variable size.
+                 *
+                 * First load the effective size (size * unit) into ARG3.
+                 */
+                comment("construct integer segment");
+                if (seg.effectiveSize >= 0) {
+                    mov_imm(ARG3, seg.effectiveSize);
+                } else {
+                    auto size = load_source(seg.size, TMP1);
+                    a.lsr(ARG3, size.reg, imm(_TAG_IMMED1_SIZE));
+                    if (Support::isPowerOf2(seg.unit)) {
+                        Uint trailing_bits = Support::ctz<Eterm>(seg.unit);
+                        if (trailing_bits) {
+                            a.lsl(ARG3, ARG3, imm(trailing_bits));
+                        }
+                    } else {
+                        mov_imm(TMP1, seg.unit);
+                        a.mul(ARG3, ARG3, TMP1);
+                    }
+                }
+
+                if (bit_offset % 8 == 0 && seg.src.isSmall() &&
+                    seg.src.as<ArgSmall>().getSigned() == 0) {
+                    /* Optimize the special case of setting a known
+                     * byte-aligned segment to zero. */
+                    comment("optimized setting segment to 0");
+                    set_zero(seg.effectiveSize);
+                } else {
+                    /* Call the helper function to fetch and store the
+                     * integer into the binary. */
+                    mov_arg(ARG2, seg.src);
+                    mov_imm(ARG4, seg.flags);
+                    load_erl_bits_state(ARG1);
+
+                    emit_enter_runtime(Live.get());
+                    runtime_call<4>(erts_new_bs_put_integer);
+                    emit_leave_runtime(Live.get());
+
+                    if (exact_type(seg.src, BEAM_TYPE_INTEGER)) {
+                        comment("skipped test for success because construction "
+                                "can't fail");
+                    } else {
+                        if (Fail.get() == 0) {
+                            mov_arg(ARG3, seg.src);
+                            mov_imm(ARG4,
+                                    beam_jit_update_bsc_reason_info(
+                                            seg.error_info,
+                                            BSC_REASON_BADARG,
+                                            BSC_INFO_TYPE,
+                                            BSC_VALUE_ARG3));
+                        }
+                        a.cbz(ARG1, resolve_label(error, disp1MB));
+                    }
                 }
-                a.cbz(ARG1, resolve_label(error, disp1MB));
             }
             break;
         case am_string: {
@@ -2016,6 +2551,15 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             ASSERT(0);
             break;
         }
+
+        if (bit_offset >= 0 && (seg.action == BscSegment::action::DIRECT ||
+                                seg.action == BscSegment::action::STORE)) {
+            if (seg.effectiveSize >= 0) {
+                bit_offset += seg.effectiveSize;
+            } else {
+                bit_offset = -1;
+            }
+        }
     }
 
     comment("done");
diff --git a/erts/emulator/beam/jit/beam_jit_args.hpp b/erts/emulator/beam/jit/beam_jit_args.hpp
index 8e1e0359ee..4dba1b3f4f 100644
--- a/erts/emulator/beam/jit/beam_jit_args.hpp
+++ b/erts/emulator/beam/jit/beam_jit_args.hpp
@@ -246,6 +246,11 @@ struct ArgRegister : public ArgSource {
     constexpr int typeIndex() const {
         return (int)(val >> 10);
     }
+
+    template<typename T>
+    constexpr T copy(int n) const {
+        return T(n | (val & ~REG_MASK));
+    }
 };
 
 struct ArgXRegister : public ArgRegister {
diff --git a/erts/emulator/beam/jit/x86/beam_asm.hpp b/erts/emulator/beam/jit/x86/beam_asm.hpp
index da037aa235..94ec4dcffa 100644
--- a/erts/emulator/beam/jit/x86/beam_asm.hpp
+++ b/erts/emulator/beam/jit/x86/beam_asm.hpp
@@ -1263,6 +1263,15 @@ protected:
     void emit_bs_get_utf16(const ArgRegister &Ctx,
                            const ArgLabel &Fail,
                            const ArgWord &Flags);
+    void update_bin_state(x86::Gp bin_base,
+                          x86::Gp bin_offset,
+                          Sint bit_offset,
+                          Sint size,
+                          x86::Gp size_reg);
+    bool need_mask(const ArgVal Val, Sint size);
+    void set_zero(Sint effectiveSize);
+    bool bs_maybe_enter_runtime(bool entered);
+    void bs_maybe_leave_runtime(bool entered);
 
     void emit_raise_exception();
     void emit_raise_exception(const ErtsCodeMFA *exp);
diff --git a/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl b/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl
index 917a2815e5..2fa14f3ad9 100755
--- a/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl
+++ b/erts/emulator/beam/jit/x86/beam_asm_global.hpp.pl
@@ -48,6 +48,7 @@ my @beam_global_funcs = qw(
     garbage_collect
     generic_bp_global
     generic_bp_local
+    get_sint64_shared
     debug_bp
     fconv_shared
     handle_call_fun_error
diff --git a/erts/emulator/beam/jit/x86/generators.tab b/erts/emulator/beam/jit/x86/generators.tab
index 4b0b2ad043..143e91dab7 100644
--- a/erts/emulator/beam/jit/x86/generators.tab
+++ b/erts/emulator/beam/jit/x86/generators.tab
@@ -635,6 +635,28 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) {
         Flags.val = flags;
         $NativeEndian(Flags);
         op->a[i+fixed_args+3] = Flags;
+
+        /*
+         * Replace short string segments with integer segments.
+         * Integer segments can be combined with adjacent integer
+         * segments for better performance.
+         */
+        if (op->a[i+fixed_args+0].val == am_string) {
+            Sint num_chars = op->a[i+fixed_args+5].val;
+            if (num_chars <= 4) {
+                Sint index = op->a[i+fixed_args+4].val;
+                const byte* s = S->beam.strings.data + index;
+                Uint num = 0;
+                op->a[i+fixed_args+0].val = am_integer;
+                op->a[i+fixed_args+2].val = 8;
+                op->a[i+fixed_args+5].val = num_chars;
+                while (num_chars-- > 0) {
+                    num = num << 8 | *s++;
+                }
+                op->a[i+fixed_args+4].type = TAG_i;
+                op->a[i+fixed_args+4].val = num;
+            }
+        }
     }
 
     if (op->a[4].val == am_private_append && Alloc.val != 0) {
diff --git a/erts/emulator/beam/jit/x86/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp
index 91438fbdc2..1c308bb6c8 100644
--- a/erts/emulator/beam/jit/x86/instr_bs.cpp
+++ b/erts/emulator/beam/jit/x86/instr_bs.cpp
@@ -1582,10 +1582,52 @@ void BeamGlobalAssembler::emit_bs_create_bin_error_shared() {
     a.jmp(labels[raise_exception_shared]);
 }
 
+/*
+ * ARG1 = tagged bignum term
+ *
+ * On return, Z is set if ARG1 is not a bignum. Otherwise, Z is clear and
+ * ARG1 is the 64 least significant bits of the bignum.
+ */
+void BeamGlobalAssembler::emit_get_sint64_shared() {
+    Label success = a.newLabel();
+    Label fail = a.newLabel();
+
+    emit_is_boxed(fail, ARG1);
+    x86::Gp boxed_ptr = emit_ptr_val(ARG4, ARG1);
+    a.mov(ARG2, emit_boxed_val(boxed_ptr));
+    a.mov(ARG3, emit_boxed_val(boxed_ptr, sizeof(Eterm)));
+    a.and_(ARG2, imm(_TAG_HEADER_MASK));
+    a.cmp(ARG2, imm(POS_BIG_SUBTAG));
+    a.je(success);
+
+    a.cmp(ARG2, imm(NEG_BIG_SUBTAG));
+    a.jne(fail);
+
+    a.neg(ARG3);
+
+    a.bind(success);
+    {
+        a.mov(ARG1, ARG3);
+        /* Clear Z flag.
+         *
+         * ARG2 is known to be POS_BIG_SUBTAG or NEG_BIG_SUBTAG at this point.
+         */
+        ERTS_CT_ASSERT(POS_BIG_SUBTAG != 0 && NEG_BIG_SUBTAG != 0);
+        a.test(ARG2, ARG2);
+        a.ret();
+    }
+
+    a.bind(fail);
+    {
+        a.xor_(ARG2, ARG2); /* Set Z flag */
+        a.ret();
+    }
+}
+
 struct BscSegment {
     BscSegment()
             : type(am_false), unit(1), flags(0), src(ArgNil()), size(ArgNil()),
-              error_info(0), effectiveSize(-1) {
+              error_info(0), effectiveSize(-1), action(action::DIRECT) {
     }
 
     Eterm type;
@@ -1596,8 +1638,226 @@ struct BscSegment {
 
     Uint error_info;
     Sint effectiveSize;
+
+    /* Here are sub actions for storing integer segments.
+     *
+     * We use the ACCUMULATE_FIRST and ACCUMULATE actions to shift the
+     * values of segments with known, small sizes (no more than 64 bits)
+     * into an accumulator register.
+     *
+     * When no more segments can be accumulated, the STORE action is
+     * used to store the value of the accumulator into the binary.
+     *
+     * The DIRECT action is used when it is not possible to use the
+     * accumulator (for unknown or too large sizes).
+     */
+    enum class action { DIRECT, ACCUMULATE_FIRST, ACCUMULATE, STORE } action;
 };
 
+static std::vector<BscSegment> bs_combine_segments(
+        const std::vector<BscSegment> segments) {
+    std::vector<BscSegment> segs;
+
+    for (auto seg : segments) {
+        switch (seg.type) {
+        case am_integer: {
+            if (!(0 < seg.effectiveSize && seg.effectiveSize <= 64)) {
+                /* Unknown or too large size. Handle using the default
+                 * DIRECT action. */
+                segs.push_back(seg);
+                continue;
+            }
+
+            if (seg.flags & BSF_LITTLE || segs.size() == 0 ||
+                segs.back().action == BscSegment::action::DIRECT) {
+                /* There are no previous compatible ACCUMULATE / STORE
+                 * actions. Create the first ones. */
+                seg.action = BscSegment::action::ACCUMULATE_FIRST;
+                segs.push_back(seg);
+                seg.action = BscSegment::action::STORE;
+                segs.push_back(seg);
+                continue;
+            }
+
+            auto prev = segs.back();
+            if (prev.flags & BSF_LITTLE) {
+                /* Little-endian segments cannot be combined with other
+                 * segments. Create new ACCUMULATE_FIRST / STORE actions. */
+                seg.action = BscSegment::action::ACCUMULATE_FIRST;
+                segs.push_back(seg);
+                seg.action = BscSegment::action::STORE;
+                segs.push_back(seg);
+                continue;
+            }
+
+            /* The current segment is compatible with the previous
+             * segment. Try combining them. */
+            if (prev.effectiveSize + seg.effectiveSize <= 64) {
+                /* The combined values of the segments fits in the
+                 * accumulator. Insert an ACCUMULATE action for the
+                 * current segment before the pre-existing STORE
+                 * action. */
+                segs.pop_back();
+                prev.effectiveSize += seg.effectiveSize;
+                seg.action = BscSegment::action::ACCUMULATE;
+                segs.push_back(seg);
+                segs.push_back(prev);
+            } else {
+                /* The size exceeds 64 bits. Can't combine. */
+                seg.action = BscSegment::action::ACCUMULATE_FIRST;
+                segs.push_back(seg);
+                seg.action = BscSegment::action::STORE;
+                segs.push_back(seg);
+            }
+            break;
+        }
+        default:
+            segs.push_back(seg);
+            break;
+        }
+    }
+    return segs;
+}
+
+void BeamModuleAssembler::update_bin_state(x86::Gp bin_base,
+                                           x86::Gp bin_offset,
+                                           Sint bit_offset,
+                                           Sint size,
+                                           x86::Gp size_reg) {
+    const int x_reg_offset = offsetof(ErtsSchedulerRegisters, x_reg_array.d);
+    const int cur_bin_base =
+            offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) +
+            offsetof(struct erl_bits_state, erts_current_bin_);
+    const int cur_bin_offset =
+            offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) +
+            offsetof(struct erl_bits_state, erts_bin_offset_);
+
+    x86::Mem mem_bin_base =
+            x86::Mem(registers, cur_bin_base - x_reg_offset, sizeof(UWord));
+    x86::Mem mem_bin_offset =
+            x86::Mem(registers, cur_bin_offset - x_reg_offset, sizeof(UWord));
+
+    if (bit_offset % 8 != 0 || !Support::isInt32(bit_offset + size)) {
+        /* The bit offset is unknown or not byte-aligned. Alternatively,
+         * the sum of bit_offset and size does not fit in an immediate. */
+        a.mov(bin_base, mem_bin_base);
+        a.mov(bin_offset, mem_bin_offset);
+
+        a.mov(ARG1, bin_offset);
+        if (size_reg.isValid()) {
+            a.add(mem_bin_offset, size_reg);
+        } else {
+            a.add(mem_bin_offset, imm(size));
+        }
+        a.shr(ARG1, imm(3));
+        a.add(ARG1, bin_base);
+    } else {
+        ASSERT(size >= 0);
+        ASSERT(bit_offset % 8 == 0);
+
+        comment("optimized updating of binary construction state");
+        a.mov(ARG1, mem_bin_base);
+        if (bit_offset) {
+            a.add(ARG1, imm(bit_offset >> 3));
+        }
+        a.mov(mem_bin_offset, imm(bit_offset + size));
+    }
+}
+
+bool BeamModuleAssembler::need_mask(const ArgVal Val, Sint size) {
+    if (size == 64) {
+        return false;
+    } else if (always_small(Val)) {
+        auto [min, max] = getIntRange(Val);
+        return !(0 <= min && max >> size == 0);
+    } else {
+        return true;
+    }
+}
+
+/*
+ * The size of the segment is assumed to be in ARG3.
+ */
+void BeamModuleAssembler::set_zero(Sint effectiveSize) {
+    update_bin_state(RET, ARG2, -1, -1, ARG3);
+
+    mov_imm(RET, 0);
+
+    if (effectiveSize < 0 || effectiveSize > 128) {
+        /* Size is unknown or greater than 128. Modern CPUs have an
+         * enhanced "rep stosb" instruction that in most circumstances
+         * is the fastest way to clear blocks of more than 128
+         * bytes. */
+        Label done = a.newLabel();
+
+        if (effectiveSize < 0) {
+            a.test(ARG3, ARG3);
+            a.short_().jz(done);
+        }
+
+        if (ARG1 != x86::rdi) {
+            a.mov(x86::rdi, ARG1);
+        }
+        a.mov(x86::rcx, ARG3);
+        a.add(x86::rcx, imm(7));
+        a.shr(x86::rcx, imm(3));
+        a.rep().stosb();
+
+        a.bind(done);
+    } else {
+        /* The size is known and it is at most 128 bits. */
+        Uint offset = 0;
+
+        ASSERT(0 <= effectiveSize && effectiveSize <= 128);
+
+        if (effectiveSize == 128) {
+            a.mov(x86::Mem(ARG1, offset, 8), RET);
+            offset += 8;
+        }
+
+        if (effectiveSize >= 64) {
+            a.mov(x86::Mem(ARG1, offset, 8), RET);
+            offset += 8;
+        }
+
+        if ((effectiveSize & 63) >= 32) {
+            a.mov(x86::Mem(ARG1, offset, 4), RETd);
+            offset += 4;
+        }
+
+        if ((effectiveSize & 31) >= 16) {
+            a.mov(x86::Mem(ARG1, offset, 2), RET.r16());
+            offset += 2;
+        }
+
+        if ((effectiveSize & 15) >= 8) {
+            a.mov(x86::Mem(ARG1, offset, 1), RET.r8());
+            offset += 1;
+        }
+
+        if ((effectiveSize & 7) > 0) {
+            a.mov(x86::Mem(ARG1, offset, 1), RET.r8());
+        }
+    }
+}
+
+bool BeamModuleAssembler::bs_maybe_enter_runtime(bool entered) {
+    if (!entered) {
+        comment("enter runtime");
+        emit_enter_runtime<Update::eReductions | Update::eStack |
+                           Update::eHeap>();
+    }
+    return true;
+}
+
+void BeamModuleAssembler::bs_maybe_leave_runtime(bool entered) {
+    if (entered) {
+        comment("leave runtime");
+        emit_leave_runtime<Update::eReductions | Update::eStack |
+                           Update::eHeap>();
+    }
+}
+
 void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
                                                const ArgWord &Alloc,
                                                const ArgWord &Live0,
@@ -1606,10 +1866,12 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
     Uint num_bits = 0;
     std::size_t n = args.size();
     std::vector<BscSegment> segments;
-    Label error = a.newLabel();
-    Label past_error = a.newLabel();
+    Label error; /* Intentionally uninitialized */
     ArgWord Live = Live0;
     x86::Gp sizeReg;
+    Sint allocated_size = -1;
+    bool need_error_handler = false;
+    bool runtime_entered = false;
 
     /*
      * Collect information about each segment and calculate sizes of
@@ -1654,13 +1916,33 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
          */
         seg.error_info = beam_jit_set_bsc_segment_op(bsc_segment, bsc_op);
 
+        /*
+         * Test whether we can omit the code for the error handler.
+         */
+        switch (seg.type) {
+        case am_integer:
+            if (!always_one_of(seg.src, BEAM_TYPE_INTEGER)) {
+                need_error_handler = true;
+            }
+            break;
+        case am_private_append:
+        case am_string:
+            break;
+        default:
+            need_error_handler = true;
+            break;
+        }
+
         /*
          * As soon as we have entered runtime mode, Y registers can no
          * longer be accessed in the usual way. Therefore, if the source
-         * and/or size are in Y register, copy them to X registers.
+         * and/or size are in Y registers, copy them to X registers. Be
+         * careful to preserve any associated type information.
          */
         if (seg.src.isYRegister()) {
-            ArgVal reg = ArgXRegister(Live.get());
+            auto reg =
+                    seg.src.as<ArgYRegister>().copy<ArgXRegister>(Live.get());
+            ASSERT(reg.typeIndex() == seg.src.as<ArgYRegister>().typeIndex());
             mov_arg(reg, seg.src);
 
             Live = Live + 1;
@@ -1668,7 +1950,9 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
         }
 
         if (seg.size.isYRegister()) {
-            ArgVal reg = ArgXRegister(Live.get());
+            auto reg =
+                    seg.size.as<ArgYRegister>().copy<ArgXRegister>(Live.get());
+            ASSERT(reg.typeIndex() == seg.size.as<ArgYRegister>().typeIndex());
             mov_arg(reg, seg.size);
 
             Live = Live + 1;
@@ -1689,16 +1973,64 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
         if (seg.effectiveSize < 0 && seg.type != am_append &&
             seg.type != am_private_append) {
             sizeReg = FCALLS;
+            need_error_handler = true;
         }
 
         segments.insert(segments.end(), seg);
     }
 
-    emit_enter_runtime<Update::eReductions | Update::eStack | Update::eHeap>();
+    /*
+     * Test whether a heap binary of fixed size will result from the
+     * construction. If so, allocate and construct the binary now
+     * before entering the runtime mode.
+     */
+    if (!sizeReg.isValid() && num_bits % 8 == 0 &&
+        num_bits / 8 <= ERL_ONHEAP_BIN_LIMIT && segments[0].type != am_append &&
+        segments[0].type != am_private_append) {
+        const int x_reg_offset =
+                offsetof(ErtsSchedulerRegisters, x_reg_array.d);
+        const int cur_bin_base =
+                offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) +
+                offsetof(struct erl_bits_state, erts_current_bin_);
+        const int cur_bin_offset =
+                offsetof(ErtsSchedulerRegisters, aux_regs.d.erl_bits_state) +
+                offsetof(struct erl_bits_state, erts_bin_offset_);
+        x86::Mem mem_bin_base =
+                x86::qword_ptr(registers, cur_bin_base - x_reg_offset);
+        x86::Mem mem_bin_offset =
+                x86::qword_ptr(registers, cur_bin_offset - x_reg_offset);
+        Uint num_bytes = num_bits / 8;
+
+        comment("allocate heap binary");
+        allocated_size = (num_bytes + 7) & (-8);
+
+        /* Ensure that there is enough room on the heap. */
+        Uint need = heap_bin_size(num_bytes) + Alloc.get();
+        emit_gc_test(ArgWord(0), ArgWord(need), Live);
+
+        /* Create the heap binary. */
+        a.lea(RET, x86::qword_ptr(HTOP, TAG_PRIMARY_BOXED));
+        a.mov(TMP_MEM1q, RET);
+        a.mov(x86::qword_ptr(HTOP, 0), imm(header_heap_bin(num_bytes)));
+        a.mov(x86::qword_ptr(HTOP, sizeof(Eterm)), imm(num_bytes));
+
+        /* Initialize the erl_bin_state struct. */
+        a.add(HTOP, imm(sizeof(Eterm[2])));
+        a.mov(mem_bin_base, HTOP);
+        a.mov(mem_bin_offset, imm(0));
+
+        /* Update HTOP. */
+        a.add(HTOP, imm(allocated_size));
+    }
+
+    if (!need_error_handler) {
+        comment("(cannot fail)");
+    } else {
+        Label past_error = a.newLabel();
+
+        runtime_entered = bs_maybe_enter_runtime(false);
+        a.short_().jmp(past_error);
 
-    a.short_().jmp(past_error);
-    a.bind(error);
-    {
         /*
          * ARG1 = optional bad size value; valid if BSC_VALUE_ARG1 is set in
          * ARG4
@@ -1708,17 +2040,18 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
          *
          * ARG4 = packed error information
          */
+        error = a.newLabel();
+        a.bind(error);
+        bs_maybe_leave_runtime(runtime_entered);
         comment("handle error");
-        emit_leave_runtime<Update::eReductions | Update::eStack |
-                           Update::eHeap>();
         if (Fail.get() != 0) {
             a.jmp(resolve_beam_label(Fail));
         } else {
             safe_fragment_call(ga->get_bs_create_bin_error_shared());
         }
-    }
 
-    a.bind(past_error);
+        a.bind(past_error);
+    }
 
     /* We count the total number of bits in an unsigned integer. To
      * avoid having to check for overflow when adding to the counter,
@@ -1742,6 +2075,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
 
         if (seg.size.isAtom() && seg.size.as<ArgAtom>().get() == am_all &&
             seg.type == am_binary) {
+            runtime_entered = bs_maybe_enter_runtime(runtime_entered);
             comment("size of an entire binary");
             mov_arg(ARG1, seg.src);
             runtime_call<1>(beam_jit_bs_bit_size);
@@ -1886,9 +2220,12 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
         }
     }
 
+    segments = bs_combine_segments(segments);
+
     /* Allocate the binary. */
     if (segments[0].type == am_append) {
         BscSegment seg = segments[0];
+        runtime_entered = bs_maybe_enter_runtime(runtime_entered);
         comment("append to binary");
         mov_arg(ARG3, Live);
         if (sizeReg.isValid()) {
@@ -1912,8 +2249,10 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
         }
         emit_test_the_non_value(RET);
         a.je(error);
+        a.mov(TMP_MEM1q, RET);
     } else if (segments[0].type == am_private_append) {
         BscSegment seg = segments[0];
+        runtime_entered = bs_maybe_enter_runtime(runtime_entered);
         comment("private append to binary");
         ASSERT(Alloc.get() == 0);
         mov_arg(ARG2, seg.src);
@@ -1926,38 +2265,45 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
         a.mov(ARG1, c_p);
         runtime_call<4>(erts_bs_private_append_checked);
         /* There is no way the call can fail on a 64-bit architecture. */
+        a.mov(TMP_MEM1q, RET);
+    } else if (allocated_size >= 0) {
+        /* The binary has already been allocated. */
     } else {
         comment("allocate binary");
+        runtime_entered = bs_maybe_enter_runtime(runtime_entered);
         mov_arg(ARG5, Alloc);
         mov_arg(ARG6, Live);
         load_erl_bits_state(ARG3);
         load_x_reg_array(ARG2);
         a.mov(ARG1, c_p);
         if (sizeReg.isValid()) {
-            comment("(size in bits)");
             a.mov(ARG4, sizeReg);
             runtime_call<6>(beam_jit_bs_init_bits);
-        } else if (num_bits % 8 == 0) {
-            comment("(size in bytes)");
-            mov_imm(ARG4, num_bits / 8);
-            runtime_call<6>(beam_jit_bs_init);
         } else {
+            allocated_size = (num_bits + 7) / 8;
+            if (allocated_size <= ERL_ONHEAP_BIN_LIMIT) {
+                allocated_size = (allocated_size + 7) & (-8);
+            }
             mov_imm(ARG4, num_bits);
             runtime_call<6>(beam_jit_bs_init_bits);
         }
+        a.mov(TMP_MEM1q, RET);
     }
-    a.mov(TMP_MEM1q, RET);
+
+    Sint bit_offset = 0;
 
     /* Build each segment of the binary. */
     for (auto seg : segments) {
         switch (seg.type) {
         case am_append:
         case am_private_append:
+            bit_offset = -1;
             break;
         case am_binary: {
             Uint error_info;
             bool can_fail = true;
 
+            runtime_entered = bs_maybe_enter_runtime(runtime_entered);
             comment("construct a binary segment");
             if (seg.effectiveSize >= 0) {
                 /* The segment has a literal size. */
@@ -2016,6 +2362,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             break;
         }
         case am_float:
+            runtime_entered = bs_maybe_enter_runtime(runtime_entered);
             comment("construct float segment");
             if (seg.effectiveSize >= 0) {
                 mov_imm(ARG3, seg.effectiveSize);
@@ -2044,42 +2391,277 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             a.jne(error);
             break;
         case am_integer:
-            comment("construct integer segment");
-            if (seg.effectiveSize >= 0) {
-                mov_imm(ARG3, seg.effectiveSize);
-            } else {
-                mov_arg(ARG3, seg.size);
-                a.sar(ARG3, imm(_TAG_IMMED1_SIZE));
-                if (seg.unit != 1) {
-                    mov_imm(RET, seg.unit);
-                    a.mul(ARG3); /* CLOBBERS RDX = ARG3! */
-                    a.mov(ARG3, RET);
+            switch (seg.action) {
+            case BscSegment::action::ACCUMULATE_FIRST:
+            case BscSegment::action::ACCUMULATE: {
+                /* Shift an integer of known size (no more than 64 bits)
+                 * into a word-size accumulator. */
+                Label accumulate = a.newLabel();
+                Label value_is_small = a.newLabel();
+                x86::Gp tmp = ARG4;
+                x86::Gp bin_data = ARG5;
+
+                comment("accumulate value for integer segment");
+                if (seg.action == BscSegment::action::ACCUMULATE_FIRST) {
+                    mov_imm(bin_data, 0);
+                } else if (seg.effectiveSize < 64) {
+                    a.shl(bin_data, imm(seg.effectiveSize));
+                }
+                mov_arg(ARG1, seg.src);
+
+                if (!always_small(seg.src)) {
+                    if (always_one_of(seg.src,
+                                      BEAM_TYPE_INTEGER |
+                                              BEAM_TYPE_MASK_ALWAYS_BOXED)) {
+                        comment("simplified small test since all other types "
+                                "are boxed");
+                        emit_is_boxed(value_is_small, seg.src, ARG1);
+                    } else {
+                        a.mov(ARG2d, ARG1d);
+                        a.and_(ARG2d, imm(_TAG_IMMED1_MASK));
+                        a.cmp(ARG2d, imm(_TAG_IMMED1_SMALL));
+                        a.short_().je(value_is_small);
+                    }
+
+                    /* The value is boxed. If it is a bignum, extract the
+                     * least significant 64 bits. */
+                    safe_fragment_call(ga->get_get_sint64_shared());
+                    if (always_one_of(seg.src, BEAM_TYPE_INTEGER)) {
+                        a.short_().jmp(accumulate);
+                    } else {
+                        a.short_().jne(accumulate);
+
+                        /* Not a bignum. Signal error. */
+                        if (Fail.get() == 0) {
+                            mov_imm(ARG4,
+                                    beam_jit_update_bsc_reason_info(
+                                            seg.error_info,
+                                            BSC_REASON_BADARG,
+                                            BSC_INFO_TYPE,
+                                            BSC_VALUE_ARG1));
+                        }
+                        a.jmp(error);
+                    }
+                }
+
+                a.bind(value_is_small);
+                a.sar(ARG1, imm(_TAG_IMMED1_SIZE));
+
+                /* Mask (if needed) and accumulate. */
+                a.bind(accumulate);
+                if (seg.effectiveSize == 64) {
+                    a.mov(bin_data, ARG1);
+                } else if (!need_mask(seg.src, seg.effectiveSize)) {
+                    comment("skipped masking because the value always fits");
+                    a.or_(bin_data, ARG1);
+                } else if (seg.effectiveSize == 32) {
+                    a.mov(ARG1d, ARG1d);
+                    a.or_(bin_data, ARG1);
+                } else if (seg.effectiveSize < 32) {
+                    a.and_(ARG1, (1ULL << seg.effectiveSize) - 1);
+                    a.or_(bin_data, ARG1);
+                } else {
+                    mov_imm(tmp, (1ULL << seg.effectiveSize) - 1);
+                    a.and_(ARG1, tmp);
+                    a.or_(bin_data, ARG1);
                 }
+                break;
             }
-            mov_arg(ARG2, seg.src);
-            mov_imm(ARG4, seg.flags);
-            load_erl_bits_state(ARG1);
-            runtime_call<4>(erts_new_bs_put_integer);
-            if (exact_type(seg.src, BEAM_TYPE_INTEGER)) {
-                comment("skipped test for success because construction can't "
-                        "fail");
-            } else {
-                if (Fail.get() == 0) {
-                    mov_arg(ARG1, seg.src);
-                    mov_imm(ARG4,
-                            beam_jit_update_bsc_reason_info(seg.error_info,
-                                                            BSC_REASON_BADARG,
-                                                            BSC_INFO_TYPE,
-                                                            BSC_VALUE_ARG1));
+            case BscSegment::action::STORE: {
+                /* The accumulator is now full or the next segment is
+                 * not possible to accumulate, so it's time to store
+                 * the accumulator to the current position in the
+                 * binary. */
+                Label store = a.newLabel();
+                Label done = a.newLabel();
+                x86::Gp bin_ptr = ARG1;
+                x86::Gp bin_base = ARG2;
+                x86::Gp bin_offset = ARG3;
+                x86::Gp tmp = ARG4;
+                x86::Gp bin_data = ARG5;
+
+                comment("construct integer segment from accumulator");
+
+                /* First we'll need to ensure that the value in the
+                 * accumulator is in little endian format. */
+                if (seg.effectiveSize % 8 != 0) {
+                    if ((seg.flags & BSF_LITTLE) == 0) {
+                        a.shl(bin_data, imm(64 - seg.effectiveSize));
+                        a.bswap(bin_data);
+                    }
+                } else if ((seg.flags & BSF_LITTLE) == 0) {
+                    switch (seg.effectiveSize) {
+                    case 8:
+                        break;
+                    case 32:
+                        a.bswap(bin_data.r32());
+                        break;
+                    case 64:
+                        a.bswap(bin_data);
+                        break;
+                    default:
+                        a.bswap(bin_data);
+                        a.shr(bin_data, imm(64 - seg.effectiveSize));
+                        break;
+                    }
                 }
-                a.test(RETd, RETd);
-                a.je(error);
+
+                update_bin_state(bin_base,
+                                 bin_offset,
+                                 bit_offset,
+                                 seg.effectiveSize,
+                                 x86::Gp());
+
+                if (bit_offset < 0) {
+                    /* Bit offset is unknown. Must emit an alignment test. */
+                    a.test(bin_offset, imm(7));
+                    a.short_().je(store);
+                }
+
+                if (bit_offset % 8 != 0) {
+                    /* Bit offset is unknown or known to be unaligned. */
+                    runtime_entered = bs_maybe_enter_runtime(runtime_entered);
+                    a.mov(TMP_MEM2q, bin_data); /* MEM1q is already in use. */
+                    a.lea(ARG1, TMP_MEM2q);
+                    mov_imm(tmp, seg.effectiveSize);
+
+                    runtime_call<4>(erts_copy_bits_restricted);
+
+                    if (bit_offset < 0) {
+                        /* Need to jump around the store code. */
+                        a.short_().jmp(done);
+                    }
+                }
+
+                a.bind(store);
+                if (bit_offset <= 0 || bit_offset % 8 == 0) {
+                    /* Bit offset is tested or known to be
+                     * byte-aligned. Emit inline code to store the
+                     * value of the accumulator into the binary. */
+                    int num_bytes = (seg.effectiveSize + 7) / 8;
+
+                    /* If more than one instruction is required for
+                     * doing the store, test whether it would be safe
+                     * to do a single 32 or 64 bit store. */
+                    switch (num_bytes) {
+                    case 3:
+                        if (bit_offset >= 0 &&
+                            allocated_size * 8 - bit_offset >= 32) {
+                            comment("simplified complicated store");
+                            num_bytes = 4;
+                        }
+                        break;
+                    case 5:
+                    case 6:
+                    case 7:
+                        if (bit_offset >= 0 &&
+                            allocated_size * 8 - bit_offset >= 64) {
+                            comment("simplified complicated store");
+                            num_bytes = 8;
+                        }
+                        break;
+                    }
+
+                    do {
+                        switch (num_bytes) {
+                        case 1:
+                            a.mov(x86::Mem(bin_ptr, 0, 1), bin_data.r8());
+                            break;
+                        case 2:
+                            a.mov(x86::Mem(bin_ptr, 0, 2), bin_data.r16());
+                            break;
+                        case 3:
+                            a.mov(x86::Mem(bin_ptr, 0, 2), bin_data.r16());
+                            a.shr(bin_data, imm(16));
+                            a.mov(x86::Mem(bin_ptr, 2, 1), bin_data.r8());
+                            break;
+                        case 4:
+                            a.mov(x86::Mem(bin_ptr, 0, 4), bin_data.r32());
+                            break;
+                        case 5:
+                        case 6:
+                        case 7:
+                            a.mov(x86::Mem(bin_ptr, 0, 4), bin_data.r32());
+                            a.add(bin_ptr, imm(4));
+                            a.shr(bin_data, imm(32));
+                            break;
+                        case 8:
+                            a.mov(x86::Mem(bin_ptr, 0, 8), bin_data);
+                            num_bytes = 0;
+                            break;
+                        default:
+                            ASSERT(0);
+                        }
+                        num_bytes -= 4;
+                    } while (num_bytes > 0);
+                }
+
+                a.bind(done);
+                break;
+            }
+            case BscSegment::action::DIRECT:
+                /* This segment either has a size exceeding the maximum
+                 * accumulator size of 64 bits or has a variable size.
+                 *
+                 * First load the effective size (size * unit) into ARG3.
+                 */
+                comment("construct integer segment");
+                if (seg.effectiveSize >= 0) {
+                    mov_imm(ARG3, seg.effectiveSize);
+                } else {
+                    mov_arg(ARG3, seg.size);
+                    a.sar(ARG3, imm(_TAG_IMMED1_SIZE));
+                    if (Support::isPowerOf2(seg.unit)) {
+                        Uint trailing_bits = Support::ctz<Eterm>(seg.unit);
+                        if (trailing_bits) {
+                            a.shl(ARG3, imm(trailing_bits));
+                        }
+                    } else {
+                        mov_imm(RET, seg.unit);
+                        a.mul(ARG3); /* CLOBBERS RDX = ARG3! */
+                        a.mov(ARG3, RET);
+                    }
+                }
+
+                if (bit_offset % 8 == 0 && seg.src.isSmall() &&
+                    seg.src.as<ArgSmall>().getSigned() == 0) {
+                    /* Optimize the special case of setting a known
+                     * byte-aligned segment to zero. */
+                    comment("optimized setting segment to 0");
+                    set_zero(seg.effectiveSize);
+                } else {
+                    /* Call the helper function to fetch and store the
+                     * integer into the binary. */
+                    runtime_entered = bs_maybe_enter_runtime(runtime_entered);
+                    mov_arg(ARG2, seg.src);
+                    mov_imm(ARG4, seg.flags);
+                    load_erl_bits_state(ARG1);
+                    runtime_call<4>(erts_new_bs_put_integer);
+                    if (exact_type(seg.src, BEAM_TYPE_INTEGER)) {
+                        comment("skipped test for success because construction "
+                                "can't fail");
+                    } else {
+                        if (Fail.get() == 0) {
+                            mov_arg(ARG1, seg.src);
+                            mov_imm(ARG4,
+                                    beam_jit_update_bsc_reason_info(
+                                            seg.error_info,
+                                            BSC_REASON_BADARG,
+                                            BSC_INFO_TYPE,
+                                            BSC_VALUE_ARG1));
+                        }
+                        a.test(RETd, RETd);
+                        a.je(error);
+                    }
+                }
+                break;
             }
             break;
         case am_string: {
             ArgBytePtr string_ptr(
                     ArgVal(ArgVal::BytePtr, seg.src.as<ArgWord>().get()));
 
+            runtime_entered = bs_maybe_enter_runtime(runtime_entered);
             comment("insert string");
             ASSERT(seg.effectiveSize >= 0);
             mov_imm(ARG3, seg.effectiveSize / 8);
@@ -2088,6 +2670,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             runtime_call<3>(erts_new_bs_put_string);
         } break;
         case am_utf8:
+            runtime_entered = bs_maybe_enter_runtime(runtime_entered);
             mov_arg(ARG2, seg.src);
             load_erl_bits_state(ARG1);
             runtime_call<2>(erts_bs_put_utf8);
@@ -2103,6 +2686,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             a.je(error);
             break;
         case am_utf16:
+            runtime_entered = bs_maybe_enter_runtime(runtime_entered);
             mov_arg(ARG2, seg.src);
             a.mov(ARG3, seg.flags);
             load_erl_bits_state(ARG1);
@@ -2119,6 +2703,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             a.je(error);
             break;
         case am_utf32:
+            runtime_entered = bs_maybe_enter_runtime(runtime_entered);
             mov_arg(ARG2, seg.src);
             mov_imm(ARG3, 4 * 8);
             a.mov(ARG4, seg.flags);
@@ -2139,10 +2724,19 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgLabel &Fail,
             ASSERT(0);
             break;
         }
+
+        if (bit_offset >= 0 && (seg.action == BscSegment::action::DIRECT ||
+                                seg.action == BscSegment::action::STORE)) {
+            if (seg.effectiveSize >= 0) {
+                bit_offset += seg.effectiveSize;
+            } else {
+                bit_offset = -1;
+            }
+        }
     }
 
+    bs_maybe_leave_runtime(runtime_entered);
     comment("done");
-    emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>();
     a.mov(RET, TMP_MEM1q);
     mov_arg(Dst, RET);
 }
diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl
index 0d083bb8d7..c7a787bf13 100644
--- a/erts/emulator/test/bs_construct_SUITE.erl
+++ b/erts/emulator/test/bs_construct_SUITE.erl
@@ -28,7 +28,7 @@
 	 copy_writable_binary/1, kostis/1, dynamic/1, bs_add/1,
 	 otp_7422/1, zero_width/1, bad_append/1, bs_append_overflow/1,
          bs_append_offheap/1,
-         reductions/1, fp16/1, error_info/1]).
+         reductions/1, fp16/1, zero_init/1, error_info/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
@@ -41,7 +41,8 @@ all() ->
      huge_float_field, system_limit, badarg,
      copy_writable_binary, kostis, dynamic, bs_add, otp_7422, zero_width,
      bad_append, bs_append_overflow, bs_append_offheap,
-     reductions, fp16, error_info].
+     reductions, fp16, zero_init,
+     error_info].
 
 init_per_suite(Config) ->
     Config.
@@ -49,14 +50,6 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     application:stop(os_mon).
 
-big(1) ->
-    57285702734876389752897683.
-
-i(X) -> X.
-
-r(L) ->
-    lists:reverse(L).
-
 -define(T(B, L), {B, ??B, L}).
 -define(N(B), {B, ??B, unknown}).
 
@@ -89,7 +82,7 @@ l(I_13, I_big1) ->
      ?T(<<57285702734876389752897684:32>>,
 	[138, 99, 0, 148]),
      ?T(<<I_big1:32/little>>,
-	r([138, 99, 0, 147])),
+	lists:reverse([138, 99, 0, 147])),
      ?T(<<-1:17/unit:8>>,
 	lists:duplicate(17, 255)),
 
@@ -142,9 +135,74 @@ l(I_13, I_big1) ->
      ?T(<<<<5:3>>/bitstring>>, <<5:3>>),
      ?T(<<42,<<7:4>>/binary-unit:4>>, <<42,7:4>>),
      ?T(<<<<344:17>>/binary-unit:17>>, <<344:17>>),
-     ?T(<<<<42,3,7656:16>>/binary-unit:16>>, <<42,3,7656:16>>)
-
-     ].
+     ?T(<<<<42,3,7656:16>>/binary-unit:16>>, <<42,3,7656:16>>),
+
+     %% Different sizes and types. First without types.
+     ?T(<<I_big1:8>>, [147]),
+     ?T(<<I_big1:16>>, [0, 147]),
+     ?T(<<I_big1:24>>, [99, 0, 147]),
+     ?T(<<I_big1:32>>, [138, 99, 0, 147]),
+     ?T(<<I_big1:40>>, [5, 138, 99, 0, 147]),
+     ?T(<<I_big1:48>>, [229, 5, 138, 99, 0, 147]),
+     ?T(<<I_big1:56>>, [249, 229, 5, 138, 99, 0, 147]),
+     ?T(<<I_big1:64>>, [42, 249, 229, 5, 138, 99, 0, 147]),
+
+     %% Known integer with range.
+     ?T(<<(I_big1 band ((1 bsl 56) - 1)):8>>, [147]),
+     ?T(<<(I_big1 band ((1 bsl 56) - 1)):16>>, [0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 56) - 1)):24>>, [99, 0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 56) - 1)):32>>, [138, 99, 0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 56) - 1)):40>>, [5, 138, 99, 0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 56) - 1)):48>>, [229, 5, 138, 99, 0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 56) - 1)):56>>, [249, 229, 5, 138, 99, 0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 64) - 1)):64>>, [42, 249, 229, 5, 138, 99, 0, 147]),
+
+     %% Known integer with exact range.
+     ?T(<<(I_big1 band ((1 bsl  8) - 1)):8>>, [147]),
+     ?T(<<(I_big1 band ((1 bsl 16) - 1)):16>>, [0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 24) - 1)):24>>, [99, 0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 32) - 1)):32>>, [138, 99, 0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 40) - 1)):40>>, [5, 138, 99, 0, 147]),
+     ?T(<<(I_big1 band ((1 bsl 48) - 1)):48>>, [229, 5, 138, 99, 0, 147]),
+
+     %% Known integer without range.
+     ?T(<<(I_big1 + 0):8>>, [147]),
+     ?T(<<(I_big1 + 0):16>>, [0, 147]),
+     ?T(<<(I_big1 + 0):24>>, [99, 0, 147]),
+     ?T(<<(I_big1 + 0):32>>, [138, 99, 0, 147]),
+     ?T(<<(I_big1 + 0):40>>, [5, 138, 99, 0, 147]),
+     ?T(<<(I_big1 + 0):48>>, [229, 5, 138, 99, 0, 147]),
+     ?T(<<(I_big1 + 0):56>>, [249, 229, 5, 138, 99, 0, 147]),
+     ?T(<<(I_big1 + 0):64>>, [42, 249, 229, 5, 138, 99, 0, 147]),
+
+     %% Known integer. Verify that the value does not bleed into the
+     %% previous segment.
+     ?T(<<1, (I_big1 + 0):8>>,  [1, 147]),
+     ?T(<<2, (I_big1 + 0):16>>, [2, 0, 147]),
+     ?T(<<3, (I_big1 + 0):24>>, [3, 99, 0, 147]),
+     ?T(<<4, (I_big1 + 0):32>>, [4, 138, 99, 0, 147]),
+     ?T(<<5, (I_big1 + 0):40>>, [5, 5, 138, 99, 0, 147]),
+     ?T(<<6, (I_big1 + 0):48>>, [6, 229, 5, 138, 99, 0, 147]),
+     ?T(<<7, (I_big1 + 0):56>>, [7, 249, 229, 5, 138, 99, 0, 147]),
+     ?T(<<8, (I_big1 + 0):64>>, [8, 42, 249, 229, 5, 138, 99, 0, 147]),
+
+     %% Test non-byte sizes.
+     ?T(<<I_big1:33>>, <<197,49,128,73,1:1>>),
+     ?T(<<I_big1:39>>, <<11,20,198,1,19:7>>),
+
+     ?T(<<I_big1:57>>, <<124,242,130,197,49,128,73,1:1>>),
+     ?T(<<I_big1:58>>, <<190,121,65,98,152,192,36,3:2>>),
+     ?T(<<I_big1:59>>, <<95,60,160,177,76,96,18,3:3>>),
+     ?T(<<I_big1:60>>, <<175,158,80,88,166,48,9,3:4>>),
+     ?T(<<I_big1:61>>, <<87,207,40,44,83,24,4,19:5>>),
+     ?T(<<I_big1:62>>, <<171,231,148,22,41,140,2,19:6>>),
+     ?T(<<I_big1:63>>, <<85,243,202,11,20,198,1,19:7>>),
+
+     %% Test non-byte sizes and also that the value does not bleed
+     %% into the previous segment.
+     ?T(<<17, I_big1:33>>, <<17, 197,49,128,73,1:1>>),
+     ?T(<<19, I_big1:39>>, <<19, 11,20,198,1,19:7>>)
+    ].
 
 native_3798() ->
     case <<1:16/native>> of
@@ -274,10 +332,10 @@ fail_check(Res, _, _) ->
 
 %%% Simple working cases
 test1(Config) when is_list(Config) ->
-    I_13 = i(13),
-    I_big1 = big(1),
+    I_13 = id(13),
+    I_big1 = id(57285702734876389752897683),
     Vars = [{'I_13', I_13},
-		  {'I_big1', I_big1}],
+            {'I_big1', I_big1}],
     lists:foreach(fun one_test/1, eval_list(l(I_13, I_big1), Vars)).
 
 %%% Misc
@@ -717,6 +775,21 @@ dynamic_big(Bef, N, Int, Lpad, Rpad) ->
     Bin = id(<<Lpad:Bef,NumBin/bitstring,Rpad:(128-Bef-N)>>),
     Bin = <<Lpad:Bef,Int:N,Rpad:(128-Bef-N)>>,
 
+    %% Units are seldom used with integer segments even in our test
+    %% suites, and I have never seen non-power-of-two units in
+    %% production code.
+    if
+        Bef rem 8 =:= 0 ->
+            Bin = <<Lpad:(Bef div 8)/unit:8,Int:N,Rpad:(128-Bef-N)>>;
+        Bef rem 7 =:= 0 ->
+            Bin = <<Lpad:(Bef div 7)/unit:7,Int:N,Rpad:(128-Bef-N)>>;
+        (128-Bef-N) rem 5 =:= 0 ->
+            Aft = (128 - Bef - N) div 5,
+            Bin = <<Lpad:Bef,Int:N,Rpad:Aft/unit:5>>;
+        true ->
+            ok
+    end,
+
     %% Further verify the result by matching.
     LpadMasked = Lpad band ((1 bsl Bef) - 1),
     RpadMasked = Rpad band ((1 bsl (128-Bef-N)) - 1),
@@ -733,6 +806,20 @@ dynamic_little(Bef, N, Int, Lpad, Rpad) ->
     Bin = id(<<Lpad:Bef/little,NumBin/bitstring,Rpad:(128-Bef-N)/little>>),
     Bin = <<Lpad:Bef/little,Int:N/little,Rpad:(128-Bef-N)/little>>,
 
+    if
+        Bef rem 8 =:= 0 ->
+            Bin = <<Lpad:(Bef div 8)/little-unit:8,
+                    Int:N/little,Rpad:(128-Bef-N)/little>>;
+        Bef rem 9 =:= 0 ->
+            Bin = <<Lpad:(Bef div 9)/little-unit:9,
+                    Int:N/little,Rpad:(128-Bef-N)/little>>;
+        (128-Bef-N) rem 17 =:= 0 ->
+            Aft = (128 - Bef - N) div 17,
+            Bin = <<Lpad:Bef/little,Int:N/little,Rpad:Aft/little-unit:17>>;
+        true ->
+            ok
+    end,
+
     %% Further verify the result by matching.
     LpadMasked = Lpad band ((1 bsl Bef) - 1),
     RpadMasked = Rpad band ((1 bsl (128-Bef-N)) - 1),
@@ -1000,6 +1087,233 @@ fp16(_Config) ->
     ?FP16(16#4000, 2.0),
     ok.
 
+zero_init(_Config) ->
+    <<LPad:64/bits,RPad:64/bits>> = id(erlang:md5([42])),
+    Sizes = [511,512,513, 767,768,769, 1023,1024,1025,
+             16#7fff,16#ffff,16#10000] ++ lists:seq(0, 257),
+    _ = [do_zero_init(Size, LPad, RPad) || Size <- Sizes],
+    ok.
+
+do_zero_init(Size, LPad, RPad) ->
+    try
+        do_zero_init_1(Size, LPad, RPad)
+    catch
+        C:R:Stk ->
+            io:format("Size = ~p, LPad = ~p, RPad = ~p\n", [Size, LPad, RPad]),
+            erlang:raise(C, R, Stk)
+    end.
+
+do_zero_init_1(Size, LPad, RPad) ->
+    Zeroes = id(<<0:Size>>),
+    <<0:Size>> = Zeroes,
+    Bin = id(<<LPad:64/bits, Zeroes/bits, RPad:64/bits>>),
+    Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>),
+    Bin = id(<<LPad:(id(64))/bits, 0:Size, RPad:64/bits>>),
+
+    if
+        Size rem 11 =:= 0 ->
+            Bin = id(<<LPad:64/bits, 0:(Size div 11)/unit:11, RPad:64/bits>>);
+        true ->
+            ok
+    end,
+
+    case Size of
+        0 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        1 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        2 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        3 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        4 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        5 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        6 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        7 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        8 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        9 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        10 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        11 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        12 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        13 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        14 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        15 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        16 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        31 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        32 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        33 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        47 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        48 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        49 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        63 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        64 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        65 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        79 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        80 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        81 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        90 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        91 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        92 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        93 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        94 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        95 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        96 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        97 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        98 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        99 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        100 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        101 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        102 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        103 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        104 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        105 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        106 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        107 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        108 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        109 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        127 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        128 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        129 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        130 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        131 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        132 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        133 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        134 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        135 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        136 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        137 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        138 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        139 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        140 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        141 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        142 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        143 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        144 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        145 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        159 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        160 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        161 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        191 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        192 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        193 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        255 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        256 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        257 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        511 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        512 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        513 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        767 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        768 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        769 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        1023 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        1024 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        1025 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+
+        16#7fff ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        16#ffff ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        16#10000 ->
+            Bin = id(<<LPad:64/bits, 0:Size, RPad:64/bits>>);
+        _ ->
+            ok
+    end.
+
+
 -define(ERROR_INFO(Expr),
         fun() ->
                 try Expr of
-- 
2.35.3

Places

File 1123-JIT-Optimize-creation-of-binaries.patch of Package erlang (Revision 8f70100112790b2ecaef2042dd03930d)

Places