File 4631-erts-On-halt-and-delayed-halt-support-for-NIFs.patch of Package erlang

From 988a0e9f44d2d6a0c114acd1d19bc21b3dee2aea Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 6 Oct 2022 11:59:31 +0200
Subject: [PATCH] [erts] On-halt and delayed halt support for NIFs

---
 erts/doc/src/erl_nif.xml                      | 175 +++++++
 erts/doc/src/erlang.xml                       | 190 ++++++--
 erts/emulator/beam/break.c                    |   2 +-
 erts/emulator/beam/erl_db.c                   |  42 +-
 erts/emulator/beam/erl_init.c                 |  49 +-
 erts/emulator/beam/erl_lock_check.c           |   1 +
 erts/emulator/beam/erl_nif.c                  | 280 ++++++++++-
 erts/emulator/beam/erl_nif.h                  |  12 +-
 erts/emulator/beam/erl_nif_api_funcs.h        |   3 +
 erts/emulator/beam/erl_process.c              |  12 +-
 erts/emulator/beam/erl_process.h              |   5 +-
 erts/emulator/beam/global.h                   |   8 +-
 erts/emulator/beam/io.c                       |   2 +-
 erts/emulator/test/dirty_nif_SUITE.erl        | 444 +++++++++++++++++-
 .../test/dirty_nif_SUITE_data/Makefile.src    |   2 +-
 .../dirty_nif_SUITE_data/dirty_nif_SUITE.c    | 281 ++++++++++-
 .../test/dirty_nif_SUITE_data/on_halt_a.c     |  23 +
 .../test/dirty_nif_SUITE_data/on_halt_b.c     |  23 +
 .../test/dirty_nif_SUITE_data/on_halt_c.c     |  23 +
 .../test/dirty_nif_SUITE_data/on_halt_d.c     |  23 +
 .../test/dirty_nif_SUITE_data/on_halt_e.c     |  23 +
 .../test/dirty_nif_SUITE_data/on_halt_f.c     |  23 +
 .../test/dirty_nif_SUITE_data/on_halt_nif.c   |  96 ++++
 erts/preloaded/src/erlang.erl                 |  26 +-
 24 files changed, 1661 insertions(+), 107 deletions(-)
 create mode 100644 erts/emulator/test/dirty_nif_SUITE_data/on_halt_a.c
 create mode 100644 erts/emulator/test/dirty_nif_SUITE_data/on_halt_b.c
 create mode 100644 erts/emulator/test/dirty_nif_SUITE_data/on_halt_c.c
 create mode 100644 erts/emulator/test/dirty_nif_SUITE_data/on_halt_d.c
 create mode 100644 erts/emulator/test/dirty_nif_SUITE_data/on_halt_e.c
 create mode 100644 erts/emulator/test/dirty_nif_SUITE_data/on_halt_f.c
 create mode 100644 erts/emulator/test/dirty_nif_SUITE_data/on_halt_nif.c

diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml
index 4208aa269a..cb300a976d 100644
--- a/erts/doc/src/erl_nif.xml
+++ b/erts/doc/src/erl_nif.xml
@@ -785,6 +785,41 @@ typedef struct {
 	  To compare two monitors, <seecref marker="#enif_compare_monitors">
           <c>enif_compare_monitors</c></seecref> must be used.</p>
       </item>
+      <tag><marker id="ErlNifOnHaltCallback"/><c>ErlNifOnHaltCallback</c></tag>
+      <item>
+        <code type="none">
+typedef void ErlNifOnHaltCallback(void *priv_data);</code>
+        <p>
+          The function prototype of an <i>on halt</i> callback function.
+        </p>
+	<p>
+          An <i>on halt</i> callback can be installed using
+          <seecref marker="#on_halt"><c>enif_set_option()</c></seecref>. Such
+          an installed callback will be called when the runtime system is
+          halting.
+        </p>
+      </item>
+      <tag><marker id="ErlNifOption"/><c>ErlNifOption</c></tag>
+      <item>
+        <p>
+          An enumeration of the options that can be set using
+          <seecref marker="#enif_set_option"><c>enif_set_option()</c></seecref>.
+        </p>
+        <p>Currently valid options:</p>
+        <taglist>
+          <tag><seecref marker="#delay_halt"><c>ERL_NIF_OPT_DELAY_HALT</c></seecref></tag>
+          <item><p>
+            Enable delay of runtime system halt with flushing enabled until
+            all calls to NIFs in the NIF library have returned.
+          </p></item>
+          <tag><seecref marker="#on_halt"><c>ERL_NIF_OPT_ON_HALT</c></seecref></tag>
+          <item><p>
+            Install a callback that will be called when the runtime system
+            halts with flushing enabled.
+          </p></item>
+        </taglist>
+      </item>
+
       <tag><marker id="ErlNifPid"/><c>ErlNifPid</c></tag>
       <item>
         <p>A process identifier (pid). In contrast to pid terms (instances of
@@ -3412,6 +3447,146 @@ if (retval &amp; ERL_NIF_SELECT_STOP_CALLED) {
       </desc>
     </func>
 
+    <func>
+      <name since="OTP 26.0">
+        <ret>int</ret><nametext>enif_set_option(ErlNifEnv *env, ErlNifOption opt, ...)</nametext>
+      </name>
+      <fsummary>
+        Set an option.
+      </fsummary>
+      <desc>
+        <marker id="enif_set_option"/>
+        <p>
+          Set an option. On success, zero will be returned. On failure, a non
+          zero value will be returned. Currently the following options can be set:
+        </p>
+        <taglist>
+          <tag><marker id="delay_halt"/>
+          <c>int enif_set_option(ErlNifEnv *env,
+          </c><seecref marker="#ErlNifOption"><c>ERL_NIF_OPT_DELAY_HALT</c></seecref><c>)</c>
+          </tag>
+          <item>
+            <p>
+              Enable delay of runtime system halt with flushing enabled until
+              all calls to NIFs in the NIF library have returned. If the
+              <i>delay halt</i> feature has not been enabled, a halt with
+              flushing enabled may complete even though processes are still
+              executing inside NIFs in the NIF library. Note that by
+              <i>returning</i> we here mean the first point where the NIF
+              returns control back to the runtime system, and <em>not</em> the
+              point where a call to a NIF return a value back to the Erlang
+              code that called the NIF. That is, if you schedule execution of
+              a NIF, using <seecref marker="#enif_schedule_nif">
+              <c>enif_schedule_nif()</c></seecref>, from within a NIF while
+              the system is halting, the scheduled NIF call will <em>not</em>
+              be executed even though <i>delay halt</i> has been enabled for
+              the NIF library.
+            </p>
+            <p>
+              The runtime system halts when one of the
+              <seemfa marker="erlang#halt/0"><c>erlang:halt()</c></seemfa>
+              BIFs are called. By default flushing is enabled, but can be
+              disabled using the
+              <seemfa marker="erlang#halt/2"><c>erlang:halt/2</c></seemfa> BIF.
+              When flushing has been disabled, the <i>delay halt</i> setting
+              will have no effect. That is, the runtime system will halt without
+              waiting for NIFs to return even if the <i>delay halt</i> setting
+              has been enabled. See the
+              <seeerl marker="erlang#halt_flush"><c>{flush, boolean()}</c></seeerl>
+              option of <c>erlang:halt/2</c> for more information.
+            </p>
+            <p>
+              The <c>ERL_NIF_OPT_DELAY_HALT</c> option can only be set during
+              loading of a NIF library in a call to <c>enif_set_option()</c>
+              inside a NIF library
+              <seecref marker="#load"><c>load()</c></seecref> or
+              <seecref marker="#upgrade"><c>upgrade()</c></seecref> call,
+              and will fail if set somewhere else. The <c>env</c>
+              argument <i>must</i> be the callback environment passed to the
+              <c>load()</c> or the <c>upgrade()</c> call. This option can also
+              only be set once. That is, the <i>delay halt</i> setting cannot
+              be changed once it has been enabled. The <i>delay halt</i>
+              setting is tied to the module instance with which the NIF library
+              instance has been loaded. That is, in case both a new and old
+              version of a module using the NIF library are loaded, they can
+              have the same or different <i>delay halt</i> settings.
+            </p>
+            <p>
+              The <i>delay halt</i> feature can be used in combination with an
+              <seecref marker="#on_halt"><i>on halt</i></seecref> callback.
+              The <i>on halt</i> callback is in this case typically used to
+              notify processes blocked in NIFs in the library that it is time
+              to return in order to let the runtime system complete the
+              halting. Such NIFs should be dirty NIFs, since ordinary NIFs
+              should never block for a long time.
+            </p>
+          </item>
+          <tag><marker id="on_halt"/>
+          <c>int enif_set_option(ErlNifEnv *env,
+          </c><seecref marker="#ErlNifOption"><c>ERL_NIF_OPT_ON_HALT</c></seecref><c>,
+          </c><seecref marker="#ErlNifOnHaltCallback"><c>ErlNifOnHaltCallback</c></seecref><c>
+          *on_halt)</c>
+          </tag>
+          <item>
+            <p>
+              Install a callback that will be called when the runtime system
+              halts with flushing enabled.
+            </p>
+            <p>
+              The runtime system halts when one of the
+              <seemfa marker="erlang#halt/0"><c>erlang:halt()</c></seemfa> BIFs
+              are called. By default flushing is enabled, but can be disabled
+              using the
+              <seemfa marker="erlang#halt/2"><c>erlang:halt/2</c></seemfa> BIF.
+              When flushing has been disabled, the runtime system will halt
+              without calling any <i>on halt</i> callbacks even if such are
+              installed. See the
+              <seeerl marker="erlang#halt_flush"><c>{flush, boolean()}</c></seeerl>
+              option of <c>erlang:halt/2</c> for more information.
+            </p>
+            <p>
+              The <c>ERL_NIF_OPT_ON_HALT</c> option can only be set during
+              loading of a NIF library in a call to <c>enif_set_option()</c>
+              inside a NIF library
+              <seecref marker="#load"><c>load()</c></seecref> or
+              <seecref marker="#upgrade"><c>upgrade()</c></seecref> call,
+              and will fail if called somewhere else. The <c>env</c>
+              argument <i>must</i> be the callback environment passed to the
+              <c>load()</c> or the <c>upgrade()</c> call. The <c>on_halt</c>
+              argument should be a function pointer to the callback to install.
+              The <i>on halt</i> callback will be tied to the module instance
+              with which the NIF library instance has been loaded. That is, in
+              case both a new and old version of a module using the NIF library
+              are loaded, they can both have different, none, or the same
+              <i>on halt</i> callbacks installed. When unloading the NIF
+              library during a
+              <seemfa marker="kernel:code#purge/1">code purge</seemfa>, an
+              installed <i>on halt</i> callback will be uninstalled.
+              The <c>ERL_NIF_OPT_ON_HALT</c> option can also only be set
+              once. That is, the <i>on halt</i> callback cannot be changed
+              or removed once it has been installed by any other means than
+              purging the module instance that loaded the NIF library.
+            </p>
+            <p>
+              When the installed <i>on halt</i> callback is called, it will be
+              passed a pointer to <c>priv_data</c> as argument. The
+              <c>priv_data</c> pointer can be set when loading the NIF library.
+            </p>
+            <p>
+              The <i>on halt</i> callback can be used in combination with
+              <seecref marker="#delay_halt"><i>delay of halt</i></seecref> until
+              all calls into the library have returned. The <i>on halt</i>
+              callback is in this case typically used to notify processes
+              blocked in NIFs in the library that it is time to return in order
+              to let the runtime system complete the halting. Such NIFs should
+              be dirty NIFs, since ordinary NIFs should never block for a long
+              time.
+            </p>
+          </item>
+        </taglist>
+      </desc>
+    </func>
+
     <func>
       <name since="OTP 22.0"><ret>void</ret>
         <nametext>enif_set_pid_undefined(ErlNifPid* pid)</nametext></name>
diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml
index 8a8a219b97..ece6764017 100644
--- a/erts/doc/src/erlang.xml
+++ b/erts/doc/src/erlang.xml
@@ -2925,7 +2925,7 @@ uncompiled code with the same arity are mapped to the same list by
       <fsummary>Halt the Erlang runtime system and indicate normal exit to
         the calling environment.</fsummary>
       <desc>
-        <p>The same as
+        <p>The same as calling
           <seemfa marker="#halt/2"><c>halt(0, [])</c></seemfa>. Example:</p>
         <pre>
 > <input>halt().</input>
@@ -2934,10 +2934,11 @@ os_prompt%</pre>
     </func>
 
     <func>
-      <name name="halt" arity="1" since=""/>
+      <name name="halt" arity="1" clause_i="1"
+            anchor="halt_status_code_1" since=""/>
       <fsummary>Halt the Erlang runtime system.</fsummary>
       <desc>
-        <p>The same as <seemfa marker="#halt/2">
+        <p>The same as calling <seemfa marker="#halt/2">
           <c>halt(<anno>Status</anno>, [])</c></seemfa>. Example:</p>
         <pre>
 > <input>halt(17).</input>
@@ -2948,44 +2949,161 @@ os_prompt%</pre>
     </func>
 
     <func>
-      <name name="halt" arity="2" since="OTP R15B01"/>
+      <name name="halt" arity="1" clause_i="2"
+            anchor="halt_abort_1" since="OTP R15B01"/>
+      <fsummary>Halt the Erlang runtime system by aborting.</fsummary>
+      <desc>
+        <p>
+          The same as calling <seeerl marker="#halt_abort_2">
+          <c>halt(abort, [])</c></seeerl>.
+        </p>
+      </desc>
+    </func>
+
+    <func>
+      <name name="halt" arity="1" clause_i="3"
+            anchor="halt_crash_dump_1" since=""/>
+      <fsummary>
+        Halt the Erlang runtime system and create an Erlang crash dump.
+      </fsummary>
+      <desc>
+        <p>
+          The same as calling <seeerl marker="#halt_crash_dump_2">
+          <c>halt(<anno>CrashDumpSlogan</anno>, [])</c></seeerl>.
+        </p>
+      </desc>
+    </func>
+
+    <func>
+      <name name="halt" arity="2" clause_i="1"
+            anchor="halt_status_code_2" since="OTP R15B01"/>
       <fsummary>Halt the Erlang runtime system.</fsummary>
+      <type name="halt_options"/>
       <desc>
-        <p><c><anno>Status</anno></c> must be a non-negative integer, a string,
-          or the atom <c>abort</c>.
-          Halts the Erlang runtime system. Has no return value.
-          Depending on <c><anno>Status</anno></c>, the following occurs:</p>
+        <p>
+          Halt the runtime system with status code
+          <c><anno>Status</anno></c>.
+        </p>
+	<note>
+	  <p>
+            On many platforms, the OS supports only status codes 0-255. A too
+            large status code is truncated by clearing the high bits.
+          </p>
+	</note>
+        <p>
+          Currently the following options are valid:
+        </p>
         <taglist>
-          <tag>integer()</tag>
-          <item>The runtime system exits with integer value
-            <c><anno>Status</anno></c>
-            as status code to the calling environment (OS).
-	    <note>
-	      <p>On many platforms, the OS supports only status
-	      codes 0-255. A too large status code is truncated by clearing
-	      the high bits.</p>
-	    </note>
-          </item>
-          <tag>string()</tag>
-          <item>An Erlang crash dump is produced with <c><anno>Status</anno></c>
-            as slogan. Then the runtime system exits with status code <c>1</c>.
-	    The string will be truncated if longer than 200 characters.
-	    <note>
-	      <p>Before ERTS 9.1 (OTP-20.1) only code points in the range 0-255
-	         was accepted in the string. Now any unicode string is valid.</p>
-	    </note>
-          </item>
-          <tag><c>abort</c></tag>
-          <item>The runtime system aborts producing a core dump, if that is
-            enabled in the OS.
+          <tag><marker id="halt_flush"/><c>{flush, EnableFlushing}</c></tag>
+          <item>
+            <p>
+              If <c>EnableFlushing</c> equals <c>true</c>, which also is the
+              default behavior, the runtime system will perform the following
+              operations before terminating:
+            </p>
+            <list>
+              <item><p>
+                Flush all outstanding output.
+              </p></item>
+              <item><p>
+                Send all Erlang ports exit signals and wait for them
+                to exit.
+              </p></item>
+              <item><p>
+                Wait for all async threads to complete all outstanding
+                async jobs.
+              </p></item>
+              <item><p>
+                Call all installed <seecref marker="erl_nif#on_halt">NIF
+                <i>on halt</i> callbacks</seecref>.
+              </p></item>
+              <item><p>
+                Wait for all ongoing <seecref marker="erl_nif#delay_halt">NIF
+                calls with the <i>delay halt</i> setting</seecref> enabled to
+                return.
+              </p></item>
+              <item><p>
+                Call all installed <c>atexit</c>/<c>on_exit</c> callbacks.
+              </p></item>
+            </list>
+            <p>
+              If <c>EnableFlushing</c> equals <c>false</c>, the runtime system
+              will terminate immediately without performing any of the above
+              listed operations.
+            </p>
+            <p>
+              Behavior changes compared to earlier versions:
+            </p>
+            <list>
+              <item>
+                <p>
+                  Runtime systems prior to OTP 26.0 called all installed
+                  <c>atexit</c>/<c>on_exit</c> callbacks also when <c>flush</c>
+                  was disabled, but as of OTP 26.0 this is no longer the case.
+                </p>
+              </item>
+            </list>
           </item>
         </taglist>
-        <p>For integer <c><anno>Status</anno></c>, the Erlang runtime system
-          closes all ports and allows async threads to finish their
-          operations before exiting. To exit without such flushing, use
-          <c><anno>Option</anno></c> as <c>{flush,false}</c>.</p>
-        <p>For statuses <c>string()</c> and <c>abort</c>, option
-          <c>flush</c> is ignored and flushing is <em>not</em> done.</p>
+      </desc>
+    </func>
+
+    <func>
+      <name name="halt" arity="2" clause_i="2"
+            anchor="halt_abort_2" since="OTP R15B01"/>
+      <fsummary>Halt the Erlang runtime system by aborting.</fsummary>
+      <type name="halt_options"/>
+      <desc>
+        <p>
+          Halt the Erlang runtime system by aborting and produce a core dump
+          if core dumping has been enabled in the environment that the
+          runtime system is executing in.
+        </p>
+        <note><p>
+          The <seeerl marker="#halt_flush"><c>{flush, boolean()}</c></seeerl>
+          option will be ignored, and flushing will be disabled.
+        </p></note>
+      </desc>
+    </func>
+
+    <func>
+      <name name="halt" arity="2" clause_i="3"
+            anchor="halt_crash_dump_2" since="OTP R15B01"/>
+      <fsummary>
+        Halt the Erlang runtime system and create an Erlang crash dump.
+      </fsummary>
+      <type name="halt_options"/>
+      <desc>
+        <p>
+          Halt the Erlang runtime system and generate an
+	  <seeguide marker="crash_dump">Erlang crash dump</seeguide>. The
+          string <c><anno>CrashDumpSlogan</anno></c> will be used as slogan
+          in the Erlang crash dump created. The slogan will be trunkated if
+          <c><anno>CrashDumpSlogan</anno></c> is longer than 1023 characters.
+        </p>
+        <note><p>
+          The <seeerl marker="#halt_flush"><c>{flush, boolean()}</c></seeerl>
+          option will be ignored, and flushing will be disabled.
+        </p></note>
+        <p>
+          Behavior changes compared to earlier versions:
+        </p>
+        <list>
+          <item>
+            <p>
+              Before OTP 24.2, the slogan was truncated if
+              <c><anno>CrashDumpSlogan</anno></c> was longer than 200
+              characters. Now it will be truncated if longer than 1023
+              characters.
+            </p>
+          </item>
+          <item>
+            <p>
+              Before OTP 20.1, only code points in the range 0-255 were
+              accepted in the slogan. Now any Unicode string is valid.
+            </p>
+          </item>
+        </list>
       </desc>
     </func>
 
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 3b6de45587..8097ed337b 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -781,7 +781,7 @@ crash_dump_limited_writer(void* vfdp, char* buf, size_t len)
     }
 
     /* We assume that crash dump was called from erts_exit_vv() */
-    erts_exit_epilogue();
+    erts_exit_epilogue(0);
 }
 
 /* XXX THIS SHOULD BE IN SYSTEM !!!! */
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index bdcb136a05..3c2df49138 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -506,13 +506,13 @@ save_sched_table(Process *c_p, DbTable *tb)
     DbTable *first;
 
     ASSERT(esdp);
-    erts_atomic_inc_nob(&esdp->ets_tables.count);
+    erts_atomic_inc_nob(&esdp->u.ets_tables.count);
     erts_refc_inc(&tb->common.refc, 1);
 
-    first = esdp->ets_tables.clist;
+    first = esdp->u.ets_tables.clist;
     if (!first) {
         tb->common.all.next = tb->common.all.prev = tb;
-        esdp->ets_tables.clist = tb;
+        esdp->u.ets_tables.clist = tb;
     }
     else {
         tb->common.all.prev = first->common.all.prev;
@@ -530,14 +530,14 @@ remove_sched_table(ErtsSchedulerData *esdp, DbTable *tb)
     ASSERT(erts_get_ref_numbers_thr_id(ERTS_MAGIC_BIN_REFN(tb->common.btid))
            == (Uint32) esdp->no);
 
-    ASSERT(erts_atomic_read_nob(&esdp->ets_tables.count) > 0);
-    erts_atomic_dec_nob(&esdp->ets_tables.count);
+    ASSERT(erts_atomic_read_nob(&esdp->u.ets_tables.count) > 0);
+    erts_atomic_dec_nob(&esdp->u.ets_tables.count);
 
     eaydp = ERTS_SCHED_AUX_YIELD_DATA(esdp, ets_all);
     if (eaydp->ongoing) {
         /* ets:all() op process list from last to first... */
         if (eaydp->tab == tb) {
-            if (eaydp->tab == esdp->ets_tables.clist)
+            if (eaydp->tab == esdp->u.ets_tables.clist)
                 eaydp->tab = NULL;
             else
                 eaydp->tab = tb->common.all.prev;
@@ -546,23 +546,23 @@ remove_sched_table(ErtsSchedulerData *esdp, DbTable *tb)
 
     if (tb->common.all.next == tb) {
         ASSERT(tb->common.all.prev == tb);
-        ASSERT(esdp->ets_tables.clist == tb);
-        esdp->ets_tables.clist = NULL;
+        ASSERT(esdp->u.ets_tables.clist == tb);
+        esdp->u.ets_tables.clist = NULL;
     }
     else {
 #ifdef DEBUG
-        DbTable *tmp = esdp->ets_tables.clist;
+        DbTable *tmp = esdp->u.ets_tables.clist;
         do {
             if (tmp == tb) break;
             tmp = tmp->common.all.next;
-        } while (tmp != esdp->ets_tables.clist);
+        } while (tmp != esdp->u.ets_tables.clist);
         ASSERT(tmp == tb);
 #endif
         tb->common.all.prev->common.all.next = tb->common.all.next;
         tb->common.all.next->common.all.prev = tb->common.all.prev;
 
-        if (esdp->ets_tables.clist == tb)
-            esdp->ets_tables.clist = tb->common.all.next;
+        if (esdp->u.ets_tables.clist == tb)
+            esdp->u.ets_tables.clist = tb->common.all.next;
 
     }
 
@@ -3207,7 +3207,7 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp,
         hp = &hfragp->mem[hfragp->used_size];
         list = *hp;
         hfragp->used_size = hfragp->alloc_size;
-        first = esdp->ets_tables.clist;
+        first = esdp->u.ets_tables.clist;
         tb = *tablepp;
     }
     else {
@@ -3215,7 +3215,7 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp,
         ASSERT(!*tablepp);
 
         /* Max heap size needed... */
-        sz = erts_atomic_read_nob(&esdp->ets_tables.count);
+        sz = erts_atomic_read_nob(&esdp->u.ets_tables.count);
         sz *= ERTS_MAGIC_REF_THING_SIZE + 2;
         sz += 3 + ERTS_REF_THING_SIZE;
         hfragp = new_message_buffer(sz);
@@ -3223,7 +3223,7 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp,
         hp = &hfragp->mem[0];
         ohp = &hfragp->off_heap;
         list = NIL;
-        first = esdp->ets_tables.clist;
+        first = esdp->u.ets_tables.clist;
         tb = first ? first->common.all.prev : NULL;
     }
 
@@ -3309,7 +3309,7 @@ erts_handle_yielded_ets_all_request(ErtsAuxWorkData *awdp)
                 return 0; /* All work completed! */
 
             if (yc < ERTS_ETS_ALL_TB_YCNT_START &&
-                yc > erts_atomic_read_nob(&esdp->ets_tables.count))
+                yc > erts_atomic_read_nob(&esdp->u.ets_tables.count))
                 return 1; /* Yield! */
 
             eaydp->ongoing = ongoing = eaydp->queue;
@@ -4511,8 +4511,8 @@ erts_ets_sched_spec_data_init(ErtsSchedulerData *esdp)
     eaydp->hfrag = NULL;
     eaydp->tab = NULL;
     eaydp->queue = NULL;
-    esdp->ets_tables.clist = NULL;
-    erts_atomic_init_nob(&esdp->ets_tables.count, 0);
+    esdp->u.ets_tables.clist = NULL;
+    erts_atomic_init_nob(&esdp->u.ets_tables.count, 0);
 }
 
 
@@ -5349,7 +5349,7 @@ erts_db_foreach_table(void (*func)(DbTable *, void *), void *arg, int alive_only
 
     for (ix = 0; ix < erts_no_schedulers; ix++) {
         ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
-        DbTable *first = esdp->ets_tables.clist;
+        DbTable *first = esdp->u.ets_tables.clist;
         if (first) {
             DbTable *tb = first;
             do {
@@ -5393,7 +5393,7 @@ Uint erts_ets_table_count(void)
 
     for (six = 0; six < erts_no_schedulers; six++) {
         ErtsSchedulerData *esdp = &erts_aligned_scheduler_data[six].esd;
-        tb_count += erts_atomic_read_nob(&esdp->ets_tables.count);
+        tb_count += erts_atomic_read_nob(&esdp->u.ets_tables.count);
     }
     return tb_count;
 }
@@ -5460,7 +5460,7 @@ static void lcnt_update_db_locks_per_sched(void *enable) {
     DbTable *head;
 
     esdp = erts_get_scheduler_data();
-    head = esdp->ets_tables.clist;
+    head = esdp->u.ets_tables.clist;
 
     if(head) {
         DbTable *iterator = head;
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 95e41226fc..ee22672391 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2538,7 +2538,7 @@ __decl_noreturn void erts_thr_fatal_error(int err, const char *what)
 
 
 static void
-system_cleanup(int flush_async)
+system_cleanup(int flush)
 {
     /*
      * Make sure only one thread exits the runtime system.
@@ -2568,24 +2568,43 @@ system_cleanup(int flush_async)
      *    (in threaded non smp case).
      */
 
-    if (!flush_async
-	|| !erts_initialized
-	)
+    if (!flush || !erts_initialized)
 	return;
 
+    /*
+     * We only flush as a result of calling erts_halt() (which in turn
+     * is called from the erlang:halt() BIF when flushing is enabled);
+     * otherwise, flushing wont work properly. If erts_halt() has
+     * been called, 'erts_halt_code' won't equal INT_MIN...
+     */
+    ASSERT(erts_halt_code != INT_MIN);
+
+    /*
+     * Nif on-halt handlers may have been added after we initiated
+     * a halt. If so, make sure that these late added handlers are
+     * executed as well..
+     */
+    erts_nif_execute_on_halt();
+
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0);
 #endif
 
     erts_exit_flush_async();
+
+    /*
+     * Wait for all NIF calls with delayed halt functionality
+     * enabled to complete before we continue...
+     */
+    erts_nif_wait_calls();
 }
 
 static int erts_exit_code;
 
 static __decl_noreturn void __noreturn
-erts_exit_vv(int n, int flush_async, const char *fmt, va_list args1, va_list args2)
+erts_exit_vv(int n, int flush, const char *fmt, va_list args1, va_list args2)
 {
-    system_cleanup(flush_async);
+    system_cleanup(flush);
 
     if (fmt != NULL && *fmt != '\0')
 	erl_error(fmt, args2);	/* Print error message. */
@@ -2598,25 +2617,25 @@ erts_exit_vv(int n, int flush_async, const char *fmt, va_list args1, va_list arg
 	erl_crash_dump_v((char*) NULL, 0, fmt, args1);
     }
 
-    erts_exit_epilogue();
+    erts_exit_epilogue(flush);
 }
 
-__decl_noreturn void __noreturn erts_exit_epilogue(void)
+__decl_noreturn void __noreturn erts_exit_epilogue(int flush)
 {
     int n = erts_exit_code;
 
     sys_tty_reset(n);
 
     if (n == ERTS_INTR_EXIT)
-	exit(0);
+	(void) (flush ? exit(0) : _exit(0));
     else if (n == ERTS_DUMP_EXIT)
 	ERTS_EXIT_AFTER_DUMP(1);
     else if (n == ERTS_ERROR_EXIT || n == ERTS_ABORT_EXIT)
         abort();
-    exit(n);
+    (void) (flush ? exit(n) : _exit(n));
 }
 
-/* Exit without flushing async threads */
+/* Exit without flushing */
 __decl_noreturn void __noreturn erts_exit(int n, const char *fmt, ...)
 {
     va_list args1, args2;
@@ -2627,8 +2646,12 @@ __decl_noreturn void __noreturn erts_exit(int n, const char *fmt, ...)
     va_end(args1);
 }
 
-/* Exit after flushing async threads */
-__decl_noreturn void __noreturn erts_flush_async_exit(int n, char *fmt, ...)
+/*
+ * Exit after flushing. This is a continuation of erts_halt() and wont
+ * work properly if called by its own without proper initialization
+ * as made in erts_halt().
+ */
+__decl_noreturn void __noreturn erts_flush_exit(int n, char *fmt, ...)
 {
     va_list args1, args2;
     va_start(args1, fmt);
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index c6f127e5a4..f299889dcc 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -108,6 +108,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"fun_tab",				NULL			},
     {	"environ",				NULL			},
     {	"release_literal_areas",		NULL			},
+    {   "on_halt",                              NULL                    },
     {	"drv_ev_state_grow",			NULL,   		},
     {	"drv_ev_state",				"address"		},
     {	"safe_hash",				"address"		},
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 50ac6b92d9..f3cc8be6ed 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -71,6 +71,17 @@
 #include <limits.h>
 #include <stddef.h> /* offsetof */
 
+#define ERTS_NIF_HALT_INFO_FLAG_BLOCK               (1 << 0)
+#define ERTS_NIF_HALT_INFO_FLAG_HALTING             (1 << 1)
+#define ERTS_NIF_HALT_INFO_FLAG_WAITING             (1 << 2)
+
+typedef struct ErtsNifOnHaltData_ ErtsNifOnHaltData;
+struct ErtsNifOnHaltData_ {
+    ErtsNifOnHaltData *next;
+    ErtsNifOnHaltData *prev;
+    ErlNifOnHaltCallback *callback;
+};
+
 /* Information about a loaded nif library.
  * Each successful call to erlang:load_nif will allocate an instance of
  * erl_module_nif. Two calls opening the same library will thus have the same
@@ -95,11 +106,21 @@ struct erl_module_nif {
                                  +1 for each owned resource type with callbacks
                                  +1 for each ongoing dirty NIF call
                                */
+    int flags;
+    ErtsNifOnHaltData on_halt;
     Module* mod;    /* Can be NULL if purged and dynlib_refc > 0 */
 
     ErlNifFunc _funcs_copy_[1];  /* only used for old libs */
 };
 
+#define ERTS_MOD_NIF_FLG_LOADING                (1 << 0)
+#define ERTS_MOD_NIF_FLG_DELAY_HALT             (1 << 1)
+#define ERTS_MOD_NIF_FLG_ON_HALT                (1 << 2)
+
+static erts_atomic_t halt_tse;
+static erts_mtx_t on_halt_mtx;
+static ErtsNifOnHaltData *on_halt_requests;
+
 typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]);
 
 #ifdef DEBUG
@@ -131,6 +152,8 @@ void dtrace_nifenv_str(ErlNifEnv *, char *);
 
 #define MIN_HEAP_FRAG_SZ 200
 static Eterm* alloc_heap_heavy(ErlNifEnv* env, size_t need, Eterm* hp);
+static void install_on_halt_callback(ErtsNifOnHaltData *ohdp);
+static void uninstall_on_halt_callback(ErtsNifOnHaltData *ohdp);
 
 static ERTS_INLINE int
 is_scheduler(void)
@@ -370,6 +393,25 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
     return (ERL_NIF_TERM) THE_NON_VALUE;
 }
 
+static ERTS_NOINLINE void
+eternal_sleep(void)
+{
+    while (!0)
+        erts_milli_sleep(1000*1000);
+}
+
+static ERTS_NOINLINE void
+handle_halting_unblocked_halt(erts_aint32_t info)
+{
+    if (info & ERTS_NIF_HALT_INFO_FLAG_WAITING) {
+        erts_tse_t *tse;
+        ERTS_THR_MEMORY_BARRIER;
+        tse = (erts_tse_t *) erts_atomic_read_nob(&halt_tse);
+        ASSERT(tse);
+        erts_tse_set(tse);
+    }
+    eternal_sleep();
+}
 
 static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
 static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
@@ -419,7 +461,34 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp,
 
     erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
 
-    result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */
+    if (!(env.mod_nif->flags & ERTS_MOD_NIF_FLG_DELAY_HALT)) {
+        result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */
+    }
+    else {
+        erts_atomic32_t *dirty_nif_halt_info = &esdp->u.dirty_nif_halt_info;
+        erts_aint32_t info;
+        info = erts_atomic32_cmpxchg_nob(dirty_nif_halt_info,
+                                         ERTS_NIF_HALT_INFO_FLAG_BLOCK,
+                                         0);
+        if (info != 0) {
+            ASSERT(info == ERTS_NIF_HALT_INFO_FLAG_HALTING
+                   || info == (ERTS_NIF_HALT_INFO_FLAG_HALTING
+                               | ERTS_NIF_HALT_INFO_FLAG_WAITING));
+            eternal_sleep();
+        }
+        result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */
+        info = erts_atomic32_read_band_relb(dirty_nif_halt_info,
+                                            ~ERTS_NIF_HALT_INFO_FLAG_BLOCK);
+        if (info & ERTS_NIF_HALT_INFO_FLAG_HALTING) {
+            ASSERT(info == (ERTS_NIF_HALT_INFO_FLAG_BLOCK
+                            | ERTS_NIF_HALT_INFO_FLAG_HALTING)
+                   || info == (ERTS_NIF_HALT_INFO_FLAG_BLOCK
+                               | ERTS_NIF_HALT_INFO_FLAG_HALTING
+                               | ERTS_NIF_HALT_INFO_FLAG_WAITING));
+            handle_halting_unblocked_halt(info);
+        }
+        ASSERT(info == ERTS_NIF_HALT_INFO_FLAG_BLOCK);
+    }
 
     erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 
@@ -2277,6 +2346,9 @@ static void close_dynlib(struct erl_module_nif* lib)
     ASSERT(lib->handle != NULL);
     ASSERT(erts_refc_read(&lib->dynlib_refc,0) == 0);
 
+    if (lib->flags & ERTS_MOD_NIF_FLG_ON_HALT)
+        uninstall_on_halt_callback(&lib->on_halt);
+
     if (lib->entry.unload != NULL) {
 	struct enif_msg_environment_t msg_env;
         pre_nif_noproc(&msg_env, lib, NULL);
@@ -4524,6 +4596,8 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args)
         lib->handle = handle;
         erts_refc_init(&lib->refc, 2);  /* Erlang code + NIF code */
         erts_refc_init(&lib->dynlib_refc, 1);
+        lib->flags = 0;
+        lib->on_halt.callback = NULL;
         ASSERT(opened_rt_list == NULL);
         lib->mod = module_p;
 
@@ -4642,7 +4716,7 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args)
     /* Call load or upgrade:
      */
     env.mod_nif = lib;
-
+    lib->flags |= ERTS_MOD_NIF_FLG_LOADING;
     lib->priv_data = NULL;
     if (prev_mi->nif != NULL) { /**************** Upgrade ***************/
         void* prev_old_data = prev_mi->nif->priv_data;
@@ -4676,11 +4750,14 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args)
             ret = load_nif_error(c_p, "load", "Library load-call unsuccessful (%d).", veto);
         }
     }
+    lib->flags &= ~ERTS_MOD_NIF_FLG_LOADING;
 
     if (ret == am_ok) {
 	/*
          * Everything ok, make NIF code callable.
 	 */
+        if (lib->flags & ERTS_MOD_NIF_FLG_ON_HALT)
+            install_on_halt_callback(&lib->on_halt);
 	this_mi->nif = lib;
         prepare_opened_rt(lib);
         /*
@@ -5046,6 +5123,20 @@ void erl_nif_init()
 
     nif_call_table_init();
     static_nifs_init();
+
+    erts_atomic_init_nob(&halt_tse, (erts_aint_t) NULL);
+    erts_mtx_init(&on_halt_mtx, "on_halt", NIL,
+                  ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
+    on_halt_requests = NULL;
+}
+
+void
+erts_nif_sched_init(ErtsSchedulerData *esdp)
+{
+    if (esdp->type == ERTS_SCHED_DIRTY_CPU
+        || esdp->type == ERTS_SCHED_DIRTY_IO) {
+        erts_atomic32_init_nob(&esdp->u.dirty_nif_halt_info, 0);
+    }
 }
 
 int erts_nif_get_funcs(struct erl_module_nif* mod,
@@ -5133,6 +5224,191 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
     return nif_result;
 }
 
+/*
+ * Set options...
+ */
+
+int
+enif_set_option(ErlNifEnv *env, ErlNifOption opt, ...)
+{
+    if (!env)
+        return EINVAL;
+
+    switch (opt) {
+
+    case ERL_NIF_OPT_DELAY_HALT: {
+        struct erl_module_nif *m = env->mod_nif;
+
+        if (!m || !(m->flags & ERTS_MOD_NIF_FLG_LOADING)
+            || (m->flags & ERTS_MOD_NIF_FLG_DELAY_HALT)) {
+            return EINVAL;
+        }
+
+        m->flags |= ERTS_MOD_NIF_FLG_DELAY_HALT;
+
+        return 0;
+    }
+
+    case ERL_NIF_OPT_ON_HALT: {
+        struct erl_module_nif *m = env->mod_nif;
+        ErlNifOnHaltCallback *on_halt;
+        va_list argp;
+
+        if (!m || ((m->flags & (ERTS_MOD_NIF_FLG_LOADING
+                                | ERTS_MOD_NIF_FLG_ON_HALT))
+                   != ERTS_MOD_NIF_FLG_LOADING)) {
+            return EINVAL;
+        }
+
+        ASSERT(!m->on_halt.callback);
+
+        va_start(argp, opt);
+        on_halt = va_arg(argp, ErlNifOnHaltCallback *);
+        va_end(argp);
+        if (!on_halt)
+            return EINVAL;
+
+        m->on_halt.callback = on_halt;
+        m->flags |= ERTS_MOD_NIF_FLG_ON_HALT;
+
+        return 0;
+    }
+
+    default:
+        return EINVAL;
+
+    }
+}
+
+/*
+ * Halt functionality...
+ */
+
+void
+erts_nif_execute_on_halt(void)
+{
+    ErtsNifOnHaltData *ohdp;
+
+    erts_mtx_lock(&on_halt_mtx);
+    for (ohdp = on_halt_requests; ohdp; ohdp = ohdp->next) {
+        struct erl_module_nif *m;
+        m = ErtsContainerStruct(ohdp, struct erl_module_nif, on_halt);
+        ohdp->callback(m->priv_data);
+    }
+    on_halt_requests = NULL;
+    erts_mtx_unlock(&on_halt_mtx);
+}
+
+void
+erts_nif_notify_halt(void)
+{
+    int ix;
+
+    erts_nif_execute_on_halt();
+
+    for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+        ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
+        ASSERT(esdp->type == ERTS_SCHED_DIRTY_CPU);
+        (void) erts_atomic32_read_bor_nob(&esdp->u.dirty_nif_halt_info,
+                                          ERTS_NIF_HALT_INFO_FLAG_HALTING);
+    }
+    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+        ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
+        ASSERT(esdp->type == ERTS_SCHED_DIRTY_IO);
+        (void) erts_atomic32_read_bor_nob(&esdp->u.dirty_nif_halt_info,
+                                          ERTS_NIF_HALT_INFO_FLAG_HALTING);
+    }
+}
+
+static ERTS_INLINE void
+wait_dirty_call_blocking_halt(ErtsSchedulerData *esdp, erts_tse_t *tse)
+{
+    erts_atomic32_t *dirty_nif_halt_info = &esdp->u.dirty_nif_halt_info;
+    erts_aint32_t info = erts_atomic32_read_acqb(dirty_nif_halt_info);
+    ASSERT(info & ERTS_NIF_HALT_INFO_FLAG_HALTING);
+    if (!(info & ERTS_NIF_HALT_INFO_FLAG_BLOCK))
+        return;
+    info = erts_atomic32_read_bor_mb(dirty_nif_halt_info,
+                                     ERTS_NIF_HALT_INFO_FLAG_WAITING);
+    if (!(info & ERTS_NIF_HALT_INFO_FLAG_BLOCK))
+        return;
+    while (!0) {
+        erts_tse_reset(tse);
+        info = erts_atomic32_read_acqb(dirty_nif_halt_info);
+        if (!(info & ERTS_NIF_HALT_INFO_FLAG_BLOCK))
+            return;
+        while (!0) {
+            if (erts_tse_wait(tse) != EINTR)
+                break;
+        }
+    }
+}
+
+void
+erts_nif_wait_calls(void)
+{
+    erts_tse_t *tse;
+    int ix;
+
+    tse = erts_tse_fetch();
+    erts_atomic_set_nob(&halt_tse, (erts_aint_t) tse);
+
+    for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
+        ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
+        ASSERT(esdp->type == ERTS_SCHED_DIRTY_CPU);
+        wait_dirty_call_blocking_halt(esdp, tse);
+    }
+    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
+        ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
+        ASSERT(esdp->type == ERTS_SCHED_DIRTY_IO);
+        wait_dirty_call_blocking_halt(esdp, tse);
+    }
+}
+
+static void
+install_on_halt_callback(ErtsNifOnHaltData *ohdp)
+{
+    ASSERT(ohdp->callback);
+
+    erts_mtx_lock(&on_halt_mtx);
+    ohdp->next = on_halt_requests;
+    ohdp->prev = NULL;
+    if (on_halt_requests)
+        on_halt_requests->prev = ohdp;
+    on_halt_requests = ohdp;
+    erts_mtx_unlock(&on_halt_mtx);
+}
+
+static void
+uninstall_on_halt_callback(ErtsNifOnHaltData *ohdp)
+{
+    erts_mtx_lock(&on_halt_mtx);
+    ohdp->callback = NULL;
+    if (ohdp->prev) {
+        ASSERT(on_halt_requests != ohdp);
+        ohdp->prev->next = ohdp->next;
+    }
+    else if (on_halt_requests == ohdp) {
+        ASSERT(erts_halt_code == INT_MIN);
+        on_halt_requests = ohdp->next;
+    }
+    else {
+        /*
+         * Uninstall during halt; and our callback
+         * has already been called...
+         */
+        ASSERT(erts_halt_code != INT_MIN);
+    }
+    if (ohdp->next) {
+        ohdp->next->prev = ohdp->prev;
+    }
+    erts_mtx_unlock(&on_halt_mtx);
+}
+
+/*
+ * End of halt functionality...
+ */
+
 #ifdef USE_VM_PROBES
 void dtrace_nifenv_str(ErlNifEnv *env, char *process_buf)
 {
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index a9a81483f6..1ed1201cf2 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -57,9 +57,10 @@
 ** 2.15: 22.0 ERL_NIF_SELECT_CANCEL, enif_select_(read|write)
 **            enif_term_type
 ** 2.16: 24.0 enif_init_resource_type, enif_dynamic_resource_call
+** 2.17  26.0 enif_set_option
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 16
+#define ERL_NIF_MINOR_VERSION 17
 
 /*
  * WHEN CHANGING INTERFACE VERSION, also replace erts version below with
@@ -70,7 +71,7 @@
  * If you're not on the OTP team, you should use a placeholder like
  * erts-@MyName@ instead.
  */
-#define ERL_NIF_MIN_ERTS_VERSION "erts-12.0"
+#define ERL_NIF_MIN_ERTS_VERSION "erts-12.3"
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
@@ -201,6 +202,8 @@ typedef struct
 
 typedef ErlDrvMonitor ErlNifMonitor;
 
+typedef void ErlNifOnHaltCallback(void *priv_data);
+
 typedef struct enif_resource_type_t ErlNifResourceType;
 typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
 typedef void ErlNifResourceStop(ErlNifEnv*, void*, ErlNifEvent, int is_direct_call);
@@ -325,6 +328,11 @@ typedef enum {
 #define ERL_NIF_THR_DIRTY_CPU_SCHEDULER 2
 #define ERL_NIF_THR_DIRTY_IO_SCHEDULER 3
 
+typedef enum {
+    ERL_NIF_OPT_DELAY_HALT = 1,
+    ERL_NIF_OPT_ON_HALT = 2
+} ErlNifOption;
+
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS
 typedef struct {
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index f9004eb64b..a2a1052b59 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -219,6 +219,8 @@ ERL_NIF_API_FUNC_DECL(ErlNifTermType,enif_term_type,(ErlNifEnv* env, ERL_NIF_TER
 ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_init_resource_type,(ErlNifEnv*, const char* name_str, const ErlNifResourceTypeInit*, ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
 ERL_NIF_API_FUNC_DECL(int,enif_dynamic_resource_call,(ErlNifEnv*, ERL_NIF_TERM mod, ERL_NIF_TERM name, ERL_NIF_TERM rsrc, void* call_data));
 
+ERL_NIF_API_FUNC_DECL(int, enif_set_option, (ErlNifEnv *env, ErlNifOption opt, ...));
+
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
 */
@@ -408,6 +410,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_dynamic_resource_call,(ErlNifEnv*, ERL_NIF_TERM m
 #  define enif_term_type ERL_NIF_API_FUNC_MACRO(enif_term_type)
 #  define enif_init_resource_type ERL_NIF_API_FUNC_MACRO(enif_init_resource_type)
 #  define enif_dynamic_resource_call ERL_NIF_API_FUNC_MACRO(enif_dynamic_resource_call)
+#  define enif_set_option ERL_NIF_API_FUNC_MACRO(enif_set_option)
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
 */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 17780a7e87..2e45c00cb8 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2498,7 +2498,7 @@ notify_reap_ports_relb(void)
 }
 
 erts_atomic32_t erts_halt_progress;
-int erts_halt_code;
+int erts_halt_code = INT_MIN;
 
 static ERTS_INLINE erts_aint32_t
 handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
@@ -2543,7 +2543,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 	    erts_port_release(prt);
 	}
 	if (erts_atomic32_dec_read_nob(&erts_halt_progress) == 0) {
-	    erts_flush_async_exit(erts_halt_code, "");
+	    erts_flush_exit(erts_halt_code, "");
 	}
     }
     return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS;
@@ -6298,7 +6298,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
 
 
     erts_atomic32_init_relb(&erts_halt_progress, -1);
-    erts_halt_code = 0;
+    erts_halt_code = INT_MIN;
 
 
 }
@@ -8656,6 +8656,7 @@ sched_thread_func(void *vesdp)
 
     erts_ets_sched_spec_data_init(esdp);
     erts_utils_sched_spec_data_init();
+    erts_nif_sched_init(esdp);
 
     process_main(esdp);
 
@@ -8706,6 +8707,8 @@ sched_dirty_cpu_thread_func(void *vesdp)
 
     erts_proc_lock_prepare_proc_lock_waiter();
 
+    erts_nif_sched_init(esdp);
+
     erts_dirty_process_main(esdp);
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
@@ -8754,6 +8757,8 @@ sched_dirty_io_thread_func(void *vesdp)
 
     erts_proc_lock_prepare_proc_lock_waiter();
 
+    erts_nif_sched_init(esdp);
+
     erts_dirty_process_main(esdp);
     /* No schedulers should *ever* terminate */
     erts_exit(ERTS_ABORT_EXIT,
@@ -14820,6 +14825,7 @@ void erts_halt(int code)
         ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING);
         ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING);
 	erts_halt_code = code;
+        erts_nif_notify_halt();
     }
 }
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 8dc55a8da0..a5922e0bf3 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -727,7 +727,10 @@ struct ErtsSchedulerData_ {
     ErtsSchedWallTime sched_wall_time;
     ErtsGCInfo gc_info;
     ErtsPortTaskHandle nosuspend_port_task_handle;
-    ErtsEtsTables ets_tables;
+    union {
+        ErtsEtsTables ets_tables;
+        erts_atomic32_t dirty_nif_halt_info;
+    } u;
 #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
     erts_alloc_verify_func_t verify_unused_temp_alloc;
     Allctr_t *verify_unused_temp_alloc_data;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 8c4dd90966..a6b26bc50b 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -127,6 +127,10 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args);
 
 void erts_unload_nif(struct erl_module_nif* nif);
 extern void erl_nif_init(void);
+extern void erts_nif_sched_init(ErtsSchedulerData *esdp);
+extern void erts_nif_execute_on_halt(void);
+extern void erts_nif_notify_halt(void);
+extern void erts_nif_wait_calls(void);
 extern int erts_nif_get_funcs(struct erl_module_nif*,
                               struct enif_func_t **funcs);
 extern Module *erts_nif_get_module(struct erl_module_nif*);
@@ -1044,9 +1048,9 @@ double erts_get_positive_zero_float(void);
 
 /* config.c */
 
-__decl_noreturn void __noreturn erts_exit_epilogue(void);
+__decl_noreturn void __noreturn erts_exit_epilogue(int flush);
 __decl_noreturn void __noreturn erts_exit(int n, const char*, ...);
-__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...);
+__decl_noreturn void __noreturn erts_flush_exit(int n, char*, ...);
 void erl_error(const char*, va_list);
 
 /* This controls whether sharing-preserving copy is used by Erlang */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index dfdbe475b3..81c3ca29ae 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -3772,7 +3772,7 @@ terminate_port(Port *prt)
     if ((state & ERTS_PORT_SFLG_HALT)
 	&& (erts_atomic32_dec_read_nob(&erts_halt_progress) == 0)) {
 	erts_port_release(prt); /* We will exit and never return */
-	erts_flush_async_exit(erts_halt_code, "");
+	erts_flush_exit(erts_halt_code, "");
     }
     if (is_internal_port(send_closed_port_id))
 	deliver_result(NULL, send_closed_port_id, connected_id, am_closed);
diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl
index 59d791eb2b..2bba329234 100644
--- a/erts/emulator/test/dirty_nif_SUITE.erl
+++ b/erts/emulator/test/dirty_nif_SUITE.erl
@@ -29,13 +29,49 @@
 -export([all/0, suite/0,
 	 init_per_suite/1, end_per_suite/1,
 	 init_per_testcase/2, end_per_testcase/2,
+         init_per_group/2, end_per_group/2, groups/0,
 	 dirty_nif/1, dirty_nif_send/1,
 	 dirty_nif_exception/1, call_dirty_nif_exception/1,
 	 dirty_scheduler_exit/1, dirty_call_while_terminated/1,
 	 dirty_heap_access/1, dirty_process_info/1,
 	 dirty_process_register/1, dirty_process_trace/1,
 	 code_purge/1, literal_area/1, dirty_nif_send_traced/1,
-	 nif_whereis/1, nif_whereis_parallel/1, nif_whereis_proxy/1]).
+	 nif_whereis/1, nif_whereis_parallel/1, nif_whereis_proxy/1,
+         set_halt_options_from_nif/1,
+         delay_halt/1,
+         delay_halt_old_code/1,
+         delay_halt_old_and_new_code/1,
+         flush_false/1,
+         on_halt/1,
+         on_halt_old_code/1,
+         on_halt_old_and_new_code/1,
+         sync_halt/1,
+         many_delay_halt/1,
+         many_on_halt/1]).
+
+-export([load_nif/2]).
+
+-nifs([lib_loaded/0,
+       call_dirty_nif/3,
+       send_from_dirty_nif/1,
+       send_wait_from_dirty_nif/1,
+       call_dirty_nif_exception/1,
+       call_dirty_nif_zero_args/0,
+       dirty_call_while_terminated_nif/1,
+       dirty_sleeper/0,
+       dirty_sleeper/1,
+       dirty_heap_access_nif/1,
+       whereis_term/2,
+       whereis_send/3,
+       dirty_terminating_literal_access/2,
+       delay_halt_normal/3,
+       delay_halt_io_bound/3,
+       delay_halt_cpu_bound/3,
+       sync_halt_io_bound/2,
+       sync_halt_cpu_bound/2,
+       set_halt_option_from_nif_normal/1,
+       set_halt_option_from_nif_io_bound/1,
+       set_halt_option_from_nif_cpu_bound/1]).
 
 -define(nif_stub,nif_stub_error(?LINE)).
 
@@ -55,23 +91,38 @@ all() ->
      literal_area,
      dirty_nif_send_traced,
      nif_whereis,
-     nif_whereis_parallel].
+     nif_whereis_parallel,
+     {group, halt_normal},
+     {group, halt_dirty_cpu},
+     {group, halt_dirty_io},
+     {group, halt_misc}].
+
+halt_sched_tests() ->
+    [set_halt_options_from_nif, delay_halt, delay_halt_old_code, delay_halt_old_and_new_code].
+halt_dirty_sched_tests() ->
+    [sync_halt, flush_false].
+
+groups() ->
+    [{halt_normal, [parallel], halt_sched_tests()},
+     {halt_dirty_cpu, [parallel], halt_sched_tests()++halt_dirty_sched_tests()},
+     {halt_dirty_io, [parallel], halt_sched_tests()++halt_dirty_sched_tests()},
+     {halt_misc, [parallel], [on_halt, on_halt_old_code, on_halt_old_and_new_code, many_on_halt, many_delay_halt]}].
+
+init_per_group(Group, Config) ->
+    [{group, Group} | Config].
+
+end_per_group(_, Config) ->
+    proplists:delete(group, Config).
 
 init_per_suite(Config) ->
-    case erlang:system_info(dirty_cpu_schedulers) of
-	N when N > 0 ->
-	    case lib_loaded() of
-		false ->
-		    ok = erlang:load_nif(
-			   filename:join(?config(data_dir, Config),
-					 "dirty_nif_SUITE"), []);
-		true ->
-		    ok
-	    end,
-	    Config;
-        _ ->
-	    {skipped, "No dirty scheduler support"}
-    end.
+    case lib_loaded() of
+        false ->
+            ok = erlang:load_nif(filename:join(?config(data_dir, Config),
+                                               "dirty_nif_SUITE"), []);
+        true ->
+            ok
+    end,
+    Config.
 
 end_per_suite(_Config) ->
     ok.
@@ -82,6 +133,9 @@ init_per_testcase(Case, Config) ->
 end_per_testcase(_Case, _Config) ->
     ok.
 
+load_nif(NifLib, LibInfo) ->
+    erlang:load_nif(NifLib, LibInfo).
+
 dirty_nif(Config) when is_list(Config) ->
     Val1 = 42,
     Val2 = "Erlang",
@@ -536,6 +590,356 @@ literal_area(Config) when is_list(Config) ->
     literal_area_collector_test:check_idle(5000),
     {comment, "Waited "++integer_to_list(TMO)++" milliseconds after purge"}.
 
+set_halt_options_from_nif(Config) when is_list(Config) ->
+    case ?config(group, Config) of
+        halt_normal ->
+            error = set_halt_option_from_nif_normal(set_on_halt_handler),
+            error = set_halt_option_from_nif_normal(delay_halt);
+        halt_dirty_cpu ->
+            error = set_halt_option_from_nif_cpu_bound(set_on_halt_handler),
+            error = set_halt_option_from_nif_cpu_bound(delay_halt);
+        halt_dirty_io ->
+            error = set_halt_option_from_nif_io_bound(set_on_halt_handler),
+            error = set_halt_option_from_nif_io_bound(delay_halt)
+    end,
+    ok.
+
+delay_halt(Config) when is_list(Config) ->
+    delay_halt(Config, new_code).
+
+delay_halt_old_code(Config) when is_list(Config) ->
+    delay_halt(Config, old_code).
+
+delay_halt_old_and_new_code(Config) when is_list(Config) ->
+    delay_halt(Config, old_and_new_code).
+
+delay_halt(Config, Type) ->
+    Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING),
+    Priv = proplists:get_value(priv_dir, Config),
+    TypeSuffix = "_"++atom_to_list(Type),
+    {Fun, FileName} = case ?config(group, Config) of
+                          halt_normal ->
+                              {fun delay_halt_normal/3, "delay_halt_normal"++TypeSuffix};
+                          halt_dirty_io ->
+                              {fun delay_halt_io_bound/3, "delay_halt_io_bound"++TypeSuffix};
+                          halt_dirty_cpu ->
+                              {fun delay_halt_cpu_bound/3, "delay_halt_cpu_bound"++TypeSuffix}
+                      end,
+    Tester = self(),
+    NifFileName = filename:join(Priv, FileName),
+    {ok, Peer, Node} = ?CT_PEER(),
+    Mon = erlang:monitor(process, Peer),
+    ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {delay_halt}]),
+    ok = erpc:call(Node, file, set_cwd, [Priv]),
+    Proxy = spawn_link(Node,
+                       fun () ->
+                               receive
+                                   {delay_halt, _} = Msg ->
+                                       unlink(Tester),
+                                       Tester ! Msg
+                               end
+                       end),
+    Start = erlang:monotonic_time(millisecond),
+    ok = erpc:cast(Node, fun () -> Fun(Proxy, FileName, 2) end),
+    receive {delay_halt, Pid} when is_pid(Pid), Node == node(Pid) -> ok end,
+    case Type of
+        new_code ->
+            ok;
+        old_code ->
+            true = erpc:call(Node, erlang, delete_module, [dirty_nif_SUITE]);
+        old_and_new_code ->
+            true = erpc:call(Node, erlang, delete_module, [dirty_nif_SUITE]),
+            ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {delay_halt}]),
+            Proxy2 = spawn_link(Node,
+                                fun () ->
+                                        receive
+                                            {delay_halt, _} = Msg ->
+                                                unlink(Tester),
+                                                Tester ! Msg
+                                        end
+                                end),
+            ok = erpc:cast(Node, fun () -> Fun(Proxy2, FileName++"_new_code", 2) end),
+            receive {delay_halt, Pid2} when is_pid(Pid2), Node == node(Pid2) -> ok end
+    end,
+    ok = erpc:cast(Node, erlang, halt, []),
+    ok = wait_until(fun () ->
+                            {ok, <<"ok">>} == file:read_file(NifFileName)
+                                andalso
+                                  (Type /= old_and_new_code
+                                   orelse {ok, <<"ok">>} == file:read_file(NifFileName++"_new_code"))
+                    end,
+                    6000),
+    Time = erlang:monotonic_time(millisecond) - Start,
+    ct:log("~s time=~pms", [FileName, Time]),
+    true = Time >= 2000,
+    receive {'DOWN', Mon, process, Peer, _} -> ok end,
+    ok.
+
+flush_false(Config) when is_list(Config) ->
+    Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING),
+    Priv = proplists:get_value(priv_dir, Config),
+    {Fun, FileName} = case ?config(group, Config) of
+                          halt_dirty_io ->
+                              {fun delay_halt_io_bound/3, "flush_false_io_bound"};
+                          halt_dirty_cpu ->
+                              {fun delay_halt_cpu_bound/3, "flush_false_cpu_bound"}
+                      end,
+    Tester = self(),
+    NifFileName = filename:join(Priv, FileName),
+    OnHaltBaseName = FileName++"_on_halt",
+    OnHaltFileName = filename:join(Priv, OnHaltBaseName),
+    {ok, Peer, Node} = ?CT_PEER(),
+    Mon = erlang:monitor(process, Peer),
+    ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {sync_halt, OnHaltBaseName, 1}]),
+    ok = erpc:call(Node, file, set_cwd, [Priv]),
+    Proxy = spawn_link(Node,
+                       fun () ->
+                               receive
+                                   {delay_halt, _} = Msg ->
+                                       unlink(Tester),
+                                       Tester ! Msg
+                               end
+                       end),
+    Start = erlang:monotonic_time(millisecond),
+    ok = erpc:cast(Node, fun () -> Fun(Proxy, FileName, 1) end),
+    receive {delay_halt, Pid} when is_pid(Pid), Node == node(Pid) -> ok end,
+    ok = erpc:cast(Node, erlang, halt, [0, [{flush,false}]]),
+    receive {'DOWN', Mon, process, Peer, _} -> ok end,
+    Time = erlang:monotonic_time(millisecond) - Start,
+    ct:log("~s time=~pms", [FileName, Time]),
+    Wait = 3000-Time,
+    if Wait > 0 -> receive after Wait -> ok end;
+       true -> ok
+    end,
+    {error,enoent} = file:read_file(NifFileName),
+    {error,enoent} = file:read_file(OnHaltFileName),
+    ok.
+
+many_delay_halt(Config) when is_list(Config) ->
+    try
+        many_delay_halt_test(Config)
+    catch
+        throw:{skip, _} = Skip ->
+            Skip
+    end.
+
+many_delay_halt_test(Config) ->
+    Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING),
+    Priv = proplists:get_value(priv_dir, Config),
+    Tester = self(),
+    {ok, Peer, Node} = ?CT_PEER(),
+    Chk = fun () ->
+                  case erlang:system_info(schedulers_online) of
+                      1 -> throw({skip, "Too few schedulers online"});
+                      _ -> ok
+                  end,
+                  case erlang:system_info(dirty_cpu_schedulers_online) of
+                      1 -> throw({skip, "Too few dirty cpu schedulers online"});
+                      _ -> ok
+                  end,
+                  case erlang:system_info(dirty_io_schedulers) of
+                      1 -> throw({skip, "Too few dirty io schedulers online"});
+                      _ -> ok
+                  end
+          end,
+    try
+        erpc:call(Node, Chk)
+    catch
+        throw:{skip, _} = Skip ->
+            peer:stop(Peer),
+            throw(Skip)
+    end,
+    Mon = erlang:monitor(process, Peer),
+    ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {delay_halt}]),
+    ok = erpc:call(Node, file, set_cwd, [Priv]),
+    ProxyFun = fun (Tag) ->
+                       fun () ->
+                               receive
+                                   {delay_halt, _} = Msg ->
+                                       unlink(Tester),
+                                       Tester ! {Tag, Msg}
+                               end
+                       end
+               end,
+    [P1, P2, P3, P4, P5] = [spawn_link(Node, ProxyFun(X)) || X <- lists:seq(1, 5)],
+    Start = erlang:monotonic_time(millisecond),
+    ok = erpc:cast(Node, fun () -> delay_halt_io_bound(P1, "many_delay_halt_io2", 2) end),
+    ok = erpc:cast(Node, fun () -> delay_halt_io_bound(P2, "many_delay_halt_io1", 1) end),
+    ok = erpc:cast(Node, fun () -> delay_halt_cpu_bound(P3, "many_delay_halt_cpu1", 1) end),
+    ok = erpc:cast(Node, fun () -> delay_halt_cpu_bound(P4, "many_delay_halt_cpu2", 2) end),
+    _ = [receive
+             {X, {delay_halt, Pid}} when is_pid(Pid), Node == node(Pid) ->
+                 ok
+         end || X <- lists:seq(1, 4)],
+    ok = erpc:cast(Node, fun () -> delay_halt_normal(P5, "many_delay_halt_normal", 1) end),
+    receive
+        {5, {delay_halt, Pid}} when is_pid(Pid), Node == node(Pid) ->
+            ok
+    end,
+    ok = erpc:cast(Node, erlang, halt, []),
+    ok = wait_until(fun () ->
+                            {ok, <<"ok">>} == file:read_file(filename:join(Priv, "many_delay_halt_io2"))
+                                andalso {ok, <<"ok">>} == file:read_file(filename:join(Priv, "many_delay_halt_cpu2"))
+                    end,
+                    3000),
+    {ok, <<"ok">>} = file:read_file(filename:join(Priv, "many_delay_halt_cpu1")),
+    {ok, <<"ok">>} = file:read_file(filename:join(Priv, "many_delay_halt_io1")),
+    {ok, <<"ok">>} = file:read_file(filename:join(Priv, "many_delay_halt_normal")),
+    Time = erlang:monotonic_time(millisecond) - Start,
+    ct:log("many_delay_halt time=~pms", [Time]),
+    true = Time >= 2000,
+    receive {'DOWN', Mon, process, Peer, _} -> ok end,
+    ok.
+
+on_halt(Config) when is_list(Config) ->
+    on_halt(Config, new_code).
+
+on_halt_old_code(Config) when is_list(Config) ->
+    on_halt(Config, old_code).
+
+on_halt_old_and_new_code(Config) when is_list(Config) ->
+    on_halt(Config, old_and_new_code).
+
+on_halt(Config, Type) ->
+    Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING),
+    Priv = proplists:get_value(priv_dir, Config),
+    FileName = "on_halt_"++atom_to_list(Type),
+    OnHaltFileName = filename:join(Priv, FileName),
+    {ok, Peer, Node} = ?CT_PEER(),
+    Mon = erlang:monitor(process, Peer),
+    ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {on_halt, FileName, 1}]),
+    case Type of
+        new_code ->
+            ok;
+        old_code ->
+            true = erpc:call(Node, erlang, delete_module, [dirty_nif_SUITE]);
+        old_and_new_code ->
+            true = erpc:call(Node, erlang, delete_module, [dirty_nif_SUITE]),
+            ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {on_halt, FileName++"_new", 1}])
+    end,
+    ok = erpc:call(Node, file, set_cwd, [Priv]),
+    Start = erlang:monotonic_time(millisecond),
+    ok = erpc:cast(Node, erlang, halt, []),
+    ok = wait_until(fun () ->
+                            {ok, <<"ok">>} == file:read_file(OnHaltFileName)
+                                andalso (Type /= old_and_new_code
+                                         orelse {ok, <<"ok">>} == file:read_file(OnHaltFileName++"_new"))
+                    end,
+                    3000),
+    Time = erlang:monotonic_time(millisecond) - Start,
+    ct:log("~s time=~pms", [FileName, Time]),
+    if Type == old_and_new_code -> true = Time >= 2000;
+       true -> true = Time >= 1000
+       end,
+    receive {'DOWN', Mon, process, Peer, _} -> ok end,
+    ok.
+
+on_halt_module_code_format() ->
+    lists:flatten(["-module(~s).~n",
+                   "-export([load/1, lib_loaded/0]).~n",
+                   "-nifs([lib_loaded/0]).~n",
+                   "load(SoFile) -> erlang:load_nif(SoFile, ?MODULE_STRING).~n",
+                   "lib_loaded() -> false.~n"]).
+
+many_on_halt(Config) when is_list(Config) ->
+    DDir = ?config(data_dir, Config),
+    Priv = proplists:get_value(priv_dir, Config),
+    OnHaltModules = ["on_halt_a","on_halt_b","on_halt_c","on_halt_d","on_halt_e","on_halt_f"],
+    DeleteOnHaltModules = ["on_halt_a","on_halt_c","on_halt_d","on_halt_f"],
+    PurgeOnHaltModules = DeleteOnHaltModules -- ["on_halt_d"],
+    ActiveOnHaltModules = OnHaltModules -- PurgeOnHaltModules,
+    lists:foreach(fun (ModStr) ->
+                          Code = io_lib:format(on_halt_module_code_format(), [ModStr]),
+                          ok = file:write_file(filename:join(DDir, ModStr++".erl"), Code)
+                  end,
+                  OnHaltModules),
+    {ok, Peer, Node} = ?CT_PEER(),
+    Mon = erlang:monitor(process, Peer),
+    ok = erpc:call(Node,
+                   fun () ->
+                           ok = file:set_cwd(Priv),
+                           lists:foreach(fun (ModStr) ->
+                                                 AbsModStr = filename:join(DDir, ModStr),
+                                                 {ok,Mod,Bin} = compile:file(AbsModStr, [binary]),
+                                                 {module, Mod} = erlang:load_module(Mod, Bin),
+                                                 ok = Mod:load(AbsModStr),
+                                                 true = Mod:lib_loaded()
+                                         end,
+                                         OnHaltModules),
+                           lists:foreach(fun (ModStr) ->
+                                                 Mod = list_to_atom(ModStr),
+                                                 true = erlang:delete_module(Mod)
+                                         end, DeleteOnHaltModules),
+                           lists:foreach(fun (ModStr) ->
+                                                 Mod = list_to_atom(ModStr),
+                                                 true = erlang:purge_module(Mod)
+                                         end, PurgeOnHaltModules),
+                           ok
+                   end),
+    Start = erlang:monotonic_time(millisecond),
+    ok = erpc:cast(Node, erlang, halt, []),
+    ok = wait_until(fun () ->
+                            try
+                                lists:foreach(fun (ModStr) ->
+                                                      FileName = filename:join(Priv, ModStr),
+                                                      {ok, <<"ok">>} = file:read_file(FileName)
+                                              end, ActiveOnHaltModules),
+                                true
+                            catch
+                                _:_ ->
+                                    false
+                            end
+                    end,
+                    1000*(length(ActiveOnHaltModules)+1)),
+    Time = erlang:monotonic_time(millisecond) - Start,
+    ct:log("many_on_halt time=~pms", [Time]),
+    true = Time >= length(ActiveOnHaltModules)*1000,
+    receive {'DOWN', Mon, process, Peer, _} -> ok end,
+    ok.
+
+sync_halt(Config) when is_list(Config) ->
+    Suite = filename:join(?config(data_dir, Config), ?MODULE_STRING),
+    Priv = proplists:get_value(priv_dir, Config),
+    {Fun, FileName} = case ?config(group, Config) of
+                          halt_dirty_io ->
+                              {fun sync_halt_io_bound/2, "sync_halt_io_bound"};
+                          halt_dirty_cpu ->
+                              {fun sync_halt_cpu_bound/2, "sync_halt_cpu_bound"}
+                      end,
+    Tester = self(),
+    NifFileName = filename:join(Priv, FileName),
+    OnHaltBaseFileName = FileName++".onhalt",
+    OnHaltFileName = filename:join(Priv, OnHaltBaseFileName),
+    {ok, Peer, Node} = ?CT_PEER(),
+    Mon = erlang:monitor(process, Peer),
+    ok = erpc:call(Node, ?MODULE, load_nif, [Suite, {sync_halt, OnHaltBaseFileName, 1}]),
+    ok = erpc:call(Node, file, set_cwd, [Priv]),
+    Proxy = spawn_link(Node,
+                       fun () ->
+                               receive
+                                   {sync_halt, _} = Msg ->
+                                       unlink(Tester),
+                                       Tester ! Msg
+                               end
+                       end),
+    ok = erpc:cast(Node, fun () -> Fun(Proxy, FileName) end),
+    receive {sync_halt, Pid} when is_pid(Pid), Node == node(Pid) -> ok end,
+    Start = erlang:monotonic_time(millisecond),
+    ok = erpc:cast(Node, erlang, halt, []),
+    ok = wait_until(fun () ->
+                            {ok, <<"ok">>} == file:read_file(OnHaltFileName)
+                    end,
+                    2000),
+    ok = wait_until(fun () ->
+                            {ok, <<"ok">>} == file:read_file(NifFileName)
+                    end,
+                    4000),
+    Time = erlang:monotonic_time(millisecond) - Start,
+    ct:log("~s time=~pms", [FileName, Time]),
+    true = Time >= 1000,
+    receive {'DOWN', Mon, process, Peer, _} -> ok end,
+    ok.
+
 %%
 %% Internal...
 %%
@@ -774,6 +1178,14 @@ dirty_heap_access_nif(_) -> ?nif_stub.
 whereis_term(_Type,_Name) -> ?nif_stub.
 whereis_send(_Type,_Name,_Msg) -> ?nif_stub.
 dirty_terminating_literal_access(_Me, _Literal) -> ?nif_stub.
+delay_halt_normal(_Pid, _FileName, _Delay) -> ?nif_stub.
+delay_halt_io_bound(_Pid, _FileName, _Delay) -> ?nif_stub.
+delay_halt_cpu_bound(_Pid, _FileName, _Delay) -> ?nif_stub.
+sync_halt_io_bound(_Pid, _FileName) -> ?nif_stub.
+sync_halt_cpu_bound(_Pid, _FileName) -> ?nif_stub.
+set_halt_option_from_nif_normal(_Op) -> ?nif_stub.
+set_halt_option_from_nif_io_bound(_Op) -> ?nif_stub.
+set_halt_option_from_nif_cpu_bound(_Op) -> ?nif_stub.
 
 nif_stub_error(Line) ->
     exit({nif_not_loaded,module,?MODULE,line,Line}).
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src b/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src
index 4462afd815..55ca552cb2 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src
+++ b/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src
@@ -1,5 +1,5 @@
 
-NIF_LIBS = dirty_nif_SUITE@dll@
+NIF_LIBS = dirty_nif_SUITE@dll@ on_halt_a@dll@ on_halt_b@dll@ on_halt_c@dll@ on_halt_d@dll@ on_halt_e@dll@ on_halt_f@dll@
 
 all: $(NIF_LIBS) echo_drv@dll@
 
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
index fb5146278b..7f0fc9ea46 100644
--- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
+++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c
@@ -19,11 +19,14 @@
  */
 #include <erl_nif.h>
 #include <assert.h>
+#include <errno.h>
 #ifdef __WIN32__
 #include <windows.h>
 #else
 #include <unistd.h>
 #endif
+#include <stdio.h>
+#include <string.h>
 
 /*
  * Hack to get around this function missing from the NIF API.
@@ -43,8 +46,128 @@ static ERL_NIF_TERM atom_pid;
 static ERL_NIF_TERM atom_port;
 static ERL_NIF_TERM atom_send;
 
+typedef struct {
+    int halting;
+    int on_halt_wait;
+    ErlNifMutex *mtx;
+    ErlNifCond *cnd;
+    char *filename;
+} PrivData;
+
+static PrivData *make_priv_data(void)
+{
+    PrivData *pdata = enif_alloc(sizeof(PrivData));
+    if (!pdata)
+        return NULL;
+    pdata->halting = 0;
+    pdata->on_halt_wait = 0;
+    pdata->mtx = NULL;
+    pdata->cnd = NULL;
+    pdata->filename = NULL;
+    return pdata;
+}
+
+static void unload(ErlNifEnv *env, void *priv_data)
+{
+    if (priv_data) {
+        PrivData *pdata = priv_data;
+        if (pdata->mtx)
+            enif_mutex_destroy(pdata->mtx);
+        if (pdata->cnd)
+            enif_cond_destroy(pdata->cnd);
+        if (pdata->filename)
+            enif_free(pdata->filename);
+        enif_free(pdata);
+    }
+}
+
+static void on_halt(void *priv_data);
+
 static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 {
+    int arity;
+    const ERL_NIF_TERM *array;
+    if (enif_get_tuple(env, load_info, &arity, &array)) {
+        char atom_text[32];
+        int err;
+        unsigned filename_len;
+        PrivData *pdata = NULL;
+
+        if (arity < 1 || 3 < arity)
+            return __LINE__;
+        if (!enif_get_atom(env, array[0], &atom_text[0],
+                           sizeof(atom_text), ERL_NIF_LATIN1)) {
+            return __LINE__;
+        }
+        pdata = make_priv_data();
+        if (!pdata)
+            return __LINE__;
+        if (strcmp(atom_text, "on_halt") == 0) {
+            if (0 != enif_set_option(env, ERL_NIF_OPT_ON_HALT, on_halt)) {
+                unload(env, pdata);
+                return __LINE__;
+            }
+        }
+        else if (strcmp(atom_text, "delay_halt") == 0) {
+            if (0 != enif_set_option(env, ERL_NIF_OPT_DELAY_HALT)) {
+                unload(env, pdata);
+                return __LINE__;
+            }
+        }
+        else if (strcmp(atom_text, "sync_halt") == 0) {
+            if (0 != enif_set_option(env, ERL_NIF_OPT_ON_HALT, on_halt)) {
+                unload(env, pdata);
+                return __LINE__;
+            }
+            if (0 != enif_set_option(env, ERL_NIF_OPT_DELAY_HALT)) {
+                unload(env, pdata);
+                return __LINE__;
+            }
+            pdata->mtx = enif_mutex_create("sync_halt_dirty_nif_SUITE");
+            pdata->cnd = enif_cond_create("sync_halt_dirty_nif_SUITE");
+            if (!pdata->mtx || !pdata->cnd) {
+                unload(env, pdata);
+                return __LINE__;
+            }
+        }
+        else {
+            unload(env, pdata);
+            return __LINE__;
+        }
+
+        if (arity >= 2) {
+            if (!enif_get_list_length(env, array[1], &filename_len)) {
+                unload(env, pdata);
+                return __LINE__;
+            }
+            if (filename_len > 0) {
+                filename_len++;
+                pdata->filename = enif_alloc(filename_len);
+                if (!pdata->filename) {
+                    unload(env, pdata);
+                    return __LINE__;
+                }
+                if (filename_len != enif_get_string(env,
+                                                    array[1],
+                                                    pdata->filename,
+                                                    filename_len,
+                                                    ERL_NIF_LATIN1)) {
+                    unload(env, pdata);
+                    return __LINE__;
+                }
+            }
+        }
+        if (arity == 3) {
+            if (!enif_get_int(env, array[2], &pdata->on_halt_wait)
+                || pdata->on_halt_wait < 0
+                || pdata->on_halt_wait*1000 < 0) {
+                unload(env, pdata);
+                return __LINE__;
+            }
+        }
+        *priv_data = (void *) pdata;
+    }
+
     atom_badarg = enif_make_atom(env, "badarg");
     atom_error = enif_make_atom(env, "error");
     atom_false = enif_make_atom(env,"false");
@@ -57,6 +180,11 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     return 0;
 }
 
+static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+{
+    return load(env, priv_data, load_info);
+}
+
 static ERL_NIF_TERM lib_loaded(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     return enif_make_atom(env, "true");
@@ -468,6 +596,149 @@ static ERL_NIF_TERM dirty_terminating_literal_access(ErlNifEnv* env, int argc, c
     return self_term;
 }
 
+static int fn_write_ok(char *filename)
+{
+    FILE *file = fopen(filename, "w");
+    if (!file)
+        return EINVAL;
+    if (1 != fwrite("ok", 2, 1, file))
+        return EINVAL;
+    fclose(file);
+    return 0;
+}
+
+static int efn_write_ok(ErlNifEnv *env, const ERL_NIF_TERM arg)
+{
+    int res;
+    unsigned filename_len;
+    char *filename;
+
+    if (!enif_get_list_length(env, arg, &filename_len) || filename_len < 2) {
+        res = EINVAL;
+        goto done;
+    }
+    filename_len++;
+    filename = enif_alloc(filename_len);
+    if (!filename) {
+        res = ENOMEM;
+        goto done;
+    }
+    if (filename_len != enif_get_string(env,
+                                        arg,
+                                        filename,
+                                        filename_len,
+                                        ERL_NIF_LATIN1)) {
+        res = EINVAL;
+        goto done;
+    }
+    res = fn_write_ok(filename);
+done:
+    if (filename)
+        enif_free(filename);
+    switch (res) {
+    case 0:
+        return atom_ok;
+    case ENOMEM:
+        return enif_raise_exception(env, enif_make_atom(env, "enomem"));
+    default:
+        return enif_make_badarg(env);
+    }
+}
+
+static ERL_NIF_TERM delay_halt(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM msg;
+    ErlNifPid receiver, self;
+    int res, secs;
+
+    if (argc != 3)
+        return enif_make_badarg(env);
+    if (!enif_get_int(env, argv[2], &secs))
+        return enif_make_badarg(env);
+    if (secs < 0 || secs*1000 < 0)
+        return enif_make_badarg(env);
+    if (!enif_self(env, &self))
+	return enif_make_badarg(env);
+    if (!enif_get_local_pid(env, argv[0], &receiver))
+	return enif_make_badarg(env);
+    msg = enif_make_tuple2(env, enif_make_atom(env, "delay_halt"), enif_make_pid(env, &self));
+    res = enif_send(env, &receiver, NULL, msg);
+    if (!res)
+	return enif_make_badarg(env);
+
+#ifdef __WIN32__
+    Sleep(secs*1000);
+#else
+    sleep(secs);
+#endif
+    return efn_write_ok(env, argv[1]);
+}
+
+static ERL_NIF_TERM sync_halt(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM msg;
+    ErlNifPid receiver, self;
+    int res;
+    PrivData *pdata = enif_priv_data(env);
+    if (!pdata)
+        return enif_raise_exception(env, enif_make_atom(env, "missing_priv_data"));
+    if (argc != 2)
+        return enif_make_badarg(env);
+    if (!enif_self(env, &self))
+	return enif_make_badarg(env);
+    if (!enif_get_local_pid(env, argv[0], &receiver))
+	return enif_make_badarg(env);
+    msg = enif_make_tuple2(env, enif_make_atom(env, "sync_halt"), enif_make_pid(env, &self));
+    res = enif_send(env, &receiver, NULL, msg);
+    if (!res)
+	return enif_make_badarg(env);
+    enif_mutex_lock(pdata->mtx);
+    while (!pdata->halting)
+        enif_cond_wait(pdata->cnd, pdata->mtx);
+    enif_mutex_unlock(pdata->mtx);
+    return efn_write_ok(env, argv[1]);
+}
+
+static ERL_NIF_TERM set_halt_option_from_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    unsigned len;
+    char atom_text[32];
+    if (argc != 1)
+        return enif_make_badarg(env);
+    if (!enif_get_atom(env, argv[0], &atom_text[0], sizeof(atom_text), ERL_NIF_LATIN1))
+        return enif_make_badarg(env);
+    if (strcmp(atom_text, "set_on_halt_handler") == 0) {
+        if (0 == enif_set_option(env, ERL_NIF_OPT_ON_HALT, on_halt))
+            return atom_ok;
+        return atom_error;
+    }
+    else if (strcmp(atom_text, "delay_halt") == 0) {
+        if (0 == enif_set_option(env, ERL_NIF_OPT_DELAY_HALT))
+            return atom_ok;
+        return atom_error;
+    }
+    return enif_make_badarg(env);
+}
+
+static void on_halt(void *priv_data)
+{
+    PrivData *pdata = (PrivData *)priv_data;
+    int res;
+#ifdef __WIN32__
+    Sleep(pdata->on_halt_wait*1000);
+#else
+    sleep(pdata->on_halt_wait);
+#endif
+    if (pdata->mtx) {
+        enif_mutex_lock(pdata->mtx);
+        assert(!pdata->halting);
+        pdata->halting = !0;
+        enif_cond_broadcast(pdata->cnd);
+        enif_mutex_unlock(pdata->mtx);
+    }
+    res = fn_write_ok(pdata->filename);
+    assert(res == 0);
+}
 
 static ErlNifFunc nif_funcs[] =
 {
@@ -484,6 +755,14 @@ static ErlNifFunc nif_funcs[] =
     {"whereis_send", 3, whereis_send, ERL_NIF_DIRTY_JOB_IO_BOUND},
     {"whereis_term", 2, whereis_term, ERL_NIF_DIRTY_JOB_CPU_BOUND},
     {"dirty_terminating_literal_access", 2, dirty_terminating_literal_access, ERL_NIF_DIRTY_JOB_CPU_BOUND},
+    {"delay_halt_normal", 3, delay_halt, 0},
+    {"delay_halt_io_bound", 3, delay_halt, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"delay_halt_cpu_bound", 3, delay_halt, ERL_NIF_DIRTY_JOB_CPU_BOUND},
+    {"sync_halt_io_bound", 2, sync_halt, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"sync_halt_cpu_bound", 2, sync_halt, ERL_NIF_DIRTY_JOB_CPU_BOUND},
+    {"set_halt_option_from_nif_normal", 1, set_halt_option_from_nif, 0},
+    {"set_halt_option_from_nif_io_bound", 1, set_halt_option_from_nif, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"set_halt_option_from_nif_cpu_bound", 1, set_halt_option_from_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}
 };
 
-ERL_NIF_INIT(dirty_nif_SUITE,nif_funcs,load,NULL,NULL,NULL)
+ERL_NIF_INIT(dirty_nif_SUITE,nif_funcs,load,NULL,upgrade,unload)
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_a.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_a.c
new file mode 100644
index 0000000000..73d50a2bf1
--- /dev/null
+++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_a.c
@@ -0,0 +1,23 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2022. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "on_halt_nif.c"
+
+ERL_NIF_INIT(on_halt_a,nif_funcs,load,NULL,NULL,unload)
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_b.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_b.c
new file mode 100644
index 0000000000..b9e13a17fa
--- /dev/null
+++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_b.c
@@ -0,0 +1,23 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2022. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "on_halt_nif.c"
+
+ERL_NIF_INIT(on_halt_b,nif_funcs,load,NULL,NULL,unload)
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_c.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_c.c
new file mode 100644
index 0000000000..db875e7f18
--- /dev/null
+++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_c.c
@@ -0,0 +1,23 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2022. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "on_halt_nif.c"
+
+ERL_NIF_INIT(on_halt_c,nif_funcs,load,NULL,NULL,unload)
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_d.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_d.c
new file mode 100644
index 0000000000..e3b64245ce
--- /dev/null
+++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_d.c
@@ -0,0 +1,23 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2022. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "on_halt_nif.c"
+
+ERL_NIF_INIT(on_halt_d,nif_funcs,load,NULL,NULL,unload)
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_e.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_e.c
new file mode 100644
index 0000000000..73357c9b9d
--- /dev/null
+++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_e.c
@@ -0,0 +1,23 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2022. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "on_halt_nif.c"
+
+ERL_NIF_INIT(on_halt_e,nif_funcs,load,NULL,NULL,unload)
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_f.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_f.c
new file mode 100644
index 0000000000..58b4955d4f
--- /dev/null
+++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_f.c
@@ -0,0 +1,23 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2022. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "on_halt_nif.c"
+
+ERL_NIF_INIT(on_halt_f,nif_funcs,load,NULL,NULL,unload)
diff --git a/erts/emulator/test/dirty_nif_SUITE_data/on_halt_nif.c b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_nif.c
new file mode 100644
index 0000000000..610b80d3f7
--- /dev/null
+++ b/erts/emulator/test/dirty_nif_SUITE_data/on_halt_nif.c
@@ -0,0 +1,96 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2022. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+#include "erl_nif.h"
+#include <errno.h>
+#include <assert.h>
+#ifdef __WIN32__
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+
+
+static int fn_write_ok(char *filename)
+{
+    FILE *file = fopen(filename, "w");
+    if (!file)
+        return EINVAL;
+    if (1 != fwrite("ok", 2, 1, file))
+        return EINVAL;
+    fclose(file);
+    return 0;
+}
+
+static void on_halt(void *priv_data)
+{
+    int res;
+#ifdef __WIN32__
+    Sleep(1000);
+#else
+    sleep(1);
+#endif
+    assert(priv_data);
+    res = fn_write_ok((char *) priv_data);
+    assert(res == 0);
+}
+
+static void unload(ErlNifEnv *env, void *priv_data)
+{
+    if (priv_data)
+        enif_free(priv_data);
+}
+
+static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    unsigned filename_len;
+    char *filename;
+    if (0 != enif_set_option(env, ERL_NIF_OPT_ON_HALT, on_halt))
+        return __LINE__;
+    if (!enif_get_list_length(env, load_info, &filename_len))
+        return __LINE__;
+    if (filename_len == 0)
+        return __LINE__;
+    filename_len++;
+    filename = enif_alloc(filename_len);
+    if (!filename)
+        return __LINE__;
+    if (filename_len != enif_get_string(env,
+                                        load_info,
+                                        filename,
+                                        filename_len,
+                                        ERL_NIF_LATIN1)) {
+        enif_free(filename);
+        return __LINE__;
+    }
+    *priv_data = (void *) filename;
+    return 0;
+}
+
+static ERL_NIF_TERM lib_loaded(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    return enif_make_atom(env, "true");
+}
+
+static ErlNifFunc nif_funcs[] =
+{
+    {"lib_loaded", 0, lib_loaded}
+};
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 470f861e08..84125f139a 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -1204,8 +1204,13 @@ halt() ->
 
 %% halt/1
 %% Shadowed by erl_bif_types: erlang:halt/1
--spec halt(Status) -> no_return() when
-      Status :: non_neg_integer() | 'abort' | string().
+-spec halt(Status :: non_neg_integer()) ->
+          no_return();
+          (Abort :: abort) ->
+          no_return();
+          (CrashDumpSlogan :: string()) ->
+          no_return().
+
 -dialyzer({no_return, halt/1}).
 halt(Status) ->
     try
@@ -1216,11 +1221,18 @@ halt(Status) ->
 
 %% halt/2
 %% Shadowed by erl_bif_types: erlang:halt/2
--spec halt(Status, Options) -> no_return() when
-      Status :: non_neg_integer() | 'abort' | string(),
-      Options :: [Option],
-      Option :: {flush, boolean()}.
-halt(_Status, _Options) ->
+-type halt_options() ::
+        [{flush, boolean()}].
+
+-spec halt(Status :: non_neg_integer(), Options :: halt_options()) ->
+          no_return();
+          (Abort :: abort, Options :: halt_options()) ->
+          no_return();
+          (CrashDumpSlogan :: string(), Options :: halt_options()) ->
+          no_return().
+
+-dialyzer({no_return, halt/2}).
+halt(_, _) ->
     erlang:nif_error(undefined).
 
 %% has_prepared_code_on_load/1
-- 
2.35.3

openSUSE Build Service is sponsored by