File 2873-Implement-rand-bytes-1-and-rand-bytes_s-2.... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 2873-Implement-rand-bytes-1-and-rand-bytes_s-2.patch of Package erlang (Revision b63e1c137af11f81d27f49a213322c4e)

Currently displaying revision b63e1c137af11f81d27f49a213322c4e , Show latest

From c0fbb37cd4e8220f8d331eeb3eac540fd7f20470 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Fri, 7 Aug 2020 17:12:07 +0200
Subject: [PATCH 3/6] Implement rand:bytes/1 and rand:bytes_s/2

---
 lib/stdlib/doc/src/rand.xml    |  29 +++++
 lib/stdlib/src/rand.erl        |  57 ++++++++++
 lib/stdlib/test/rand_SUITE.erl | 197 ++++++++++++++++++++++++++-------
 3 files changed, 245 insertions(+), 38 deletions(-)

diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml
index 0eed1c2e49..9e3c628920 100644
--- a/lib/stdlib/doc/src/rand.xml
+++ b/lib/stdlib/doc/src/rand.xml
@@ -350,6 +350,35 @@ tests. We suggest to use a sign test to extract a random Boolean value.</pre>
   </datatypes>
 
   <funcs>
+    <func>
+      <name name="bytes" arity="1" since="OTP 24.0"/>
+      <fsummary>Return a random binary.</fsummary>
+      <desc><marker id="bytes-1"/>
+      <p>
+        Returns, for a specified integer <c><anno>N</anno> >= 0</c>,
+        a <c>binary()</c> with that number of random bytes.
+        Generates as many random numbers as required using
+        the selected algorithm to compose the binary,
+        and updates the state in the process dictionary accordingly.
+      </p>
+      </desc>
+    </func>
+
+    <func>
+      <name name="bytes_s" arity="2" since="OTP 24.0"/>
+      <fsummary>Return a random binary.</fsummary>
+      <desc><marker id="bytes-1"/>
+      <p>
+        Returns, for a specified integer <c><anno>N</anno> >= 0</c>
+        and a state, a <c>binary()</c> with that number of random bytes,
+        and a new state.
+        Generates as many random numbers as required using
+        the selected algorithm to compose the binary,
+        and the new state.
+      </p>
+      </desc>
+    </func>
+
     <func>
       <name name="export_seed" arity="0" since="OTP 18.0"/>
       <fsummary>Export the random number generation state.</fsummary>
diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl
index 1de78a23ac..d5906d3730 100644
--- a/lib/stdlib/src/rand.erl
+++ b/lib/stdlib/src/rand.erl
@@ -31,6 +31,7 @@
 	 export_seed/0, export_seed_s/1,
          uniform/0, uniform/1, uniform_s/1, uniform_s/2,
          uniform_real/0, uniform_real_s/1,
+         bytes/1, bytes_s/2,
          jump/0, jump/1,
          normal/0, normal/2, normal_s/1, normal_s/3
 	]).
@@ -539,6 +540,62 @@ uniform_real_s(#{max:=_} = AlgHandler, Next, M0, BitNo, R0) ->
     {V1, R1} = Next(R0),
     uniform_real_s(AlgHandler, Next, M0, BitNo, R1, ?MASK(56, V1), 56).
 
+
+%% bytes/1: given a number N,
+%% returns a random binary with N bytes
+
+-spec bytes(N :: non_neg_integer()) -> Bytes :: binary().
+bytes(N) ->
+    {Bytes, State} = bytes_s(N, seed_get()),
+    _ = seed_put(State),
+    Bytes.
+
+
+%% bytes_s/2: given a number N and a state,
+%% returns a random binary with N bytes and a new state
+
+-spec bytes_s(N :: non_neg_integer(), State :: state()) ->
+                     {Bytes :: binary(), NewState :: state()}.
+bytes_s(N, {#{bits:=Bits, next:=Next} = AlgHandler, R})
+  when is_integer(N), 0 =< N ->
+    WeakLowBits = maps:get(weak_low_bits, AlgHandler, 0),
+    bytes_r(N, AlgHandler, Next, R, Bits, WeakLowBits);
+bytes_s(N, {#{max:=Mask, next:=Next} = AlgHandler, R})
+  when is_integer(N), 0 =< N, ?MASK(58) =< Mask ->
+    %% Old spec - assume 58 bits and 2 weak low bits
+    %% giving 56 bits i.e precisely 7 bytes per generated number
+    Bits = 58,
+    WeakLowBits = 2,
+    bytes_r(N, AlgHandler, Next, R, Bits, WeakLowBits).
+
+%% N:           Number of bytes to generate
+%% Bits:        Number of bits in the generated word
+%% WeakLowBits: Number of low bits in the generated word
+%%              to waste due to poor quality
+bytes_r(N, AlgHandler, Next, R, Bits, WeakLowBits) ->
+    %% We use whole bytes from each generator word,
+    %% GoodBytes: that number of bytes
+    GoodBytes = (Bits - WeakLowBits) bsr 3,
+    GoodBits = GoodBytes bsl 3,
+    %% Shift: how many bits of each generator word to waste
+    %% by shifting right - we use the bits from the big end
+    Shift = Bits - GoodBits,
+    bytes_r(N, AlgHandler, Next, R, <<>>, GoodBytes, GoodBits, Shift).
+%%
+bytes_r(N0, AlgHandler, Next, R0, Bytes0, GoodBytes, GoodBits, Shift)
+  when GoodBytes < N0 ->
+    {V, R1} = Next(R0),
+    Bytes1 = <<Bytes0/binary, (V bsr Shift):GoodBits>>,
+    N1 = N0 - GoodBytes,
+    bytes_r(N1, AlgHandler, Next, R1, Bytes1, GoodBytes, GoodBits, Shift);
+bytes_r(N, AlgHandler, Next, R0, Bytes, _GoodBytes, GoodBits, _Shift) ->
+    {V, R1} = Next(R0),
+    Bits = N bsl 3,
+    %% Use the big end bits
+    Shift = GoodBits - Bits,
+    {<<Bytes/binary, (V bsr Shift):Bits>>, {AlgHandler, R1}}.
+
+
 %% jump/1: given a state, jump/1
 %% returns a new state which is equivalent to that
 %% after a large number of call defined for each algorithm.
diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl
index 230020857f..207cff00e5 100644
--- a/lib/stdlib/test/rand_SUITE.erl
+++ b/lib/stdlib/test/rand_SUITE.erl
@@ -33,6 +33,7 @@ suite() ->
 
 all() ->
     [seed, interval_int, interval_float,
+     bytes_count,
      api_eq,
      reference,
      {group, basic_stats},
@@ -45,7 +46,7 @@ all() ->
 
 groups() ->
     [{basic_stats, [parallel],
-      [basic_stats_uniform_1, basic_stats_uniform_2,
+      [basic_stats_uniform_1, basic_stats_uniform_2, basic_stats_bytes,
        basic_stats_standard_normal]},
      {distr_stats, [parallel],
       [stats_standard_normal_box_muller,
@@ -81,6 +82,17 @@ test() ->
 algs() ->
     [exsss, exrop, exsp, exs1024s, exs64, exsplus, exs1024, exro928ss].
 
+crypto_support() ->
+    try crypto:strong_rand_bytes(1) of
+        <<_>> ->
+            ok
+    catch
+        error : low_entropy ->
+            low_entropy;
+        error : undef ->
+            no_crypto
+    end.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 %% Test that seed and seed_s and export_seed/0 is working.
@@ -164,7 +176,9 @@ api_eq_1(S00) ->
 		    V1 = rand:uniform(1000000),
 		    {V2, S2} = rand:normal_s(S1),
 		    V2 = rand:normal(),
-		    S2
+                    B3 = rand:bytes(64),
+                    {B3, S3} = rand:bytes_s(64, S2),
+		    S3
 	    end,
     S1 = lists:foldl(Check, S00, lists:seq(1, 200)),
     S1 = get(rand_seed),
@@ -251,6 +265,25 @@ interval_float_1(N) ->
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
+%% Check that bytes/1 and bytes_s/2 generates
+%% the right number of bytes
+
+bytes_count(Config) when is_list(Config) ->
+    Algs = [default|algs()],
+    Counts = lists:seq(0, 255),
+    [begin
+         _ = rand:seed(Alg),
+         [begin
+              ExportState = rand:export_seed(),
+              B = rand:bytes(N),
+              {B, _NewState} = rand:bytes_s(N, rand:seed_s(ExportState)),
+              N = byte_size(B)
+          end || N <- Counts]
+     end || Alg <- Algs],
+    ok.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
 %% Check if each algorithm generates the proper sequence.
 reference(Config) when is_list(Config) ->
     [reference_1(Alg) || Alg <- algs()],
@@ -327,6 +360,13 @@ basic_stats_uniform_2(Config) when is_list(Config) ->
      || Alg <- [default|algs()]],
     ok.
 
+basic_stats_bytes(Config) when is_list(Config) ->
+    ct:timetrap({minutes,15}), %% valgrind needs a lot of time
+    [basic_bytes(
+       ?LOOP div 100, rand:seed_s(Alg), 0, array:new(256, [{default, 0}]))
+     || Alg <- [default|algs()]],
+    ok.
+
 basic_stats_standard_normal(Config) when is_list(Config) ->
     ct:timetrap({minutes,6}), %% valgrind needs a lot of time
     io:format("Testing standard normal~n",[]),
@@ -357,6 +397,7 @@ basic_stats_normal(Config) when is_list(Config) ->
       end,
       IntendedMeanVariancePairs).
 
+
 basic_uniform_1(N, S0, Sum, A0) when N > 0 ->
     {X,S} =
         case N band 1 of
@@ -369,39 +410,78 @@ basic_uniform_1(N, S0, Sum, A0) when N > 0 ->
     A = array:set(I, 1+array:get(I,A0), A0),
     basic_uniform_1(N-1, S, Sum+X, A);
 basic_uniform_1(0, {#{type:=Alg}, _}, Sum, A) ->
-    AverN = Sum / ?LOOP,
-    io:format("~.12w: Average: ~.4f~n", [Alg, AverN]),
+    Loop = ?LOOP,
+    AverExp = 1.0 / 2,
+    Buckets = 100,
     Counters = array:to_list(A),
     Min = lists:min(Counters),
     Max = lists:max(Counters),
-    io:format("~.12w: Min: ~p Max: ~p~n", [Alg, Min, Max]),
-
-    %% Verify that the basic statistics are ok
-    %% be gentle we don't want to see to many failing tests
-    abs(0.5 - AverN) < 0.005 orelse ct:fail({average, Alg, AverN}),
-    abs(?LOOP div 100 - Min) < 1000 orelse ct:fail({min, Alg, Min}),
-    abs(?LOOP div 100 - Max) < 1000 orelse ct:fail({max, Alg, Max}),
-    ok.
+    basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max).
 
 basic_uniform_2(N, S0, Sum, A0) when N > 0 ->
     {X,S} = rand:uniform_s(100, S0),
     A = array:set(X-1, 1+array:get(X-1,A0), A0),
     basic_uniform_2(N-1, S, Sum+X, A);
 basic_uniform_2(0, {#{type:=Alg}, _}, Sum, A) ->
-    AverN = Sum / ?LOOP,
-    io:format("~.12w: Average: ~.4f~n", [Alg, AverN]),
+    Loop = ?LOOP,
+    AverExp = ((100 - 1) / 2) + 1,
+    Buckets = 100,
     Counters = tl(array:to_list(A)),
     Min = lists:min(Counters),
     Max = lists:max(Counters),
-    io:format("~.12w: Min: ~p Max: ~p~n", [Alg, Min, Max]),
+    basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max).
+
+basic_bytes(N, S0, Sum0, A0) when N > 0 ->
+    ByteSize = 100,
+    {Bin,S} = rand:bytes_s(ByteSize, S0),
+    {Sum,A} = basic_bytes_incr(Bin, Sum0, A0),
+    basic_bytes(N-1, S, Sum, A);
+basic_bytes(0, {#{type:=Alg}, _}, Sum, A) ->
+    ByteSize = 100,
+    Loop = (?LOOP * ByteSize) div 100,
+    Buckets = 256,
+    AverExp = (Buckets - 1) / 2,
+    Counters = array:to_list(A),
+    Min = lists:min(Counters),
+    Max = lists:max(Counters),
+    basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max).
 
+basic_bytes_incr(Bin, Sum, A) ->
+    basic_bytes_incr(Bin, Sum, A, 0).
+%%
+basic_bytes_incr(Bin, Sum, A, N) ->
+    case Bin of
+        <<_:N/binary, B, _/binary>> ->
+            basic_bytes_incr(
+              Bin, Sum+B, array:set(B, array:get(B, A)+1, A), N+1);
+        <<_/binary>> ->
+            {Sum,A}
+    end.
+
+basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max) ->
+    AverDiff = AverExp * 0.01,
+    Aver = Sum / Loop,
+    io:format(
+      "~.12w: Expected Average: ~.4f, Allowed Diff: ~.4f, Average: ~.4f~n",
+      [Alg, AverExp, AverDiff, Aver]),
+    %%
+    CountExp = Loop / Buckets,
+    CountDiff = CountExp * 0.1,
+    io:format(
+      "~.12w: Expected Count: ~p, Allowed Diff: ~p, Min: ~p, Max: ~p~n",
+      [Alg, CountExp, CountDiff, Min, Max]),
+    %%
     %% Verify that the basic statistics are ok
     %% be gentle we don't want to see to many failing tests
-    abs(50.5 - AverN) < 0.5 orelse ct:fail({average, Alg, AverN}),
-    abs(?LOOP div 100 - Min) < 1000 orelse ct:fail({min, Alg, Min}),
-    abs(?LOOP div 100 - Max) < 1000 orelse ct:fail({max, Alg, Max}),
+    abs(Aver - AverExp) < AverDiff orelse
+        ct:fail({average, Alg, Aver, AverExp, AverDiff}),
+    abs(Min - CountExp) < CountDiff orelse
+        ct:fail({min, Alg, Min, CountExp, CountDiff}),
+    abs(Max - CountExp) < CountDiff orelse
+        ct:fail({max, Alg, Max, CountExp, CountDiff}),
     ok.
 
+
 basic_normal_1(N, IntendedMean, IntendedVariance, S0, StandardSum, StandardSq) when N > 0 ->
     {X,S} = normal_s(IntendedMean, IntendedVariance, S0),
     % We now shape X into a standard normal distribution (in case it wasn't already)
@@ -417,6 +497,7 @@ basic_normal_1(0, _IntendedMean, _IntendedVariance, {#{type:=Alg}, _}, StandardS
     StandardStdDev =  math:sqrt(StandardVariance),
     io:format("~.12w: Standardised Average: ~7.4f, Standardised StdDev ~6.4f~n",
               [Alg, StandardMean, StandardStdDev]),
+    %%
     %% Verify that the basic statistics are ok
     %% be gentle we don't want to see to many failing tests
     abs(StandardMean) < 0.005 orelse ct:fail({average, Alg, StandardMean}),
@@ -779,8 +860,8 @@ rand_state(Gen) ->
 
 %% Test that the user can write algorithms.
 plugin(Config) when is_list(Config) ->
-    try crypto:strong_rand_bytes(1) of
-        <<_>> ->
+    case crypto_support() of
+        ok ->
             _ = lists:foldl(
                   fun(_, S0) ->
                           {V1, S1} = rand:uniform_s(10000, S0),
@@ -789,12 +870,9 @@ plugin(Config) when is_list(Config) ->
                           true = is_float(V2),
                           S2
                   end, crypto64_seed(), lists:seq(1, 200)),
-            ok
-    catch
-        error:low_entropy ->
-            {skip,low_entropy};
-        error:undef ->
-            {skip,no_crypto}
+            ok;
+        Problem ->
+            {skip,Problem}
     end.
 
 %% Test implementation
@@ -855,16 +933,19 @@ measure(Config) ->
             {(X), (St)} when is_float(X) ->
                 St
         end).
+-define(CHECK_BYTE_SIZE(Gen, Size, Bin, St),
+        case (Gen) of
+            {(Bin), (St)} when byte_size(Bin) =:= (Size) ->
+                St
+        end).
 
 do_measure(_Config) ->
     Algs =
-        algs() ++
-        try crypto:strong_rand_bytes(1) of
-            <<_>> ->
-		[crypto64, crypto_cache, crypto_aes, crypto]
-        catch
-            error:low_entropy -> [];
-            error:undef -> []
+        case crypto_support() of
+            ok ->
+                algs() ++ [crypto64, crypto_cache, crypto_aes, crypto];
+            _ ->
+                algs()
         end,
     %%
     ct:pal("~nRNG uniform integer range 10000 performance~n",[]),
@@ -1021,6 +1102,27 @@ do_measure(_Config) ->
           end,
           Algs),
     %%
+    ByteSize = 16, % At about 100 bytes crypto_bytes breaks even to exsss
+    ct:pal("~nRNG ~w bytes performance~n",[ByteSize]),
+    _ =
+        measure_1(
+          fun (_) -> ByteSize end,
+          fun (State, Size, Mod) ->
+                  measure_loop(
+                    fun (St0) ->
+                            ?CHECK_BYTE_SIZE(
+                               Mod:bytes_s(Size, St0), Size,
+                               Bin, St1)
+                    end,
+                    State)
+          end,
+          case crypto_support() of
+              ok ->
+                  Algs ++ [crypto_bytes, crypto_bytes_cached];
+              _ ->
+                  Algs
+          end),
+    %%
     ct:pal("~nRNG uniform float performance~n",[]),
     _ =
         measure_1(
@@ -1116,6 +1218,10 @@ measure_1(RangeFun, Fun, Alg, TMark) ->
 		   crypto_aes, crypto:strong_rand_bytes(256))};
             random ->
                 {random, random:seed(os:timestamp()), get(random_seed)};
+            crypto_bytes ->
+                {?MODULE, ignored_state};
+            crypto_bytes_cached ->
+                {?MODULE, <<>>};
             _ ->
                 {rand, rand:seed_s(Alg)}
         end,
@@ -1139,6 +1245,21 @@ measure_1(RangeFun, Fun, Alg, TMark) ->
 	{Pid, Msg} -> Msg
     end.
 
+%% Comparison algorithm for rand:bytes_s/2 vs. crypto:strong_rand_bytes/1
+bytes_s(N, Cache) when is_binary(Cache) ->
+    %% crypto_bytes_cached
+    case Cache of
+        <<Bytes:N/binary, Rest/binary>> ->
+            {Bytes, Rest};
+        <<Part/binary>> ->
+            <<Bytes:N/binary, Rest/binary>> =
+                <<Part/binary, (crypto:strong_rand_bytes(N * 16))/binary>>,
+            {Bytes, Rest}
+    end;
+bytes_s(N, ignored_state = St) ->
+    %% crypto_bytes
+    {crypto:strong_rand_bytes(N), St}.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% The jump sequence tests has two parts
 %% for those with the functional API (jump/1)
@@ -1268,16 +1389,16 @@ short_jump(Config) when is_list(Config) ->
       fun ({Alg,AlgState}) ->
 	      {Alg,rand:exro928_jump_2pow20(AlgState)}
       end),
-    try crypto:strong_rand_bytes(1) of
-        _ ->
+    case crypto_support() of
+        ok ->
             short_jump(
               crypto:rand_seed_alg_s(crypto_aes, integer_to_list(Seed)),
               fun ({Alg,AlgState}) ->
                       {Alg,crypto:rand_plugin_aes_jump_2pow20(AlgState)}
               end),
-            ok
-    catch error:undef ->
-            {skip,no_crypto}
+            ok;
+        Problem ->
+            {skip,Problem}
     end.
 
 short_jump({#{bits := Bits},_} = State_0, Jump2Pow20) ->
-- 
2.26.2

Places

File 2873-Implement-rand-bytes-1-and-rand-bytes_s-2.patch of Package erlang (Revision b63e1c137af11f81d27f49a213322c4e)

Places