File 2873-Implement-rand-bytes-1-and-rand-bytes_s-2.patch of Package erlang
From c0fbb37cd4e8220f8d331eeb3eac540fd7f20470 Mon Sep 17 00:00:00 2001
From: Raimo Niskanen <raimo@erlang.org>
Date: Fri, 7 Aug 2020 17:12:07 +0200
Subject: [PATCH 3/6] Implement rand:bytes/1 and rand:bytes_s/2
---
lib/stdlib/doc/src/rand.xml | 29 +++++
lib/stdlib/src/rand.erl | 57 ++++++++++
lib/stdlib/test/rand_SUITE.erl | 197 ++++++++++++++++++++++++++-------
3 files changed, 245 insertions(+), 38 deletions(-)
diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml
index 0eed1c2e49..9e3c628920 100644
--- a/lib/stdlib/doc/src/rand.xml
+++ b/lib/stdlib/doc/src/rand.xml
@@ -350,6 +350,35 @@ tests. We suggest to use a sign test to extract a random Boolean value.</pre>
</datatypes>
<funcs>
+ <func>
+ <name name="bytes" arity="1" since="OTP 24.0"/>
+ <fsummary>Return a random binary.</fsummary>
+ <desc><marker id="bytes-1"/>
+ <p>
+ Returns, for a specified integer <c><anno>N</anno> >= 0</c>,
+ a <c>binary()</c> with that number of random bytes.
+ Generates as many random numbers as required using
+ the selected algorithm to compose the binary,
+ and updates the state in the process dictionary accordingly.
+ </p>
+ </desc>
+ </func>
+
+ <func>
+ <name name="bytes_s" arity="2" since="OTP 24.0"/>
+ <fsummary>Return a random binary.</fsummary>
+ <desc><marker id="bytes-1"/>
+ <p>
+ Returns, for a specified integer <c><anno>N</anno> >= 0</c>
+ and a state, a <c>binary()</c> with that number of random bytes,
+ and a new state.
+ Generates as many random numbers as required using
+ the selected algorithm to compose the binary,
+ and the new state.
+ </p>
+ </desc>
+ </func>
+
<func>
<name name="export_seed" arity="0" since="OTP 18.0"/>
<fsummary>Export the random number generation state.</fsummary>
diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl
index 1de78a23ac..d5906d3730 100644
--- a/lib/stdlib/src/rand.erl
+++ b/lib/stdlib/src/rand.erl
@@ -31,6 +31,7 @@
export_seed/0, export_seed_s/1,
uniform/0, uniform/1, uniform_s/1, uniform_s/2,
uniform_real/0, uniform_real_s/1,
+ bytes/1, bytes_s/2,
jump/0, jump/1,
normal/0, normal/2, normal_s/1, normal_s/3
]).
@@ -539,6 +540,62 @@ uniform_real_s(#{max:=_} = AlgHandler, Next, M0, BitNo, R0) ->
{V1, R1} = Next(R0),
uniform_real_s(AlgHandler, Next, M0, BitNo, R1, ?MASK(56, V1), 56).
+
+%% bytes/1: given a number N,
+%% returns a random binary with N bytes
+
+-spec bytes(N :: non_neg_integer()) -> Bytes :: binary().
+bytes(N) ->
+ {Bytes, State} = bytes_s(N, seed_get()),
+ _ = seed_put(State),
+ Bytes.
+
+
+%% bytes_s/2: given a number N and a state,
+%% returns a random binary with N bytes and a new state
+
+-spec bytes_s(N :: non_neg_integer(), State :: state()) ->
+ {Bytes :: binary(), NewState :: state()}.
+bytes_s(N, {#{bits:=Bits, next:=Next} = AlgHandler, R})
+ when is_integer(N), 0 =< N ->
+ WeakLowBits = maps:get(weak_low_bits, AlgHandler, 0),
+ bytes_r(N, AlgHandler, Next, R, Bits, WeakLowBits);
+bytes_s(N, {#{max:=Mask, next:=Next} = AlgHandler, R})
+ when is_integer(N), 0 =< N, ?MASK(58) =< Mask ->
+ %% Old spec - assume 58 bits and 2 weak low bits
+ %% giving 56 bits i.e precisely 7 bytes per generated number
+ Bits = 58,
+ WeakLowBits = 2,
+ bytes_r(N, AlgHandler, Next, R, Bits, WeakLowBits).
+
+%% N: Number of bytes to generate
+%% Bits: Number of bits in the generated word
+%% WeakLowBits: Number of low bits in the generated word
+%% to waste due to poor quality
+bytes_r(N, AlgHandler, Next, R, Bits, WeakLowBits) ->
+ %% We use whole bytes from each generator word,
+ %% GoodBytes: that number of bytes
+ GoodBytes = (Bits - WeakLowBits) bsr 3,
+ GoodBits = GoodBytes bsl 3,
+ %% Shift: how many bits of each generator word to waste
+ %% by shifting right - we use the bits from the big end
+ Shift = Bits - GoodBits,
+ bytes_r(N, AlgHandler, Next, R, <<>>, GoodBytes, GoodBits, Shift).
+%%
+bytes_r(N0, AlgHandler, Next, R0, Bytes0, GoodBytes, GoodBits, Shift)
+ when GoodBytes < N0 ->
+ {V, R1} = Next(R0),
+ Bytes1 = <<Bytes0/binary, (V bsr Shift):GoodBits>>,
+ N1 = N0 - GoodBytes,
+ bytes_r(N1, AlgHandler, Next, R1, Bytes1, GoodBytes, GoodBits, Shift);
+bytes_r(N, AlgHandler, Next, R0, Bytes, _GoodBytes, GoodBits, _Shift) ->
+ {V, R1} = Next(R0),
+ Bits = N bsl 3,
+ %% Use the big end bits
+ Shift = GoodBits - Bits,
+ {<<Bytes/binary, (V bsr Shift):Bits>>, {AlgHandler, R1}}.
+
+
%% jump/1: given a state, jump/1
%% returns a new state which is equivalent to that
%% after a large number of call defined for each algorithm.
diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl
index 230020857f..207cff00e5 100644
--- a/lib/stdlib/test/rand_SUITE.erl
+++ b/lib/stdlib/test/rand_SUITE.erl
@@ -33,6 +33,7 @@ suite() ->
all() ->
[seed, interval_int, interval_float,
+ bytes_count,
api_eq,
reference,
{group, basic_stats},
@@ -45,7 +46,7 @@ all() ->
groups() ->
[{basic_stats, [parallel],
- [basic_stats_uniform_1, basic_stats_uniform_2,
+ [basic_stats_uniform_1, basic_stats_uniform_2, basic_stats_bytes,
basic_stats_standard_normal]},
{distr_stats, [parallel],
[stats_standard_normal_box_muller,
@@ -81,6 +82,17 @@ test() ->
algs() ->
[exsss, exrop, exsp, exs1024s, exs64, exsplus, exs1024, exro928ss].
+crypto_support() ->
+ try crypto:strong_rand_bytes(1) of
+ <<_>> ->
+ ok
+ catch
+ error : low_entropy ->
+ low_entropy;
+ error : undef ->
+ no_crypto
+ end.
+
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Test that seed and seed_s and export_seed/0 is working.
@@ -164,7 +176,9 @@ api_eq_1(S00) ->
V1 = rand:uniform(1000000),
{V2, S2} = rand:normal_s(S1),
V2 = rand:normal(),
- S2
+ B3 = rand:bytes(64),
+ {B3, S3} = rand:bytes_s(64, S2),
+ S3
end,
S1 = lists:foldl(Check, S00, lists:seq(1, 200)),
S1 = get(rand_seed),
@@ -251,6 +265,25 @@ interval_float_1(N) ->
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% Check that bytes/1 and bytes_s/2 generates
+%% the right number of bytes
+
+bytes_count(Config) when is_list(Config) ->
+ Algs = [default|algs()],
+ Counts = lists:seq(0, 255),
+ [begin
+ _ = rand:seed(Alg),
+ [begin
+ ExportState = rand:export_seed(),
+ B = rand:bytes(N),
+ {B, _NewState} = rand:bytes_s(N, rand:seed_s(ExportState)),
+ N = byte_size(B)
+ end || N <- Counts]
+ end || Alg <- Algs],
+ ok.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
%% Check if each algorithm generates the proper sequence.
reference(Config) when is_list(Config) ->
[reference_1(Alg) || Alg <- algs()],
@@ -327,6 +360,13 @@ basic_stats_uniform_2(Config) when is_list(Config) ->
|| Alg <- [default|algs()]],
ok.
+basic_stats_bytes(Config) when is_list(Config) ->
+ ct:timetrap({minutes,15}), %% valgrind needs a lot of time
+ [basic_bytes(
+ ?LOOP div 100, rand:seed_s(Alg), 0, array:new(256, [{default, 0}]))
+ || Alg <- [default|algs()]],
+ ok.
+
basic_stats_standard_normal(Config) when is_list(Config) ->
ct:timetrap({minutes,6}), %% valgrind needs a lot of time
io:format("Testing standard normal~n",[]),
@@ -357,6 +397,7 @@ basic_stats_normal(Config) when is_list(Config) ->
end,
IntendedMeanVariancePairs).
+
basic_uniform_1(N, S0, Sum, A0) when N > 0 ->
{X,S} =
case N band 1 of
@@ -369,39 +410,78 @@ basic_uniform_1(N, S0, Sum, A0) when N > 0 ->
A = array:set(I, 1+array:get(I,A0), A0),
basic_uniform_1(N-1, S, Sum+X, A);
basic_uniform_1(0, {#{type:=Alg}, _}, Sum, A) ->
- AverN = Sum / ?LOOP,
- io:format("~.12w: Average: ~.4f~n", [Alg, AverN]),
+ Loop = ?LOOP,
+ AverExp = 1.0 / 2,
+ Buckets = 100,
Counters = array:to_list(A),
Min = lists:min(Counters),
Max = lists:max(Counters),
- io:format("~.12w: Min: ~p Max: ~p~n", [Alg, Min, Max]),
-
- %% Verify that the basic statistics are ok
- %% be gentle we don't want to see to many failing tests
- abs(0.5 - AverN) < 0.005 orelse ct:fail({average, Alg, AverN}),
- abs(?LOOP div 100 - Min) < 1000 orelse ct:fail({min, Alg, Min}),
- abs(?LOOP div 100 - Max) < 1000 orelse ct:fail({max, Alg, Max}),
- ok.
+ basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max).
basic_uniform_2(N, S0, Sum, A0) when N > 0 ->
{X,S} = rand:uniform_s(100, S0),
A = array:set(X-1, 1+array:get(X-1,A0), A0),
basic_uniform_2(N-1, S, Sum+X, A);
basic_uniform_2(0, {#{type:=Alg}, _}, Sum, A) ->
- AverN = Sum / ?LOOP,
- io:format("~.12w: Average: ~.4f~n", [Alg, AverN]),
+ Loop = ?LOOP,
+ AverExp = ((100 - 1) / 2) + 1,
+ Buckets = 100,
Counters = tl(array:to_list(A)),
Min = lists:min(Counters),
Max = lists:max(Counters),
- io:format("~.12w: Min: ~p Max: ~p~n", [Alg, Min, Max]),
+ basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max).
+
+basic_bytes(N, S0, Sum0, A0) when N > 0 ->
+ ByteSize = 100,
+ {Bin,S} = rand:bytes_s(ByteSize, S0),
+ {Sum,A} = basic_bytes_incr(Bin, Sum0, A0),
+ basic_bytes(N-1, S, Sum, A);
+basic_bytes(0, {#{type:=Alg}, _}, Sum, A) ->
+ ByteSize = 100,
+ Loop = (?LOOP * ByteSize) div 100,
+ Buckets = 256,
+ AverExp = (Buckets - 1) / 2,
+ Counters = array:to_list(A),
+ Min = lists:min(Counters),
+ Max = lists:max(Counters),
+ basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max).
+basic_bytes_incr(Bin, Sum, A) ->
+ basic_bytes_incr(Bin, Sum, A, 0).
+%%
+basic_bytes_incr(Bin, Sum, A, N) ->
+ case Bin of
+ <<_:N/binary, B, _/binary>> ->
+ basic_bytes_incr(
+ Bin, Sum+B, array:set(B, array:get(B, A)+1, A), N+1);
+ <<_/binary>> ->
+ {Sum,A}
+ end.
+
+basic_verify(Alg, Loop, Sum, AverExp, Buckets, Min, Max) ->
+ AverDiff = AverExp * 0.01,
+ Aver = Sum / Loop,
+ io:format(
+ "~.12w: Expected Average: ~.4f, Allowed Diff: ~.4f, Average: ~.4f~n",
+ [Alg, AverExp, AverDiff, Aver]),
+ %%
+ CountExp = Loop / Buckets,
+ CountDiff = CountExp * 0.1,
+ io:format(
+ "~.12w: Expected Count: ~p, Allowed Diff: ~p, Min: ~p, Max: ~p~n",
+ [Alg, CountExp, CountDiff, Min, Max]),
+ %%
%% Verify that the basic statistics are ok
%% be gentle we don't want to see to many failing tests
- abs(50.5 - AverN) < 0.5 orelse ct:fail({average, Alg, AverN}),
- abs(?LOOP div 100 - Min) < 1000 orelse ct:fail({min, Alg, Min}),
- abs(?LOOP div 100 - Max) < 1000 orelse ct:fail({max, Alg, Max}),
+ abs(Aver - AverExp) < AverDiff orelse
+ ct:fail({average, Alg, Aver, AverExp, AverDiff}),
+ abs(Min - CountExp) < CountDiff orelse
+ ct:fail({min, Alg, Min, CountExp, CountDiff}),
+ abs(Max - CountExp) < CountDiff orelse
+ ct:fail({max, Alg, Max, CountExp, CountDiff}),
ok.
+
basic_normal_1(N, IntendedMean, IntendedVariance, S0, StandardSum, StandardSq) when N > 0 ->
{X,S} = normal_s(IntendedMean, IntendedVariance, S0),
% We now shape X into a standard normal distribution (in case it wasn't already)
@@ -417,6 +497,7 @@ basic_normal_1(0, _IntendedMean, _IntendedVariance, {#{type:=Alg}, _}, StandardS
StandardStdDev = math:sqrt(StandardVariance),
io:format("~.12w: Standardised Average: ~7.4f, Standardised StdDev ~6.4f~n",
[Alg, StandardMean, StandardStdDev]),
+ %%
%% Verify that the basic statistics are ok
%% be gentle we don't want to see to many failing tests
abs(StandardMean) < 0.005 orelse ct:fail({average, Alg, StandardMean}),
@@ -779,8 +860,8 @@ rand_state(Gen) ->
%% Test that the user can write algorithms.
plugin(Config) when is_list(Config) ->
- try crypto:strong_rand_bytes(1) of
- <<_>> ->
+ case crypto_support() of
+ ok ->
_ = lists:foldl(
fun(_, S0) ->
{V1, S1} = rand:uniform_s(10000, S0),
@@ -789,12 +870,9 @@ plugin(Config) when is_list(Config) ->
true = is_float(V2),
S2
end, crypto64_seed(), lists:seq(1, 200)),
- ok
- catch
- error:low_entropy ->
- {skip,low_entropy};
- error:undef ->
- {skip,no_crypto}
+ ok;
+ Problem ->
+ {skip,Problem}
end.
%% Test implementation
@@ -855,16 +933,19 @@ measure(Config) ->
{(X), (St)} when is_float(X) ->
St
end).
+-define(CHECK_BYTE_SIZE(Gen, Size, Bin, St),
+ case (Gen) of
+ {(Bin), (St)} when byte_size(Bin) =:= (Size) ->
+ St
+ end).
do_measure(_Config) ->
Algs =
- algs() ++
- try crypto:strong_rand_bytes(1) of
- <<_>> ->
- [crypto64, crypto_cache, crypto_aes, crypto]
- catch
- error:low_entropy -> [];
- error:undef -> []
+ case crypto_support() of
+ ok ->
+ algs() ++ [crypto64, crypto_cache, crypto_aes, crypto];
+ _ ->
+ algs()
end,
%%
ct:pal("~nRNG uniform integer range 10000 performance~n",[]),
@@ -1021,6 +1102,27 @@ do_measure(_Config) ->
end,
Algs),
%%
+ ByteSize = 16, % At about 100 bytes crypto_bytes breaks even to exsss
+ ct:pal("~nRNG ~w bytes performance~n",[ByteSize]),
+ _ =
+ measure_1(
+ fun (_) -> ByteSize end,
+ fun (State, Size, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_BYTE_SIZE(
+ Mod:bytes_s(Size, St0), Size,
+ Bin, St1)
+ end,
+ State)
+ end,
+ case crypto_support() of
+ ok ->
+ Algs ++ [crypto_bytes, crypto_bytes_cached];
+ _ ->
+ Algs
+ end),
+ %%
ct:pal("~nRNG uniform float performance~n",[]),
_ =
measure_1(
@@ -1116,6 +1218,10 @@ measure_1(RangeFun, Fun, Alg, TMark) ->
crypto_aes, crypto:strong_rand_bytes(256))};
random ->
{random, random:seed(os:timestamp()), get(random_seed)};
+ crypto_bytes ->
+ {?MODULE, ignored_state};
+ crypto_bytes_cached ->
+ {?MODULE, <<>>};
_ ->
{rand, rand:seed_s(Alg)}
end,
@@ -1139,6 +1245,21 @@ measure_1(RangeFun, Fun, Alg, TMark) ->
{Pid, Msg} -> Msg
end.
+%% Comparison algorithm for rand:bytes_s/2 vs. crypto:strong_rand_bytes/1
+bytes_s(N, Cache) when is_binary(Cache) ->
+ %% crypto_bytes_cached
+ case Cache of
+ <<Bytes:N/binary, Rest/binary>> ->
+ {Bytes, Rest};
+ <<Part/binary>> ->
+ <<Bytes:N/binary, Rest/binary>> =
+ <<Part/binary, (crypto:strong_rand_bytes(N * 16))/binary>>,
+ {Bytes, Rest}
+ end;
+bytes_s(N, ignored_state = St) ->
+ %% crypto_bytes
+ {crypto:strong_rand_bytes(N), St}.
+
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% The jump sequence tests has two parts
%% for those with the functional API (jump/1)
@@ -1268,16 +1389,16 @@ short_jump(Config) when is_list(Config) ->
fun ({Alg,AlgState}) ->
{Alg,rand:exro928_jump_2pow20(AlgState)}
end),
- try crypto:strong_rand_bytes(1) of
- _ ->
+ case crypto_support() of
+ ok ->
short_jump(
crypto:rand_seed_alg_s(crypto_aes, integer_to_list(Seed)),
fun ({Alg,AlgState}) ->
{Alg,crypto:rand_plugin_aes_jump_2pow20(AlgState)}
end),
- ok
- catch error:undef ->
- {skip,no_crypto}
+ ok;
+ Problem ->
+ {skip,Problem}
end.
short_jump({#{bits := Bits},_} = State_0, Jump2Pow20) ->
--
2.26.2
