File 2664-stdlib-Minor-optimization-of-the-Erlang-scanner.patch of Package erlang
From d76eec9ac43ba86f7d26748ab12e3afeb40339c2 Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Wed, 25 Sep 2019 10:43:03 +0200
Subject: [PATCH 4/8] stdlib: Minor optimization of the Erlang scanner
Remove underscores from numbers only when necessary.
---
lib/stdlib/src/erl_scan.erl | 126 +++++++++++++++++++++++++-------------------
1 file changed, 72 insertions(+), 54 deletions(-)
diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl
index 35ac83eb13..4779f4c508 100644
--- a/lib/stdlib/src/erl_scan.erl
+++ b/lib/stdlib/src/erl_scan.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1996-2015. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2019. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -379,7 +379,7 @@ scan1([$\%|Cs], St, Line, Col, Toks) when not St#erl_scan.comment ->
scan1([$\%=C|Cs], St, Line, Col, Toks) ->
scan_comment(Cs, St, Line, Col, Toks, [C]);
scan1([C|Cs], St, Line, Col, Toks) when ?DIGIT(C) ->
- scan_number(Cs, St, Line, Col, Toks, [C]);
+ scan_number(Cs, St, Line, Col, Toks, [C], no_underscore);
scan1("..."++Cs, St, Line, Col, Toks) ->
tok2(Cs, St, Line, Col, Toks, "...", '...', 3);
scan1(".."=Cs, _St, Line, Col, Toks) ->
@@ -938,30 +938,33 @@ escape_char($s) -> $\s; % \s = SPC
escape_char($d) -> $\d; % \d = DEL
escape_char(C) -> C.
-scan_number([C|Cs], St, Line, Col, Toks, Ncs) when ?DIGIT(C) ->
- scan_number(Cs, St, Line, Col, Toks, [C|Ncs]);
-scan_number([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs) when
+scan_number(Cs, St, Line, Col, Toks, {Ncs, Us}) ->
+ scan_number(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_number([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) ->
+ scan_number(Cs, St, Line, Col, Toks, [C|Ncs], Us);
+scan_number([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs, _Us) when
?DIGIT(Next) andalso ?DIGIT(Prev) ->
- scan_number(Cs, St, Line, Col, Toks, [Next,$_|Ncs]);
-scan_number([$.,C|Cs], St, Line, Col, Toks, Ncs) when ?DIGIT(C) ->
- scan_fraction(Cs, St, Line, Col, Toks, [C,$.|Ncs]);
-scan_number([$.]=Cs, _St, Line, Col, Toks, Ncs) ->
- {more,{Cs,Col,Toks,Line,Ncs,fun scan_number/6}};
-scan_number([$#|Cs]=Cs0, St, Line, Col, Toks, Ncs0) ->
+ scan_number(Cs, St, Line, Col, Toks, [Next,$_|Ncs], with_underscore);
+scan_number([$.,C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) ->
+ scan_fraction(Cs, St, Line, Col, Toks, [C,$.|Ncs], Us);
+scan_number([$.]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+ {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_number/6}};
+scan_number([$#|Cs]=Cs0, St, Line, Col, Toks, Ncs0, Us) ->
Ncs = lists:reverse(Ncs0),
- case catch list_to_integer(remove_digit_separators(Ncs)) of
+ case catch list_to_integer(remove_digit_separators(Ncs, Us)) of
B when B >= 2, B =< 1+$Z-$A+10 ->
Bcs = Ncs++[$#],
- scan_based_int(Cs, St, Line, Col, Toks, {B,[],Bcs});
+ scan_based_int(Cs, St, Line, Col, Toks, {B,[],Bcs}, no_underscore);
B ->
Len = length(Ncs),
scan_error({base,B}, Line, Col, Line, incr_column(Col, Len), Cs0)
end;
-scan_number([]=Cs, _St, Line, Col, Toks, Ncs) ->
- {more,{Cs,Col,Toks,Line,Ncs,fun scan_number/6}};
-scan_number(Cs, St, Line, Col, Toks, Ncs0) ->
+scan_number([]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+ {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_number/6}};
+scan_number(Cs, St, Line, Col, Toks, Ncs0, Us) ->
Ncs = lists:reverse(Ncs0),
- case catch list_to_integer(remove_digit_separators(Ncs)) of
+ case catch list_to_integer(remove_digit_separators(Ncs, Us)) of
N when is_integer(N) ->
tok3(Cs, St, Line, Col, Toks, integer, Ncs, N);
_ ->
@@ -969,7 +972,9 @@ scan_number(Cs, St, Line, Col, Toks, Ncs0) ->
scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs)
end.
-remove_digit_separators(Number) ->
+remove_digit_separators(Number, no_underscore) ->
+ Number;
+remove_digit_separators(Number, with_underscore) ->
[C || C <- Number, C =/= $_].
-define(BASED_DIGIT(C, B),
@@ -977,17 +982,20 @@ remove_digit_separators(Number) ->
orelse (C >= $A andalso B > 10 andalso C < $A + B - 10)
orelse (C >= $a andalso B > 10 andalso C < $a + B - 10))).
-scan_based_int([C|Cs], St, Line, Col, Toks, {B,Ncs,Bcs}) when
+scan_based_int(Cs, St, Line, Col, Toks, {State,Us}) ->
+ scan_based_int(Cs, St, Line, Col, Toks, State, Us).
+
+scan_based_int([C|Cs], St, Line, Col, Toks, {B,Ncs,Bcs}, Us) when
?BASED_DIGIT(C, B) ->
- scan_based_int(Cs, St, Line, Col, Toks, {B,[C|Ncs],Bcs});
-scan_based_int([$_,Next|Cs], St, Line, Col, Toks, {B,[Prev|_]=Ncs,Bcs}) when
+ scan_based_int(Cs, St, Line, Col, Toks, {B,[C|Ncs],Bcs}, Us);
+scan_based_int([$_,Next|Cs], St, Line, Col, Toks, {B,[Prev|_]=Ncs,Bcs}, _Us) when
?BASED_DIGIT(Next, B) andalso ?BASED_DIGIT(Prev, B) ->
- scan_based_int(Cs, St, Line, Col, Toks, {B,[Next,$_|Ncs],Bcs});
-scan_based_int([]=Cs, _St, Line, Col, Toks, State) ->
- {more,{Cs,Col,Toks,Line,State,fun scan_based_int/6}};
-scan_based_int(Cs, St, Line, Col, Toks, {B,Ncs0,Bcs}) ->
+ scan_based_int(Cs, St, Line, Col, Toks, {B,[Next,$_|Ncs],Bcs}, with_underscore);
+scan_based_int([]=Cs, _St, Line, Col, Toks, State, Us) ->
+ {more,{Cs,Col,Toks,Line,{State,Us},fun scan_based_int/6}};
+scan_based_int(Cs, St, Line, Col, Toks, {B,Ncs0,Bcs}, Us) ->
Ncs = lists:reverse(Ncs0),
- case catch erlang:list_to_integer(remove_digit_separators(Ncs), B) of
+ case catch erlang:list_to_integer(remove_digit_separators(Ncs, Us), B) of
N when is_integer(N) ->
tok3(Cs, St, Line, Col, Toks, integer, Bcs++Ncs, N);
_ ->
@@ -996,38 +1004,48 @@ scan_based_int(Cs, St, Line, Col, Toks, {B,Ncs0,Bcs}) ->
scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs)
end.
-scan_fraction([C|Cs], St, Line, Col, Toks, Ncs) when ?DIGIT(C) ->
- scan_fraction(Cs, St, Line, Col, Toks, [C|Ncs]);
-scan_fraction([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs) when
+scan_fraction(Cs, St, Line, Col, Toks, {Ncs,Us}) ->
+ scan_fraction(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_fraction([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) ->
+ scan_fraction(Cs, St, Line, Col, Toks, [C|Ncs], Us);
+scan_fraction([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs, _Us) when
?DIGIT(Next) andalso ?DIGIT(Prev) ->
- scan_fraction(Cs, St, Line, Col, Toks, [Next,$_|Ncs]);
-scan_fraction([E|Cs], St, Line, Col, Toks, Ncs) when E =:= $e; E =:= $E ->
- scan_exponent_sign(Cs, St, Line, Col, Toks, [E|Ncs]);
-scan_fraction([]=Cs, _St, Line, Col, Toks, Ncs) ->
- {more,{Cs,Col,Toks,Line,Ncs,fun scan_fraction/6}};
-scan_fraction(Cs, St, Line, Col, Toks, Ncs) ->
- float_end(Cs, St, Line, Col, Toks, Ncs).
-
-scan_exponent_sign([C|Cs], St, Line, Col, Toks, Ncs) when C =:= $+; C =:= $- ->
- scan_exponent(Cs, St, Line, Col, Toks, [C|Ncs]);
-scan_exponent_sign([]=Cs, _St, Line, Col, Toks, Ncs) ->
- {more,{Cs,Col,Toks,Line,Ncs,fun scan_exponent_sign/6}};
-scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs) ->
- scan_exponent(Cs, St, Line, Col, Toks, Ncs).
-
-scan_exponent([C|Cs], St, Line, Col, Toks, Ncs) when ?DIGIT(C) ->
- scan_exponent(Cs, St, Line, Col, Toks, [C|Ncs]);
-scan_exponent([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs) when
+ scan_fraction(Cs, St, Line, Col, Toks, [Next,$_|Ncs], with_underscore);
+scan_fraction([E|Cs], St, Line, Col, Toks, Ncs, Us) when E =:= $e; E =:= $E ->
+ scan_exponent_sign(Cs, St, Line, Col, Toks, [E|Ncs], Us);
+scan_fraction([]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+ {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_fraction/6}};
+scan_fraction(Cs, St, Line, Col, Toks, Ncs, Us) ->
+ float_end(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_exponent_sign(Cs, St, Line, Col, Toks, {Ncs, Us}) ->
+ scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_exponent_sign([C|Cs], St, Line, Col, Toks, Ncs, Us) when
+ C =:= $+; C =:= $- ->
+ scan_exponent(Cs, St, Line, Col, Toks, [C|Ncs], Us);
+scan_exponent_sign([]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+ {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_exponent_sign/6}};
+scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs, Us) ->
+ scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_exponent(Cs, St, Line, Col, Toks, {Ncs, Us}) ->
+ scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_exponent([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) ->
+ scan_exponent(Cs, St, Line, Col, Toks, [C|Ncs], Us);
+scan_exponent([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs, _) when
?DIGIT(Next) andalso ?DIGIT(Prev) ->
- scan_exponent(Cs, St, Line, Col, Toks, [Next,$_|Ncs]);
-scan_exponent([]=Cs, _St, Line, Col, Toks, Ncs) ->
- {more,{Cs,Col,Toks,Line,Ncs,fun scan_exponent/6}};
-scan_exponent(Cs, St, Line, Col, Toks, Ncs) ->
- float_end(Cs, St, Line, Col, Toks, Ncs).
+ scan_exponent(Cs, St, Line, Col, Toks, [Next,$_|Ncs], with_underscore);
+scan_exponent([]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+ {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_exponent/6}};
+scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us) ->
+ float_end(Cs, St, Line, Col, Toks, Ncs, Us).
-float_end(Cs, St, Line, Col, Toks, Ncs0) ->
+float_end(Cs, St, Line, Col, Toks, Ncs0, Us) ->
Ncs = lists:reverse(Ncs0),
- case catch list_to_float(remove_digit_separators(Ncs)) of
+ case catch list_to_float(remove_digit_separators(Ncs, Us)) of
F when is_float(F) ->
tok3(Cs, St, Line, Col, Toks, float, Ncs, F);
_ ->
--
2.16.4