File 2664-stdlib-Minor-optimization-of-the-Erlang-sc... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 2664-stdlib-Minor-optimization-of-the-Erlang-scanner.patch of Package erlang (Revision aca0373827bf6cd8dc0a3e04bae29ec6)

Currently displaying revision aca0373827bf6cd8dc0a3e04bae29ec6 , Show latest

From d76eec9ac43ba86f7d26748ab12e3afeb40339c2 Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Wed, 25 Sep 2019 10:43:03 +0200
Subject: [PATCH 4/8] stdlib: Minor optimization of the Erlang scanner

Remove underscores from numbers only when necessary.
---
 lib/stdlib/src/erl_scan.erl | 126 +++++++++++++++++++++++++-------------------
 1 file changed, 72 insertions(+), 54 deletions(-)

diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl
index 35ac83eb13..4779f4c508 100644
--- a/lib/stdlib/src/erl_scan.erl
+++ b/lib/stdlib/src/erl_scan.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2015. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2019. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -379,7 +379,7 @@ scan1([$\%|Cs], St, Line, Col, Toks) when not St#erl_scan.comment ->
 scan1([$\%=C|Cs], St, Line, Col, Toks) ->
     scan_comment(Cs, St, Line, Col, Toks, [C]);
 scan1([C|Cs], St, Line, Col, Toks) when ?DIGIT(C) ->
-    scan_number(Cs, St, Line, Col, Toks, [C]);
+    scan_number(Cs, St, Line, Col, Toks, [C], no_underscore);
 scan1("..."++Cs, St, Line, Col, Toks) ->
     tok2(Cs, St, Line, Col, Toks, "...", '...', 3);
 scan1(".."=Cs, _St, Line, Col, Toks) ->
@@ -938,30 +938,33 @@ escape_char($s) -> $\s;                         % \s = SPC
 escape_char($d) -> $\d;                         % \d = DEL
 escape_char(C) -> C.
 
-scan_number([C|Cs], St, Line, Col, Toks, Ncs) when ?DIGIT(C) ->
-    scan_number(Cs, St, Line, Col, Toks, [C|Ncs]);
-scan_number([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs) when
+scan_number(Cs, St, Line, Col, Toks, {Ncs, Us}) ->
+    scan_number(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_number([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) ->
+    scan_number(Cs, St, Line, Col, Toks, [C|Ncs], Us);
+scan_number([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs, _Us) when
       ?DIGIT(Next) andalso ?DIGIT(Prev) ->
-    scan_number(Cs, St, Line, Col, Toks, [Next,$_|Ncs]);
-scan_number([$.,C|Cs], St, Line, Col, Toks, Ncs) when ?DIGIT(C) ->
-    scan_fraction(Cs, St, Line, Col, Toks, [C,$.|Ncs]);
-scan_number([$.]=Cs, _St, Line, Col, Toks, Ncs) ->
-    {more,{Cs,Col,Toks,Line,Ncs,fun scan_number/6}};
-scan_number([$#|Cs]=Cs0, St, Line, Col, Toks, Ncs0) ->
+    scan_number(Cs, St, Line, Col, Toks, [Next,$_|Ncs], with_underscore);
+scan_number([$.,C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) ->
+    scan_fraction(Cs, St, Line, Col, Toks, [C,$.|Ncs], Us);
+scan_number([$.]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+    {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_number/6}};
+scan_number([$#|Cs]=Cs0, St, Line, Col, Toks, Ncs0, Us) ->
     Ncs = lists:reverse(Ncs0),
-    case catch list_to_integer(remove_digit_separators(Ncs)) of
+    case catch list_to_integer(remove_digit_separators(Ncs, Us)) of
         B when B >= 2, B =< 1+$Z-$A+10 ->
             Bcs = Ncs++[$#],
-            scan_based_int(Cs, St, Line, Col, Toks, {B,[],Bcs});
+            scan_based_int(Cs, St, Line, Col, Toks, {B,[],Bcs}, no_underscore);
         B ->
             Len = length(Ncs),
             scan_error({base,B}, Line, Col, Line, incr_column(Col, Len), Cs0)
     end;
-scan_number([]=Cs, _St, Line, Col, Toks, Ncs) ->
-    {more,{Cs,Col,Toks,Line,Ncs,fun scan_number/6}};
-scan_number(Cs, St, Line, Col, Toks, Ncs0) ->
+scan_number([]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+    {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_number/6}};
+scan_number(Cs, St, Line, Col, Toks, Ncs0, Us) ->
     Ncs = lists:reverse(Ncs0),
-    case catch list_to_integer(remove_digit_separators(Ncs)) of
+    case catch list_to_integer(remove_digit_separators(Ncs, Us)) of
         N when is_integer(N) ->
             tok3(Cs, St, Line, Col, Toks, integer, Ncs, N);
         _ ->
@@ -969,7 +972,9 @@ scan_number(Cs, St, Line, Col, Toks, Ncs0) ->
             scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs)
     end.
 
-remove_digit_separators(Number) ->
+remove_digit_separators(Number, no_underscore) ->
+    Number;
+remove_digit_separators(Number, with_underscore) ->
     [C || C <- Number, C =/= $_].
 
 -define(BASED_DIGIT(C, B),
@@ -977,17 +982,20 @@ remove_digit_separators(Number) ->
          orelse (C >= $A andalso B > 10 andalso C < $A + B - 10)
          orelse (C >= $a andalso B > 10 andalso C < $a + B - 10))).
 
-scan_based_int([C|Cs], St, Line, Col, Toks, {B,Ncs,Bcs}) when
+scan_based_int(Cs, St, Line, Col, Toks, {State,Us}) ->
+    scan_based_int(Cs, St, Line, Col, Toks, State, Us).
+
+scan_based_int([C|Cs], St, Line, Col, Toks, {B,Ncs,Bcs}, Us) when
       ?BASED_DIGIT(C, B) ->
-    scan_based_int(Cs, St, Line, Col, Toks, {B,[C|Ncs],Bcs});
-scan_based_int([$_,Next|Cs], St, Line, Col, Toks, {B,[Prev|_]=Ncs,Bcs}) when
+    scan_based_int(Cs, St, Line, Col, Toks, {B,[C|Ncs],Bcs}, Us);
+scan_based_int([$_,Next|Cs], St, Line, Col, Toks, {B,[Prev|_]=Ncs,Bcs}, _Us) when
       ?BASED_DIGIT(Next, B) andalso ?BASED_DIGIT(Prev, B) ->
-    scan_based_int(Cs, St, Line, Col, Toks, {B,[Next,$_|Ncs],Bcs});
-scan_based_int([]=Cs, _St, Line, Col, Toks, State) ->
-    {more,{Cs,Col,Toks,Line,State,fun scan_based_int/6}};
-scan_based_int(Cs, St, Line, Col, Toks, {B,Ncs0,Bcs}) ->
+    scan_based_int(Cs, St, Line, Col, Toks, {B,[Next,$_|Ncs],Bcs}, with_underscore);
+scan_based_int([]=Cs, _St, Line, Col, Toks, State, Us) ->
+    {more,{Cs,Col,Toks,Line,{State,Us},fun scan_based_int/6}};
+scan_based_int(Cs, St, Line, Col, Toks, {B,Ncs0,Bcs}, Us) ->
     Ncs = lists:reverse(Ncs0),
-    case catch erlang:list_to_integer(remove_digit_separators(Ncs), B) of
+    case catch erlang:list_to_integer(remove_digit_separators(Ncs, Us), B) of
         N when is_integer(N) ->
             tok3(Cs, St, Line, Col, Toks, integer, Bcs++Ncs, N);
         _ ->
@@ -996,38 +1004,48 @@ scan_based_int(Cs, St, Line, Col, Toks, {B,Ncs0,Bcs}) ->
             scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs)
     end.
 
-scan_fraction([C|Cs], St, Line, Col, Toks, Ncs) when ?DIGIT(C) ->
-    scan_fraction(Cs, St, Line, Col, Toks, [C|Ncs]);
-scan_fraction([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs) when
+scan_fraction(Cs, St, Line, Col, Toks, {Ncs,Us}) ->
+    scan_fraction(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_fraction([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) ->
+    scan_fraction(Cs, St, Line, Col, Toks, [C|Ncs], Us);
+scan_fraction([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs, _Us) when
       ?DIGIT(Next) andalso ?DIGIT(Prev) ->
-    scan_fraction(Cs, St, Line, Col, Toks, [Next,$_|Ncs]);
-scan_fraction([E|Cs], St, Line, Col, Toks, Ncs) when E =:= $e; E =:= $E ->
-    scan_exponent_sign(Cs, St, Line, Col, Toks, [E|Ncs]);
-scan_fraction([]=Cs, _St, Line, Col, Toks, Ncs) ->
-    {more,{Cs,Col,Toks,Line,Ncs,fun scan_fraction/6}};
-scan_fraction(Cs, St, Line, Col, Toks, Ncs) ->
-    float_end(Cs, St, Line, Col, Toks, Ncs).
-
-scan_exponent_sign([C|Cs], St, Line, Col, Toks, Ncs) when C =:= $+; C =:= $- ->
-    scan_exponent(Cs, St, Line, Col, Toks, [C|Ncs]);
-scan_exponent_sign([]=Cs, _St, Line, Col, Toks, Ncs) ->
-    {more,{Cs,Col,Toks,Line,Ncs,fun scan_exponent_sign/6}};
-scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs) ->
-    scan_exponent(Cs, St, Line, Col, Toks, Ncs).
-
-scan_exponent([C|Cs], St, Line, Col, Toks, Ncs) when ?DIGIT(C) ->
-    scan_exponent(Cs, St, Line, Col, Toks, [C|Ncs]);
-scan_exponent([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs) when
+    scan_fraction(Cs, St, Line, Col, Toks, [Next,$_|Ncs], with_underscore);
+scan_fraction([E|Cs], St, Line, Col, Toks, Ncs, Us) when E =:= $e; E =:= $E ->
+    scan_exponent_sign(Cs, St, Line, Col, Toks, [E|Ncs], Us);
+scan_fraction([]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+    {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_fraction/6}};
+scan_fraction(Cs, St, Line, Col, Toks, Ncs, Us) ->
+    float_end(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_exponent_sign(Cs, St, Line, Col, Toks, {Ncs, Us}) ->
+    scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_exponent_sign([C|Cs], St, Line, Col, Toks, Ncs, Us) when
+      C =:= $+; C =:= $- ->
+    scan_exponent(Cs, St, Line, Col, Toks, [C|Ncs], Us);
+scan_exponent_sign([]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+    {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_exponent_sign/6}};
+scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs, Us) ->
+    scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_exponent(Cs, St, Line, Col, Toks, {Ncs, Us}) ->
+    scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us).
+
+scan_exponent([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) ->
+    scan_exponent(Cs, St, Line, Col, Toks, [C|Ncs], Us);
+scan_exponent([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs, _) when
       ?DIGIT(Next) andalso ?DIGIT(Prev) ->
-    scan_exponent(Cs, St, Line, Col, Toks, [Next,$_|Ncs]);
-scan_exponent([]=Cs, _St, Line, Col, Toks, Ncs) ->
-    {more,{Cs,Col,Toks,Line,Ncs,fun scan_exponent/6}};
-scan_exponent(Cs, St, Line, Col, Toks, Ncs) ->
-    float_end(Cs, St, Line, Col, Toks, Ncs).
+    scan_exponent(Cs, St, Line, Col, Toks, [Next,$_|Ncs], with_underscore);
+scan_exponent([]=Cs, _St, Line, Col, Toks, Ncs, Us) ->
+    {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_exponent/6}};
+scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us) ->
+    float_end(Cs, St, Line, Col, Toks, Ncs, Us).
 
-float_end(Cs, St, Line, Col, Toks, Ncs0) ->
+float_end(Cs, St, Line, Col, Toks, Ncs0, Us) ->
     Ncs = lists:reverse(Ncs0),
-    case catch list_to_float(remove_digit_separators(Ncs)) of
+    case catch list_to_float(remove_digit_separators(Ncs, Us)) of
         F when is_float(F) ->
             tok3(Cs, St, Line, Col, Toks, float, Ncs, F);
         _ ->
-- 
2.16.4

Places

File 2664-stdlib-Minor-optimization-of-the-Erlang-scanner.patch of Package erlang (Revision aca0373827bf6cd8dc0a3e04bae29ec6)

Places