File 3731-Implement-simple-command-line-serving-via-... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 3731-Implement-simple-command-line-serving-via-httpd.patch of Package erlang (Revision b9af3d1f35ade773f4f570c4ea2d6c63)

Currently displaying revision b9af3d1f35ade773f4f570c4ea2d6c63 , Show latest

From 76e11f0990b64b3bf0769b8ee9d903a1addf1901 Mon Sep 17 00:00:00 2001
From: Johannes Christ <jc@jchri.st>
Date: Thu, 25 May 2023 11:52:10 +0200
Subject: [PATCH] Implement simple command-line serving via httpd

Often times people want to quickly have a directory index of some folder
to either view directly or share on the local network. For instance, in
Python, `python3 -m http.server` implements a simple server
accomplishing exactly this.

`httpd` has the needed functionality to do this, but did not define a
command-line entrypoint. Users who wanted to quickly serve files with
Erlang needed hacks like shown here:
https://gist.github.com/willurd/5720255#erlang.

This commit adds support for the following forms of hosting a simple
directory listing with support for `index.html` and basic mimetypes,
with the following commands:

erl -S httpd
    erl -S httpd serve
    erl -S httpd serve path/to/dir

The original suggestion cropped up here:
https://erlangforums.com/t/httpc-httpd-improvements/2622/8
---
 lib/inets/doc/src/http_server.xml             |  22 ++
 lib/inets/src/http_server/httpd.erl           | 115 +++++++++
 lib/inets/test/Makefile                       |   3 +-
 lib/inets/test/httpd_serve_SUITE.erl          | 230 ++++++++++++++++++
 .../test/httpd_serve_SUITE_data/aaa/info.txt  |   1 +
 .../test/httpd_serve_SUITE_data/index.html    |   1 +
 .../httpd_serve_SUITE_data/sample_file.html   |   1 +
 .../httpd_serve_SUITE_data/subdir/index.html  |   1 +
 8 files changed, 373 insertions(+), 1 deletion(-)
 create mode 100644 lib/inets/test/httpd_serve_SUITE.erl
 create mode 100644 lib/inets/test/httpd_serve_SUITE_data/aaa/info.txt
 create mode 100644 lib/inets/test/httpd_serve_SUITE_data/index.html
 create mode 100644 lib/inets/test/httpd_serve_SUITE_data/sample_file.html
 create mode 100644 lib/inets/test/httpd_serve_SUITE_data/subdir/index.html

diff --git a/lib/inets/doc/src/http_server.xml b/lib/inets/doc/src/http_server.xml
index bc7507f703..78d1ee641c 100644
--- a/lib/inets/doc/src/http_server.xml
+++ b/lib/inets/doc/src/http_server.xml
@@ -612,6 +612,28 @@ start() ->
 	received.</p>
     </section>
   </section>
+  <section>
+    <title>Serving files from the command line</title>
+    <p>httpd includes functionality to quickly serve files from the command
+      line. In its simplest form, <c>erl -S httpd</c> will serve files in
+      the local directory on localhost.</p>
+    <taglist>
+      <tag><c>--port</c></tag>
+      <item>Sets the port to bind on. Defaults to <c>8000</c>.</item>
+
+      <tag><c>--bind</c></tag>
+      <item>Sets the bind address to listen on. Defaults to <c>127.0.0.1</c>.</item>
+
+      <tag><i>DIRECTORY</i></tag>
+      <item>Sets the directory to serve data from. Defaults to the current directory.</item>
+    </taglist>
+    <p>For example, to serve files from directory <c>test_results</c> on port <c>4000</c>:</p>
+    <code>
+      erl -S httpd serve --port 4000 test_results
+    </code>
+    <p>For a full reference of all options, run
+      <c>erl -S httpd serve --help</c>.</p>
+  </section>
 </chapter>
 
 
diff --git a/lib/inets/src/http_server/httpd.erl b/lib/inets/src/http_server/httpd.erl
index 0c662522fa..97e78e10fd 100644
--- a/lib/inets/src/http_server/httpd.erl
+++ b/lib/inets/src/http_server/httpd.erl
@@ -43,6 +43,8 @@
          info/3,
          info/4
         ]).
+%% Command line interface
+-export([start/1, serve/1]).
 
 -deprecated({parse_query, 1,
             "use uri_string:dissect_query/1 instead"}).
@@ -392,6 +394,119 @@ service_info(Pid) ->
 	    {error, service_not_available} 
     end.
 
+%%%--------------------------------------------------------------
+%%% Command line interface
+%%%--------------------------------------------------------------------
+
+parse_ip_address(Input) ->
+    case inet:parse_address(Input) of
+        {ok, Address} -> Address;
+        {error, einval} -> error(badarg)
+    end.
+
+%% Try to locate good mime types to use for the server.
+%% If none were found on the host, uses a slim default.
+default_mime_types() ->
+    Locations = [
+        "/etc/mime.types"
+        % Note nginx installations also occasionally host a `mime.types` file,
+        % but this is usually in nginx's own configuration file format. Apache,
+        % on the other hand, uses the standard format and can be used.
+    ],
+    find_mime_types(Locations).
+
+find_mime_types([Path | Paths]) ->
+    case filelib:is_file(Path) of
+        true -> Path;
+        false -> find_mime_types(Paths)
+    end;
+
+find_mime_types([]) ->
+    [
+        {"html", "text/html"}, {"htm", "text/html"}, {"js", "text/javascript"},
+        {"css","text/css"}, {"gif", "image/gif"}, {"jpg", "image/jpeg"},
+        {"jpeg", "image/jpeg"}, {"png", "image/png"}
+    ].
+
+serve_cli() ->
+    #{
+      arguments => [
+        #{
+          name => directory,
+          type => string,
+          help => "Directory to serve data from.",
+          default => "."
+        },
+        #{
+          name => help,
+          type => boolean,
+          short => $h,
+          long => "-help",
+          help => "Show this description."
+        },
+        #{
+          name => port,
+          type => {integer, [{min, 0}, {max, 65535}]},
+          short => $p,
+          long => "-port",
+          default => 8000,
+          help => (
+            "Port to bind on. Use '0' for the OS to automatically assign "
+            "a port which can then be seen on server startup."
+          )
+        },
+        #{
+          name => address,
+          type => {custom, fun parse_ip_address/1},
+          short => $b,
+          long => "-bind",
+          default => {127, 0, 0, 1},
+          help => "IP address to listen on. Use 0.0.0.0 or :: for all interfaces."
+        }
+      ],
+      help => "Start a HTTP server serving files from DIRECTORY.",
+      handler => fun do_serve/1
+    }.
+
+start(Args) ->
+    %% `-S` without a function and without arguments
+    serve(Args).
+
+serve(Args) ->
+    argparse:run(Args, serve_cli(), #{progname => "erl -S httpd serve"}).
+
+do_serve(#{help := true}) ->
+    io:format("~ts", [argparse:help(serve_cli())]),
+    erlang:halt(0);
+do_serve(#{address := Address, port := Port, directory := Path}) ->
+    AbsPath = string:trim(filename:absname(Path), trailing, "/."),
+    inets:start(),
+    IpFamilyOpts = case Address of 
+        {_, _, _, _} -> [];
+        _ -> [{ipfamily, inet6}]
+    end,
+    {ok, Pid} = start_service(
+      [
+         {bind_address, Address},
+         {document_root, AbsPath},
+         {server_root, AbsPath},
+         {directory_index, ["index.html"]},
+         {port, Port},
+         {mime_type, "application/octet-stream"},
+         {mime_types, default_mime_types()},
+         {modules, [mod_alias, mod_dir, mod_get]}
+      ] ++ IpFamilyOpts
+    ),
+    % This is needed to support random port assignment (--port 0)
+    [{port, ActualPort}] = info(Pid, [port]),
+    io:fwrite("~nStarted HTTP server on http://~s:~w at ~s~n",
+              [inet:ntoa(Address), ActualPort, AbsPath]),
+    receive
+        {From, shutdown} ->
+            ok = stop_service(Pid),
+            From ! done
+    end.
+
 %%%--------------------------------------------------------------
 %%% Internal functions
 %%%--------------------------------------------------------------------
diff --git a/lib/inets/test/Makefile b/lib/inets/test/Makefile
index da0805bc4d..0f8ba516a0 100644
--- a/lib/inets/test/Makefile
+++ b/lib/inets/test/Makefile
@@ -129,6 +129,7 @@ MODULES =                 		\
 	httpd_bench_SUITE               \
 	http_test_lib    		\
 	httpd_basic_SUITE		\
+	httpd_serve_SUITE       	\
 	httpd_load        		\
 	httpd_time_test	  		\
 	httpd_1_1         		\
@@ -164,7 +165,7 @@ INETS_FILES = inets.config $(INETS_SPECS)
 
 
 INETS_DATADIRS = inets_SUITE_data inets_socketwrap_SUITE_data
-HTTPD_DATADIRS = httpd_test_data httpd_SUITE_data httpd_basic_SUITE_data httpd_bench_SUITE_data
+HTTPD_DATADIRS = httpd_test_data httpd_SUITE_data httpd_basic_SUITE_data httpd_bench_SUITE_data httpd_serve_SUITE_data
 
 HTTPC_DATADIRS = httpc_SUITE_data httpc_proxy_SUITE_data
 
diff --git a/lib/inets/test/httpd_serve_SUITE.erl b/lib/inets/test/httpd_serve_SUITE.erl
new file mode 100644
index 0000000000..c49391c35c
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE.erl
@@ -0,0 +1,230 @@
+%% Tests for the `erl -S httpd serve` functionality.
+-module(httpd_serve_SUITE).
+-export([suite/0, all/0, groups/0]).
+-export([
+    argless_start/1,
+    argless_serve/1,
+    simple_random_port_serve/1,
+    serve_on_all_interfaces_v4/1,
+    serve_on_localhost_v4/1,
+    serve_on_all_interfaces_v6/1,
+    serve_on_localhost_v6/1,
+    serve_custom_directory/1
+]).
+
+%% When starting up servers for tests, these variables define how long to
+%% wait for the server to report that it has started up, and after how
+%% many retries to quit waiting for its report altogether.
+-define(STARTUP_WAIT_NAPTIME_MS, 20).
+-define(STARTUP_WAIT_RETRIES, 100).
+
+%% Default assertions to run in all tests.
+-define(DEFAULT_ASSERTIONS, [directory_index, random_file]).
+
+suite() ->
+    [{ct_hooks, [ts_install_cth]},
+     {timetrap, {seconds, 30}}].
+
+all() ->
+    [{group, httpd_serve_on_default_port},
+     {group, httpd_serve_on_random_ports}].
+
+groups() ->
+    [{httpd_serve_on_default_port, [sequence], [
+        argless_serve,
+        argless_start]},
+     {httpd_serve_on_random_ports, [parallel], [
+        simple_random_port_serve,
+        serve_on_all_interfaces_v4,
+        serve_on_localhost_v4,
+        serve_on_all_interfaces_v6,
+        serve_on_localhost_v6,
+        serve_custom_directory
+      ]}].
+
+%%
+%% Test cases
+%%
+
+%% Fixed ports (must be run one at a time)
+
+argless_start(_Config) ->
+    ServerFun = fun () -> httpd:start([]) end,
+    verify_server(ServerFun).
+
+argless_serve(_Config) ->
+    ServerFun = fun () -> httpd:serve([]) end,
+    verify_server(ServerFun).
+
+%% Random ports (can run in parallel)
+
+simple_random_port_serve(Config) ->
+    verify_server(["--port", "0", suite_data(Config)]).
+
+serve_on_all_interfaces_v4(Config) ->
+    verify_server(["--port", "0", "--bind", "0.0.0.0", suite_data(Config)]).
+
+serve_on_localhost_v4(Config) ->
+    verify_server(["--port", "0", "--bind", "127.0.0.1", suite_data(Config)]).
+
+serve_on_all_interfaces_v6(Config) ->
+    verify_server(["--port", "0", "--bind", "::", suite_data(Config)]).
+
+serve_on_localhost_v6(Config) ->
+    verify_server(["--port", "0", "--bind", "::1", suite_data(Config)]).
+
+serve_custom_directory(Config) ->
+    SuiteData = suite_data(Config),
+    verify_server(["--port", "0", filename:join(SuiteData, "subdir")]).
+
+%%
+%% Assertion functions
+%%
+
+%% Assert that the server responds properly.
+run_server_assertions(Response) ->
+    run_server_assertions(Response, ?DEFAULT_ASSERTIONS).
+
+%% Assert that the server responds properly.
+run_server_assertions({ok, {Ip, Port, Path}}, Assertions) when is_integer(Port) ->
+    % From the `filelib:wildcard/1` docs:
+    % "Directory separators must always be written as /, even on Windows."
+    IpToRequest = case Ip of
+        {_, _, _, _} -> inet:ntoa(Ip);
+        {_, _, _, _, _, _, _, _} -> "[" ++ inet:ntoa(Ip) ++ "]"
+    end,
+
+    ct:log("Validating custom assertions"),
+    DirectoryUrl = "http://" ++ IpToRequest ++ ":" ++ integer_to_list(Port) ++ "/",
+    ServerInfo = #{
+        url => DirectoryUrl,
+        bind_ip => Ip,
+        path => Path
+    },
+    ok = verify_assertions(Assertions, ServerInfo),
+    ct:comment("Ran ~w assertion(s).", [length(Assertions)]).
+
+%%
+%% Assertion helper functions
+%%
+
+verify_200_at(Url) ->
+    HttpcOpts = [{socket_opts, [{ipfamily, inet6fb4}]}],
+    Request = {Url, []},
+    Response = httpc:request(get, Request, [{autoredirect, false}], HttpcOpts),
+    case Response of
+        {ok, {{_Version, 200, _}, _Headers, _Body}} ->
+            Response;
+        {ok, {{_Version, 301, _}, Headers, _Body}} ->
+            % To be resilient against the case where the server is not
+            % reachable under its `server_name`, for instance, in test
+            % containers or other hosts with unexpected networking setups,
+            % replace the suggested hostname with the hostname we came from.
+            {_, SuggestedTarget} = proplists:lookup("location", Headers),
+            #{path := SuggestedPath} = uri_string:parse(SuggestedTarget),
+            OurUri = uri_string:parse(Url),
+            DecomposedTarget = maps:put(path, SuggestedPath, OurUri),
+            RedirectTarget = uri_string:recompose(DecomposedTarget),
+            RedirectedRequest = {RedirectTarget, []},
+            RedirectedResponse = httpc:request(get, RedirectedRequest, [], HttpcOpts),
+            ct:log("Following redirect (rewritten from ~s to ~s)", [SuggestedTarget, RedirectTarget]),
+            {ok, {{_, 200, _}, _, _}} = RedirectedResponse;
+        {error, {failed_connect, [{to_address, {"::", _Port}},
+                                  {inet6, [inet6], eaddrnotavail},
+                                  {inet, [inet], nxdomain}]}} ->
+            % In this case we could bind on all IPv6 interfaces (::) just
+            % fine, but could not issue requests to it due to an OS error.
+            % Write it off as a networking misconfiguration and skip.
+            exit({skip, "Could not reach host on IPv6 address ::"})
+    end.
+
+verify_assertions([], _ServerInfo) ->
+    ok;
+
+verify_assertions([directory_index | Assertions], #{url := Url} = ServerInfo) ->
+    ct:log("Validating directory index at ~s", [Url]),
+    verify_200_at(Url),
+    ct:log("Directory index received with a 200"),
+    verify_assertions(Assertions, ServerInfo);
+
+verify_assertions([random_file | Assertions],  #{url := Url, path := Path} = ServerInfo) ->
+    Files = filelib:wildcard(Path ++ "/*"),
+    File = lists:nth(rand:uniform(length(Files)), Files),
+    Basename = filename:basename(File),
+    FileUrl = Url ++ Basename,
+    ct:log("Validating random file at ~s", [FileUrl]),
+    verify_200_at(FileUrl),
+    ct:log("File received with a 200"),
+    verify_assertions(Assertions, ServerInfo).
+
+%%
+%% Helper functions
+%%
+
+suite_data(Config) ->
+    proplists:get_value(data_dir, Config).
+
+verify_server(FunOrArgs) ->
+    TestFun = fun run_server_assertions/1,
+    with_server(FunOrArgs, TestFun).
+
+with_server(Args, TestFun) when is_list(Args) ->
+    ServerFun = fun () -> httpd:serve(Args) end,
+    run_with_server(ServerFun, TestFun);
+
+with_server(ServerFun, TestFun) when is_function(ServerFun) ->
+    run_with_server(ServerFun, TestFun).
+
+run_with_server(ServerFun, TestFun) ->
+    ct:log("Starting server"),
+    ct:capture_start(),
+    {Child, _Reference} = spawn_monitor(ServerFun),
+    StartupResult = wait_for_startup_line(?STARTUP_WAIT_RETRIES),
+    ct:capture_stop(),
+    {ok, Line} = StartupResult,
+    Parsed = parse_startup_line(Line),
+    ct:log("Running test function"),
+    Result = TestFun(Parsed),
+    ct:log("Test function finished, shutting down server"),
+    Child ! {self(), shutdown},
+    receive done -> ok after 5000 -> ct:fail("No server shutdown after 5s") end,
+    ct:log("Server stopped"),
+    Result.
+
+%% Wait for `ct:capture_get' to give us the output we're looking for.
+wait_for_startup_line(Tries) ->
+    wait_for_startup_line([], [], Tries).
+
+wait_for_startup_line([], [], 0) ->
+    {error, no_output_at_all};
+wait_for_startup_line([], Unexpected, 0) ->
+    {error, {no_startup_line, unexpected_output, Unexpected}};
+wait_for_startup_line([], Unexpected, Tries) when Tries > 0 ->
+    receive
+        {'DOWN', _Reference, process, _Child, Info} ->
+            case Info of
+                {{badmatch, {error, {listen, eaddrnotavail}}}, _} ->
+                    exit({skip, "Adress not available to listen"});
+                _ ->
+                    ct:fail("Child process has died: ~w", [Info])
+            end
+    after 
+        0 -> ok
+    end,
+    timer:sleep(?STARTUP_WAIT_NAPTIME_MS),
+    wait_for_startup_line(ct:capture_get(), Unexpected, Tries - 1);
+wait_for_startup_line(["\nStarted HTTP" ++ _Rest = Line | _Lines], _Unexpected, _Tries) ->
+    {ok, Line};
+wait_for_startup_line([Line | Lines], Unexpected, Tries) ->
+    wait_for_startup_line(Lines, [Line | Unexpected], Tries).
+
+%% Parse the given line into a tuple.
+%% Example line:
+%%   Started HTTP server on http://127.0.0.1:8000 at /path/to/lib/inets/make_test_dir/ct_logs/ct_run.test_server@zulu.2023-06-06_12.07.27\n"
+parse_startup_line(Line) ->
+    {match, [_, RawIp, RawPort, Path]} = re:run(
+        Line, "^\nStarted HTTP server on http://(.+):(\\d+) at (.*)\\n$", [{capture, all, list}]
+    ),
+    {ok, Ip} = inet:parse_address(RawIp),
+    Port = list_to_integer(RawPort),
+    {ok, {Ip, Port, Path}}.
diff --git a/lib/inets/test/httpd_serve_SUITE_data/aaa/info.txt b/lib/inets/test/httpd_serve_SUITE_data/aaa/info.txt
new file mode 100644
index 0000000000..890527d5ba
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE_data/aaa/info.txt
@@ -0,0 +1 @@
+This directory serves to test proper redirects for subdirectories.
diff --git a/lib/inets/test/httpd_serve_SUITE_data/index.html b/lib/inets/test/httpd_serve_SUITE_data/index.html
new file mode 100644
index 0000000000..0741e1d445
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE_data/index.html
@@ -0,0 +1 @@
+<h1>httpd_serve_SUITE_data/index.html</h1>
diff --git a/lib/inets/test/httpd_serve_SUITE_data/sample_file.html b/lib/inets/test/httpd_serve_SUITE_data/sample_file.html
new file mode 100644
index 0000000000..5bbbe438b3
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE_data/sample_file.html
@@ -0,0 +1 @@
+<h1>httpd_serve_SUITE_data/sample_file.html</h1>
diff --git a/lib/inets/test/httpd_serve_SUITE_data/subdir/index.html b/lib/inets/test/httpd_serve_SUITE_data/subdir/index.html
new file mode 100644
index 0000000000..6add0208af
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE_data/subdir/index.html
@@ -0,0 +1 @@
+<h1>httpd_serve_SUITE_data/subdir/index.html</h1>
-- 
2.35.3

Places

File 3731-Implement-simple-command-line-serving-via-httpd.patch of Package erlang (Revision b9af3d1f35ade773f4f570c4ea2d6c63)

Places