File 4731-Implement-simple-command-line-serving-via-httpd.patch of Package erlang
From 76e11f0990b64b3bf0769b8ee9d903a1addf1901 Mon Sep 17 00:00:00 2001
From: Johannes Christ <jc@jchri.st>
Date: Thu, 25 May 2023 11:52:10 +0200
Subject: [PATCH] Implement simple command-line serving via httpd
Often times people want to quickly have a directory index of some folder
to either view directly or share on the local network. For instance, in
Python, `python3 -m http.server` implements a simple server
accomplishing exactly this.
`httpd` has the needed functionality to do this, but did not define a
command-line entrypoint. Users who wanted to quickly serve files with
Erlang needed hacks like shown here:
https://gist.github.com/willurd/5720255#erlang.
This commit adds support for the following forms of hosting a simple
directory listing with support for `index.html` and basic mimetypes,
with the following commands:
erl -S httpd
erl -S httpd serve
erl -S httpd serve path/to/dir
The original suggestion cropped up here:
https://erlangforums.com/t/httpc-httpd-improvements/2622/8
---
lib/inets/doc/src/http_server.xml | 22 ++
lib/inets/src/http_server/httpd.erl | 115 +++++++++
lib/inets/test/Makefile | 3 +-
lib/inets/test/httpd_serve_SUITE.erl | 230 ++++++++++++++++++
.../test/httpd_serve_SUITE_data/aaa/info.txt | 1 +
.../test/httpd_serve_SUITE_data/index.html | 1 +
.../httpd_serve_SUITE_data/sample_file.html | 1 +
.../httpd_serve_SUITE_data/subdir/index.html | 1 +
8 files changed, 373 insertions(+), 1 deletion(-)
create mode 100644 lib/inets/test/httpd_serve_SUITE.erl
create mode 100644 lib/inets/test/httpd_serve_SUITE_data/aaa/info.txt
create mode 100644 lib/inets/test/httpd_serve_SUITE_data/index.html
create mode 100644 lib/inets/test/httpd_serve_SUITE_data/sample_file.html
create mode 100644 lib/inets/test/httpd_serve_SUITE_data/subdir/index.html
diff --git a/lib/inets/doc/src/http_server.xml b/lib/inets/doc/src/http_server.xml
index bc7507f703..78d1ee641c 100644
--- a/lib/inets/doc/src/http_server.xml
+++ b/lib/inets/doc/src/http_server.xml
@@ -612,6 +612,28 @@ start() ->
received.</p>
</section>
</section>
+ <section>
+ <title>Serving files from the command line</title>
+ <p>httpd includes functionality to quickly serve files from the command
+ line. In its simplest form, <c>erl -S httpd</c> will serve files in
+ the local directory on localhost.</p>
+ <taglist>
+ <tag><c>--port</c></tag>
+ <item>Sets the port to bind on. Defaults to <c>8000</c>.</item>
+
+ <tag><c>--bind</c></tag>
+ <item>Sets the bind address to listen on. Defaults to <c>127.0.0.1</c>.</item>
+
+ <tag><i>DIRECTORY</i></tag>
+ <item>Sets the directory to serve data from. Defaults to the current directory.</item>
+ </taglist>
+ <p>For example, to serve files from directory <c>test_results</c> on port <c>4000</c>:</p>
+ <code>
+ erl -S httpd serve --port 4000 test_results
+ </code>
+ <p>For a full reference of all options, run
+ <c>erl -S httpd serve --help</c>.</p>
+ </section>
</chapter>
diff --git a/lib/inets/src/http_server/httpd.erl b/lib/inets/src/http_server/httpd.erl
index 0c662522fa..97e78e10fd 100644
--- a/lib/inets/src/http_server/httpd.erl
+++ b/lib/inets/src/http_server/httpd.erl
@@ -43,6 +43,8 @@
info/3,
info/4
]).
+%% Command line interface
+-export([start/1, serve/1]).
-deprecated({parse_query, 1,
"use uri_string:dissect_query/1 instead"}).
@@ -392,6 +394,119 @@ service_info(Pid) ->
{error, service_not_available}
end.
+%%%--------------------------------------------------------------
+%%% Command line interface
+%%%--------------------------------------------------------------------
+
+parse_ip_address(Input) ->
+ case inet:parse_address(Input) of
+ {ok, Address} -> Address;
+ {error, einval} -> error(badarg)
+ end.
+
+%% Try to locate good mime types to use for the server.
+%% If none were found on the host, uses a slim default.
+default_mime_types() ->
+ Locations = [
+ "/etc/mime.types"
+ % Note nginx installations also occasionally host a `mime.types` file,
+ % but this is usually in nginx's own configuration file format. Apache,
+ % on the other hand, uses the standard format and can be used.
+ ],
+ find_mime_types(Locations).
+
+find_mime_types([Path | Paths]) ->
+ case filelib:is_file(Path) of
+ true -> Path;
+ false -> find_mime_types(Paths)
+ end;
+
+find_mime_types([]) ->
+ [
+ {"html", "text/html"}, {"htm", "text/html"}, {"js", "text/javascript"},
+ {"css","text/css"}, {"gif", "image/gif"}, {"jpg", "image/jpeg"},
+ {"jpeg", "image/jpeg"}, {"png", "image/png"}
+ ].
+
+serve_cli() ->
+ #{
+ arguments => [
+ #{
+ name => directory,
+ type => string,
+ help => "Directory to serve data from.",
+ default => "."
+ },
+ #{
+ name => help,
+ type => boolean,
+ short => $h,
+ long => "-help",
+ help => "Show this description."
+ },
+ #{
+ name => port,
+ type => {integer, [{min, 0}, {max, 65535}]},
+ short => $p,
+ long => "-port",
+ default => 8000,
+ help => (
+ "Port to bind on. Use '0' for the OS to automatically assign "
+ "a port which can then be seen on server startup."
+ )
+ },
+ #{
+ name => address,
+ type => {custom, fun parse_ip_address/1},
+ short => $b,
+ long => "-bind",
+ default => {127, 0, 0, 1},
+ help => "IP address to listen on. Use 0.0.0.0 or :: for all interfaces."
+ }
+ ],
+ help => "Start a HTTP server serving files from DIRECTORY.",
+ handler => fun do_serve/1
+ }.
+
+start(Args) ->
+ %% `-S` without a function and without arguments
+ serve(Args).
+
+serve(Args) ->
+ argparse:run(Args, serve_cli(), #{progname => "erl -S httpd serve"}).
+
+do_serve(#{help := true}) ->
+ io:format("~ts", [argparse:help(serve_cli())]),
+ erlang:halt(0);
+do_serve(#{address := Address, port := Port, directory := Path}) ->
+ AbsPath = string:trim(filename:absname(Path), trailing, "/."),
+ inets:start(),
+ IpFamilyOpts = case Address of
+ {_, _, _, _} -> [];
+ _ -> [{ipfamily, inet6}]
+ end,
+ {ok, Pid} = start_service(
+ [
+ {bind_address, Address},
+ {document_root, AbsPath},
+ {server_root, AbsPath},
+ {directory_index, ["index.html"]},
+ {port, Port},
+ {mime_type, "application/octet-stream"},
+ {mime_types, default_mime_types()},
+ {modules, [mod_alias, mod_dir, mod_get]}
+ ] ++ IpFamilyOpts
+ ),
+ % This is needed to support random port assignment (--port 0)
+ [{port, ActualPort}] = info(Pid, [port]),
+ io:fwrite("~nStarted HTTP server on http://~s:~w at ~s~n",
+ [inet:ntoa(Address), ActualPort, AbsPath]),
+ receive
+ {From, shutdown} ->
+ ok = stop_service(Pid),
+ From ! done
+ end.
+
%%%--------------------------------------------------------------
%%% Internal functions
%%%--------------------------------------------------------------------
diff --git a/lib/inets/test/Makefile b/lib/inets/test/Makefile
index da0805bc4d..0f8ba516a0 100644
--- a/lib/inets/test/Makefile
+++ b/lib/inets/test/Makefile
@@ -129,6 +129,7 @@ MODULES = \
httpd_bench_SUITE \
http_test_lib \
httpd_basic_SUITE \
+ httpd_serve_SUITE \
httpd_load \
httpd_time_test \
httpd_1_1 \
@@ -164,7 +165,7 @@ INETS_FILES = inets.config $(INETS_SPECS)
INETS_DATADIRS = inets_SUITE_data inets_socketwrap_SUITE_data
-HTTPD_DATADIRS = httpd_test_data httpd_SUITE_data httpd_basic_SUITE_data httpd_bench_SUITE_data
+HTTPD_DATADIRS = httpd_test_data httpd_SUITE_data httpd_basic_SUITE_data httpd_bench_SUITE_data httpd_serve_SUITE_data
HTTPC_DATADIRS = httpc_SUITE_data httpc_proxy_SUITE_data
diff --git a/lib/inets/test/httpd_serve_SUITE.erl b/lib/inets/test/httpd_serve_SUITE.erl
new file mode 100644
index 0000000000..c49391c35c
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE.erl
@@ -0,0 +1,230 @@
+%% Tests for the `erl -S httpd serve` functionality.
+-module(httpd_serve_SUITE).
+-export([suite/0, all/0, groups/0]).
+-export([
+ argless_start/1,
+ argless_serve/1,
+ simple_random_port_serve/1,
+ serve_on_all_interfaces_v4/1,
+ serve_on_localhost_v4/1,
+ serve_on_all_interfaces_v6/1,
+ serve_on_localhost_v6/1,
+ serve_custom_directory/1
+]).
+
+%% When starting up servers for tests, these variables define how long to
+%% wait for the server to report that it has started up, and after how
+%% many retries to quit waiting for its report altogether.
+-define(STARTUP_WAIT_NAPTIME_MS, 20).
+-define(STARTUP_WAIT_RETRIES, 100).
+
+%% Default assertions to run in all tests.
+-define(DEFAULT_ASSERTIONS, [directory_index, random_file]).
+
+suite() ->
+ [{ct_hooks, [ts_install_cth]},
+ {timetrap, {seconds, 30}}].
+
+all() ->
+ [{group, httpd_serve_on_default_port},
+ {group, httpd_serve_on_random_ports}].
+
+groups() ->
+ [{httpd_serve_on_default_port, [sequence], [
+ argless_serve,
+ argless_start]},
+ {httpd_serve_on_random_ports, [parallel], [
+ simple_random_port_serve,
+ serve_on_all_interfaces_v4,
+ serve_on_localhost_v4,
+ serve_on_all_interfaces_v6,
+ serve_on_localhost_v6,
+ serve_custom_directory
+ ]}].
+
+%%
+%% Test cases
+%%
+
+%% Fixed ports (must be run one at a time)
+
+argless_start(_Config) ->
+ ServerFun = fun () -> httpd:start([]) end,
+ verify_server(ServerFun).
+
+argless_serve(_Config) ->
+ ServerFun = fun () -> httpd:serve([]) end,
+ verify_server(ServerFun).
+
+%% Random ports (can run in parallel)
+
+simple_random_port_serve(Config) ->
+ verify_server(["--port", "0", suite_data(Config)]).
+
+serve_on_all_interfaces_v4(Config) ->
+ verify_server(["--port", "0", "--bind", "0.0.0.0", suite_data(Config)]).
+
+serve_on_localhost_v4(Config) ->
+ verify_server(["--port", "0", "--bind", "127.0.0.1", suite_data(Config)]).
+
+serve_on_all_interfaces_v6(Config) ->
+ verify_server(["--port", "0", "--bind", "::", suite_data(Config)]).
+
+serve_on_localhost_v6(Config) ->
+ verify_server(["--port", "0", "--bind", "::1", suite_data(Config)]).
+
+serve_custom_directory(Config) ->
+ SuiteData = suite_data(Config),
+ verify_server(["--port", "0", filename:join(SuiteData, "subdir")]).
+
+%%
+%% Assertion functions
+%%
+
+%% Assert that the server responds properly.
+run_server_assertions(Response) ->
+ run_server_assertions(Response, ?DEFAULT_ASSERTIONS).
+
+%% Assert that the server responds properly.
+run_server_assertions({ok, {Ip, Port, Path}}, Assertions) when is_integer(Port) ->
+ % From the `filelib:wildcard/1` docs:
+ % "Directory separators must always be written as /, even on Windows."
+ IpToRequest = case Ip of
+ {_, _, _, _} -> inet:ntoa(Ip);
+ {_, _, _, _, _, _, _, _} -> "[" ++ inet:ntoa(Ip) ++ "]"
+ end,
+
+ ct:log("Validating custom assertions"),
+ DirectoryUrl = "http://" ++ IpToRequest ++ ":" ++ integer_to_list(Port) ++ "/",
+ ServerInfo = #{
+ url => DirectoryUrl,
+ bind_ip => Ip,
+ path => Path
+ },
+ ok = verify_assertions(Assertions, ServerInfo),
+ ct:comment("Ran ~w assertion(s).", [length(Assertions)]).
+
+%%
+%% Assertion helper functions
+%%
+
+verify_200_at(Url) ->
+ HttpcOpts = [{socket_opts, [{ipfamily, inet6fb4}]}],
+ Request = {Url, []},
+ Response = httpc:request(get, Request, [{autoredirect, false}], HttpcOpts),
+ case Response of
+ {ok, {{_Version, 200, _}, _Headers, _Body}} ->
+ Response;
+ {ok, {{_Version, 301, _}, Headers, _Body}} ->
+ % To be resilient against the case where the server is not
+ % reachable under its `server_name`, for instance, in test
+ % containers or other hosts with unexpected networking setups,
+ % replace the suggested hostname with the hostname we came from.
+ {_, SuggestedTarget} = proplists:lookup("location", Headers),
+ #{path := SuggestedPath} = uri_string:parse(SuggestedTarget),
+ OurUri = uri_string:parse(Url),
+ DecomposedTarget = maps:put(path, SuggestedPath, OurUri),
+ RedirectTarget = uri_string:recompose(DecomposedTarget),
+ RedirectedRequest = {RedirectTarget, []},
+ RedirectedResponse = httpc:request(get, RedirectedRequest, [], HttpcOpts),
+ ct:log("Following redirect (rewritten from ~s to ~s)", [SuggestedTarget, RedirectTarget]),
+ {ok, {{_, 200, _}, _, _}} = RedirectedResponse;
+ {error, {failed_connect, [{to_address, {"::", _Port}},
+ {inet6, [inet6], eaddrnotavail},
+ {inet, [inet], nxdomain}]}} ->
+ % In this case we could bind on all IPv6 interfaces (::) just
+ % fine, but could not issue requests to it due to an OS error.
+ % Write it off as a networking misconfiguration and skip.
+ exit({skip, "Could not reach host on IPv6 address ::"})
+ end.
+
+verify_assertions([], _ServerInfo) ->
+ ok;
+
+verify_assertions([directory_index | Assertions], #{url := Url} = ServerInfo) ->
+ ct:log("Validating directory index at ~s", [Url]),
+ verify_200_at(Url),
+ ct:log("Directory index received with a 200"),
+ verify_assertions(Assertions, ServerInfo);
+
+verify_assertions([random_file | Assertions], #{url := Url, path := Path} = ServerInfo) ->
+ Files = filelib:wildcard(Path ++ "/*"),
+ File = lists:nth(rand:uniform(length(Files)), Files),
+ Basename = filename:basename(File),
+ FileUrl = Url ++ Basename,
+ ct:log("Validating random file at ~s", [FileUrl]),
+ verify_200_at(FileUrl),
+ ct:log("File received with a 200"),
+ verify_assertions(Assertions, ServerInfo).
+
+%%
+%% Helper functions
+%%
+
+suite_data(Config) ->
+ proplists:get_value(data_dir, Config).
+
+verify_server(FunOrArgs) ->
+ TestFun = fun run_server_assertions/1,
+ with_server(FunOrArgs, TestFun).
+
+with_server(Args, TestFun) when is_list(Args) ->
+ ServerFun = fun () -> httpd:serve(Args) end,
+ run_with_server(ServerFun, TestFun);
+
+with_server(ServerFun, TestFun) when is_function(ServerFun) ->
+ run_with_server(ServerFun, TestFun).
+
+run_with_server(ServerFun, TestFun) ->
+ ct:log("Starting server"),
+ ct:capture_start(),
+ {Child, _Reference} = spawn_monitor(ServerFun),
+ StartupResult = wait_for_startup_line(?STARTUP_WAIT_RETRIES),
+ ct:capture_stop(),
+ {ok, Line} = StartupResult,
+ Parsed = parse_startup_line(Line),
+ ct:log("Running test function"),
+ Result = TestFun(Parsed),
+ ct:log("Test function finished, shutting down server"),
+ Child ! {self(), shutdown},
+ receive done -> ok after 5000 -> ct:fail("No server shutdown after 5s") end,
+ ct:log("Server stopped"),
+ Result.
+
+%% Wait for `ct:capture_get' to give us the output we're looking for.
+wait_for_startup_line(Tries) ->
+ wait_for_startup_line([], [], Tries).
+
+wait_for_startup_line([], [], 0) ->
+ {error, no_output_at_all};
+wait_for_startup_line([], Unexpected, 0) ->
+ {error, {no_startup_line, unexpected_output, Unexpected}};
+wait_for_startup_line([], Unexpected, Tries) when Tries > 0 ->
+ receive
+ {'DOWN', _Reference, process, _Child, Info} ->
+ case Info of
+ {{badmatch, {error, {listen, eaddrnotavail}}}, _} ->
+ exit({skip, "Adress not available to listen"});
+ _ ->
+ ct:fail("Child process has died: ~w", [Info])
+ end
+ after
+ 0 -> ok
+ end,
+ timer:sleep(?STARTUP_WAIT_NAPTIME_MS),
+ wait_for_startup_line(ct:capture_get(), Unexpected, Tries - 1);
+wait_for_startup_line(["\nStarted HTTP" ++ _Rest = Line | _Lines], _Unexpected, _Tries) ->
+ {ok, Line};
+wait_for_startup_line([Line | Lines], Unexpected, Tries) ->
+ wait_for_startup_line(Lines, [Line | Unexpected], Tries).
+
+%% Parse the given line into a tuple.
+%% Example line:
+%% Started HTTP server on http://127.0.0.1:8000 at /path/to/lib/inets/make_test_dir/ct_logs/ct_run.test_server@zulu.2023-06-06_12.07.27\n"
+parse_startup_line(Line) ->
+ {match, [_, RawIp, RawPort, Path]} = re:run(
+ Line, "^\nStarted HTTP server on http://(.+):(\\d+) at (.*)\\n$", [{capture, all, list}]
+ ),
+ {ok, Ip} = inet:parse_address(RawIp),
+ Port = list_to_integer(RawPort),
+ {ok, {Ip, Port, Path}}.
diff --git a/lib/inets/test/httpd_serve_SUITE_data/aaa/info.txt b/lib/inets/test/httpd_serve_SUITE_data/aaa/info.txt
new file mode 100644
index 0000000000..890527d5ba
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE_data/aaa/info.txt
@@ -0,0 +1 @@
+This directory serves to test proper redirects for subdirectories.
diff --git a/lib/inets/test/httpd_serve_SUITE_data/index.html b/lib/inets/test/httpd_serve_SUITE_data/index.html
new file mode 100644
index 0000000000..0741e1d445
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE_data/index.html
@@ -0,0 +1 @@
+<h1>httpd_serve_SUITE_data/index.html</h1>
diff --git a/lib/inets/test/httpd_serve_SUITE_data/sample_file.html b/lib/inets/test/httpd_serve_SUITE_data/sample_file.html
new file mode 100644
index 0000000000..5bbbe438b3
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE_data/sample_file.html
@@ -0,0 +1 @@
+<h1>httpd_serve_SUITE_data/sample_file.html</h1>
diff --git a/lib/inets/test/httpd_serve_SUITE_data/subdir/index.html b/lib/inets/test/httpd_serve_SUITE_data/subdir/index.html
new file mode 100644
index 0000000000..6add0208af
--- /dev/null
+++ b/lib/inets/test/httpd_serve_SUITE_data/subdir/index.html
@@ -0,0 +1 @@
+<h1>httpd_serve_SUITE_data/subdir/index.html</h1>
--
2.35.3
