File looking_glass-0.1.0-git.patch of Package looking_glass

Overview Repositories Revisions Requests Users Attributes Meta

File looking_glass-0.1.0-git.patch of Package looking_glass (Revision 0974709d8d90947c225591061a24f95f)

Currently displaying revision 0974709d8d90947c225591061a24f95f , Show latest

diff --git a/.gitignore b/.gitignore
index 83e9179..0c60974 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,7 @@
 c_src/env.mk
 c_src/*.o
 deps/
-ebin/
+ebin/*.beam
 logs/
 looking_glass.d
 priv/lg_tracer.so
diff --git a/Makefile b/Makefile
index 788b3c0..0fca5e1 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ PROJECT_DESCRIPTION = New project
 PROJECT_VERSION = 0.1.0
 
 DEPS = lz4
-dep_lz4 = git https://github.com/essen/lz4-erlang master
+dep_lz4 = git https://github.com/rabbitmq/lz4-erlang master
 
 BUILD_DEPS = nif_helpers
 dep_nif_helpers = git https://github.com/ninenines/nif_helpers master
@@ -14,7 +14,7 @@ LOCAL_DEPS = runtime_tools
 C_SRC_OUTPUT ?= $(CURDIR)/priv/lg_tracer
 
 TEST_DEPS = ct_helper
-dep_ct_helper = git https://github.com/extend/ct_helper master
+dep_ct_helper = git https://github.com/ninenines/ct_helper master
 
 include erlang.mk
 
diff --git a/README.asciidoc b/README.asciidoc
index deefe6b..19d4019 100644
--- a/README.asciidoc
+++ b/README.asciidoc
@@ -10,7 +10,7 @@ Erlang/OTP 19.0 or above.
 
 Looking Glass aims to provide a very efficient tool
 usable both in development and production settings,
-and capable of running for a very long amount of time
+and capable of running for a long amount of time
 even on busy systems.
 
 == Online documentation
@@ -25,7 +25,3 @@ even on busy systems.
 * Run `make install-docs` to install man pages on your system
 * Full documentation in Asciidoc available in `doc/src/`
 * Examples available in `examples/`
-
-== Getting help
-
-* https://github.com/rabbitmq/looking-glass/issues[Issues tracker]
diff --git a/c_src/lg_tracer.c b/c_src/lg_tracer.c
index acdd810..09cade2 100644
--- a/c_src/lg_tracer.c
+++ b/c_src/lg_tracer.c
@@ -77,6 +77,7 @@ NIF_ATOMS(NIF_ATOM_DECL)
     F(enabled_call, 3) \
     F(enabled_procs, 3) \
     F(enabled_running_procs, 3) \
+    F(enabled_send, 3) \
     F(trace, 5)
 
 NIF_FUNCTIONS(NIF_FUNCTION_H_DECL)
@@ -107,7 +108,7 @@ void unload(ErlNifEnv* env, void* priv_data)
 
 NIF_FUNCTION(enabled)
 {
-    ERL_NIF_TERM tracers, head;
+    ERL_NIF_TERM tracers, value;
     ErlNifPid tracer;
 
     // This function will only be called for trace_status.
@@ -117,27 +118,22 @@ NIF_FUNCTION(enabled)
     if (!enif_get_map_value(env, argv[1], atom_tracers, &tracers))
         return atom_remove;
 
-    // Disable the trace when the tracers option is not a list.
-    if (!enif_is_list(env, tracers))
-        return atom_remove;
-
-    while (enif_get_list_cell(env, tracers, &head, &tracers)) {
-        // Don't generate trace events for tracers.
-        if (enif_is_identical(argv[2], head))
-            return atom_discard;
+    // Because the tracers supervisor is a one_for_all, we only need
+    // to check one of the tracer processes to confirm all are alive.
 
-        // Disable the trace when one of the tracers is not a local process.
-        if (!enif_get_local_pid(env, head, &tracer))
-            return atom_remove;
+    // We know for a fact that this key exists because
+    // there's at least one tracer process.
+    enif_get_map_value(env, tracers, enif_make_int(env, 0), &value);
 
-        // Disable the trace when one of the tracers is not alive.
-        if (!enif_is_process_alive(env, &tracer))
-            return atom_remove;
-    }
+    // Disable the trace when one of the tracers is not a local process.
+    if (!enif_get_local_pid(env, value, &tracer))
+        return atom_remove;
 
-    // @todo Discard trace events for a percent of processes.
+    // Disable the trace when one of the tracers is not alive.
+    if (!enif_is_process_alive(env, &tracer))
+        return atom_remove;
 
-    return atom_trace;
+    return atom_discard;
 }
 
 NIF_FUNCTION(enabled_call)
@@ -150,14 +146,17 @@ NIF_FUNCTION(enabled_procs)
 {
     ERL_NIF_TERM mode;
 
-    // We only want the exit event when 'profile' mode is enabled.
+    // We only want the spawn and exit events when 'profile' mode
+    // is enabled. Technically we only care about exits for callgrind,
+    // but spawn is cheap to keep and useful for message profilers.
     if (enif_get_map_value(env, argv[1], atom_mode, &mode)
         && enif_is_identical(atom_profile, mode)
-        && enif_is_identical(atom_exit, argv[0])) {
-        return atom_trace;
+        && !(enif_is_identical(atom_spawn, argv[0])
+            || enif_is_identical(atom_exit, argv[0]))) {
+        return atom_discard;
     }
 
-    return atom_discard;
+    return atom_trace;
 }
 
 NIF_FUNCTION(enabled_running_procs)
@@ -166,19 +165,28 @@ NIF_FUNCTION(enabled_running_procs)
     return atom_trace;
 }
 
+NIF_FUNCTION(enabled_send)
+{
+    // We always want both send and send_to_non_existing_process.
+    return atom_trace;
+}
+
 // trace(TraceTag, TracerState, Tracee, TraceTerm, Opts)
 
 NIF_FUNCTION(trace)
 {
-    ERL_NIF_TERM tracers, head, ts, extra, msg;
+    ERL_NIF_TERM tracers, head, ts, extra, mspec, msg;
     ErlNifPid tracer;
-    unsigned int len, nth, i;
+    unsigned int nth;
+    size_t len;
+    int has_extra, has_mspec;
 
     if (!enif_get_map_value(env, argv[1], atom_tracers, &tracers))
         return atom_ok;
 
-    if (!enif_get_list_length(env, tracers, &len))
-        return atom_ok;
+    // We know for a fact that the argument is a map. And if not,
+    // no problem because we will return when trying to get a value from it.
+    enif_get_map_size(env, tracers, &len);
 
 #if (ERL_NIF_MAJOR_VERSION >= 2) && (ERL_NIF_MINOR_VERSION >= 12)
     nth = enif_hash(ERL_NIF_INTERNAL_HASH, argv[2], 0) % len;
@@ -199,27 +207,38 @@ NIF_FUNCTION(trace)
     nth = (tracee.pid >> 4) % len;
 #endif
 
-    for (i = 0; i <= nth; i++) {
-        if (!enif_get_list_cell(env, tracers, &head, &tracers))
-            return atom_ok;
-    }
+    if (!enif_get_map_value(env, tracers, enif_make_int(env, nth), &head))
+        return atom_ok;
 
     if (!enif_get_local_pid(env, head, &tracer))
         return atom_ok;
 
     // Everything good. Generate a timestamp to include in the message.
 
-    ts = enif_make_int64(env,
-        enif_monotonic_time(ERL_NIF_USEC) + enif_time_offset(ERL_NIF_USEC));
+    ts = enif_make_int64(env, enif_monotonic_time(ERL_NIF_USEC));
 
     // Build the message. There can be two different messages
     // depending on whether the extra option was set:
     //
     // - {Tag, Tracee, Ts, Term}
     // - {Tag, Tracee, Ts, Term, Extra}
+    //
+    // On top of that when match specs are enabled we may have
+    // one additional term at the end of the tuple containing
+    // the result of the match spec function.
+    //
+    // - {Tag, Tracee, Ts, Term, Result}
+    // - {Tag, Tracee, Ts, Term, Extra, Result}
+
+    has_extra = enif_get_map_value(env, argv[4], atom_extra, &extra);
+    has_mspec = enif_get_map_value(env, argv[4], atom_match_spec_result, &mspec);
 
-    if (enif_get_map_value(env, argv[4], atom_extra, &extra))
+    if (has_extra && has_mspec)
+        msg = enif_make_tuple6(env, argv[0], argv[2], ts, argv[3], extra, mspec);
+    else if (has_extra)
         msg = enif_make_tuple5(env, argv[0], argv[2], ts, argv[3], extra);
+    else if (has_mspec)
+        msg = enif_make_tuple5(env, argv[0], argv[2], ts, argv[3], mspec);
     else
         msg = enif_make_tuple4(env, argv[0], argv[2], ts, argv[3]);
 
diff --git a/doc/src/guide/book.asciidoc b/doc/src/guide/book.asciidoc
index 8cdde72..36ce0ce 100644
--- a/doc/src/guide/book.asciidoc
+++ b/doc/src/guide/book.asciidoc
@@ -5,3 +5,7 @@ include::introduction.asciidoc[Introduction]
 include::tracing.asciidoc[Tracing]
 
 include::callgrind.asciidoc[Callgrind profiling]
+
+include::flame.asciidoc[Flame graph profiling]
+
+include::messages.asciidoc[Messages profiling]
diff --git a/doc/src/guide/callgrind.asciidoc b/doc/src/guide/callgrind.asciidoc
index a1e91d5..c34444a 100644
--- a/doc/src/guide/callgrind.asciidoc
+++ b/doc/src/guide/callgrind.asciidoc
@@ -114,7 +114,7 @@ using the `scope` option:
 [source,erlang]
 ----
 1> lg_callgrind:profile_many("traces.lz4.*", "callgrind.out",
-    #{running => true}).
+    #{scope => per_process}).
 ----
 
 === Using the cachegrind tool
diff --git a/doc/src/guide/flame.asciidoc b/doc/src/guide/flame.asciidoc
new file mode 100644
index 0000000..7bb976d
--- /dev/null
+++ b/doc/src/guide/flame.asciidoc
@@ -0,0 +1,82 @@
+[[flame]]
+== Flame graph profiling
+
+As an alternative to xref:callgrind[Callgrind output],
+Looking Glass provides flame graphs. Flame graphs are
+a graphical view of stack traces that make it obvious
+where the most time is spent. It complements the other
+graphical views provided by `qcachegrind`.
+
+Looking Glass only takes care of providing an output
+that can then be converted into a flame graph using
+the usual tool (not included). This chapter will
+explain both operations.
+
+=== Required trace options
+
+In order to generate a flame graph we currently need to
+use one additional option when tracing. This option will
+result in adding stack trace information to call events.
+The option is `process_dump` and it must be set to `true`.
+
+To give an example, instead of this:
+
+[source,erlang]
+----
+1> lg:trace('_', lg_file_tracer, "traces.lz4").
+----
+
+Do this:
+
+[source,erlang]
+----
+1> lg:trace('_', lg_file_tracer, "traces.lz4",
+    #{process_dump => true}).
+----
+
+=== Profiling one file
+
+The `lg_flame` module provides a similar interface as other
+Looking Glass profilers. You can produce an intermediate
+output based on one or many files.
+
+To profile one file:
+
+[source,erlang]
+----
+1> lg_flame:profile("traces.lz4.1", "output").
+----
+
+This will create an intermediate file named 'output'.
+
+=== Profiling many files
+
+To profile many files:
+
+[source,erlang]
+----
+1> lg_flame:profile_many("traces.lz4.*", "output").
+----
+
+Note that the output is always a single file as the
+results are merged together.
+
+=== Building the flame graph
+
+https://github.com/brendangregg/FlameGraph[flamegraph.pl]
+can be used to produce actual SVG flame graphs.
+
+First we need to clone it. Anywhere will do:
+
+[source,bash]
+$ git clone https://github.com/brendangregg/FlameGraph
+
+Then we can use it on our output file to create an SVG:
+
+[source,bash]
+$ ./FlameGraph/flamegraph.pl output > output.svg
+
+You can then open the output SVG in your Web browser
+of choice. The produced SVG is interactive, you can
+click on the different functions to zoom in, and you
+can also search for a specific function call.
diff --git a/doc/src/guide/introduction.asciidoc b/doc/src/guide/introduction.asciidoc
index a25afdd..d00262f 100644
--- a/doc/src/guide/introduction.asciidoc
+++ b/doc/src/guide/introduction.asciidoc
@@ -25,9 +25,17 @@ known as `kcachegrind`) is recommended. Note that
 it is a good idea to also install `graphviz` to
 have the quite informative call graphs.
 
+=== Requirements
+
+Looking Glass should work out of the box on any system
+with the standard build tools installed.
+
 === License
 
-TODO
+Looking Glass is double-licensed under the Mozilla
+Public License 1.1 and the Apache License version 2.
+
+See the LICENSE file for more information.
 
 === Versioning
 
diff --git a/doc/src/guide/messages.asciidoc b/doc/src/guide/messages.asciidoc
new file mode 100644
index 0000000..34db683
--- /dev/null
+++ b/doc/src/guide/messages.asciidoc
@@ -0,0 +1,254 @@
+[[messages]]
+== Messages profiling
+
+Looking Glass can also be used to profile Erlang processes
+based on the messages they send. It can help you detect
+which processes are the most busy and is able to generate
+graphs and sequence diagrams to help you debug complex
+issues.
+
+=== Enabling the tracing of messages
+
+By default Looking Glass will not include the messages
+in the trace files. It needs to be enabled through the
+xref:tracing_send[send option].
+
+The output from one tracing session can then be used
+for both callgrind and message profiling.
+
+=== Profiling one file
+
+You can profile one file by calling the function
+`lg_messages:profile/1`. It takes the trace file name
+and prints out the result of the profiling.
+
+[source,erlang]
+----
+1> lg_messages:profile("traces.lz4.1").
+----
+
+It will also create a GraphViz file currently hardcoded as
+'digraph.gv' and print further instructions to use it.
+
+=== Profiling many files
+
+A convenience function is available for profiling many
+files at once: `lg_callgrind:profile_many/2,3`. It takes
+a wildcard pattern as first argument and a file name
+prefix as second argument:
+
+You can profile many files by calling the function
+`lg_messages:profile_many/1`. It takes a wildcard pattern
+and prints out the result of the profiling. The result
+is a merge of the events in the different trace files.
+
+[source,erlang]
+----
+1> lg_messages:profile_many("traces.lz4.*").
+----
+
+=== Profile output
+
+The profile step will result in four tables being printed.
+
+* The first table shows the processes that sent the most messages.
+
+* The second table shows the processes that sent the most messages
+  to processes that are either dead or never existed in the first place.
+
+* The third table shows the processes that were most frequently sending
+  messages to one specific other process (from Alice to Bob).
+
+* The last table shows the processes that were exchanging the most
+  messages (from Alice to Bob, and from Bob to Alice).
+
+.Example output
+----
+1> lg_messages:profile_many("traces.lz4.*").
+
+They sent the most messages
+===========================
+
+Process ID      Count      Most recent message
+----------      -----      -------------------
+<7782.367.0>    147327     {notify,{event,channel_closed,...}}
+<7782.356.0>    73035      {notify,{event,connection_closed,...}}
+<7782.382.0>    30514      pause
+<7782.391.0>    30052      {'$gen_cast',{deliver,{...},...}}
+<7782.365.0>    1486       {channel_exit,1,{writer,...}}
+[...]
+
+They sent the most messages to dead processes
+=============================================
+
+Process ID      Count      Most recent message
+----------      -----      -------------------
+<7782.367.0>    29         {notify,{event,channel_closed,...}}
+
+They sent the most messages to one other process
+================================================
+
+From pid        To pid          Count      Most recent message
+--------        ------          -----      -------------------
+<7782.367.0>    <7782.365.0>    74318      {notify,{event,channel_closed,...}}
+<7782.356.0>    <7782.367.0>    73001      {notify,{event,connection_closed,...}}
+<7782.367.0>    <7782.375.0>    73000      {notify,{event,channel_closed,...}}
+<7782.382.0>    <7782.391.0>    30202      pause
+<7782.391.0>    <7782.375.0>    29894      {'$gen_cast',{deliver,{...},...}}
+<7782.365.0>    <7782.375.0>    1485       {channel_exit,1,{writer,...}}
+[...]
+
+They sent the most messages to each other
+=========================================
+
+Count      Pid 1           Most recent message
+           Pid 2           from the corresponding process
+-----      -----           ------------------------------
+74318      <7782.365.0>    {channel_exit,1,{writer,...}}
+           <7782.367.0>    {notify,{event,channel_closed,...}}
+73001      <7782.356.0>    {notify,{event,connection_closed,...}}
+           <7782.367.0>    {notify,{event,channel_closed,...}}
+73000      <7782.367.0>    {notify,{event,channel_closed,...}}
+           <7782.375.0>    '<none>'
+30351      <7782.382.0>    pause
+           <7782.391.0>    {'$gen_cast',{deliver,{...},...}}
+29894      <7782.375.0>    '<none>'
+           <7782.391.0>    {'$gen_cast',{deliver,{...},...}}
+[...]
+
+The file digraph.gv was created. Use GraphViz to make a PNG.
+$ dot -Tpng -O digraph.gv
+
+You can also edit the file to remove uninteresting processes.
+One line in the file is equal to a connection between two processes.
+----
+
+At the end of the output, instructions are given to generate an
+image from a GraphViz file. This image shows the relationships
+between the processes and indicates how many messages they send
+to each other.
+
+The file generated by Looking Glass is a text file that can be
+further edited as necessary. It looks like this:
+
+----
+digraph {
+    concentrate=true;
+    splines=ortho;
+    edge [arrowhead=none, labelfontsize=12.0, minlen=3];
+
+    "error_logger" -> "<7782.354.0>" [taillabel=0, headlabel=2];
+    "<7782.32.0>" -> "<7782.380.0>" [taillabel=0, headlabel=1];
+    "<7782.388.0>" -> "<7782.391.0>" [taillabel=0, headlabel=1];
+    "error_logger" -> "<7782.355.0>" [taillabel=0, headlabel=4];
+[...]
+}
+----
+
+It is of course possible to edit this file. You may want to
+modify the style attributes, or even remove processes from
+the output entirely.
+
+=== Generating sequence diagrams
+
+Looking Glass can also be used to extract the sequence of messages
+exchanged between two or more processes. This is done using the
+`lg_messages_seqdiag` module, which works just like `lg_messages`
+ecept the functions take a second argument containing the list of
+pids you wish to investigate.
+
+To look at one file:
+
+[source,erlang]
+----
+1> lg_messages_seqdiag:profile("traces.lz4.1",
+    ["<7788.381.0>", "<7788.382.0>", "<7774.383.0>",
+     "<7774.384.0>", "<7774.386.0>"]).
+----
+
+And many files:
+
+[source,erlang]
+----
+1> lg_messages_seqdiag:profile_many("traces.lz4.*",
+    ["<7788.381.0>", "<7788.382.0>", "<7774.383.0>",
+     "<7774.384.0>", "<7774.386.0>"]).
+----
+
+The list of pids must be given as a list of strings. This is
+because the processes represented do not exist on the running
+system. Looking Glass will ignore the node information from the
+pid too, so you do not need to worry about it. This explains why
+the pids requested in the previous two snippets look as if they
+come from different nodes. The pids `"<7888.381.0>"` and
+`"<7774.381.0>"` are therefore equivalent.
+
+After running one of these commands, you will end up with a
+file 'seq.diag' that can then be used to create an image. This
+file can also be edited later on if necessary. It looks like this:
+
+----
+seqdiag {
+    edge_length = 300;
+    activation = none;
+
+    "<7774.382.0>" -> "<7774.381.0>" [label="gen:call #1 {start_child,{collector,{rabbit_queue_collector,start_link,[...]},intrinsic,30000,worker,...}}"];
+    "<7774.383.0>" -> "<7774.381.0>" [label="{ack,<7774.383.0>,{ok,<7774.383.0>}}"];
+    "<7774.381.0>" -> "<7774.382.0>" [label="#1 {ok,<7774.383.0>}"];
+[...]
+}
+----
+
+Before you can create an image from it, you will need to install
+`seqdiag`. Installation instructions will depend on your system.
+The project page is at http://blockdiag.com/en/seqdiag/
+
+.Example output
+image::seq.png[]
+
+=== Identifying processes
+
+While Looking Glass will display the pid and one sample message
+from each process, it's not always ideal to identify which process
+is which.
+
+To allievate that, Looking Glass offers a simple solution:
+sending a message to the named process `lg` while a tracer is
+running. Looking Glass will inevitably log this message in the
+trace file, recognize that the target is `lg` and use the
+message as metadata. This metadata is then available to any
+module reading from the trace file.
+
+The process is only available when Looking Glass is running,
+of course, which means we can't just send a message directly.
+The following works:
+
+[source,erlang]
+----
+is_pid(whereis(lg)) andalso (lg ! Info).
+----
+
+This can be made into a macro, of course:
+
+[source,erlang]
+----
+%% Store metadata in the trace files when message tracing is enabled.
+-define(LG_INFO(Info), is_pid(whereis(lg)) andalso (lg ! Info)).
+----
+
+And can then be used like this:
+
+[source,erlang]
+----
+?LG_INFO(#{process_type => reader}).
+----
+
+The message must always be a map. Reading the trace file
+will otherwise fail. Looking Glass only recognizes the
+`process_type` field, and uses it as a label to identify
+processes when profiling exchanges of messages. You are
+free to define any other value you need in the map.
+
+The metadata can also be updated by sending another message
+or by calling the macro a second time. The operation done
+on the map will be a merge by default.
diff --git a/doc/src/guide/seq.png b/doc/src/guide/seq.png
new file mode 100644
index 0000000..3661c2f
Binary files /dev/null and b/doc/src/guide/seq.png differ
diff --git a/doc/src/guide/tracing.asciidoc b/doc/src/guide/tracing.asciidoc
index f19b15c..ec9a4fe 100644
--- a/doc/src/guide/tracing.asciidoc
+++ b/doc/src/guide/tracing.asciidoc
@@ -321,3 +321,19 @@ of resources and so isn't enabled by default:
 1> lg:trace('_', lg_file_tracer, "traces.lz4",
     #{mode => profile, running => true}).
 ----
+
+[[tracing_send]]
+You may want to also trace the messages sent by the processes.
+To do so you need to enable the `send` option. You can then
+xref:messages[obtain detailed information about the processes
+sending messages]. To enable the tracing of messages:
+
+[source,erlang]
+----
+1> lg:trace('_', lg_file_tracer, "traces.lz4",
+    #{send => true}).
+----
+
+All the options in this section can be combined at will. It is
+possible to use the data from the same tracing session when
+profiling both functions and messages.
diff --git a/ebin/looking_glass.app b/ebin/looking_glass.app
new file mode 100644
index 0000000..657a178
--- /dev/null
+++ b/ebin/looking_glass.app
@@ -0,0 +1,9 @@
+{application, 'looking_glass', [
+	{description, "New project"},
+	{vsn, "0.1.0"},
+	{modules, ['lg','lg_callgrind','lg_file_reader','lg_file_tracer','lg_flame','lg_messages','lg_messages_seqdiag','lg_rabbit_hole','lg_raw_console_tracer','lg_socket_client','lg_socket_tracer','lg_tracer','lg_tracer_pool','looking_glass_app','looking_glass_sup']},
+	{registered, [looking_glass_sup]},
+	{applications, [kernel,stdlib,runtime_tools,lz4]},
+	{mod, {looking_glass_app, []}},
+	{env, []}
+]}.
\ No newline at end of file
diff --git a/rebar.config b/rebar.config
new file mode 100644
index 0000000..48cb621
--- /dev/null
+++ b/rebar.config
@@ -0,0 +1,6 @@
+{pre_hooks, [
+    {"(linux|darwin|solaris|win32)", clean, "make clean"},
+    {"(freebsd|netbsd|openbsd)", clean, "gmake clean"},
+    {"(linux|darwin|solaris|win32)", compile, "make"},
+    {"(freebsd|netbsd|openbsd)", compile, "gmake"}
+]}.
diff --git a/src/lg.erl b/src/lg.erl
index 2eddcbf..4552c22 100644
--- a/src/lg.erl
+++ b/src/lg.erl
@@ -39,6 +39,7 @@
     mode => trace | profile,
     pool_id => any(),
     pool_size => pos_integer(),
+    send => boolean(),
     running => boolean()
 }.
 
@@ -66,6 +67,7 @@ do_trace(Input0, TracerMod, TracerOpts, Opts) ->
     %% Start the pool of tracer processes.
     PoolID = maps:get(pool_id, Opts, default),
     PoolSize = maps:get(pool_size, Opts, erlang:system_info(schedulers)),
+    true = PoolSize > 0,
     {ok, PoolPid} = supervisor:start_child(looking_glass_sup, #{
         id => PoolID,
         start => {lg_tracer_pool, start_link, [PoolSize, TracerMod, TracerOpts]},
@@ -73,12 +75,12 @@ do_trace(Input0, TracerMod, TracerOpts, Opts) ->
         type => supervisor
     }),
     Tracers = lg_tracer_pool:tracers(PoolPid),
+    TracersMap = maps:from_list(lists:zip(lists:seq(0, length(Tracers) - 1), Tracers)),
     Mode = maps:get(mode, Opts, trace),
-    Running = maps:get(running, Opts, false),
     Input1 = flatten(Input0, []),
     Input2 = ensure_pattern(Input1),
     Input = ensure_scope(Input2),
-    trace_input(Input, #{mode => Mode, tracers => Tracers}, Running),
+    trace_input(Input, #{mode => Mode, tracers => TracersMap}, Opts),
     ok.
 
 flatten([], Acc) ->
@@ -107,7 +109,7 @@ ensure_scope(Input) ->
 
 trace_input([], _, _) ->
     ok;
-trace_input([{scope, Scope}|Tail], Opts, Running) ->
+trace_input([{scope, Scope}|Tail], TracerState, Opts) ->
     %% We currently enable the following trace flags:
     %% - call: function calls
     %% - procs: process exit events; plus others we ignore
@@ -119,18 +121,24 @@ trace_input([{scope, Scope}|Tail], Opts, Running) ->
     %%
     %% @todo It might be useful to count the number of sends
     %% or receives a function does.
+    ExtraFlags = [running || maps:get(running, Opts, false)]
+        ++ [send || maps:get(send, Opts, false)],
     _ = [erlang:trace(PidPortSpec, true, [
             call, procs, timestamp, arity, return_to, set_on_spawn,
-            {tracer, lg_tracer, Opts}
-            |[running || Running]
+            {tracer, lg_tracer, TracerState}
+            |ExtraFlags
         ])
     || PidPortSpec <- Scope],
-    trace_input(Tail, Opts, Running);
-trace_input([Mod|Tail], Opts, Running) when is_atom(Mod) ->
+    trace_input(Tail, TracerState, Opts);
+trace_input([Mod|Tail], TracerState, Opts) when is_atom(Mod) ->
+    MatchSpec = case Opts of
+        #{process_dump := true} -> [{'_', [], [{message, {process_dump}}]}];
+        _ -> true
+    end,
     %% The module must be loaded before we attempt to trace it.
     _ = code:ensure_loaded(Mod),
-    _ = erlang:trace_pattern({Mod, '_', '_'}, true, [local]),
-    trace_input(Tail, Opts, Running).
+    _ = erlang:trace_pattern({Mod, '_', '_'}, MatchSpec, [local]),
+    trace_input(Tail, TracerState, Opts).
 
 stop() ->
     stop(default).
diff --git a/src/lg_callgrind.erl b/src/lg_callgrind.erl
index 814abe9..5d90357 100644
--- a/src/lg_callgrind.erl
+++ b/src/lg_callgrind.erl
@@ -108,13 +108,26 @@ profile_many(Wildcard, Prefix, Opts) ->
     Seq = lists:seq(1, length(Files)),
     OutFiles = [Prefix ++ "." ++ integer_to_list(N) || N <- Seq],
     Many = lists:zip(Files, OutFiles),
-    %% @todo Do it in parallel.
-    _ = [profile(Input, Output, Opts) || {Input, Output} <- Many],
-    ok.
+    Refs = [monitor(process, spawn_link(?MODULE, profile, [Input, Output, Opts]))
+        || {Input, Output} <- Many],
+    wait_for_procs(Refs).
+
+%% We do not need to worry about failure because we are linked.
+wait_for_procs([]) ->
+    ok;
+wait_for_procs(Refs) ->
+    receive
+        %% We purposefully ignore any stray messages.
+        {'DOWN', R, process, _, _} ->
+            wait_for_procs(Refs -- [R])
+    end.
 
 %% We handle trace events one by one, keeping track of the
 %% execution stack for each process.
 
+%% We don't care about match spec results for callgrind.
+handle_event({call, Pid, Ts, MFA, _MSpec}, State) ->
+    handle_event({call, Pid, Ts, MFA}, State);
 handle_event({call, Pid, Ts, MFA}, State0) ->
     Proc = case is_process_profiled(Pid, State0) of
         {true, P} -> P;
diff --git a/src/lg_flame.erl b/src/lg_flame.erl
new file mode 100644
index 0000000..74d7570
--- /dev/null
+++ b/src/lg_flame.erl
@@ -0,0 +1,288 @@
+%% Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+%%
+%% This package, Looking Glass, is double-licensed under the Mozilla
+%% Public License 1.1 ("MPL") and the Apache License version 2
+%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+%% please see LICENSE-APACHE2.
+%%
+%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+%% either express or implied. See the LICENSE file for specific language governing
+%% rights and limitations of this software.
+%%
+%% If you have any questions regarding licensing, please contact us at
+%% info@rabbitmq.com.
+
+-module(lg_flame).
+
+-export([profile/2]).
+-export([profile_many/2]).
+
+-record(state, {
+          output_path="",
+          pid,
+          last_ts,
+          count=0,
+          acc=[]}). % per-process state
+
+-spec profile(file:filename_all(), file:filename_all()) -> ok.
+profile(Input, Output) ->
+    InitialState = exp1_init(Output),
+    {ok, FinalState} = lg_file_reader:fold(fun handle_event/2, InitialState, Input),
+    flush(FinalState).
+
+-spec profile_many(file:filename(), file:filename()) -> ok.
+profile_many(Wildcard, Output) ->
+    InitialState = exp1_init(Output),
+    Files = filelib:wildcard(Wildcard),
+    FinalState = lists:foldl(fun(Input, State0) ->
+        case lg_file_reader:fold(fun handle_event/2, State0, Input) of
+            {ok, State} ->
+                State;
+            {error, Reason, HumanReadable} ->
+                io:format("Error ~p while reading ~s:~n~s~n",
+                    [Reason, Input, HumanReadable]),
+                State0
+        end
+    end, InitialState, Files),
+    flush(FinalState).
+
+flush(#state{output_path=OutputPath}) ->
+    PidStates = get(),
+    {ok, FH} = file:open(OutputPath, [write, raw, binary, delayed_write]),
+    io:format("\n\nWriting to ~s for ~w processes... ", [OutputPath, length(PidStates)]),
+    [
+        [begin
+             Pid_str0 = lists:flatten(io_lib:format("~w", [Pid])),
+             Size = length(Pid_str0),
+             Pid_str = [$(, lists:sublist(Pid_str0, 2, Size-2), $)],
+             Time_str = integer_to_list(Time),
+             file:write(FH, [Pid_str, $;, intersperse($;, lists:reverse(Stack)), 32, Time_str, 10])
+         end || {Stack, Time} <- Acc]
+     || {Pid, #state{acc=Acc} = _S} <- PidStates],
+    file:close(FH),
+    io:format("finished!\n"),
+    ok.
+
+handle_event({Type, Pid, Ts, Arg}, State) ->
+    exp1({trace_ts, Pid, Type, Arg, Ts}, State);
+handle_event({Type, Pid, Ts, Arg, ExtraOrMspec}, State) ->
+    exp1({trace_ts, Pid, Type, Arg, ExtraOrMspec, Ts}, State);
+handle_event({Type, Pid, Ts, Arg, Extra, Mspec}, State) ->
+    exp1({trace_ts, Pid, Type, Arg, Extra, Mspec, Ts}, State).
+
+%% Below is Scott L. Fritchie's ISC licensed work with only a handful changes.
+
+exp1_init(OutputPath) ->
+    #state{output_path=OutputPath}.
+
+exp1(T, #state{output_path=OutputPath} = S) ->
+    trace_ts = element(1, T),
+    Pid = element(2, T),
+    PidState = case erlang:get(Pid) of
+                   undefined ->
+                       io:format("~p ", [Pid]),
+                       #state{output_path=OutputPath};
+                   SomeState ->
+                       SomeState
+               end,
+    NewPidState = exp1_inner(T, PidState),
+    erlang:put(Pid, NewPidState),
+    S.
+
+exp1_inner({trace_ts, _Pid, InOut, _MFA, _TS}, #state{last_ts=undefined} = S)
+  when InOut == in; InOut == out ->
+    %% in & out, without call context, don't help us
+    S;
+exp1_inner({trace_ts, _Pid, Return, _MFA, _TS}, #state{last_ts=undefined} = S)
+  when Return == return_from; Return == return_to ->
+    %% return_from and return_to, without call context, don't help us
+    S;
+exp1_inner({trace_ts, Pid, call, MFA, BIN, TS},
+     #state{last_ts=LastTS, acc=Acc, count=Count} = S) ->
+  try
+    %% Calculate time elapsed, TS-LastTs.
+    %% 0. If Acc is empty, then skip step #1.
+    %% 1. Credit elapsed time to the stack on the top of Acc.
+    %% 2. Push a 0 usec item with this stack onto Acc.
+    Stak = lists:filter(fun(<<"unknown function">>) -> false;
+                           (_)                      -> true
+                        end, stak_binify(BIN)),
+    Stack0 = stak_trim(Stak),
+    MFA_bin = mfa_binify(MFA),
+    Stack1 = [MFA_bin|lists:reverse(Stack0)],
+    Acc2 = case Acc of
+               [] ->
+                   [{Stack1, 0}];
+               [{LastStack, LastTime}|Tail] ->
+                   USec = TS - LastTS,
+%                   io:format("Stack1: ~p ~p\n", [Stack1, USec]),
+                   [{Stack1, 0},
+                    {LastStack, LastTime + USec}|Tail]
+           end,
+    %% TODO: more state tracking here.
+    S#state{pid=Pid, last_ts=TS, count=Count+1, acc=Acc2}
+  catch XX:YY ->
+            io:format(user, "~p: ~p:~p @ ~p\n", [?LINE, XX, YY, erlang:get_stacktrace()]),
+            S
+  end;
+exp1_inner({trace_ts, _Pid, return_to, MFA, TS}, #state{last_ts=LastTS, acc=Acc} = S) ->
+  try
+    %% Calculate time elapsed, TS-LastTs.
+    %% 1. Credit elapsed time to the stack on the top of Acc.
+    %% 2. Push a 0 usec item with the "best" stack onto Acc.
+    %%    "best" = MFA exists in the middle of the stack onto Acc,
+    %%    or else MFA exists at the top of a stack elsewhere in Acc.
+    [{LastStack, LastTime}|Tail] = Acc,
+    MFA_bin = mfa_binify(MFA),
+    BestStack = lists:dropwhile(fun(SomeMFA) when SomeMFA /= MFA_bin -> true;
+                                   (_)                               -> false
+                                end, find_matching_stack(MFA_bin, Acc)),
+    USec = TS - LastTS,
+    Acc2 = [{BestStack, 0},
+            {LastStack, LastTime + USec}|Tail],
+%    io:format(user, "return-to: ~p\n", [lists:sublist(Acc2, 4)]),
+    S#state{last_ts=TS, acc=Acc2}
+  catch XX:YY ->
+            io:format(user, "~p: ~p:~p @ ~p\n", [?LINE, XX, YY, erlang:get_stacktrace()]),
+            S
+  end;
+    
+exp1_inner({trace_ts, _Pid, gc_start, _Info, TS}, #state{last_ts=LastTS, acc=Acc} = S) ->
+  try
+    %% Push a 0 usec item onto Acc.
+    [{LastStack, LastTime}|Tail] = Acc,
+    NewStack = [<<"GARBAGE-COLLECTION">>|LastStack],
+    USec = TS - LastTS,
+    Acc2 = [{NewStack, 0},
+            {LastStack, LastTime + USec}|Tail],
+%    io:format(user, "GC 1: ~p\n", [lists:sublist(Acc2, 4)]),
+    S#state{last_ts=TS, acc=Acc2}
+  catch _XX:_YY ->
+            %% io:format(user, "~p: ~p:~p @ ~p\n", [?LINE, _XX, _YY, erlang:get_stacktrace()]),
+            S
+  end;
+exp1_inner({trace_ts, _Pid, gc_end, _Info, TS}, #state{last_ts=LastTS, acc=Acc} = S) ->
+  try
+    %% Push the GC time onto Acc, then push 0 usec item from last exec
+    %% stack onto Acc.
+    [{GCStack, GCTime},{LastExecStack,_}|Tail] = Acc,
+    USec = TS - LastTS,
+    Acc2 = [{LastExecStack, 0}, {GCStack, GCTime + USec}|Tail],
+%    io:format(user, "GC 2: ~p\n", [lists:sublist(Acc2, 4)]),
+    S#state{last_ts=TS, acc=Acc2}
+  catch _XX:_YY ->
+            %% io:format(user, "~p: ~p:~p @ ~p\n", [?LINE, _XX, _YY, erlang:get_stacktrace()]),
+            S
+  end;
+
+exp1_inner({trace_ts, _Pid, out, MFA, TS}, #state{last_ts=LastTS, acc=Acc} = S) ->
+  try
+    %% Push a 0 usec item onto Acc.
+    %% The MFA reported here probably doesn't appear in the stacktrace
+    %% given to us by the last 'call', so add it here.
+    [{LastStack, LastTime}|Tail] = Acc,
+    MFA_bin = mfa_binify(MFA),
+    NewStack = [<<"SLEEP">>,MFA_bin|LastStack],
+    USec = TS - LastTS,
+    Acc2 = [{NewStack, 0},
+            {LastStack, LastTime + USec}|Tail],
+    S#state{last_ts=TS, acc=Acc2}
+  catch XX:YY ->
+            io:format(user, "~p: ~p:~p @ ~p\n", [?LINE, XX, YY, erlang:get_stacktrace()]),
+            S
+  end;
+exp1_inner({trace_ts, _Pid, in, MFA, TS}, #state{last_ts=LastTS, acc=Acc} = S) ->
+  try
+    %% Push the Sleep time onto Acc, then push 0 usec item from last
+    %% exec stack onto Acc.
+    %% The MFA reported here probably doesn't appear in the stacktrace
+    %% given to us by the last 'call', so add it here.
+    MFA_bin = mfa_binify(MFA),
+    [{SleepStack, SleepTime},{LastExecStack,_}|Tail] = Acc,
+    USec = TS - LastTS,
+    Acc2 = [{[MFA_bin|LastExecStack], 0}, {SleepStack, SleepTime + USec}|Tail],
+    S#state{last_ts=TS, acc=Acc2}
+  catch XX:YY ->
+            io:format(user, "~p: ~p:~p @ ~p\n", [?LINE, XX, YY, erlang:get_stacktrace()]),
+            S
+  end;
+
+exp1_inner(end_of_trace = _Else, #state{pid=Pid, output_path=OutputPath, acc=Acc} = S) ->
+    {ok, FH} = file:open(OutputPath, [write, raw, binary, delayed_write]),
+    io:format("Writing to ~s ... ", [OutputPath]),
+    [begin
+         Pid_str = io_lib:format("~w", [Pid]),
+         Time_str = integer_to_list(Time),
+         file:write(FH, [Pid_str, $;, intersperse($;, lists:reverse(Stack)), 32, Time_str, 10])
+     end || {Stack, Time} <- Acc],
+    file:close(FH),
+    io:format("finished\n"),
+    S;
+exp1_inner(_Else, S) ->
+%    io:format("?? ~P\n", [_Else, 10]),
+    S.
+
+find_matching_stack(MFA_bin, [{H,_Time}|_] = Acc) ->
+    case lists:member(MFA_bin, H) of
+        true ->
+            H;
+        false ->
+            find_matching_stack2(MFA_bin, Acc)
+    end.
+
+find_matching_stack2(MFA_bin, [{[MFA_bin|_StackTail]=Stack,_Time}|_]) ->
+    Stack;
+find_matching_stack2(MFA_bin, [_H|T]) ->
+    find_matching_stack2(MFA_bin, T);
+find_matching_stack2(_MFA_bin, []) ->
+    [<<"FIND-MATCHING-FAILED">>].    
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+intersperse(_, []) -> [];
+intersperse(_, [X]) -> [X];
+intersperse(Sep, [X | Xs]) -> [X, Sep | intersperse(Sep, Xs)].
+
+stak_trim([<<"proc_lib:init_p_do_apply/3">>,<<"gen_fsm:decode_msg/9">>,<<"gen_fsm:handle_msg/7">>,<<"gen_fsm:loop/7">>|T]) ->
+    stak_trim([<<"GEN-FSM">>|T]);
+stak_trim([<<"GEN-FSM">>,<<"gen_fsm:decode_msg/9">>,<<"gen_fsm:handle_msg/7">>,<<"gen_fsm:loop/7">>|T]) ->
+    stak_trim([<<"GEN-FSM">>|T]);
+stak_trim(Else) ->
+    Else.
+
+stak_binify(Bin) when is_binary(Bin) ->
+    [list_to_binary(X) || X <- stak(Bin)];
+stak_binify(X) ->
+    list_to_binary(io_lib:format("~w", [X])).
+
+mfa_binify({M,F,A}) ->
+    list_to_binary(io_lib:format("~w:~w/~w", [M, F, A]));
+mfa_binify(X) ->
+    list_to_binary(io_lib:format("~w", [X])).
+
+%% Borrowed from redbug.erl
+
+stak(Bin) ->
+  lists:foldl(fun munge/2,[],string:tokens(binary_to_list(Bin),"\n")).
+
+munge(I,Out) ->
+  case I of %% lists:reverse(I) of
+    "..."++_ -> ["truncated!!!"|Out];
+    _ ->
+      case string:str(I, "Return addr") of
+        0 ->
+          case string:str(I, "cp = ") of
+            0 -> Out;
+            _ -> [mfaf(I)|Out]
+          end;
+        _ ->
+          case string:str(I, "erminate process normal") of
+            0 -> [mfaf(I)|Out];
+            _ -> Out
+          end
+      end
+  end.
+
+mfaf(I) ->
+  [_, C|_] = string:tokens(I,"()+"),
+  string:strip(C).
diff --git a/src/lg_messages.erl b/src/lg_messages.erl
new file mode 100644
index 0000000..a35874d
--- /dev/null
+++ b/src/lg_messages.erl
@@ -0,0 +1,214 @@
+%% Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+%%
+%% This package, Looking Glass, is double-licensed under the Mozilla
+%% Public License 1.1 ("MPL") and the Apache License version 2
+%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+%% please see LICENSE-APACHE2.
+%%
+%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+%% either express or implied. See the LICENSE file for specific language governing
+%% rights and limitations of this software.
+%%
+%% If you have any questions regarding licensing, please contact us at
+%% info@rabbitmq.com.
+
+-module(lg_messages).
+
+-export([profile/1]).
+-export([profile_many/1]).
+
+-record(state, {
+    meta = #{} :: map(),
+    senders = #{} :: #{pid() => pos_integer()},
+    receivers = #{} :: #{pid() => pos_integer()},
+    pairs = #{} :: #{{pid(), pid()} => pos_integer()},
+    non_existing = #{} :: #{pid() => pos_integer()},
+    last_msgs = #{} :: #{pid() => atom()}
+}).
+
+-spec profile(file:filename_all()) -> ok.
+profile(Input) ->
+    {ok, FinalState} = lg_file_reader:fold(fun handle_event/2, #state{}, Input),
+    flush(FinalState).
+
+-spec profile_many(file:filename()) -> ok.
+profile_many(Wildcard) ->
+    Files = filelib:wildcard(Wildcard),
+    FinalState = lists:foldl(fun(Input, State0) ->
+        {ok, State} = lg_file_reader:fold(fun handle_event/2, State0, Input),
+        State
+    end, #state{}, Files),
+    flush(FinalState).
+
+%% @todo Later we may want to look at the latency of gen_server call/reply.
+%% @todo Later we may want to look at particular messages, have some sort of callback.
+handle_event({send, From, _, Info, lg}, State=#state{meta=Meta0}) ->
+    Meta = case Meta0 of
+        #{From := Info0} -> Meta0#{From => maps:merge(Info0, Info)};
+        _ -> Meta0#{From => Info}
+    end,
+    State#state{meta=Meta};
+handle_event({send, From, _, Msg, To},
+        State=#state{senders=Senders, receivers=Receivers, pairs=Pairs, last_msgs=Msgs}) ->
+    SendersCount = maps:get(From, Senders, 0),
+    ReceiversCount = maps:get(To, Receivers, 0),
+    PairsCount = maps:get({From, To}, Pairs, 0),
+    State#state{
+        senders=Senders#{From => SendersCount + 1},
+        receivers=Receivers#{To => ReceiversCount + 1},
+        pairs=Pairs#{{From, To} => PairsCount + 1},
+        last_msgs=Msgs#{From => Msg}};
+handle_event({send_to_non_existing_process, From, _, Msg, _},
+        State=#state{non_existing=Map, last_msgs=Msgs}) ->
+    Count = maps:get(From, Map, 0),
+    State#state{
+        non_existing=Map#{From => Count + 1},
+        last_msgs=Msgs#{From => Msg}};
+%% Ignore all other events. We only care about messages.
+handle_event(_, State) ->
+    State.
+
+%% Output of the profiling.
+
+flush(State) ->
+    flush_most_active_senders(State),
+    flush_most_active_receivers(State),
+    flush_most_non_existing(State),
+    flush_most_active_pair_unidirectional(State),
+    flush_most_active_pair_bidirectional(State),
+    io:format("~n"),
+    flush_digraph(State),
+    ok.
+
+flush_most_active_senders(State=#state{senders=Procs}) ->
+    List = lists:sublist(
+        lists:reverse(lists:keysort(2, maps:to_list(Procs))),
+        1, 100),
+    format_by_count("They sent the most messages", List, State).
+
+flush_most_active_receivers(State=#state{receivers=Procs}) ->
+    List = lists:sublist(
+        lists:reverse(lists:keysort(2, maps:to_list(Procs))),
+        1, 100),
+    format_by_count("They received the most messages", List, State).
+
+flush_most_non_existing(State=#state{non_existing=Procs}) ->
+    List = lists:sublist(
+        lists:reverse(lists:keysort(2, maps:to_list(Procs))),
+        1, 100),
+    format_by_count("They sent the most messages to dead processes", List, State).
+
+format_by_count(Title, List, State) ->
+    MsgCols = case io:columns() of
+        {ok, Cols} -> Cols;
+        _ -> 80
+    end,
+    io:format(
+        "~n~s~n~s~n~n"
+        "Process ID      Count      (Label) OR Message sent~n"
+        "----------      -----      -----------------------~n",
+        [Title, lists:duplicate(length(Title), $=)]),
+    _ = [begin
+        {Prefix, Label, Suffix} = label_or_msg(P, State),
+        io:format("~-15w ~-10b ~s~" ++ integer_to_list(MsgCols) ++ "P~s~n",
+            [P, C, Prefix, Label, 5, Suffix])
+    end || {P, C} <- List],
+    ok.
+
+label_or_msg(P, #state{meta=Meta, last_msgs=Msgs}) ->
+    case maps:get(P, Meta, #{}) of
+        #{process_type := PT} -> {"(", PT, ")"};
+        _ -> {"", maps:get(P, Msgs, '<none>'), ""}
+    end.
+
+flush_most_active_pair_unidirectional(State=#state{pairs=Procs}) ->
+    List = lists:sublist(
+        lists:reverse(lists:keysort(2, maps:to_list(Procs))),
+        1, 100),
+    Title = "They sent the most messages to one other process",
+    MsgCols = case io:columns() of
+        {ok, Cols} -> Cols;
+        _ -> 80
+    end,
+    io:format(
+        "~n~s~n~s~n~n"
+        "From pid        To pid          Count      (Label) OR Message sent~n"
+        "--------        ------          -----      -----------------------~n",
+        [Title, lists:duplicate(length(Title), $=)]),
+    _ = [begin
+        {Prefix, Label, Suffix} = label_or_msg(F, State),
+        io:format("~-15w ~-15w ~-10b ~s~" ++ integer_to_list(MsgCols) ++ "P~s~n",
+            [F, T, C, Prefix, Label, 5, Suffix])
+    end || {{F, T}, C} <- List],
+    ok.
+
+flush_most_active_pair_bidirectional(State=#state{pairs=Procs0}) ->
+    Procs = maps:fold(fun merge_pairs/3, #{}, Procs0),
+    List = lists:sublist(
+        lists:reverse(lists:keysort(2, maps:to_list(Procs))),
+        1, 100),
+    Title = "They sent the most messages to each other",
+    MsgCols = case io:columns() of
+        {ok, Cols} -> Cols;
+        _ -> 80
+    end,
+    io:format(
+        "~n~s~n~s~n~n"
+        "Count      Pid 1           (Label) OR Message sent~n"
+        "           Pid 2           by the corresponding process~n"
+        "-----      -----           ----------------------------~n",
+        [Title, lists:duplicate(length(Title), $=)]),
+    _ = [begin
+        {FPrefix, FLabel, FSuffix} = label_or_msg(F, State),
+        {TPrefix, TLabel, TSuffix} = label_or_msg(T, State),
+        io:format(
+            "~-10b ~-15w ~s~" ++ integer_to_list(MsgCols) ++ "P~s~n"
+            "           ~-15w ~s~" ++ integer_to_list(MsgCols) ++ "P~s~n",
+            [C, F, FPrefix, FLabel, 5, FSuffix,
+                T, TPrefix, TLabel, 5, TSuffix])
+    end || {{F, T}, C} <- List],
+    ok.
+
+flush_digraph(State=#state{pairs=Procs0}) ->
+    Procs = maps:fold(fun group_pairs/3, #{}, Procs0),
+    List = maps:to_list(Procs),
+    ok = file:write_file("digraph.gv", [
+        "digraph {\n"
+        "    concentrate=true;\n"
+        "    splines=ortho;\n"
+        "    edge [arrowhead=none, labelfontsize=12.0, minlen=3];\n"
+        "\n",
+        [io_lib:format("    \"~w~s\" -> \"~w~s\" [taillabel=~b, headlabel=~b];~n",
+            [F, label(F, State), T, label(T, State), FC, TC]) || {{F, T}, {FC, TC}} <- List],
+        "}\n"
+    ]),
+    io:format(
+        "The file digraph.gv was created. Use GraphViz to make a PNG.~n"
+        "$ dot -Tpng -O digraph.gv~n"
+        "~n"
+        "You can also edit the file to remove uninteresting processes.~n"
+        "One line in the file is equal to a connection between two processes.~n"),
+    ok.
+
+label(P, #state{meta=Meta}) ->
+    case maps:get(P, Meta, #{}) of
+        #{process_type := PT} -> io_lib:format(" (~w)", [PT]);
+        _ -> ""
+    end.
+
+merge_pairs({From, To}, Count, Acc) ->
+    Key = if
+        From < To -> {From, To};
+        true -> {To, From}
+    end,
+    AccCount = maps:get(Key, Acc, 0),
+    Acc#{Key => AccCount + Count}.
+
+group_pairs({From, To}, Count, Acc) when From < To ->
+    Key = {From, To},
+    {_, AccCount} = maps:get(Key, Acc, {0, 0}),
+    Acc#{Key => {Count, AccCount}};
+group_pairs({From, To}, Count, Acc) ->
+    Key = {To, From},
+    {AccCount, _} = maps:get(Key, Acc, {0, 0}),
+    Acc#{Key => {AccCount, Count}}.
diff --git a/src/lg_messages_seqdiag.erl b/src/lg_messages_seqdiag.erl
new file mode 100644
index 0000000..3033a66
--- /dev/null
+++ b/src/lg_messages_seqdiag.erl
@@ -0,0 +1,144 @@
+%% Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+%%
+%% This package, Looking Glass, is double-licensed under the Mozilla
+%% Public License 1.1 ("MPL") and the Apache License version 2
+%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+%% please see LICENSE-APACHE2.
+%%
+%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+%% either express or implied. See the LICENSE file for specific language governing
+%% rights and limitations of this software.
+%%
+%% If you have any questions regarding licensing, please contact us at
+%% info@rabbitmq.com.
+
+-module(lg_messages_seqdiag).
+
+-export([profile/2]).
+-export([profile_many/2]).
+
+-record(state, {
+    meta = #{} :: map(),
+    events = [],
+    pids
+}).
+
+-spec profile(file:filename_all(), list()) -> ok.
+profile(Input, Pids) ->
+    {ok, FinalState} = lg_file_reader:fold(fun handle_event/2,
+        #state{pids=prepare_pids(Pids)}, Input),
+    flush(FinalState).
+
+-spec profile_many(file:filename(), list()) -> ok.
+profile_many(Wildcard, Pids) ->
+    Files = filelib:wildcard(Wildcard),
+    FinalState = lists:foldl(fun(Input, State0) ->
+        {ok, State} = lg_file_reader:fold(fun handle_event/2, State0, Input),
+        State
+    end, #state{pids=prepare_pids(Pids)}, Files),
+    flush(FinalState).
+
+handle_event({send, From, _, Info, lg}, State=#state{meta=Meta0}) ->
+    Meta = case Meta0 of
+        #{From := Info0} -> Meta0#{From => maps:merge(Info0, Info)};
+        _ -> Meta0#{From => Info}
+    end,
+    State#state{meta=Meta};
+handle_event(Event = {Type, From, _, _, To}, State)
+        when Type =:= send; Type =:= send_to_non_existing_process ->
+    maybe_keep_event(Event, From, To, State);
+handle_event(Event = {spawn, From, _, To, _}, State) ->
+    maybe_keep_event(Event, From, To, State);
+handle_event(Event = {exit, Pid0, _, _}, State=#state{events=Events, pids=Pids}) ->
+    Pid = hide_pid_node(Pid0),
+    case lists:member(Pid, Pids) of
+        true -> State#state{events=[Event|Events]};
+        _ -> State
+    end;
+%% Ignore all other events. We only care about messages and spawns/exits.
+handle_event(_, State) ->
+    State.
+
+maybe_keep_event(Event, From0, To0, State=#state{events=Events, pids=Pids}) ->
+    From = hide_pid_node(From0),
+    To = hide_pid_node(To0),
+    case {lists:member(From, Pids), lists:member(To, Pids)} of
+        {true, true} -> State#state{events=[Event|Events]};
+        _ -> State
+    end.
+
+prepare_pids(Pids) ->
+    [hide_pid_node(Pid) || Pid <- Pids].
+
+hide_pid_node(Pid) when is_pid(Pid) -> hide_pid_node(pid_to_list(Pid));
+hide_pid_node([$<, _, $.|Tail]) -> "<***." ++ Tail;
+hide_pid_node([$<, _, _, $.|Tail]) -> "<***." ++ Tail;
+hide_pid_node([$<, _, _, _, $.|Tail]) -> "<***." ++ Tail;
+hide_pid_node([$<, _, _, _, _, $.|Tail]) -> "<***." ++ Tail;
+hide_pid_node([$<, _, _, _, _, _, $.|Tail]) -> "<***." ++ Tail;
+hide_pid_node(Name) -> Name.
+
+flush(State=#state{events=Events0}) ->
+    %% Sort by timestamp from oldest to newest.
+    Events = lists:keysort(3, Events0),
+    %% Initialize the formatting state.
+    put(num_calls, 0),
+    %% Output everything.
+    ok = file:write_file("seq.diag", [
+        "seqdiag {\n"
+        "    edge_length = 300;\n"
+        "    activation = none;\n"
+        "\n",
+        [format_event(Event, State) || Event <- Events],
+        "}\n"
+    ]),
+    io:format(
+        "The file seq.diag was created. Use seqdiag to make a PNG.~n"
+        "$ seqdiag -Tpng --no-transparency seq.diag~n"
+        "~n"
+        "To use a custom font, use the -f modifier:~n"
+        "$ seqdiag -Tpng --no-transparency -f /usr/share/fonts/TTF/verdana.ttf seq.diag~n"
+        "~n"
+        "You can also edit the file to remove uninteresting messages.~n"
+        "One line in the file is equal to a message sent by a process to another.~n"),
+    ok.
+
+format_event({spawn, From, _, To, MFA}, State) ->
+    io_lib:format("    \"~w~s\" ->> \"~w~s\" [label=\"spawn ~9999P\"];~n", [
+        From, label(From, State), To, label(To, State), MFA, 8]);
+format_event({exit, Pid, _, Reason}, State) ->
+    PidLabel = label(Pid, State),
+    io_lib:format("    \"~w~s\" ->> \"~w~s\" [label=\"exit ~9999P\"];~n", [
+        Pid, PidLabel, Pid, PidLabel, Reason, 8]);
+format_event({Type, From, _, {'$gen_call', {From, Ref}, Msg}, To}, State) ->
+    NumCalls = get(num_calls) + 1,
+    put(num_calls, NumCalls),
+    put(Ref, NumCalls),
+    io_lib:format("    \"~w~s\" ~s \"~w~s\" [label=\"gen:call #~w ~9999P\"];~n", [
+        From, label(From, State),
+        case Type of send -> "->"; _ -> "-->" end,
+        To, label(To, State), NumCalls, Msg, 8]);
+format_event(Event={Type, From, _, {Ref, Msg}, To}, State) ->
+    case get(Ref) of
+        undefined ->
+            default_format_event(Event, State);
+        NumCall ->
+            io_lib:format("    \"~w~s\" ~s \"~w~s\" [label=\"#~w ~9999P\"];~n", [
+                From, label(From, State),
+                case Type of send -> "->"; _ -> "-->" end,
+                To, label(To, State), NumCall, Msg, 8])
+    end;
+format_event(Event, State) ->
+    default_format_event(Event, State).
+
+default_format_event({Type, From, _, Msg, To}, State) ->
+    io_lib:format("    \"~w~s\" ~s \"~w~s\" [label=\"~9999P\"];~n", [
+        From, label(From, State),
+        case Type of send -> "->"; _ -> "-->" end,
+        To, label(To, State), Msg, 8]).
+
+label(P, #state{meta=Meta}) ->
+    case maps:get(P, Meta, #{}) of
+        #{process_type := PT} -> io_lib:format(" (~w)", [PT]);
+        _ -> ""
+    end.
diff --git a/src/lg_rabbit_hole.erl b/src/lg_rabbit_hole.erl
new file mode 100644
index 0000000..688e0ce
--- /dev/null
+++ b/src/lg_rabbit_hole.erl
@@ -0,0 +1,59 @@
+%% Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+%%
+%% This package, Looking Glass, is double-licensed under the Mozilla
+%% Public License 1.1 ("MPL") and the Apache License version 2
+%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+%% please see LICENSE-APACHE2.
+%%
+%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+%% either express or implied. See the LICENSE file for specific language governing
+%% rights and limitations of this software.
+%%
+%% If you have any questions regarding licensing, please contact us at
+%% info@rabbitmq.com.
+
+%% The purpose of this process is to be the target of messages
+%% sent by traced processes. The messages contain metadata that
+%% we want to log when we are tracing and later use when profiling
+%% the sending of messages. This process does not need them, it
+%% just needs to exist, and therefore it discards everything.
+
+-module(lg_rabbit_hole).
+-behaviour(gen_server).
+
+%% API.
+-export([start_link/0]).
+
+%% gen_server.
+-export([init/1]).
+-export([handle_call/3]).
+-export([handle_cast/2]).
+-export([handle_info/2]).
+-export([terminate/2]).
+-export([code_change/3]).
+
+%% API.
+
+-spec start_link() -> {ok, pid()}.
+start_link() ->
+    gen_server:start_link({local, lg}, ?MODULE, [], []).
+
+%% gen_server.
+
+init([]) ->
+    {ok, undefined}.
+
+handle_call(_Request, _From, State) ->
+    {reply, ignored, State}.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
diff --git a/src/lg_raw_console_tracer.erl b/src/lg_raw_console_tracer.erl
index 4259cff..a6f02f0 100644
--- a/src/lg_raw_console_tracer.erl
+++ b/src/lg_raw_console_tracer.erl
@@ -35,7 +35,9 @@ loop(Parent) ->
     receive
         {system, From, Request} ->
             sys:handle_system_msg(Request, From, Parent, ?MODULE, [], Parent);
-        Msg ->
+        Msg0 ->
+            %% Convert the event's monotonic time to its system time.
+            Msg = setelement(3, Msg0, erlang:time_offset(microsecond) + element(3, Msg0)),
             erlang:display(Msg),
             loop(Parent)
     end.
diff --git a/src/lg_socket_client.erl b/src/lg_socket_client.erl
new file mode 100644
index 0000000..21aa5ae
--- /dev/null
+++ b/src/lg_socket_client.erl
@@ -0,0 +1,117 @@
+%% Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+%%
+%% This package, Looking Glass, is double-licensed under the Mozilla
+%% Public License 1.1 ("MPL") and the Apache License version 2
+%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+%% please see LICENSE-APACHE2.
+%%
+%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+%% either express or implied. See the LICENSE file for specific language governing
+%% rights and limitations of this software.
+%%
+%% If you have any questions regarding licensing, please contact us at
+%% info@rabbitmq.com.
+
+-module(lg_socket_client).
+-behavior(gen_statem).
+
+-export([start_link/2]).
+-export([stop/1]).
+
+%% gen_statem.
+-export([callback_mode/0]).
+-export([init/1]).
+-export([connect/3]).
+-export([open_file/3]).
+-export([process_events/3]).
+-export([close_file/3]).
+-export([code_change/4]).
+-export([terminate/3]).
+
+-record(state, {
+    port :: inet:port_number(),
+    base_filename :: file:filename_all(),
+    nth = 0 :: non_neg_integer(),
+    socket :: inet:socket() | undefined,
+    io_device :: file:io_device() | undefined,
+    events_per_frame = 100000 :: pos_integer(),
+    events_this_frame = 0 :: non_neg_integer(),
+    buffer = <<>> :: binary()
+}).
+
+start_link(Port, BaseFilename) ->
+    gen_statem:start_link(?MODULE, [Port, BaseFilename], []).
+
+stop(Pid) ->
+    gen_statem:stop(Pid).
+
+callback_mode() ->
+    state_functions.
+
+init([Port, BaseFilename]) ->
+    %% Store all messages off the heap to avoid unnecessary GC.
+    process_flag(message_queue_data, off_heap),
+    %% We need to trap exit signals in order to shutdown properly.
+    process_flag(trap_exit, true),
+    {ok, connect, #state{port=Port, base_filename=BaseFilename},
+        {next_event, internal, run}}.
+
+connect(internal, _, State) ->
+    do_connect(State);
+connect({timeout, retry}, retry, State) ->
+    do_connect(State);
+connect(_, _, State) ->
+    {keep_state, State}.
+
+do_connect(State=#state{port=Port}) ->
+    case gen_tcp:connect("localhost", Port, [binary, {packet, 2}, {active, true}]) of
+        {ok, Socket} ->
+            {next_state, open_file, State#state{socket=Socket},
+                {next_event, internal, run}};
+        {error, _} ->
+            {keep_state, State, [{{timeout, retry}, 1000, retry}]}
+    end.
+
+open_file(internal, _, State=#state{base_filename=Filename0, nth=Nth}) ->
+    Filename = filename:flatten([Filename0, ".", integer_to_list(Nth)]),
+    {ok, IoDevice} = file:open(Filename, [write, raw]),
+    {next_state, process_events, State#state{nth=Nth + 1, io_device=IoDevice}}.
+
+process_events(info, {tcp, Socket, Bin}, State=#state{socket=Socket, io_device=IoDevice,
+        events_per_frame=MaxEvents, events_this_frame=NumEvents0, buffer=Buffer0}) ->
+    BinSize = byte_size(Bin),
+    Buffer = <<Buffer0/binary, BinSize:16, Bin/binary>>,
+    NumEvents = NumEvents0 + 1,
+    if
+        MaxEvents =:= NumEvents ->
+            ok = file:write(IoDevice, lz4f:compress_frame(Buffer)),
+            {keep_state, State#state{events_this_frame=0, buffer= <<>>}};
+        true ->
+            {keep_state, State#state{events_this_frame=NumEvents, buffer=Buffer}}
+    end;
+process_events(info, {tcp_closed, Socket}, State=#state{socket=Socket}) ->
+    {next_state, close_file, State#state{socket=undefined},
+        {next_event, internal, run}};
+process_events(info, {tcp_error, Socket, _}, State=#state{socket=Socket}) ->
+    _ = gen_tcp:close(Socket),
+    {next_state, close_file, State#state{socket=undefined},
+        {next_event, internal, run}}.
+
+close_file(internal, _, State) ->
+    do_close_file(State),
+    {next_state, connect, State#state{io_device=undefined},
+        {next_event, internal, run}}.
+
+do_close_file(#state{io_device=IoDevice, buffer=Buffer}) ->
+    _ = file:write(IoDevice, lz4f:compress_frame(Buffer)),
+    _ = file:close(IoDevice),
+    ok.
+
+code_change(_OldVsn, OldState, OldData, _Extra) ->
+    {callback_mode(), OldState, OldData}.
+
+terminate(_, _, #state{io_device=undefined}) ->
+    ok;
+terminate(_, _, State) ->
+    do_close_file(State),
+    ok.
diff --git a/src/lg_socket_tracer.erl b/src/lg_socket_tracer.erl
new file mode 100644
index 0000000..0238f2d
--- /dev/null
+++ b/src/lg_socket_tracer.erl
@@ -0,0 +1,148 @@
+%% Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+%%
+%% This package, Looking Glass, is double-licensed under the Mozilla
+%% Public License 1.1 ("MPL") and the Apache License version 2
+%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+%% please see LICENSE-APACHE2.
+%%
+%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+%% either express or implied. See the LICENSE file for specific language governing
+%% rights and limitations of this software.
+%%
+%% If you have any questions regarding licensing, please contact us at
+%% info@rabbitmq.com.
+
+-module(lg_socket_tracer).
+
+-export([start_link/2]).
+-export([init/2]).
+
+-export([system_continue/3]).
+-export([system_terminate/4]).
+-export([system_code_change/4]).
+
+-record(state, {
+    parent :: pid(),
+    lsocket :: inet:socket(),
+    timeout_ref :: reference() | undefined
+}).
+
+start_link(Nth, BasePort) ->
+    Pid = proc_lib:spawn_link(?MODULE, init, [self(), BasePort + Nth - 1]),
+    {ok, Pid}.
+
+init(Parent, Port) ->
+    %% Store all messages off the heap to avoid unnecessary GC.
+    process_flag(message_queue_data, off_heap),
+    %% We need to trap exit signals in order to shutdown properly.
+    process_flag(trap_exit, true),
+    %% Open the listening socket.
+    {ok, LSocket} = gen_tcp:listen(Port, [
+        binary, {reuseaddr, true}, {nodelay, true},
+        %% We encode all events to binary inside a 2-byte length frame.
+        {packet, 2},
+        %% We expect the client to send pings every second or so and
+        %% nothing else, so using active mode is faster and still safe.
+        {active, true},
+        %% We only expect one connection at a time. We don't need
+        %% a backlog except for the cases where the connection is
+        %% lost and will reconnect immediately before we get a
+        %% chance to accept again.
+        {backlog, 1}
+        %% We are using non-blocking TCP send. We therefore do not
+        %% need to configure send timeout options.
+    ]),
+    %% We reject all messages until we get a connection.
+    accept(#state{parent=Parent, lsocket=LSocket}).
+
+accept(State=#state{lsocket=LSocket}) ->
+    {ok, AcceptRef} = prim_inet:async_accept(LSocket, -1),
+    accept_loop(State, AcceptRef).
+
+accept_loop(State=#state{parent=Parent, lsocket=LSocket}, AcceptRef) ->
+    receive
+        {'EXIT', Parent, Reason} ->
+            exit(Reason);
+        {system, From, Request} ->
+            sys:handle_system_msg(Request, From, Parent, ?MODULE, [],
+                {accept_loop, State, AcceptRef});
+        {inet_async, LSocket, AcceptRef, {ok, CSocket}} ->
+            trace_loop(set_timeout(State), CSocket);
+        {inet_async, LSocket, AcceptRef, Error} ->
+            exit({accept_error, Error});
+        %% We discard all trace events when no client is connected.
+        %% We may also end up discarding old timeouts or TCP messages.
+        _ ->
+            accept_loop(State, AcceptRef)
+    end.
+
+trace_loop(State=#state{parent=Parent, timeout_ref=TRef}, CSocket) ->
+    receive
+        {'EXIT', Parent, Reason} ->
+            exit(Reason);
+        {system, From, Request} ->
+            sys:handle_system_msg(Request, From, Parent, ?MODULE, [],
+                {trace_loop, State, CSocket});
+        %% Reset the timeout when we receive data.
+        {tcp, CSocket, _} ->
+            trace_loop(reset_timeout(State), CSocket);
+        {tcp_closed, CSocket} ->
+            close(State, CSocket);
+        {tcp_error, CSocket, _} ->
+            close(State, CSocket);
+        {timeout, TRef, ?MODULE} ->
+            close(State, CSocket);
+        %% Discard the non-blocking send reply when successful.
+        {inet_reply, CSocket, ok} ->
+            trace_loop(State, CSocket);
+        %% And close the socket when an error occured.
+        {inet_reply, CSocket, _} ->
+            close(State, CSocket);
+        %% Discard TCP messages from closed sockets.
+        {tcp, _, _} ->
+            trace_loop(State, CSocket);
+        {tcp_closed, _} ->
+            trace_loop(State, CSocket);
+        {tcp_error, _, _} ->
+            trace_loop(State, CSocket);
+        %% Discard any previous timeout.
+        {timeout, _, ?MODULE} ->
+            trace_loop(State, CSocket);
+        Msg ->
+            Bin = term_to_binary(Msg),
+            _ = byte_size(Bin),
+            case erlang:port_command(CSocket, <<Bin/binary>>, [nosuspend]) of
+                true ->
+                    trace_loop(State, CSocket);
+                %% The send buffer is full.
+                false ->
+                    close(State, CSocket)
+            end
+    end.
+
+close(State, CSocket) ->
+    _ = gen_tcp:close(CSocket),
+    accept(cancel_timeout(State)).
+
+system_continue(_, _, {accept_loop, State, AcceptRef}) ->
+    accept_loop(State, AcceptRef);
+system_continue(_, _, {trace_loop, State, CSocket}) ->
+    trace_loop(State, CSocket).
+
+-spec system_terminate(any(), _, _, _) -> no_return().
+system_terminate(Reason, _, _, _) ->
+    exit(Reason).
+
+system_code_change(Misc, _, _, _) ->
+    {ok, Misc}.
+
+reset_timeout(State) ->
+    set_timeout(cancel_timeout(State)).
+
+set_timeout(State) ->
+    TRef = erlang:start_timer(5000, self(), ?MODULE),
+    State#state{timeout_ref=TRef}.
+
+cancel_timeout(State=#state{timeout_ref=TRef}) ->
+    _ = erlang:cancel_timer(TRef, [{async, true}, {info, false}]),
+    State#state{timeout_ref=undefined}.
diff --git a/src/lg_tracer.erl b/src/lg_tracer.erl
index e1cb5af..1eba511 100644
--- a/src/lg_tracer.erl
+++ b/src/lg_tracer.erl
@@ -19,6 +19,7 @@
 -export([enabled_call/3]).
 -export([enabled_procs/3]).
 -export([enabled_running_procs/3]).
+-export([enabled_send/3]).
 -export([trace/5]).
 
 -on_load(on_load/0).
@@ -42,5 +43,8 @@ enabled_procs(_, _, _) ->
 enabled_running_procs(_, _, _) ->
     erlang:nif_error({not_loaded, ?MODULE}).
 
+enabled_send(_, _, _) ->
+    erlang:nif_error({not_loaded, ?MODULE}).
+
 trace(_, _, _, _, _) ->
     erlang:nif_error({not_loaded, ?MODULE}).
diff --git a/src/looking_glass_sup.erl b/src/looking_glass_sup.erl
index c93b55d..613ea87 100644
--- a/src/looking_glass_sup.erl
+++ b/src/looking_glass_sup.erl
@@ -22,5 +22,8 @@ start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init([]) ->
-    Procs = [],
+    Procs = [
+        {lg_rabbit_hole, {lg_rabbit_hole, start_link, []},
+            permanent, 5000, worker, [lg_rabbit_hole]}
+    ],
     {ok, {{one_for_one, 1, 5}, Procs}}.
diff --git a/test/lg_SUITE.erl b/test/lg_SUITE.erl
index 93a6734..a8b5c86 100644
--- a/test/lg_SUITE.erl
+++ b/test/lg_SUITE.erl
@@ -35,7 +35,7 @@ do_callback() ->
     [{scope, [self()]}, lists].
 
 file_tracer(Config) ->
-    doc("Save events to a files on disk."),
+    doc("Save events to files on disk."),
     lg:trace(lists, lg_file_tracer, config(priv_dir, Config) ++ "/file_tracer.lz4"),
     lists:seq(1,10),
     lg:stop(),
@@ -66,13 +66,95 @@ raw_console_tracer(_) ->
     ct:print("Stop tracing to the console.").
 
 running_true(Config) ->
-    doc("Trace a specific module in with running option enabled."),
+    doc("Trace a specific module with running option enabled."),
     lg:trace(lists, lg_file_tracer, config(priv_dir, Config) ++ "/running_true.lz4",
         #{running => true}),
     lists:seq(1,10),
     lg:stop(),
     do_ensure_decompress(config(priv_dir, Config) ++ "/running_true.lz4").
 
+send_true(Config) ->
+    doc("Trace a specific module with send option enabled."),
+    lg:trace(lists, lg_file_tracer, config(priv_dir, Config) ++ "/send_true.lz4",
+        #{send => true}),
+    Self = self(),
+    %% Send a message to and from an existing process.
+    Pid = spawn(fun() ->
+        receive {msg_from, Self} ->
+            Self ! {msg_from, self()}
+        end
+    end),
+    Pid ! {msg_from, Self},
+    receive {msg_from, Pid} -> ok end,
+    %% Also send a message to a non existing process.
+    DeadPid = spawn(fun() -> ok end),
+    receive after 100 -> ok end,
+    DeadPid ! {msg_from, Self},
+    lg:stop(),
+    do_ensure_decompress(config(priv_dir, Config) ++ "/send_true.lz4").
+
+socket_tracer(_) ->
+    doc("Send events to a socket."),
+    Port = 61234,
+    lg:trace(lists, lg_socket_tracer, Port, #{pool_size => 1}),
+    {ok, Socket} = gen_tcp:connect("localhost", Port,
+        [binary, {packet, 2}, {active, true}]),
+    lists:seq(1,10),
+    lg:stop(),
+    do_socket_tracer_recv(Socket).
+
+socket_tracer_client(Config) ->
+    doc("Send events to a socket client."),
+    Port = 61234,
+    lg:trace(lists, lg_socket_tracer, Port, #{pool_size => 1}),
+    BaseFilename = config(priv_dir, Config) ++ "/socket_tracer_client.lz4",
+    {ok, Pid} = lg_socket_client:start_link(Port, BaseFilename),
+    timer:sleep(1000),
+    lists:seq(1,10),
+    lg:stop(),
+    lg_socket_client:stop(Pid),
+    {ok, File} = file:read_file(BaseFilename ++ ".0"),
+    _ = lz4f:decompress(File),
+    true = filelib:file_size(BaseFilename ++ ".0") > 0,
+    ok.
+
+socket_tracer_many(_) ->
+    doc("Send events to many sockets."),
+    Port = 61234,
+    lg:trace(lists, lg_socket_tracer, Port, #{pool_size => 5}),
+    {ok, _} = gen_tcp:connect("localhost", Port, []),
+    {ok, _} = gen_tcp:connect("localhost", Port + 1, []),
+    {ok, _} = gen_tcp:connect("localhost", Port + 2, []),
+    {ok, _} = gen_tcp:connect("localhost", Port + 3, []),
+    {ok, _} = gen_tcp:connect("localhost", Port + 4, []),
+    {error, _} = gen_tcp:connect("localhost", Port + 5, []),
+    lg:stop().
+
+socket_tracer_reconnect(_) ->
+    doc("Confirm we can reconnect to the tracer."),
+    Port = 61234,
+    lg:trace(lists, lg_socket_tracer, Port, #{pool_size => 1}),
+    {ok, Socket0} = gen_tcp:connect("localhost", Port,
+        [binary, {packet, 2}, {active, true}]),
+    ok = gen_tcp:close(Socket0),
+    {ok, Socket} = gen_tcp:connect("localhost", Port,
+        [binary, {packet, 2}, {active, true}]),
+    lists:seq(1,10),
+    lg:stop(),
+    do_socket_tracer_recv(Socket).
+
+do_socket_tracer_recv(Socket) ->
+    receive
+        {tcp, Socket, Data} ->
+            Term = binary_to_term(Data),
+            true = is_tuple(Term),
+            do_socket_tracer_recv(Socket);
+        {tcp_closed, Socket} ->
+            ok
+    after 1000 ->
+        error(timeout)
+    end.
+
 stop_while_trace_is_running(Config) ->
     doc("Stop tracing while events are still coming in."),
     Self = self(),

Places

File looking_glass-0.1.0-git.patch of Package looking_glass (Revision 0974709d8d90947c225591061a24f95f)

Places