File gcc12-fifo-jobserver-support.patch of Package gcc12
From 1270ccda70ca09f7d4fe76b5156dca8992bd77a6 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 9 Aug 2022 13:59:32 +0200
Subject: [PATCH 1/3] Factor out jobserver_active_p.
gcc/ChangeLog:
* gcc.cc (driver::detect_jobserver): Remove and move to
jobserver.h.
* lto-wrapper.cc (jobserver_active_p): Likewise.
(run_gcc): Likewise.
* opts-jobserver.h: New file.
* opts-common.cc (jobserver_info::jobserver_info): New function.
---
gcc/gcc.cc | 37 +++++--------------------------------
gcc/lto-wrapper.cc | 44 +++++++++-----------------------------------
gcc/opts-common.cc | 41 +++++++++++++++++++++++++++++++++++++++++
gcc/opts-jobserver.h | 44 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 99 insertions(+), 67 deletions(-)
create mode 100644 gcc/opts-jobserver.h
diff --git a/gcc/gcc.cc b/gcc/gcc.cc
index 5cbb38560b2..cac11c1a117 100644
--- a/gcc/gcc.cc
+++ b/gcc/gcc.cc
@@ -27,6 +27,7 @@ CC recognizes how to compile each input file by suffixes in the file names.
Once it knows which kind of compilation to perform, the procedure for
compilation is specified by a string called a "spec". */
+#define INCLUDE_STRING
#include "config.h"
#include "system.h"
#include "coretypes.h"
@@ -43,6 +44,7 @@ compilation is specified by a string called a "spec". */
#include "opts.h"
#include "filenames.h"
#include "spellcheck.h"
+#include "opts-jobserver.h"
@@ -9178,38 +9180,9 @@ driver::final_actions () const
void
driver::detect_jobserver () const
{
- /* Detect jobserver and drop it if it's not working. */
- const char *makeflags = env.get ("MAKEFLAGS");
- if (makeflags != NULL)
- {
- const char *needle = "--jobserver-auth=";
- const char *n = strstr (makeflags, needle);
- if (n != NULL)
- {
- int rfd = -1;
- int wfd = -1;
-
- bool jobserver
- = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
- && rfd > 0
- && wfd > 0
- && is_valid_fd (rfd)
- && is_valid_fd (wfd));
-
- /* Drop the jobserver if it's not working now. */
- if (!jobserver)
- {
- unsigned offset = n - makeflags;
- char *dup = xstrdup (makeflags);
- dup[offset] = '\0';
-
- const char *space = strchr (makeflags + offset, ' ');
- if (space != NULL)
- strcpy (dup + offset, space);
- xputenv (concat ("MAKEFLAGS=", dup, NULL));
- }
- }
- }
+ jobserver_info jinfo;
+ if (!jinfo.is_active && !jinfo.skipped_makeflags.empty ())
+ xputenv (jinfo.skipped_makeflags.c_str ());
}
/* Determine what the exit code of the driver should be. */
diff --git a/gcc/lto-wrapper.cc b/gcc/lto-wrapper.cc
index 795ab74555c..1e8eba16dfb 100644
--- a/gcc/lto-wrapper.cc
+++ b/gcc/lto-wrapper.cc
@@ -37,6 +37,7 @@ along with GCC; see the file COPYING3. If not see
./ccCJuXGv.lto.ltrans.o
*/
+#define INCLUDE_STRING
#include "config.h"
#include "system.h"
#include "coretypes.h"
@@ -49,6 +50,8 @@ along with GCC; see the file COPYING3. If not see
#include "lto-section-names.h"
#include "collect-utils.h"
#include "opts-diagnostic.h"
+#include "opt-suggestions.h"
+#include "opts-jobserver.h"
/* Environment variable, used for passing the names of offload targets from GCC
driver to lto-wrapper. */
@@ -1336,35 +1339,6 @@ init_num_threads (void)
#endif
}
-/* Test and return reason why a jobserver cannot be detected. */
-
-static const char *
-jobserver_active_p (void)
-{
- #define JS_PREFIX "jobserver is not available: "
- #define JS_NEEDLE "--jobserver-auth="
-
- const char *makeflags = getenv ("MAKEFLAGS");
- if (makeflags == NULL)
- return JS_PREFIX "%<MAKEFLAGS%> environment variable is unset";
-
- const char *n = strstr (makeflags, JS_NEEDLE);
- if (n == NULL)
- return JS_PREFIX "%<" JS_NEEDLE "%> is not present in %<MAKEFLAGS%>";
-
- int rfd = -1;
- int wfd = -1;
-
- if (sscanf (n + strlen (JS_NEEDLE), "%d,%d", &rfd, &wfd) == 2
- && rfd > 0
- && wfd > 0
- && is_valid_fd (rfd)
- && is_valid_fd (wfd))
- return NULL;
- else
- return JS_PREFIX "cannot access %<" JS_NEEDLE "%> file descriptors";
-}
-
/* Print link to -flto documentation with a hint message. */
void
@@ -1422,7 +1396,6 @@ run_gcc (unsigned argc, char *argv[])
bool jobserver_requested = false;
int auto_parallel = 0;
bool no_partition = false;
- const char *jobserver_error = NULL;
bool fdecoded_options_first = true;
vec<cl_decoded_option> fdecoded_options;
fdecoded_options.create (16);
@@ -1653,14 +1626,14 @@ run_gcc (unsigned argc, char *argv[])
}
else
{
- jobserver_error = jobserver_active_p ();
- if (jobserver && jobserver_error != NULL)
+ jobserver_info jinfo;
+ if (jobserver && !jinfo.is_active)
{
/* Fall back to auto parallelism. */
jobserver = 0;
auto_parallel = 1;
}
- else if (!jobserver && jobserver_error == NULL)
+ else if (!jobserver && jinfo.is_active)
{
parallel = 1;
jobserver = 1;
@@ -1971,9 +1944,10 @@ cont:
if (nr > 1)
{
- if (jobserver_requested && jobserver_error != NULL)
+ jobserver_info jinfo;
+ if (jobserver_requested && !jinfo.is_active)
{
- warning (0, jobserver_error);
+ warning (0, jinfo.error_msg.c_str ());
print_lto_docs_link ();
}
else if (parallel == 0)
diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index 8097c058c72..4d4f424df13 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3. If not see
<http://www.gnu.org/licenses/>. */
+#define INCLUDE_STRING
#include "config.h"
#include "system.h"
#include "intl.h"
@@ -25,6 +26,7 @@ along with GCC; see the file COPYING3. If not see
#include "options.h"
#include "diagnostic.h"
#include "spellcheck.h"
+#include "opts-jobserver.h"
static void prune_options (struct cl_decoded_option **, unsigned int *);
@@ -2005,3 +2007,42 @@ void prepend_xassembler_to_collect_as_options (const char *collect_as_options,
obstack_1grow (o, '\'');
}
}
+
+jobserver_info::jobserver_info ()
+{
+ /* Detect jobserver and drop it if it's not working. */
+ string js_needle = "--jobserver-auth=";
+
+ const char *envval = getenv ("MAKEFLAGS");
+ if (envval != NULL)
+ {
+ string makeflags = envval;
+ size_t n = makeflags.rfind (js_needle);
+ if (n != string::npos)
+ {
+ if (sscanf (makeflags.c_str () + n + js_needle.size (),
+ "%d,%d", &rfd, &wfd) == 2
+ && rfd > 0
+ && wfd > 0
+ && is_valid_fd (rfd)
+ && is_valid_fd (wfd))
+ is_active = true;
+ else
+ {
+ string dup = makeflags.substr (0, n);
+ size_t pos = makeflags.find (' ', n);
+ if (pos != string::npos)
+ dup += makeflags.substr (pos);
+ skipped_makeflags = "MAKEFLAGS=" + dup;
+ error_msg
+ = "cannot access %<" + js_needle + "%> file descriptors";
+ }
+ }
+ error_msg = "%<" + js_needle + "%> is not present in %<MAKEFLAGS%>";
+ }
+ else
+ error_msg = "%<MAKEFLAGS%> environment variable is unset";
+
+ if (!error_msg.empty ())
+ error_msg = "jobserver is not available: " + error_msg;
+}
diff --git a/gcc/opts-jobserver.h b/gcc/opts-jobserver.h
new file mode 100644
index 00000000000..68ce188b84a
--- /dev/null
+++ b/gcc/opts-jobserver.h
@@ -0,0 +1,44 @@
+/* GNU make's jobserver related functionality.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>.
+
+See dbgcnt.def for usage information. */
+
+#ifndef GCC_JOBSERVER_H
+#define GCC_JOBSERVER_H
+
+using namespace std;
+
+struct jobserver_info
+{
+ /* Default constructor. */
+ jobserver_info ();
+
+ /* Error message if there is a problem. */
+ string error_msg = "";
+ /* Skipped MAKEFLAGS where --jobserver-auth is skipped. */
+ string skipped_makeflags = "";
+ /* File descriptor for reading used for jobserver communication. */
+ int rfd = -1;
+ /* File descriptor for writing used for jobserver communication. */
+ int wfd = -1;
+ /* Return true if jobserver is active. */
+ bool is_active = false;
+};
+
+#endif /* GCC_JOBSERVER_H */
--
2.37.2
From 53e3b2bf16a486c15c20991c6095f7be09012b55 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 9 Aug 2022 13:59:36 +0200
Subject: [PATCH 2/3] lto: support --jobserver-style=fifo for recent GNU make
gcc/ChangeLog:
* opts-jobserver.h: Add one member.
* opts-common.cc (jobserver_info::jobserver_info): Parse FIFO
format of --jobserver-auth.
---
gcc/opts-common.cc | 17 +++++++++++++++--
gcc/opts-jobserver.h | 2 ++
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index 4d4f424df13..c2993f9140a 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -2010,8 +2010,14 @@ void prepend_xassembler_to_collect_as_options (const char *collect_as_options,
jobserver_info::jobserver_info ()
{
+ /* Traditionally, GNU make uses opened pipes for jobserver-auth,
+ e.g. --jobserver-auth=3,4.
+ Starting with GNU make 4.4, one can use --jobserver-style=fifo
+ and then named pipe is used: --jobserver-auth=fifo:/tmp/hcsparta. */
+
/* Detect jobserver and drop it if it's not working. */
string js_needle = "--jobserver-auth=";
+ string fifo_prefix = "fifo:";
const char *envval = getenv ("MAKEFLAGS");
if (envval != NULL)
@@ -2020,8 +2026,15 @@ jobserver_info::jobserver_info ()
size_t n = makeflags.rfind (js_needle);
if (n != string::npos)
{
- if (sscanf (makeflags.c_str () + n + js_needle.size (),
- "%d,%d", &rfd, &wfd) == 2
+ string ending = makeflags.substr (n + js_needle.size ());
+ if (ending.find (fifo_prefix) == 0)
+ {
+ ending = ending.substr (fifo_prefix.size ());
+ pipe_path = ending.substr (0, ending.find (' '));
+ is_active = true;
+ }
+ else if (sscanf (makeflags.c_str () + n + js_needle.size (),
+ "%d,%d", &rfd, &wfd) == 2
&& rfd > 0
&& wfd > 0
&& is_valid_fd (rfd)
diff --git a/gcc/opts-jobserver.h b/gcc/opts-jobserver.h
index 68ce188b84a..98ea2579962 100644
--- a/gcc/opts-jobserver.h
+++ b/gcc/opts-jobserver.h
@@ -37,6 +37,8 @@ struct jobserver_info
int rfd = -1;
/* File descriptor for writing used for jobserver communication. */
int wfd = -1;
+ /* Named pipe path. */
+ string pipe_path = "";
/* Return true if jobserver is active. */
bool is_active = false;
};
--
2.37.2
From fed766af32ed6cd371016cc24e931131e19b4eb1 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 9 Aug 2022 13:59:39 +0200
Subject: [PATCH 3/3] lto: respect jobserver in parallel WPA streaming
PR lto/106328
gcc/ChangeLog:
* opts-jobserver.h (struct jobserver_info): Add pipefd.
(jobserver_info::connect): New.
(jobserver_info::disconnect): Likewise.
(jobserver_info::get_token): Likewise.
(jobserver_info::return_token): Likewise.
* opts-common.cc: Implement the new functions.
gcc/lto/ChangeLog:
* lto.cc (wait_for_child): Decrement nruns once a process
finishes.
(stream_out_partitions): Use job server if active.
(do_whole_program_analysis): Likewise.
---
gcc/lto/lto.cc | 58 +++++++++++++++++++++++++++++++++-----------
gcc/opts-common.cc | 40 ++++++++++++++++++++++++++++++
gcc/opts-jobserver.h | 14 +++++++++++
3 files changed, 98 insertions(+), 14 deletions(-)
diff --git a/gcc/lto/lto.cc b/gcc/lto/lto.cc
index 31b0c1862f7..c82307f4f7e 100644
--- a/gcc/lto/lto.cc
+++ b/gcc/lto/lto.cc
@@ -18,6 +18,7 @@ You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3. If not see
<http://www.gnu.org/licenses/>. */
+#define INCLUDE_STRING
#include "config.h"
#include "system.h"
#include "coretypes.h"
@@ -54,11 +55,17 @@ along with GCC; see the file COPYING3. If not see
#include "attribs.h"
#include "builtins.h"
#include "lto-common.h"
+#include "opts-jobserver.h"
-
-/* Number of parallel tasks to run, -1 if we want to use GNU Make jobserver. */
+/* Number of parallel tasks to run. */
static int lto_parallelism;
+/* Number of active WPA streaming processes. */
+static int nruns = 0;
+
+/* GNU make's jobserver info. */
+static jobserver_info *jinfo = NULL;
+
/* Return true when NODE has a clone that is analyzed (i.e. we need
to load its body even if the node itself is not needed). */
@@ -205,6 +212,12 @@ wait_for_child ()
"streaming subprocess was killed by signal");
}
while (!WIFEXITED (status) && !WIFSIGNALED (status));
+
+ --nruns;
+
+ /* Return token to the jobserver if active. */
+ if (jinfo != NULL && jinfo->is_active)
+ jinfo->return_token ();
}
#endif
@@ -228,25 +241,35 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max,
bool ARG_UNUSED (last))
{
#ifdef HAVE_WORKING_FORK
- static int nruns;
-
if (lto_parallelism <= 1)
{
stream_out_partitions_1 (temp_filename, blen, min, max);
return;
}
- /* Do not run more than LTO_PARALLELISM streamings
- FIXME: we ignore limits on jobserver. */
if (lto_parallelism > 0 && nruns >= lto_parallelism)
- {
- wait_for_child ();
- nruns --;
- }
+ wait_for_child ();
+
/* If this is not the last parallel partition, execute new
streaming process. */
if (!last)
{
+ if (jinfo != NULL && jinfo->is_active)
+ while (true)
+ {
+ if (jinfo->get_token ())
+ break;
+ if (nruns > 0)
+ wait_for_child ();
+ else
+ {
+ /* There are no free tokens, lets do the job outselves. */
+ stream_out_partitions_1 (temp_filename, blen, min, max);
+ asm_nodes_output = true;
+ return;
+ }
+ }
+
pid_t cpid = fork ();
if (!cpid)
@@ -264,10 +287,12 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max,
/* Last partition; stream it and wait for all children to die. */
else
{
- int i;
stream_out_partitions_1 (temp_filename, blen, min, max);
- for (i = 0; i < nruns; i++)
+ while (nruns > 0)
wait_for_child ();
+
+ if (jinfo != NULL && jinfo->is_active)
+ jinfo->disconnect ();
}
asm_nodes_output = true;
#else
@@ -460,9 +485,14 @@ do_whole_program_analysis (void)
lto_parallelism = 1;
- /* TODO: jobserver communication is not supported, yet. */
if (!strcmp (flag_wpa, "jobserver"))
- lto_parallelism = param_max_lto_streaming_parallelism;
+ {
+ jinfo = new jobserver_info ();
+ if (jinfo->is_active)
+ jinfo->connect ();
+
+ lto_parallelism = param_max_lto_streaming_parallelism;
+ }
else
{
lto_parallelism = atoi (flag_wpa);
diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index c2993f9140a..5d79f174a38 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -2059,3 +2059,43 @@ jobserver_info::jobserver_info ()
if (!error_msg.empty ())
error_msg = "jobserver is not available: " + error_msg;
}
+
+void
+jobserver_info::connect ()
+{
+ if (!pipe_path.empty ())
+ pipefd = open (pipe_path.c_str (), O_RDWR);
+}
+
+void
+jobserver_info::disconnect ()
+{
+ if (!pipe_path.empty ())
+ {
+ gcc_assert (close (pipefd) == 0);
+ pipefd = -1;
+ }
+}
+
+bool
+jobserver_info::get_token ()
+{
+ int fd = pipe_path.empty () ? rfd : pipefd;
+ char c;
+ unsigned n = read (fd, &c, 1);
+ if (n != 1)
+ {
+ gcc_assert (errno == EAGAIN);
+ return false;
+ }
+ else
+ return true;
+}
+
+void
+jobserver_info::return_token ()
+{
+ int fd = pipe_path.empty () ? wfd : pipefd;
+ char c = 'G';
+ gcc_assert (write (fd, &c, 1) == 1);
+}
diff --git a/gcc/opts-jobserver.h b/gcc/opts-jobserver.h
index 98ea2579962..76c1d9b2882 100644
--- a/gcc/opts-jobserver.h
+++ b/gcc/opts-jobserver.h
@@ -29,6 +29,18 @@ struct jobserver_info
/* Default constructor. */
jobserver_info ();
+ /* Connect to the server. */
+ void connect ();
+
+ /* Disconnect from the server. */
+ void disconnect ();
+
+ /* Get token from the server. */
+ bool get_token ();
+
+ /* Return token to the server. */
+ void return_token ();
+
/* Error message if there is a problem. */
string error_msg = "";
/* Skipped MAKEFLAGS where --jobserver-auth is skipped. */
@@ -39,6 +51,8 @@ struct jobserver_info
int wfd = -1;
/* Named pipe path. */
string pipe_path = "";
+ /* Pipe file descriptor. */
+ int pipefd = -1;
/* Return true if jobserver is active. */
bool is_active = false;
};
--
2.37.2
From 3f1c2f89f6b8b8d23a9072f8549b0a2c1de06b03 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 16 Aug 2022 08:22:29 +0200
Subject: [PATCH] jobserver: fix fifo mode by opening pipe in proper mode
The current jobserver_info relies on non-blocking FDs,
thus one the pipe in such mode.
gcc/ChangeLog:
* opts-common.cc (jobserver_info::connect): Open fifo
in non-blocking mode.
---
gcc/opts-common.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index 5d79f174a38..4dec9f94447 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -2064,7 +2064,7 @@ void
jobserver_info::connect ()
{
if (!pipe_path.empty ())
- pipefd = open (pipe_path.c_str (), O_RDWR);
+ pipefd = open (pipe_path.c_str (), O_RDWR | O_NONBLOCK);
}
void
--
2.37.2
