diff --git a/configure.ac b/configure.ac index f3aa9c9a..50eab4b8 100644 --- a/configure.ac +++ b/configure.ac @@ -76,6 +76,34 @@ else AC_DEFINE([LEAF_OPTIMISATION], 1, [Define if you want to use the leaf optimisation (this can still be turned off with -noleaf)]) fi +dnl The --enable-load-limiter option will automatically test for the +dnl presence of a usable /proc/loadavg file, but it can also be set +dnl manually in order to override the test result. +AC_LANG(C) +AC_USE_SYSTEM_EXTENSIONS +AC_CACHE_CHECK(for usable /proc/loadavg, ac_cv_proc_loadavg, + [AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include +#include +int main() { + int fd, i, n; char str[64]; + fd = open("/proc/loadavg", O_RDONLY | O_CLOEXEC); + if (fd < 0 || read(fd, str, 64) < 0 || lseek(fd, 0, SEEK_SET) < 0) return 1; + for (str[63] = i = n = 0; i < 3 && n < 64; ++n) if (str[n] == ' ') ++i; + return i < 3 || strtoul(&str[n], NULL, 10) < 1 || close(fd) < 0; } ]])], + ac_cv_proc_loadavg=yes, + ac_cv_proc_loadavg=no)]) + +AC_MSG_CHECKING(for load limiter) +AC_ARG_ENABLE(load-limiter, + AS_HELP_STRING([--enable-load-limiter={yes|no}],[Allow to limit the processes started by xargs to not exceed the total number of active processes on a system [default=autodetect]]), + ac_cv_load_limiter=$enableval, + ac_cv_load_limiter=$ac_cv_proc_loadavg) +AS_IF(test x$ac_cv_load_limiter = xyes, + AC_DEFINE(ENABLE_LOAD_LIMITER, 1, [Defined when load limiter support should be built in])) +AC_MSG_RESULT($ac_cv_load_limiter) + AC_ARG_VAR([DEFAULT_ARG_SIZE], [Default size of arguments to child processes of find and xargs, 128k if unspecified]) if test -n "$DEFAULT_ARG_SIZE"; then diff --git a/xargs/xargs.1 b/xargs/xargs.1 index 3bedd67f..1ca4a0b8 100644 --- a/xargs/xargs.1 +++ b/xargs/xargs.1 @@ -225,6 +225,20 @@ arrange for each process to produce a separate output file (or otherwise use separate resources). .TP .PD +.BI \-G " max-load\fR, \fI" \-\-load\-limit "\fR=\fImax-load" +Limit the number of parallel processes started by +.B \-\-max\-procs +with respect to the current system load. When the number of active +processes on a system exceeds +.I max-load +will +.B xargs +run at most one process at a time. This allows to run load-intensive +processses safely in parallel and for nesting multiple +.B xargs +commands without overstressing a system. +.TP +.PD .B \-o, \-\-open\-tty Reopen stdin as .I /dev/tty diff --git a/xargs/xargs.c b/xargs/xargs.c index a2917e48..0171f5e1 100644 --- a/xargs/xargs.c +++ b/xargs/xargs.c @@ -142,6 +142,14 @@ static size_t pids_alloc = 0u; /* Process ID of the parent xargs process. */ static pid_t parent; +#ifdef ENABLE_LOAD_LIMITER +/* When nonzero, limits the number of parallel processes with respect + to the system load. */ +#define PROC_LOADAVG_SIZE 64 +static unsigned long int load_max = 0; +static int proc_loadavg_fd = -2; +#endif + /* If nonzero, we've been signaled that we can start more child processes. */ static volatile sig_atomic_t stop_waiting = 0; @@ -196,6 +204,7 @@ static struct option const longopts[] = {"show-limits", no_argument, NULL, 'S'}, {"exit", no_argument, NULL, 'x'}, {"max-procs", required_argument, NULL, 'P'}, + {"load-limit", required_argument, NULL, 'G'}, {"process-slot-var", required_argument, NULL, PROCESS_SLOT_VAR}, {"version", no_argument, NULL, 'v'}, {"help", no_argument, NULL, 'h'}, @@ -513,7 +522,7 @@ main (int argc, char **argv) bc_use_sensible_arg_max (&bc_ctl); } - while ((optc = getopt_long (argc, argv, "+0a:E:e::i::I:l::L:n:oprs:txP:d:", + while ((optc = getopt_long (argc, argv, "+0a:E:e::i::I:l::L:n:oprs:txP:G:d:", longopts, &option_index)) != -1) { switch (optc) @@ -630,6 +639,15 @@ main (int argc, char **argv) proc_max = parse_num (optarg, 'P', 0L, MAX_PROC_MAX, 1); break; + case 'G': +#ifdef ENABLE_LOAD_LIMITER + /* Global load limiter for parallel processes. */ + load_max = parse_num (optarg, 'G', 0L, MAX_PROC_MAX, 1); +#else + error (EXIT_FAILURE, 0, _("option -G not supported")); +#endif + break; + case 'a': input_file = optarg; break; @@ -699,6 +717,14 @@ main (int argc, char **argv) #endif /* SIGUSR2 */ #endif /* SIGUSR1 */ +#ifdef ENABLE_LOAD_LIMITER + proc_loadavg_fd = open ("/proc/loadavg", O_RDONLY | O_CLOEXEC); + if (proc_loadavg_fd < 0) + { + error (EXIT_FAILURE, errno, _("Cannot open /proc/loadavg")); + proc_loadavg_fd = -1; + } +#endif if (0 == strcmp (input_file, "-")) { @@ -747,6 +773,11 @@ main (int argc, char **argv) fprintf (stderr, _("Maximum parallelism (--max-procs must be no greater): %" PRIuMAX "\n"), (uintmax_t)MAX_PROC_MAX); +#ifdef ENABLE_LOAD_LIMITER + fprintf (stderr, + _("Global load limiter (-G, --load-limit must be no greater): %" PRIuMAX "\n"), + (uintmax_t)MAX_PROC_MAX); +#endif if (isatty (STDIN_FILENO)) { @@ -1249,6 +1280,51 @@ xargs_do_exec (struct buildcmd_control *ctl, void *usercontext, int argc, char * } } +#ifdef ENABLE_LOAD_LIMITER + while (procs_executing > 0 && load_max > 0 && proc_loadavg_fd >= 0) + { + char proc_loadavg[PROC_LOADAVG_SIZE]; + int i, p; + unsigned long int load; + + p = read (proc_loadavg_fd, proc_loadavg, PROC_LOADAVG_SIZE); + if (p < 0) + { + error (EXIT_FAILURE, errno, _("Cannot read from /proc/loadavg")); + close (proc_loadavg_fd); + proc_loadavg_fd = -1; + return 0; + } + /* Reset the read position so we can read from the file repeatedly. */ + if (lseek (proc_loadavg_fd, 0, SEEK_SET) < 0) + { + error (EXIT_FAILURE, errno, _("Cannot seek on /proc/loadavg")); + close (proc_loadavg_fd); + proc_loadavg_fd = -1; + return 0; + } + proc_loadavg[p < PROC_LOADAVG_SIZE ? p : PROC_LOADAVG_SIZE - 1] = '\0'; + for (i = p = 0; i < 3 && p < PROC_LOADAVG_SIZE; ++p) + { + if (proc_loadavg[p] == ' ') + ++i; + } + if (i < 3) + { + error (EXIT_FAILURE, ENOTSUP,_("unsupported format of /proc/loadavg")); + close (proc_loadavg_fd); + proc_loadavg_fd = -1; + return 0; + } + load = strtoul (&proc_loadavg[p], NULL, 10); + + if (load > load_max) + wait_for_proc (false, 1u); + else + break; + } +#endif + if (!query_before_executing || print_args (true)) { if (!query_before_executing && print_command) @@ -1696,6 +1772,8 @@ usage (int status) " before executing the command; useful to run an\n" " interactive application.\n")); HTL (_(" -P, --max-procs=MAX-PROCS run at most MAX-PROCS processes at a time\n")); + HTL (_(" -G, --load-limit=MAX-PROCS limit the number of parallel processes to not\n" + " exceed MAX-PROCS of active processes system-wide\n")); HTL (_(" -p, --interactive prompt before running commands\n")); HTL (_(" --process-slot-var=VAR set environment variable VAR in child processes\n")); HTL (_(" -r, --no-run-if-empty if there are no arguments, then do not run COMMAND;\n"