[gnuastro-commits] master f30efcfe: Makefile extensions: function to group parallelization in batches

[Mohammad Akhlaghi]

branch: master
commit f30efcfe925050d31180fb5be91e070b0b1323fe
Author: Mohammad Akhlaghi <mohammad@akhlaghi.org>
Commit: Mohammad Akhlaghi <mohammad@akhlaghi.org>

    Makefile extensions: function to group parallelization in batches
    
    Until now, to limit parallelization we just had the 'ast-text-prev-in-list'
    function that would completely avoid parallelization. However, it is
    sometimes necessary to just decrease parallelization, not to completely
    disable it.
    
    With this commit, a new 'ast-text-prev-batch-in-list' function has been
    added for this purpose with a fully working example.
---
 NEWS                |  7 +++++
 doc/gnuastro.texi   | 75 +++++++++++++++++++++++++++++++++++++++++++----------
 lib/gnuastro/type.h | 21 +++++++++++++++
 lib/makeplugin.c    | 58 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 147 insertions(+), 14 deletions(-)

diff --git a/NEWS b/NEWS
index 5ba99dc8..98037924 100644
--- a/NEWS
+++ b/NEWS
@@ -40,6 +40,13 @@ See the end of the file for license conditions.
   - $(ast-text-prev-in-list TARGET, LIST): select the word that is previous
     to 'TARGET' in a list of words. See the documentation for a fully
     working example and how this can be useful.
+
+  - $(ast-text-prev-batch-in-list TARGET, NUM, LIST): select the previous
+    "batch" of 'NUM' words (in relation to the batch that contains
+    'TARGET'). This is useful for steps in your pipelines were you need to
+    limit the parallelization to batches. See the example in the book for a
+    fully working example.
+
 ** Removed features
 ** Changed features
 *** All programs
diff --git a/doc/gnuastro.texi b/doc/gnuastro.texi
index 87eb562d..9911fcbf 100644
--- a/doc/gnuastro.texi
+++ b/doc/gnuastro.texi
@@ -35889,6 +35889,31 @@ They are therefore generic (not limited to 
astronomy/FITS), but because they are
 The names of these functions start with @code{ast-text-*} and each has a fully 
working example to demonstrate its usage.
 
 @table @code
+@item $(ast-text-to-upper STRING)
+Returns the input string but with all characters in UPPER-CASE.
+For example, the following minimal Makefile will print @code{FOOO BAAR UGGH} 
word of the list.
+
+@example
+load /usr/local/lib/libgnuastro_make.so
+
+list   = fOOo bAar UggH
+ulist := $(ast-text-to-upper $(list))
+all:; echo $(ulist)
+@end example
+
+@item $(ast-text-to-lower STRING)
+Returns the input string but with all characters in lower-case.
+For example, the following minimal Makefile will print @code{fooo baar uggh} 
word of the list.
+
+@example
+load /usr/local/lib/libgnuastro_make.so
+
+list  = fOOo bAar UggH
+list := $(ast-text-to-lower $(list))
+all:; echo $(ulist)
+@end example
+
+
 @item $(ast-text-contains STRING, TEXT)
 Returns all white-space-separated words in @code{TEXT} that contain the 
@code{STRING}, removing any words that @emph{do not} match.
 For example, the following minimal Makefile will only print the @code{bAaz 
Aah} word of the list.
@@ -35971,29 +35996,51 @@ Without this function, make first builds all the 
sub-sub-components, then goes t
 There can be any level of components between these, allowing this operation to 
be as complex as necessary in your data analysis pipeline.
 Unfortunately the @code{.NOTPARALLEL} target of GNU Make doesn't allow this 
level of customization.
 
+@item $(ast-text-prev-batch-in-list TARGET, NUM, LIST)
+Returns the previous batch of @code{NUM} words in @code{LIST} (in relation to 
the batch containing @code{TARGET}).
+In the special case that @code{NUM=1}, this is equivalent to the 
@code{ast-text-prev-in-list} function that is described above.
 
-@item $(ast-text-to-upper STRING)
-Returns the input string but with all characters in UPPER-CASE.
-For example, the following minimal Makefile will print @code{FOOO BAAR UGGH} 
word of the list.
+Here is one scenario where this function is useful: in astronomy datasets are 
can easily be very large.
+Therefore, some Make recipes in your pipeline may require a lot of memory; 
such that executing them on all the available threads (for example 12 threads 
with @code{-j12}) will immediately occupy all your RAM, causing a crash in your 
pipeline.
+However, let's assume that you have sufficient RAM to execute 4 targets of 
those recipes in parallel.
+Therefore while you want all the other steps of your pipeline to be using all 
12 threads, you want one rule to only build 4 targets at any time.
+
+The example below demonstrates the usage of this function in a minimal working 
example of the scenario above: we want to build 15 targets, but in batches of 4 
target at a time, irrespective of how many threads Make was executed with.
 
 @example
 load /usr/local/lib/libgnuastro_make.so
 
-list   = fOOo bAar UggH
-ulist := $(ast-text-to-upper $(list))
-all:; echo $(ulist)
+.SECONDEXPANSION:
+
+targets := $(foreach i,$(shell seq 15),a-$(i).fits)
+
+all: $(targets)
+
+$(targets): $$(ast-text-prev-batch-in-list $$@@,4,$(targets))
+     @@echo "$@@: $^"
 @end example
 
-@item $(ast-text-to-lower STRING)
-Returns the input string but with all characters in lower-case.
-For example, the following minimal Makefile will print @code{fooo baar uggh} 
word of the list.
+@noindent
+If you place the example above in a plain-text file called @file{Makefile} 
(correcting for the TAB at the start of the recipe), and run Make on 12 
threads, you will see an output like below.
+The targets in each batch are not ordered (and the order may change in 
different runs) because they have been run in parallel.
 
 @example
-load /usr/local/lib/libgnuastro_make.so
-
-list  = fOOo bAar UggH
-list := $(ast-text-to-lower $(list))
-all:; echo $(ulist)
+$ make -j12
+a-1.fits:
+a-3.fits:
+a-2.fits:
+a-4.fits:
+a-5.fits: a-1.fits a-2.fits a-3.fits a-4.fits
+a-6.fits: a-1.fits a-2.fits a-3.fits a-4.fits
+a-8.fits: a-1.fits a-2.fits a-3.fits a-4.fits
+a-7.fits: a-1.fits a-2.fits a-3.fits a-4.fits
+a-9.fits: a-5.fits a-6.fits a-7.fits a-8.fits
+a-11.fits: a-5.fits a-6.fits a-7.fits a-8.fits
+a-12.fits: a-5.fits a-6.fits a-7.fits a-8.fits
+a-10.fits: a-5.fits a-6.fits a-7.fits a-8.fits
+a-13.fits: a-9.fits a-10.fits a-11.fits a-12.fits
+a-15.fits: a-9.fits a-10.fits a-11.fits a-12.fits
+a-14.fits: a-9.fits a-10.fits a-11.fits a-12.fits
 @end example
 
 @end table
diff --git a/lib/gnuastro/type.h b/lib/gnuastro/type.h
index 7f01ef6a..0d3c51f4 100644
--- a/lib/gnuastro/type.h
+++ b/lib/gnuastro/type.h
@@ -1,3 +1,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 /*********************************************************************
 Type -- Type information and basic operations.
 This is part of GNU Astronomy Utilities (Gnuastro) package.
diff --git a/lib/makeplugin.c b/lib/makeplugin.c
index db7fbb6f..8967eb3d 100644
--- a/lib/makeplugin.c
+++ b/lib/makeplugin.c
@@ -54,6 +54,7 @@ static char 
*text_to_lower=MAKEPLUGIN_FUNC_PREFIX"-text-to-lower";
 static char *text_contains_name=MAKEPLUGIN_FUNC_PREFIX"-text-contains";
 static char *text_prev_in_list=MAKEPLUGIN_FUNC_PREFIX"-text-prev-in-list";
 static char *text_not_contains_name=MAKEPLUGIN_FUNC_PREFIX"-text-not-contains";
+static char 
*text_prev_batch_in_list=MAKEPLUGIN_FUNC_PREFIX"-text-prev-batch-in-list";
 
 /* Gnuastro analysis functions */
 static char *version_is_name=MAKEPLUGIN_FUNC_PREFIX"-version-is";
@@ -228,6 +229,58 @@ makeplugin_text_prev_in_list(const char *caller, unsigned 
int argc,
 
 
 
+/* Return the previous word in the given list. */
+static char *
+makeplugin_text_prev_batch_in_list(const char *caller, unsigned int argc,
+                                   char **argv)
+{
+  void *nptr;
+  char *out, *target=argv[0];
+  size_t i=0, num, batch, start;
+  gal_list_str_t *tmp, *olist=NULL, *list=gal_list_str_extract(argv[2]);
+
+  /* Interpret the number. */
+  nptr=&num;
+  if( gal_type_from_string(&nptr, argv[1], GAL_TYPE_SIZE_T) )
+    error(EXIT_SUCCESS, 0, "'%s' could not be read as an "
+          "unsigned integer", argv[1]);
+
+  /* Parse the input list. */
+  for(tmp=list; tmp!=NULL; tmp=tmp->next)
+    { if( !strcmp(tmp->v,target) ) { break; } ++i; }
+
+  /* Set the starting counter for the batch strings to return. f we are in
+     the first batch, or if the given string isn't in the list at all, we
+     don't want to retun anything. */
+  batch=i/num;
+  if(batch==0 || tmp==NULL) return NULL;
+  else
+    {
+      /* We want to return the previous batch, so we'll decrement the
+         batch, and calculate the starting index from that. */
+      start = --batch*num;
+
+      /* Add the strings to the output string. */
+      i=0;
+      for(tmp=list; tmp!=NULL; tmp=tmp->next)
+        {
+          if(i>=start && i<start+num) gal_list_str_add(&olist, tmp->v, 0);
+          ++i;
+        }
+    }
+
+  /* Build the output pointer, clean up and return. */
+  gal_list_str_reverse(&olist);
+  out=gal_list_str_cat(olist, ' ');
+  gal_list_str_free(list,  1);  /* This is the original list.     */
+  gal_list_str_free(olist, 0);  /* We didn't allocate new copies. */
+  return out;
+}
+
+
+
+
+
 
 
 
@@ -392,6 +445,11 @@ libgnuastro_make_gmk_setup()
   gmk_add_function(text_prev_in_list, makeplugin_text_prev_in_list,
                    2, 2, GMK_FUNC_DEFAULT);
 
+  /* Select batch of previous 'num' elements in list.*/
+  gmk_add_function(text_prev_batch_in_list,
+                   makeplugin_text_prev_batch_in_list,
+                   3, 3, GMK_FUNC_DEFAULT);
+