[gnuastro-commits] master 74c9bd4c 1/2: Library (arithmetic.h): new to-1d operator for removing dims

[Mohammad Akhlaghi]

branch: master
commit 74c9bd4cf0485216d67248daaf4888fe7b262f69
Author: Mohammad Akhlaghi <mohammad@akhlaghi.org>
Commit: Mohammad Akhlaghi <mohammad@akhlaghi.org>

    Library (arithmetic.h): new to-1d operator for removing dims
    
    Until now, there was no easy way to remove the dimensionality of an input
    dataset (to have a 1D array). But this is sometimes necessary (for example
    to do the "flattening" phase of convolutional neural networks.
    
    With this commit, the necessary operator has been added to the arithmetic
    library and the documentation contains a fully working example of how to
    use it to simulate flattening.
    
    This was suggested by Faezeh Bidjarchian.
---
 NEWS                      |   2 +
 doc/gnuastro.texi         |  63 ++++++++++++++++++++++++
 lib/arithmetic.c          | 121 ++++++++++++++++++++++++++++++++--------------
 lib/gnuastro/arithmetic.h |   1 +
 4 files changed, 152 insertions(+), 35 deletions(-)

diff --git a/NEWS b/NEWS
index d25b8643..678dd5d9 100644
--- a/NEWS
+++ b/NEWS
@@ -27,6 +27,8 @@ See the end of the file for license conditions.
 
   Arithmetic:
   - New operators:
+    - to-1d: convert the input operand into a 1D array, no matter how many
+      dimensions it has. Suggested by Faezeh Bidjarchian.
     - pool-min: Min-pooling to reduce the size of the input by calculating
                 the minimum of a the pixels within the pooling window. See
                 the new "Pooling opeators" section of the book for
diff --git a/doc/gnuastro.texi b/doc/gnuastro.texi
index 7b687d93..fe9b4c95 100644
--- a/doc/gnuastro.texi
+++ b/doc/gnuastro.texi
@@ -19021,6 +19021,69 @@ Every pixel in this 2D image will have the flux of the 
sum of the 100 slices.
 
 @table @command
 
+@item to-1d
+Convert the input operand into a 1D array; irrespective of the number of 
dimensions it has.
+This operator only takes a single operand (the input array) and just updates 
the metadata.
+Therefore it does not change the layout of the array contents in memory and is 
very fast.
+
+If no further operation is requested on the 1D array, recall that Arithmetic 
will write a 1D array as a table column by default.
+In case you want the output to be saved as a 1D image, or to see it on the 
standard output, please use the @code{--onedasimage} or @code{--onedonstdout} 
options respectively (see @ref{Invoking astarithmetic}).
+
+This operator is useful in scenarios where after some operations on a 2D image 
or 3D cube, the dimensionality is no longer relevant for you and you just care 
about the values.
+In the example below, we will first make a simple 2D image from a plain-text 
file, then convert it to a 1D array:
+
+@example
+## Contents of 'a.txt' to start with.
+$ cat a.txt
+# Image 1: DEMO [counts, uint8] An example image
+1 2 3
+4 5 6
+7 8 9
+
+## Convert the text image into a FITS image.
+$ astconvertt a.txt -o a.fits
+
+## Convert it into a table column (1D):
+$ astarithmetic a.fits to-1d -o table.fits
+
+## Convert it into a 1D image:
+$ astarithmetic a.fits to-1d -o table.fits --onedasimage
+@end example
+
+@cindex Flattening (CNNs)
+A more real-world example would be the following: assume you want to 
``flatten'' two images into a single 1D array (as commonly done in 
convolutional neural networks, or 
CNNs@footnote{@url{https://en.wikipedia.org/wiki/Convolutional_neural_network}}).
+First, we show the contents of a new @mymath{2\times2} image in plain-text 
image, then convert it to a 2D FITS image (@file{b.fits}).
+We will then use arithmetic to make both @file{a.fits} (from the example 
above) and @file{b.fits} into a 1D array and stitch them together into a single 
1D image with one call to Arithmetic.
+For a description of the @code{stitch} operator, see below (same section).
+
+@example
+## Contents of 'b.txt':
+$ cat b.txt
+# Image 1: DEMO [counts, uint8] An example image
+10 11
+12 13
+
+## Convert the text image into a FITS image.
+$ astconvertt b.txt -o b.fits
+
+# Flatten the two images into a single 1D image:
+$ astarithmetic a.fits to-1d b.fits to-1d 2 1 stitch -g1 \
+                --onedonstdout --quiet
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+@end example
+
 @item stitch
 Stitch (connect) any number of given images together along the given dimension.
 The output has the same number of dimensions as the input, but the number of 
pixels along the requested dimension will be different from the inputs.
diff --git a/lib/arithmetic.c b/lib/arithmetic.c
index abc2ebf1..48a62846 100644
--- a/lib/arithmetic.c
+++ b/lib/arithmetic.c
@@ -1126,6 +1126,26 @@ arithmetic_size(int operator, int flags, gal_data_t *in, 
gal_data_t *arg)
 
 
 
+/* Stitch multiple operands along a given dimension. */
+static gal_data_t *
+arithmetic_to_1d(int operator, int flags, gal_data_t *input)
+{
+  size_t i;
+
+  /* Set absurd value to cause a crash if values that shouldn't be used are
+     used! */
+  for(i=1;i<input->ndim;++i) input->dsize[i]=GAL_BLANK_UINT64;
+
+  /* Reset the metadata and return.*/
+  input->ndim=1;
+  input->dsize[0]=input->size;
+  return input;
+}
+
+
+
+
+
 static size_t
 arithmetic_stitch_sanity_check(gal_data_t *list, gal_data_t *fdim)
 {
@@ -1134,7 +1154,7 @@ arithmetic_stitch_sanity_check(gal_data_t *list, 
gal_data_t *fdim)
   size_t c, dim, otherdim;
 
   /* Currently we only have the stitch operator for 2D datasets. */
-  if(list->ndim!=2)
+  if(list->ndim>2)
     error(EXIT_FAILURE, 0, "currently the 'stitch' operator only "
           "works with 2D datasets (images) but you have given a "
           "%zu dimensional dataset. Please get in touch with us "
@@ -1208,7 +1228,7 @@ arithmetic_stitch(int operator, int flags, gal_data_t 
*list,
   void *oarr;
   gal_data_t *tmp, *out;
   uint8_t type=list->type;
-  size_t i, dim, sum, dsize[2];
+  size_t i, dim, sum, dsize[2]={GAL_BLANK_SIZE_T, GAL_BLANK_SIZE_T};
 
   /* In case we are dealing with a single-element list, just return it! */
   if(list->next==NULL) return list;
@@ -1218,8 +1238,8 @@ arithmetic_stitch(int operator, int flags, gal_data_t 
*list,
 
   /* Find the size of the final output dataset. */
   sum=0; for(tmp=list; tmp!=NULL; tmp=tmp->next) sum+=tmp->dsize[dim];
-  dsize[0] = dim==0 ? sum            : list->dsize[0];
-  dsize[1] = dim==0 ? list->dsize[1] : sum;
+  dsize[0] = dim==0 ? sum : list->dsize[0];
+  if(list->ndim>1) dsize[1] = dim==0 ? list->dsize[1] : sum;
 
   /* Allocate the output dataset. */
   out=gal_data_alloc(NULL, list->type, list->ndim, dsize, list->wcs,
@@ -1229,39 +1249,63 @@ arithmetic_stitch(int operator, int flags, gal_data_t 
*list,
   /* Write the individual datasets into the output. Note that 'dim' is the
      dimension counter of the C standard. */
   oarr=out->array;
-  for(tmp=list; tmp!=NULL; tmp=tmp->next)
+  switch(list->ndim)
     {
-      switch(dim)
+    case 1: /* 1D stitching. */
+      sum=0;
+      for(tmp=list; tmp!=NULL; tmp=tmp->next)
         {
+          memcpy(gal_pointer_increment(oarr, sum, type),
+                 tmp->array, gal_type_sizeof(type)*tmp->size);
+          sum+=tmp->size;
+        }
+      break;
 
-        /* Vertical stitching (second FITS axis is the vertical axis). */
-        case 0:
-          memcpy(oarr,tmp->array, gal_type_sizeof(type)*tmp->size);
-          oarr += gal_type_sizeof(type)*tmp->size;
-          break;
-
-        /* Horizontal stitching (first FITS axis is the horizontal axis) */
-        case 1:
-
-          /* Copy row-by-row. */
-          for(i=0;i<dsize[0];++i)
-            memcpy(gal_pointer_increment(oarr,       i*dsize[1],      type),
-                   gal_pointer_increment(tmp->array, i*tmp->dsize[1], type),
-                   gal_type_sizeof(type)*tmp->dsize[1]);
-
-          /* Copying has finished, increment the start for the next
-             image. Note that in this scenario, the starting pixel for the
-             next image is on the first row, but tmp->dsize[1] pixels
-             away. */
-          oarr += gal_type_sizeof(type)*tmp->dsize[1];
-          break;
-
-        default:
-          error(EXIT_FAILURE, 0, "%s: a bug! Please contact us at '%s' "
-                "to find and fix the problem. The value of 'dim' is "
-                "'%zu' is not understood", __func__, PACKAGE_BUGREPORT,
-                dim);
+    case 2: /* 2D stitching. */
+      for(tmp=list; tmp!=NULL; tmp=tmp->next)
+        {
+          switch(dim)
+            {
+
+            /* Vertical stitching (second FITS axis is the vertical
+               axis). */
+            case 0:
+              memcpy(oarr,tmp->array, gal_type_sizeof(type)*tmp->size);
+              oarr += gal_type_sizeof(type)*tmp->size;
+              break;
+
+            /* Horizontal stitching (first FITS axis is the horizontal
+               axis) */
+            case 1:
+
+              /* Copy row-by-row. */
+              for(i=0;i<dsize[0];++i)
+                memcpy(gal_pointer_increment(oarr, i*dsize[1], type),
+                       gal_pointer_increment(tmp->array, i*tmp->dsize[1],
+                                             type),
+                       gal_type_sizeof(type)*tmp->dsize[1]);
+
+              /* Copying has finished, increment the start for the next
+                 image. Note that in this scenario, the starting pixel for the
+                 next image is on the first row, but tmp->dsize[1] pixels
+                 away. */
+              oarr += gal_type_sizeof(type)*tmp->dsize[1];
+              break;
+
+            default:
+              error(EXIT_FAILURE, 0, "%s: a bug! Please contact us at '%s' "
+                    "to find and fix the problem. The value of 'dim' is "
+                    "'%zu' is not understood", __func__, PACKAGE_BUGREPORT,
+                    dim);
+            }
         }
+      break;
+
+    default:
+      error(EXIT_FAILURE, 0, "%s: a bug! Please contact us at '%s' to "
+            "fix the problem. The value %zu of 'ndim' should have been "
+            "checked before this point!", __func__, PACKAGE_BUGREPORT,
+            list->ndim);
     }
 
   /* Clean up and return */
@@ -3294,7 +3338,9 @@ gal_arithmetic_set_operator(char *string, size_t 
*num_operands)
   else if (!strcmp(string, "random-from-hist-raw"))
     { op=GAL_ARITHMETIC_OP_RANDOM_FROM_HIST_RAW; *num_operands=3; }
 
-  /* Stitching */
+  /* Dimensionality changing */
+  else if (!strcmp(string, "to-1d"))
+    { op=GAL_ARITHMETIC_OP_TO1D;              *num_operands=1;  }
   else if (!strcmp(string, "stitch"))
     { op=GAL_ARITHMETIC_OP_STITCH;            *num_operands=-1; }
 
@@ -3531,6 +3577,7 @@ gal_arithmetic_operator_string(int operator)
     case GAL_ARITHMETIC_OP_RANDOM_FROM_HIST_RAW:return "random-from-hist-raw";
 
     case GAL_ARITHMETIC_OP_SIZE:            return "size";
+    case GAL_ARITHMETIC_OP_TO1D:            return "to-1d";
     case GAL_ARITHMETIC_OP_STITCH:          return "stitch";
 
     case GAL_ARITHMETIC_OP_TO_UINT8:        return "uchar";
@@ -3767,7 +3814,11 @@ gal_arithmetic(int operator, size_t numthreads, int 
flags, ...)
         out=arithmetic_mknoise(operator, flags, d1, d2);
       break;
 
-    /* Stitch multiple datasets. */
+    /* Dimensionality changing operators. */
+    case GAL_ARITHMETIC_OP_TO1D:
+      d1 = va_arg(va, gal_data_t *);
+      out=arithmetic_to_1d(operator, flags, d1);
+      break;
     case GAL_ARITHMETIC_OP_STITCH:
       d1 = va_arg(va, gal_data_t *);
       d2 = va_arg(va, gal_data_t *);
diff --git a/lib/gnuastro/arithmetic.h b/lib/gnuastro/arithmetic.h
index fdaa4f49..ec5be066 100644
--- a/lib/gnuastro/arithmetic.h
+++ b/lib/gnuastro/arithmetic.h
@@ -196,6 +196,7 @@ enum gal_arithmetic_operators
   GAL_ARITHMETIC_OP_RANDOM_FROM_HIST_RAW,/* Randoms from a histogram (raw).*/
 
   GAL_ARITHMETIC_OP_STITCH,       /* Stitch multiple datasets together.    */
+  GAL_ARITHMETIC_OP_TO1D,         /* Make they output into a 1D array.     */
 
   GAL_ARITHMETIC_OP_TO_UINT8,     /* Convert to uint8_t.                   */
   GAL_ARITHMETIC_OP_TO_INT8,      /* Convert to int8_t.                    */