[GNUnet-SVN] [gnunet] 01/02: support compressed JSON uploads

[gnunet]

This is an automated email from the git hooks/post-receive script.

grothoff pushed a commit to branch master
in repository gnunet.

commit 782565417337605572b66758bf13d7123e26f744
Author: Christian Grothoff <address@hidden>
AuthorDate: Fri May 3 16:18:59 2019 +0200

    support compressed JSON uploads
---
 src/json/.gitignore      |   1 +
 src/json/Makefile.am     |   5 +-
 src/json/json_mhd.c      | 122 +++++++++++++++++++++++++++++++++++++++++++++++
 src/json/test_json_mhd.c |  47 ++++++++++++++++--
 4 files changed, 169 insertions(+), 6 deletions(-)

diff --git a/src/json/.gitignore b/src/json/.gitignore
index 6709c749d..347bffd7b 100644
--- a/src/json/.gitignore
+++ b/src/json/.gitignore
@@ -1 +1,2 @@
 test_json
+test_json_mhd
diff --git a/src/json/Makefile.am b/src/json/Makefile.am
index f030c3016..dfe185d5e 100644
--- a/src/json/Makefile.am
+++ b/src/json/Makefile.am
@@ -22,7 +22,9 @@ libgnunetjson_la_LIBADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   -ljansson \
-  $(XLIB)
+  -lmicrohttpd \
+  $(XLIB) \
+  $(Z_LIBS)
 
 check_PROGRAMS = \
   test_json \
@@ -57,6 +59,7 @@ test_json_mhd_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
   -ljansson \
   -lmicrohttpd \
+  $(Z_LIBS) \
   $(LIB_GNURL)
 test_json_mhd_CPPFLAGS = \
  $(CPP_GNURL) $(AM_CPPFLAGS)
diff --git a/src/json/json_mhd.c b/src/json/json_mhd.c
index 30b29b88e..b6ab2d116 100644
--- a/src/json/json_mhd.c
+++ b/src/json/json_mhd.c
@@ -26,6 +26,7 @@
  */
 #include "platform.h"
 #include "gnunet_json_lib.h"
+#include <zlib.h>
 
 
 /**
@@ -55,6 +56,11 @@ struct Buffer
    * Number of allocated bytes in buffer.
    */
   size_t alloc;
+
+  /**
+   * Maximum buffer size allowed.
+   */
+  size_t max;
 };
 
 
@@ -80,7 +86,10 @@ buffer_init (struct Buffer *buf,
   if (data_size > alloc_size)
     alloc_size = data_size;
   buf->data = GNUNET_malloc (alloc_size);
+  buf->alloc = alloc_size;
   GNUNET_memcpy (buf->data, data, data_size);
+  buf->fill = data_size;
+  buf->max = max_size;
   return GNUNET_OK;
 }
 
@@ -137,6 +146,99 @@ buffer_append (struct Buffer *buf,
 }
 
 
+/**
+ * Decompress data in @a buf.
+ *
+ * @param buf input data to inflate
+ * @return result code indicating the status of the operation
+ */
+static enum GNUNET_JSON_PostResult
+inflate_data (struct Buffer *buf)
+{
+  z_stream z;
+  char *tmp;
+  size_t tmp_size;
+  int ret;
+
+  memset (&z, 0, sizeof (z));
+  z.next_in = (Bytef *) buf->data;
+  z.avail_in = buf->fill;
+  tmp_size = GNUNET_MIN (buf->max, buf->fill * 4);
+  tmp = GNUNET_malloc (tmp_size);
+  z.next_out = (Bytef *) tmp;
+  z.avail_out = tmp_size;
+  ret = inflateInit (&z);
+  switch (ret)
+  {
+  case Z_MEM_ERROR:
+    GNUNET_break (0);
+    return GNUNET_JSON_PR_OUT_OF_MEMORY;
+  case Z_STREAM_ERROR:
+    GNUNET_break_op (0);
+    return GNUNET_JSON_PR_JSON_INVALID;
+  case Z_OK:
+    break;
+  }
+  while (1)
+  {
+    ret = inflate (&z, 0);
+    switch (ret)
+    {
+    case Z_MEM_ERROR:
+      GNUNET_break (0);
+      GNUNET_break (Z_OK == inflateEnd (&z));
+      GNUNET_free (tmp);
+      return GNUNET_JSON_PR_OUT_OF_MEMORY;
+    case Z_DATA_ERROR:
+      GNUNET_break (0);
+      GNUNET_break (Z_OK == inflateEnd (&z));
+      GNUNET_free (tmp);
+      return GNUNET_JSON_PR_JSON_INVALID;
+    case Z_NEED_DICT:
+      GNUNET_break (0);
+      GNUNET_break (Z_OK == inflateEnd (&z));
+      GNUNET_free (tmp);
+      return GNUNET_JSON_PR_JSON_INVALID;
+    case Z_OK:
+      if ((0 < z.avail_out) && (0 == z.avail_in))
+      {
+        /* truncated input stream */
+        GNUNET_break (0);
+        GNUNET_break (Z_OK == inflateEnd (&z));
+        GNUNET_free (tmp);
+        return GNUNET_JSON_PR_JSON_INVALID;
+      }
+      if (0 < z.avail_out)
+        continue; /* just call it again */
+      /* output buffer full, can we grow it? */
+      if (tmp_size == buf->max)
+      {
+        /* already at max */
+        GNUNET_break (0);
+        GNUNET_break (Z_OK == inflateEnd (&z));
+        GNUNET_free (tmp);
+        return GNUNET_JSON_PR_OUT_OF_MEMORY;
+      }
+      if (tmp_size * 2 < tmp_size)
+        tmp_size = buf->max;
+      else
+        tmp_size = GNUNET_MIN (buf->max, tmp_size * 2);
+      tmp = GNUNET_realloc (tmp, tmp_size);
+      z.next_out = (Bytef *) &tmp[z.total_out];
+      continue;
+    case Z_STREAM_END:
+      /* decompression successful, make 'tmp' the new 'data' */
+      GNUNET_free (buf->data);
+      buf->data = tmp;
+      buf->alloc = tmp_size;
+      buf->fill = z.total_out;
+      GNUNET_break (Z_OK == inflateEnd (&z));
+      return GNUNET_JSON_PR_SUCCESS; /* at least for now */
+    }
+  } /* while (1) */
+}
+
+
 /**
  * Process a POST request containing a JSON object.  This function
  * realizes an MHD POST processor that will (incrementally) process
@@ -161,10 +263,13 @@ GNUNET_JSON_post_parser (size_t buffer_max,
                          json_t **json)
 {
   struct Buffer *r = *con_cls;
+  const char *ce;
+  int ret;
 
   *json = NULL;
   if (NULL == *con_cls)
   {
+
     /* We are seeing a fresh POST request. */
     r = GNUNET_new (struct Buffer);
     if (GNUNET_OK != buffer_init (r,
@@ -202,12 +307,29 @@ GNUNET_JSON_post_parser (size_t buffer_max,
   }
 
   /* We have seen the whole request. */
+  ce = MHD_lookup_connection_value (connection,
+                                    MHD_HEADER_KIND,
+                                    MHD_HTTP_HEADER_CONTENT_ENCODING);
+  if ((NULL != ce) && (0 == strcasecmp ("deflate", ce)))
+  {
+    ret = inflate_data (r);
+    if (GNUNET_JSON_PR_SUCCESS != ret)
+    {
+      buffer_deinit (r);
+      GNUNET_free (r);
+      *con_cls = NULL;
+      return ret;
+    }
+  }
 
   *json = json_loadb (r->data, r->fill, 0, NULL);
   if (NULL == *json)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 "Failed to parse JSON request body\n");
+    buffer_deinit (r);
+    GNUNET_free (r);
+    *con_cls = NULL;
     return GNUNET_JSON_PR_JSON_INVALID;
   }
   buffer_deinit (r);
diff --git a/src/json/test_json_mhd.c b/src/json/test_json_mhd.c
index 665fd140e..13338b32c 100644
--- a/src/json/test_json_mhd.c
+++ b/src/json/test_json_mhd.c
@@ -27,6 +27,7 @@
 #include "gnunet_util_lib.h"
 #include "gnunet_json_lib.h"
 #include "gnunet_curl_lib.h"
+#include <zlib.h>
 
 #define MAX_SIZE 1024 * 1024
 
@@ -69,7 +70,7 @@ access_handler_cb (void *cls,
       global_ret = 6;
     }
     json_decref (json);
-    resp = MHD_create_response_from_buffer (2, "OK", MHD_RESPMEM_PERSISTENT);
+    resp = MHD_create_response_from_buffer (3, "OK\n", MHD_RESPMEM_PERSISTENT);
     ret = MHD_queue_response (connection, MHD_HTTP_OK, resp);
     MHD_destroy_response (resp);
     return ret;
@@ -100,8 +101,12 @@ main (int argc, const char *const argv[])
   uint16_t port;
   CURL *easy;
   char *url;
+  char *str;
+  size_t slen;
   long post_data_size;
   void *post_data;
+  uLongf dlen;
+  struct curl_slist *json_header;
 
   GNUNET_log_setup ("test-json-mhd", "WARNING", NULL);
   global_ret = 2;
@@ -123,28 +128,60 @@ main (int argc, const char *const argv[])
     GNUNET_snprintf (tmp, sizeof (tmp), "%u", i);
     json_object_set_new (bigj, tmp, json_string (tmp));
   }
-  post_data = json_dumps (bigj, JSON_INDENT (2));
-  post_data_size = strlen (post_data);
-
+  str = json_dumps (bigj, JSON_INDENT (2));
+  slen = strlen (str);
+
+#ifdef compressBound
+  dlen = compressBound (slen);
+#else
+  dlen = slen + slen / 100 + 20;
+  /* documentation says 100.1% oldSize + 12 bytes, but we
+   * should be able to overshoot by more to be safe */
+#endif
+  post_data = GNUNET_malloc (dlen);
+  if (Z_OK !=
+      compress2 ((Bytef *) post_data, &dlen, (const Bytef *) str, slen, 9))
+  {
+    GNUNET_break (0);
+    MHD_stop_daemon (daemon);
+    GNUNET_free (url);
+    json_decref (bigj);
+    GNUNET_free (post_data);
+    GNUNET_free (str);
+    return 1;
+  }
+  post_data_size = (long) dlen;
   port = MHD_get_daemon_info (daemon, MHD_DAEMON_INFO_BIND_PORT)->port;
   easy = curl_easy_init ();
   GNUNET_asprintf (&url, "http://localhost:%u/";, (unsigned int) port);
-  curl_easy_setopt (easy, CURLOPT_VERBOSE, 1);
+  curl_easy_setopt (easy, CURLOPT_VERBOSE, 0);
   curl_easy_setopt (easy, CURLOPT_URL, url);
   curl_easy_setopt (easy, CURLOPT_POST, 1);
   curl_easy_setopt (easy, CURLOPT_POSTFIELDS, post_data);
   curl_easy_setopt (easy, CURLOPT_POSTFIELDSIZE, post_data_size);
+
+  json_header = curl_slist_append (NULL, "Content-Type: application/json");
+  json_header = curl_slist_append (json_header, "Content-Encoding: deflate");
+  curl_easy_setopt (easy, CURLOPT_HTTPHEADER, json_header);
   if (0 != curl_easy_perform (easy))
   {
     GNUNET_break (0);
     MHD_stop_daemon (daemon);
     GNUNET_free (url);
     json_decref (bigj);
+    GNUNET_free (post_data);
+    GNUNET_free (str);
+    curl_slist_free_all (json_header);
+    curl_easy_cleanup (easy);
     return 1;
   }
   MHD_stop_daemon (daemon);
   GNUNET_free (url);
   json_decref (bigj);
+  GNUNET_free (post_data);
+  GNUNET_free (str);
+  curl_slist_free_all (json_header);
+  curl_easy_cleanup (easy);
   return global_ret;
 }
 

-- 
To stop receiving notification emails like this one, please contact
address@hidden