[gnunet] branch master updated: -transactional bulk import

[gnunet]

This is an automated email from the git hooks/post-receive script.

martin-schanzenbach pushed a commit to branch master
in repository gnunet.

The following commit(s) were added to refs/heads/master by this push:
     new 220c5b1ab -transactional bulk import
220c5b1ab is described below

commit 220c5b1abfc9a056cc8df092dc9df661db6461e0
Author: Martin Schanzenbach <schanzen@gnunet.org>
AuthorDate: Thu Oct 6 14:43:10 2022 +0900

    -transactional bulk import
---
 src/namestore/plugin_rest_namestore.c | 143 ++++++++++++++++++++++++----------
 1 file changed, 101 insertions(+), 42 deletions(-)

diff --git a/src/namestore/plugin_rest_namestore.c 
b/src/namestore/plugin_rest_namestore.c
index efd309d6c..a9b49e19a 100644
--- a/src/namestore/plugin_rest_namestore.c
+++ b/src/namestore/plugin_rest_namestore.c
@@ -205,6 +205,11 @@ struct RequestHandle
    */
   struct GNUNET_NAMESTORE_QueueEntry *ns_qe;
 
+  /**
+   * For bulk import, we need a dedicated Namestore handle
+   */
+  struct GNUNET_NAMESTORE_Handle *nc;
+
   /**
    * Response object
    */
@@ -319,7 +324,8 @@ cleanup_handle (void *cls)
     GNUNET_NAMESTORE_zone_iteration_stop (handle->list_it);
   if (NULL != handle->ns_qe)
     GNUNET_NAMESTORE_cancel (handle->ns_qe);
-
+  if (NULL != handle->nc)
+    GNUNET_NAMESTORE_disconnect (handle->nc);
   if (NULL != handle->resp_object)
   {
     json_decref (handle->resp_object);
@@ -726,20 +732,16 @@ ns_lookup_cb (void *cls,
   }
 }
 
-/**
- * Import callback
- *
- * @param cls the `struct RequestHandle`
- * @param success the success indicating integer, GNUNET_OK on success
- * @param emsg the error message (can be NULL)
- */
+
 static void
-import_finished_cb (void *cls, int32_t success, const char *emsg)
+bulk_tx_commit_cb (void *cls, int32_t success, const char *emsg)
 {
   struct RequestHandle *handle = cls;
   struct MHD_Response *resp;
 
   handle->ns_qe = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Commit finished, %d\n", success);
   if (GNUNET_YES != success)
   {
     if (NULL != emsg)
@@ -748,7 +750,7 @@ import_finished_cb (void *cls, int32_t success, const char 
*emsg)
       GNUNET_SCHEDULER_add_now (&do_error, handle);
       return;
     }
-    handle->emsg = GNUNET_strdup ("Error importing records");
+    handle->emsg = GNUNET_strdup ("Error importing records on commit");
     GNUNET_SCHEDULER_add_now (&do_error, handle);
     return;
   }
@@ -759,45 +761,46 @@ import_finished_cb (void *cls, int32_t success, const 
char *emsg)
 
 
 /**
- * Handle namestore POST import
+ * Import callback
  *
- * @param con_handle the connection handle
- * @param url the url
- * @param cls the RequestHandle
+ * @param cls the `struct RequestHandle`
+ * @param success the success indicating integer, GNUNET_OK on success
+ * @param emsg the error message (can be NULL)
  */
-void
-namestore_import (struct GNUNET_REST_RequestHandle *con_handle,
-                  const char *url,
-                  void *cls)
+static void
+import_finished_cb (void *cls, int32_t success, const char *emsg)
 {
   struct RequestHandle *handle = cls;
-  struct EgoEntry *ego_entry;
-  char *egoname;
-  json_t *data_js;
-  json_error_t err;
 
-  char term_data[handle->rest_handle->data_size + 1];
-  // set zone to name if given
-  if (strlen (GNUNET_REST_API_NS_NAMESTORE_IMPORT) + 1 >= strlen (handle->url))
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Import finished, %d\n", success);
+  handle->ns_qe = NULL;
+  if (GNUNET_YES != success)
   {
-    handle->response_code = MHD_HTTP_NOT_FOUND;
-    handle->emsg = GNUNET_strdup (GNUNET_REST_IDENTITY_NOT_FOUND);
+    if (NULL != emsg)
+    {
+      handle->emsg = GNUNET_strdup (emsg);
+      GNUNET_SCHEDULER_add_now (&do_error, handle);
+      return;
+    }
+    handle->emsg = GNUNET_strdup ("Error importing records");
     GNUNET_SCHEDULER_add_now (&do_error, handle);
     return;
   }
-  ego_entry = NULL;
-
-  egoname = &handle->url[strlen (GNUNET_REST_API_NS_NAMESTORE_IMPORT) + 1];
-  ego_entry = get_egoentry_namestore (handle, egoname);
+  handle->ns_qe = GNUNET_NAMESTORE_transaction_commit (handle->nc,
+                                                       &bulk_tx_commit_cb,
+                                                       handle);
+}
 
-  if (NULL == ego_entry)
-  {
-    handle->response_code = MHD_HTTP_NOT_FOUND;
-    handle->emsg = GNUNET_strdup (GNUNET_REST_IDENTITY_NOT_FOUND);
-    GNUNET_SCHEDULER_add_now (&do_error, handle);
-    return;
-  }
+static void
+bulk_tx_start (void *cls, int32_t success, const char *emsg)
+{
+  struct RequestHandle *handle = cls;
+  json_t *data_js;
+  json_error_t err;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Transaction started...\n");
   if (0 >= handle->rest_handle->data_size)
   {
     handle->response_code = MHD_HTTP_BAD_REQUEST;
@@ -805,6 +808,7 @@ namestore_import (struct GNUNET_REST_RequestHandle 
*con_handle,
     GNUNET_SCHEDULER_add_now (&do_error, handle);
     return;
   }
+  char term_data[handle->rest_handle->data_size + 1];
   term_data[handle->rest_handle->data_size] = '\0';
   GNUNET_memcpy (term_data,
                  handle->rest_handle->data,
@@ -851,10 +855,9 @@ namestore_import (struct GNUNET_REST_RequestHandle 
*con_handle,
                   "Parsed record set for name %s\n", ri[index].a_label);
     }
   }
-  //json_decref (data_js);
+  // json_decref (data_js);
 
-  handle->zone_pkey = GNUNET_IDENTITY_ego_get_private_key (ego_entry->ego);
-  handle->ns_qe = GNUNET_NAMESTORE_records_store2 (ns_handle,
+  handle->ns_qe = GNUNET_NAMESTORE_records_store2 (handle->nc,
                                                    handle->zone_pkey,
                                                    rd_set_count,
                                                    ri,
@@ -868,6 +871,61 @@ namestore_import (struct GNUNET_REST_RequestHandle 
*con_handle,
   }
 }
 
+
+
+/**
+ * Handle namestore POST import
+ *
+ * @param con_handle the connection handle
+ * @param url the url
+ * @param cls the RequestHandle
+ */
+void
+namestore_import (struct GNUNET_REST_RequestHandle *con_handle,
+                  const char *url,
+                  void *cls)
+{
+  struct RequestHandle *handle = cls;
+  struct EgoEntry *ego_entry;
+  char *egoname;
+
+  // set zone to name if given
+  if (strlen (GNUNET_REST_API_NS_NAMESTORE_IMPORT) + 1 >= strlen (
+        handle->url))
+  {
+    handle->response_code = MHD_HTTP_NOT_FOUND;
+    handle->emsg = GNUNET_strdup (GNUNET_REST_IDENTITY_NOT_FOUND);
+    GNUNET_SCHEDULER_add_now (&do_error, handle);
+    return;
+  }
+  ego_entry = NULL;
+
+  egoname = &handle->url[strlen (GNUNET_REST_API_NS_NAMESTORE_IMPORT) + 1];
+  ego_entry = get_egoentry_namestore (handle, egoname);
+
+  if (NULL == ego_entry)
+  {
+    handle->response_code = MHD_HTTP_NOT_FOUND;
+    handle->emsg = GNUNET_strdup (GNUNET_REST_IDENTITY_NOT_FOUND);
+    GNUNET_SCHEDULER_add_now (&do_error, handle);
+    return;
+  }
+  handle->zone_pkey = GNUNET_IDENTITY_ego_get_private_key (ego_entry->ego);
+
+  // We need a per-client connection for a transactional bulk import
+  handle->nc = GNUNET_NAMESTORE_connect (cfg);
+  if (NULL == handle->nc)
+  {
+    handle->response_code = MHD_HTTP_BAD_REQUEST;
+    handle->emsg = GNUNET_strdup (GNUNET_REST_NAMESTORE_NO_DATA);
+    GNUNET_SCHEDULER_add_now (&do_error, handle);
+    return;
+  }
+  handle->ns_qe = GNUNET_NAMESTORE_transaction_begin (handle->nc,
+                                                      &bulk_tx_start,
+                                                      handle);
+}
+
 /**
  * Handle namestore POST/PUT request
  *
@@ -1188,7 +1246,8 @@ rest_process_request (struct GNUNET_REST_RequestHandle 
*rest_handle,
   struct GNUNET_REST_RequestHandlerError err;
   static const struct GNUNET_REST_RequestHandler handlers[] =
   { { MHD_HTTP_METHOD_GET, GNUNET_REST_API_NS_NAMESTORE, &namestore_get },
-    { MHD_HTTP_METHOD_POST, GNUNET_REST_API_NS_NAMESTORE_IMPORT, 
&namestore_import },
+    { MHD_HTTP_METHOD_POST, GNUNET_REST_API_NS_NAMESTORE_IMPORT,
+      &namestore_import },
     { MHD_HTTP_METHOD_POST, GNUNET_REST_API_NS_NAMESTORE, &namestore_add },
     { MHD_HTTP_METHOD_PUT, GNUNET_REST_API_NS_NAMESTORE, &namestore_update },
     { MHD_HTTP_METHOD_DELETE, GNUNET_REST_API_NS_NAMESTORE,

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.