[GNUnet-SVN] [gnunet] branch master updated (b91002ea1 -> 7fc3ecc8e)

[gnunet]

This is an automated email from the git hooks/post-receive script.

grothoff pushed a change to branch master
in repository gnunet.

    from b91002ea1 ++
     new 32cc463b6 fix scanf calls
     new 7fc3ecc8e start preparations for flow control by namestore monitors

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 src/include/gnunet_namestore_service.h   | 34 +++++++++++++++++-
 src/include/gnunet_protocols.h           |  9 ++++-
 src/namestore/gnunet-service-namestore.c | 59 ++++++++++++++++++++++++++++++--
 src/namestore/namestore.h                | 28 ++++++++++++++-
 src/namestore/namestore_api_monitor.c    | 39 +++++++++++++++++++--
 src/namestore/plugin_namestore_flat.c    | 43 ++++++++++++++---------
 6 files changed, 189 insertions(+), 23 deletions(-)

diff --git a/src/include/gnunet_namestore_service.h 
b/src/include/gnunet_namestore_service.h
index 4828f72ad..b8c7ea810 100644
--- a/src/include/gnunet_namestore_service.h
+++ b/src/include/gnunet_namestore_service.h
@@ -109,6 +109,12 @@ typedef void
  * it is replaced with the new record.  Use an empty array to
  * remove all records under the given name.
  *
+ * The continuation is called after the value has been stored in the
+ * database. Monitors may be notified asynchronously (basically with
+ * a buffer). However, if any monitor is consistently too slow to
+ * keep up with the changes, calling @a cont will be delayed until the
+ * monitors do keep up.
+ *
  * @param h handle to the namestore
  * @param pkey private key of the zone
  * @param label name that is being mapped
@@ -313,7 +319,7 @@ struct GNUNET_NAMESTORE_ZoneMonitor;
  *         that monitoring is down. You need to still explicitly call
  *         #GNUNET_NAMESTORE_zone_monitor_stop().
  * @param error_cb_cls closure for @a error_cb
- * @param monitor function to call on zone changes
+ * @param monitor function to call on zone changes, with an initial limit of 1
  * @param monitor_cls closure for @a monitor
  * @param sync_cb function called when we're in sync with the namestore
  * @param sync_cb_cls closure for @a sync_cb
@@ -332,6 +338,32 @@ GNUNET_NAMESTORE_zone_monitor_start (const struct 
GNUNET_CONFIGURATION_Handle *c
 
 
 /**
+ * Calls the monitor processor specified in 
#GNUNET_NAMESTORE_zone_monitor_start
+ * for the next record(s).  This function is used to allow clients that merely
+ * monitor the NAMESTORE to still throttle namestore operations, so we can be
+ * sure that the monitors can keep up.
+ *
+ * Note that #GNUNET_NAMESTORE_records_store() only waits for this
+ * call if the previous limit set by the client was already reached.
+ * Thus, by using a @a limit greater than 1, monitors basically enable
+ * a queue of notifications to be processed asynchronously with some
+ * delay.  Note that even with a limit of 1 the
+ * #GNUNET_NAMESTORE_records_store() function will run asynchronously
+ * and the continuation may be invoked before the monitors completed
+ * (or even started) processing the notification.  Thus, monitors will
+ * only closely track the current state of the namestore, but not
+ * be involved in the transactions.
+ *
+ * @param zm the monitor
+ * @param limit number of records to return to the iterator in one shot
+ *        (before #GNUNET_NAMESTORE_zone_monitor_next is to be called again)
+ */
+void
+GNUNET_NAMESTORE_zone_monitor_next (struct GNUNET_NAMESTORE_ZoneMonitor *zm,
+                                    uint64_t limit);
+
+
+/**
  * Stop monitoring a zone for changes.
  *
  * @param zm handle to the monitor activity to stop
diff --git a/src/include/gnunet_protocols.h b/src/include/gnunet_protocols.h
index bf1b48679..1ef00cfc8 100644
--- a/src/include/gnunet_protocols.h
+++ b/src/include/gnunet_protocols.h
@@ -1306,13 +1306,20 @@ extern "C"
 #define GNUNET_MESSAGE_TYPE_NAMESTORE_RECORD_RESULT 443
 
 /**
+ * Client to service: I am now ready for the next (set of) monitor
+ * events. Monitoring equivlaent of
+ * #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT.
+ */
+#define GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT 444
+
+/**
  * Client to service: please start iteration; receives
  * "GNUNET_MESSAGE_TYPE_NAMESTORE_LOOKUP_NAME_RESPONSE" messages in return.
  */
 #define GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_START 445
 
 /**
- * Client to service: next record in iteration please.
+ * Client to service: next record(s) in iteration please.
  */
 #define GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT 447
 
diff --git a/src/namestore/gnunet-service-namestore.c 
b/src/namestore/gnunet-service-namestore.c
index fa189dbc3..f47c8776b 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -169,6 +169,12 @@ struct ZoneMonitor
    */
   uint64_t seq;
 
+  /**
+   * Current limit of how many more messages we are allowed
+   * to queue to this monitor.
+   */
+  uint64_t limit;
+
 };
 
 
@@ -1667,7 +1673,7 @@ monitor_iterate_cb (void *cls,
  * Handles a #GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_START message
  *
  * @param cls the client sending the message
- * @param message message from the client
+ * @param zis_msg message from the client
  */
 static void
 handle_monitor_start (void *cls,
@@ -1685,7 +1691,7 @@ handle_monitor_start (void *cls,
                               monitor_tail,
                               zm);
   GNUNET_SERVICE_client_mark_monitor (nc->client);
-  GNUNET_SERVICE_client_disable_continue_warning (nc->client);
+  GNUNET_SERVICE_client_continue (nc->client);
   GNUNET_notification_context_add (monitor_nc,
                                   nc->mq);
   if (GNUNET_YES == ntohl (zis_msg->iterate_first))
@@ -1733,6 +1739,51 @@ monitor_next (void *cls)
 
 
 /**
+ * Handles a #GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT message
+ *
+ * @param cls the client sending the message
+ * @param nm message from the client
+ */
+static void
+handle_monitor_next (void *cls,
+                     const struct ZoneMonitorNextMessage *nm)
+{
+  struct NamestoreClient *nc = cls;
+  struct ZoneMonitor *zm;
+  uint64_t inc;
+
+  inc = GNUNET_ntohll (nm->limit);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Received ZONE_MONITOR_NEXT message with limit %llu\n",
+              (unsigned long long) inc);
+  for (zm = monitor_head; NULL != zm; zm = zm->next)
+    if (zm->nc == nc)
+      break;
+  if (NULL == zm)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (nc->client);
+    return;
+  }
+  GNUNET_SERVICE_client_continue (nc->client);
+  if (zm->limit + inc < zm->limit)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (nc->client);
+    return;
+  }
+  zm->limit += inc;
+#if 0
+  if (GNUNET_YES == ntohl (zis_msg->iterate_first))
+    zm->task = GNUNET_SCHEDULER_add_now (&monitor_next,
+                                        zm);
+  else
+    monitor_sync (zm);
+#endif
+}
+
+
+/**
  * Process namestore requests.
  *
  * @param cls closure
@@ -1831,6 +1882,10 @@ GNUNET_SERVICE_MAIN
                          GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_START,
                          struct ZoneMonitorStartMessage,
                          NULL),
+ GNUNET_MQ_hd_fixed_size (monitor_next,
+                         GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT,
+                         struct ZoneMonitorNextMessage,
+                         NULL),
  GNUNET_MQ_handler_end ());
 
 
diff --git a/src/namestore/namestore.h b/src/namestore/namestore.h
index 207b35662..679ca3d3d 100644
--- a/src/namestore/namestore.h
+++ b/src/namestore/namestore.h
@@ -161,7 +161,7 @@ struct LabelLookupResponseMessage
    * Length of serialized record data
    */
   uint16_t rd_len GNUNET_PACKED;
-  
+
   /**
    * Number of records contained
    */
@@ -319,6 +319,32 @@ struct ZoneMonitorStartMessage
 
 
 /**
+ * Ask for next result of zone iteration for the given operation
+ */
+struct ZoneMonitorNextMessage
+{
+  /**
+   * Type will be #GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Always zero.
+   */
+  uint32_t reserved;
+
+  /**
+   * Number of records to return to the iterator in one shot
+   * (before #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_MONITOR_NEXT
+   * should be send again). In NBO.
+   */
+  uint64_t limit;
+
+};
+
+
+
+/**
  * Start a zone iteration for the given zone
  */
 struct ZoneIterationStartMessage
diff --git a/src/namestore/namestore_api_monitor.c 
b/src/namestore/namestore_api_monitor.c
index 8e6d39ad7..9ba90833b 100644
--- a/src/namestore/namestore_api_monitor.c
+++ b/src/namestore/namestore_api_monitor.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013, 2016 GNUnet e.V.
+     Copyright (C) 2013, 2016, 2018 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,7 +17,6 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
 /**
  * @file namestore/namestore_api_monitor.c
  * @brief API to monitor changes in the NAMESTORE
@@ -340,6 +339,42 @@ GNUNET_NAMESTORE_zone_monitor_start (const struct 
GNUNET_CONFIGURATION_Handle *c
 
 
 /**
+ * Calls the monitor processor specified in 
#GNUNET_NAMESTORE_zone_monitor_start
+ * for the next record(s).  This function is used to allow clients that merely
+ * monitor the NAMESTORE to still throttle namestore operations, so we can be
+ * sure that the monitors can keep up.
+ *
+ * Note that #GNUNET_NAMESTORE_records_store() only waits for this
+ * call if the previous limit set by the client was already reached.
+ * Thus, by using a @a limit greater than 1, monitors basically enable
+ * a queue of notifications to be processed asynchronously with some
+ * delay.  Note that even with a limit of 1 the
+ * #GNUNET_NAMESTORE_records_store() function will run asynchronously
+ * and the continuation may be invoked before the monitors completed
+ * (or even started) processing the notification.  Thus, monitors will
+ * only closely track the current state of the namestore, but not
+ * be involved in the transactions.
+ *
+ * @param zm the monitor
+ * @param limit number of records to return to the iterator in one shot
+ *        (before #GNUNET_NAMESTORE_zone_monitor_next is to be called again)
+ */
+void
+GNUNET_NAMESTORE_zone_monitor_next (struct GNUNET_NAMESTORE_ZoneMonitor *zm,
+                                    uint64_t limit)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct ZoneMonitorNextMessage *nm;
+
+  env = GNUNET_MQ_msg (nm,
+                       GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT);
+  nm->limit = GNUNET_htonll (limit);
+  GNUNET_MQ_send (zm->mq,
+                  env);
+}
+
+
+/**
  * Stop monitoring a zone for changes.
  *
  * @param zm handle to the monitor activity to stop
diff --git a/src/namestore/plugin_namestore_flat.c 
b/src/namestore/plugin_namestore_flat.c
index 88b3ce9b4..bbb9e3c62 100644
--- a/src/namestore/plugin_namestore_flat.c
+++ b/src/namestore/plugin_namestore_flat.c
@@ -140,7 +140,7 @@ database_setup (struct Plugin *plugin)
   char *rvalue;
   char *record_count;
   size_t record_data_size;
-  size_t size;
+  uint64_t size;
   size_t key_len;
   struct GNUNET_HashCode hkey;
   struct GNUNET_DISK_FileHandle *fh;
@@ -218,7 +218,8 @@ database_setup (struct Plugin *plugin)
   if (0 < size)
   {
     line = strtok (buffer, "\n");
-    while (line != NULL) {
+    while (line != NULL)
+    {
       zone_private_key = strtok (line, ",");
       if (NULL == zone_private_key)
         break;
@@ -236,23 +237,33 @@ database_setup (struct Plugin *plugin)
         break;
       line = strtok (NULL, "\n");
       entry = GNUNET_new (struct FlatFileEntry);
-      if (1 != sscanf (rvalue,
-                      "%lu",
-                      &entry->rvalue))
       {
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Error parsing entry\n");
-        GNUNET_free (entry);
-        break;
+        unsigned long long ll;
+
+        if (1 != sscanf (rvalue,
+                         "%llu",
+                         &ll))
+        {
+          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                      "Error parsing entry\n");
+          GNUNET_free (entry);
+          break;
+        }
+        entry->rvalue = (uint64_t) ll;
       }
-      if (1 != sscanf (record_count,
-                      "%u",
-                      &entry->record_count))
       {
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Error parsing entry\n");
-        GNUNET_free (entry);
-        break;
+        unsigned int ui;
+
+        if (1 != sscanf (record_count,
+                         "%u",
+                         &ui))
+        {
+          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                      "Error parsing entry\n");
+          GNUNET_free (entry);
+          break;
+        }
+        entry->record_count = (uint32_t) ui;
       }
       entry->label = GNUNET_strdup (label);
       record_data_size

-- 
To stop receiving notification emails like this one, please contact
address@hidden