[GNUnet-SVN] r18613 - in gnunet/src: datastore include

[gnunet]

Author: grothoff
Date: 2011-12-15 15:55:51 +0100 (Thu, 15 Dec 2011)
New Revision: 18613

Modified:
   gnunet/src/datastore/gnunet-service-datastore.c
   gnunet/src/include/gnunet_datastore_plugin.h
Log:
adding support for detection quota changes / missing bloomfilter files and 
reconstruction/recovery code

Modified: gnunet/src/datastore/gnunet-service-datastore.c
===================================================================
--- gnunet/src/datastore/gnunet-service-datastore.c     2011-12-15 14:55:34 UTC 
(rev 18612)
+++ gnunet/src/datastore/gnunet-service-datastore.c     2011-12-15 14:55:51 UTC 
(rev 18613)
@@ -1444,6 +1444,24 @@
 
 
 /**
+ * Adds a given key to the bloomfilter 'count' times.
+ *
+ * @param cls the bloomfilter
+ * @param key key to add
+ * @param count number of times to add key
+ */
+static void
+add_key_to_bloomfilter (void *cls,
+                       const GNUNET_HashCode *key,
+                       unsigned int count)
+{
+  struct GNUNET_CONTAINER_BloomFilter *bf = cls;
+  while (0 < count--)
+    GNUNET_CONTAINER_bloomfilter_add (bf, key);
+}
+
+
+/**
  * Process datastore requests.
  *
  * @param cls closure
@@ -1477,6 +1495,7 @@
   };
   char *fn;
   unsigned int bf_size;
+  int refresh_bf;
 
   cfg = c;
   if (GNUNET_OK !=
@@ -1509,9 +1528,58 @@
     fn = NULL;
   }
   if (fn != NULL)
-    filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* 
approx. 3% false positives at max use */
+  {
+    if (GNUNET_YES == GNUNET_DISK_file_test (fn))
+    {
+      filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* 
approx. 3% false positives at max use */
+      if (NULL == filter)
+      {
+       /* file exists but not valid, remove and try again, but refresh */
+       if (0 != UNLINK (fn))
+       {
+         /* failed to remove, run without file */
+         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                     _("Failed to remove bogus bloomfilter file `%s'\n"),
+                     fn);
+         GNUNET_free (fn);
+         fn = NULL;
+         filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5);        
/* approx. 3% false positives at max use */
+         refresh_bf = GNUNET_YES;
+       }
+       else
+       {
+         /* try again after remove */
+         filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        
/* approx. 3% false positives at max use */
+         refresh_bf = GNUNET_YES;
+         if (NULL == filter)
+         {
+           /* failed yet again, give up on using file */
+           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                       _("Failed to remove bogus bloomfilter file `%s'\n"),
+                       fn);
+           GNUNET_free (fn);
+           fn = NULL;
+           filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5);      
  /* approx. 3% false positives at max use */
+         }
+       }
+      }
+      else
+      {
+       /* normal case: have an existing valid bf file, no need to refresh */
+       refresh_bf = GNUNET_NO;
+      }
+    }
+    else
+    {
+      filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* 
approx. 3% false positives at max use */
+      refresh_bf = GNUNET_YES;
+    }
+  }
   else
+  {
     filter = GNUNET_CONTAINER_bloomfilter_init (NULL, bf_size, 5);      /* 
approx. 3% false positives at max use */
+    refresh_bf = GNUNET_YES;
+  }
   GNUNET_free_non_null (fn);
   if (filter == NULL)
   {
@@ -1542,6 +1610,19 @@
                              &process_stat_in, plugin);
   GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL);
   GNUNET_SERVER_add_handlers (server, handlers);
+  if (GNUNET_YES == refresh_bf)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+               _("Rebuilding bloomfilter.  Please be patient.\n"));
+    if (NULL != plugin->api->get_keys)
+      plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, 
filter);  
+    else
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                 _("Plugin does not support get_keys function. Please 
fix!\n"));
+
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+               _("Bloomfilter construction complete.\n"));
+  }
   expired_kill_task =
       GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
                                           &delete_expired, NULL);

Modified: gnunet/src/include/gnunet_datastore_plugin.h
===================================================================
--- gnunet/src/include/gnunet_datastore_plugin.h        2011-12-15 14:55:34 UTC 
(rev 18612)
+++ gnunet/src/include/gnunet_datastore_plugin.h        2011-12-15 14:55:51 UTC 
(rev 18613)
@@ -135,6 +135,29 @@
 
 
 /**
+ * An processor over a set of keys stored in the datastore.
+ *
+ * @param cls closure
+ * @param key key in the data store
+ * @param count how many values are stored under this key in the datastore
+ */
+typedef void (*PluginKeyProcessor) (void *cls, 
+                                  const GNUNET_HashCode *key,
+                                  unsigned int count);
+
+
+/**
+ * Get all of the keys in the datastore.
+ *
+ * @param cls closure
+ * @param proc function to call on each key
+ * @param proc_cls closure for proc
+ */
+typedef void (*PluginGetKeys) (void *cls,
+                              PluginKeyProcessor proc, void *proc_cls);
+
+
+/**
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
@@ -148,6 +171,8 @@
  *        there may be!
  * @param type entries of which type are relevant?
  *     Use 0 for any type.
+ * @param min find the smallest key that is larger than the given min,
+ *            NULL for no minimum (return smallest key)
  * @param proc function to call on the matching value;
  *        proc should be called with NULL if there is no result
  * @param proc_cls closure for proc
@@ -159,7 +184,6 @@
                               PluginDatumProcessor proc, void *proc_cls);
 
 
-
 /**
  * Get a random item (additional constraints may apply depending on
  * the specific implementation).  Calls 'proc' with all values ZERO or
@@ -174,6 +198,8 @@
                                  void *proc_cls);
 
 
+
+
 /**
  * Update the priority for a particular key in the datastore.  If
  * the expiration time in value is different than the time found in
@@ -296,6 +322,11 @@
    */
   PluginDrop drop;
 
+  /**
+   * Iterate over all keys in the database.
+   */
+  PluginGetKeys get_keys;
+  
 };