[GNUnet-SVN] r24900 - in gnunet: contrib src/mesh

[gnunet]

Author: szengel
Date: 2012-11-11 15:23:35 +0100 (Sun, 11 Nov 2012)
New Revision: 24900

Modified:
   gnunet/contrib/regex_profiler_infiniband.conf
   gnunet/src/mesh/gnunet-daemon-regexprofiler.c
   gnunet/src/mesh/gnunet-regex-profiler.c
   gnunet/src/mesh/gnunet-service-mesh.c
Log:
Better mesh/regex profiler performance.


Modified: gnunet/contrib/regex_profiler_infiniband.conf
===================================================================
--- gnunet/contrib/regex_profiler_infiniband.conf       2012-11-11 07:44:00 UTC 
(rev 24899)
+++ gnunet/contrib/regex_profiler_infiniband.conf       2012-11-11 14:23:35 UTC 
(rev 24900)
@@ -3,20 +3,21 @@
 PORT = 11999
 ACCEPT_FROM = 127.0.0.1; 192.168.1.0/24;
 HOSTNAME = localhost
-MAX_PARALLEL_OVERLAY_CONNECT_OPERATIONS = 10
+MAX_PARALLEL_OVERLAY_CONNECT_OPERATIONS = 20
 MAX_PARALLEL_OPERATIONS = 1000
 MAX_PARALLEL_SERVICE_CONNECTIONS = 1000
 MAX_PARALLEL_TOPOLOGY_CONFIG_OPERATIONS = 50
 
 [regexprofiler]
-BINARY = /home/szengel/gnunet/src/mesh/.libs/gnunet-service-regexprofiler
+BINARY = /home/szengel/gnunet/src/mesh/.libs/gnunet-daemon-regexprofiler
 REGEX_PREFIX = "GNVPN-0001-PAD"
+ANNOUNCE_DELAY = 500 ms
 
 [mesh]
 AUTOSTART = YES
 ACCEPT_FROM = 127.0.0.1; 192.168.1.0/24;
-APP_ANNOUNCE_TIME = 30 s
-ID_ANNOUNCE_TIME = 30 s
+APP_ANNOUNCE_TIME = 60 s
+ID_ANNOUNCE_TIME = 60 s
 CONNECT_TIMEOUT = 30 s
 PORT = 12001
 DHT_REPLICATION_LEVEL = 10
@@ -26,11 +27,11 @@
 ACCEPT_FROM = 127.0.0.1; 192.168.1.0/24;
 #PORT = 12002
 HOSTNAME = localhost
-DISABLE_TRY_CONNECT = YES
+DISABLE_TRY_CONNECT = NO
 
 [dhtcache]
 DATABASE = sqlite
-QUOTA = 10 MB
+QUOTA = 1000 MB
 
 [arm]
 DEFAULTSERVICES = core mesh dht statistics regexprofiler
@@ -42,8 +43,8 @@
 [resolver]
 AUTOSTART = NO
 
-[block]
-plugins = dht mesh
+#[block]
+#plugins = dht mesh
 
 [transport]
 AUTOSTART = YES
@@ -102,3 +103,4 @@
 
 [nat]
 RETURN_LOCAL_ADDRESSES = YES
+DISABLEV6 = YES

Modified: gnunet/src/mesh/gnunet-daemon-regexprofiler.c
===================================================================
--- gnunet/src/mesh/gnunet-daemon-regexprofiler.c       2012-11-11 07:44:00 UTC 
(rev 24899)
+++ gnunet/src/mesh/gnunet-daemon-regexprofiler.c       2012-11-11 14:23:35 UTC 
(rev 24900)
@@ -20,7 +20,7 @@
 
 /**
  * @file mesh/gnunet-daemon-regexprofiler.c
- * @brief service that uses mesh to announce a regular expression. Used in
+ * @brief daemon that uses mesh to announce a regular expression. Used in
  * conjunction with gnunet-regex-profiler to announce regexes on serveral peers
  * without the need to explicitly connect to the mesh service running on the
  * peer from within the profiler.
@@ -71,8 +71,23 @@
  */
 static char * regex_prefix;
 
+/**
+ * Time to wait between announcing regexes.
+ */
+static struct GNUNET_TIME_Relative announce_delay = { 500 };
 
 /**
+ * Regexes to announce read from 'policy_filename'.
+ */
+static char **regexes;
+
+/**
+ * Number of regexes read from 'policy_filename'.
+ */
+static unsigned int num_regexes;
+
+
+/**
  * Task run during shutdown.
  *
  * @param cls unused
@@ -121,6 +136,93 @@
 
 
 /**
+ * Task run to iterate over all policies and announce them using mesh.
+ *
+ * @param cls unused
+ * @param tc unused
+ */
+static void
+do_announce_policies (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  static unsigned int num_rx_announced;
+
+  announce_regex (regexes[num_rx_announced]);
+
+  if (++num_rx_announced < num_regexes)
+  {
+    GNUNET_SCHEDULER_add_delayed (announce_delay, &do_announce_policies, NULL);
+    return;
+  }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "All regexes announced.\n");
+}
+
+/**
+ * Load regular expressions from filename into 'rxes' array. Array needs to be 
freed.
+ *
+ * @param filename filename of the file containing the regexes, one per line.
+ * @param rxes array with all regexes, needs to be freed.
+ */
+static unsigned int
+load_regexes (const char *filename, char ***rxes)
+{
+  char *data;
+  char *buf;
+  uint64_t filesize;
+  unsigned int offset;
+  unsigned int rx_cnt;
+  unsigned int i;
+
+  if (GNUNET_YES != GNUNET_DISK_file_test (policy_filename))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Could not find policy file %s\n", policy_filename);
+    return 0;
+  }
+  if (GNUNET_OK != GNUNET_DISK_file_size (policy_filename, &filesize, 
GNUNET_YES, GNUNET_YES))
+    filesize = 0;
+  if (0 == filesize)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Policy file %s is empty.\n", 
policy_filename);
+    return 0;
+  }
+  data = GNUNET_malloc (filesize);
+  if (filesize != GNUNET_DISK_fn_read (policy_filename, data, filesize))
+  {
+    GNUNET_free (data);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Could not read policy file %s.\n",
+                policy_filename);
+    return 0;
+  }
+  buf = data;
+  offset = 0;
+  rx_cnt = 0;
+  while (offset < (filesize - 1))
+  {
+    offset++;
+    if (((data[offset] == '\n')) && (buf != &data[offset]))
+    {
+      data[offset] = '\0';
+      rx_cnt++;
+      buf = &data[offset + 1];
+    }
+    else if ((data[offset] == '\n') || (data[offset] == '\0'))
+      buf = &data[offset + 1];
+  }
+  *rxes = GNUNET_malloc (sizeof (char *) * rx_cnt);
+  offset = 0;
+  for (i = 0; i < rx_cnt; i++)
+  {
+    GNUNET_asprintf (&(*rxes)[i], "%s%s", regex_prefix, &data[offset]);
+    offset += strlen (&data[offset]) + 1;
+  }
+  GNUNET_free (data);
+
+  return rx_cnt;
+}
+
+
+/**
  * @brief Main function that will be run by the scheduler.
  *
  * @param cls closure
@@ -133,13 +235,7 @@
      const char *cfgfile GNUNET_UNUSED,
      const struct GNUNET_CONFIGURATION_Handle *cfg_)
 {
-  char *regex;
-  char *data;
-  char *buf;
-  uint64_t filesize;
-  unsigned int offset;
-  GNUNET_MESH_ApplicationType app;
-
+  const GNUNET_MESH_ApplicationType app = (GNUNET_MESH_ApplicationType)0;
   static struct GNUNET_MESH_MessageHandler handlers[] = {
     {NULL, 0, 0}
   };
@@ -185,10 +281,19 @@
     return;
   }
 
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER", 
"ANNOUNCE_DELAY",
+                                          &announce_delay))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _
+                ("%s service is lacking key configuration settings (%s).  
Using default value: %s.\n"),
+                "regexprofiler", "announce_delay", 
+               GNUNET_STRINGS_relative_time_to_string (announce_delay, 
GNUNET_NO));
+  }
+
   stats_handle = GNUNET_STATISTICS_create ("regexprofiler", cfg);
 
-  app = (GNUNET_MESH_ApplicationType)0;
-
   mesh_handle =
     GNUNET_MESH_connect (cfg, NULL, NULL, NULL, handlers, &app);
 
@@ -200,48 +305,20 @@
     return;
   }
 
-  /* Announcing regexes from policy_filename */
-  if (GNUNET_YES != GNUNET_DISK_file_test (policy_filename))
+  /* Read regexes from policy files */
+  if ((num_regexes = load_regexes (policy_filename, &regexes)) == 0)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Could not find policy file %s\n", policy_filename);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 
+               "Policy file %s contains no policies. Exiting.\n", 
+               policy_filename);
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  if (GNUNET_OK != GNUNET_DISK_file_size (policy_filename, &filesize, 
GNUNET_YES, GNUNET_YES))
-    filesize = 0;
-  if (0 == filesize)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Policy file %s is empty.\n", 
policy_filename);
-    return;
-  }
-  data = GNUNET_malloc (filesize);
-  if (filesize != GNUNET_DISK_fn_read (policy_filename, data, filesize))
-  {
-    GNUNET_free (data);
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Could not read policy file %s.\n",
-                policy_filename);
-    return;
-  }
-  buf = data;
-  offset = 0;
-  regex = NULL;
-  while (offset < (filesize - 1))
-  {
-    offset++;
-    if (((data[offset] == '\n')) && (buf != &data[offset]))
-    {
-      data[offset] = '\0';
-      GNUNET_assert (NULL != buf);
-      GNUNET_asprintf (&regex, "%s%s", regex_prefix, buf);
-      announce_regex (regex);
-      GNUNET_free (regex);
-      buf = &data[offset + 1];
-    }
-    else if ((data[offset] == '\n') || (data[offset] == '\0'))
-          buf = &data[offset + 1];
-  }
-  GNUNET_free (data);
 
+  /* Announcing regexes from policy_filename */
+  GNUNET_SCHEDULER_add_delayed (announce_delay, &do_announce_policies, NULL);
+
   /* Scheduled the task to clean up when shutdown is called */
   GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_task,
                                 NULL);

Modified: gnunet/src/mesh/gnunet-regex-profiler.c
===================================================================
--- gnunet/src/mesh/gnunet-regex-profiler.c     2012-11-11 07:44:00 UTC (rev 
24899)
+++ gnunet/src/mesh/gnunet-regex-profiler.c     2012-11-11 14:23:35 UTC (rev 
24900)
@@ -444,6 +444,33 @@
 mesh_da (void *cls, void *op_result);
 
 
+/**
+ * Function called by testbed once we are connected to stats
+ * service. Get the statistics for the services of interest.
+ *
+ * @param cls the 'struct RegexPeer' for which we connected to stats
+ * @param op connect operation handle
+ * @param ca_result handle to stats service
+ * @param emsg error message on failure
+ */
+static void
+stats_connect_cb (void *cls,
+                  struct GNUNET_TESTBED_Operation *op,
+                  void *ca_result,
+                  const char *emsg);
+
+
+/**
+ * Task to collect all statistics from all peers, will shutdown the
+ * profiler, when done.
+ *
+ * @param cls NULL
+ * @param tc the task context
+ */
+static void
+do_collect_stats (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+
+
 
/******************************************************************************/
 /********************************  SHUTDOWN  
**********************************/
 
/******************************************************************************/
@@ -482,12 +509,16 @@
       size =
         GNUNET_snprintf (output_buffer,
                          sizeof (output_buffer),
-                         "Search string not found: %s (%d)\nOn peer: %u 
(%p)\nWith policy file: %s\nAfter: %s\n",
+                         "%p Search string not found: %s (%d)\n%p On peer: %u 
(%p)\n%p With policy file: %s\n%p After: %s\n",
+                        peer,
                          peer->search_str,
                          peer->search_str_matched,
+                        peer,
                          peer->id,
                          peer,
+                        peer,
                          peer->policy_file,
+                        peer,
                          GNUNET_STRINGS_relative_time_to_string (prof_time, 
GNUNET_NO));
       if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
         GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
@@ -660,11 +691,26 @@
   GNUNET_TESTBED_operation_done (peer->stats_op_handle);
   peer->stats_op_handle = NULL;
 
-  if (++peer_cnt == num_search_strings)
+  peer_cnt++;
+  peer = &peers[peer_cnt];
+
+  if (peer_cnt == num_peers)
   {
     struct GNUNET_TIME_Relative delay = { 100 };
     shutdown_task = GNUNET_SCHEDULER_add_delayed (delay, &do_shutdown, NULL);
   }
+  else
+  {
+    peer->stats_op_handle =
+      GNUNET_TESTBED_service_connect (NULL,
+                                     peer->peer_handle,
+                                     "statistics",
+                                     &stats_connect_cb,
+                                     peer,
+                                     &stats_ca,
+                                     &stats_da,
+                                     peer);
+  }
 }
 
 
@@ -697,41 +743,43 @@
 
   peer->stats_handle = ca_result;
 
-  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, "mesh", NULL,
+  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, NULL, NULL,
                                     GNUNET_TIME_UNIT_FOREVER_REL,
-                                    NULL,
-                                       &stats_iterator, peer))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-               "Could not get mesh statistics of peer %u!\n", peer->id);
-  }
-  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, "regexprofiler", NULL,
-                                    GNUNET_TIME_UNIT_FOREVER_REL,
-                                    NULL,
-                                    &stats_iterator, peer))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-               "Could not get regexprofiler statistics of peer %u!\n", 
peer->id);
-  }
-  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, "transport", NULL,
-                                    GNUNET_TIME_UNIT_FOREVER_REL,
-                                    NULL,
-                                    &stats_iterator, peer))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-               "Could not get transport statistics of peer %u!\n", peer->id);
-  }
-  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, "dht", NULL,
-                                    GNUNET_TIME_UNIT_FOREVER_REL,
                                     &stats_cb,
                                     &stats_iterator, peer))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-               "Could not get dht statistics of peer %u!\n", peer->id);
+               "Could not get statistics of peer %u!\n", peer->id);
   }
 }
 
 
+/**
+ * Task to collect all statistics from all peers, will shutdown the
+ * profiler, when done.
+ *
+ * @param cls NULL
+ * @param tc the task context
+ */
+static void
+do_collect_stats (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct RegexPeer *peer = &peers[0];
+
+  GNUNET_assert (NULL != peer->peer_handle);
+
+  peer->stats_op_handle =
+    GNUNET_TESTBED_service_connect (NULL,
+                                   peer->peer_handle,
+                                   "statistics",
+                                   &stats_connect_cb,
+                                   peer,
+                                   &stats_ca,
+                                   &stats_da,
+                                   peer);
+}
+
+
 
/******************************************************************************/
 /************************  MESH SERVICE CONNECTIONS  
**************************/
 
/******************************************************************************/
@@ -831,16 +879,6 @@
     }
   }
 
-  peer->stats_op_handle =
-    GNUNET_TESTBED_service_connect (NULL,
-                                   peer->peer_handle,
-                                   "statistics",
-                                   &stats_connect_cb,
-                                   peer,
-                                   &stats_ca,
-                                   &stats_da,
-                                   peer);
-
   GNUNET_TESTBED_operation_done (peer->mesh_op_handle);
   peer->mesh_op_handle = NULL;
 
@@ -850,11 +888,14 @@
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "All strings successfully matched in %s\n",
                 GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO));
-    printf ("All strings successfully matched. Shutting down.\n");
+    printf ("All strings successfully matched.\n");
     fflush (stdout);
 
     if (GNUNET_SCHEDULER_NO_TASK != search_timeout_task)
       GNUNET_SCHEDULER_cancel (search_timeout_task);
+
+    printf ("Collecting stats and shutting down.\n");
+    GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL);
   }
 }
 
@@ -876,11 +917,11 @@
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Found %i of %i strings\n", peers_found, num_search_strings);
 
-  printf ("Search timed out after %s. Shutting down.\n", 
+  printf ("Search timed out after %s. Collecting stats and shutting down.\n", 
          GNUNET_STRINGS_relative_time_to_string (search_timeout, GNUNET_NO));
   fflush (stdout);
 
-  shutdown_task = GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
+  GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL);
 }
 
 
@@ -1633,7 +1674,7 @@
     GNUNET_asprintf (&(*strings)[i], "%s%s", regex_prefix, &data[offset]);
     offset += strlen (&data[offset]) + 1;
   }
-  free (data);
+  GNUNET_free (data);
   return str_cnt;
 }
 

Modified: gnunet/src/mesh/gnunet-service-mesh.c
===================================================================
--- gnunet/src/mesh/gnunet-service-mesh.c       2012-11-11 07:44:00 UTC (rev 
24899)
+++ gnunet/src/mesh/gnunet-service-mesh.c       2012-11-11 14:23:35 UTC (rev 
24900)
@@ -182,6 +182,8 @@
      */
   char *regex;
 
+  struct GNUNET_REGEX_Automaton *dfa;
+
     /**
      * How many characters per edge can we squeeze?
      */
@@ -1651,18 +1653,18 @@
  * @param regex The regular expresion.
  */
 static void
-regex_put (const struct MeshRegexDescriptor *regex)
+regex_put (struct MeshRegexDescriptor *regex)
 {
-  struct GNUNET_REGEX_Automaton *dfa;
+  if (NULL == regex->dfa)
+  {
+    regex->dfa = GNUNET_REGEX_construct_dfa (regex->regex,
+                                            strlen (regex->regex),
+                                            regex->compression);
+  }
 
   DEBUG_DHT ("  regex_put (%s) start\n", regex->regex);
-  dfa = GNUNET_REGEX_construct_dfa (regex->regex,
-                                    strlen(regex->regex),
-                                    regex->compression);
-  GNUNET_REGEX_iterate_all_edges (dfa, &regex_iterator, NULL);
-  GNUNET_REGEX_automaton_destroy (dfa);
+  GNUNET_REGEX_iterate_all_edges (regex->dfa, &regex_iterator, NULL);
   DEBUG_DHT ("  regex_put (%s) end\n", regex);
-
 }
 
 /**
@@ -6662,6 +6664,8 @@
     for (i = 0; i < c->n_regex; i++)
     {
       GNUNET_free (c->regexes[i].regex);
+      if (NULL != c->regexes[i].dfa)
+       GNUNET_REGEX_automaton_destroy (c->regexes[i].dfa);
     }
     GNUNET_free_non_null (c->regexes);
     if (GNUNET_SCHEDULER_NO_TASK != c->regex_announce_task)
@@ -6813,6 +6817,7 @@
   regex[len] = '\0';
   rd.regex = regex;
   rd.compression = ntohs (msg->compression_characters);
+  rd.dfa = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  length %u\n", len);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  regex %s\n", regex);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  cm %u\n", ntohs(rd.compression));