[GNUnet-SVN] r9972 - in gnunet: . src/hostlist

[gnunet]

Author: grothoff
Date: 2010-01-11 00:40:03 +0100 (Mon, 11 Jan 2010)
New Revision: 9972

Modified:
   gnunet/TODO
   gnunet/src/hostlist/Makefile.am
   gnunet/src/hostlist/gnunet-daemon-hostlist.c
   gnunet/src/hostlist/hostlist-client.c
   gnunet/src/hostlist/hostlist-server.c
   gnunet/src/hostlist/test_gnunet_daemon_hostlist.c
   gnunet/src/hostlist/test_gnunet_daemon_hostlist_peer1.conf
   gnunet/src/hostlist/test_gnunet_daemon_hostlist_peer2.conf
Log:
working on hostlist

Modified: gnunet/TODO
===================================================================
--- gnunet/TODO 2010-01-10 22:30:32 UTC (rev 9971)
+++ gnunet/TODO 2010-01-10 23:40:03 UTC (rev 9972)
@@ -15,6 +15,8 @@
 * TEST:
   - topology
   - hostlist
+    * is 'update_response' / 'finish_response' etc ever called? 
(hostlist-server)
+    * we still have serious shutdown problems in the testcase (on failure)
 * CORE:
   - soliciting traffic for clients that registered for it is not implemented
     (in the service, client API supports 
GNUNET_MESSAGE_TYPE_CORE_SOLICIT_TRAFFIC
@@ -38,6 +40,13 @@
   - load: need way to determine network load (up/down)
   - trust: need *fast* way to check/update trust in peers
            (async peerinfo would not be right)
+  - OS: existing waitpid call is not nice (not integratable with scheduler! 
fix this!)
+* STATISTICS:
+  - synchronous/asynchronous API (& implementation) is not nice
+  - does not seem to work with timeouts (especially if service is not running)
+* ARM:
+  - need to get rid of synchronous API for service starts (cause all kinds of 
problems)
+    [=> eliminate for need to tell ARM about service starts most of the time!]
 * Module features to implement:
   - advanced FS API parts
     + namespaces: fundamental namespace API

Modified: gnunet/src/hostlist/Makefile.am
===================================================================
--- gnunet/src/hostlist/Makefile.am     2010-01-10 22:30:32 UTC (rev 9971)
+++ gnunet/src/hostlist/Makefile.am     2010-01-10 23:40:03 UTC (rev 9972)
@@ -27,12 +27,10 @@
 gnunet_daemon_hostlist_CPPFLAGS = \
  @LIBCURL_CPPFLAGS@ 
 
-
-
 check_PROGRAMS = \
  test_gnunet_daemon_hostlist
 
-# TESTS = $(check_PROGRAMS) $(check_SCRIPTS)
+# TESTS = $(check_PROGRAMS)
 
 test_gnunet_daemon_hostlist_SOURCES = \
  test_gnunet_daemon_hostlist.c

Modified: gnunet/src/hostlist/gnunet-daemon-hostlist.c
===================================================================
--- gnunet/src/hostlist/gnunet-daemon-hostlist.c        2010-01-10 22:30:32 UTC 
(rev 9971)
+++ gnunet/src/hostlist/gnunet-daemon-hostlist.c        2010-01-10 23:40:03 UTC 
(rev 9972)
@@ -118,13 +118,21 @@
 static void
 cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
+  if (bootstrapping)
+    {
+      GNUNET_HOSTLIST_client_stop ();
+    }
+  if (provide_hostlist)
+    {      
+      GNUNET_HOSTLIST_server_stop ();
+    }
+  GNUNET_CORE_disconnect (core);
   if (stats != NULL)
     {
       GNUNET_STATISTICS_destroy (stats,
                                 GNUNET_NO);
       stats = NULL;
     }
-  GNUNET_CORE_disconnect (core);
 }
 
 

Modified: gnunet/src/hostlist/hostlist-client.c
===================================================================
--- gnunet/src/hostlist/hostlist-client.c       2010-01-10 22:30:32 UTC (rev 
9971)
+++ gnunet/src/hostlist/hostlist-client.c       2010-01-10 23:40:03 UTC (rev 
9972)
@@ -132,7 +132,9 @@
 
   total = size * nmemb;
   if ( (total == 0) || (bogus_url) )
-    return total;  /* ok, no data or bogus data */
+    {
+      return total;  /* ok, no data or bogus data */
+    }
   left = total;
   while (left > 0)
     {
@@ -161,6 +163,9 @@
        break;
       if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message*)msg) == msize)
        {
+         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                     "Received valid `%s' message from hostlist server.\n",
+                     "HELLO");
          GNUNET_TRANSPORT_offer_hello (transport, msg);
        }
       else
@@ -318,6 +323,11 @@
   struct CURLMsg *msg;
   CURLMcode mret;
 
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    {
+      clean_up ();
+      return;
+    }
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_TIMEOUT))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
@@ -365,6 +375,9 @@
       clean_up ();
       return;
     }
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+             _("Download of hostlist `%s' completed.\n"),
+             current_url);
   clean_up ();
 }
 
@@ -521,6 +534,7 @@
 check_task (void *cls,
            const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
+  
   if (connection_count < MIN_CONNECTIONS)
     download_hostlist ();
   else
@@ -597,6 +611,8 @@
 static void
 primary_task (void *cls, int success)
 {
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Statistics request done, scheduling hostlist download\n");
   schedule_hostlist_task ();
 }
 
@@ -608,6 +624,9 @@
              uint64_t value,
              int is_persistent)
 {
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+             _("Initial time between hostlist downloads is %llums\n"),
+             (unsigned long long) value);
   hostlist_delay.value = value;
   return GNUNET_OK;
 }
@@ -656,8 +675,9 @@
 void
 GNUNET_HOSTLIST_client_stop ()
 {
-  GNUNET_SCHEDULER_cancel (sched,
-                          current_task);
+  if (current_task != GNUNET_SCHEDULER_NO_TASK)
+    GNUNET_SCHEDULER_cancel (sched,
+                            current_task);
   GNUNET_free_non_null (proxy);
   proxy = NULL;
   if (sched != NULL)

Modified: gnunet/src/hostlist/hostlist-server.c
===================================================================
--- gnunet/src/hostlist/hostlist-server.c       2010-01-10 22:30:32 UTC (rev 
9971)
+++ gnunet/src/hostlist/hostlist-server.c       2010-01-10 23:40:03 UTC (rev 
9972)
@@ -144,6 +144,9 @@
                           GNUNET_TIME_UNIT_MINUTES,
                           &host_processor,
                           results);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+             _("Created new hostlist response with %u bytes\n"),
+             results->size);  
 }
 
 
@@ -174,6 +177,8 @@
   
   if (0 != strcmp (method, MHD_HTTP_METHOD_GET))
     return MHD_NO;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+             _("Received request for our hostlist\n"));
   if (NULL == *con_cls)
     {
       (*con_cls) = &dummy;
@@ -282,6 +287,9 @@
                                                   "HTTPPORT", 
                                                   &port))
     return GNUNET_SYSERR;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+             _("Hostlist service starts on port %llu\n"),
+             port);
   daemon_handle = MHD_start_daemon (MHD_USE_IPv6,
                                     (unsigned short) port,
                                     &accept_policy_callback,

Modified: gnunet/src/hostlist/test_gnunet_daemon_hostlist.c
===================================================================
--- gnunet/src/hostlist/test_gnunet_daemon_hostlist.c   2010-01-10 22:30:32 UTC 
(rev 9971)
+++ gnunet/src/hostlist/test_gnunet_daemon_hostlist.c   2010-01-10 23:40:03 UTC 
(rev 9972)
@@ -40,12 +40,15 @@
 static int ok;
 
 static struct GNUNET_SCHEDULER_Handle *sched;
+
+static GNUNET_SCHEDULER_TaskIdentifier timeout_task;
     
 struct PeerContext
 {
   struct GNUNET_CONFIGURATION_Handle *cfg;
   struct GNUNET_TRANSPORT_Handle *th;
   struct GNUNET_MessageHeader *hello;
+  struct GNUNET_ARM_Handle *arm;
 #if START_ARM
   pid_t arm_pid;
 #endif
@@ -57,7 +60,53 @@
 
 
 
+/**
+ * Timeout, give up.
+ */
 static void
+timeout_error (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+             "Timeout trying to connect peers, test failed.\n");
+  GNUNET_TRANSPORT_disconnect (p1.th);
+  p1.th = NULL;
+  GNUNET_TRANSPORT_disconnect (p2.th);
+  p2.th = NULL;
+  GNUNET_SCHEDULER_shutdown (sched);
+}
+
+
+/**
+ * Function called to notify transport users that another
+ * peer connected to us.
+ *
+ * @param cls closure
+ * @param peer the peer that connected
+ * @param latency current latency of the connection
+ */
+static void
+notify_connect (void *cls,
+               const struct GNUNET_PeerIdentity * peer,
+               struct GNUNET_TIME_Relative latency)
+{
+  if (peer == NULL)
+    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Peers connected, shutting down.\n");
+  GNUNET_assert (ok == 4);
+  ok = 0;
+
+  GNUNET_SCHEDULER_cancel (sched,
+                          timeout_task);
+  GNUNET_TRANSPORT_disconnect (p1.th);
+  p1.th = NULL;
+  GNUNET_TRANSPORT_disconnect (p2.th);
+  p2.th = NULL;
+}
+
+
+static void
 process_hello (void *cls,
                struct GNUNET_TIME_Relative latency,
                const struct GNUNET_PeerIdentity *peer,
@@ -65,22 +114,13 @@
 {
   struct PeerContext *p = cls;
 
-  GNUNET_assert (peer != NULL);
+  if (message == NULL)
+    return;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received (my) `%s' from transport service of `%4s'\n",
-              "HELLO", GNUNET_i2s (peer));
-  GNUNET_assert (message != NULL);
-  p->hello = GNUNET_malloc (ntohs (message->size));
-  memcpy (p->hello, message, ntohs (message->size));
-  if ((p == &p1) && (p2.th != NULL))
-    GNUNET_TRANSPORT_offer_hello (p2.th, message);
-  if ((p == &p2) && (p1.th != NULL))
-    GNUNET_TRANSPORT_offer_hello (p1.th, message);
-
-  if ((p == &p1) && (p2.hello != NULL))
-    GNUNET_TRANSPORT_offer_hello (p1.th, p2.hello);
-  if ((p == &p2) && (p1.hello != NULL))
-    GNUNET_TRANSPORT_offer_hello (p2.th, p1.hello);
+             "Received HELLO, starting hostlist service.\n");
+  GNUNET_assert ( (ok >= 2) && (ok <= 3) );
+  ok++;
+  GNUNET_ARM_start_services (p->cfg, sched, "hostlist", NULL);
 }
 
 
@@ -98,13 +138,75 @@
 #endif
   GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname));
   GNUNET_ARM_start_services (p->cfg, sched, "core", NULL);
-  p->th = GNUNET_TRANSPORT_connect (sched, p->cfg, p, NULL, NULL, NULL);
+  p->th = GNUNET_TRANSPORT_connect (sched, p->cfg, p, NULL, 
+                                   &notify_connect, NULL);
   GNUNET_assert (p->th != NULL);
   GNUNET_TRANSPORT_get_hello (p->th, TIMEOUT, &process_hello, p);
 }
 
 
 static void
+waitpid_task (void *cls, 
+             const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerContext *p = cls;
+
+#if START_ARM 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Killing ARM process.\n");
+  if (0 != PLIBC_KILL (p->arm_pid, SIGTERM))
+    GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
+  if (GNUNET_OS_process_wait(p->arm_pid) != GNUNET_OK)
+    GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid");
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "ARM process %u stopped\n", p->arm_pid);
+#endif
+  GNUNET_CONFIGURATION_destroy (p->cfg);
+}
+
+
+static void
+stop_cb (void *cls, 
+        int success)
+{
+  struct PeerContext *p = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             success 
+             ? "ARM stopped core service\n" 
+             : "ARM failed to stop core service\n");
+  GNUNET_ARM_disconnect (p->arm);
+  p->arm = NULL;
+  /* make sure this runs after all other tasks are done */
+  GNUNET_SCHEDULER_add_delayed (sched,
+                               GNUNET_TIME_UNIT_SECONDS,
+                               &waitpid_task, p);
+}
+
+
+static void
+stop_arm (struct PeerContext *p)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Asking ARM to stop core service\n");
+  p->arm = GNUNET_ARM_connect (p->cfg, sched, NULL);
+  GNUNET_ARM_stop_service (p->arm, "core", GNUNET_TIME_UNIT_SECONDS,
+                          &stop_cb, p);
+}
+
+
+/**
+ * Try again to connect to transport service.
+ */
+static void
+shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  stop_arm (&p1);
+  stop_arm (&p2);
+}
+
+
+static void
 run (void *cls,
      struct GNUNET_SCHEDULER_Handle *s,
      char *const *args,
@@ -114,27 +216,20 @@
   GNUNET_assert (ok == 1);
   ok++;
   sched = s;
+  timeout_task = GNUNET_SCHEDULER_add_delayed (sched,
+                                              GNUNET_TIME_relative_multiply 
(GNUNET_TIME_UNIT_SECONDS,
+                                                                             
15),
+                                              &timeout_error,
+                                              NULL);
+  GNUNET_SCHEDULER_add_delayed (sched,
+                               GNUNET_TIME_UNIT_FOREVER_REL,
+                               &shutdown_task,
+                               NULL);
   setup_peer (&p1, "test_gnunet_daemon_hostlist_peer1.conf");
   setup_peer (&p2, "test_gnunet_daemon_hostlist_peer2.conf");
 }
 
 
-static void
-stop_arm (struct PeerContext *p)
-{
-  GNUNET_ARM_stop_services (p->cfg, sched, "core", NULL);
-#if START_ARM
-  if (0 != PLIBC_KILL (p->arm_pid, SIGTERM))
-    GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill");
-  if (GNUNET_OS_process_wait(p->arm_pid) != GNUNET_OK)
-    GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid");
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "ARM process %u stopped\n", p->arm_pid);
-#endif
-  GNUNET_CONFIGURATION_destroy (p->cfg);
-}
-
-
 static int
 check ()
 {
@@ -152,8 +247,6 @@
   GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
                       argv, "test-gnunet-daemon-hostlist",
                      "nohelp", options, &run, &ok);
-  stop_arm (&p1);
-  stop_arm (&p2);
   return ok;
 }
 

Modified: gnunet/src/hostlist/test_gnunet_daemon_hostlist_peer1.conf
===================================================================
--- gnunet/src/hostlist/test_gnunet_daemon_hostlist_peer1.conf  2010-01-10 
22:30:32 UTC (rev 9971)
+++ gnunet/src/hostlist/test_gnunet_daemon_hostlist_peer1.conf  2010-01-10 
23:40:03 UTC (rev 9972)
@@ -12,6 +12,7 @@
 
 [arm]
 PORT = 12966
+DEFAULTSERVICES = resolver transport core statistics
 
 [statistics]
 PORT = 12967
@@ -27,3 +28,8 @@
 
 [testing]
 WEAKRANDOM = YES
+
+[hostlist]
+HTTPPORT = 12980
+SERVERS = http://localhost:12981/
+OPTIONS = -b -p -L DEBUG
\ No newline at end of file

Modified: gnunet/src/hostlist/test_gnunet_daemon_hostlist_peer2.conf
===================================================================
--- gnunet/src/hostlist/test_gnunet_daemon_hostlist_peer2.conf  2010-01-10 
22:30:32 UTC (rev 9971)
+++ gnunet/src/hostlist/test_gnunet_daemon_hostlist_peer2.conf  2010-01-10 
23:40:03 UTC (rev 9972)
@@ -12,6 +12,7 @@
 
 [arm]
 PORT = 22966
+DEFAULTSERVICES = resolver transport core statistics
 
 [statistics]
 PORT = 22967
@@ -27,3 +28,8 @@
 
 [testing]
 WEAKRANDOM = YES
+
+[hostlist]
+HTTPPORT = 12981
+SERVERS = http://localhost:12980/
+OPTIONS = -b -p -L DEBUG