[GNUnet-SVN] r3904 - in GNUnet/src: applications/datastore applications/gap applications/pingpong server transports util/os

[grothoff]

Author: grothoff
Date: 2006-12-09 20:51:14 -0800 (Sat, 09 Dec 2006)
New Revision: 3904

Modified:
   GNUnet/src/applications/datastore/prefetch.c
   GNUnet/src/applications/gap/gap.h
   GNUnet/src/applications/pingpong/pingpong.c
   GNUnet/src/server/connection.c
   GNUnet/src/transports/tcp.c
   GNUnet/src/util/os/cpustatus.c
   GNUnet/src/util/os/statuscalls.c
   GNUnet/src/util/os/statuscallstest.c
Log:
tuning

Modified: GNUnet/src/applications/datastore/prefetch.c
===================================================================
--- GNUnet/src/applications/datastore/prefetch.c        2006-12-10 03:47:17 UTC 
(rev 3903)
+++ GNUnet/src/applications/datastore/prefetch.c        2006-12-10 04:51:14 UTC 
(rev 3904)
@@ -140,8 +140,7 @@
   if (load < 10)
     load = 10;    /* never sleep less than 500 ms */
   if (load > 100)
-    load = 100;   /* never sleep longer than 5 seconds since that
-                    might show up badly in the shutdown sequence... */
+    load = 100;   /* never sleep longer than 5 seconds */
   if (doneSignal)
     return SYSERR;
   /* the higher the load, the longer the sleep */
@@ -166,8 +165,7 @@
     if (load < 10)
       load = 10;    /* never sleep less than 500 ms */
     if (load > 100)
-      load = 100;   /* never sleep longer than 5 seconds since that
-                      might show up badly in the shutdown sequence... */
+      load = 100;   /* never sleep longer than 5 seconds */
     PTHREAD_SLEEP(50 * cronMILLIS * load);
   }
   return NULL;

Modified: GNUnet/src/applications/gap/gap.h
===================================================================
--- GNUnet/src/applications/gap/gap.h   2006-12-10 03:47:17 UTC (rev 3903)
+++ GNUnet/src/applications/gap/gap.h   2006-12-10 04:51:14 UTC (rev 3904)
@@ -46,7 +46,8 @@
 
 /**
  * Until which load do we consider the peer idle and do not
- * charge at all?
+ * charge at all? (should be larger than IDLE_LOAD_THRESHOLD used
+ * by server/connection.c!)
  */
 #define IDLE_LOAD_THRESHOLD 75
 

Modified: GNUnet/src/applications/pingpong/pingpong.c
===================================================================
--- GNUnet/src/applications/pingpong/pingpong.c 2006-12-10 03:47:17 UTC (rev 
3903)
+++ GNUnet/src/applications/pingpong/pingpong.c 2006-12-10 04:51:14 UTC (rev 
3904)
@@ -496,7 +496,7 @@
         0,
         sizeof(PingPongEntry)*MAX_PING_PONG);
   GE_LOG(ectx,
-        GE_INFO | GE_USER | GE_BULK,
+        GE_DEBUG | GE_USER | GE_REQUEST,
         _("`%s' registering handlers %d %d (plaintext and ciphertext)\n"),
         "pingpong",
         p2p_PROTO_PING,

Modified: GNUnet/src/server/connection.c
===================================================================
--- GNUnet/src/server/connection.c      2006-12-10 03:47:17 UTC (rev 3903)
+++ GNUnet/src/server/connection.c      2006-12-10 04:51:14 UTC (rev 3904)
@@ -64,10 +64,22 @@
 
 #define DEBUG_CONNECTION NO
 
-/* output knapsack priorities into a file? */
+/**
+ * output knapsack priorities into a file? 
+ */
 #define DEBUG_COLLECT_PRIO NO
 
 /**
+ * Until which load do we consider the peer overly idle
+ * (which means that we would like to use more resources).<p>
+ *
+ * Note that we use 50 to leave some room for applications
+ * to consume resources "idly" (i.e. up to 75%) and then
+ * still have some room for "paid for" resource consumption.
+ */
+#define IDLE_LOAD_THRESHOLD 50
+
+/**
  * If an attempt to establish a connection is not answered
  * within 150s, drop.
  */
@@ -406,16 +418,6 @@
   cron_t lastSendAttempt;
 
   /**
-   * How frequent (per connection!) may we attempt to solve the knapsack
-   * problem and send a message out? Note that setting this value higher
-   * reduces the CPU overhead while a lower value can improve thoughput.
-   *
-   * The value is adjusted according to how fast we perceive the CPU
-   * to be (and is also proportional too how much bandwidth we have)...
-   */
-  cron_t MAX_SEND_FREQUENCY;
-
-  /**
    * a hash collision overflow chain 
    */
   struct BufferEntry_ *overflowChain;
@@ -641,7 +643,6 @@
  */
 static BufferEntry * initBufferEntry() {
   BufferEntry * be;
-  int load;
 
   be = MALLOC(sizeof(BufferEntry));
   memset(be, 0, sizeof(BufferEntry));
@@ -658,10 +659,6 @@
   be->idealized_limit = MIN_BPM_PER_PEER;
   be->max_transmitted_limit = MIN_BPM_PER_PEER;
   be->lastSendAttempt = 0;      /* never */
-  load = os_cpu_get_load(ectx, cfg);
-  if (load == -1)
-    load = 50; /* failed to determine load, assume 50% */
-  be->MAX_SEND_FREQUENCY = 50 * cronMILLIS * load;
   be->inSendBuffer = NO;
   be->last_bps_update = get_time(); /* now */
   return be;
@@ -939,27 +936,28 @@
  * @return OK if sending a message now is acceptable
  */
 static int checkSendFrequency(BufferEntry * be) {
-  if(be->max_bpm == 0)
+  cron_t msf;
+
+  if (be->max_bpm == 0)
     be->max_bpm = 1;
 
-  if(be->session.mtu == 0) {
-    be->MAX_SEND_FREQUENCY =    /* ms per message */
+  if (be->session.mtu == 0) {
+    msf =    /* ms per message */
       EXPECTED_MTU / (be->max_bpm * cronMINUTES / cronMILLIS) /* bytes per ms 
*/
       /2;
-  }
-  else {
-    be->MAX_SEND_FREQUENCY =    /* ms per message */
+  } else {
+    msf =    /* ms per message */
       be->session.mtu           /* byte per message */
       / (be->max_bpm * cronMINUTES / cronMILLIS)  /* bytes per ms */
-      /2;                       /* some head-room */
+      / 2;                       /* some head-room */
   }
   /* Also: allow at least 2 * MINIMUM_SAMPLE_COUNT knapsack
      solutions for any MIN_SAMPLE_TIME! */
-  if (be->MAX_SEND_FREQUENCY > 2 * MIN_SAMPLE_TIME / MINIMUM_SAMPLE_COUNT)
-    be->MAX_SEND_FREQUENCY = 2 * MIN_SAMPLE_TIME / MINIMUM_SAMPLE_COUNT;
+  if (msf > 2 * MIN_SAMPLE_TIME / MINIMUM_SAMPLE_COUNT)
+    msf = 2 * MIN_SAMPLE_TIME / MINIMUM_SAMPLE_COUNT;
 
-  if (be->lastSendAttempt + be->MAX_SEND_FREQUENCY > get_time()) {
-#if DEBUG_CONNECTION || 1
+  if (be->lastSendAttempt + msf > get_time()) {
+#if DEBUG_CONNECTION
     GE_LOG(ectx,
           GE_DEBUG | GE_REQUEST | GE_USER,
           "Send frequency too high (CPU load), send deferred.\n");
@@ -1179,7 +1177,7 @@
   if(msgCap < EXPECTED_MTU)
     msgCap = EXPECTED_MTU;
   if (load < 50) {                  /* afford more if CPU load is low */
-    if(load == 0)
+    if (load == 0)
       load = 1; /* avoid division by zero */
     msgCap += (MAX_SEND_BUFFER_SIZE - EXPECTED_MTU) / load;
   }
@@ -2418,13 +2416,14 @@
        total_allowed_sent += root->max_bpm;
        total_allowed_recv += root->idealized_limit;
         if ( (now > root->isAlive) && /* concurrency might make this false... 
*/
-            (now - root->isAlive > SECONDS_INACTIVE_DROP * cronSECONDS)) {
+            (now - root->isAlive > SECONDS_INACTIVE_DROP * cronSECONDS) ) {
           EncName enc;
 
           /* switch state form UP to DOWN: too much inactivity */
           IF_GELOG(ectx,
                   GE_DEBUG | GE_REQUEST | GE_USER,
-                  hash2enc(&root->session.sender.hashPubKey, &enc));
+                  hash2enc(&root->session.sender.hashPubKey, 
+                           &enc));
           GE_LOG(ectx,
                 GE_DEBUG | GE_REQUEST | GE_USER,
                 "closing connection with `%s': "
@@ -2438,8 +2437,9 @@
         if ( (root->available_send_window >= 60000) &&
             (root->sendBufferSize < 4) &&
             (scl_nextHead != NULL) &&
-            (os_network_monitor_get_load(load_monitor, Upload) < 25) &&
-            (os_cpu_get_load(ectx, cfg) < 50) ) {
+            (os_network_monitor_get_load(load_monitor, 
+                                         Upload) < IDLE_LOAD_THRESHOLD) &&
+            (os_cpu_get_load(ectx, cfg) < IDLE_LOAD_THRESHOLD) ) {
           /* create some traffic by force! */
           char * msgBuf;
           unsigned int mSize;
@@ -2524,8 +2524,10 @@
   GE_ASSERT(ectx, sender != NULL);
   hash2enc(&sender->hashPubKey, &enc);
   if(size < sizeof(P2P_PACKET_HEADER)) {
-    GE_LOG(ectx, GE_WARNING | GE_BULK | GE_USER,
-        _("Message from `%s' discarded: invalid format.\n"), &enc);
+    GE_LOG(ectx, 
+          GE_WARNING | GE_BULK | GE_USER,
+          _("Message from `%s' discarded: invalid format.\n"), 
+          &enc);
     return SYSERR;
   }
   hash2enc(&sender->hashPubKey, &enc);
@@ -2540,7 +2542,10 @@
     stats->change(stat_received, size);
 
 #if DEBUG_CONNECTION
-  GE_LOG(ectx, GE_DEBUG | GE_REQUEST | GE_USER, "Decrypting message from host 
`%s'\n", &enc);
+  GE_LOG(ectx, 
+        GE_DEBUG | GE_REQUEST | GE_USER, 
+        "Decrypting message from host `%s'\n", 
+        &enc);
 #endif
   MUTEX_LOCK(lock);
   be = lookForHost(sender);
@@ -2565,9 +2570,10 @@
                      tmp);
   hash(tmp, size - sizeof(HashCode512), &hc);
   if(!((res != OK) && equalsHashCode512(&hc, &msg->hash))) {
-    GE_LOG(ectx, GE_INFO | GE_BULK | GE_USER,
-        "Decrypting message from host `%s' failed, wrong sessionkey!\n",
-        &enc);
+    GE_LOG(ectx, 
+          GE_INFO | GE_BULK | GE_USER,
+          "Decrypting message from host `%s' failed, wrong sessionkey!\n",
+          &enc);
 #if DEBUG_CONNECTION
     printMsg("Wrong sessionkey", sender,
              &be->skey_remote, (const INITVECTOR *) &msg->hash,
@@ -2596,10 +2602,11 @@
       }
     }
     if(res == SYSERR) {
-      GE_LOG(ectx, GE_WARNING | GE_REQUEST | GE_USER,
-          _("Invalid sequence number"
-            " %u <= %u, dropping message.\n"),
-          sequenceNumber, be->lastSequenceNumberReceived);
+      GE_LOG(ectx,
+            GE_WARNING | GE_REQUEST | GE_USER,
+            _("Invalid sequence number"
+              " %u <= %u, dropping message.\n"),
+            sequenceNumber, be->lastSequenceNumberReceived);
       MUTEX_UNLOCK(lock);
       return SYSERR;
     }
@@ -3447,12 +3454,9 @@
   MUTEX_LOCK(lock);
   be = lookForHost(hi);
   MUTEX_UNLOCK(lock);
-  if(be == NULL) {
+  if (be == NULL) 
     return NO;
-  }
-  else {
-    return (be->status == STAT_UP);
-  }
+  return (be->status == STAT_UP);  
 }
 
 /**
@@ -3464,7 +3468,8 @@
 unsigned int computeIndex(const PeerIdentity * hostId) {
   unsigned int res = (((unsigned int) hostId->hashPubKey.bits[0]) &
                       ((unsigned int) (CONNECTION_MAX_HOSTS_ - 1)));
-  GE_ASSERT(ectx, res < CONNECTION_MAX_HOSTS_);
+  GE_ASSERT(ectx, 
+           res < CONNECTION_MAX_HOSTS_);
   return res;
 }
 
@@ -3484,12 +3489,12 @@
   ENTRY();
   MUTEX_LOCK(lock);
   be = lookForHost(node);
-  if((be != NULL) && (be->status == STAT_UP)) {
+  if ( (be != NULL) &&
+       (be->status == STAT_UP) ) {
     ret = be->idealized_limit;
     if(ret == 0)
       ret = 1;
-  }
-  else {
+  } else {
     ret = 0;
   }
   MUTEX_UNLOCK(lock);

Modified: GNUnet/src/transports/tcp.c
===================================================================
--- GNUnet/src/transports/tcp.c 2006-12-10 03:47:17 UTC (rev 3903)
+++ GNUnet/src/transports/tcp.c 2006-12-10 04:51:14 UTC (rev 3904)
@@ -184,7 +184,7 @@
     return NULL;
   }
   GE_LOG(ectx,
-        GE_DEBUG | GE_USER | GE_BULK,
+        GE_DEBUG | GE_USER | GE_REQUEST,
         "TCP uses IP address %u.%u.%u.%u.\n",
         PRIP(ntohl(*(int*)&haddr->ip)));
   haddr->port = htons(port);

Modified: GNUnet/src/util/os/cpustatus.c
===================================================================
--- GNUnet/src/util/os/cpustatus.c      2006-12-10 03:47:17 UTC (rev 3903)
+++ GNUnet/src/util/os/cpustatus.c      2006-12-10 04:51:14 UTC (rev 3904)
@@ -363,7 +363,7 @@
 int os_cpu_get_load(struct GE_Context * ectx,
                    struct GC_Configuration * cfg) {
   static int currentLoad;
-  static int lastRet = -1;
+  static int agedLoad = -1;
   static cron_t lastCall;
   unsigned long long maxCPULoad;
   int ret;
@@ -378,31 +378,31 @@
                                              &maxCPULoad))
     return -1;
   MUTEX_LOCK(statusMutex);
-  ret = (100 * currentLoad) / maxCPULoad;
   now = get_time();
-  if ( (lastRet != -1) &&
-       (now - lastCall < 250 * cronMILLIS) ) {
+  if ( (agedLoad == -1) ||
+       (now - lastCall > 500 * cronMILLIS) ) {
     /* use smoothing, but do NOT update lastRet at frequencies higher
-       than 250ms; this makes the smoothing (mostly) independent from
-       the frequency at which getCPULoad is called. */
-    ret = (ret + 7 * lastRet)/8;
-    MUTEX_UNLOCK(statusMutex);
-    return ret;
-  }
-  currentLoad = updateCpuUsage();
-  if (currentLoad == -1) {
-    lastRet = -1;
-    MUTEX_UNLOCK(statusMutex);
-    return -1;
-  }
-
-  ret = (100 * currentLoad) / maxCPULoad;
-  /* for CPU, we don't do the 'fast increase' since CPU is much
-     more jitterish to begin with */
-  if (lastRet != -1)
-    ret = (ret + 7 * lastRet) / 8;
-  lastRet = ret;
-  lastCall = now;
+       than 500ms; this makes the smoothing (mostly) independent from
+       the frequency at which getCPULoad is called (and we don't spend
+       more time measuring CPU than actually computing something). */
+    currentLoad = updateCpuUsage();    
+    lastCall = now;
+    if (currentLoad == -1) {
+      agedLoad = -1;
+    } else {
+      if (agedLoad == -1) {
+       agedLoad = currentLoad;
+      } else {
+       /* for CPU, we don't do the 'fast increase' since CPU is much
+          more jitterish to begin with */
+       agedLoad = (agedLoad * 31 + currentLoad) / 32; 
+      }
+    }
+  }  
+  if (agedLoad == -1)
+    ret = -1;
+  else
+    ret = (100 * agedLoad) / maxCPULoad;
   MUTEX_UNLOCK(statusMutex);
   return ret;
 }

Modified: GNUnet/src/util/os/statuscalls.c
===================================================================
--- GNUnet/src/util/os/statuscalls.c    2006-12-10 03:47:17 UTC (rev 3903)
+++ GNUnet/src/util/os/statuscalls.c    2006-12-10 04:51:14 UTC (rev 3904)
@@ -376,6 +376,8 @@
   return -1;
 }
 
+#define INCREMENTAL_INTERVAL (60 * cronSECONDS)
+
 /**
  * Get the load of the network relative to what is allowed.
  * @return the network load as a percentage of allowed
@@ -433,10 +435,10 @@
   }
 
   maxExpect = (now - di->lastCall) * di->max / cronSECONDS;
-  if (now - di->lastCall < 5 * cronSECONDS) {
+  if (now - di->lastCall < INCREMENTAL_INTERVAL) {
     /* return weighted average between last return value and
        load in the last interval */
-    weight = (now - di->lastCall) * 100 / (5 * cronSECONDS); /* how close are 
we to lastCall? */
+    weight = (now - di->lastCall) * 100 / INCREMENTAL_INTERVAL; /* how close 
are we to lastCall? */
     if (maxExpect == 0)
       ret = di->lastValue;
     else

Modified: GNUnet/src/util/os/statuscallstest.c
===================================================================
--- GNUnet/src/util/os/statuscallstest.c        2006-12-10 03:47:17 UTC (rev 
3903)
+++ GNUnet/src/util/os/statuscallstest.c        2006-12-10 04:51:14 UTC (rev 
3904)
@@ -49,7 +49,7 @@
   start = get_time();
   ret = os_cpu_get_load(ectx,
                        cfg);
-  while (start + 12 * cronSECONDS > get_time())
+  while (start + 60 * cronSECONDS > get_time())
     sqrt(245.2523); /* do some processing to drive load up */
   if (ret > os_cpu_get_load(ectx,
                            cfg)) {