[GNUnet-SVN] r299 - GNUnet/src/applications/sqstore_sqlite

[grothoff]

Author: grothoff
Date: 2005-02-24 20:29:47 -0800 (Thu, 24 Feb 2005)
New Revision: 299

Modified:
   GNUnet/src/applications/sqstore_sqlite/sqlite.c
   GNUnet/src/applications/sqstore_sqlite/sqlitetest.c
Log:
better, but still not quite there

Modified: GNUnet/src/applications/sqstore_sqlite/sqlite.c
===================================================================
--- GNUnet/src/applications/sqstore_sqlite/sqlite.c     2005-02-25 02:21:58 UTC 
(rev 298)
+++ GNUnet/src/applications/sqstore_sqlite/sqlite.c     2005-02-25 04:29:47 UTC 
(rev 299)
@@ -64,11 +64,7 @@
   unsigned int lastSync;
   
   /** Precompiled SQL */
-  sqlite3_stmt *exists, *countContent, *updPrio, *insertContent, *migr;
-  
-  /** Last migrated block */
-  cron_t lastExp;
-  unsigned int lastPrio;
+  sqlite3_stmt *exists, *countContent, *updPrio, *insertContent;
 } sqliteHandle;
 
 static sqliteHandle *dbh;
@@ -164,16 +160,6 @@
 }
 
 /**
- * @brief Suspend migration and unlock table
- */
-static void suspendMigration() {
-  if (dbh->migr) {
-    sqlite3_finalize(dbh->migr);
-    dbh->migr = NULL;
-  }
-}
-
-/**
  * Given a full row from gn070 table 
(size,type,prio,anonLevel,expire,hash,value),
  * assemble it into a Datastore_Datum representation.
  */
@@ -226,8 +212,6 @@
   sqlite3_stmt *stmt;
   double ret = SYSERR;
   
-  suspendMigration();
-
   i = sq_prepare("Select anonLevel from gn070 where hash = ?",
                 &stmt);
   if (i == SQLITE_OK) {
@@ -268,8 +252,6 @@
                   double val) {
   sqlite3_stmt *stmt;
   
-  suspendMigration();
-
   if (sq_prepare("REPLACE into gn070(hash, anonLevel, type) values (?, ?, ?)",
                 &stmt) == SQLITE_OK) {
     sqlite3_bind_text(stmt,
@@ -312,110 +294,149 @@
  *        Use 0 for any type.
  * @param callback the callback method
  * @param data second argument to all callback calls
- * @param sort 0 to order by expiration, 1 to order by prio
+ * @param sortByPriority 0 to order by expiration, 1 to order by prio
  * @return the number of items stored in the content database
  */
 static int sqlite_iterate(unsigned int type, 
                          Datum_Iterator iter,
-                         void *closure, 
-                         int sort) {
-       
-  sqlite3_stmt *stmt;
-  int count = 0;
-  char scratch[200], *ptr;
+                         void * closure, 
+                         int sortByPriority) { 
+  sqlite3_stmt * stmt;
+  int count;
+  char scratch[512];
   Datastore_Datum * datum;
-  int bind;
+  unsigned int lastPrio;
+  unsigned long long lastExp;
+  unsigned long hashLen;
+  char * lastHash;
+  HashCode160 key;
 
 #if DEBUG_SQLITE
   LOG(LOG_DEBUG, "SQLite: iterating through the database\n");
 #endif 
 
   MUTEX_LOCK(&dbh->DATABASE_Lock_);
-  
-  if ((sort == 0 && !dbh->migr) || sort != 0) {
-    strcpy(scratch, 
-          "SELECT size, type, prio, anonLevel, expire, hash, value "
-          "FROM gn070 ");
-    ptr = scratch + 67;
-    
-    if (type) {
-      strcpy(ptr, "where type = :1 ");
-      ptr += 16;
-    }    
-    if (sort) {
-      strcpy(ptr, "order by prio ASC");
-    } else {
-      if (!type) {
-        strcpy(ptr, "where ");
-        ptr += 6;
-      }
-      strcpy(ptr, "expire > :2 and prio >= :3 order by expire ASC, prio ASC");
-    }
-  
-    if (sq_prepare(scratch, 
-                  &stmt) != SQLITE_OK) {
-      LOG_SQLITE(LOG_ERROR, "sqlite_query");
-      MUTEX_UNLOCK(&dbh->DATABASE_Lock_);
-      return(SYSERR);
-    }
-    
-    dbh->migr = stmt;
-  } else
-    if (sort == 0)
-      stmt = dbh->migr;
-  
-  bind = 1;
-  
-  if (type)
-    sqlite3_bind_int(stmt, 
-                    bind++,
-                    type);
-    
-  if (!sort) {
-    sqlite3_bind_int(stmt,
-                    bind++,
-                    dbh->lastExp);
-    sqlite3_bind_int(stmt, 
-                    bind, 
-                    dbh->lastPrio);
+ 
+  strcpy(scratch, 
+        "SELECT size, type, prio, anonLevel, expire, hash, value FROM gn070"
+        " WHERE ((hash > :1 AND expire == :2 AND prio == :3) OR ");
+  if (sortByPriority) 
+    strcat(scratch,
+          "(expire > :4 AND prio == :5) OR prio > :6)");
+  else
+    strcat(scratch,
+          "(prio > :4 AND expire == :5) OR expire > :6)");
+  if (type) 
+    strcat(scratch, " AND type = :7");
+  if (sortByPriority) 
+    strcat(scratch, " ORDER BY prio ASC, expire ASC, hash ASC");
+  else 
+    strcat(scratch, " ORDER BY expire ASC, prio ASC, hash ASC");  
+  strcat(scratch, " LIMIT 1");
+  if (sq_prepare(scratch, 
+                &stmt) != SQLITE_OK) {
+    LOG_SQLITE(LOG_ERROR, "sqlite3_prepare");
+    MUTEX_UNLOCK(&dbh->DATABASE_Lock_);
+    return SYSERR;
   }
 
-  while (sqlite3_step(stmt) == SQLITE_ROW) {
-    datum = assembleDatum(stmt);
-    
-    if (datum == NULL) {
-      LOG(LOG_WARNING,
-         _("Invalid data in database.  Please verify integrity!\n"));
-      continue; 
-    }    
-    MUTEX_UNLOCK(&dbh->DATABASE_Lock_);
+  count    = 0;
+  lastPrio = 0;
+  lastExp  = 0x8000000000000000LL; /* MIN long long; sqlite does not know 
about unsigned... */
+  memset(&key, 0, sizeof(HashCode160));
+  lastHash = MALLOC(sizeof(HashCode160)*2 + 2);
+  while (1) {
+    hashLen = sqlite_encode_binary((const char *) &key, 
+                                  sizeof(HashCode160),
+                                  lastHash);
 
-    if( SYSERR == iter(&datum->key, &datum->value, closure) ) {
-      count = SYSERR;
+    sqlite3_bind_blob(stmt,
+                     1,
+                     lastHash,
+                     hashLen,
+                     SQLITE_TRANSIENT);
+    sqlite3_bind_int64(stmt,
+                      2,
+                      lastExp);
+    sqlite3_bind_int(stmt, 
+                    3,
+                    lastPrio);
+    if (sortByPriority) {
+      sqlite3_bind_int(stmt, 
+                      4,
+                      lastPrio);
+      sqlite3_bind_int64(stmt,
+                        5,
+                        lastExp);
+      sqlite3_bind_int(stmt, 
+                      6,
+                      lastPrio);
+    } else {
+      sqlite3_bind_int64(stmt,
+                        4,
+                        lastExp);
+      sqlite3_bind_int(stmt, 
+                      5,
+                      lastPrio);
+      sqlite3_bind_int64(stmt,
+                        6,
+                        lastExp);
+    }
+    if (type)
+      sqlite3_bind_int(stmt, 
+                      7,
+                      type);   
+    if (sqlite3_step(stmt) == SQLITE_ROW) {
+      datum = assembleDatum(stmt);
+      
+      if (datum == NULL) {
+       LOG(LOG_WARNING,
+           _("Invalid data in database.  Please verify integrity!\n"));
+       continue; 
+      }    
+      
+      /*      printf("FOUND %4u prio %4u exp %20llu old: %4u, %20llu\n",
+            (ntohl(datum->value.size) - sizeof(Datastore_Value))/8,
+            ntohl(datum->value.prio),
+            ntohll(datum->value.expirationTime),
+            lastPrio,
+            lastExp);
+      */
+      printf("FOUND %4u: exp %20lld old: %20lld - %d\n",
+            (ntohl(datum->value.size) - sizeof(Datastore_Value))/8,
+            ntohll(datum->value.expirationTime),
+            lastExp,
+            ntohll(datum->value.expirationTime) > lastExp);
+      
+      if (iter != NULL) {
+       MUTEX_UNLOCK(&dbh->DATABASE_Lock_);
+       if (SYSERR == iter(&datum->key, 
+                          &datum->value, 
+                          closure) ) {
+         count = SYSERR;
+         FREE(datum);
+         MUTEX_LOCK(&dbh->DATABASE_Lock_);    
+         break;
+       }    
+       MUTEX_LOCK(&dbh->DATABASE_Lock_);    
+      }
+      key = datum->key;
+      lastPrio = ntohl(datum->value.prio);
+      lastExp  = ntohll(datum->value.expirationTime);
       FREE(datum);
+      count++;
+    } else
       break;
-    }
-    
-    MUTEX_LOCK(&dbh->DATABASE_Lock_);
-    
-    dbh->lastPrio = sqlite3_column_int(stmt, 2);
-    dbh->lastExp = sqlite3_column_int64(stmt, 4);
-    
-    FREE(datum);
-
-    count++;
+    sqlite3_reset(stmt);           
   }
-  
-  if (sort)
-    sqlite3_finalize(stmt);
-    
+  FREE(lastHash);
+  sqlite3_finalize(stmt);    
   MUTEX_UNLOCK(&dbh->DATABASE_Lock_);
   
 #if DEBUG_SQLITE
   LOG(LOG_DEBUG,
       "SQLite: reached end of database\n");
 #endif
-
   return count;
 }
 
@@ -465,7 +486,6 @@
   sqlite3_finalize(dbh->exists);
   sqlite3_finalize(dbh->updPrio);
   sqlite3_finalize(dbh->insertContent);
-  suspendMigration();  
   syncStats();
   if (sqlite3_close(dbh->dbf) != SQLITE_OK)
     LOG_SQLITE(LOG_ERROR, "sqlite_close");
@@ -517,10 +537,8 @@
 #endif 
 
   MUTEX_LOCK(&dbh->DATABASE_Lock_); 
-  suspendMigration();
   
-  strcpy(scratch, "SELECT ");
-  
+  strcpy(scratch, "SELECT "); 
   if (iter == NULL)
     strcat(scratch, "count(*)");
   else
@@ -593,10 +611,10 @@
       } else
        count += sqlite3_column_int(stmt, 0);
     } 
-    FREENONNULL(escapedHash);
     if (ret != SQLITE_DONE) {
       LOG_SQLITE(LOG_ERROR, "sqlite_query");
       sqlite3_finalize(stmt);
+      FREENONNULL(escapedHash);
       MUTEX_UNLOCK(&dbh->DATABASE_Lock_);
       return SYSERR;
     }
@@ -610,6 +628,7 @@
 #if DEBUG_SQLITE
   LOG(LOG_DEBUG, "SQLite: done reading content\n");
 #endif 
+  FREENONNULL(escapedHash);
   
   return count;
 }
@@ -639,10 +658,8 @@
       ntohl(*(int*)&value[1]));
 #endif
 
-  MUTEX_LOCK(&dbh->DATABASE_Lock_);
+  MUTEX_LOCK(&dbh->DATABASE_Lock_);  
   
-  suspendMigration();
-  
   if (dbh->lastSync > 1000)
     syncStats(dbh);
   
@@ -714,7 +731,6 @@
 #endif 
 
   MUTEX_LOCK(&dbh->DATABASE_Lock_); 
-  suspendMigration();
   
   if (dbh->lastSync > 1000)
     syncStats(dbh);
@@ -817,9 +833,7 @@
       "SQLite: update block\n");
 #endif 
 
-  MUTEX_LOCK(&dbh->DATABASE_Lock_);
-  
-  suspendMigration();
+  MUTEX_LOCK(&dbh->DATABASE_Lock_);  
 
   contentSize = ntohl(value->size)-sizeof(Datastore_Value);
   
@@ -847,9 +861,9 @@
   sqlite3_bind_int(dbh->updPrio,
                   4,
                   delta);
-  sqlite3_bind_int64(dbh->updPrio, 
-                    5, 
-                    MAX_PRIORITY);
+  sqlite3_bind_int(dbh->updPrio, 
+                  5, 
+                  MAX_PRIORITY);
   
   n = sqlite3_step(dbh->updPrio);
   sqlite3_reset(dbh->updPrio);
@@ -957,10 +971,6 @@
   
   dbh->payload = getStat("PAYLOAD");
   
-  dbh->migr = NULL;
-  dbh->lastPrio = 0;
-  dbh->lastExp = 0;
-  
   if (dbh->payload == SYSERR) {
     FREE(dbh->fn);
     FREE(dbh);

Modified: GNUnet/src/applications/sqstore_sqlite/sqlitetest.c
===================================================================
--- GNUnet/src/applications/sqstore_sqlite/sqlitetest.c 2005-02-25 02:21:58 UTC 
(rev 298)
+++ GNUnet/src/applications/sqstore_sqlite/sqlitetest.c 2005-02-25 04:29:47 UTC 
(rev 299)
@@ -20,7 +20,7 @@
   value = MALLOC(sizeof(Datastore_Value) + 8 * i);
   value->size = htonl(sizeof(Datastore_Value) + 8 * i);
   value->type = htonl(i);
-  value->prio = htonl(i);
+  value->prio = htonl(i+1);
   value->anonymityLevel = i;
   value->expirationTime = now - i * cronSECONDS;
   memset(&value[1], i, 8*i);
@@ -41,8 +41,13 @@
                    value,
                    ntohl(val->size)) ) )
     ret = OK;
-  else
-    ret = SYSERR;                
+  else {
+    printf("Wanted: %u, %llu; got %u, %llu - %d\n",
+          ntohl(value->size), ntohll(value->expirationTime),
+          ntohl(val->size), ntohll(val->expirationTime),
+          memcmp(val, value, ntohl(val->size)));
+    ret = SYSERR;
+  }
   FREE(value);
   return ret;
 }
@@ -62,7 +67,6 @@
                       int * closure) {
   int ret;
 
-  printf("IDOWN: %u\n", *closure);
   (*closure) -= 2;  
   ret = checkValue(key, val, closure);
   return ret;
@@ -86,6 +90,12 @@
     FREE(value);
   }
   ASSERT(oldSize < api->getSize());
+  ASSERT(256 == api->iterateLowPriority(ANY_BLOCK,
+                                       NULL,
+                                       NULL));
+  ASSERT(256 == api->iterateExpirationTime(ANY_BLOCK,
+                                          NULL,
+                                          NULL));
   for (i=255;i>=0;i--) {
     memset(&key, 256-i, sizeof(HashCode160)); 
     ASSERT(1 == api->get(&key, i, &checkValue, (void*) &i));