[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r5335 - GNUnet/src/applications/sqstore_sqlite
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] r5335 - GNUnet/src/applications/sqstore_sqlite |
|
Date: |
Sun, 22 Jul 2007 00:37:25 -0600 (MDT) |
Author: grothoff
Date: 2007-07-22 00:37:25 -0600 (Sun, 22 Jul 2007)
New Revision: 5335
Modified:
GNUnet/src/applications/sqstore_sqlite/sqlite.c
GNUnet/src/applications/sqstore_sqlite/sqlitetest.c
GNUnet/src/applications/sqstore_sqlite/sqlitetest2.c
GNUnet/src/applications/sqstore_sqlite/sqlitetest3.c
Log:
tests
Modified: GNUnet/src/applications/sqstore_sqlite/sqlite.c
===================================================================
--- GNUnet/src/applications/sqstore_sqlite/sqlite.c 2007-07-22 06:12:53 UTC
(rev 5334)
+++ GNUnet/src/applications/sqstore_sqlite/sqlite.c 2007-07-22 06:37:25 UTC
(rev 5335)
@@ -240,7 +240,7 @@
if (sqlite3_step (stmt) == SQLITE_DONE)
{
if (sqlite3_exec (ret->dbh,
- "CREATE TABLE gn071 ("
+ "CREATE TABLE gn070 ("
" size INTEGER NOT NULL DEFAULT 0,"
" type INTEGER NOT NULL DEFAULT 0,"
" prio INTEGER NOT NULL DEFAULT 0,"
@@ -362,6 +362,27 @@
/* benchmarking shows 2-12% overhead */
}
+static int delete_by_rowid(sqliteHandle * handle,
+ unsigned long long rid) {
+ sqlite3_stmt *stmt;
+
+ if (sq_prepare (handle->dbh,
+ "DELETE FROM gn070 WHERE _ROWID_ = ?", &stmt) != SQLITE_OK) {
+ LOG_SQLITE (handle,
+ GE_ERROR | GE_ADMIN | GE_USER | GE_BULK, "sq_prepare");
+ return SYSERR;
+ }
+ sqlite3_bind_int64 (stmt, 1, rid);
+ if (SQLITE_DONE != sqlite3_step (stmt)) {
+ LOG_SQLITE (handle,
+ GE_ERROR | GE_ADMIN | GE_USER | GE_BULK, "sqlite_step");
+ sqlite3_finalize (stmt);
+ return SYSERR;
+ }
+ sqlite3_finalize (stmt);
+ return OK;
+}
+
/**
* Given a full row from gn070 table
(size,type,prio,anonLevel,expire,hash,value),
* assemble it into a Datastore_Value representation.
@@ -435,6 +456,7 @@
value->prio = htonl (sqlite3_column_int (stmt, 2));
value->anonymityLevel = htonl (sqlite3_column_int (stmt, 3));
value->expirationTime = htonll (sqlite3_column_int64 (stmt, 4));
+ memcpy (key, sqlite3_column_blob (stmt, 5), sizeof(HashCode512));
memcpy (&value[1], sqlite3_column_blob (stmt, 6), contentSize);
*rowid = sqlite3_column_int64 (stmt, 7);
return value;
@@ -564,7 +586,7 @@
sqliteHandle *handle;
int ret;
cron_t now;
- unsigned long long rowid;
+ unsigned long long rowid;
handle = getDBHandle ();
dbh = handle->dbh;
@@ -691,12 +713,19 @@
count++;
if (iter != NULL)
{
- if (SYSERR == iter (&key, datum, closure, rowid))
+ ret = iter (&key, datum, closure, rowid);
+ if (ret == SYSERR)
{
FREE (datum);
count = SYSERR;
break;
}
+ if (ret == NO) {
+ MUTEX_LOCK(db->DATABASE_Lock_);
+ db->payload -= getContentDatastoreSize (datum);
+ MUTEX_UNLOCK(db->DATABASE_Lock_);
+ delete_by_rowid(handle, rowid);
+ }
}
}
lastPrio = ntohl (datum->prio);
@@ -828,12 +857,17 @@
payload += getContentDatastoreSize (datum);
if (iter != NULL)
{
- if (SYSERR == iter (&key, datum, closure, rowid))
+ ret = iter (&key, datum, closure, rowid);
+ if (ret == SYSERR)
{
FREE (datum);
count = SYSERR;
break;
}
+ if (ret == NO) {
+ payload -= getContentDatastoreSize (datum);
+ delete_by_rowid(handle, rowid);
+ }
}
FREE (datum);
count++;
@@ -1011,7 +1045,8 @@
"Found in database block with type %u.\n",
ntohl (*(int *) &((&datum->value)[1])));
#endif
- if (SYSERR == iter (&rkey, datum, closure, rowid))
+ ret = iter (&rkey, datum, closure, rowid);
+ if (ret == SYSERR)
{
count = SYSERR;
@@ -1019,6 +1054,12 @@
ret = SQLITE_DONE;
break;
}
+ if (ret == NO) {
+ MUTEX_LOCK(db->DATABASE_Lock_);
+ db->payload -= getContentDatastoreSize (datum);
+ MUTEX_UNLOCK(db->DATABASE_Lock_);
+ delete_by_rowid(handle, rowid);
+ }
FREE (datum);
count++;
}
Modified: GNUnet/src/applications/sqstore_sqlite/sqlitetest.c
===================================================================
--- GNUnet/src/applications/sqstore_sqlite/sqlitetest.c 2007-07-22 06:12:53 UTC
(rev 5334)
+++ GNUnet/src/applications/sqstore_sqlite/sqlitetest.c 2007-07-22 06:37:25 UTC
(rev 5335)
@@ -1,6 +1,6 @@
/*
This file is part of GNUnet.
- (C) 2004, 2005, 2006 Christian Grothoff (and other contributing authors)
+ (C) 2004, 2005, 2006, 2007 Christian Grothoff (and other contributing
authors)
GNUnet is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published
@@ -52,7 +52,8 @@
static int
checkValue (const HashCode512 * key,
- const Datastore_Value * val, void *closure)
+ const Datastore_Value * val, void *closure,
+ unsigned long long uid)
{
int i;
int ret;
@@ -77,39 +78,49 @@
}
static int
-iterateUp (const HashCode512 * key, const Datastore_Value * val, int *closure)
+iterateUp (const HashCode512 * key, const Datastore_Value * val, int *closure,
+ unsigned long long uid)
{
int ret;
- ret = checkValue (key, val, closure);
+ ret = checkValue (key, val, closure, uid);
(*closure) += 2;
return ret;
}
static int
iterateDown (const HashCode512 * key,
- const Datastore_Value * val, int *closure)
+ const Datastore_Value * val, int *closure,
+ unsigned long long uid)
{
int ret;
(*closure) -= 2;
- ret = checkValue (key, val, closure);
+ ret = checkValue (key, val, closure, uid);
return ret;
}
static int
iterateDelete (const HashCode512 * key,
- const Datastore_Value * val, SQstore_ServiceAPI * api)
+ const Datastore_Value * val, void * closure,
+ unsigned long long uid)
{
- if (1 == api->del (key, val))
- return OK;
- else
- return SYSERR;
+ return NO;
}
static int
+iteratePriority (const HashCode512 * key,
+ const Datastore_Value * val, SQstore_ServiceAPI * api,
+ unsigned long long uid)
+{
+ api->update(uid, 4, 0);
+ return OK;
+}
+
+static int
priorityCheck (const HashCode512 * key,
- const Datastore_Value * val, int *closure)
+ const Datastore_Value * val, int *closure,
+ unsigned long long uid)
{
int id;
@@ -122,7 +133,8 @@
static int
multipleCheck (const HashCode512 * key,
- const Datastore_Value * val, Datastore_Value ** last)
+ const Datastore_Value * val, Datastore_Value ** last,
+ unsigned long long uid)
{
if (*last != NULL)
{
@@ -171,7 +183,7 @@
{
memset (&key, 256 - i, sizeof (HashCode512));
value = initValue (i);
- ASSERT (1 == api->del (&key, value));
+ ASSERT (1 == api->get (&key, 0, &iterateDelete, NULL));
FREE (value);
}
ASSERT (oldSize > api->getSize ());
@@ -197,7 +209,8 @@
ASSERT (1 == api->iterateExpirationTime (ANY_BLOCK,
(Datum_Iterator) & priorityCheck,
&i));
- api->update (&key, value, 4, 0);
+ ASSERT (1 == api->iterateAllNow((Datum_Iterator) & iteratePriority,
+ api));
i += 4;
ASSERT (1 == api->iterateExpirationTime (ANY_BLOCK,
(Datum_Iterator) & priorityCheck,
@@ -214,8 +227,8 @@
(Datum_Iterator) & multipleCheck,
&value));
FREE (value);
- api->del (&key, NULL);
- api->del (&key, NULL);
+ ASSERT (2 == api->iterateAllNow ((Datum_Iterator) & iterateDelete,
+ api));
ASSERT (0 == api->iterateExpirationTime (ANY_BLOCK, NULL, NULL));
api->drop ();
Modified: GNUnet/src/applications/sqstore_sqlite/sqlitetest2.c
===================================================================
--- GNUnet/src/applications/sqstore_sqlite/sqlitetest2.c 2007-07-22
06:12:53 UTC (rev 5334)
+++ GNUnet/src/applications/sqstore_sqlite/sqlitetest2.c 2007-07-22
06:37:25 UTC (rev 5335)
@@ -1,6 +1,6 @@
/*
This file is part of GNUnet.
- (C) 2004, 2005, 2006 Christian Grothoff (and other contributing authors)
+ (C) 2004, 2005, 2006, 2007 Christian Grothoff (and other contributing
authors)
GNUnet is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published
@@ -22,14 +22,23 @@
* @brief Test for the sqstore implementations.
* @author Christian Grothoff
*
- * This testcase inserts a bunch of (variable size) data and then
- * deletes data until the (reported) database size drops below a given
- * threshold. This is iterated 10 times, with the actual size of the
- * content stored, the database size reported and the file size on
- * disk being printed for each iteration. The deletion strategy
- * alternates between "lowest priority" and "earliest expiration".
+ * This testcase inserts a bunch of (variable size) data and then deletes
+ * data until the (reported) database size drops below a given threshold.
+ * This is iterated 10 times, with the actual size of the content stored,
+ * the database size reported and the file size on disk being printed for
+ * each iteration. The code also prints a "I" for every 40 blocks
+ * inserted and a "D" for every 40 blocks deleted. The deletion
+ * strategy alternates between "lowest priority" and "earliest expiration".
* Priorities and expiration dates are set using a pseudo-random value
* within a realistic range.
+ * <p>
+ *
+ * Note that the disk overhead calculations are not very sane for
+ * MySQL: we take the entire /var/lib/mysql directory (best we can
+ * do for ISAM), which may contain other data and which never
+ * shrinks. The scanning of the entire mysql directory during
+ * each report is also likely to be the cause of a minor
+ * slowdown compared to sqlite.<p>
*/
#include "platform.h"
@@ -44,12 +53,36 @@
#define ASSERT(x) do { if (! (x)) { printf("Error at %s:%d\n", __FILE__,
__LINE__); goto FAILURE;} } while (0)
/**
- * Target datastore size (in bytes). Realistic sizes are
- * more like 16 GB (not the default of 16 MB); however,
- * those take too long to run them in the usual "make check"
- * sequence. Hence the value used for shipping is tiny.
+ * Target datastore size (in bytes).
+ * <p>
+ * Example impact of total size on the reported number
+ * of operations (insert and delete) per second (once
+ * roughly stabilized -- this is not "sound" experimental
+ * data but just a rough idea) for a particular machine:
+ * <pre>
+ * 4: 60 at 7k ops total
+ * 8: 50 at 3k ops total
+ * 16: 48 at 8k ops total
+ * 32: 46 at 8k ops total
+ * 64: 61 at 9k ops total
+ * 128: 89 at 9k ops total
+ * 4092: 11 at 383k ops total (12 GB stored, 14.8 GB DB size on disk, 2.5 GB
reported)
+ * </pre>
+ * Pure insertion performance into an empty DB initially peaks
+ * at about 400 ops. The performance seems to drop especially
+ * once the existing (fragmented) ISAM space is filled up and
+ * the DB needs to grow on disk. This could be explained with
+ * ISAM looking more carefully for defragmentation opportunities.
+ * <p>
+ * MySQL disk space overheads (for otherwise unused database when
+ * run with 128 MB target data size; actual size 651 MB, useful
+ * data stored 520 MB) are quite large in the range of 25-30%.
+ * <p>
+ * This kind of processing seems to be IO bound (system is roughly
+ * at 90% wait, 10% CPU). This is with MySQL 5.0.
+ *
*/
-#define MAX_SIZE 1024LL * 1024 * 128
+#define MAX_SIZE 1024LL * 1024 * 16
/**
* Report progress outside of major reports? Should probably be YES if
@@ -76,8 +109,11 @@
/**
* Name of the database on disk.
+ * You may have to adjust this path and the access
+ * permission to the respective directory in order
+ * to obtain all of the performance information.
*/
-#define DB_NAME "/tmp/gnunet-sqlite-sqstore-test/data/fs/content/gnunet.dat"
+#define DB_NAME "/var/lib/mysql"
static unsigned long long stored_bytes;
@@ -130,7 +166,8 @@
static int
iterateDelete (const HashCode512 * key,
- const Datastore_Value * val, void *cls)
+ const Datastore_Value * val, void *cls,
+ unsigned long long uid)
{
SQstore_ServiceAPI *api = cls;
static int dc;
@@ -144,10 +181,9 @@
if (dc % REP_FREQ == 0)
fprintf (stderr, "D");
#endif
- GE_ASSERT (NULL, 1 == api->del (key, val));
stored_bytes -= ntohl (val->size);
stored_entries--;
- return OK;
+ return NO;
}
/**
@@ -156,14 +192,13 @@
static int
test (SQstore_ServiceAPI * api)
{
- unsigned long long lops;
int i;
int j;
unsigned long long size;
int have_file;
+ struct stat sbuf;
- lops = 0;
- have_file = OK == disk_file_test (NULL, DB_NAME);
+ have_file = 0 == stat (DB_NAME, &sbuf);
for (i = 0; i < ITERATIONS; i++)
{
@@ -184,7 +219,6 @@
else
api->iterateExpirationTime (0, &iterateDelete, api);
- /* every 10 iterations print status */
size = 0;
if (have_file)
disk_file_size (NULL, DB_NAME, &size, NO);
@@ -192,14 +226,12 @@
#if REPORT_ID
"\n"
#endif
- "%u: Useful %llu, API %llu, disk %llu (%.2f%%) / %lluk ops /
%llu ops/s\n", i, stored_bytes / 1024, /* used size in k */
+ "Useful %llu, API %llu, disk %llu (%.2f%%) / %lluk ops / %llu
ops/s\n", stored_bytes / 1024, /* used size in k */
api->getSize () / 1024, /* API-reported size in k */
size / 1024, /* disk size in kb */
(100.0 * size / stored_bytes) - 100, /* overhead */
(stored_ops * 2 - stored_entries) / 1024, /* total
operations (in k) */
- 1000 * ((stored_ops * 2 - stored_entries) - lops) / (1 +
get_time () - start_time)); /* operations per second */
- lops = stored_ops * 2 - stored_entries;
- start_time = get_time ();
+ 1000 * (stored_ops * 2 - stored_entries) / (1 + get_time () -
start_time)); /* operations per second */
if (GNUNET_SHUTDOWN_TEST () == YES)
break;
}
@@ -244,4 +276,4 @@
return 0;
}
-/* end of sqlitetest2.c */
+/* end of mysqltest2.c */
Modified: GNUnet/src/applications/sqstore_sqlite/sqlitetest3.c
===================================================================
--- GNUnet/src/applications/sqstore_sqlite/sqlitetest3.c 2007-07-22
06:12:53 UTC (rev 5334)
+++ GNUnet/src/applications/sqstore_sqlite/sqlitetest3.c 2007-07-22
06:37:25 UTC (rev 5335)
@@ -18,7 +18,7 @@
Boston, MA 02111-1307, USA.
*/
/*
- * @file applications/sqstore_sqlite/sqlitetest3.c
+ * @file applications/sqstore_mysql/mysqltest3.c
* @brief Profile sqstore iterators.
* @author Christian Grothoff
*/
@@ -40,7 +40,7 @@
*/
#define MAX_SIZE 1024LL * 1024 * 128
-#define ITERATIONS 10
+#define ITERATIONS 1
/**
* Number of put operations equivalent to 1/10th of MAX_SIZE
@@ -65,11 +65,13 @@
/* most content is 32k */
size = sizeof (Datastore_Value) + 32 * 1024;
+
if (weak_randomi (16) == 0) /* but some of it is less! */
size = sizeof (Datastore_Value) + weak_randomi (32 * 1024);
size = size - (size & 7); /* always multiple of 8 */
- GE_ASSERT (NULL, size >= sizeof (Datastore_Value));
+
/* generate random key */
+ key.bits[0] = (unsigned int) get_time();
hash (&key, sizeof (HashCode512), &key);
value = MALLOC (size);
value->size = htonl (size);
@@ -86,10 +88,6 @@
return SYSERR;
}
ic++;
-#if REPORT_ID
- if (ic % REP_FREQ == 0)
- fprintf (stderr, "I");
-#endif
stored_bytes += ntohl (value->size);
stored_ops++;
stored_entries++;
@@ -98,7 +96,8 @@
}
static int
-iterateDummy (const HashCode512 * key, const Datastore_Value * val, void *cls)
+iterateDummy (const HashCode512 * key, const Datastore_Value * val, void *cls,
+ unsigned long long uid)
{
if (GNUNET_SHUTDOWN_TEST () == YES)
return SYSERR;
@@ -110,6 +109,7 @@
{
int i;
int j;
+ int ret;
cron_t start;
cron_t end;
@@ -129,43 +129,37 @@
if (GNUNET_SHUTDOWN_TEST () == YES)
break;
start = get_time ();
- api->iterateLowPriority (0, &iterateDummy, api);
+ ret = api->iterateLowPriority (0, &iterateDummy, api);
end = get_time ();
- printf ("%3u low priority iteration took %20llums\n", i, end - start);
+ printf ("%3u low priority iteration took %20llums (%d)\n", i, end -
start, ret);
if (GNUNET_SHUTDOWN_TEST () == YES)
break;
start = get_time ();
- api->iterateExpirationTime (0, &iterateDummy, api);
+ ret = api->iterateExpirationTime (0, &iterateDummy, api);
end = get_time ();
- printf ("%3u expiration t iteration took %20llums\n", i, end - start);
+ printf ("%3u expiration t iteration took %20llums (%d)\n", i, end -
start, ret);
if (GNUNET_SHUTDOWN_TEST () == YES)
break;
start = get_time ();
- api->iterateNonAnonymous (0, NO, &iterateDummy, api);
+ ret = api->iterateNonAnonymous (0, &iterateDummy, api);
end = get_time ();
- printf ("%3u non anonymou iteration took %20llums\n", i, end - start);
+ printf ("%3u non anonymou iteration took %20llums (%d)\n", i, end -
start, ret);
if (GNUNET_SHUTDOWN_TEST () == YES)
break;
start = get_time ();
- api->iterateNonAnonymous (0, YES, &iterateDummy, api);
+ ret = api->iterateMigrationOrder (&iterateDummy, api);
end = get_time ();
- printf ("%3u non anon YES iteration took %20llums\n", i, end - start);
+ printf ("%3u migration or iteration took %20llums (%d)\n", i, end -
start, ret);
if (GNUNET_SHUTDOWN_TEST () == YES)
break;
start = get_time ();
- api->iterateMigrationOrder (&iterateDummy, api);
+ ret = api->iterateAllNow (&iterateDummy, api);
end = get_time ();
- printf ("%3u migration or iteration took %20llums\n", i, end - start);
+ printf ("%3u all now iteration took %20llums (%d)\n", i, end -
start, ret);
if (GNUNET_SHUTDOWN_TEST () == YES)
break;
- start = get_time ();
- api->iterateAllNow (&iterateDummy, api);
- end = get_time ();
- printf ("%3u all now iteration took %20llums\n", i, end - start);
- if (GNUNET_SHUTDOWN_TEST () == YES)
- break;
}
- api->drop ();
+ api->drop();
return OK;
}
@@ -202,4 +196,4 @@
return 0;
}
-/* end of sqlitetest3.c */
+/* end of mysqltest3.c */
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r5335 - GNUnet/src/applications/sqstore_sqlite,
gnunet <=