[GNUnet-SVN] r27342 - in gnunet: doc/man src/datastore src/include src/util

[gnunet]

Author: grothoff
Date: 2013-05-30 21:10:02 +0200 (Thu, 30 May 2013)
New Revision: 27342

Added:
   gnunet/doc/man/gnunet-datastore.1
   gnunet/src/datastore/gnunet-datastore.c
Modified:
   gnunet/doc/man/Makefile.am
   gnunet/src/datastore/Makefile.am
   gnunet/src/include/gnunet_datastore_service.h
   gnunet/src/util/gnunet-resolver.c
Log:
adding tool to convert/merge datastores (#2875)

Modified: gnunet/doc/man/Makefile.am
===================================================================
--- gnunet/doc/man/Makefile.am  2013-05-30 18:34:35 UTC (rev 27341)
+++ gnunet/doc/man/Makefile.am  2013-05-30 19:10:02 UTC (rev 27342)
@@ -7,6 +7,7 @@
   gnunet-core.1 \
   gnunet-directory.1 \
   gnunet-dns2gns.1 \
+  gnunet-datastore.1 \
   gnunet-download.1 \
   gnunet-download-manager.1 \
   gnunet-ecc.1 \

Added: gnunet/doc/man/gnunet-datastore.1
===================================================================
--- gnunet/doc/man/gnunet-datastore.1                           (rev 0)
+++ gnunet/doc/man/gnunet-datastore.1   2013-05-30 19:10:02 UTC (rev 27342)
@@ -0,0 +1,41 @@
+.TH gnunet\-datastore "1" "30 May 2013" "GNUnet"
+.SH NAME
+gnunet\-datastore \- merge or convert GNUnet datastore databases
+
+.SH SYNOPSIS
+.B gnunet\-datastore 
+[\fIOPTIONS\fR]
+.SH DESCRIPTION
+.PP
+
+gnunet\-datastore can be used to convert or merge GNUnet datastores.  This is 
useful if a datastore is to be migrated between SQL databases, i.e. from sqlite 
to postgres or vice versa.  gnunet\-datastore basically takes two configuration 
files (which must specify different databases) and reads in all of the data 
from the datasource (\-s option) and copies it to the destination (\-c option). 
 Note that replication level information is lost in the process at this time.
+
+.TP
+\fB\-c \fIFILENAME\fR, \fB\-\-config=FILENAME\fR
+configuration file to use for the destination database
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+print help page
+.TP
+\fB\-i\fR, \fB\-\-list-indexed\fR
+print information about files that are currently indexed by file-sharing
+.TP
+\fB\-L \fILOGLEVEL\fR, \fB\-\-loglevel=LOGLEVEL\fR
+Change the loglevel.  Possible values for LOGLEVEL are ERROR, WARNING, INFO 
and DEBUG.
+.TP
+\fB\-s \fIFILENAME\fR, \fB\-\-sourcecfg=FILENAME\fR
+configuration file to use for the source database
+.TP
+\fB\-v\fR, \fB\-\-version\fR
+print the version number
+.TP
+\fB\-V\fR, \fB\-\-verbose\fR
+be verbose
+
+.SH NOTES
+
+
+.SH "REPORTING BUGS"
+Report bugs by using mantis <https://gnunet.org/bugs/> or by sending 
electronic mail to <address@hidden>
+.SH "SEE ALSO"
+\fBgnunet\-publish\fP(1)

Modified: gnunet/src/datastore/Makefile.am
===================================================================
--- gnunet/src/datastore/Makefile.am    2013-05-30 18:34:35 UTC (rev 27341)
+++ gnunet/src/datastore/Makefile.am    2013-05-30 19:10:02 UTC (rev 27342)
@@ -32,6 +32,8 @@
   $(GN_LIB_LDFLAGS) $(WINFLAGS) \
   -version-info 1:0:0
 
+bin_PROGRAMS = \
+ gnunet-datastore
 
 libexec_PROGRAMS = \
  gnunet-service-datastore
@@ -43,6 +45,13 @@
   $(top_builddir)/src/util/libgnunetutil.la \
   $(GN_LIBINTL)
 
+gnunet_datastore_SOURCES = \
+ gnunet-datastore.c 
+gnunet_datastore_LDADD = \
+  $(top_builddir)/src/datastore/libgnunetdatastore.la \
+  $(top_builddir)/src/util/libgnunetutil.la \
+  $(GN_LIBINTL)
+
 if HAVE_MYSQL
  MYSQL_PLUGIN = libgnunet_plugin_datastore_mysql.la
 if HAVE_BENCHMARKS

Added: gnunet/src/datastore/gnunet-datastore.c
===================================================================
--- gnunet/src/datastore/gnunet-datastore.c                             (rev 0)
+++ gnunet/src/datastore/gnunet-datastore.c     2013-05-30 19:10:02 UTC (rev 
27342)
@@ -0,0 +1,262 @@
+/*
+     This file is part of GNUnet.
+     (C) 2010, 2013 Christian Grothoff (and other contributing authors)
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+     Boston, MA 02111-1307, USA.
+*/
+
+/**
+ * @file datastore/gnunet-datastore.c
+ * @brief tool to manipulate datastores
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_datastore_service.h"
+
+
+/**
+ * Name of the second configuration file.
+ */
+static char *alternative_cfg;
+
+/**
+ * Global return value.
+ */
+static int ret;
+
+/**
+ * Our offset on 'get'.
+ */
+static uint64_t offset;
+
+/**
+ * First UID ever returned.
+ */
+static uint64_t first_uid;
+
+/**
+ * Configuration for the source database.
+ */
+static struct GNUNET_CONFIGURATION_Handle *scfg;
+
+/**
+ * Handle for database source.
+ */
+static struct GNUNET_DATASTORE_Handle *db_src;
+
+/**
+ * Handle for database destination.
+ */
+static struct GNUNET_DATASTORE_Handle *db_dst;
+
+/**
+ * Current operation.
+ */
+static struct GNUNET_DATASTORE_QueueEntry *qe;
+
+
+static void
+do_shutdown (void *cls,
+            const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  if (NULL != qe)
+    GNUNET_DATASTORE_cancel (qe);
+  GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO);
+  GNUNET_DATASTORE_disconnect (db_dst, GNUNET_NO);
+  GNUNET_CONFIGURATION_destroy (scfg);
+}
+
+
+/**
+ * Perform next GET operation.
+ */
+static void 
+do_get (void);
+
+
+/**
+ * Continuation called to notify client about result of the
+ * operation.
+ *
+ * @param cls closure
+ * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
+ *                GNUNET_NO if content was already there
+ *                GNUNET_YES (or other positive value) on success
+ * @param min_expiration minimum expiration time required for 0-priority 
content to be stored
+ *                by the datacache at this time, zero for unknown, forever if 
we have no
+ *                space for 0-priority content
+ * @param msg NULL on success, otherwise an error message
+ */
+static void 
+do_finish (void *cls,
+          int32_t success,
+          struct GNUNET_TIME_Absolute min_expiration,
+          const char *msg)
+{
+  qe = NULL;
+  if (GNUNET_SYSERR == success)
+  {
+    fprintf (stderr,
+            _("Failed to store item: %s, aborting\n"),
+            msg);
+    ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  do_get ();
+}
+
+
+/**
+ * Process a datum that was stored in the datastore.
+ *
+ * @param cls closure
+ * @param key key for the content
+ * @param size number of bytes in data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ *        maybe 0 if no unique identifier is available
+ */
+static void 
+do_put (void *cls,
+       const struct GNUNET_HashCode *key,
+       size_t size, const void *data,
+       enum GNUNET_BLOCK_Type type,
+       uint32_t priority,
+       uint32_t anonymity,
+       struct GNUNET_TIME_Absolute
+       expiration, uint64_t uid)
+{
+  qe = NULL;
+  if ( (0 != offset) &&
+       (uid == first_uid) )
+  {
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  if (0 == offset)
+    first_uid = uid;
+  qe = GNUNET_DATASTORE_put (db_dst, 0,
+                            key, size, data, type,
+                            priority, anonymity, 
+                            0 /* FIXME: replication is lost... */, 
+                            expiration,
+                            0, 1, GNUNET_TIME_UNIT_FOREVER_REL,
+                            &do_finish, NULL);
+}
+
+
+/**
+ * Perform next GET operation.
+ */
+static void 
+do_get ()
+{
+  qe = GNUNET_DATASTORE_get_key (db_src,
+                                offset,
+                                NULL, GNUNET_BLOCK_TYPE_ANY,
+                                0, 1,
+                                GNUNET_TIME_UNIT_FOREVER_REL,
+                                &do_put, NULL);
+}
+
+
+
+/**
+ * Main function that will be run by the scheduler.
+ *
+ * @param cls closure
+ * @param args remaining command-line arguments
+ * @param cfgfile name of the configuration file used 
+ * @param cfg configuration -- for destination datastore
+ */
+static void
+run (void *cls, char *const *args, const char *cfgfile,
+     const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  if (NULL == alternative_cfg)
+    return; /* nothing to be done */
+  if (0 == strcmp (cfgfile, alternative_cfg))
+  {
+    fprintf (stderr, 
+            _("Cannot use the same configuration for source and 
destination\n"));
+    ret = 1;
+    return;
+  }
+  scfg = GNUNET_CONFIGURATION_create ();
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_load (scfg,
+                                alternative_cfg))
+  {
+    GNUNET_CONFIGURATION_destroy (scfg);
+    ret = 1;
+    return;
+  }
+  db_src = GNUNET_DATASTORE_connect (scfg);
+  if (NULL == db_src)
+  {
+    GNUNET_CONFIGURATION_destroy (scfg);
+    ret = 1;
+    return;
+  }
+  db_dst = GNUNET_DATASTORE_connect (cfg);
+  if (NULL == db_dst)
+  {
+    GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO);
+    GNUNET_CONFIGURATION_destroy (scfg);
+    ret = 1;
+    return;
+  }
+  GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
+                               &do_shutdown, NULL);
+  do_get ();
+}
+
+
+/**
+ * The main function to manipulate datastores.
+ *
+ * @param argc number of arguments from the command line
+ * @param argv command line arguments
+ * @return 0 ok, 1 on error
+ */
+int
+main (int argc, char *const *argv)
+{
+  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
+    { 's', "sourcecfg", "FILENAME",
+      gettext_noop ("specifies the configuration to use to access an 
alternative datastore; will merge that datastore into our current datastore"),
+      1, &GNUNET_GETOPT_set_filename, &alternative_cfg },
+    GNUNET_GETOPT_OPTION_END
+  };
+  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+    return 2;
+
+  if (GNUNET_OK !=
+      GNUNET_PROGRAM_run (argc, argv, "gnunet-datastore",
+                         gettext_noop ("Manipulate GNUnet datastore"),
+                         options, &run, NULL)) 
+    ret = 1;
+  GNUNET_free ((void*) argv);
+  return ret;
+}
+
+/* end of gnunet-datastore.c */

Modified: gnunet/src/include/gnunet_datastore_service.h
===================================================================
--- gnunet/src/include/gnunet_datastore_service.h       2013-05-30 18:34:35 UTC 
(rev 27341)
+++ gnunet/src/include/gnunet_datastore_service.h       2013-05-30 19:10:02 UTC 
(rev 27342)
@@ -153,7 +153,7 @@
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h, uint32_t rid,
-                      const struct GNUNET_HashCode * key, size_t size,
+                      const struct GNUNET_HashCode *key, size_t size,
                       const void *data, enum GNUNET_BLOCK_Type type,
                       uint32_t priority, uint32_t anonymity,
                       uint32_t replication,
@@ -300,7 +300,7 @@
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h, uint64_t offset,
-                          const struct GNUNET_HashCode * key,
+                          const struct GNUNET_HashCode *key,
                           enum GNUNET_BLOCK_Type type,
                           unsigned int queue_priority,
                           unsigned int max_queue_size,

Modified: gnunet/src/util/gnunet-resolver.c
===================================================================
--- gnunet/src/util/gnunet-resolver.c   2013-05-30 18:34:35 UTC (rev 27341)
+++ gnunet/src/util/gnunet-resolver.c   2013-05-30 19:10:02 UTC (rev 27342)
@@ -133,8 +133,9 @@
                                NULL);
 }
 
+
 /**
- * The main function to obtain statistics in GNUnet.
+ * The main function to access GNUnet's DNS resolver.
  *
  * @param argc number of arguments from the command line
  * @param argv command line arguments