[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[gnunet] branch master updated: modernize datastore postgres implementat
|
From: |
gnunet |
|
Subject: |
[gnunet] branch master updated: modernize datastore postgres implementation |
|
Date: |
Tue, 22 Nov 2022 15:38:25 +0100 |
This is an automated email from the git hooks/post-receive script.
grothoff pushed a commit to branch master
in repository gnunet.
The following commit(s) were added to refs/heads/master by this push:
new bf175e5d2 modernize datastore postgres implementation
bf175e5d2 is described below
commit bf175e5d2785033528128909457e382cb1929165
Author: Christian Grothoff <christian@grothoff.org>
AuthorDate: Tue Nov 22 15:38:22 2022 +0100
modernize datastore postgres implementation
---
src/datastore/Makefile.am | 9 ++-
src/datastore/datastore-0001.sql | 49 +++++++++++++
src/datastore/datastore-drop.sql | 25 +++++++
src/datastore/datastore.conf.in | 1 +
src/datastore/plugin_datastore_postgres.c | 113 ++++++++++--------------------
src/include/gnunet_datastore_plugin.h | 22 +++---
src/namecache/namecache.conf.in | 3 +-
src/namestore/namestore.conf.in | 2 -
8 files changed, 132 insertions(+), 92 deletions(-)
diff --git a/src/datastore/Makefile.am b/src/datastore/Makefile.am
index 07ae004b3..b73a0497b 100644
--- a/src/datastore/Makefile.am
+++ b/src/datastore/Makefile.am
@@ -10,6 +10,12 @@ libexecdir= $(pkglibdir)/libexec/
pkgcfg_DATA = \
datastore.conf
+sqldir = $(prefix)/share/gnunet/sql/
+
+sql_DATA = \
+ datastore-0001.sql \
+ datastore-drop.sql
+
if USE_COVERAGE
AM_CFLAGS = --coverage -O0
XLIBS = -lgcov
@@ -318,4 +324,5 @@ EXTRA_DIST = \
test_plugin_datastore_data_mysql.conf \
test_datastore_api_data_postgres.conf \
perf_plugin_datastore_data_postgres.conf \
- test_plugin_datastore_data_postgres.conf
+ test_plugin_datastore_data_postgres.conf \
+ $(sql_DATA)
diff --git a/src/datastore/datastore-0001.sql b/src/datastore/datastore-0001.sql
new file mode 100644
index 000000000..0d4758be2
--- /dev/null
+++ b/src/datastore/datastore-0001.sql
@@ -0,0 +1,49 @@
+--
+-- This file is part of GNUnet
+-- Copyright (C) 2014--2022 GNUnet e.V.
+--
+-- GNUnet is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR
+-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- GNUnet; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+--
+
+-- Everything in one big transaction
+BEGIN;
+
+-- Check patch versioning is in place.
+SELECT _v.register_patch('datastore-0001', NULL, NULL);
+
+-------------------- Schema ----------------------------
+
+CREATE SCHEMA datastore;
+COMMENT ON SCHEMA datastore IS 'gnunet-datastore data';
+
+SET search_path TO datastore;
+
+CREATE TABLE IF NOT EXISTS gn090 (
+ repl INTEGER NOT NULL DEFAULT 0,
+ type INTEGER NOT NULL DEFAULT 0,
+ prio INTEGER NOT NULL DEFAULT 0,
+ anonLevel INTEGER NOT NULL DEFAULT 0,
+ expire BIGINT NOT NULL DEFAULT 0,
+ rvalue BIGINT NOT NULL DEFAULT 0,
+ hash BYTEA NOT NULL DEFAULT '',
+ vhash BYTEA NOT NULL DEFAULT '',
+ value BYTEA NOT NULL DEFAULT '',
+ oid BIGINT GENERATED BY DEFAULT AS IDENTITY);
+
+CREATE INDEX IF NOT EXISTS oid_hash ON gn090 (oid);
+CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash);
+CREATE INDEX IF NOT EXISTS idx_prio_anon ON gn090 (prio,anonLevel);
+CREATE INDEX IF NOT EXISTS idx_prio_hash_anon ON gn090 (prio,hash,anonLevel);
+CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue);
+CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash);
+
+COMMIT;
diff --git a/src/datastore/datastore-drop.sql b/src/datastore/datastore-drop.sql
new file mode 100644
index 000000000..67fee303d
--- /dev/null
+++ b/src/datastore/datastore-drop.sql
@@ -0,0 +1,25 @@
+--
+-- This file is part of GNUnet
+-- Copyright (C) 2014--2022 GNUnet e.V.
+--
+-- GNUnet is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR
+-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- GNUnet; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+--
+
+-- Everything in one big transaction
+BEGIN;
+
+
+SELECT _v.unregister_patch('datastore-0001');
+
+DROP SCHEMA datastore CASCADE;
+
+COMMIT;
diff --git a/src/datastore/datastore.conf.in b/src/datastore/datastore.conf.in
index 21d24bb52..bcd495c8f 100644
--- a/src/datastore/datastore.conf.in
+++ b/src/datastore/datastore.conf.in
@@ -18,6 +18,7 @@ FILENAME = $GNUNET_DATA_HOME/datastore/sqlite.db
[datastore-postgres]
CONFIG = postgres:///gnunet
+SQL_DIR = ${DATADIR}/sql/
[datastore-mysql]
DATABASE = gnunet
diff --git a/src/datastore/plugin_datastore_postgres.c
b/src/datastore/plugin_datastore_postgres.c
index e49564dd9..5fcacc17b 100644
--- a/src/datastore/plugin_datastore_postgres.c
+++ b/src/datastore/plugin_datastore_postgres.c
@@ -1,6 +1,6 @@
/*
This file is part of GNUnet
- Copyright (C) 2009-2017 GNUnet e.V.
+ Copyright (C) 2009-2017, 2022 GNUnet e.V.
GNUnet is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License as published
@@ -64,102 +64,67 @@ struct Plugin
* @param plugin global context
* @return #GNUNET_OK on success, #GNUNET_SYSERR on error
*/
-static int
+static enum GNUNET_GenericReturnValue
init_connection (struct Plugin *plugin)
{
- struct GNUNET_PQ_ExecuteStatement es[] = {
- /* FIXME: PostgreSQL does not have unsigned integers! This is ok for the
type column because
- * we only test equality on it and can cast it to/from uint32_t. For repl,
prio, and anonLevel
- * we do math or inequality tests, so we can't handle the entire range of
uint32_t.
- * This will also cause problems for expiration times after
294247-01-10-04:00:54 UTC.
- */
- GNUNET_PQ_make_try_execute (
- "CREATE SEQUENCE IF NOT EXISTS gn090_oid_seq"),
- GNUNET_PQ_make_execute ("CREATE TABLE IF NOT EXISTS gn090 ("
- " repl INTEGER NOT NULL DEFAULT 0,"
- " type INTEGER NOT NULL DEFAULT 0,"
- " prio INTEGER NOT NULL DEFAULT 0,"
- " anonLevel INTEGER NOT NULL DEFAULT 0,"
- " expire BIGINT NOT NULL DEFAULT 0,"
- " rvalue BIGINT NOT NULL DEFAULT 0,"
- " hash BYTEA NOT NULL DEFAULT '',"
- " vhash BYTEA NOT NULL DEFAULT '',"
- " value BYTEA NOT NULL DEFAULT '',"
- " oid OID NOT NULL DEFAULT
nextval('gn090_oid_seq'))"),
- GNUNET_PQ_make_try_execute (
- "ALTER SEQUENCE gn090_oid_seq OWNED BY gn090.oid"),
- GNUNET_PQ_make_try_execute (
- "CREATE INDEX IF NOT EXISTS oid_hash ON gn090 (oid)"),
- GNUNET_PQ_make_try_execute (
- "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)"),
- GNUNET_PQ_make_try_execute (
- "CREATE INDEX IF NOT EXISTS idx_prio ON gn090 (prio)"),
- GNUNET_PQ_make_try_execute (
- "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire)"),
- GNUNET_PQ_make_try_execute (
- "CREATE INDEX IF NOT EXISTS idx_prio_anon ON gn090 (prio,anonLevel)"),
- GNUNET_PQ_make_try_execute (
- "CREATE INDEX IF NOT EXISTS idx_prio_hash_anon ON gn090
(prio,hash,anonLevel)"),
- GNUNET_PQ_make_try_execute (
- "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)"),
- GNUNET_PQ_make_try_execute (
- "CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash)"),
- GNUNET_PQ_make_execute (
- "ALTER TABLE gn090 ALTER value SET STORAGE EXTERNAL"),
- GNUNET_PQ_make_execute ("ALTER TABLE gn090 ALTER hash SET STORAGE PLAIN"),
- GNUNET_PQ_make_execute ("ALTER TABLE gn090 ALTER vhash SET STORAGE PLAIN"),
- GNUNET_PQ_EXECUTE_STATEMENT_END
- };
-
#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, oid"
struct GNUNET_PQ_PreparedStatement ps[] = {
GNUNET_PQ_make_prepare ("get",
- "SELECT " RESULT_COLUMNS " FROM gn090"
+ "SELECT " RESULT_COLUMNS
+ " FROM datastore.gn090"
" WHERE oid >= $1::bigint AND"
" (rvalue >= $2 OR 0 = $3::smallint) AND"
" (hash = $4 OR 0 = $5::smallint) AND"
" (type = $6 OR 0 = $7::smallint)"
" ORDER BY oid ASC LIMIT 1"),
GNUNET_PQ_make_prepare ("put",
- "INSERT INTO gn090 (repl, type, prio, anonLevel,
expire, rvalue, hash, vhash, value) "
+ "INSERT INTO datastore.gn090"
+ " (repl, type, prio, anonLevel, expire, rvalue,
hash, vhash, value) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"),
GNUNET_PQ_make_prepare ("update",
- "UPDATE gn090"
+ "UPDATE datastore.gn090"
" SET prio = prio + $1,"
" repl = repl + $2,"
" expire = GREATEST(expire, $3)"
" WHERE hash = $4 AND vhash = $5"),
GNUNET_PQ_make_prepare ("decrepl",
- "UPDATE gn090 SET repl = GREATEST (repl - 1, 0) "
- "WHERE oid = $1"),
+ "UPDATE datastore.gn090"
+ " SET repl = GREATEST (repl - 1, 0)"
+ " WHERE oid = $1"),
GNUNET_PQ_make_prepare ("select_non_anonymous",
- "SELECT " RESULT_COLUMNS " FROM gn090 "
- "WHERE anonLevel = 0 AND type = $1 AND oid >=
$2::bigint "
- "ORDER BY oid ASC LIMIT 1"),
+ "SELECT " RESULT_COLUMNS
+ " FROM datastore.gn090"
+ " WHERE anonLevel = 0 AND type = $1 AND oid >=
$2::bigint"
+ " ORDER BY oid ASC LIMIT 1"),
GNUNET_PQ_make_prepare ("select_expiration_order",
- "(SELECT " RESULT_COLUMNS " FROM gn090 "
- "WHERE expire < $1 ORDER BY prio ASC LIMIT 1) "
+ "(SELECT " RESULT_COLUMNS
+ " FROM datastore.gn090"
+ " WHERE expire < $1 ORDER BY prio ASC LIMIT 1) "
"UNION "
- "(SELECT " RESULT_COLUMNS " FROM gn090 "
- "ORDER BY prio ASC LIMIT 1) "
- "ORDER BY expire ASC LIMIT 1"),
+ "(SELECT " RESULT_COLUMNS
+ " FROM datastore.gn090"
+ " ORDER BY prio ASC LIMIT 1)"
+ " ORDER BY expire ASC LIMIT 1"),
GNUNET_PQ_make_prepare ("select_replication_order",
- "SELECT " RESULT_COLUMNS " FROM gn090 "
- "ORDER BY repl DESC,RANDOM() LIMIT 1"),
+ "SELECT " RESULT_COLUMNS
+ " FROM datastore.gn090"
+ " ORDER BY repl DESC,RANDOM() LIMIT 1"),
GNUNET_PQ_make_prepare ("delrow",
- "DELETE FROM gn090 "
- "WHERE oid=$1"),
+ "DELETE FROM datastore.gn090"
+ " WHERE oid=$1"),
GNUNET_PQ_make_prepare ("remove",
- "DELETE FROM gn090"
+ "DELETE FROM datastore.gn090"
" WHERE hash = $1 AND"
" value = $2"),
GNUNET_PQ_make_prepare ("get_keys",
- "SELECT hash FROM gn090"),
+ "SELECT hash"
+ " FROM datastore.gn090"),
GNUNET_PQ_make_prepare ("estimate_size",
"SELECT CASE WHEN NOT EXISTS"
- " (SELECT 1 FROM gn090)"
+ " (SELECT 1 FROM datastore.gn090)"
" THEN 0"
- " ELSE (SELECT SUM(LENGTH(value))+256*COUNT(*)
FROM gn090)"
+ " ELSE (SELECT SUM(LENGTH(value))+256*COUNT(*)"
+ " FROM datastore.gn090)"
"END AS total"),
GNUNET_PQ_PREPARED_STATEMENT_END
};
@@ -167,8 +132,8 @@ init_connection (struct Plugin *plugin)
plugin->dbh = GNUNET_PQ_connect_with_cfg (plugin->env->cfg,
"datastore-postgres",
+ "datastore-",
NULL,
- es,
ps);
if (NULL == plugin->dbh)
return GNUNET_SYSERR;
@@ -389,7 +354,7 @@ process_result (void *cls,
for (unsigned int i = 0; i < num_results; i++)
{
int iret;
- uint32_t rowid;
+ uint64_t rowid;
uint32_t utype;
uint32_t anonymity;
uint32_t replication;
@@ -406,7 +371,7 @@ process_result (void *cls,
GNUNET_PQ_result_spec_absolute_time ("expire", &expiration_time),
GNUNET_PQ_result_spec_auto_from_type ("hash", &key),
GNUNET_PQ_result_spec_variable_size ("value", &data, &size),
- GNUNET_PQ_result_spec_uint32 ("oid", &rowid),
+ GNUNET_PQ_result_spec_uint64 ("oid", &rowid),
GNUNET_PQ_result_spec_end
};
@@ -439,7 +404,7 @@ process_result (void *cls,
if (iret == GNUNET_NO)
{
struct GNUNET_PQ_QueryParam param[] = {
- GNUNET_PQ_query_param_uint32 (&rowid),
+ GNUNET_PQ_query_param_uint64 (&rowid),
GNUNET_PQ_query_param_end
};
@@ -635,9 +600,8 @@ repl_proc (void *cls,
struct ReplCtx *rc = cls;
struct Plugin *plugin = rc->plugin;
int ret;
- uint32_t oid = (uint32_t) uid;
struct GNUNET_PQ_QueryParam params[] = {
- GNUNET_PQ_query_param_uint32 (&oid),
+ GNUNET_PQ_query_param_uint64 (&uid),
GNUNET_PQ_query_param_end
};
enum GNUNET_DB_QueryStatus qret;
@@ -940,9 +904,6 @@ libgnunet_plugin_datastore_postgres_init (void *cls)
api->get_keys = &postgres_plugin_get_keys;
api->drop = &postgres_plugin_drop;
api->remove_key = &postgres_plugin_remove_key;
- GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
- "datastore-postgres",
- _ ("Postgres database running\n"));
return api;
}
diff --git a/src/include/gnunet_datastore_plugin.h
b/src/include/gnunet_datastore_plugin.h
index 3de08b788..4d1a7ff67 100644
--- a/src/include/gnunet_datastore_plugin.h
+++ b/src/include/gnunet_datastore_plugin.h
@@ -101,17 +101,17 @@ struct GNUNET_DATASTORE_PluginEnvironment
* @return #GNUNET_OK to keep the item
* #GNUNET_NO to delete the item
*/
-typedef int
-(*PluginDatumProcessor) (void *cls,
- const struct GNUNET_HashCode *key,
- uint32_t size,
- const void *data,
- enum GNUNET_BLOCK_Type type,
- uint32_t priority,
- uint32_t anonymity,
- uint32_t replication,
- struct GNUNET_TIME_Absolute expiration,
- uint64_t uid);
+typedef enum GNUNET_GenericReturnValue
+(*PluginDatumProcessor)(void *cls,
+ const struct GNUNET_HashCode *key,
+ uint32_t size,
+ const void *data,
+ enum GNUNET_BLOCK_Type type,
+ uint32_t priority,
+ uint32_t anonymity,
+ uint32_t replication,
+ struct GNUNET_TIME_Absolute expiration,
+ uint64_t uid);
/**
diff --git a/src/namecache/namecache.conf.in b/src/namecache/namecache.conf.in
index cf1340846..6bf75454b 100644
--- a/src/namecache/namecache.conf.in
+++ b/src/namecache/namecache.conf.in
@@ -24,7 +24,6 @@ FILENAME = $GNUNET_DATA_HOME/namecache/flat.db
[namecache-postgres]
CONFIG = postgres:///gnunet
-TEMPORARY_TABLE = NO
-
+SQL_DIR = ${DATADIR}/sql/
diff --git a/src/namestore/namestore.conf.in b/src/namestore/namestore.conf.in
index 697d07612..d817f3f95 100644
--- a/src/namestore/namestore.conf.in
+++ b/src/namestore/namestore.conf.in
@@ -27,8 +27,6 @@ FILENAME = $GNUNET_DATA_HOME/namestore/sqlite.db
[namestore-postgres]
# How to connect to the database
CONFIG = postgres:///gnunet
-# Use temporary tables
-TEMPORARY_TABLE = NO
# Use asynchronous commit (SET synchronous_commit TO OFF).
ASYNC_COMMIT = NO
INIT_ON_CONNECT = YES
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [gnunet] branch master updated: modernize datastore postgres implementation,
gnunet <=