[taler-grid5k] 113/141: tune postgres config

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit c94f4116d865c5a4c7425b567707e49011bf8504
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Sun Nov 7 10:05:06 2021 +0100

    tune postgres config
---
 docker/entrypoint.sh              |  11 +++-
 experiment/run.sh                 |   6 +-
 experiment/scripts/createusers.sh |   1 -
 experiment/scripts/database.sh    |  51 ++++++++++-------
 experiment/scripts/merchant.sh    |  46 ++++++++-------
 experiment/scripts/monitor.sh     | 117 +++++++++++++++++++-------------------
 6 files changed, 131 insertions(+), 101 deletions(-)

diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index a4386a6..1224f8b 100644
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -75,12 +75,21 @@ then
          exp_continue
         }
         eof {}
-      }"
+      }" > /tmp/expect.log
     else 
       ssh-add /root/cert.pem
     fi
 
   fi
+
+  if ! ssh-add -L > /dev/null ; then
+    echo "ERROR could not add certificate"
+    cat /tmp/expect.log
+    echo ""
+    exit 1
+  else
+    echo "INFO added certificate"
+  fi
 fi
 
 install gnunet \
diff --git a/experiment/run.sh b/experiment/run.sh
index f3bb6cd..7052d7f 100644
--- a/experiment/run.sh
+++ b/experiment/run.sh
@@ -43,7 +43,7 @@ case "${HOSTNAME}" in
   ${DATABASE_HOSTS}) 
     set_host database
     set_ddn ${DATABASE_DOMAIN}
-    exec ~/scripts/database.sh 
+    exec ~/scripts/database.sh init
     ;;
   ${EXCHANGE_HOSTS}) 
     set_host exchange
@@ -54,12 +54,12 @@ case "${HOSTNAME}" in
   ${MERCHANT_HOSTS})
     set_host merchant
     set_ddn ${MERCHANT_DOMAIN}
-    exec ~/scripts/merchant.sh
+    exec ~/scripts/merchant.sh init
     ;;
   ${MONITOR_HOSTS})
     set_host monitor
     set_ddn ${MONITOR_DOMAIN}
-    exec ~/scripts/monitor.sh
+    exec ~/scripts/monitor.sh init
     ;;
   ${PROXY_HOSTS})
     set_host proxy
diff --git a/experiment/scripts/createusers.sh 
b/experiment/scripts/createusers.sh
index 1efcf7b..a7de2a7 100755
--- a/experiment/scripts/createusers.sh
+++ b/experiment/scripts/createusers.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-
 set -e
 
 . /usr/share/debconf/confmodule
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index da0fb17..9961271 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -6,24 +6,34 @@ source ~/scripts/helpers.sh
 # move to tmp to prevent change dir errors
 cd /tmp 
 
-sed -i "s\<DB_URL_HERE>\postgresql:///${DB_NAME}\g" \
-       /etc/taler/secrets/exchange-db.secret.conf
-
-echo "
-listen_addresses='*'
-log_destination=syslog
-syslog_ident='taler-database'
-log_min_duration_statement=500
-shared_preload_libraries='pg_stat_statements,auto_explain'
-" >> /etc/postgresql/13/main/postgresql.conf
-
-# Enable password for taler since this is commonly the case
-# For the postgres user do not enable authentication (used in metrics)
-echo "
-host all ${DB_USER} 172.16.0.0/12 md5 
-host all postgres 172.16.0.0/12 trust
-" >> /etc/postgresql/13/main/pg_hba.conf
-
+if [[ "$1" == "init" ]];
+then
+  sed -i "s\<DB_URL_HERE>\postgresql:///${DB_NAME}\g" \
+       /etc/taler/secrets/exchange-db.secret.conf
+  
+  echo "
+  listen_addresses='*'
+  log_destination=syslog
+  syslog_ident='taler-database'
+  log_min_duration_statement=500
+  shared_preload_libraries='pg_stat_statements,auto_explain'
+  
+  # use 25% of the available memory 
+  # (https://www.postgresql.org/docs/13/runtime-config-resource.html)
+  shared_buffers=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) / 4 ))kB
+  # 
(https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-MAX-WAL-SIZE)
+  max_wal_size=2GB 
+  max_worker_processes=12
+  " >> /etc/postgresql/13/main/postgresql.conf
+  
+  # Enable password for taler since this is commonly the case
+  # For the postgres user do not enable authentication (used in metrics)
+  echo "
+  host all ${DB_USER} 172.16.0.0/12 md5 
+  host all postgres 172.16.0.0/12 trust
+  " >> /etc/postgresql/13/main/pg_hba.conf
+fi
+  
 systemctl restart postgresql 
 
 su postgres << EOF
@@ -35,7 +45,10 @@ psql -tAc "SELECT 1 FROM pg_database WHERE 
datname='${DB_NAME}'" | \
   createdb -O taler-exchange-httpd "${DB_NAME}"
 EOF
 
-sudo -u taler-exchange-httpd taler-exchange-dbinit
+if [[ "$1" == "init" ]]; 
+then
+  sudo -u taler-exchange-httpd taler-exchange-dbinit
+fi
 
 su postgres << EOF
 psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='${DB_USER}'" | \
diff --git a/experiment/scripts/merchant.sh b/experiment/scripts/merchant.sh
index 2861d71..ceeaa2e 100755
--- a/experiment/scripts/merchant.sh
+++ b/experiment/scripts/merchant.sh
@@ -7,25 +7,28 @@ cd /tmp
 
 restart_rsyslog
 
-wait_for_keys "${PROXY_DOMAIN}"
-
-MASTER_KEY=$(
-  curl -k -f \
-     "${PROXY_DOMAIN}/keys" \
-     | jq -r '.master_public_key'
-)
-
-sed -i -e "s\<EXCHANGE_URL_HERE>\http://${EXCHANGE_DOMAIN}/\g"; \
-       -e "s/<EXCHANGE_MASTER_KEY_HERE>/${MASTER_KEY}/g" \
-        /etc/taler/conf.d/merchant.conf
-
-sed -i "s/local\s* all\s* postgres\s* peer/local all postgres trust/g" \
-       /etc/postgresql/13/main/pg_hba.conf
-
-echo "
-shared_preload_libraries='pg_stat_statements,auto_explain'
-" >> /etc/postgresql/13/main/postgresql.conf
-
+if [[ "$1" == "init" ]];
+then
+  wait_for_keys "${PROXY_DOMAIN}"
+  
+  MASTER_KEY=$(
+    curl -k -f \
+       "${PROXY_DOMAIN}/keys" \
+       | jq -r '.master_public_key'
+  )
+  
+  sed -i -e "s\<EXCHANGE_URL_HERE>\http://${EXCHANGE_DOMAIN}/\g"; \
+         -e "s/<EXCHANGE_MASTER_KEY_HERE>/${MASTER_KEY}/g" \
+          /etc/taler/conf.d/merchant.conf
+  
+  sed -i "s/local\s* all\s* postgres\s* peer/local all postgres trust/g" \
+       /etc/postgresql/13/main/pg_hba.conf
+  
+  echo "
+  shared_preload_libraries='pg_stat_statements,auto_explain'
+  " >> /etc/postgresql/13/main/postgresql.conf
+fi
+  
 systemctl restart postgresql
 
 su postgres << EOF
@@ -39,7 +42,10 @@ psql
 CREATE EXTENSION pg_stat_statements;
 EOF
 
-sudo -u taler-merchant-httpd taler-merchant-dbinit
+if [[ "$1" == "init" ]];
+then
+  sudo -u taler-merchant-httpd taler-merchant-dbinit
+fi
 
 sed -i "s\<DB_URL_HERE>\user=postgres host=/run/postgresql 
dbname=taler-merchant\g" \
        /etc/default/prometheus-postgres-exporter
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
index bf20b60..a80e393 100755
--- a/experiment/scripts/monitor.sh
+++ b/experiment/scripts/monitor.sh
@@ -3,62 +3,65 @@ set -eux
 
 source ~/scripts/helpers.sh
 
-AUTH_HEADER="Authorization: Bearer ${GRAFANA_API_KEY}"
-GRAFANA_API="https://$(host ${DNS_HOST} | sed -n 1p | awk '{print 
$4}'):3000/api"
-
-if ! curl -k -f -H "${AUTH_HEADER}" \
-  "${GRAFANA_API}/datasources" \
-  -o ds.json ;
-then 
-  echo "Failed to retrieve datasources from Grafana"
-  exit $?
-fi
-
-function update_datasource() {
-  ID=$(jq --arg name "$1" '.[] | select(.name == $name) | .id' ds.json)
-
-  jq --arg url "https://$(hostname | cut -d "." -f 1,2 
-).$2.proxy.grid5000.fr" \
-     --arg name "$1" \
-     '.[] | select(.name == $name) | .url = $url' \
-     ds.json | tee /dev/tty | curl -X PUT -k -f -d @- \
-       -H "${AUTH_HEADER}" \
-       -H "Content-Type: application/json" \
-       -H "Accept: application/json" \
-        "${GRAFANA_API}/datasources/${ID}"
-}
-
-update_datasource "${PROMETHEUS_DATASOURCE_NAME}" 
"${PROMETHEUS_G5K_PROXY_PORT}"
-update_datasource "${LOKI_DATASOURCE_NAME}" "${LOKI_G5K_PROXY_PORT}"
-
-systemctl restart loki \
-                 promtail
-
-sed -i "s/<MERCHANT_HOST_HERE>/${MERCHANT_DOMAIN}/g" \
-       /etc/monitor/prometheus.yaml
-
-sed -i "s\<DB_URL_HERE>\postgresql://postgres@${DATABASE_DOMAIN}:5432\g" \
-       /etc/default/prometheus-postgres-exporter
-
-sed -i "s\<PROXY_URL_HERE>\http://${PROXY_DOMAIN}/stub_status\g"; \
-       /etc/default/prometheus-nginx-exporter
-
-wait_for_db
-
-# Initialize prometheus after the db is ready, then all dns records have been 
set for sure
-if [[ "${ENABLE_EXPORTERS}" == "true" ]];
+if [[ "$1" == "init" ]];
 then
-  cat /etc/monitor/node-exporters.yaml.tpl >> /etc/monitor/prometheus.yaml
-  for WALLET in $(get_wallet_domains);
-  do
-    sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ - 
'wallet.${WALLET}.perf.taler:9100'" \
-           /etc/monitor/prometheus.yaml
-  done
+  AUTH_HEADER="Authorization: Bearer ${GRAFANA_API_KEY}"
+  GRAFANA_API="https://$(host ${DNS_HOST} | sed -n 1p | awk '{print 
$4}'):3000/api"
+  
+  if ! curl -k -f -H "${AUTH_HEADER}" \
+    "${GRAFANA_API}/datasources" \
+    -o ds.json ;
+  then 
+    echo "Failed to retrieve datasources from Grafana"
+    exit $?
+  fi
+  
+  function update_datasource() {
+    ID=$(jq --arg name "$1" '.[] | select(.name == $name) | .id' ds.json)
+  
+    jq --arg url "https://$(hostname | cut -d "." -f 1,2 
-).$2.proxy.grid5000.fr" \
+       --arg name "$1" \
+       '.[] | select(.name == $name) | .url = $url' \
+       ds.json | tee /dev/tty | curl -X PUT -k -f -d @- \
+         -H "${AUTH_HEADER}" \
+         -H "Content-Type: application/json" \
+         -H "Accept: application/json" \
+          "${GRAFANA_API}/datasources/${ID}"
+  }
+  
+  update_datasource "${PROMETHEUS_DATASOURCE_NAME}" 
"${PROMETHEUS_G5K_PROXY_PORT}"
+  update_datasource "${LOKI_DATASOURCE_NAME}" "${LOKI_G5K_PROXY_PORT}"
+  
+  systemctl restart loki \
+                 promtail
+  
+  sed -i "s/<MERCHANT_HOST_HERE>/${MERCHANT_DOMAIN}/g" \
+       /etc/monitor/prometheus.yaml
+  
+  sed -i "s\<DB_URL_HERE>\postgresql://postgres@${DATABASE_DOMAIN}:5432\g" \
+       /etc/default/prometheus-postgres-exporter
+  
+  sed -i "s\<PROXY_URL_HERE>\http://${PROXY_DOMAIN}/stub_status\g"; \
+       /etc/default/prometheus-nginx-exporter
+  
+  wait_for_db
+  
+  # Initialize prometheus after the db is ready, then all dns records have 
been set for sure
+  if [[ "${ENABLE_EXPORTERS}" == "true" ]];
+  then
+    cat /etc/monitor/node-exporters.yaml.tpl >> /etc/monitor/prometheus.yaml
+    for WALLET in $(get_wallet_domains);
+    do
+      sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ - 
'wallet.${WALLET}.perf.taler:9100'" \
+           /etc/monitor/prometheus.yaml
+    done
+  fi
+  
+  systemctl restart prometheus
+  
+  # Proxy takes longer to start
+  wait_for_keys "${PROXY_DOMAIN}"
+  
+  systemctl restart prometheus-postgres-exporter \
+                 prometheus-nginx-exporter
 fi
-
-systemctl restart prometheus
-
-# Proxy takes longer to start
-wait_for_keys "${PROXY_DOMAIN}"
-
-systemctl restart prometheus-postgres-exporter \
-                 prometheus-nginx-exporter

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.