[taler-grid5k] 106/141: update scripts and docker build image

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit 64e56752746f27e65beb903b65f33b06792cf5bc
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Sat Nov 6 00:01:43 2021 +0100

    update scripts and docker build image
---
 .../usr/lib/systemd/system/taler-wallet@.service   |  2 +-
 docker/Dockerfile                                  | 12 +++-
 docker/README.md                                   | 11 +++-
 docker/docker-compose.yaml                         |  9 ++-
 docker/entrypoint.sh                               | 77 ++++++++++++++++------
 experiment/run.sh                                  |  2 +-
 experiment/scripts/exchange.sh                     | 77 ++++++++++++----------
 .../scripts/{test.sh => taler-wallet-bench.sh}     |  5 +-
 experiment/scripts/wallet.sh                       | 14 ++--
 image/taler-debian11.yaml                          | 62 ++++++++++-------
 10 files changed, 174 insertions(+), 97 deletions(-)

diff --git a/configs/usr/lib/systemd/system/taler-wallet@.service 
b/configs/usr/lib/systemd/system/taler-wallet@.service
index f6ccdfc..4415f54 100755
--- a/configs/usr/lib/systemd/system/taler-wallet@.service
+++ b/configs/usr/lib/systemd/system/taler-wallet@.service
@@ -7,7 +7,7 @@ Type=simple
 Restart=always
 RestartSec=2
 EnvironmentFile=/etc/environment
-ExecStart=/root/scripts/test.sh %I
+ExecStart=/root/scripts/taler-wallet-bench.sh %I
 
 StandardOutput=journal
 StandardError=journal
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 20a941d..f8a96f0 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -5,7 +5,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 ENV GRID5K_DEST=lyon,lille
 ENV GNUNET_COMMIT_SHA=master
 ENV EXCHANGE_COMMIT_SHA=master
-ENV MEERCHANT_COMMIT_SHA=master
+ENV MERCHANT_COMMIT_SHA=master
+ENV WALLET_COMMIT_SHA=master
 ENV GRID5K_COMMIT_SHA=master
 
 RUN apt update && \
@@ -52,14 +53,18 @@ RUN apt update && \
         gnupg \
         gnupg-agent \
         expect \
-        rsync && \
+        rsync \
+        nodejs \
+        npm && \
     pip install jinja2 && \
+    npm install -g pnpm && \
     gem install kameleon-builder
 
 RUN mkdir taler && cd taler && \
     git clone https://git.gnunet.org/gnunet.git && \
     git clone https://git.taler.net/exchange.git && \
     git clone https://git.taler.net/merchant.git && \
+    git clone https://git.taler.net/wallet-core.git && \
     git clone https://git.taler.net/grid5k.git
 
 COPY entrypoint.sh /entrypoint.sh
@@ -67,4 +72,5 @@ RUN chmod +x entrypoint.sh
 
 VOLUME ["/taler"]
 
-CMD ["/bin/bash", "-c", "/entrypoint.sh"]
+ENTRYPOINT ["./entrypoint.sh"]
+CMD [""]
diff --git a/docker/README.md b/docker/README.md
index ff9e6fd..0e1edeb 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -27,7 +27,7 @@ docker run -it --rm \
            -e GRID5K_USER=<user> \
            -e GRID5K_CERT_PASSWD=<cert_passwd> \
            -v <cert_path>:/root/cert.pem \
-           taler:build
+           taler:build <ARGUMENTS>
 ```
 
 ### docker-compose
@@ -42,9 +42,11 @@ GRID5K_CERT=<cert_path>
 the build can be started with:
 
 ```bash
-GRID5K_CERT_PASSWD=<cert_passwd> docker-compose up --build
+ARGUMENTS=<ARGUMENTS> GRID5K_CERT_PASSWD=<cert_passwd> docker-compose up 
--build
 ```
 
+Or place the password (also arguments) in .env too but make sure its protected.
+
 ### Notes
 
 #### Environment Variables
@@ -53,12 +55,17 @@ GRID5K_CERT_PASSWD=<cert_passwd> docker-compose up --build
 **GRID5K_CERT**: the certificate which is used to login to the Grid5000 nodes 
(docker-compose only)
 **GRID5K_CERT_PASSWD**: the password to decrypt `GRID5K_CERT`
 **GRID5K_DEST**: comma separated list of where to copy the image to in the 
grid (lille,lyon)
+**ARGUMENTS**: args to pass to entrypoint, one of 
+  -c|--clean (run make clean) 
+  -r|--rebuild (rebuild the image - automatically set if -c is used) 
+ As per default, running the docker command again will not clean or rebuild 
the image
 
 ##### Additional
 
 **GNUNET_COMMIT_SHA**: Which commit to use of gnunet (master)
 **EXCHANGE_COMMIT_SHA**: Which commit to use of taler-exchange (master)
 **MERCHANT_COMMIT_SHA**: Which commit to use of taler-merchant (master)
+**WALLET_COMMIT_SHA**: Which commit to use of wallet-core (master)
 **GRID5K_COMMIT_SHA**: Which commit to use of this repo (master)
 
 #### VNC
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index fcca896..843c0d7 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -3,6 +3,7 @@ version: "3"
 services:
   taler-build:
     image: taler:build
+    container_name: taler-g5k
     build: .
     ports:
       - "5900:5900"
@@ -14,9 +15,10 @@ services:
       - /dev/fuse
       - /dev/net/tun
     environment:
-      GNUNET_COMMIT_SHA: dda40766b9b6df3644c945429e12f28cc9e7e0c8
-      EXCHANGE_COMMIT_SHA: b338ce6f88a6f34a663ef55b740086ed4272362b
-      MERCHANT_COMMIT_SHA: 99e7729ede69e218198a8c846e8096ad83127f6b
+      GNUNET_COMMIT_SHA: master
+      EXCHANGE_COMMIT_SHA: master
+      MERCHANT_COMMIT_SHA: master
+      WALLET_COMMIT_SHA: benchmark
       GRID5K_COMMIT_SHA: node-setup
       GRID5K_USER: ${GRID5K_USER}
       GRID5K_CERT_PASSWD: ${GRID5K_CERT_PASSWD}
@@ -25,6 +27,7 @@ services:
     volumes:
       - build:/taler
       - ${GRID5K_CERT}:/root/cert.pem
+    command: "${ARGUMENTS}"
 
 volumes:
   build:
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 3cb281d..a4eaddd 100644
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -1,51 +1,86 @@
 #!/bin/bash
 set -e
 
+while [[ $# -gt 0 ]]; do
+  case "$1" in 
+    -c|--clean)
+      CLEAN=true
+      REBUILD=true
+      echo "INFO will clean previous builds and image"
+      shift
+      ;;
+    -r|--rebuild)
+      REBUILD=true
+      echo "INFO will rebuild image"
+      shift
+      ;;
+    *)
+      shift
+      ;;
+  esac
+done
+
 TALER_HOME=/taler
 
-function checkout() {
+function prepare() {
   cd "${TALER_HOME}/$1"
   git checkout master && git pull
   git checkout "$2"
 }
 
-function install() {
-  checkout "$1" "$2"
-  ./bootstrap 
-  ./configure --enable-logging=verbose --prefix=/usr
+function build() {
+  prepare "$1" "$2"
   if [ "$CLEAN" = true ]; then
-    make clean
+    make clean || true
+  fi
+  if ! make; then
+    ./bootstrap 
+    ./configure --enable-logging=verbose --prefix=/usr || ./configure
   fi
   if [ ! -z "$3" ]; then
     /bin/bash -c "$3"
   fi
-  make -j 6
-  make install
-  make -j 6 dist
+  make dist
+}
+
+function install() {
+  build "$1" "$2" "$3"
+  if [[ $(ldconfig -p | grep "$1") == "" ]] || \
+     [ "$CLEAN" = true ]; then
+    make install
+    ldconfig
+  fi
 }
 
 install gnunet \
-        "${GNUNET_COMMIT_SHA}"
+      "${GNUNET_COMMIT_SHA}"
 
 install exchange \
-        "${EXCHANGE_COMMIT_SHA}" \
-        'find . -name "*Makefile*" -exec sed -i "/x-taler-bank.fee/d" {} \;'
+      "${EXCHANGE_COMMIT_SHA}" \
+      'find . -name "*Makefile*" -exec sed -i "/x-taler-bank.fee/d" {} \;'
 
 install merchant \
-        "${MERCHANT_COMMIT_SHA}"
+      "${MERCHANT_COMMIT_SHA}"
 
-checkout grid5k \
-         "${GRID5K_COMMIT_SHA}" 
+build wallet-core \
+      "${WALLET_COMMIT_SHA}"
 
-cd image
+prepare grid5k \
+      "${GRID5K_COMMIT_SHA}" 
 
-mv "${TALER_HOME}"/gnunet/gnunet*.tar.gz grid5000/steps/data/gnunet.tar.gz
-mv "${TALER_HOME}"/exchange/taler-exchange*.tar.gz 
grid5000/steps/data/exchange.tar.gz
-mv "${TALER_HOME}"/merchant/taler-merchant*.tar.gz 
grid5000/steps/data/merchant.tar.gz
+cd image
 
-rm -rf build
+if [ ! -f "build/taler-debian11/taler-debian11.tar.zst" ] || \
+   [ "$REBUILD" = true ];
+then
+  mv "${TALER_HOME}"/gnunet/gnunet*.tar.gz grid5000/steps/data/gnunet.tar.gz
+  mv "${TALER_HOME}"/exchange/taler-exchange*.tar.gz 
grid5000/steps/data/exchange.tar.gz
+  mv "${TALER_HOME}"/merchant/taler-merchant*.tar.gz 
grid5000/steps/data/merchant.tar.gz
+  mv "${TALER_HOME}"/wallet-core/taler-wallet*.tar.gz 
grid5000/steps/data/wallet.tar.gz
 
-yes r | kameleon build taler-debian11.yaml
+  rm -rf build || true
+  yes r | kameleon build taler-debian11.yaml
+fi
 
 cd build/taler-debian11
 
diff --git a/experiment/run.sh b/experiment/run.sh
index 582b6a2..f3bb6cd 100644
--- a/experiment/run.sh
+++ b/experiment/run.sh
@@ -49,7 +49,7 @@ case "${HOSTNAME}" in
     set_host exchange
     set_ddn ${EXCHANGE_DOMAIN}
     enable_netdelay ${DATABASE_DOMAIN}
-    exec ~/scripts/exchange.sh 
+    exec ~/scripts/exchange.sh init
     ;;
   ${MERCHANT_HOSTS})
     set_host merchant
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index 6aac170..09da484 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -1,42 +1,47 @@
 #!/bin/bash
-set -eux
-
-source ~/scripts/helpers.sh
-
-restart_rsyslog
-
-sed -i 
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}/${DB_NAME}\g"
 \
-       /etc/taler/secrets/exchange-db.secret.conf
-
-sed -i "s/<BANK_HOST_HERE>/${BANK_DOMAIN}/g" \
-       /etc/taler/secrets/exchange-accountcredentials.secret.conf
-
-MASTER_KEY=$(sudo -u taler-exchange-offline taler-exchange-offline setup)
-
-sed -i -e "s/<BANK_HOST_HERE>/${BANK_DOMAIN}/g" \
-       -e "s\<BASE_URL_HERE>\http://${EXCHANGE_DOMAIN}/\g"; \
-       -e "s/<MASTER_KEY_HERE>/${MASTER_KEY}/g" \
-       /etc/taler/conf.d/exchange-business.conf
-
-wait_for_db
-# Wait another second to make sure user has permissions
-sleep 1
-
-systemctl restart taler-exchange.target
-
-for (( i=1; i < ${NUM_EXCHANGE_PROCESSES}; i++ ))
+set -ex
+
+if [[ "$1" == "init" ]];
+then   
+  source ~/scripts/helpers.sh
+  restart_rsyslog
+  
+  sed -i 
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}/${DB_NAME}\g"
 \
+       /etc/taler/secrets/exchange-db.secret.conf
+  
+  sed -i "s/<BANK_HOST_HERE>/${BANK_DOMAIN}/g" \
+       /etc/taler/secrets/exchange-accountcredentials.secret.conf
+  
+  MASTER_KEY=$(sudo -u taler-exchange-offline taler-exchange-offline setup)
+  
+  sed -i -e "s/<BANK_HOST_HERE>/${BANK_DOMAIN}/g" \
+         -e "s\<BASE_URL_HERE>\http://${EXCHANGE_DOMAIN}/\g"; \
+         -e "s/<MASTER_KEY_HERE>/${MASTER_KEY}/g" \
+       /etc/taler/conf.d/exchange-business.conf
+  
+  wait_for_db
+  # Wait another second to make sure user has permissions
+  sleep 1
+
+  systemctl restart taler-exchange.target
+
+  wait_for_keys "${EXCHANGE_DOMAIN}/management"
+  
+  taler-exchange-offline download > sig-req.json
+  taler-exchange-offline sign < sig-req.json > sig-res.json
+  taler-exchange-offline enable-account $(taler-config -s exchange-account-1 
-o payto_uri) > acct-res.json
+  taler-exchange-offline wire-fee 2021 x-taler-bank KUDOS:0 KUDOS:0 > 
fee-res.json
+  taler-exchange-offline upload < sig-res.json
+  taler-exchange-offline upload < acct-res.json
+  taler-exchange-offline upload < fee-res.json
+fi
+  
+RUNNING=$(ps -aux | grep "[taler]-exchange-httpd" | wc -l)
+
+for i in $(seq ${NUM_EXCHANGE_PROCESSES});
 do
+  let "i+=${RUNNING}"
   systemctl restart taler-exchange-httpd@100"${i}".socket 
taler-exchange-httpd@100"${i}".service
 done
 
-wait_for_keys "${EXCHANGE_DOMAIN}/management"
-
-taler-exchange-offline download > sig-req.json
-taler-exchange-offline sign < sig-req.json > sig-res.json
-taler-exchange-offline enable-account $(taler-config -s exchange-account-1 -o 
payto_uri) > acct-res.json
-taler-exchange-offline wire-fee 2021 x-taler-bank KUDOS:0 KUDOS:0 > 
fee-res.json
-taler-exchange-offline upload < sig-res.json
-taler-exchange-offline upload < acct-res.json
-taler-exchange-offline upload < fee-res.json
-
 exit 0
diff --git a/experiment/scripts/test.sh 
b/experiment/scripts/taler-wallet-bench.sh
similarity index 85%
rename from experiment/scripts/test.sh
rename to experiment/scripts/taler-wallet-bench.sh
index 24b25f8..be0462f 100755
--- a/experiment/scripts/test.sh
+++ b/experiment/scripts/taler-wallet-bench.sh
@@ -1,12 +1,15 @@
 #!/bin/bash
 
+LOG_LEVEL=NONE
+
 case $1 in
    *logging*)
-   echo "enable logging of wallet here"
+   LOG_LEVEL=INFO
    ;;
 esac
 
 taler-wallet-cli \
+    -L ${LOG_LEVEL} \
     advanced bench1 \
       --config-json "
 {
diff --git a/experiment/scripts/wallet.sh b/experiment/scripts/wallet.sh
index 204afd9..f53e93a 100755
--- a/experiment/scripts/wallet.sh
+++ b/experiment/scripts/wallet.sh
@@ -2,15 +2,19 @@
 
 NUM_PROCESSES=${NUM_WALLET_PROCESSES:-10}
 
-if [[ "$1" == "init" ]] && [ "$NUM_PROCESSES" -gt "0" ]; 
+if [[ "$1" == "init" ]];  
 then
   source ~/scripts/helpers.sh
+  restart_rsyslog
   wait_for_keys "${PROXY_DOMAIN}"
   sleep 5
 
-  # Start one process with logging enabled
-  let "NUM_PROCESSES-=1"
-  systemctl restart taler-wallet@logging.service
+  if [ "$NUM_PROCESSES" -gt "0" ];
+  then
+    # Start one process with logging enabled
+    let "NUM_PROCESSES-=1"
+    systemctl restart taler-wallet@logging.service
+  fi
 fi
 
 # count the running wallets
@@ -21,3 +25,5 @@ do
   let "i+=${RUNNING}"
   systemctl restart taler-wallet@${i}.service
 done
+
+exit 0
diff --git a/image/taler-debian11.yaml b/image/taler-debian11.yaml
index ed2cc7b..1acfa7b 100644
--- a/image/taler-debian11.yaml
+++ b/image/taler-debian11.yaml
@@ -43,7 +43,15 @@ global:
   other_packages_no_clean: "$${taler_packages} 
$${taler_packages_no_recommends} $${taler_build_packages}" 
   taler_disable_services: "nginx postgresql prometheus 
prometheus-postgres-exporter prometheus-nginx-exporter prometheus-node-exporter"
   taler_loki_version: "v2.3.0"
-
+  gnunet: gnunet.tar.gz
+  exchange: exchange.tar.gz
+  merchant: merchant.tar.gz
+  wallet: wallet.tar.gz
+  taler_path: /root/taler
+  gnunet_path: $${taler_path}/$${gnunet}
+  exchange_path: $${taler_path}/$${exchange}
+  merchant_path: $${taler_path}/$${merchant}
+  wallet_path: $${taler_path}/$${wallet}
   ## Other parameters can be changed, see kameleon info debian10-taler.yaml
 
 bootstrap:
@@ -59,12 +67,6 @@ setup:
   ## kameleon dryrun debian10_custom.yaml to see the resulting steps in the 
build.
   ## The following is given as example only, replace with your steps.
   - import:
-    - gnunet: gnunet.tar.gz
-    - exchange: exchange.tar.gz
-    - merchant: merchant.tar.gz
-    - gnunet_path: /root/$${gnunet}
-    - exchange_path: /root/$${exchange}
-    - merchant_path: /root/$${merchant}
     - import_gnunet:
       # kameleon_data_dir=./grid5000/steps/data
       - local2in:
@@ -78,6 +80,10 @@ setup:
       - local2in:
         - $${kameleon_data_dir}/$${merchant}
         - $${merchant_path}
+    - import_wallet:
+      - local2in:
+        - $${kameleon_data_dir}/$${wallet}
+        - $${wallet_path}
   - install:
     - packages:
       - exec_in: |
@@ -87,34 +93,40 @@ setup:
          apt upgrade
          apt install -y $${taler_packages} $${taler_build_packages}
          apt install -y --no-install-recommends 
$${taler_packages_no_recommends}
-    - build_taler: 
+    - gnunet: 
       - exec_in: |
-         cd /root 
+         cd $${taler_path} 
 
-         # GNUNet
-         tar -xvf gnunet.tar.gz
-         cd gnunet-* && ./configure --enable-logging=verbose --prefix=/usr && 
make && make install 
+         tar -xvf $${gnunet}
+         cd gnunet-* 
+         ./configure --enable-logging=verbose --prefix=/usr && make && make 
install 
          ldconfig
+    - exchange:
+      - exec_in: |
+         cd $${taler_path}
 
-         # Exchange
-         cd /root
          pip install jinja2
-         tar -xvf exchange.tar.gz
-         cd taler-exchange* && ./configure --enable-logging=verbose 
--prefix=/usr && make && make install
+         tar -xvf $${exchange}
+         cd taler-exchange*
+         ./configure --enable-logging=verbose --prefix=/usr && make && make 
install
          ldconfig
 
-         # Merchant
-         cd /root
-         tar -xvf merchant.tar.gz
-         cd taler-merchant* && ./configure --enable-logging=verbose 
--prefix=/usr && make && make install
+    - merchant:
+      - exec_in: |
+         cd $${taler_path}
+
+         tar -xvf $${merchant}
+         cd taler-merchant* 
+         ./configure --enable-logging=verbose --prefix=/usr && make && make 
install
          ldconfig
+    - wallet: |
+      - exec_in: |
+         cd $${taler_path}
 
-         # Wallet
-         cd /root
-         git clone https://git.taler.net/wallet-core.git
-         cd wallet-core && git checkout benchmark
+         tar -xvf $${wallet}
+         cd taler-wallet*
          npm install -g pnpm
-         ./bootstrap && ./configure && make install
+         ./configure && make install
     - loki-promtail:
       - exec_in: |
          apt-get install -y unzip curl

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.