[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[taler-deployment] 02/02: taler-local: prepare key material
|
From: |
gnunet |
|
Subject: |
[taler-deployment] 02/02: taler-local: prepare key material |
|
Date: |
Sun, 26 Sep 2021 23:45:35 +0200 |
This is an automated email from the git hooks/post-receive script.
ms pushed a commit to branch master
in repository deployment.
commit 78a499ecc87544b5ca95acf1d6addd05c7153cf5
Author: ms <ms@taler.net>
AuthorDate: Sun Sep 26 23:44:43 2021 +0200
taler-local: prepare key material
---
bin/WIP/taler-local | 292 +++++++++++++++++++++++++++++++++++++---------------
1 file changed, 208 insertions(+), 84 deletions(-)
diff --git a/bin/WIP/taler-local b/bin/WIP/taler-local
index 6863e90..af7d633 100755
--- a/bin/WIP/taler-local
+++ b/bin/WIP/taler-local
@@ -15,6 +15,7 @@
# You should have received a copy of the GNU General Public License
# along with GNU Taler. If not, see <https://www.gnu.org/licenses/>.
+import shutil
import atexit
import click
import types
@@ -24,12 +25,14 @@ import os.path
import subprocess
import time
import random
+import logging
from os import listdir
from os.path import isdir, join
from pathlib import Path
from dataclasses import dataclass
from typing import List, Callable
from shutil import copy
+from multiprocessing import Process
from string import ascii_letters, ascii_uppercase
from sys import exit
from urllib.parse import urljoin
@@ -38,8 +41,12 @@ import requests
from collections import OrderedDict
import errno
from pathlib import Path
-from subprocess import check_call as Popen, DEVNULL
+from subprocess import Popen, DEVNULL, PIPE
from datetime import datetime
+from requests_unixsocket import Session
+from flask import Flask, request, Response
+from werkzeug.datastructures import Headers
+from werkzeug.exceptions import HTTPException
@dataclass
class Repo:
@@ -320,6 +327,7 @@ def get_stale_repos(repos: List[Repo]) -> List[Repo]:
if not s.exists():
timestamps[r.name] = time.time()
stale.append(r)
+ continue
ts = timestamps[r.name] = s.stat().st_mtime
for dep in r.deps:
if timestamps[dep] > ts:
@@ -334,7 +342,9 @@ def get_stale_repos(repos: List[Repo]) -> List[Repo]:
of the repositories to _exclude_ from compilation",
default="")
def build(without_repos) -> None:
+
"""Build the deployment from source."""
+
exclude = split_repos_list(without_repos)
# Get the repositories names from the source directory
repos_names = get_repos_names()
@@ -385,47 +395,132 @@ def checkout_repos(repos: List[Repo]):
def bootstrap(repos) -> None:
"""Clone all the specified repositories."""
-
home = Path.home()
reposList = split_repos_list(repos)
checkout_repos(load_repos(reposList))
+class TalerReverseProxy(Flask):
+ def stop(self):
+ self.proc.terminate()
+ self.proc.join()
+ del os.environ["WERKZEUG_RUN_MAIN"]
+
+ def start(self):
+ logging.basicConfig(filename="/tmp/reverse-proxy.log", filemode="a")
+ logger = logging.getLogger("werkzeug")
+ logger.setLevel(logging.DEBUG)
+ self.proc = Process(
+ target=self.run,
+ kwargs=dict(debug=False, port=8080, host="localhost")
+ )
+ os.environ["WERKZEUG_RUN_MAIN"] = "true"
+ self.proc.start()
+ atexit.register(self.stop)
+
+ def index(self):
+ return "I'm the Taler reverse proxy."
+
+ def proxy(self, component, path=""):
+ logger = logging.getLogger("werkzeug")
+ logger.setLevel(logging.DEBUG)
+ s = Session()
+ uri = f"http+unix://%2Ftmp%2F{component}.sock/{path}"
+ raw_data = request.get_data()
+ if len(request.args) > 0:
+ uri += f"?{request.query_string.decode()}"
+ logger.debug("Proxying to: " + uri)
+ try:
+ if request.method == "GET":
+ method = s.get
+ if request.method == "POST":
+ method = s.post
+ if request.method == "PUT":
+ method = s.put
+ if request.method == "DELETE":
+ method = s.delete
+ resp = method(uri, headers=request.headers, data=raw_data)
+ except Exception as error:
+ logger.error(error)
+ return "Could not connect to upstream", 500
+ logger.debug(f"Upstream responds: {resp.text}")
+ headers = Headers()
+ for k in resp.headers.keys():
+ headers.set(k, resp.headers[k])
+ return Response(
+ response=resp.text,
+ status=resp.status_code,
+ headers=headers
+ )
+
+ def get_app(self):
+ return self
+
+ def __init__(self):
+ super().__init__("taler-proxy")
+ all_methods = [
+ "GET", "POST", "HEAD",
+ "DELETE", "OPTIONS", "PUT"
+ ]
+ self.add_url_rule("/", view_func=self.index)
+ self.add_url_rule("/<component>", view_func=self.proxy,
methods=all_methods)
+ self.add_url_rule("/<component>/", view_func=self.proxy,
methods=all_methods)
+ self.add_url_rule("/<component>/<path:path>", view_func=self.proxy,
methods=all_methods)
+
@cli.command()
def prepare():
+
+ """Generate configuration, run-time blobs, instances, euFin accounts."""
+
+ def fail(reason=None):
+ if reason:
+ print("ERROR: " + reason)
+ print("See logs in /tmp/$component.log")
+ exit(1)
+
# Print No Newline.
def print_nn(msg):
- print(msg, env="")
+ print(msg, end="")
# Runs a command synchronously.
- def cmd(args, env=os.environ, custom_name=None):
- handle = launch(args, env, custom_name)
+ def cmd(
+ args, env=os.environ,
+ custom_name=None, return_stdout=False
+ ):
+ handle = launch(
+ args, env=env,
+ custom_name=custom_name,
+ return_stdout=return_stdout
+ )
return_code = handle.wait()
if return_code != 0:
- print("Command: " + " ".join(args) + " failed, return code: " +
str(return_code))
- print(f"See logs in /tmp/$command.log")
- return False
+ print("\nCommand: " + " ".join(args) + " failed, return code: " +
str(return_code))
+ name = custom_name if custom_name else args[0]
+ print(f"See logs in /tmp/{name}.log")
+ exit(return_code)
return handle
- def kill(s):
- s.terminate()
- s.wait()
+ def kill(proc):
+ proc.terminate()
+ proc.wait()
# Runs a command in the background.
- def launch(cmd, env=os.environ, custom_name=None):
+ def launch(cmd, env=os.environ, custom_name=None, return_stdout=False):
name = custom_name if custom_name else cmd[0]
- log_file = open("/tmp/{name}.log", "a+")
- handle = Popen(
- cmd,
- stdin=DEVNULL,
- stdout=log_file,
- stderr=log_file,
- env=env
- )
+ log_file = open(f"/tmp/{name}.log", "a+")
+ try:
+ handle = Popen(
+ cmd,
+ stdin=DEVNULL,
+ stdout=log_file if not return_stdout else PIPE,
+ stderr=log_file,
+ env=env
+ )
+ except Exception as error:
+ fail(f"Could not launch: {name}: {error}")
atexit.register(lambda: kill(handle))
return handle
def check_running(check_url):
- print_nn(f"Checking {check_url} is served correctly...")
for i in range(10):
print_nn(".")
try:
@@ -434,11 +529,13 @@ def prepare():
# Raises if the request gets a non 200 OK.
response.raise_for_status()
except:
- if i == 10:
- print("ERROR: {name} is not running (correctly)")
+ if i == 9:
+ print("FAIL")
return False
- print(" OK")
- return True
+ time.sleep(0.5)
+ continue
+ break
+ return True
def get_nexus_cli_env(
username, password,
@@ -457,7 +554,12 @@ def prepare():
env["LIBEUFIN_SANDBOX_USERNAME"] = username
env["LIBEUFIN_SANDBOX_PASSWORD"] = password
return env
-
+
+ def get_sandbox_server_env(db_file):
+ env = os.environ.copy()
+ env["LIBEUFIN_SANDBOX_DB_CONNECTION"] = f"jdbc:sqlite:{db_file}"
+ return env
+
def prepare_nexus_account(
ebics_url,
ebics_host_id,
@@ -606,11 +708,13 @@ def prepare():
currency,
rev_proxy_url,
wire_method,
+ exchange_master_pub,
exchange_wire_address,
merchant_wire_address,
exchange_wire_gateway_username,
exchange_wire_gateway_password,
frontend_api_key,
+ taler_runtime_dir
):
def coin(
obj,
@@ -637,10 +741,11 @@ def prepare():
obj.cfg_put(sec, "fee_deposit", currency + ":" + f_deposit)
obj.cfg_put(sec, "rsa_keysize", rsa_keysize)
-
obj = ConfigFile("taler.conf")
obj.cfg_put("paths", "TALER_DATA_HOME", "${HOME}/.taler-data")
- obj.cfg_put("paths", "TALER_RUNTIME_DIR", "${HOME}/.taler-runtime")
+ if not os.path.isdir(taler_runtime_dir):
+ os.makedirs(taler_runtime_dir)
+ obj.cfg_put("paths", "TALER_RUNTIME_DIR", str(taler_runtime_dir))
obj.cfg_put("taler", "CURRENCY", currency)
obj.cfg_put("taler", "CURRENCY_ROUND_UNIT", f"{currency}:0.01")
@@ -689,6 +794,10 @@ def prepare():
"merchant-exchange-{}".format(currency),
"exchange_base_url", rev_proxy_url + "/exchange/",
)
+ obj.cfg_put(
+ "merchant-exchange-{}".format(currency),
+ "master_key", exchange_master_pub,
+ )
obj.cfg_put("auditor", "serve", "unix")
# FIXME: both below used?
@@ -700,17 +809,18 @@ def prepare():
obj.cfg_put("taler-exchange-secmod-eddsa", "unixpath",
"/tmp/exchange-secmod-eddsa.sock")
obj.cfg_put("taler-exchange-secmod-rsa", "unixpath",
"/tmp/exchange-secmod-rsa.sock")
obj.cfg_put("taler-exchange-secmod-rsa", "sm_priv_key",
-
"/tmp/taler-data/taler-exchange-secmod-rsa/secmod-private-key"
+
"/tmp/.taler-data/taler-exchange-secmod-rsa/secmod-private-key"
)
obj.cfg_put("exchange", "base_url", rev_proxy_url + "/exchange/")
+ obj.cfg_put("exchange", "master_public_key", exchange_master_pub)
obj.cfg_put("exchange", "serve", "unix")
obj.cfg_put("exchange", "unixpath", "/tmp/exchange.sock")
obj.cfg_put("exchange", "terms_etag", "0")
- obj.cfg_put("exchange", "terms_dir",
"$HOME/local/share/taler-exchange/tos")
+ obj.cfg_put("exchange", "terms_dir",
"$HOME/.local/share/taler-exchange/tos")
obj.cfg_put("exchange", "privacy_etag", "0")
- obj.cfg_put("exchange", "privacy_dir",
"$HOME/local/share/taler-exchange/pp")
+ obj.cfg_put("exchange", "privacy_dir",
"$HOME/.local/share/taler-exchange/pp")
obj.cfg_put("exchangedb-postgres", "db_conn_str", "postgres:///taler")
@@ -777,10 +887,14 @@ def prepare():
obj.cfg_put("authorization-question", "cost", f"{currency}:0")
obj.cfg_put("authorization-question", "enabled", "yes")
obj.cfg_write(outdir)
-
+
CURRENCY = "EUR"
WIRE_METHOD = "sepa"
+
+ # Directories
CFG_OUTDIR = Path.home() / ".config"
+ TALER_RUNTIME_DIR = Path.home() / ".taler-runtime"
+ TALER_DATA_DIR = Path.home() / ".taler-data"
# IBANs
IBAN_EXCHANGE = "EX00000000000000000000"
@@ -814,17 +928,33 @@ def prepare():
EXCHANGE_BANK_CONNECTION = "exchange-ebics-connection"
EXCHANGE_FACADE_NAME = "exchange-taler-facade"
+ print_nn("Remove stale data..")
+ if TALER_DATA_DIR.exists():
+ shutil.rmtree(TALER_DATA_DIR)
+ if TALER_RUNTIME_DIR.exists():
+ shutil.rmtree(TALER_RUNTIME_DIR)
+ print(" OK")
+ print_nn("Generate exchange's master key...")
+ res = cmd(
+ ["taler-exchange-offline", "setup"],
+ return_stdout=True
+ )
+ EXCHANGE_MASTER_PUB = res.communicate()[0].decode("utf-8").rstrip()
+ print(" OK")
+
mc = config_main(
"taler.conf",
outdir=CFG_OUTDIR,
currency=CURRENCY,
rev_proxy_url=REV_PROXY,
wire_method=WIRE_METHOD,
+ exchange_master_pub=EXCHANGE_MASTER_PUB,
exchange_wire_address=IBAN_EXCHANGE,
merchant_wire_address=IBAN_MERCHANT,
exchange_wire_gateway_username=EXCHANGE_NEXUS_USERNAME,
exchange_wire_gateway_password=EXCHANGE_NEXUS_PASSWORD,
- frontend_api_key=FRONTENDS_API_TOKEN
+ frontend_api_key=FRONTENDS_API_TOKEN,
+ taler_runtime_dir=TALER_RUNTIME_DIR
)
config_sync(
"sync.conf",
@@ -851,24 +981,32 @@ def prepare():
custom_name="remove-taler-data"
)
print(" OK")
-
+
+ print_nn("Launching the reverse proxy...")
+ rev_proxy = TalerReverseProxy()
+ rev_proxy.start()
+ if not check_running(REV_PROXY + "/"):
+ fail("Reverse proxy did not start correctly.")
+ # Do check.
+ print(" OK")
+ print_nn("Launching the exchange RSA helper...")
+ exchange_rsa_handle = launch(["taler-exchange-secmod-rsa"])
+ print(" OK")
+ print_nn("Launching the exchange EDDSA helper...")
+ exchange_eddsa_handle = launch(["taler-exchange-secmod-eddsa"])
+ print(" OK")
print_nn("Launching the exchange...")
exchange_handle = launch(["taler-exchange-httpd"])
- if not check_running(REV_PROXY + "/exchange/keys"):
- exit(1)
+ if not check_running(REV_PROXY + "/exchange/"):
+ fail("Exchange did not start correctly.")
print(" OK")
- print_nn("Launching the exchange RSA helper...")
- exchange_rsa_handle = launch("exchange-rsa", ["taler-exchange-secmod-rsa"])
- print_nn("Launching the exchange RSA helper...")
- exchange_eddsa_handle = launch("exchange-eddsa",
["taler-exchange-secmod-eddsa"])
- print_nn("exchange-offline: signing the exchange keys, hence testing the
crypto helpers are correct...")
+ print_nn("exchange-offline: signing key material...")
cmd(["taler-exchange-offline", "download", "sign", "upload"])
print(" OK")
- PAYTO_URI=mc.sections["exchange-account-1"]["payto-uri"]
- print_nn("exchange-offline: enabling {PAYTO_URI}...")
+ PAYTO_URI=mc.sections["exchange-account-1"]["payto_uri"]
+ print_nn(f"exchange-offline: enabling {PAYTO_URI}...")
cmd(["taler-exchange-offline", "enable-account", PAYTO_URI, "upload"])
- print(" OK")
-
+ print(" OK")
# Set up wire fees for next 5 years
NOW = datetime.now()
YEAR = NOW.year
@@ -878,7 +1016,7 @@ def prepare():
[
"taler-exchange-offline",
"wire-fee",
- year,
+ str(year),
WIRE_METHOD,
CURRENCY + ":0.01",
CURRENCY + ":0.01",
@@ -887,25 +1025,20 @@ def prepare():
custom_name="set-wire-fee"
)
print(" OK")
- print_nn("Getting exchange master public key via /keys..")
- try:
- response = requests.get(REV_PROXY + "/exchange/keys")
- response.raise_for_status()
- except:
- EXCHANGE_MASTER_PUB = response.json().get("master_public_key")
- print(" OK")
print_nn("Stopping exchange HTTP daemon and crypto helpers...")
kill(exchange_rsa_handle)
kill(exchange_eddsa_handle)
+ kill(exchange_handle)
print(" OK")
print_nn("Add this exchange to the auditor...")
cmd(
[
"taler-auditor-exchange",
"-m", EXCHANGE_MASTER_PUB,
- "-u", REV_PROXY + "/exchange"
+ "-u", REV_PROXY + "/exchange/"
],
)
+ print(" OK")
## Step 4: Set up euFin
print_nn("Resetting euFin databases...")
try:
@@ -924,17 +1057,22 @@ def prepare():
"--password", SANDBOX_ADMIN_PASSWORD
],
custom_name="sandbox-superuser",
+ env=get_sandbox_server_env(SANDBOX_DB_FILE)
)
print(" OK")
print_nn("Launching Sandbox...")
- handle_sandbox = launch("sandbox", [
- "libeufin-sandbox", "serve",
- "--with-unix-socket", "/tmp/sandbox.sock"
- ])
+ handle_sandbox = launch(
+ [
+ "libeufin-sandbox", "serve",
+ "--with-unix-socket", "/tmp/sandbox.sock",
+ ],
+ env=get_sandbox_server_env(SANDBOX_DB_FILE)
+ )
if not check_running(SANDBOX_URL):
- exit(1)
+ fail("Sandbox did not start correctly")
print(" OK")
-
+ exit(22)
+
print_nn("Make Sandbox EBICS host...")
cmd(
[
@@ -1009,7 +1147,7 @@ def prepare():
"--with-unix-socket", "/tmp/nexus.sock"
])
if not check_running(NEXUS_URL):
- exit(1)
+ fail("Nexus did not start correctly")
print(" OK")
prepare_nexus_account(
@@ -1054,8 +1192,8 @@ def prepare():
)
response.raise_for_status()
except Exception as error:
- print(error)
- exit(1)
+ fail(error)
+
FACADE_URL=response.json().get("facade")[0].get("baseUrl")
print_nn("Terminating Nexus...")
@@ -1065,17 +1203,6 @@ def prepare():
kill(handle_sandbox)
print(" OK")
- # Finish configuration now:
- cmd(
- [
- "taler-config", "-s"
- f"merchant-exchange-{CURRENCY}",
- "-o" "master_key", "-V",
- EXCHANGE_MASTER_PUB
- ],
- custom_name="specify-exchange-pub-for-merchant",
- )
-
# Point the exchange to the facade.
cmd(
[
@@ -1086,7 +1213,6 @@ def prepare():
],
custom_name="specify-facade-url",
)
-
cmd(
[
"taler-config", "-s"
@@ -1151,7 +1277,7 @@ def prepare():
if resp.status_code < 200 or resp.status_code >= 300:
print(f"Could not create (or patch) instance '{instance_id}'")
print(f"Backend responds: {resp.status_code}/{resp.text}")
- exit(1)
+ fail()
print_nn("Start merchant (with TALER_MERCHANT_TOKEN into the env)...")
auth_env = os.environ.copy()
@@ -1161,15 +1287,13 @@ def prepare():
env=auth_env
)
if not check_running("/merchant-backend"):
- # check_running logs errors already.
- exit(1)
+ fail("Merchant backend did not start correctly")
print(" OK")
ensure_instance(
currency=CURRENCY,
instance_id="default",
backend_url = REV_PROXY + "/merchant-backend",
- auth_token = TALER_MERCHANT_TOKEN,
bank_hostname = REV_PROXY + "/sandbox",
wire_method = "sepa",
merchant_wire_address = "UNUSED_IBAN", # Won't pass Camt validation.
@@ -1178,18 +1302,17 @@ def prepare():
print_nn("Stopping the merchant...")
kill(merchant_handle)
print(" OK")
- print_nn("Restartin the merchant WITHOUT the auth-token in the env...")
+ print_nn("Restarting the merchant WITHOUT the auth-token in the env...")
merchant_handle = launch(["taler-merchant-httpd"])
if not check_running("/merchant-backend"):
# check_running logs errors already.
- exit(1)
+ fail("Merchant backend did not start correctly")
print(" OK")
print_nn("Creating the 'donations' instance...")
ensure_instance(
currency=CURRENCY,
instance_id="donations",
backend_url = REV_PROXY + "/merchant-backend",
- auth_token = TALER_MERCHANT_TOKEN,
bank_hostname = REV_PROXY + "/sandbox",
wire_method = "sepa",
merchant_wire_address = "UNUSED_IBAN", # Won't pass Camt validation.
@@ -1201,7 +1324,6 @@ def prepare():
currency=CURRENCY,
instance_id="gnunet",
backend_url = REV_PROXY + "/merchant-backend",
- auth_token = TALER_MERCHANT_TOKEN,
bank_hostname = REV_PROXY + "/sandbox",
wire_method = "sepa",
merchant_wire_address = IBAN_MERCHANT,
@@ -1211,6 +1333,8 @@ def prepare():
print_nn("Stopping the merchant backend...")
kill(merchant_handle)
print(" OK")
-
+ print_nn("Stopping the reverse proxy...")
+ rev_proxy.stop()
+ print(" OK")
if __name__ == "__main__":
cli()
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.