[taler-deployment] branch master updated: documenting the master.cfg a bit better, minor refactoring, adding deployment checkout as first step to factories, and adding new job for updating the buildmaster if the deployment changes

[gnunet]

This is an automated email from the git hooks/post-receive script.

grothoff pushed a commit to branch master
in repository deployment.

The following commit(s) were added to refs/heads/master by this push:
     new 3566669  documenting the master.cfg a bit better, minor refactoring, 
adding deployment checkout as first step to factories, and adding new job for 
updating the buildmaster if the deployment changes
3566669 is described below

commit 3566669088b674cd6f8a5947941325c94c900cd5
Author: Christian Grothoff <address@hidden>
AuthorDate: Sun Dec 15 16:31:11 2019 +0100

    documenting the master.cfg a bit better, minor refactoring, adding 
deployment checkout as first step to factories, and adding new job for updating 
the buildmaster if the deployment changes
---
 bootstrap-buildmaster |  25 ++++++++++++
 buildbot/master.cfg   | 111 +++++++++++++++++++++++++++++++++-----------------
 2 files changed, 99 insertions(+), 37 deletions(-)

diff --git a/bootstrap-buildmaster b/bootstrap-buildmaster
new file mode 100755
index 0000000..793c2bf
--- /dev/null
+++ b/bootstrap-buildmaster
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Bootstrap the Taler setup for the buildmaster.
+
+set -eu
+
+BRANCH=master
+REPOS="deployment"
+
+cd $HOME
+
+for component in $REPOS; do
+  if ! test -d $HOME/$component; then
+    git clone git://git.taler.net/$component.git
+  fi
+done
+
+for component in $REPOS; do
+  echo "Checking out $component to $BRANCH"
+  git -C $HOME/$component checkout $BRANCH
+done
+
+if [[ ! -d worker ]]; then
+  buildbot-worker create-worker --umask=0o22 ~/worker localhost:9989 
buildmaster-worker buildmaster-pass
+fi
diff --git a/buildbot/master.cfg b/buildbot/master.cfg
index c29868f..70b1dc1 100644
--- a/buildbot/master.cfg
+++ b/buildbot/master.cfg
@@ -45,6 +45,11 @@ c["buildbotNetUsageData"] = "basic"
 # be configured on the worker.
 c["workers"] = [
 
+    ##
+    # This worker restarts the buildmaster itself on
+    # changes to this file.
+    worker.Worker("buildmaster-worker", "buildmaster-pass"),
+
     ##
     # This worker makes the code coverage and publishes it
     # under the "lcov" Website.
@@ -115,7 +120,7 @@ c["change_source"] = [ALLCS]
 # Configure the Schedulers, which decide how to react to incoming
 # changes.
 
-
+# The web page changed if 'www' changed OR if 'web' in the 'twister' repo 
changed:
 def twister_web_page(change):
     _change = change.asDict()
     repo = _change.get("project")
@@ -127,10 +132,9 @@ def twister_web_page(change):
         if re.search(r"web", file.get("name", "")) \
                 and "twister" == repo:
             return True
-
     return False
 
-
+# Jobs we run at 6am every day: coverage analysis and auditor.
 NIGHTLY_SCHEDULER = schedulers.Nightly(
     name="nightly-scheduler",
     builderNames=["lcov-builder",
@@ -141,15 +145,17 @@ NIGHTLY_SCHEDULER = schedulers.Nightly(
     minute=0
 )
 
+# Docs run if master or stable branch of 'docs' (or deployment) changed.
 DOC_SCHEDULER = schedulers.SingleBranchScheduler(
     name="periodic-doc-scheduler",
     builderNames=["doc-builder"],
     change_filter=util.ChangeFilter(
-        branch_re="(master|stable)", project_re="(docs)"
+        branch_re="(master|stable)", project_re="(docs|deployment)"
     ),
     treeStableTimer=None
 )
 
+# Sites are re-build whenever deployment, www or twister changes.
 SITES_SCHEDULER = schedulers.SingleBranchScheduler(
     name="sites-scheduler",
     builderNames=["sites-builder"],
@@ -157,48 +163,43 @@ SITES_SCHEDULER = schedulers.SingleBranchScheduler(
         branch_re="(master|stable)",
         # Given that filter_fn is used, the line below could be
         # removed (?)
-        project_re="(www|twister)",
+        project_re="(www|twister|deployment)",
         filter_fn=twister_web_page
     ),
     treeStableTimer=None
 )
 
+# Wallet is re-build whenever wallet-core or deployment changes
 WALLET_SCHEDULER = schedulers.SingleBranchScheduler(
     name="wallet-scheduler",
     change_filter=util.ChangeFilter(
         branch="master",
-        project_re="(wallet|deployment)"
+        project_re="(wallet-core|deployment)"
     ),
     treeStableTimer=None,
     builderNames=["wallet-builder"]
 )
 
-TALER_TEST_HEALTHCHECK = schedulers.SingleBranchScheduler(
-    name="taler-test-healthcheck-scheduler",
+# Healthchecks are done on wallet or deployment changes to master
+TALER_HEALTHCHECK_SCHEDULER = schedulers.SingleBranchScheduler(
+    name="taler-healthcheck-scheduler",
     change_filter=util.ChangeFilter(
         branch="master",
         project_re="(wallet|deployment)"
     ),
     treeStableTimer=None,
-    builderNames=["taler-test-healthcheck-builder"]
-)
-
-TALER_DEMO_HEALTHCHECK = schedulers.SingleBranchScheduler(
-    name="taler-demo-healthcheck-scheduler",
-    change_filter=util.ChangeFilter(
-        branch="master",
-        project_re="(wallet|deployment)"
-    ),
-    treeStableTimer=None,
-    builderNames=["taler-demo-healthcheck-builder"]
+    builderNames=["taler-test-healthcheck-builder",
+                  "taler-demo-healthcheck-builder"]
 )
 
+# We check demo once per hour.
 DEMO_SERVICES_CHECKER_SCHEDULER = schedulers.Periodic(
     name="demo-services-checker-scheduler",
     periodicBuildTimer=60 * 60,  # 1 hour
     builderNames=["demo-services-checker-builder"]
 )
 
+# We top-up the tipping reserve every 10 days.
 TIP_RESERVE_TOPPER_SCHEDULER = schedulers.Periodic(
     name="tip-reserve-topper-scheduler",
     periodicBuildTimer=60 * 60 * 24 * 10,  # 10 days
@@ -206,11 +207,12 @@ TIP_RESERVE_TOPPER_SCHEDULER = schedulers.Periodic(
                   "tip-reserve-topper-builder-test"]
 )
 
+# Scheduler that triggers if anything changes
 ALL_SCHEDULER = schedulers.SingleBranchScheduler(
     name="all-scheduler",
     change_filter=util.ChangeFilter(
         branch_re="(master|stable)",
-        project_re="(backoffice|wallet|bank|exchange|"
+        project_re="(backoffice|wallet-core|bank|exchange|"
         "merchant|deployment|donations|twister|"
         "blog|help|survey|landing)"
     ),
@@ -220,6 +222,7 @@ ALL_SCHEDULER = schedulers.SingleBranchScheduler(
 
 # Scheduler monitoring the help.git repo; a forgotten repo we
 # use to test BB.
+# NOTE: scheduler is NOT active! (commented out below)
 DEBUG_SCHEDULER = schedulers.SingleBranchScheduler(
     name="debug-scheduler",
     change_filter=util.ChangeFilter(branch="master", project="help"),
@@ -228,6 +231,7 @@ DEBUG_SCHEDULER = schedulers.SingleBranchScheduler(
 )
 
 # Consider adding other Python parts, like the various frontends.
+# NOTE: scheduler is NOT active! (commented out below)
 LINT_SCHEDULER = schedulers.SingleBranchScheduler(
     name="lint-scheduler",
     change_filter=util.ChangeFilter(
@@ -237,8 +241,7 @@ LINT_SCHEDULER = schedulers.SingleBranchScheduler(
     builderNames=["lint-builder"]
 )
 
-# Provide "force" button in the web UI.  To be removed in the
-# future ?
+# Provide "force" button in the web UI.
 FORCE_SCHEDULER = schedulers.ForceScheduler(
     name="force-scheduler",
     builderNames=[
@@ -265,8 +268,7 @@ c["schedulers"] = [
     ALL_SCHEDULER,
     FORCE_SCHEDULER,
     WALLET_SCHEDULER,
-    TALER_DEMO_HEALTHCHECK,
-    TALER_TEST_HEALTHCHECK,
+    TALER_HEALTHCHECK_SCHEDULER,
 
     ##
     # Rarely/never used, excluding.
@@ -282,7 +284,10 @@ c["schedulers"] = [
 # one worker.
 
 
-def git_step(repo):
+# Convenience function that checks out a Git repository.
+# First argument is the URL of the Git to clone, second
+# the desired branch. Default is 'master'.
+def git_step(repo,target_branch="master"):
     return Git(
         repourl=repo,
         mode="full",
@@ -290,10 +295,32 @@ def git_step(repo):
         logEnviron=False,
         alwaysUseLatest=True,
         haltOnFailure=True,
-        branch="master"
+        branch=target_branch
+    )
+
+# Convenience function that checks out the deployment.
+def update_deployment(factory):
+    return factory.addStep(git_step("git://git.taler.net/deployment.git"))
+
+# Create a FACTORY with a deployment.git checkout as the first step.
+def create_factory_with_deployment():
+    f = util.BuildFactory()
+    update_deployment (f);
+    return f
+
+# Factory for a job to update the buildmaster itself.
+BUILDMASTER_FACTORY = create_factory_with_deployment()
+BUILDMASTER_FACTORY.addStep(
+    ShellCommand(
+        name="restart buildmaster",
+        description="trigger buildmaster restart with new configuration",
+        descriptionDone="Buildmaster updated",
+        command=["buildbot", "reconfig"],
+        workdir="master/"
     )
+)
 
-TALER_DEMO_HEALTHCHECK_FACTORY = util.BuildFactory()
+TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment()
 
TALER_DEMO_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
 TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
     ShellCommand(
@@ -323,7 +350,7 @@ TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
     )
 )
 
-TALER_TEST_HEALTHCHECK_FACTORY = util.BuildFactory()
+TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment()
 
TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
 TALER_TEST_HEALTHCHECK_FACTORY.addStep(
     ShellCommand(
@@ -354,7 +381,7 @@ TALER_TEST_HEALTHCHECK_FACTORY.addStep(
 )
 
 
-WALLET_FACTORY = util.BuildFactory()
+WALLET_FACTORY = create_factory_with_deployment()
 WALLET_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
 WALLET_FACTORY.addStep(
     ShellCommand(
@@ -437,7 +464,7 @@ LINT_FACTORY.addStep(
     )
 )
 
-LCOV_FACTORY = util.BuildFactory()
+LCOV_FACTORY = create_factory_with_deployment()
 LCOV_FACTORY.addStep(
     ShellCommand(
         name="build",
@@ -461,7 +488,7 @@ LCOV_FACTORY.addStep(
 )
 
 # FIXME: 'demo' reports generator missing.
-AUDITOR_FACTORY = util.BuildFactory()
+AUDITOR_FACTORY = create_factory_with_deployment()
 AUDITOR_FACTORY.addStep(
     ShellCommand(
         name="Auditor reports generator",
@@ -472,7 +499,7 @@ AUDITOR_FACTORY.addStep(
     )
 )
 
-TIP_RESERVE_TOPPER_FACTORY = util.BuildFactory()
+TIP_RESERVE_TOPPER_FACTORY = create_factory_with_deployment()
 TIP_RESERVE_TOPPER_FACTORY.addStep(
     ShellCommand(
         name="tip reserve topper",
@@ -483,7 +510,7 @@ TIP_RESERVE_TOPPER_FACTORY.addStep(
     )
 )
 
-BUILD_FACTORY = util.BuildFactory()
+BUILD_FACTORY = create_factory_with_deployment()
 BUILD_FACTORY.addStep(
     ShellCommand(
         name="build",
@@ -554,7 +581,7 @@ BUILD_FACTORY.addStep(
     )
 )
 
-SELENIUM_FACTORY = util.BuildFactory()
+SELENIUM_FACTORY = create_factory_with_deployment()
 SELENIUM_FACTORY.addStep(
     ShellCommand(
         name="selenium",
@@ -565,7 +592,7 @@ SELENIUM_FACTORY.addStep(
     )
 )
 
-DOC_FACTORY = util.BuildFactory()
+DOC_FACTORY = create_factory_with_deployment()
 DOC_FACTORY.addStep(
     ShellCommand(
         name="build docs",
@@ -577,7 +604,7 @@ DOC_FACTORY.addStep(
     )
 )
 
-SITES_FACTORY = util.BuildFactory()
+SITES_FACTORY = create_factory_with_deployment()
 SITES_FACTORY.addStep(
     ShellCommand(
         name="build Web sites",
@@ -589,7 +616,7 @@ SITES_FACTORY.addStep(
     )
 )
 
-DEMO_SERVICES_CHECKER_FACTORY = util.BuildFactory()
+DEMO_SERVICES_CHECKER_FACTORY = create_factory_with_deployment()
 DEMO_SERVICES_CHECKER_FACTORY.addStep(
     ShellCommand(
         name="demo services checker",
@@ -609,6 +636,14 @@ DEBUG_BUILDER = util.BuilderConfig(
     factory=DEBUG_FACTORY
 )
 
+# Builder to update the buildmaster itself.
+BUILDMASTER_BUILDER = util.BuilderConfig(
+    name="buildmaster-builder",
+    workernames=["buildmaster-worker"],
+    factory=BUILDMASTER_FACTORY
+)
+
+# This builder is NOT ACTIVE!
 LINT_BUILDER = util.BuilderConfig(
     name="lint-builder",
     workernames=["lint-worker"],
@@ -688,6 +723,7 @@ TALER_DEMO_HEALTHCHECK_BUILDER = util.BuilderConfig(
 )
 
 c["builders"] = [
+    BUILDMASTER_BUILDER,
     LCOV_BUILDER,
     AUDITOR_BUILDER_TEST,
     AUDITOR_BUILDER_DEMO,
@@ -755,6 +791,7 @@ c["buildbotURL"] = "https://buildbot.taler.net/";
 from taler_bb_userpass_db import USER_PASSWORD_DB
 
 BUILDER_LIST = [
+    "buildmaster-builder",
     "doc-builder",
     "lcov-builder",
     "auditor-builder",

-- 
To stop receiving notification emails like this one, please contact
address@hidden.