[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
branch master updated: hydra: Add configuration for beid.
|
From: |
Christopher Baines |
|
Subject: |
branch master updated: hydra: Add configuration for beid. |
|
Date: |
Sat, 03 Jun 2023 06:32:52 -0400 |
This is an automated email from the git hooks/post-receive script.
cbaines pushed a commit to branch master
in repository maintenance.
The following commit(s) were added to refs/heads/master by this push:
new 97cb27d hydra: Add configuration for beid.
97cb27d is described below
commit 97cb27d0bb2d536f654e6707e7d4e3b3b21f09e8
Author: Christopher Baines <mail@cbaines.net>
AuthorDate: Sat Jun 3 11:27:43 2023 +0100
hydra: Add configuration for beid.
This is a Hetzner VM I've had for a while. It's currently a key part
in the patch and branch testing since it runs data.qa.guix.gnu.org,
git.guix-patches.cbaines.net and patches.guix-patches.cbaines.net.
Publishing the configuration is long overdue.
* hydra/beid.scm: New file.
---
hydra/beid.scm | 493 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 493 insertions(+)
diff --git a/hydra/beid.scm b/hydra/beid.scm
new file mode 100644
index 0000000..db42323
--- /dev/null
+++ b/hydra/beid.scm
@@ -0,0 +1,493 @@
+;; Configuration for the machine hosting data.qa.guix.gnu.org,
+;; git.guix-patches.cbaines.net and patches.guix-patches.cbaines.net
+;;
+;; Currently this is deployed on a Hetzner VM hosted in
+;; Falkenstein. It has 16 cores, 32GB of RAM and 360GB root storage
+;; with 200GB of attached storage
+;;
+;; Copyright © 2023 Christopher Baines <mail@cbaines.net>
+;; Released under the GNU GPLv3 or any later version.
+
+(use-modules (srfi srfi-1)
+ (ice-9 match)
+ (gnu)
+ (guix gexp)
+ (guix utils)
+ (guix build-system gnu)
+ (guix build utils)
+ (guix git-download)
+ (guix packages))
+(use-service-modules networking ssh web certbot databases vpn
+ mail mcron getmail guix monitoring ci
+ version-control cgit)
+(use-package-modules admin certs databases package-management
+ patchutils ci web version-control guile pkg-config
autotools
+ guile-xyz python-web gnupg django)
+
+(define my-gitolite-rc-file
+ (plain-file
+ "gitolite.rc"
+ "%RC = (
+ UMASK => 0027,
+ GIT_CONFIG_KEYS => '.*',
+ ROLES => {
+ READERS => 1,
+ WRITERS => 1,
+ },
+ LOCAL_CODE => \"$rc{GL_ADMIN_BASE}/local\",
+
+ ENABLE => [
+ 'help',
+ 'desc',
+ 'info',
+ 'perms',
+ 'writable',
+ 'ssh-authkeys',
+ 'git-config',
+ 'daemon',
+ 'gitweb',
+ 'cgit',
+ 'update-gitweb-access-list',
+ 'repo-specific-hooks',
+ 'git-annex-shell ua',
+ ],
+);
+1;"))
+
+(define my-git-http-configuration
+ (git-http-configuration
+ (git-root "/var/lib/gitolite/repositories")))
+
+(define my-cgit-configuration-nginx
+ (list
+ (nginx-server-configuration
+ (server-name '("git.guix-patches.cbaines.net"))
+ (listen '("80"))
+ (locations
+ (list
+ (nginx-location-configuration
+ (uri "^~ /.well-known/acme-challenge/")
+ (body '("root /srv/http/beid.cbaines.net;")))
+ (nginx-location-configuration
+ (uri "= /.well-known/acme-challenge/")
+ (body '("return 404;")))
+ (nginx-location-configuration
+ (uri "/")
+ (body '("return 301
https://git.guix-patches.cbaines.net$request_uri;";))))))
+ (nginx-server-configuration
+ (server-name '("git.guix-patches.cbaines.net"))
+ (root cgit)
+ (locations
+ (list
+ (git-http-nginx-location-configuration my-git-http-configuration)
+ (nginx-location-configuration
+ (uri "@cgit")
+ (body '("fastcgi_param SCRIPT_FILENAME
$document_root/lib/cgit/cgit.cgi;"
+ "fastcgi_param PATH_INFO $uri;"
+ "fastcgi_param QUERY_STRING $args;"
+ "fastcgi_param HTTP_HOST $server_name;"
+ "fastcgi_pass 127.0.0.1:9000;")))))
+ (try-files (list "$uri" "@cgit"))
+ (listen '("443 ssl"))
+ (ssl-certificate
+ "/etc/letsencrypt/live/git.guix-patches.cbaines.net/fullchain.pem")
+ (ssl-certificate-key
+ "/etc/letsencrypt/live/git.guix-patches.cbaines.net/privkey.pem"))))
+
+(define my-gitolite-configuration
+ (gitolite-configuration
+ (rc-file my-gitolite-rc-file)
+ (admin-pubkey (local-file "keys/ssh/cbaines.pub"))))
+
+(define my-cgit-configuration
+ (cgit-configuration
+ (enable-git-config? #t)
+ (repository-directory "/var/lib/gitolite/repositories")
+ (project-list "/var/lib/gitolite/projects.list")
+ (root-title "Guix Patches")
+ (root-desc "Git Repositories")
+ (clone-url '("https://git.guix-patches.cbaines.net/git/$CGIT_REPO_URL";
+ "git@git.guix-patches.cbaines.net:$CGIT_REPO_URL"))
+ (max-repo-count 100)
+ (snapshots '("tar" "tar.gz"))
+ (remove-suffix? #t)
+ (enable-index-owner? #f)
+ (enable-index-links? #t)
+ (enable-commit-graph? #t)
+ (nginx my-cgit-configuration-nginx)))
+
+(define patchwork-nginx-server-configuration
+ (nginx-server-configuration
+ (server-name '("patches.guix-patches.cbaines.net"))
+ (listen '("443 ssl"))
+ (root "/srv/http/patches.guix-patches.cbaines.net")
+ (ssl-certificate
"/etc/letsencrypt/live/git.guix-patches.cbaines.net/fullchain.pem")
+ (ssl-certificate-key
"/etc/letsencrypt/live/git.guix-patches.cbaines.net/privkey.pem")
+ (locations
+ (list
+ (nginx-location-configuration
+ (uri "/register")
+ (body '("return 200 'Registration disabled due to spam, please email
mail@cbaines.net\n';"
+ "add_header Content-Type text/plain;")))
+ (nginx-location-configuration
+ (uri "/")
+ (body '("proxy_pass http://patchwork-proxy;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-For $remote_addr;")))))))
+
+(define guix-data-service-nginx-server-configuration
+ (nginx-server-configuration
+ (server-name '("data.qa.guix.gnu.org"))
+ (listen '("443 ssl"))
+ (root "/srv/http/data.qa.guix.gnu.org")
+ (ssl-certificate "/etc/letsencrypt/live/data.qa.guix.gnu.org/fullchain.pem")
+ (ssl-certificate-key
"/etc/letsencrypt/live/data.qa.guix.gnu.org/privkey.pem")
+ (locations
+ (list
+ (nginx-location-configuration
+ (uri "/")
+ (body '("try_files $uri $uri/ @guix-data-service;")))
+ (nginx-location-configuration
+ (uri "~ /dumps/(.+)")
+ (body `(("root /var/lib/guix-data-service;"
+ "try_files $uri =404;"))))
+ (nginx-named-location-configuration
+ (name "guix-data-service")
+ (body '("proxy_pass http://guix-data-service-proxy;";
+ "proxy_read_timeout 300s;"
+ "proxy_set_header Host $host;"
+ "proxy_set_header X-Forwarded-For $remote_addr;"
+
+ "proxy_cache guix-data-service;"
+ "proxy_cache_revalidate on;"
+ "proxy_cache_min_uses 3;"
+ "proxy_cache_use_stale error timeout updating http_500 http_502
http_503 http_504;"
+ "proxy_cache_background_update on;"
+ "proxy_cache_lock on;"
+ "add_header X-Cache-Status $upstream_cache_status;"
+
+ "gzip on;"
+ "gzip_types text/html application/json;"
+ "gzip_proxied any;")))))))
+
+(define my-nginx-service-extra-content "
+ types {
+ text/plain run;
+ }
+
+ proxy_cache_path /var/cache/nginx/guix-data-service
+ levels=2
+ inactive=2d
+ keys_zone=guix-data-service:4m # ~32K keys
+ max_size=1g
+ use_temp_path=off;")
+
+(define my-nginx-service
+ (service nginx-service-type
+ (nginx-configuration
+ (nginx
+ (package
+ (inherit nginx)
+ (arguments
+ (append
+ '(#:configure-flags '("--with-http_gzip_static_module"
+ "--with-http_gunzip_module"))
+ (package-arguments nginx)))))
+ (server-names-hash-bucket-size 64)
+ (extra-content my-nginx-service-extra-content)
+ (upstream-blocks
+ (list
+ (nginx-upstream-configuration
+ (name "patchwork-proxy")
+ (servers '("localhost:8080")))
+ (nginx-upstream-configuration
+ (name "guix-data-service-proxy")
+ (servers '("localhost:8765")))))
+ (server-blocks
+ (list
+ (nginx-server-configuration
+ (server-name '("patches.guix-patches.cbaines.net"))
+ (listen '("80"))
+ (locations
+ (list
+ (nginx-location-configuration
+ (uri "/")
+ (body
+ '("return 301
https://patches.guix-patches.cbaines.net$request_uri;";)))
+ (nginx-location-configuration
+ (uri "^~ /.well-known/acme-challenge/")
+ (body '("root /srv/http/beid.cbaines.net;")))
+ (nginx-location-configuration
+ (uri "= /.well-known/acme-challenge/")
+ (body '("return 404;"))))))
+ patchwork-nginx-server-configuration
+
+ guix-data-service-nginx-server-configuration
+ (nginx-server-configuration
+ (inherit guix-data-service-nginx-server-configuration)
+ (listen '("80"))
+ (ssl-certificate #f)
+ (ssl-certificate-key #f)
+ (locations
+ (append
+ (nginx-server-configuration-locations
+ guix-data-service-nginx-server-configuration)
+ (list
+ (nginx-location-configuration
+ (uri "^~ /.well-known/acme-challenge/")
+ (body '("root /srv/http/beid.cbaines.net;")))
+ (nginx-location-configuration
+ (uri "= /.well-known/acme-challenge/")
+ (body '("return 404;"))))))))))))
+
+(define (guix-data-service-derivation-cleanup guix-data-service)
+ (program-file
+ "guix-data-service-derivation-cleanup"
+ (with-extensions (cons* guix-data-service
+ guile-gcrypt
+ (map (match-lambda
+ ((name pkg) pkg))
+ (package-propagated-inputs guix-data-service)))
+ #~(begin
+ (setvbuf (current-output-port) 'line)
+ (setvbuf (current-error-port) 'line)
+
+ (simple-format #t "~A: start:
guix-data-service-derivation-cleanup\n"
+ (strftime "%c" (localtime
(current-time))))
+ (use-modules (guix-data-service data-deletion))
+ (begin
+ (delete-unreferenced-derivations))))
+ #:guile
+ (car
+ (assoc-ref (package-native-inputs guix-data-service)
+ "guile"))))
+
+(define (guix-data-service-branch-cleanup guix-data-service)
+ (program-file
+ "guix-data-service-branch-cleanup"
+ (with-extensions (cons* guix-data-service
+ guile-gcrypt
+ (map (match-lambda
+ ((name pkg) pkg))
+ (package-propagated-inputs guix-data-service)))
+ #~(begin
+ (setvbuf (current-output-port) 'line)
+ (setvbuf (current-error-port) 'line)
+
+ (simple-format #t "~A: start:
guix-data-service-branch-cleanup\n"
+ (strftime "%c" (localtime
(current-time))))
+ (use-modules (guix-data-service data-deletion))
+ (begin
+ (delete-data-for-all-deleted-branches)
+
(delete-revisions-for-all-branches-except-most-recent-n 350))))
+ #:guile
+ (car
+ (assoc-ref (package-native-inputs guix-data-service)
+ "guile"))))
+
+(define mcron-service-configuration
+ (mcron-configuration
+ (jobs (list #~(job "30 */6 * * *"
+ "guix gc -F 10G")
+ #~(job "0 1 * * *"
+ "guix gc")
+ #~(job "0 0 * * *"
+ #$(guix-data-service-branch-cleanup
my-guix-data-service))
+ #~(job "0 0 * * 0"
+ #$(guix-data-service-derivation-cleanup
my-guix-data-service))))))
+
+(operating-system
+ (host-name "beid")
+ (timezone "Europe/London")
+ (locale "en_GB.utf8")
+
+ (bootloader (bootloader-configuration
+ (bootloader grub-bootloader)
+ (target "/dev/sda")))
+
+ (initrd-modules (append (list "virtio_scsi")
+ %base-initrd-modules))
+
+ (file-systems (cons* (file-system
+ (device (file-system-label "root"))
+ (mount-point "/")
+ (type "ext4"))
+ (file-system
+ (device "/dev/disk/by-id/scsi-0HC_Volume_10097981")
+ (mount-point "/mnt/beid-postgresql-additional-data")
+ (type "ext4")
+ (options "discard"))
+ %base-file-systems))
+
+ (swap-devices '("/swapfile"))
+
+ (users (cons (user-account
+ (name "chris")
+ (group "users")
+
+ (supplementary-groups '("wheel"))
+ (home-directory "/home/chris"))
+ %base-user-accounts))
+
+ (packages (cons* nss-certs
+ python-git-multimail
+ %base-packages))
+
+ (services (cons*
+ (service dhcp-client-service-type)
+
+ (extra-special-file "/usr/bin/guix-data-service-branch-cleanup"
+ (guix-data-service-branch-cleanup
my-guix-data-service))
+ (extra-special-file
"/usr/bin/guix-data-service-derivation-cleanup"
+ (guix-data-service-derivation-cleanup
my-guix-data-service))
+
+ (service ntp-service-type)
+ (service openssh-service-type
+ (openssh-configuration
+ (permit-root-login 'without-password)
+ (password-authentication? #f)))
+ (service certbot-service-type
+ (certbot-configuration
+ (certificates
+ (list (certificate-configuration
+ (domains
+ '(;; Domains for the guix patches testing setup
+ "git.guix-patches.cbaines.net"
+ "patches.guix-patches.cbaines.net")))
+ (certificate-configuration
+ (domains
+ '("data.qa.guix.gnu.org")))))
+ (email "mail@cbaines.net")
+ (webroot "/srv/http/beid.cbaines.net")))
+ my-nginx-service
+
+ (service guix-data-service-type
+ (guix-data-service-configuration
+ (package my-guix-data-service)
+ (getmail-idle-mailboxes '("INBOX"))
+ (extra-options
+ '("--postgresql-statement-timeout=300000"
+ "--thread-pool-threads=32"))
+ (extra-process-jobs-options
+ '("--max-processes=1"
+ "--latest-branch-revision-max-processes=6"))
+ (commits-getmail-retriever-configuration
+ (getmail-retriever-configuration
+ (type "SimpleIMAPSSLRetriever")
+ (server "imap.cbaines.net")
+ (port 993)
+ (username "guix-patches")
+ (password-command
+ (list (file-append coreutils "/bin/cat")
+
"/etc/guix-data-service-guix-patches-imap-password"))
+ (extra-parameters
+ '((mailboxes . ("INBOX"))))))))
+
+ (service prometheus-node-exporter-service-type)
+ (service httpd-service-type
+ (httpd-configuration
+ (config
+ (httpd-config-file
+ (listen '("8080"))))))
+ (service mcron-service-type
+ mcron-service-configuration)
+ (service postgresql-service-type
+ (postgresql-configuration
+ (postgresql postgresql-13)
+ (config-file
+ (postgresql-config-file
+ (log-destination "stderr")
+ (hba-file
+ (plain-file "pg_hba.conf"
+ "
+local all all trust
+host all all 127.0.0.1/32 md5
+host all all ::1/128 md5"))
+ (extra-config
+ '(("session_preload_libraries" "auto_explain")
+ ("auto_explain.log_min_duration" "1000ms")
+ ("max_connections" 500)
+ ("random_page_cost" 1.0)
+ ("work_mem" "1GB")
+ ("shared_buffers" "1GB")
+ ("effective_cache_size" "28 GB")
+ ("max_worker_processes" 4)
+ ("max_parallel_workers_per_gather" 4)
+ ("autovacuum_vacuum_cost_limit" 2000)
+ ("autovacuum_vacuum_threshold" 0)
+ ("default_statistics_target" 10000)
+ ("effective_io_concurrency" 200)
+ ("logging_collector" #t)
+ ("log_directory" "/var/log/postgresql")))))))
+ (service gitolite-service-type my-gitolite-configuration)
+ (simple-service 'gitolite-home-permissions
+ activation-service-type
+ #~(chmod "/var/lib/gitolite" #o750))
+
+ (service cgit-service-type my-cgit-configuration)
+ (service fcgiwrap-service-type
+ (fcgiwrap-configuration
+ (group "git")))
+
+ (service patchwork-service-type
+ (patchwork-configuration
+ (patchwork
+ (package
+ (inherit patchwork)
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://git.cbaines.net/git/patchwork";)
+ (commit "add-captcha")))
+ (sha256
+ (base32
+
"130mnj1az1f543sh8ph42kdqh032zzmm5ymd1ap0yyl90122xrpb"))))
+ (propagated-inputs
+ `(,@(package-propagated-inputs patchwork)
+ ("python-django-simple-math-captcha"
+ ,python-django-simple-math-captcha)))))
+ (settings-module
+ (patchwork-settings-module
+ (allowed-hosts '("patches.guix-patches.cbaines.net"))
+ (default-from-email "patchwork@cbaines.net")
+ (admins
+ '(("Christopher Baines" "mail@cbaines.net")))
+ (extra-settings
+ "
+SESSION_COOKIE_DOMAIN = '.guix-patches.cbaines.net'
+
+#REST_RESULTS_PER_PAGE=500
+MAX_REST_RESULTS_PER_PAGE=1000
+FORCE_HTTPS_LINKS=True
+
+EMAIL_HOST = 'smtp.cbaines.net'
+EMAIL_PORT = 587
+EMAIL_HOST_USER = 'patchwork'
+EMAIL_HOST_PASSWORD =
open('/etc/getmail-patchwork-imap-password').readline().rstrip()
+EMAIL_USE_TLS = True
+
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
+")))
+ (domain "patches.guix-patches.cbaines.net")
+ (getmail-retriever-config
+ (getmail-retriever-configuration
+ (type "SimpleIMAPSSLRetriever")
+ (server "imap.cbaines.net")
+ (port 993)
+ (username "patchwork")
+ (password-command
+ (list (file-append coreutils "/bin/cat")
+ "/etc/getmail-patchwork-imap-password"))
+ (extra-parameters
+ '((mailboxes . ("Patches"))))))))
+ (modify-services
+ %base-services
+ (guix-service-type
+ config => (guix-configuration
+ (inherit config)
+ (max-silent-time (* 60 60 3))
+ (timeout (* 60 60 24))
+ (build-accounts 32)
+ (extra-options '("--max-jobs=4"))))))))
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- branch master updated: hydra: Add configuration for beid.,
Christopher Baines <=