Improve backups, introduce remote restore

Container volume backup logic has been moved to a (largely equivalent)
external script, allowing for future expansion of functionality. In
addition, a `rclone-pull@` service has been added and set up as a default
dependency for the `container-volume-restore@` service, allowing for
automatic set up of servers based on latest remote backups.

host/base/service/container-volume-backup

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+#
+# Archive all files in source directory into destination directory.
+
+set -euo pipefail
+
+# Parse command-line arguments.
+unset SRCDIR DESTDIR BASENAME
+
+while getopts 's:d:n:' arg; do
+  case "${arg}" in
+    s) SRCDIR="${OPTARG}" ;;
+    d) DESTDIR="${OPTARG}" ;;
+    n) BASENAME="${OPTARG}-" ;;
+    *) echo "Unknown argument '${arg}' given. Exiting." >&2 && exit 1 ;;
+  esac
+done
+
+# Check and return true if source files are newer than any archive already created.
+function archive-should-update() {
+    local latest="${DESTDIR}/${BASENAME}latest.tar.gz"
+
+    if ! test -e "${latest}"; then
+        return 0
+    elif test "$(find "${SRCDIR}" -type f -newer "${latest}" -print -quit | wc -l)" -gt 0; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Create compressed archive for source path into destination path for pre-defined, date-based name.
+function archive-create() {
+    local name="${BASENAME}$(date +%w%H)"
+
+    # Create uncompressed archive for source path and compress separately to ensure minimal blocking.
+    tar --verbose --create --file "${DESTDIR}/${name}.tar" --directory "${SRCDIR}" .
+    gzip --force "${DESTDIR}/${name}.tar"
+
+    # Point the latest archive to a "special" name for future use.
+    (cd /backup && ln --symbolic --force "${name}.tar.gz" "${BASENAME}latest.tar.gz")
+}
+
+# Entry-point for script. Create tar archive for source directory into destination directory, checking
+# for any changes since last run, and ensuring files are archived correctly for subsequent restore.
+function main() {
+    if archive-should-update "${SRCDIR}" "${DESTDIR}"; then
+        archive-create "${SRCDIR}" "${DESTDIR}"
+    fi
+}
+
+main "$@"

host/base/systemd/container-volume-backup@.service

@@ -5,15 +5,11 @@ Description=Backup for Container Volume %I
 Type=oneshot
 SyslogIdentifier=%N
 ExecStartPre=/bin/install --mode 0700 --directory %S/backups/coreos-home-server/%i
-ExecStart=/bin/podman run --replace --rm --name %p-%i --entrypoint /bin/bash \
+ExecStart=/bin/podman run --replace --rm --name %p-%i --entrypoint /opt/container-volume-backup \
                           --volume %i:/data:z,ro \
+                          --volume %E/coreos-home-server/base/service:/opt:z,ro \
                           --volume %S/backups/coreos-home-server/%i:/backup:z \
-                          docker.io/debian:bullseye-slim -c \
-                          'test -f /backup/%i-latest.tar.gz && \
-                           test "$(find /data -type f -newer /backup/%i-latest.tar.gz -print -quit | wc -l)" -eq 0 && exit 0; \
-                           name="%i-$(date +%%w%%H)" && \
-                           tar --verbose --create --gzip --file "/backup/$name.tar.gz" --directory /data . && \
-                           cd /backup && ln --symbolic --force "$name.tar.gz" %i-latest.tar.gz'
+                          docker.io/debian:bullseye-slim -s /data -d /backup -n "%i"
 
 [Install]
 WantedBy=multi-user.target

host/virtual/spec.bu

@@ -134,12 +134,8 @@ systemd:
           contents: |
             [Service]
             ExecStart=
-            ExecStart=/bin/podman create --replace --pull never --name letsencrypt-register-%i \
-                                         --volume letsencrypt:/var/lib/letsencrypt:z \
-                                         --entrypoint true localhost/letsencrypt:latest
-            ExecStartPost=/bin/podman init letsencrypt-register-%i
-            ExecStartPost=/bin/podman cp /etc/ssl/private/certificates/ letsencrypt-register-%i:/var/lib/letsencrypt
-            ExecStartPost=/bin/podman rm letsencrypt-register-%i
+            ExecStart=/bin/sh -c "V=$(podman volume mount letsencrypt) && cp -Rv /etc/ssl/private/certificates $V"
+            ExecStartPost=/bin/podman volume unmount letsencrypt
 
 storage:
   files:

service/rclone/systemd/container-volume-restore@.service.d/before-rclone.conf

@@ -0,0 +1,3 @@
+[Unit]
+Wants=rclone-pull@var-lib-backups-coreos\x2dhome\x2dserver-%i.service
+Before=rclone-pull@var-lib-backups-coreos\x2dhome\x2dserver-%i.service

service/rclone/systemd/rclone-pull@.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Rclone Pull for /%I
+Wants=container-build@rclone.service
+After=container-build@rclone.service
+ConditionDirectoryNotEmpty=!/%I
+
+[Service]
+Type=oneshot
+SyslogIdentifier=%N
+Environment=SOURCE=crypt:%i
+ExecStartPre=/bin/install --mode 0700 -d /%I
+ExecStart=/bin/podman run --rm --env-file %E/coreos-home-server/rclone/rclone.env \
+                          --volume /%I:/data:z localhost/rclone:latest \
+                          sync ${SOURCE} /data
+
+[Install]
+WantedBy=multi-user.target