services/nomad/build/build-rsyncd,buildbot: improve sync/index of packages

* to prevent re-syncing packages between each package build, use --compare-dest
* to prevent races with syncing and signing, update the repo as a pre-xfer
  script and collect the log to show in the builder output

Author: classabbyamp

services/nomad/build/build-rsyncd.nomad

@@ -65,8 +65,8 @@ job "build-rsyncd" {
       }
 
       template {
-        data        = file("rsync-post-xfer")
-        destination = "local/rsync-post-xfer"
+        data        = file("rsync-update-repo")
+        destination = "local/rsync-update-repo"
         perms       = "0755"
       }
 
@@ -114,8 +114,13 @@ EOF
 [incoming-${template.value}]
 path = /incoming/${template.value}
 auth users = buildsync:rw
-filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .*
-post-xfer exec = /local/rsync-post-xfer
+filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .* - *.log
+
+[updaterepo-${template.value}]
+path = /incoming/${template.value}
+auth users = buildsync:r
+filter = + update-repo.log - * - .*
+early exec = /local/rsync-update-repo
 EOF
           destination = "local/rsyncd.conf.d/${template.value}.conf.inc"
         }

services/nomad/build/buildbot.cfg

@@ -218,10 +218,17 @@ def make_xbps_bulk_cmd(props):
 
 @util.renderer
 def make_rsync_cmd(props):
+    target = props.getProperty('target')
+    if target.startswith("aarch64"):
+        final_dest = "/mirror/current/aarch64"
+    elif target.endswith("musl"):
+        final_dest = "/mirror/current/musl"
+    else:
+        final_dest = "/mirror/current"
     return ['bash', '-e', '-c',
-            util.Interpolate("""
+            util.Interpolate(f"""
 echo "=> Syncing packages to the shadow repository..."
-rsync -vurk --delete-after \
+rsync -vurk --delete-after --compare-dest={final_dest} \
 --filter='+ */' --filter='+ *.%(prop:target)s.xbps' \
 --filter='- .*' --filter='- *' \
 --password-file=/secrets/rsync/password /hostdir/binpkgs/ \
@@ -235,6 +242,28 @@ esac
 """)]
 
 
+@util.renderer
+def make_update_repo_cmd(props):
+    return ['bash', '-e', '-c',
+            util.Interpolate(f"""
+echo "=> Moving packages to the shadow repository..."
+rsync -vurk --delete-after --password-file=/secrets/rsync/password \
+{{ range nomadService 1 $allocID "build-rsyncd" -}}
+rsync://buildsync@{{ .Address }}:{{ .Port }}/updaterepo-%(prop:target)s/update-repo.log ./update-repo.log
+{{ end -}}
+case "$?" in
+    0|23|24)
+        cat ./update-repo.log
+        exit 0
+        ;;
+    *)
+        cat ./update-repo.log
+        exit 1
+        ;;
+esac
+""")]
+
+
 @util.renderer
 def make_clean_cmd(props):
     return [
@@ -258,6 +287,11 @@ def build_packages(props):
             logname=f'sync:{p}',
             haltOnFailure=True,
         ))
+        cmds.append(util.ShellArg(
+            command=make_update_repo_cmd,
+            logname=f'update-repo:{p}',
+            haltOnFailure=True,
+        ))
     if cmds:
         cmds.append(util.ShellArg(
             command=['make', 'clean'],
@@ -372,6 +406,18 @@ factory.addStep(steps.ShellCommand(
     timeout=14400,
 ))
 
+factory.addStep(steps.ShellCommand(
+    command=make_update_repo_cmd,
+    name='sync_packages',
+    description='updating shadow repository with built packages',
+    descriptionDone='updated shadow repository with built packages',
+    alwaysRun=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir=builddir(),
+    timeout=14400,
+))
+
 factory.addStep(steps.ShellCommand(
     command=make_clean_cmd,
     name='clean_packages',

services/nomad/build/rsync-post-xfer services/nomad/build/rsync-update-reposervices/nomad/build/rsync-post-xfer renamed to services/nomad/build/rsync-update-repo

@@ -1,6 +1,8 @@
 #!/bin/sh
 
-set -e
+set -ex
+
+exec >"$RSYNC_MODULE_PATH"/update-repo.log 2>&1
 
 TGT_BASE="/mirror/current"
 
@@ -16,9 +18,6 @@ process_repo() {
     XBPS_TARGET_ARCH="$ARCH" xbps-rindex -c "$TGT_BASE/$TGT/$repo"
     # remove old packages
     XBPS_TARGET_ARCH="$ARCH" xbps-rindex -r "$TGT_BASE/$TGT/$repo"
-    # Remove signatures that don't have a corresponding package
-    find "$TGT_BASE/$TGT/$repo" -maxdepth 1 \( -name '*.xbps.sig' -o -name '*.xbps.sig2' \) \
-        -exec sh -c 'for x in "$@"; do [ -e "${x%.sig*}" ] || rm -- $x; done' _ {} +
 }
 
 ARCH="${RSYNC_MODULE_NAME#*-}"
@@ -46,6 +45,6 @@ case "$ARCH" in
 esac
 
 # clean up incoming
-rm -r "${RSYNC_MODULE_PATH:?}"/*
+rm -r "${RSYNC_MODULE_PATH:?}"/*.xbps
 
 exit 0