Mirror master and release branches.

Having postgresql-cfbot/postgresql mirror the master branch keeps ccache
recent.  Since master is the default branch, other branches are allowed
to use its ccache.

Author: macdice

cfbot_api.py

@@ -53,6 +53,10 @@ def github_webhook():
             cfbot_github.handle_workflow_run_webhook(conn, event)
             conn.commit()
             return "OK"
+        elif event_type == "push":
+            cfbot_github.handle_push_webhook(conn, event)
+            conn.commit()
+            return "OK"
         else:
             return "unhandled event type " + event_type
     except:

cfbot_github.py

@@ -800,6 +800,27 @@ def handle_workflow_job_webhook(conn, event):
     ingest_task(conn, build_id, task_id, task_status, task_name, "webhook")
 
 
+def handle_push(conn, event):
+    repo = event["repository"]["full_name"]
+    if repo != cfbot_config.GITHUB_MIRROR_FULL_REPO:
+        # We only mirror branches from postgres/postgres (which is itself a
+        # mirror of the actual self-hosted postgresql.org repo).
+        return
+    ref = event["ref"]
+    if not ref.startswith("refs/head/"):
+        # We don't mirror refs/tag/... should we?
+        return
+    branch = ref[10:]
+    if not re.match(cfbot_config.GITHUB_MIRROR_BRANCH_PATTERN, branch):
+        # Only mirror branches that match our configured pattern (though
+        # we don't actually expect postgres/postgres to have any non-matching
+        # branches).
+        return
+    cursor = conn.cursor()
+    # Actual mirroring work is handed off to a cfbot worker.
+    cfbot_work_queue.insert_work_queue(cursor, "push-mirror-branch", branch)
+
+
 # ======================================================================
 # Functions called by cfbot workers servicing work_queue items.
 # ======================================================================

cfbot_patch.py

@@ -165,11 +165,20 @@ def choose_submission(conn, cf_ids):
     return commitfest_id, submission_id
 
 
-def update_patchbase_tree(repo_dir):
-    """Pull changes from PostgreSQL master and return the HEAD commit ID."""
+def checkout_patchbase_branch(repo_dir, branch):
+    """Switch patchbase repo to a given branch."""
     subprocess.call(
-        "cd %s && git checkout . -q > /dev/null && git clean -fd > /dev/null && git checkout -q master && git pull -q"
-        % repo_dir,
+        "cd %s && git checkout . -q > /dev/null && git clean -fd > /dev/null && git checkout -q %s"
+        % (repo_dir, branch),
+        shell=True,
+    )
+
+
+def update_patchbase_tree(repo_dir, branch):
+    """Switch branch and pull changes from tracked remote."""
+    checkout_patchbase_branch(repo_dir, branch)
+    subprocess.call(
+        "cd %s && git pull -q" % (repo_dir, branch),
         shell=True,
     )
 
@@ -310,6 +319,29 @@ def update_submission(conn, message_id, commit_id, commitfest_id, submission_id)
     )
 
 
+def mirror_branch(branch):
+    template_repo_path = patchburner_ctl("template-repo-path").strip()
+
+    # Everything else in this file assumes that
+    # cfbot_periodic_minutely.py acquired the "big lock" while
+    # applying patches etc, but this function is reached by cfbot
+    # workers, and needs interlocking so it does that explicitly here.
+    with cfbot_util.lock():
+        logging.info("updating branch %s in template repo", branch)
+        update_patchbase_tree(template_repo_path, branch)
+        if cfbot_config.GIT_REMOTE_NAME:
+            logging.info("pushing branch %s (automatic mirror)", branch)
+            my_env = os.environ.copy()
+            my_env["GIT_SSH_COMMAND"] = cfbot_config.GIT_SSH_COMMAND
+            subprocess.check_call(
+                "cd %s && git push -q -f %s %s"
+                % (template_repo_path, cfbot_config.GIT_REMOTE_NAME, branch),
+                env=my_env,
+                shell=True,
+                stderr=subprocess.DEVNULL,
+            )
+
+
 def reset_repo_to_good_master_commit(conn, repo_path):
     # find the most recent commit ID that succeeded on master
     cursor = conn.cursor()
@@ -324,16 +356,18 @@ def reset_repo_to_good_master_commit(conn, repo_path):
         (good_commit_id,) = result
     else:
         good_commit_id = "origin/master"
+    logging.info("selected master commit %s as base", good_commit_id)
 
+    checkout_patchbase_branch(repo_path, "master")
     commit_id = get_commit_id(repo_path)
-    if commit_id != good_commit_id:
-        logging.info("updating repo to find last good master commit %s", good_commit_id)
-        update_patchbase_tree(repo_path)
 
-    reset_commit_id(repo_path, good_commit_id)
-    commit_id = get_commit_id(repo_path)
-    if good_commit_id != "origin/master":
-        assert commit_id == good_commit_id
+    if commit_id != good_commit_id:
+        logging.info("updating template repo")
+        update_patchbase_tree(repo_path, "master")
+        reset_commit_id(repo_path, good_commit_id)
+        commit_id = get_commit_id(repo_path)
+        if good_commit_id != "origin/master":
+            assert commit_id == good_commit_id
 
     return commit_id
 

cfbot_work_queue.py

@@ -138,6 +138,9 @@ def process_one_job(conn, fetch_only):
             cfbot_github.poll_github_commit(conn, key)
         elif type == "poll-github-run":
             cfbot_github.poll_github_run(conn, key)
+        # Mirroring master, REL_*_STABLE
+        elif type == "push-mirror-branch":
+            cfbot_patch.mirror_branch(key)
         # Notifying the Commitfest app
         elif type == "post-task-status":
             cfbot_commitfest.post_task_status(conn, key)