fix(ci): rebuild Go backends on linked pkg/ changes and on matrix entry edits

scripts/changed-backends.js

@@ -5,13 +5,14 @@ import { Octokit } from "@octokit/core";
 import {
   getAllBackendPaths,
   filterMatrix,
+  BACKEND_MATRIX_FILE,
 } from "./lib/backend-filter.mjs";
 
 // Matrix data lives in a small data-only YAML so both backend.yml (master push)
 // and backend_pr.yml (pull_request) can use a dynamic `matrix: ${{ fromJson(...) }}`
 // for the live job, while this script remains the single source of truth for
 // "what backends does the project know about".
-const matrixYml = yaml.load(fs.readFileSync(".github/backend-matrix.yml", "utf8"));
+const matrixYml = yaml.load(fs.readFileSync(BACKEND_MATRIX_FILE, "utf8"));
 const includes = matrixYml.include;
 const includesDarwin = matrixYml.includeDarwin;
 
@@ -76,6 +77,45 @@ async function getChangedFilesForPush(event) {
   return res.data.files.map(f => f.filename);
 }
 
+// The matrix file's contents at the base revision, so filterMatrix can rebuild
+// only the entries whose fields actually changed instead of all 417 (or, as
+// before, none of them). Returns null when the previous revision cannot be
+// resolved, which filterMatrix treats as "rebuild everything".
+//
+// Only called when the changed-file list actually names the matrix file, so the
+// common path costs no extra API request.
+async function getPreviousMatrix(event) {
+  const ref = event.pull_request ? event.pull_request.base.sha : event.before;
+  if (!ref || /^0+$/.test(ref)) return null;
+  const owner = event.repository.owner.login;
+  const repo = event.repository.name;
+  try {
+    const res = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', {
+      owner,
+      repo,
+      path: BACKEND_MATRIX_FILE,
+      ref,
+      mediaType: { format: 'raw' },
+    });
+    // `format: raw` yields a string; fall back to the JSON representation in
+    // case a proxy or a future Octokit version ignores the media type.
+    const raw = typeof res.data === 'string'
+      ? res.data
+      : Buffer.from(res.data.content, 'base64').toString('utf8');
+    const previous = yaml.load(raw);
+    return {
+      include: previous.include || [],
+      includeDarwin: previous.includeDarwin || [],
+    };
+  } catch (err) {
+    console.log(
+      `could not read ${BACKEND_MATRIX_FILE} at ${ref}, falling back to run-all:`,
+      err.message
+    );
+    return null;
+  }
+}
+
 // Group matrix entries by tag-suffix and emit a merge-matrix entry per group.
 // Both multi-leg groups (per-arch fan-out) and singletons get one entry each:
 // the build job pushes by digest only with no tags applied, so every backend
@@ -200,13 +240,14 @@ function emitFullMatrix() {
   }
 }
 
-function emitFilteredMatrix(changedFiles) {
+function emitFilteredMatrix(changedFiles, previousMatrix) {
   console.log("Changed files:", changedFiles);
 
   const { filtered, filteredDarwin, changedBackends } = filterMatrix({
     includes,
     includesDarwin,
     changedFiles,
+    previousMatrix,
   });
 
   console.log("Filtered files:", filtered);
@@ -260,5 +301,10 @@ function emitFilteredMatrix(changedFiles) {
     emitFullMatrix();
     return;
   }
-  emitFilteredMatrix(changedFiles);
+
+  const previousMatrix = changedFiles.includes(BACKEND_MATRIX_FILE)
+    ? await getPreviousMatrix(event)
+    : null;
+
+  emitFilteredMatrix(changedFiles, previousMatrix);
 })();

scripts/lib/backend-filter.mjs

@@ -137,9 +137,39 @@ const isDarwinGenericGo = item =>
 
 const isLinuxPython = item => item.dockerfile.endsWith("python");
 
+// Dockerfile.golang is the only Linux dockerfile that compiles Go (it runs
+// `make protogen-go && make -C backend/go/<backend> build`); every other one
+// builds C++, Rust or a Python venv and links no Go code at all.
+const isLinuxGo = item => item.dockerfile.endsWith("golang");
+
 const never = () => false;
 const always = () => true;
 
+// The pkg/ subtrees that end up inside a Go backend binary. Not a guess:
+//
+//   go list -deps ./backend/go/... | grep LocalAI/pkg
+//
+// returns exactly pkg/audio, pkg/grpc (+ base, grpcerrors, proto), pkg/httpclient,
+// pkg/sound, pkg/store and pkg/utils — identical for GOOS/GOARCH in
+// {linux,darwin} x {amd64,arm64}, so one list covers both matrices. Notably
+// absent: pkg/model, pkg/downloader, pkg/functions and the other ~21 pkg
+// subtrees, which are core-server-only.
+//
+// Enumerating rather than taking all of pkg/ is the whole point. All of pkg/
+// changes in ~8.6% of commits and would fire a 199-entry Go matrix that often;
+// these six subtrees change in 2.0%, which is the same order as the already
+// accepted scripts/build/ rule (1.9%). If the enumeration ever drifts from the
+// `go list` output the tests pin both directions — a listed subtree must
+// trigger, an unlisted one must not.
+const GO_BACKEND_PKG_PREFIXES = [
+  "pkg/audio/",
+  "pkg/grpc/", // covers base/, grpcerrors/ and the generated proto/
+  "pkg/httpclient/",
+  "pkg/sound/",
+  "pkg/store/",
+  "pkg/utils/",
+];
+
 // Shared build inputs: files that end up in, or decide the contents of, images
 // belonging to backends whose own directory they do not live under. The
 // per-backend prefix match in filterMatrix() structurally cannot see these, so
@@ -173,6 +203,21 @@ export const SHARED_BUILD_INPUTS = [
     linux: isLinuxPython,
     darwin: isDarwinPython,
   },
+  {
+    // Compiled into every Go backend binary (see GO_BACKEND_PKG_PREFIXES).
+    // `_test.go` files never reach a binary, same carve-out as the
+    // `*_test.sh` one on the scripts/build/ catch-all below.
+    //
+    // Darwin's bespoke builders (llama-cpp, ds4, privacy-filter) carry
+    // lang=go for runner selection only — their sources are C++ and link no
+    // Go, so isDarwinGenericGo is the correct predicate here, exactly as it
+    // is for scripts/build/golang-darwin.sh.
+    matches: file =>
+      GO_BACKEND_PKG_PREFIXES.some(prefix => file.startsWith(prefix)) &&
+      !file.endsWith("_test.go"),
+    linux: isLinuxGo,
+    darwin: isDarwinGenericGo,
+  },
   {
     // The reusable build workflows own build-args, packaging and push for
     // their whole OS family, so a change there can alter what lands in every
@@ -247,6 +292,51 @@ function dockerfileChanged(item, changedFiles) {
   return df !== "" && changedFiles.includes(df);
 }
 
+// .github/backend-matrix.yml is a shared build input like the ones above, but
+// it cannot be handled by a path rule: every entry lives in it, so matching the
+// path would rebuild all 417 Linux entries on every PR that adds a backend —
+// and new backends already trigger via their own directory. Excluding it
+// wholesale is what the rest of this file does, and that leaves a real hole:
+// editing an existing entry's base-image / build-type / cuda version changes
+// the image that entry produces while touching no file the prefix match can
+// see, so it rebuilt nothing.
+//
+// The fix needs the file's *previous* contents, which a changed-path list
+// cannot carry. changed-backends.js fetches the base revision via the GitHub
+// contents API and hands it in as `previousMatrix`; everything below stays
+// pure. When the file did not change, `previousMatrix` is never consulted.
+export const BACKEND_MATRIX_FILE = ".github/backend-matrix.yml";
+
+// Identity of a matrix entry across revisions. tag-suffix names the image;
+// per-arch legs of the same image are distinguished by platform-tag. Verified
+// unique across all 417 Linux and 56 Darwin entries.
+export function matrixEntryKey(item) {
+  return JSON.stringify([item["tag-suffix"] || "", item["platform-tag"] || ""]);
+}
+
+// Full field set, order-independent, so a reordered or reindented YAML entry
+// compares equal and only a real value change counts.
+function entryFingerprint(item) {
+  return JSON.stringify(
+    Object.keys(item).sort().map(key => [key, item[key]])
+  );
+}
+
+// Keys of entries that are new or whose fields differ from the previous
+// revision. Removed entries produce nothing to build.
+function changedEntryKeys(current, previous) {
+  const before = new Map();
+  for (const item of previous) {
+    before.set(matrixEntryKey(item), entryFingerprint(item));
+  }
+  const keys = new Set();
+  for (const item of current) {
+    const key = matrixEntryKey(item);
+    if (before.get(key) !== entryFingerprint(item)) keys.add(key);
+  }
+  return keys;
+}
+
 function matchedSharedRules(changedFiles) {
   const rules = [];
   for (const file of changedFiles) {
@@ -259,20 +349,46 @@ function matchedSharedRules(changedFiles) {
 // Filter both matrices against a changed-file list. Returns the surviving
 // entries plus the set of backend names considered changed, which drives the
 // per-backend boolean outputs consumed by test-extra.yml.
-export function filterMatrix({ includes, includesDarwin, changedFiles }) {
+export function filterMatrix({
+  includes,
+  includesDarwin,
+  changedFiles,
+  previousMatrix,
+}) {
   const sharedRules = matchedSharedRules(changedFiles);
 
+  const matrixFileChanged = changedFiles.includes(BACKEND_MATRIX_FILE);
+  // The matrix file changed but we could not resolve what it used to say (API
+  // failure, shallow history, first push). Claiming "no entry changed" would
+  // reintroduce exactly the silent-empty-matrix failure this file exists to
+  // prevent, so fall back to rebuilding everything — the same posture
+  // changed-backends.js takes for a truncated or unavailable diff.
+  const matrixDiffUnavailable = matrixFileChanged && !previousMatrix;
+
+  const changedLinuxKeys =
+    matrixFileChanged && previousMatrix
+      ? changedEntryKeys(includes, previousMatrix.include || [])
+      : null;
+  const changedDarwinKeys =
+    matrixFileChanged && previousMatrix
+      ? changedEntryKeys(includesDarwin, previousMatrix.includeDarwin || [])
+      : null;
+
   const filtered = includes.filter(item => {
     const backendPath = inferBackendPath(item);
     if (!backendPath) return false;
     if (backendChanged(item.backend, backendPath, changedFiles)) return true;
     if (dockerfileChanged(item, changedFiles)) return true;
+    if (matrixDiffUnavailable) return true;
+    if (changedLinuxKeys && changedLinuxKeys.has(matrixEntryKey(item))) return true;
     return sharedRules.some(rule => rule.linux(item));
   });
 
   const filteredDarwin = includesDarwin.filter(item => {
     const backendPath = inferBackendPathDarwin(item);
     if (changedFiles.some(file => file.startsWith(backendPath))) return true;
+    if (matrixDiffUnavailable) return true;
+    if (changedDarwinKeys && changedDarwinKeys.has(matrixEntryKey(item))) return true;
     return sharedRules.some(rule => rule.darwin(item));
   });