fix(ci): pre-fetch common-ancestor objects for config sync in 'ci pr' (#10464)

`bit ci pr --keep-lane` kept logging `... was not found on the
filesystem` / `skipping config sync from main` for every component, even
after #10460.

#10460 pre-fetched main's **head** Version object before
`syncConfigFromMain`'s 3-way config merge. But that merge loads
**three** Version objects per component: the lane head, main's head, and
their **common ancestor** (the lane's fork point). On a fresh CI runner
the fork-point object is never fetched — switching to the lane brings
the parent *graph* (version-history), not each ancestor's full Version —
so `loadVersion(baseSnap)` throws `VersionNotFoundOnFS`, the
per-component catch swallows it, and the merge silently no-ops for every
diverged component (which is all of them on a reused lane once main has
advanced).

Confirmed from a failing run: every reported hash is the fork-point tag
(e.g. `1.0.1038`), not the current main head (`1.0.1041`) — main's head
loaded fine, the base is what was missing.

**Fix:** after the existing head pre-fetch, resolve each component's
diverge state (the graph is already local) and batch-fetch the
common-ancestor base objects too, before the merge loop. Same
best-effort contract as the head pre-fetch; the base is always an
ancestor of main's head, so it's always fetchable.

Adds an e2e regression that reproduces the fresh-runner state by
dropping the fork-point object before the second `ci pr` and asserts
config sync still lands.

Author: davidfirst

e2e/harmony/ci-commands.e2e.ts

@@ -365,6 +365,81 @@ describe('ci commands', function () {
     });
   });
 
+  /**
+   * Regression for the "was not found on the filesystem" config-sync failure (#10460 follow-up).
+   *
+   * `syncConfigFromMain` does a 3-way merge that loads three Version objects per component: the
+   * lane head, main's head, and their COMMON ANCESTOR (the lane's fork point). On a real CI runner
+   * the workspace is fresh: switching to the lane fetches the lane heads plus the version-history
+   * (the parent graph) — enough to compute divergence — but NOT the full Version object of the fork
+   * point, which lives only on main. #10460 pre-fetched main's head; this test guards that the fork
+   * point is pre-fetched too. We reproduce the fresh-runner state by deleting the fork-point object
+   * from the local scope before the second `bit ci pr` — without the fix, loading it throws
+   * VersionNotFoundOnFS, the sync silently skips the component, and main's config never lands.
+   */
+  describe('bit ci pr syncs main config even when the fork-point object is not on the filesystem', () => {
+    let secondPrOutput: string;
+    let forkPointHash: string;
+    before(() => {
+      helper.scopeHelper.setWorkspaceWithRemoteScope();
+      setupGitRemote();
+      const defaultBranch = setupComponentsAndInitialCommit();
+      // The fork point: comp1's head on main right now, before the lane snaps on top of it. This is
+      // the common ancestor the config merge will need as its base.
+      forkPointHash = helper.command.getHead('comp1');
+
+      // First PR commit + ci pr — lane forks here and snaps comp1 on top of the fork point.
+      helper.command.runCmd('git checkout -b feature/fork-point-test');
+      helper.fs.outputFile('comp1/comp1.js', 'console.log("pr commit 1");');
+      helper.command.runCmd('git add comp1/comp1.js');
+      helper.command.runCmd('git commit -m "feat: pr commit 1"');
+      helper.command.runCmd('bit ci pr --keep-lane --message "first pr commit"');
+
+      // Main moves ahead with a deps-set config change, so the lane and main have both snapped since
+      // the fork (they diverge — the merge base is the fork point, not either head).
+      helper.command.runCmd(`git checkout ${defaultBranch}`);
+      helper.npm.addFakeNpmPackage('is-positive', '1.0.0');
+      helper.workspaceJsonc.addPolicyToDependencyResolver({ dependencies: { 'is-positive': '1.0.0' } });
+      helper.command.dependenciesSet('comp1', 'is-positive@1.0.0');
+      helper.command.tagAllWithoutBuild();
+      helper.command.export();
+      helper.command.runCmd('git add .');
+      helper.command.runCmd('git commit -m "chore: bump comp1 deps on main"');
+      helper.command.runCmd(`git push origin ${defaultBranch}`);
+
+      // Simulate the fresh CI runner: it never fetched the fork-point Version object (only the lane
+      // heads + parent graph). Assert it's actually present first — so a wrong hash or setup change
+      // can't turn this into a no-op — then drop it so the merge base is missing on disk.
+      helper.fs.expectFileToExist(helper.general.getHashPathOfObject(forkPointHash, true));
+      helper.fs.deleteObject(helper.general.getHashPathOfObject(forkPointHash));
+
+      // Second PR commit + ci pr — must re-fetch the fork point and sync main's deps change.
+      helper.command.runCmd('git checkout feature/fork-point-test');
+      helper.command.runCmd(`git merge ${defaultBranch}`);
+      helper.fs.outputFile('comp2/comp2.js', 'console.log("pr commit 2");');
+      helper.command.runCmd('git add comp2/comp2.js');
+      helper.command.runCmd('git commit -m "feat: pr commit 2"');
+      secondPrOutput = helper.command.runCmd('bit ci pr --keep-lane --message "second pr commit"');
+    });
+
+    it('should not skip config sync with a "was not found on the filesystem" error', () => {
+      // The exact production symptom (VersionNotFoundOnFS on the missing fork point). The functional
+      // outcome is covered by the config assertion below; this just guards the specific regression.
+      expect(secondPrOutput).to.not.include('was not found on the filesystem');
+    });
+
+    it("should still propagate main's deps-set config change to comp1 on the lane", () => {
+      helper.command.switchLocalLane('feature-fork-point-test');
+      const showConfig = helper.command.showAspectConfig('comp1', Extensions.dependencyResolver);
+      const dep = showConfig.data.dependencies.find((d: any) => d.id === 'is-positive');
+      expect(
+        dep,
+        `expected 'is-positive' on comp1's deps after merging main, got: ${JSON.stringify(showConfig.data.dependencies, null, 2)}`
+      ).to.exist;
+      expect(dep.version).to.equal('1.0.0');
+    });
+  });
+
   /**
    * Same as the deps-set propagation test, but for an `bit env set` on main. Env config has its own
    * strategy in the config merger, so it's worth covering explicitly: a long-running PR's lane must

scopes/git/ci/ci.main.runtime.ts

@@ -33,6 +33,8 @@ import type { LaneId } from '@teambit/lane-id';
 import type { ConsumerComponent } from '@teambit/legacy.consumer-component';
 import { SourceBranchDetector } from './source-branch-detector';
 import { generateRandomStr } from '@teambit/toolbox.string.random';
+import { pMapPool } from '@teambit/toolbox.promise.map-pool';
+import { concurrentComponentsLimit } from '@teambit/harmony.modules.concurrency';
 import { extractSkipTasksFromMessage } from './skip-tasks-from-message';
 
 // Two distinct conflicts can surface from the remote on a concurrent `bit ci pr` race.
@@ -420,36 +422,66 @@ export class CiMain {
     // lane-merge flows — see merge-status-provider / merge-lanes). Pass the *specific* main-head
     // version so `cache: true` still fetches it: the component already exists locally at its lane
     // version, so a version-less id would look satisfied and skip the remote.
-    if (componentsToSync.length) {
-      const mainHeadIds = componentsToSync.map(({ laneComp, mainHead }) =>
-        laneComp.id.changeVersion(mainHead.toString())
-      );
-      try {
-        await this.importer.importObjectsFromMainIfExist(mainHeadIds, { cache: true });
-      } catch (e: any) {
-        // Best-effort: a fetch hiccup shouldn't abort `bit ci pr`. The merge loop below still runs;
-        // any component whose main head is still missing just logs the existing skip.
-        this.logger.console(
-          chalk.yellow(`Could not pre-fetch main's objects for config sync (continuing): ${e?.message || e}`)
-        );
-      }
-    }
+    const mainHeadIds = componentsToSync.map(({ laneComp, mainHead }) =>
+      laneComp.id.changeVersion(mainHead.toString())
+    );
+    await this.prefetchFromMainForConfigSync(mainHeadIds, 'head objects');
+
+    // Resolve each component's diverge state up front — before the merge loop — so we can also
+    // pre-fetch the common-ancestor objects below. getDivergeData only walks the parent graph,
+    // which is already local (switchToLane brings the version-history, and the head pre-fetch above
+    // reinforced it), so no full Version object is needed yet. Keep only components where main is
+    // actually ahead or diverged; the rest have nothing to bring in from main. Bound the fan-out
+    // (getDivergeData traverses each component's version graph) so a lane with many components
+    // doesn't spawn one unbounded burst of concurrent graph walks.
+    const componentsToMerge = compact(
+      await pMapPool(
+        componentsToSync,
+        async (item) => {
+          try {
+            const divergeData = await getDivergeData({
+              repo,
+              modelComponent: item.modelComponent,
+              sourceHead: item.laneComp.head,
+              targetHead: item.mainHead,
+              throws: false,
+            });
+            if (!divergeData.isTargetAhead() && !divergeData.isDiverged()) return undefined;
+            return { ...item, divergeData };
+          } catch (e: any) {
+            // Best-effort per component (same contract as the merge loop below).
+            this.logger.console(
+              chalk.yellow(
+                `  ${item.laneComp.id.toStringWithoutVersion()}: skipping config sync from main (${e?.message || e})`
+              )
+            );
+            return undefined;
+          }
+        },
+        { concurrency: concurrentComponentsLimit() }
+      )
+    );
+
+    // The head pre-fetch above brought main's head Version plus the version-history (parent graph),
+    // but NOT the full Version object of the common ancestor (the lane's fork point) for components
+    // where the lane and main have BOTH snapped since the fork. The 3-way config merge below loads
+    // that base Version (`baseVersion.extensions`); without it, `loadVersion(baseSnap)` throws
+    // VersionNotFoundOnFS, the per-component catch swallows it as "skipping config sync from main",
+    // and the sync silently no-ops for every diverged component. The fork point lives on main and,
+    // like the head, was never fetched (includeVersionHistory carries the graph, not each ancestor's
+    // Version). Batch-fetch the bases in one call — same best-effort contract as the head pre-fetch.
+    const baseIds = compact(
+      componentsToMerge.map(({ laneComp, divergeData }) => {
+        const baseSnap = divergeData.commonSnapBeforeDiverge;
+        return baseSnap ? laneComp.id.changeVersion(baseSnap.toString()) : undefined;
+      })
+    );
+    await this.prefetchFromMainForConfigSync(baseIds, 'common-ancestor objects');
 
     const syncedIds: ComponentID[] = [];
-    for (const { laneComp, modelComponent, mainHead } of componentsToSync) {
+    for (const { laneComp, modelComponent, mainHead, divergeData } of componentsToMerge) {
       try {
         const laneHead = laneComp.head;
-        const divergeData = await getDivergeData({
-          repo,
-          modelComponent,
-          sourceHead: laneHead,
-          targetHead: mainHead,
-          throws: false,
-        });
-        // Only sync when main has snaps the lane doesn't (target ahead, or diverged). If the lane
-        // is ahead-only / equal there's nothing on main to bring in.
-        if (!divergeData.isTargetAhead() && !divergeData.isDiverged()) continue;
-
         const currentVersion = await modelComponent.loadVersion(laneHead.toString(), repo);
         const otherVersion = await modelComponent.loadVersion(mainHead.toString(), repo);
         // base = common ancestor. When the lane is strictly behind main (no divergence) the common
@@ -515,6 +547,23 @@ export class CiMain {
     this.logger.console(chalk.green(`Synced config from main for ${syncedIds.length} component(s)`));
   }
 
+  /**
+   * Batch-fetch main-side Version objects the config merge needs (heads, then common ancestors),
+   * mirroring the lane-merge flows (merge-status-provider / merge-lanes). Best-effort: a fetch
+   * hiccup shouldn't abort `bit ci pr` — the merge loop still runs and any component whose object is
+   * still missing just logs the existing per-component skip. `label` names which objects for the log.
+   */
+  private async prefetchFromMainForConfigSync(ids: ComponentID[], label: string) {
+    if (!ids.length) return;
+    try {
+      await this.importer.importObjectsFromMainIfExist(ids, { cache: true });
+    } catch (e: any) {
+      this.logger.console(
+        chalk.yellow(`Could not pre-fetch main's ${label} for config sync (continuing): ${e?.message || e}`)
+      );
+    }
+  }
+
   /**
    * Copied from `merging.main.runtime` (`filterDeletedDependenciesFromConfig`): the config merge
    * can emit deletion markers (`version: '-'`) for deps removed on main. The aspects-merger applies