feat: implement fast path resolution for Python environment managers (microsoft#1408)

- Added fast path resolution logic in SysPythonManager, VenvManager,
CondaEnvManager, PipenvManager, and PyEnvManager to directly resolve
persisted environments without waiting for full initialization.
- Enhanced error handling with logging for failed resolutions in the
fast path.
- Introduced unit tests to validate the fast path behavior across
different environment managers, ensuring correct resolution and
background initialization handling.

Author: eleanorjboyd

docs/startup-flow.md

@@ -5,19 +5,22 @@
 user opens VS Code
 python environments extension begins activation
 
-SYNC (`activate` in extension.ts):
-1. create core objects: ProjectManager, EnvironmentManagers, ManagerReady
-2. `setPythonApi()` — API object created, deferred resolved (API is now available to consumers)
-3. create views (EnvManagerView, ProjectView), status bar, terminal manager
-4. register all commands
-5. activate() returns — extension is "active" from VS Code's perspective
+**SYNC (`activate` in extension.ts):**
+1. create StatusBar, ProjectManager, EnvVarManager, EnvironmentManagers, ManagerReady
+2. create TerminalActivation, shell providers, TerminalManager
+3. create ProjectCreators
+4. `setPythonApi()` — API object created, deferred resolved (API is now available to consumers)
+5. create views (EnvManagerView, ProjectView)
+6. register all commands
+7. activate() returns — extension is "active" from VS Code's perspective
 
    📊 TELEMETRY: EXTENSION.ACTIVATION_DURATION { duration }
 
-ASYNC (setImmediate callback, still in extension.ts):
+**ASYNC (setImmediate callback, still in extension.ts):**
 1. spawn PET process (`createNativePythonFinder`)
    1. sets up a JSON-RPC connection to it over stdin/stdout
-2. register all built-in managers in parallel (Promise.all):
+2. register all built-in managers + shell env init in parallel (Promise.all):
+   - `shellStartupVarsMgr.initialize()`
    - for each manager (system, conda, pyenv, pipenv, poetry):
      1. check if tool exists (e.g. `getConda(nativeFinder)` asks PET for the conda binary)
      2. if tool not found → log, return early (manager not registered)
@@ -26,17 +29,19 @@ ASYNC (setImmediate callback, still in extension.ts):
         - fires `onDidChangeEnvironmentManager` → `ManagerReady` deferred resolves for this manager
 3. all registrations complete (Promise.all resolves)
 
---- gate point: `applyInitialEnvironmentSelection` ---
+**--- gate point: `applyInitialEnvironmentSelection` ---**
+
    📊 TELEMETRY: ENV_SELECTION.STARTED { duration (activation→here), registeredManagerCount, registeredManagerIds, workspaceFolderCount }
 
 1. for each workspace folder + global scope (no workspace case), run `resolvePriorityChainCore` to find manager:
    - P1: pythonProjects[] setting → specific manager for this project
    - P2: user-configured defaultEnvManager setting
    - P3: user-configured python.defaultInterpreterPath → nativeFinder.resolve(path)
-   - P4: auto-discovery → try venv manager (local .venv), fall back to system python
-     - for workspace scope: ask venv manager if there's a local env (.venv/venv in the folder)
-       - if found → use venv manager with that env
-       - if not found → fall back to system python manager
+   - P4: auto-discovery → try venv manager, fall back to system python
+     - for workspace scope: call `venvManager.get(scope)`
+       - if venv found (local .venv/venv) → use venv manager with that env
+       - if no local venv → venv manager may still return its `globalEnv` (system Python)
+       - if venvManager.get returns undefined → fall back to system python manager
      - for global scope: use system python manager directly
 
 2. get the environment from the winning priority level:
@@ -54,30 +59,55 @@ ASYNC (setImmediate callback, still in extension.ts):
 
    managerDiscovery — P1, P2, or P4 won (manager → interpreter):
      `resolvePriorityChainCore` returns { manager, environment: undefined }
-       → result.environment is undefined → falls through to `await result.manager.get(scope)`
-     `manager.get(scope)` (e.g. `CondaEnvManager.get()`):
-       4. `initialize()` — lazy, once-only per manager (guarded by deferred)
-          a. `nativeFinder.refresh(hardRefresh=false)`:
-             → `handleSoftRefresh()` checks in-memory cache (Map) for key 'all' (bc one big scan, shared cache, all managers benefit)
-               - on reload: cache is empty (Map was destroyed) → cache miss
-               - falls through to `handleHardRefresh()`
-             → `handleHardRefresh()`:
-               - adds request to WorkerPool queue (concurrency 1, so serialized)
-               - when its turn comes, calls `doRefresh()`:
-                 1. `configure()` — JSON-RPC to PET with search paths, conda/poetry/pipenv paths, cache dir
-                 2. `refresh` — JSON-RPC to PET, PET scans filesystem
-                    - PET may use its own on-disk cache (cacheDirectory) to speed this up
-                    - PET streams back results as 'environment' and 'manager' notifications
-                    - envs missing version/prefix get an inline resolve() call
-                 3. returns NativeInfo[] (all envs of all types)
-               - result stored in in-memory cache under key 'all'
-             → subsequent managers calling nativeFinder.refresh(false) get cache hit → instant
-          b. filter results to this manager's env type (e.g. conda filters to kind=conda)
-          c. convert NativeEnvInfo → PythonEnvironment objects → populate collection
-          d. `loadEnvMap()` — reads persisted env path from workspace state
-             → matches path against freshly discovered collection via `findEnvironmentByPath()`
-             → populates `fsPathToEnv` map
-       5. look up scope in `fsPathToEnv` → return the matched env
+       → falls through to `await result.manager.get(scope)`
+
+       **--- inner fork: fast path vs slow path (tryFastPathGet in fastPath.ts) ---**
+      Conditions checked before entering fast path:
+         a. `_initialized` deferred is undefined (never created) OR has not yet completed
+         b. scope is a `Uri` (not global/undefined)
+
+         FAST PATH (background init kickoff + optional early return):
+         **Race-condition safety (runs before any await):**
+         1. if `_initialized` doesn't exist yet:
+            - create deferred and **register immediately** via `setInitialized()` callback
+            - this blocks concurrent callers from spawning duplicate background inits
+              - kick off `startBackgroundInit()` as fire-and-forget
+                 - this happens as soon as (a) and (b) are true, **even if** no persisted path exists
+         2. get project fsPath: `getProjectFsPathForScope(api, scope)` 
+            - prefers resolved project path if available, falls back to scope.fsPath
+            - shared across all managers to avoid lambda duplication
+           3. read persisted path (only if scope is a `Uri`; may return undefined)
+           4. if a persisted path exists:
+              - attempt `resolve(persistedPath)`
+              - failure (no env, mismatched manager, etc.) → fall through to SLOW PATH
+              - success → return env immediately (background init continues in parallel)
+         **Failure recovery (in startBackgroundInit error handler):**
+         - if background init throws: `setInitialized(undefined)` — clear deferred so next `get()` call retries init
+
+       SLOW PATH — fast path conditions not met, or fast path failed:
+         4. `initialize()` — lazy, once-only per manager (guarded by `_initialized` deferred)
+            **Once-only guarantee:**
+            - first caller creates `_initialized` deferred (if not already created by fast path)
+            - concurrent callers see the existing deferred and await it instead of re-running init
+            - deferred is **not cleared on failure** here (unlike in fast-path background handler)
+              so only one init attempt runs, but subsequent calls still await the same failed init
+            **Note:** In the fast path, if background init fails, the deferred is cleared to allow retry
+            a. `nativeFinder.refresh(hardRefresh=false)`:
+               → internally calls `handleSoftRefresh()` → computes cache key from options
+                 - on reload: cache is empty (Map was destroyed) → cache miss
+                 - falls through to `handleHardRefresh()`
+               → `handleHardRefresh()` adds request to WorkerPool queue (concurrency 1):
+                   1. run `configure()` to setup PET search paths
+                   2. run `refresh` — PET scans filesystem
+                      - PET may use its own on-disk cache
+                   3. returns NativeInfo[] (all envs of all types)
+                      - result stored in in-memory cache so subsequent managers get instant cache hit
+            b. filter results to this manager's env type (e.g. conda filters to kind=conda)
+            c. convert NativeEnvInfo → PythonEnvironment objects → populate collection
+            d. `loadEnvMap()` — reads persisted env path from workspace state
+               → matches path against PET discovery results
+               → populates `fsPathToEnv` map
+         5. look up scope in `fsPathToEnv` → return the matched env
 
    📊 TELEMETRY: ENV_SELECTION.RESULT (per scope) { duration (priority chain + manager.get), scope, prioritySource, managerId, path, hasPersistedSelection }
 
@@ -86,8 +116,8 @@ ASYNC (setImmediate callback, still in extension.ts):
 
    📊 TELEMETRY: EXTENSION.MANAGER_REGISTRATION_DURATION { duration (activation→here), result, failureStage?, errorType? }
 
-POST-INIT:
+**POST-INIT:**
 1. register terminal package watcher
 2. register settings change listener (`registerInterpreterSettingsChangeListener`) — re-runs priority chain if settings change
 3.  initialize terminal manager
-4.  send telemetry (manager selection, project structure, discovery summary)
+4.  send telemetry (manager selection, project structure, discovery summary)

src/managers/builtin/sysPythonManager.ts

@@ -20,6 +20,7 @@ import {
 import { SysManagerStrings } from '../../common/localize';
 import { createDeferred, Deferred } from '../../common/utils/deferred';
 import { normalizePath } from '../../common/utils/pathUtils';
+import { getProjectFsPathForScope, tryFastPathGet } from '../common/fastPath';
 import { NativePythonFinder } from '../common/nativePythonFinder';
 import { getLatest } from '../common/utils';
 import {
@@ -145,6 +146,22 @@ export class SysPythonManager implements EnvironmentManager {
     }
 
     async get(scope: GetEnvironmentScope): Promise<PythonEnvironment | undefined> {
+        const fastResult = await tryFastPathGet({
+            initialized: this._initialized,
+            setInitialized: (deferred) => {
+                this._initialized = deferred;
+            },
+            scope,
+            label: 'system',
+            getProjectFsPath: (s) => getProjectFsPathForScope(this.api, s),
+            getPersistedPath: (fsPath) => getSystemEnvForWorkspace(fsPath),
+            resolve: (p) => resolveSystemPythonEnvironmentPath(p, this.nativeFinder, this.api, this),
+            startBackgroundInit: () => this.internalRefresh(false, SysManagerStrings.sysManagerDiscovering),
+        });
+        if (fastResult) {
+            return fastResult.env;
+        }
+
         await this.initialize();
 
         if (scope instanceof Uri) {

src/managers/builtin/venvManager.ts

@@ -35,6 +35,7 @@ import { createDeferred, Deferred } from '../../common/utils/deferred';
 import { normalizePath } from '../../common/utils/pathUtils';
 import { showErrorMessage, showInformationMessage, withProgress } from '../../common/window.apis';
 import { findParentIfFile } from '../../features/envCommands';
+import { getProjectFsPathForScope, tryFastPathGet } from '../common/fastPath';
 import { NativePythonFinder } from '../common/nativePythonFinder';
 import { getLatest, shortVersion, sortEnvironments } from '../common/utils';
 import { promptInstallPythonViaUv } from './uvPythonInstaller';
@@ -366,6 +367,22 @@ export class VenvManager implements EnvironmentManager {
     }
 
     async get(scope: GetEnvironmentScope): Promise<PythonEnvironment | undefined> {
+        const fastResult = await tryFastPathGet({
+            initialized: this._initialized,
+            setInitialized: (deferred) => {
+                this._initialized = deferred;
+            },
+            scope,
+            label: 'venv',
+            getProjectFsPath: (s) => getProjectFsPathForScope(this.api, s),
+            getPersistedPath: (fsPath) => getVenvForWorkspace(fsPath),
+            resolve: (p) => resolveVenvPythonEnvironmentPath(p, this.nativeFinder, this.api, this, this.baseManager),
+            startBackgroundInit: () => this.internalRefresh(undefined, false, VenvManagerStrings.venvInitialize),
+        });
+        if (fastResult) {
+            return fastResult.env;
+        }
+
         await this.initialize();
 
         if (!scope) {

src/managers/common/fastPath.ts

@@ -0,0 +1,101 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { Uri } from 'vscode';
+import { GetEnvironmentScope, PythonEnvironment, PythonEnvironmentApi } from '../../api';
+import { traceError, traceWarn } from '../../common/logging';
+import { createDeferred, Deferred } from '../../common/utils/deferred';
+
+/**
+ * Options for the fast-path resolution in manager.get().
+ */
+export interface FastPathOptions {
+    /** The current _initialized deferred (may be undefined if init hasn't started). */
+    initialized: Deferred<void> | undefined;
+    /** Updates the manager's _initialized deferred. */
+    setInitialized: (initialized: Deferred<void> | undefined) => void;
+    /** The scope passed to get(). */
+    scope: GetEnvironmentScope;
+    /** Label for log messages, e.g. 'venv', 'conda'. */
+    label: string;
+    /** Gets the project fsPath for a given Uri scope. */
+    getProjectFsPath: (scope: Uri) => string;
+    /** Reads the persisted env path for a workspace fsPath. */
+    getPersistedPath: (workspaceFsPath: string) => Promise<string | undefined>;
+    /** Resolves a persisted path to a full PythonEnvironment. */
+    resolve: (persistedPath: string) => Promise<PythonEnvironment | undefined>;
+    /** Starts background initialization (full discovery). Returns a promise that completes when init is done. */
+    startBackgroundInit: () => Promise<void> | Thenable<void>;
+}
+
+/**
+ * Result from a successful fast-path resolution.
+ */
+export interface FastPathResult {
+    /** The resolved environment. */
+    env: PythonEnvironment;
+}
+
+/**
+ * Gets the fsPath for a scope by preferring the resolved project path when available.
+ */
+export function getProjectFsPathForScope(api: Pick<PythonEnvironmentApi, 'getPythonProject'>, scope: Uri): string {
+    return api.getPythonProject(scope)?.uri.fsPath ?? scope.fsPath;
+}
+
+/**
+ * Attempts fast-path resolution for manager.get(): if full initialization hasn't completed yet
+ * and there's a persisted environment for the workspace, resolve it directly via nativeFinder
+ * instead of waiting for full discovery.
+ *
+ * Returns the resolved environment (with an optional new deferred) if successful, or undefined
+ * to fall through to the normal init path.
+ */
+export async function tryFastPathGet(opts: FastPathOptions): Promise<FastPathResult | undefined> {
+    if (!(opts.scope instanceof Uri)) {
+        return undefined;
+    }
+
+    if (opts.initialized?.completed) {
+        return undefined;
+    }
+
+    let deferred = opts.initialized;
+    if (!deferred) {
+        // Register deferred before any await to avoid concurrent callers starting duplicate inits.
+        deferred = createDeferred<void>();
+        opts.setInitialized(deferred);
+        const deferredRef = deferred;
+        try {
+            Promise.resolve(opts.startBackgroundInit()).then(
+                () => deferredRef.resolve(),
+                (err) => {
+                    traceError(`[${opts.label}] Background initialization failed:`, err);
+                    // Allow subsequent get()/initialize() calls to retry after a background init failure.
+                    opts.setInitialized(undefined);
+                    deferredRef.resolve();
+                },
+            );
+        } catch (syncErr) {
+            traceError(`[${opts.label}] Background initialization threw synchronously:`, syncErr);
+            opts.setInitialized(undefined);
+            deferredRef.resolve();
+        }
+    }
+
+    const fsPath = opts.getProjectFsPath(opts.scope);
+    const persistedPath = await opts.getPersistedPath(fsPath);
+
+    if (persistedPath) {
+        try {
+            const resolved = await opts.resolve(persistedPath);
+            if (resolved) {
+                return { env: resolved };
+            }
+        } catch (err) {
+            traceWarn(`[${opts.label}] Fast path resolve failed for '${persistedPath}', falling back to full init:`, err);
+        }
+    }
+
+    return undefined;
+}

src/managers/conda/condaEnvManager.ts

@@ -24,6 +24,7 @@ import { traceError } from '../../common/logging';
 import { createDeferred, Deferred } from '../../common/utils/deferred';
 import { normalizePath } from '../../common/utils/pathUtils';
 import { showErrorMessage, showInformationMessage, withProgress } from '../../common/window.apis';
+import { getProjectFsPathForScope, tryFastPathGet } from '../common/fastPath';
 import { NativePythonFinder } from '../common/nativePythonFinder';
 import { CondaSourcingStatus } from './condaSourcingUtils';
 import {
@@ -260,6 +261,32 @@ export class CondaEnvManager implements EnvironmentManager, Disposable {
         }
     }
     async get(scope: GetEnvironmentScope): Promise<PythonEnvironment | undefined> {
+        const fastResult = await tryFastPathGet({
+            initialized: this._initialized,
+            setInitialized: (deferred) => {
+                this._initialized = deferred;
+            },
+            scope,
+            label: 'conda',
+            getProjectFsPath: (s) => getProjectFsPathForScope(this.api, s),
+            getPersistedPath: (fsPath) => getCondaForWorkspace(fsPath),
+            resolve: (p) => resolveCondaPath(p, this.nativeFinder, this.api, this.log, this),
+            startBackgroundInit: () =>
+                withProgress({ location: ProgressLocation.Window, title: CondaStrings.condaDiscovering }, async () => {
+                    this.collection = await refreshCondaEnvs(false, this.nativeFinder, this.api, this.log, this);
+                    await this.loadEnvMap();
+                    this._onDidChangeEnvironments.fire(
+                        this.collection.map((e) => ({
+                            environment: e,
+                            kind: EnvironmentChangeKind.add,
+                        })),
+                    );
+                }),
+        });
+        if (fastResult) {
+            return fastResult.env;
+        }
+
         await this.initialize();
         if (scope instanceof Uri) {
             let env = this.fsPathToEnv.get(normalizePath(scope.fsPath));