feat: glob pattern support in executionEnvironments root and extraPaths

Author: jacks0n

docs/configuration/config-files.md

@@ -30,7 +30,7 @@ The following settings control the *environment* in which basedpyright will chec
 
 - **verboseOutput** [boolean]: Specifies whether output logs should be verbose. This is useful when diagnosing certain problems like import resolution issues.
 
-- **extraPaths** [array of strings, optional]: Additional search paths that will be used when searching for modules imported by files.
+- **extraPaths** [array of strings, optional]: Additional search paths that will be used when searching for modules imported by files. Glob patterns (`**`, `*`, `?`) are supported *(basedpyright exclusive)* — matching directories are expanded at configuration time.
 
 - **pythonVersion** [string, optional]: Specifies the version of Python that will be used to execute the source code. The version should be specified as a string in the format "M.m" where M is the major version and m is the minor (e.g. `"3.0"` or `"3.6"`). If a version is provided, pyright will generate errors if the source code makes use of language features that are not supported in that version. It will also tailor its use of type stub files, which conditionalizes type definitions based on the version. If no version is specified, pyright will use the version of the current python interpreter, if one is present.
 
@@ -314,13 +314,13 @@ The following settings allow more fine grained control over the **typeCheckingMo
 
 
 ## Execution Environment Options
-Pyright allows multiple “execution environments” to be defined for different portions of your source tree. For example, a subtree may be designed to run with different import search paths or a different version of the python interpreter than the rest of the source base.
+Pyright allows multiple "execution environments" to be defined for different portions of your source tree. For example, a subtree may be designed to run with different import search paths or a different version of the python interpreter than the rest of the source base.
 
-The following settings can be specified for each execution environment. Each source file within a project is associated with at most one execution environment -- the first one whose root directory contains that file.
+The following settings can be specified for each execution environment. Each source file within a project is associated with at most one execution environment -- the first one whose root matches that file. Environments are searched in array order; the first match wins.
 
-- **root** [string, required]: Root path for the code that will execute within this execution environment.
+- **root** [string, required]: Root path for the code that will execute within this execution environment. Glob patterns (`**`, `*`, `?`) are supported *(basedpyright exclusive)* — when used, import resolution falls back to the project root.
 
-- **extraPaths** [array of strings, optional]: Additional search paths (in addition to the root path) that will be used when searching for modules imported by files within this execution environment. If specified, this overrides the default extraPaths setting when resolving imports for files within this execution environment. Note that each file’s execution environment mapping is independent, so if file A is in one execution environment and imports a second file B within a second execution environment, any imports from B will use the extraPaths in the second execution environment.
+- **extraPaths** [array of strings, optional]: Additional search paths (in addition to the root path) that will be used when searching for modules imported by files within this execution environment. Glob patterns (`**`, `*`, `?`) are supported *(basedpyright exclusive)* — matching directories are expanded at configuration time. If specified, this overrides the default extraPaths setting when resolving imports for files within this execution environment. Note that each file's execution environment mapping is independent, so if file A is in one execution environment and imports a second file B within a second execution environment, any imports from B will use the extraPaths in the second execution environment.
 
 - **pythonVersion** [string, optional]: The version of Python used for this execution environment. If not specified, the global `pythonVersion` setting is used instead.
 
@@ -377,10 +377,10 @@ The following is an example of a pyright config file:
       ]
     },
     {
-      "root": "src/tests",
+      "root": "**/tests",
       "reportPrivateUsage": false,
       "extraPaths": [
-        "src/tests/e2e",
+        "**/fixtures",
         "src/sdk"
       ]
     },
@@ -413,7 +413,7 @@ pythonPlatform = "Linux"
 executionEnvironments = [
   { root = "src/web", pythonVersion = "3.5", pythonPlatform = "Windows", extraPaths = [ "src/service_libs" ], reportMissingImports = "warning" },
   { root = "src/sdk", pythonVersion = "3.0", extraPaths = [ "src/backend" ] },
-  { root = "src/tests", reportPrivateUsage = false, extraPaths = ["src/tests/e2e", "src/sdk" ]},
+  { root = "**/tests", reportPrivateUsage = false, extraPaths = ["**/fixtures", "src/sdk"] },
   { root = "src" }
 ]
 ```

packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts

@@ -277,7 +277,8 @@ export class BackgroundAnalysisProgram {
     }
 
     private _ensurePartialStubPackages(execEnv: ExecutionEnvironment) {
-        this._backgroundAnalysis?.ensurePartialStubPackages(execEnv.root?.toString());
+        const execEnvIndex = this.configOptions.getExecutionEnvironments().indexOf(execEnv);
+        this._backgroundAnalysis?.ensurePartialStubPackages(execEnvIndex);
         return this._importResolver.ensurePartialStubPackages(execEnv);
     }
 

packages/pyright-internal/src/analyzer/importResolver.ts

@@ -413,7 +413,7 @@ export class ImportResolver {
             return false;
         }
 
-        if (this.partialStubs.isPartialStubPackagesScanned(execEnv)) {
+        if (execEnv.root && this.partialStubs.isPathScanned(execEnv.root)) {
             return false;
         }
 

packages/pyright-internal/src/backgroundAnalysisBase.ts

@@ -55,7 +55,7 @@ export interface IBackgroundAnalysis extends Disposable {
     setConfigOptions(configOptions: ConfigOptions): void;
     setTrackedFiles(fileUris: Uri[]): void;
     setAllowedThirdPartyImports(importNames: string[]): void;
-    ensurePartialStubPackages(executionRoot: string | undefined): void;
+    ensurePartialStubPackages(execEnvIndex: number): void;
     setFileOpened(fileUri: Uri, version: number | null, contents: string, options: OpenFileOptions): void;
     updateChainedUri(fileUri: Uri, chainedUri: Uri | undefined): void;
     setFileClosed(fileUri: Uri, isTracked?: boolean): void;
@@ -158,8 +158,8 @@ export class BackgroundAnalysisBase implements IBackgroundAnalysis {
         this.enqueueRequest({ requestType: 'setAllowedThirdPartyImports', data: serialize(importNames) });
     }
 
-    ensurePartialStubPackages(executionRoot: string | undefined) {
-        this.enqueueRequest({ requestType: 'ensurePartialStubPackages', data: serialize({ executionRoot }) });
+    ensurePartialStubPackages(execEnvIndex: number) {
+        this.enqueueRequest({ requestType: 'ensurePartialStubPackages', data: serialize({ execEnvIndex }) });
     }
 
     setFileOpened(fileUri: Uri, version: number | null, contents: string, options: OpenFileOptions) {
@@ -589,8 +589,8 @@ export abstract class BackgroundAnalysisRunnerBase extends BackgroundThreadBase
             }
 
             case 'ensurePartialStubPackages': {
-                const { executionRoot } = deserialize(msg.data);
-                this.handleEnsurePartialStubPackages(executionRoot);
+                const { execEnvIndex } = deserialize(msg.data);
+                this.handleEnsurePartialStubPackages(execEnvIndex);
                 break;
             }
 
@@ -775,10 +775,8 @@ export abstract class BackgroundAnalysisRunnerBase extends BackgroundThreadBase
         this.program.setAllowedThirdPartyImports(importNames);
     }
 
-    protected handleEnsurePartialStubPackages(executionRoot: string | undefined) {
-        const execEnv = this._configOptions
-            .getExecutionEnvironments()
-            .find((e) => e.root?.toString() === executionRoot);
+    protected handleEnsurePartialStubPackages(execEnvIndex: number) {
+        const execEnv = this._configOptions.getExecutionEnvironments()[execEnvIndex];
         if (execEnv) {
             this.importResolver.ensurePartialStubPackages(execEnv);
         }

packages/pyright-internal/src/common/configOptions.ts

@@ -7,6 +7,9 @@
  * Class that holds the configuration options for the analyzer.
  */
 
+import { globSync } from 'node:fs';
+import { matchesGlob } from 'node:path/posix';
+
 import { ImportLogger } from '../analyzer/importLogger';
 import { getPathsFromPthFiles } from '../analyzer/pythonPathUtils';
 import * as pathConsts from '../common/pathConsts';
@@ -27,7 +30,7 @@ import { PythonVersion, latestStablePythonVersion } from './pythonVersion';
 import { ServiceKeys } from './serviceKeys';
 import { ServiceProvider } from './serviceProvider';
 import { Uri } from './uri/uri';
-import { FileSpec, getFileSpec, isDirectory } from './uri/uriUtils';
+import { FileSpec, getFileSpec, getWildcardRoot, isDirectory } from './uri/uriUtils';
 import { userFacingOptionsList } from './stringUtils';
 
 // prevent upstream changes from sneaking in and adding errors using console.error,
@@ -66,6 +69,8 @@ export class ExecutionEnvironment {
     // tools or playgrounds).
     skipNativeLibraries: boolean;
 
+    rootGlob?: string;
+
     // Default to "." which indicates every file in the project.
     constructor(
         name: string,
@@ -1526,13 +1531,21 @@ export class ConfigOptions {
     // execution environment is used.
     findExecEnvironment(file: Uri): ExecutionEnvironment {
         return (
-            this.executionEnvironments.find((env) => {
-                const envRoot = Uri.is(env.root) ? env.root : this.projectRoot.resolvePaths(env.root || '');
-                return file.startsWith(envRoot);
-            }) ?? this.getDefaultExecEnvironment()
+            this.executionEnvironments.find((env) => this._fileMatchesEnvironment(file, env)) ??
+            this.getDefaultExecEnvironment()
         );
     }
 
+    private _fileMatchesEnvironment(file: Uri, env: ExecutionEnvironment): boolean {
+        if (env.rootGlob !== undefined) {
+            const relative = this.projectRoot.getRelativePath(file)?.slice(2);
+            if (relative === undefined) return false;
+            return matchesGlob(relative, env.rootGlob + '/**');
+        }
+        const envRoot = Uri.is(env.root) ? env.root : this.projectRoot.resolvePaths(env.root || '');
+        return file.startsWith(envRoot);
+    }
+
     getExecutionEnvironments(): ExecutionEnvironment[] {
         if (this.executionEnvironments.length > 0) {
             return this.executionEnvironments;
@@ -1691,7 +1704,7 @@ export class ConfigOptions {
                     if (typeof path !== 'string') {
                         console.error(`Config "extraPaths" field ${pathIndex} must be a string.`);
                     } else {
-                        configExtraPaths!.push(configDirUri.resolvePaths(path));
+                        this._resolveExtraPath(path, configDirUri, configExtraPaths, console);
                     }
                 });
                 this.defaultExtraPaths = [...configExtraPaths];
@@ -1989,6 +2002,20 @@ export class ConfigOptions {
         return this.pythonEnvironmentName || this.pythonPath?.toString() || 'python';
     }
 
+    private _resolveExtraPath(path: string, configDirUri: Uri, out: Uri[], console: ConsoleInterface) {
+        if (/[*?]/.test(path)) {
+            try {
+                for (const match of globSync(path, { cwd: configDirUri.getFilePath() })) {
+                    out.push(configDirUri.resolvePaths(match));
+                }
+            } catch (e) {
+                console.error(`Failed to expand glob pattern "${path}": ${e}`);
+            }
+        } else {
+            out.push(configDirUri.resolvePaths(path));
+        }
+    }
+
     private _convertBoolean(value: any, fieldName: string, defaultValue: boolean): boolean {
         if (value === undefined) {
             return defaultValue;
@@ -2043,7 +2070,8 @@ export class ConfigOptions {
 
             // Validate the root.
             if (envObj.root && typeof envObj.root === 'string') {
-                newExecEnv.root = configDirUri.resolvePaths(envObj.root);
+                newExecEnv.rootGlob = envObj.root;
+                newExecEnv.root = getWildcardRoot(configDirUri, envObj.root);
             } else {
                 console.error(`Config executionEnvironments index ${index}: missing root value.`);
             }
@@ -2067,7 +2095,7 @@ export class ConfigOptions {
                                     ` extraPaths field ${pathIndex} must be a string.`
                             );
                         } else {
-                            newExecEnv.extraPaths.push(configDirUri.resolvePaths(path));
+                            this._resolveExtraPath(path, configDirUri, newExecEnv.extraPaths, console);
                         }
                     });
                 }

packages/pyright-internal/src/tests/config.test.ts

@@ -12,7 +12,11 @@ import assert from 'assert';
 import { AnalyzerService } from '../analyzer/service';
 import { deserialize, serialize } from '../backgroundThreadBase';
 import { CommandLineOptions, DiagnosticSeverityOverrides } from '../common/commandLineOptions';
-import { ConfigOptions, ExecutionEnvironment, getStandardDiagnosticRuleSet } from '../common/configOptions';
+import {
+    ConfigOptions,
+    ExecutionEnvironment,
+    getStandardDiagnosticRuleSet,
+} from '../common/configOptions';
 import { ConsoleInterface, NullConsole } from '../common/console';
 import { TaskListPriority } from '../common/diagnostic';
 import { combinePaths, normalizePath, normalizeSlashes } from '../common/pathUtils';
@@ -740,4 +744,76 @@ describe(`config test'}`, () => {
             shouldRunAnalysis: () => true,
         });
     }
+
+    describe('glob root support', () => {
+        function setupExecEnvConfig(roots: ({ root: string } & Record<string, unknown>)[]) {
+            const cwd = UriEx.file(normalizePath(process.cwd()));
+            const configOptions = new ConfigOptions(cwd);
+            const json = { executionEnvironments: roots };
+            const fs = new TestFileSystem(false);
+            const console = new ErrorTrackingNullConsole();
+            const sp = createServiceProvider(fs, console);
+            configOptions.initializeFromJson(json, cwd, sp, new NoAccessHost());
+            configOptions.setupExecutionEnvironments(json, cwd, console);
+            return { cwd, configOptions, console };
+        }
+
+        test.each([
+            'src', '**/tests', 'src/*/utils', 'src/test?', '***/tests', '   ',
+        ])('root "%s" sets rootGlob', (root) => {
+            const { configOptions, console } = setupExecEnvConfig([{ root }]);
+            assert.deepStrictEqual(console.errors, []);
+            const env = configOptions.executionEnvironments[0];
+            assert.ok(env);
+            assert.strictEqual(env.rootGlob, root);
+            assert.ok(env.root);
+        });
+
+        test('serialization round-trip preserves rootGlob', () => {
+            const { configOptions } = setupExecEnvConfig([{ root: '**/tests' }]);
+            const cloned = deserialize<ConfigOptions>(serialize(configOptions));
+            assert.strictEqual(
+                cloned.executionEnvironments[0].rootGlob,
+                configOptions.executionEnvironments[0].rootGlob
+            );
+        });
+
+        test.each([
+            { roots: ['**/tests'], file: 'tests/test_foo.py', envIndex: 0 },
+            { roots: ['**/tests'], file: 'src/tests/test_foo.py', envIndex: 0 },
+            { roots: ['**/tests'], file: 'src/lib/deep/tests/test_bar.py', envIndex: 0 },
+            { roots: ['**/tests'], file: 'src/testing/foo.py', envIndex: -1 },
+            { roots: ['src/*/utils'], file: 'src/foo/utils/helper.py', envIndex: 0 },
+            { roots: ['src/*/utils'], file: 'src/foo/bar/utils/helper.py', envIndex: -1 },
+            { roots: ['src/v?/lib'], file: 'src/v1/lib/foo.py', envIndex: 0 },
+            { roots: ['src/v?/lib'], file: 'src/v10/lib/foo.py', envIndex: -1 },
+            { roots: ['src/core', '**/tests'], file: 'src/core/tests/test_foo.py', envIndex: 0 },
+            { roots: ['src/core', '**/tests'], file: 'lib/tests/test_bar.py', envIndex: 1 },
+            { roots: ['src/core', '**/tests'], file: 'other/module.py', envIndex: -1 },
+            { roots: ['**/tests', 'src'], file: 'src/tests/test_foo.py', envIndex: 0 },
+            { roots: ['**/tests', 'src'], file: 'src/module.py', envIndex: 1 },
+            { roots: ['**/tests', '**/test'], file: 'src/tests/foo.py', envIndex: 0 },
+            { roots: ['**/tests', '**/test'], file: 'src/test/foo.py', envIndex: 1 },
+            { roots: ['**/tests', '**/test'], file: 'src/tests/test/foo.py', envIndex: 0 },
+        ])('roots $roots: "$file" -> env $envIndex', ({ roots, file, envIndex }) => {
+            const { cwd, configOptions } = setupExecEnvConfig(roots.map((root) => ({ root })));
+            const result = configOptions.findExecEnvironment(cwd.resolvePaths(file));
+            if (envIndex >= 0) {
+                assert.strictEqual(result, configOptions.executionEnvironments[envIndex]);
+            } else {
+                for (const env of configOptions.executionEnvironments) {
+                    assert.notStrictEqual(result, env);
+                }
+            }
+        });
+
+        test('glob root with diagnostic overrides', () => {
+            const { configOptions } = setupExecEnvConfig([
+                { root: '**/tests', reportPrivateUsage: false },
+            ]);
+            const env = configOptions.executionEnvironments[0];
+            assert.strictEqual(env.root!.key, configOptions.projectRoot.key);
+            assert.strictEqual(env.diagnosticRuleSet.reportPrivateUsage, 'none');
+        });
+    });
 });