feat: glob pattern support in executionEnvironments root and extraPaths

Author: jackson-cooper-serv

docs/configuration/config-files.md

@@ -30,7 +30,7 @@ The following settings control the *environment* in which basedpyright will chec
 
 - **verboseOutput** [boolean]: Specifies whether output logs should be verbose. This is useful when diagnosing certain problems like import resolution issues.
 
-- **extraPaths** [array of strings, optional]: Additional search paths that will be used when searching for modules imported by files.
+- **extraPaths** [array of strings, optional]: Additional search paths that will be used when searching for modules imported by files. Glob patterns (`**`, `*`, `?`) are supported *(basedpyright exclusive)* — matching directories are expanded at configuration time.
 
 - **pythonVersion** [string, optional]: Specifies the version of Python that will be used to execute the source code. The version should be specified as a string in the format "M.m" where M is the major version and m is the minor (e.g. `"3.0"` or `"3.6"`). If a version is provided, pyright will generate errors if the source code makes use of language features that are not supported in that version. It will also tailor its use of type stub files, which conditionalizes type definitions based on the version. If no version is specified, pyright will use the version of the current python interpreter, if one is present.
 
@@ -314,13 +314,13 @@ The following settings allow more fine grained control over the **typeCheckingMo
 
 
 ## Execution Environment Options
-Pyright allows multiple “execution environments” to be defined for different portions of your source tree. For example, a subtree may be designed to run with different import search paths or a different version of the python interpreter than the rest of the source base.
+Pyright allows multiple "execution environments" to be defined for different portions of your source tree. For example, a subtree may be designed to run with different import search paths or a different version of the python interpreter than the rest of the source base.
 
-The following settings can be specified for each execution environment. Each source file within a project is associated with at most one execution environment -- the first one whose root directory contains that file.
+The following settings can be specified for each execution environment. Each source file within a project is associated with at most one execution environment -- the first one whose root matches that file. Environments are searched in array order; the first match wins.
 
-- **root** [string, required]: Root path for the code that will execute within this execution environment.
+- **root** [string, required]: Root path for the code that will execute within this execution environment. Glob patterns (`**`, `*`, `?`) are supported *(basedpyright exclusive)* — when used, import resolution falls back to the project root.
 
-- **extraPaths** [array of strings, optional]: Additional search paths (in addition to the root path) that will be used when searching for modules imported by files within this execution environment. If specified, this overrides the default extraPaths setting when resolving imports for files within this execution environment. Note that each file’s execution environment mapping is independent, so if file A is in one execution environment and imports a second file B within a second execution environment, any imports from B will use the extraPaths in the second execution environment.
+- **extraPaths** [array of strings, optional]: Additional search paths (in addition to the root path) that will be used when searching for modules imported by files within this execution environment. Glob patterns (`**`, `*`, `?`) are supported *(basedpyright exclusive)* — matching directories are expanded at configuration time. If specified, this overrides the default extraPaths setting when resolving imports for files within this execution environment. Note that each file's execution environment mapping is independent, so if file A is in one execution environment and imports a second file B within a second execution environment, any imports from B will use the extraPaths in the second execution environment.
 
 - **pythonVersion** [string, optional]: The version of Python used for this execution environment. If not specified, the global `pythonVersion` setting is used instead.
 
@@ -377,10 +377,10 @@ The following is an example of a pyright config file:
       ]
     },
     {
-      "root": "src/tests",
+      "root": "**/tests",
       "reportPrivateUsage": false,
       "extraPaths": [
-        "src/tests/e2e",
+        "**/fixtures",
         "src/sdk"
       ]
     },
@@ -413,7 +413,7 @@ pythonPlatform = "Linux"
 executionEnvironments = [
   { root = "src/web", pythonVersion = "3.5", pythonPlatform = "Windows", extraPaths = [ "src/service_libs" ], reportMissingImports = "warning" },
   { root = "src/sdk", pythonVersion = "3.0", extraPaths = [ "src/backend" ] },
-  { root = "src/tests", reportPrivateUsage = false, extraPaths = ["src/tests/e2e", "src/sdk" ]},
+  { root = "**/tests", reportPrivateUsage = false, extraPaths = ["**/fixtures", "src/sdk"] },
   { root = "src" }
 ]
 ```

packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts

@@ -277,7 +277,8 @@ export class BackgroundAnalysisProgram {
     }
 
     private _ensurePartialStubPackages(execEnv: ExecutionEnvironment) {
-        this._backgroundAnalysis?.ensurePartialStubPackages(execEnv.root?.toString());
+        const execEnvIndex = this.configOptions.getExecutionEnvironments().indexOf(execEnv);
+        this._backgroundAnalysis?.ensurePartialStubPackages(execEnvIndex);
         return this._importResolver.ensurePartialStubPackages(execEnv);
     }
 

packages/pyright-internal/src/analyzer/importResolver.ts

@@ -182,7 +182,8 @@ export class ImportResolver {
             return suggestions;
         }
 
-        const root = getParentImportResolutionRoot(sourceFileUri, execEnv.root);
+        const importRoot = execEnv.root;
+        const root = getParentImportResolutionRoot(sourceFileUri, importRoot);
         const origin = sourceFileUri.getDirectory();
 
         let current: Uri | undefined = origin;
@@ -320,9 +321,10 @@ export class ImportResolver {
         detectPyTyped = false
     ) {
         // Cache results of the reverse of resolveImport as we cache resolveImport.
+        const importRoot = execEnv.root;
         const cache = getOrAdd(
             this._cachedModuleNameResults,
-            execEnv.root?.key,
+            importRoot?.key,
             () => new Map<string, ModuleImportInfo>()
         );
         const key = `${allowInvalidModuleName}.${detectPyTyped}.${fileUri.key}`;
@@ -372,8 +374,9 @@ export class ImportResolver {
         }
 
         // The "default" workspace has a root-less execution environment; ignore it.
-        if (execEnv.root) {
-            roots.push(execEnv.root);
+        const importRoot = execEnv.root;
+        if (importRoot) {
+            roots.push(importRoot);
         }
 
         appendArray(roots, execEnv.extraPaths);
@@ -413,7 +416,8 @@ export class ImportResolver {
             return false;
         }
 
-        if (this.partialStubs.isPartialStubPackagesScanned(execEnv)) {
+        const importRoot = execEnv.root;
+        if (importRoot && this.partialStubs.isPathScanned(importRoot)) {
             return false;
         }
 
@@ -423,7 +427,7 @@ export class ImportResolver {
 
         // Add paths to search stub packages.
         addPaths(this._configOptions.stubPath);
-        addPaths(execEnv.root ?? this._configOptions.projectRoot);
+        addPaths(importRoot ?? this._configOptions.projectRoot);
         execEnv.extraPaths.forEach((p) => addPaths(p));
         addPaths(typeshedPathEx);
 
@@ -591,7 +595,8 @@ export class ImportResolver {
         }
 
         // Check whether the given file is in the parent directory import resolution cache.
-        const root = getParentImportResolutionRoot(sourceFileUri, execEnv.root);
+        const importRoot = execEnv.root;
+        const root = getParentImportResolutionRoot(sourceFileUri, importRoot);
         if (!this.cachedParentImportResults.checkValidPath(this.fileSystem, sourceFileUri, root)) {
             return importResult;
         }
@@ -718,7 +723,8 @@ export class ImportResolver {
         // If the import is relative, include the source file path in the key.
         const relativeSourceFileUri = moduleDescriptor && moduleDescriptor.leadingDots > 0 ? sourceFileUri : undefined;
 
-        getOrAdd(this._cachedImportResults, execEnv.root?.key, () => new Map<string, ImportResult>()).set(
+        const importRoot = execEnv.root;
+        getOrAdd(this._cachedImportResults, importRoot?.key, () => new Map<string, ImportResult>()).set(
             this._getImportCacheKey(relativeSourceFileUri, importName, fromUserFile),
             importResult
         );
@@ -1117,11 +1123,12 @@ export class ImportResolver {
             }
 
             // Look for it in the root directory of the execution environment.
-            if (execEnv.root) {
+            const importRoot = execEnv.root;
+            if (importRoot) {
                 this._getCompletionSuggestionsAbsolute(
                     sourceFileUri,
                     execEnv,
-                    execEnv.root,
+                    importRoot,
                     moduleDescriptor,
                     suggestions
                 );
@@ -1221,8 +1228,9 @@ export class ImportResolver {
         }
 
         // Look for it in the root directory of the execution environment.
-        if (execEnv.root) {
-            const candidateModuleNameInfo = _getModuleNameInfoFromPath(execEnv.root, fileUri);
+        const importRoot = execEnv.root;
+        if (importRoot) {
+            const candidateModuleNameInfo = _getModuleNameInfoFromPath(importRoot, fileUri);
 
             if (candidateModuleNameInfo) {
                 if (candidateModuleNameInfo.containsInvalidCharacters) {
@@ -1330,7 +1338,7 @@ export class ImportResolver {
         }
 
         if (detectPyTyped && importType === ImportType.ThirdParty) {
-            const root = getParentImportResolutionRoot(fileUri, execEnv.root);
+            const root = getParentImportResolutionRoot(fileUri, importRoot);
 
             // Go up directories one by one looking for a py.typed file.
             let current: Uri | undefined = fileUri.getDirectory();
@@ -1580,7 +1588,8 @@ export class ImportResolver {
         moduleDescriptor: ImportedModuleDescriptor,
         fromUserFile: boolean
     ) {
-        const cacheForExecEnv = this._cachedImportResults.get(execEnv.root?.key ?? '');
+        const importRoot = execEnv.root;
+        const cacheForExecEnv = this._cachedImportResults.get(importRoot?.key ?? '');
         if (!cacheForExecEnv) {
             return undefined;
         }
@@ -1669,12 +1678,13 @@ export class ImportResolver {
         let localImport: ImportResult | undefined;
 
         // Look for it in the root directory of the execution environment.
-        if (execEnv.root) {
-            importLogger?.log(`Looking in root directory of execution environment ` + `'${execEnv.root}'`);
+        const importRoot = execEnv.root;
+        if (importRoot) {
+            importLogger?.log(`Looking in root directory of execution environment ` + `'${importRoot}'`);
 
             localImport = this.resolveAbsoluteImport(
                 sourceFileUri,
-                execEnv.root,
+                importRoot,
                 execEnv,
                 moduleDescriptor,
                 importName,
@@ -1756,7 +1766,7 @@ export class ImportResolver {
         // If a library is fully py.typed, then we have found the best match,
         // unless the execution environment is typeshed itself, in which case
         // we don't want to favor py.typed libraries. Use the typeshed lookup below.
-        if (execEnv.root !== this._getTypeshedRoot(this._configOptions.typeshedPath, importLogger)) {
+        if (importRoot !== this._getTypeshedRoot(this._configOptions.typeshedPath, importLogger)) {
             if (bestResultSoFar?.pyTypedInfo && !bestResultSoFar.isPartlyResolved) {
                 return bestResultSoFar;
             }

packages/pyright-internal/src/backgroundAnalysisBase.ts

@@ -55,7 +55,7 @@ export interface IBackgroundAnalysis extends Disposable {
     setConfigOptions(configOptions: ConfigOptions): void;
     setTrackedFiles(fileUris: Uri[]): void;
     setAllowedThirdPartyImports(importNames: string[]): void;
-    ensurePartialStubPackages(executionRoot: string | undefined): void;
+    ensurePartialStubPackages(execEnvIndex: number): void;
     setFileOpened(fileUri: Uri, version: number | null, contents: string, options: OpenFileOptions): void;
     updateChainedUri(fileUri: Uri, chainedUri: Uri | undefined): void;
     setFileClosed(fileUri: Uri, isTracked?: boolean): void;
@@ -158,8 +158,8 @@ export class BackgroundAnalysisBase implements IBackgroundAnalysis {
         this.enqueueRequest({ requestType: 'setAllowedThirdPartyImports', data: serialize(importNames) });
     }
 
-    ensurePartialStubPackages(executionRoot: string | undefined) {
-        this.enqueueRequest({ requestType: 'ensurePartialStubPackages', data: serialize({ executionRoot }) });
+    ensurePartialStubPackages(execEnvIndex: number) {
+        this.enqueueRequest({ requestType: 'ensurePartialStubPackages', data: serialize({ execEnvIndex }) });
     }
 
     setFileOpened(fileUri: Uri, version: number | null, contents: string, options: OpenFileOptions) {
@@ -589,8 +589,8 @@ export abstract class BackgroundAnalysisRunnerBase extends BackgroundThreadBase
             }
 
             case 'ensurePartialStubPackages': {
-                const { executionRoot } = deserialize(msg.data);
-                this.handleEnsurePartialStubPackages(executionRoot);
+                const { execEnvIndex } = deserialize(msg.data);
+                this.handleEnsurePartialStubPackages(execEnvIndex);
                 break;
             }
 
@@ -775,10 +775,8 @@ export abstract class BackgroundAnalysisRunnerBase extends BackgroundThreadBase
         this.program.setAllowedThirdPartyImports(importNames);
     }
 
-    protected handleEnsurePartialStubPackages(executionRoot: string | undefined) {
-        const execEnv = this._configOptions
-            .getExecutionEnvironments()
-            .find((e) => e.root?.toString() === executionRoot);
+    protected handleEnsurePartialStubPackages(execEnvIndex: number) {
+        const execEnv = this._configOptions.getExecutionEnvironments()[execEnvIndex];
         if (execEnv) {
             this.importResolver.ensurePartialStubPackages(execEnv);
         }

packages/pyright-internal/src/common/configOptions.ts

@@ -7,6 +7,9 @@
  * Class that holds the configuration options for the analyzer.
  */
 
+import { globSync } from 'node:fs';
+import { matchesGlob } from 'node:path/posix';
+
 import { ImportLogger } from '../analyzer/importLogger';
 import { getPathsFromPthFiles } from '../analyzer/pythonPathUtils';
 import * as pathConsts from '../common/pathConsts';
@@ -27,7 +30,7 @@ import { PythonVersion, latestStablePythonVersion } from './pythonVersion';
 import { ServiceKeys } from './serviceKeys';
 import { ServiceProvider } from './serviceProvider';
 import { Uri } from './uri/uri';
-import { FileSpec, getFileSpec, isDirectory } from './uri/uriUtils';
+import { FileSpec, getFileSpec, getWildcardRoot, isDirectory } from './uri/uriUtils';
 import { userFacingOptionsList } from './stringUtils';
 
 // prevent upstream changes from sneaking in and adding errors using console.error,
@@ -66,6 +69,8 @@ export class ExecutionEnvironment {
     // tools or playgrounds).
     skipNativeLibraries: boolean;
 
+    rootGlob?: string;
+
     // Default to "." which indicates every file in the project.
     constructor(
         name: string,
@@ -1526,13 +1531,21 @@ export class ConfigOptions {
     // execution environment is used.
     findExecEnvironment(file: Uri): ExecutionEnvironment {
         return (
-            this.executionEnvironments.find((env) => {
-                const envRoot = Uri.is(env.root) ? env.root : this.projectRoot.resolvePaths(env.root || '');
-                return file.startsWith(envRoot);
-            }) ?? this.getDefaultExecEnvironment()
+            this.executionEnvironments.find((env) => this._fileMatchesEnvironment(file, env)) ??
+            this.getDefaultExecEnvironment()
         );
     }
 
+    private _fileMatchesEnvironment(file: Uri, env: ExecutionEnvironment): boolean {
+        if (env.rootGlob !== undefined) {
+            const relative = this.projectRoot.getRelativePath(file)?.slice(2);
+            if (relative === undefined) return false;
+            return matchesGlob(relative, env.rootGlob + '/**');
+        }
+        const envRoot = Uri.is(env.root) ? env.root : this.projectRoot.resolvePaths(env.root || '');
+        return file.startsWith(envRoot);
+    }
+
     getExecutionEnvironments(): ExecutionEnvironment[] {
         if (this.executionEnvironments.length > 0) {
             return this.executionEnvironments;
@@ -1691,7 +1704,7 @@ export class ConfigOptions {
                     if (typeof path !== 'string') {
                         console.error(`Config "extraPaths" field ${pathIndex} must be a string.`);
                     } else {
-                        configExtraPaths!.push(configDirUri.resolvePaths(path));
+                        this._resolveExtraPath(path, configDirUri, configExtraPaths, console);
                     }
                 });
                 this.defaultExtraPaths = [...configExtraPaths];
@@ -1989,6 +2002,20 @@ export class ConfigOptions {
         return this.pythonEnvironmentName || this.pythonPath?.toString() || 'python';
     }
 
+    private _resolveExtraPath(path: string, configDirUri: Uri, out: Uri[], console: ConsoleInterface) {
+        if (/[*?]/.test(path)) {
+            try {
+                for (const match of globSync(path, { cwd: configDirUri.getFilePath() })) {
+                    out.push(configDirUri.resolvePaths(match));
+                }
+            } catch (e) {
+                console.error(`Failed to expand glob pattern "${path}": ${e}`);
+            }
+        } else {
+            out.push(configDirUri.resolvePaths(path));
+        }
+    }
+
     private _convertBoolean(value: any, fieldName: string, defaultValue: boolean): boolean {
         if (value === undefined) {
             return defaultValue;
@@ -2043,7 +2070,8 @@ export class ConfigOptions {
 
             // Validate the root.
             if (envObj.root && typeof envObj.root === 'string') {
-                newExecEnv.root = configDirUri.resolvePaths(envObj.root);
+                newExecEnv.rootGlob = envObj.root;
+                newExecEnv.root = getWildcardRoot(configDirUri, envObj.root);
             } else {
                 console.error(`Config executionEnvironments index ${index}: missing root value.`);
             }
@@ -2067,7 +2095,7 @@ export class ConfigOptions {
                                     ` extraPaths field ${pathIndex} must be a string.`
                             );
                         } else {
-                            newExecEnv.extraPaths.push(configDirUri.resolvePaths(path));
+                            this._resolveExtraPath(path, configDirUri, newExecEnv.extraPaths, console);
                         }
                     });
                 }