feat: support custom queryRoot(defaults to std, i.e. projectroot/qlsrc)

Author: Lslightly

ci/malloc_test.yaml

@@ -25,7 +25,6 @@ buildGrps:
 queryconfig:
   resultRoot: codeqlResult # resultRoot only for -collect csv. Bqrs/csv per db: <dbPath>/results/lslightly/qlstat/<queryPathNoExt>.<fmt>
   parallelCore: 20 # parallel cores to run query
-  queryRoot: qlsrc # query root. There should be codeql-pack.yaml in queryRoot directory
   queryGrps:
     - queryDBs:
         - batchmalloc

cmd/batch_clone_build/clone_test.yaml

@@ -9,4 +9,3 @@ repositories:
 
 queryconfig:
   resultRoot: codeqlResult
-  queryRoot: qlsrc

cmd/codeql_qdriver/main.go

@@ -58,7 +58,7 @@ func main() {
 		}
 		for grpi, grp := range cfg.QueryGrps {
 			fmt.Printf("Grp %d: Executing queries\n", grpi)
-			queriesExec(cfg, grp, targetDecodeFmt)
+			queriesExec(cfg, grp, grp.ResolvedQueryRoot(), targetDecodeFmt)
 		}
 	} else {
 		fmt.Println("Decoding results")
@@ -103,13 +103,13 @@ func buildGroupExternalArgs(queries []config.Query, extroot string) []string {
 //
 // Results are placed at <dbPath>/results/<queryPack>/<queryPathNoExt>.bqrs.
 // Decoded output goes to <dbPath>/results/<queryPack>/<queryPathNoExt>.<tgtFmt>.
-func runQueriesOnDB(cfg *config.Artifact, db config.DB, queries []config.Query, tgtFmt string) {
+func runQueriesOnDB(cfg *config.Artifact, db config.DB, queryRoot string, queries []config.Query, tgtFmt string) {
 	// --- Phase 1: run-queries ---
 	extArgs := buildGroupExternalArgs(queries, db.ExtDir())
 
 	queryPaths := make([]string, len(queries))
 	for i, q := range queries {
-		queryPaths[i] = q.QueryPath(cfg.QueryRoot)
+		queryPaths[i] = q.QueryPath(queryRoot)
 	}
 
 	logDir := filepath.Join(cfg.PassLogDir("query"), db.Name)
@@ -164,13 +164,13 @@ first remove all content in ${config.OutResultRoot}/${qScriptNameNoExt}
 dump error log for ${db} in ${config.OutResultRoot}/${qScriptNameNoExt}/error.log
 dump stdout/stderr for ${db} in ${config.OutResultRoot}/${qScriptNameNoExt}/log/${db}.log
 */
-func queriesExec(cfg *config.Artifact, grp config.QueryGroup, tgtFmt string) {
+func queriesExec(cfg *config.Artifact, grp config.QueryGroup, queryRoot string, tgtFmt string) {
 	dbs := cfg.ConvStrSliceToDBSlice(grp.QueryDBs)
 
 	// Create Query objects for all queries in the group
 	queries := make([]config.Query, len(grp.Queries))
 	for i, qScript := range grp.Queries {
-		queries[i] = config.CreateQuery(qScript, grp.Externals, grp.ExternalFiles)
+		queries[i] = config.CreateQuery(qScript, grp.Externals, grp.ExternalFiles, queryRoot)
 	}
 
 	bar := progressbar.Default(int64(len(dbs)))
@@ -180,7 +180,7 @@ func queriesExec(cfg *config.Artifact, grp config.QueryGroup, tgtFmt string) {
 		wg.Add(1)
 		go func(db config.DB) {
 			defer wg.Done()
-			runQueriesOnDB(cfg, db, queries, tgtFmt)
+			runQueriesOnDB(cfg, db, queryRoot, queries, tgtFmt)
 			bar.Describe(fmt.Sprintf("db: %-20s %d queries", db.Name, len(queries)))
 			bar.Add(1)
 		}(db)
@@ -207,12 +207,13 @@ func decodeResults(cfg *config.Artifact, tgtFmt string) {
 	checkDecodeTargetFmt(tgtFmt)
 
 	for grpi, grp := range cfg.QueryGrps {
+		queryRoot := grp.ResolvedQueryRoot()
 		dbs := cfg.ConvStrSliceToDBSlice(grp.QueryDBs)
 		queries := grp.Queries
 
 		bar := progressbar.Default(int64(len(queries) * len(dbs)))
 		for _, qScript := range queries {
-			query := config.CreateQuery(qScript, grp.Externals, grp.ExternalFiles)
+			query := config.CreateQuery(qScript, grp.Externals, grp.ExternalFiles, queryRoot)
 			for _, db := range dbs {
 				bqrsPath := filepath.Join(db.Path(), "results/lslightly/qlstat", query.PathNoExt()+".bqrs")
 				csvPath := filepath.Join(db.Path(), "results/lslightly/qlstat", query.PathNoExt()+"."+tgtFmt)
@@ -244,9 +245,10 @@ func decodeResults(cfg *config.Artifact, tgtFmt string) {
 
 func collectCSVs(cfg *config.Artifact) {
 	for grpi, grp := range cfg.QueryGrps {
+		queryRoot := grp.ResolvedQueryRoot()
 		bar := progressbar.Default(int64(len(grp.Queries)))
 		for _, qScript := range grp.Queries {
-			query := config.CreateQuery(qScript, grp.Externals, grp.ExternalFiles)
+			query := config.CreateQuery(qScript, grp.Externals, grp.ExternalFiles, queryRoot)
 			bar.Describe(fmt.Sprintf("Grp %d: Collecting for "+query.PathNoExt(), grpi))
 			collectCSVsForQuery(cfg, query, grp)
 			bar.Add(1)

cmd/escape_adapter/movedtoheap_test.yaml

@@ -20,7 +20,6 @@ buildGrps:
 # query
 queryconfig:
   resultRoot: codeqlResult # resultRoot only for -collect csv. Bqrs/csv per db: <dbPath>/results/lslightly/qlstat/<queryPathNoExt>.<fmt>
-  queryRoot: qlsrc # query root. There should be codeql-pack.yaml in queryRoot directory
   parallelCore: 20 # parallel cores to run query
   queryGrps:
     - queryDBs: # "*" means all repositories. "-" means repositories defined in the "repositories" section. Otherwise, use fullnames. Note that repositories with same fullname in different source will be queried.

config/artifact.go

@@ -24,7 +24,6 @@ type Artifact struct {
 
 type QueryConfig struct {
 	ResultRoot   string       `yaml:"resultRoot"`
-	QueryRoot    string       `yaml:"queryRoot"`
 	ParallelCore int          `yaml:"parallelCore"`
 	QueryGrps    []QueryGroup `yaml:"queryGrps"`
 }
@@ -37,12 +36,23 @@ type BuildGroup struct {
 }
 
 type QueryGroup struct {
+	QueryRoot     string   `yaml:"queryRoot"`
 	QueryDBs      []string `yaml:"queryDBs"`
 	Queries       []string `yaml:"queries"`
 	Externals     []string `yaml:"externals"`
 	ExternalFiles []string `yaml:"externalFiles"`
 }
 
+// ResolvedQueryRoot returns the absolute path for this group's query root.
+// "std" or empty → <projectRoot>/qlsrc (built-in queries).
+// Other values are returned as-is (absolute path or CWD-relative).
+func (g *QueryGroup) ResolvedQueryRoot() string {
+	if g.QueryRoot == "" || g.QueryRoot == "std" {
+		return filepath.Join(utils.ProjectRoot(), "qlsrc")
+	}
+	return g.QueryRoot
+}
+
 func UnmarshalArtifact(filename string) *Artifact {
 	data, err := os.ReadFile(filename)
 	if err != nil {
@@ -68,10 +78,6 @@ func UnmarshalArtifact(filename string) *Artifact {
 		cfg.ResultRoot = filepath.Join(utils.ProjectRoot(), "codeqlResult")
 		log.Println("\033[33mWARNING: resultRoot is empty, set to default value:", cfg.ResultRoot, "\033[0m")
 	}
-	if cfg.QueryRoot == "" {
-		cfg.QueryRoot = filepath.Join(utils.ProjectRoot(), "qlsrc")
-		log.Println("\033[33mWARNING: queryRoot is empty, set to default value:", cfg.QueryRoot, "\033[0m")
-	}
 	return cfg
 }
 

config/query.go

@@ -14,13 +14,17 @@ type Query struct {
 	cacheEntriesForFiles []string // cache external entries for externalFiles
 }
 
-func CreateQuery(path string, externals []string, externalFiles []string) Query {
+func CreateQuery(path string, externals []string, externalFiles []string, queryRoot string) Query {
 	if filepath.Ext(path) != ".ql" {
 		log.Fatalf("Suffix of query source %s is not .ql.", path)
 	}
-	cache := make([]string, len(externalFiles)*5)
+	cache := make([]string, 0, len(externalFiles)*5)
 	for _, extfile := range externalFiles {
-		exts, err := ReadExtsFromFile(extfile)
+		resolvedPath := extfile
+		if !filepath.IsAbs(extfile) {
+			resolvedPath = filepath.Join(queryRoot, extfile)
+		}
+		exts, err := ReadExtsFromFile(resolvedPath)
 		if err != nil {
 			log.Fatalf("Failed to read external file %s: %v", extfile, err)
 		}

demo.yaml

@@ -62,7 +62,6 @@ buildGrps:
 #   "-" — all databases defined in buildGrps
 queryconfig:
   resultRoot: codeqlResult
-  queryRoot: qlsrc
   parallelCore: 20
   queryGrps:
     - queryDBs:

doc/adapters/escape_analysis.md

@@ -38,6 +38,7 @@ queryconfig:
 ```
 
 In this configuration:
+- `queryRoot` is optional per query group. If omitted, it defaults to `std`, which resolves to the built-in query library at `<projectRoot>/qlsrc`. Custom paths can be specified instead to use queries outside the built-in library.
 - `extgenScript: goescape` instructs the system to compile repositories with escape analysis enabled
   - Note that if you use custom script, 4 environment variables will be set. You can use them in your custom script.
     - `REPO_DIR`: The directory path of the repository

example.yaml

@@ -49,7 +49,6 @@ buildGrps:
 # query
 queryconfig:
   resultRoot: codeqlResult # resultRoot only for -collect csv. Bqrs/csv per db: <dbPath>/results/lslightly/qlstat/<queryPathNoExt>.<fmt>
-  queryRoot: qlsrc # query root. There should be codeql-pack.yaml in queryRoot directory
   parallelCore: 20 # parallel cores to run query
   queryGrps:
     - queryDBs: # "*" means all repositories. "-" means repositories defined in the "repositories" section. Otherwise, use fullnames. Note that repositories with same fullname in different source will be queried.

qlsrc/pprof_ext/heapprofile_unittests/heapprofile_api_tests.yaml

@@ -34,7 +34,6 @@ buildGrps:
 queryconfig:
   resultRoot: codeqlResult # resultRoot only for -collect csv. Bqrs/csv per db: <dbPath>/results/lslightly/qlstat/<queryPathNoExt>.<fmt>
   parallelCore: 20 # parallel cores to run query
-  queryRoot: qlsrc # query root. There should be codeql-pack.yaml in queryRoot directory
   queryGrps:
     - queryDBs:
         - malloc_test