Merge branch 'worktree-redesign-repo-config'

Author: Lslightly

README.md

@@ -31,7 +31,7 @@ QLStat provides a comprehensive framework for large-scale empirical analysis of
 
 Create your `stat.yaml` config file according to [`example.yaml`](./example.yaml), [`demo.yaml`](./demo.yaml) or YAML files in [`yaml-examples/`](./yaml-examples/). The configuration supports several key sections:
 
-- `sources`: Define repository sources with prefixes and specific repositories
+- `repositories`: Define repositories with URL prefixes and optional directory layout
 - `language`: Specify the programming language for analysis (e.g., go)
 - `buildGrps`: Configure build groups with timeout, build commands, and optional extgen scripts
 - `queryconfig`: Set up query execution with parallelization options

ci/malloc_test.yaml

@@ -3,17 +3,17 @@ repoRoot: repos # repoRoot stores all repositories
 dbRoot: codeql-db # dbRoot stores all databases created by codeql indexed by repository name
 
 # clone
-sources:
-  - prefix: https://test/ # clone url prefix
-    fullnames:
+repositories:
+  - dir: test/
+    repos:
       - malloc_test
       - batchmalloc
 
 # build
 language: go # language to analyze
 buildTimeout: 3600 # timeout for building repository
 buildGrps:
-- buildRepos: # "*" means all repositories. "-" means repositories defined in "sources" attribute. You can also specify fullname of repositories to force re-build. Note that repositories with same fullname in different source will be re-built also.
+- buildRepos: # "*" means all repositories. "-" means repositories defined in the "repositories" section. You can also specify fullname of repositories to force re-build. Note that repositories with same fullname in different source will be re-built also.
     - malloc_test
   buildCmd: ci/build/malloc_test.sh
   extgenScript: ci/extgens/malloc_test.sh

cmd/batch_clone_build/build.go

@@ -127,9 +127,8 @@ func buildDirSetup(cfg *config.Artifact) (*os.File, *os.File) {
 	logFile := utils.CreateFile(logFilePath)
 	defer logFile.Close()
 
-	for _, gs := range cfg.Sources {
-		hostdir := gs.HostDir(cfg.RepoRoot)
-		utils.MkdirAll(filepath.Join(hostdir))
+	for _, rg := range cfg.Repositories {
+		rg.CreateRepoRootDir(cfg.RepoRoot)
 	}
 	return csvFile, logFile
 }

cmd/batch_clone_build/clone.go

@@ -40,9 +40,9 @@ func batchClone(cfg *config.Artifact) {
 	statusChan := make(chan cloneStatus)
 	repoCnt := 0
 	var wg sync.WaitGroup
-	for _, gs := range cfg.Sources {
-		gs.CreateRepoRootDir(cfg.RepoRoot)
-		for _, repo := range gs.GetRepos() {
+	for _, rg := range cfg.Repositories {
+		rg.CreateRepoRootDir(cfg.RepoRoot)
+		for _, repo := range rg.GetRepos() {
 			wg.Add(1)
 			repoCnt++
 			go func() {

cmd/batch_clone_build/clone_test.go

@@ -23,6 +23,6 @@ func TestCloneBranch(t *testing.T) {
 		assert.Nil(t, err, err.Error()+"\nstdout:"+outbuf.String()+"\nstderr:"+errbuf.String())
 	}
 	outbuf.Reset()
-	assert.Nil(t, utils.RuncmdWithBuf(filepath.Join(utils.ProjectRoot(), "repos/github.com/zhihu-parallel-pi-calc"), &outbuf, nil, "git", "branch", "--show-current"))
+	assert.Nil(t, utils.RuncmdWithBuf(filepath.Join(utils.ProjectRoot(), "repos/zhihu-parallel-pi-calc"), &outbuf, nil, "git", "branch", "--show-current"))
 	assert.Equal(t, "QLStat-test", strings.TrimSpace(outbuf.String()))
 }

cmd/batch_clone_build/clone_test.yaml

@@ -1,9 +1,10 @@
 logRoot: logs
 repoRoot: repos
 dbRoot: codeql-db
-sources:
-  - prefix: https://github.com/ # clone url prefix
-    fullnames:
+repositories:
+  - urlPrefix: https://github.com/
+    dir: github.com
+    repos:
       - Lslightly-courses/zhihu-parallel-pi-calc QLStat-test
 
 queryconfig:

cmd/escape_adapter/movedtoheap_test.yaml

@@ -3,17 +3,17 @@ repoRoot: repos # repoRoot stores all repositories
 dbRoot: codeql-db # dbRoot stores all databases created by codeql indexed by repository name
 
 # clone
-sources:
-  - prefix: https://test/ # clone url prefix
-    fullnames:
+repositories:
+  - dir: test/
+    repos:
       - escape
       - false-sharing
 
 # build
 language: go # language to analyze
 buildTimeout: 3600 # timeout for building repository
 buildGrps:
-  - buildRepos: # "*" means all repositories. "-" means repositories defined in "sources" attribute. You can also specify fullname of repositories to force re-build. Note that repositories with same fullname in different source will be re-built also.
+  - buildRepos: # "*" means all repositories. "-" means repositories defined in the "repositories" section. You can also specify fullname of repositories to force re-build. Note that repositories with same fullname in different source will be re-built also.
     - "-"
     extgenScript: goescape
 
@@ -23,7 +23,7 @@ queryconfig:
   queryRoot: qlsrc # query root. There should be codeql-pack.yaml in queryRoot directory
   parallelCore: 20 # parallel cores to run query
   queryGrps:
-    - queryDBs: # "*" means all repositories. "-" means repositories defined in "sources" attribute. Otherwise, use fullnames. Note that repositories with same fullname in different source will be queried.
+    - queryDBs: # "*" means all repositories. "-" means repositories defined in the "repositories" section. Otherwise, use fullnames. Note that repositories with same fullname in different source will be queried.
         - escape
       queries: # queries
         - escape_ext/moved_to_heap_var_test.ql

config/artifact.go

@@ -14,7 +14,7 @@ import (
 type Artifact struct {
 	RepoRoot     string       `yaml:"repoRoot"`
 	LogRoot      string       `yaml:"logRoot"`
-	Sources      []*GitSource `yaml:"sources"`
+	Repositories []*RepoGroup `yaml:"repositories"`
 	DBRoot       string       `yaml:"dbRoot"`
 	Lang         string       `yaml:"language"`
 	BuildTimeout int          `yaml:"buildTimeout"`
@@ -156,7 +156,7 @@ type reposType int
 
 const (
 	buildSpecific reposType = iota
-	buildWrittenInSources
+	buildWrittenInRepositories
 	buildAll
 )
 
@@ -165,7 +165,7 @@ func getReposType(repos []string) reposType {
 		repo0 := repos[0]
 		switch repo0 {
 		case "-":
-			return buildWrittenInSources
+			return buildWrittenInRepositories
 		case "*":
 			return buildAll
 		}
@@ -175,23 +175,23 @@ func getReposType(repos []string) reposType {
 
 func (art *Artifact) ConvStrSliceToRepoSlice(repos []string) (res []Repo) {
 	switch getReposType(repos) {
-	case buildWrittenInSources:
-		for _, gs := range art.Sources {
-			res = append(res, gs.GetRepos()...)
+	case buildWrittenInRepositories:
+		for _, rg := range art.Repositories {
+			res = append(res, rg.GetRepos()...)
 		}
 		return
 	case buildAll:
-		for _, gs := range art.Sources {
-			res = append(res, gs.reposInDir(art.RepoRoot)...)
+		for _, rg := range art.Repositories {
+			res = append(res, rg.reposInDir(art.RepoRoot)...)
 		}
 		return
 	default:
-		for _, gs := range art.Sources {
-			gs.calcFullName2RepoCache()
+		for _, rg := range art.Repositories {
+			rg.calcRepoCache()
 		}
 		for _, fullname := range repos {
-			for _, gs := range art.Sources {
-				if repo, ok := gs.fullName2RepoCache[fullname]; ok {
+			for _, rg := range art.Repositories {
+				if repo, ok := rg.repoCache[fullname]; ok {
 					res = append(res, repo)
 				}
 			}

config/gitsource.go

@@ -14,24 +14,39 @@ type Repo struct {
 	FullName    string
 	DirBaseName string
 	branch      string
-	*GitSource
+	*RepoGroup
 }
 
+// HasURLPrefix returns true if the repo belongs to a group with a URL prefix (remote repo).
+func (r *Repo) HasURLPrefix() bool {
+	return r.RepoGroup != nil && r.RepoGroup.URLPrefix != ""
+}
+
+// RemoteURL returns the full clone URL by joining URLPrefix and FullName.
+// Must only be called when HasURLPrefix() is true.
 func (r *Repo) RemoteURL() string {
-	url, err := url.JoinPath(r.GitSource.Prefix, r.FullName)
+	if !r.HasURLPrefix() {
+		log.Fatalf("RemoteURL called on repo %s which has no URLPrefix (local-only repo)", r.FullName)
+	}
+	u, err := url.JoinPath(r.RepoGroup.URLPrefix, r.FullName)
 	if err != nil {
 		log.Fatalf("Fail to know target url: %v", err)
 	}
-	url += ".git"
-	return url
+	u += ".git"
+	return u
 }
 
-// DirPath returns root/<host>/<repo>
+// DirPath returns root/<Dir>/<dirBaseName> if Dir is set, otherwise root/<dirBaseName>
 func (r *Repo) DirPath(root string) string {
-	return filepath.Join(r.GitSource.HostDir(root), r.DirBaseName)
+	return filepath.Join(r.RepoGroup.GroupDir(root), r.DirBaseName)
 }
 
+// Clone clones the repo from remote. Skips silently if URLPrefix is empty (local-only repo).
 func (r *Repo) Clone(root string) error {
+	if !r.HasURLPrefix() {
+		log.Printf("Skipping clone for %s: no URLPrefix (local-only repo)", r.FullName)
+		return nil
+	}
 	args := []string{"clone"}
 	args = append(args, r.RemoteURL(), r.DirPath(root))
 	var errBuf bytes.Buffer