opt:(go) add option LoadByPackages to avoid OOM and skip exludes

lang/collect/collect.go

@@ -36,6 +36,7 @@ type CollectOption struct {
 	NoNeedComment      bool
 	NeedTest           bool
 	Excludes           []string
+	LoadByPackages     bool
 }
 
 type Collector struct {

lang/golang/parser/option.go

@@ -25,6 +25,7 @@ type Options struct {
 	Excludes       []string
 	CollectComment bool
 	NeedTest       bool
+	LoadByPackages bool
 }
 
 // type Option func(options *Options)

lang/golang/parser/parser.go

@@ -164,7 +164,29 @@ func (p *GoParser) ParseModule(mod *Module, dir string) (err error) {
 		return nil
 	})
 
-	return p.loadPackages(mod, dir, "./...")
+	if p.opts.LoadByPackages {
+		var errs []error
+		filepath.Walk(dir, func(path string, info fs.FileInfo, e error) error {
+			if e != nil || !info.IsDir() || shouldIgnoreDir(path) {
+				return nil
+			}
+			for _, exclude := range p.exclues {
+				if exclude.MatchString(path) {
+					return nil
+				}
+			}
+			if err := p.parsePackage(p.pkgPathFromABS(path)); err != nil {
+				errs = append(errs, err)
+			}
+			return nil
+		})
+		if len(errs) > 0 {
+			return fmt.Errorf("parse package failed: %v", errs)
+		}
+		return nil
+	} else {
+		return p.loadPackages(mod, dir, "./...")
+	}
 }
 
 // getRepo return currently parsed golang AST
@@ -249,7 +271,7 @@ func (p *GoParser) searchName(name string) (ids []Identity, err error) {
 		if e != nil || info.IsDir() || shouldIgnoreFile(path) || shouldIgnoreDir(filepath.Dir(path)) || !strings.HasSuffix(path, ".go") {
 			return nil
 		}
-		mod, _ := p.getModuleFromPath(filepath.Dir(path))
+		mod := p.pkgPathFromABS(path)
 		m := p.repo.Modules[mod]
 		if m == nil {
 			dir, _ := filepath.Rel(p.homePageDir, path)
@@ -428,24 +450,29 @@ func (p *GoParser) getModuleFromPkg(pkg PkgPath) (name string, dir string) {
 }
 
 // path is absolute path
-func (p *GoParser) getModuleFromPath(path string) (name string, dir string) {
+func (p *GoParser) getModuleFromPath(path string) (name string, dir string, rel string) {
 	for _, m := range p.modules {
-		if len(m.dir) != 0 && strings.HasPrefix(path, m.dir) {
-			return m.name, m.dir
+		if m.dir == "" {
+			continue
+		}
+		dir := filepath.Join(p.homePageDir, m.dir)
+		if strings.HasPrefix(path, dir) {
+			rel, _ = filepath.Rel(dir, path)
+			return m.name, m.dir, rel
 		}
 	}
-	return "", ""
+	return "", "", ""
 }
 
 // FromABS converts an absolute path to local mod path
 func (p *GoParser) pkgPathFromABS(path string) PkgPath {
-	mod, dir := p.getModuleFromPath(path)
+	mod, _, rel := p.getModuleFromPath(path)
 	if mod == "" {
 		panic("not found package from " + path)
 	}
-	if rel, err := filepath.Rel(dir, path); err != nil {
-		panic("path " + path + " is not relative from mod path " + dir)
+	if rel != "" && rel != "." {
+		return mod + "/" + rel
 	} else {
-		return filepath.Join(mod, rel)
+		return mod
 	}
 }

lang/golang/parser/pkg.go

@@ -142,7 +142,7 @@ func (p *GoParser) parsePackage(pkgPath PkgPath) (err error) {
 	}
 	// fmt.Println("[parsePackage] mod:", mod, "dir:", dir, "pkgPath:", pkgPath, p.opts.ReferCodeDepth)
 
-	return p.loadPackages(lib, dir, pkgPath)
+	return p.loadPackages(lib, filepath.Join(p.homePageDir, lib.Dir), pkgPath)
 }
 
 var loadCount = 0

lang/parse.go

@@ -185,6 +185,9 @@ func callGoParser(ctx context.Context, repoPath string, opts collect.CollectOpti
 	if opts.NeedTest {
 		goopts.NeedTest = true
 	}
+	if opts.LoadByPackages {
+		goopts.LoadByPackages = true
+	}
 	goopts.Excludes = opts.Excludes
 	p := parser.NewParser(repoPath, repoPath, goopts)
 	repo, err := p.ParseRepo()

main.go

@@ -67,6 +67,7 @@ func main() {
 	flags.BoolVar(&opts.LoadExternalSymbol, "load-external-symbol", false, "load external symbols into results")
 	flags.BoolVar(&opts.NoNeedComment, "no-need-comment", false, "do not need comment (only works for Go now)")
 	flags.BoolVar(&opts.NeedTest, "need-test", false, "need parse test files (only works for Go now)")
+	flags.BoolVar(&opts.LoadByPackages, "load-by-packages", false, "load by packages (only works for Go now)")
 	flags.Var((*StringArray)(&opts.Excludes), "exclude", "exclude files or directories, support multiple values")
 	flags.StringVar(&opts.RepoID, "repo-id", "", "specify the repo id")
 	flagLsp := flags.String("lsp", "", "Specify the language server path.")