feat:(uniast) add Function.Signature (#48)

* feat: support `Repository.GetFileNodes()`

* feat:(go) support get function signature

* opt: only collect codes files

* opt: Get File only return codes file

* feat(lsp): support collect func signatures

* update ast version

* still keep files

Author: AsterDY

docs/uniast-zh.md

@@ -1,4 +1,4 @@
-# Universal Abstract-Syntax-Tree Specification (v0.1.1)
+# Universal Abstract-Syntax-Tree Specification (v0.1.2)
 
 Universal Abstract-Syntax-Tree 是 ABCoder 建立的一种 LLM 亲和、语言无关的代码上下文数据结构，表示某个仓库代码的统一抽象语法树。收集了语言实体（函数、类型、常（变）量）的定义及其相互依赖关系，用于后续的 AI 理解、coding-workflow 开发。
 
@@ -237,6 +237,7 @@ Universal Abstract-Syntax-Tree 是 ABCoder 建立的一种 LLM 亲和、语言
     "StartOffset": 3290,
     "EndOffset": 3573,
     "Content": "// BindSession binds the session with current goroutine\nfunc (self *SessionManager) BindSession(Identity SessionIdentity, s Session) {\n\tshard : = self.shards[uint64(Identity)%uint64(self.opts.ShardNumber)]\n\n\tshard.Store(Identity, s)\n\n\tif self.opts.EnableImplicitlyTransmitAsync {\n\t\ttransmitSessionIdentity(Identity)\n\t}\n}",
+    "Signature": "func (self *SessionManager) BindSession(Identity SessionIdentity, s Session)",
     "Receiver": {
         "IsPointer": true,
         "Type": {
@@ -320,6 +321,7 @@ Universal Abstract-Syntax-Tree 是 ABCoder 建立的一种 LLM 亲和、语言
 
 - IsMethod: 是否是一个方法
 
+- Signature: 函数签名，包括函数名、参数、返回值等
 
 - IsInterfaceMethod: 是否是接口的方法--这里 abcoder parse 收集 InterfaceMethod 为了方便 LLM 理解，但是实际上 write 中并不会认为其是一个语言实体
 

lang/collect/collect.go

@@ -77,6 +77,7 @@ type functionInfo struct {
 	InputsSorted     []dependency       `json:"-"`
 	Outputs          map[int]dependency `json:"outputs,omitempty"`
 	OutputsSorted    []dependency       `json:"-"`
+	Signature        string             `json:"signature,omitempty"`
 }
 
 func switchSpec(l uniast.Language) LanguageSpec {
@@ -596,11 +597,29 @@ func (c *Collector) processSymbol(ctx context.Context, sym *DocumentSymbol, dept
 				log.Error("get receiver symbol for token %v failed: %v\n", rec, err)
 			}
 		}
-
 		tsyms, ts := c.getDepsWithLimit(ctx, sym, tps, depth-1)
 		ipsyms, is := c.getDepsWithLimit(ctx, sym, ips, depth-1)
 		opsyms, os := c.getDepsWithLimit(ctx, sym, ops, depth-1)
-		c.updateFunctionInfo(sym, tsyms, ipsyms, opsyms, ts, is, os, rsym)
+
+		//get last token of params for get signature
+		lastToken := rec
+		for _, t := range tps {
+			if t > lastToken {
+				lastToken = t
+			}
+		}
+		for _, t := range ips {
+			if t > lastToken {
+				lastToken = t
+			}
+		}
+		for _, t := range ops {
+			if t > lastToken {
+				lastToken = t
+			}
+		}
+
+		c.updateFunctionInfo(sym, tsyms, ipsyms, opsyms, ts, is, os, rsym, lastToken)
 	}
 
 	// variable info: type
@@ -628,7 +647,7 @@ func (c *Collector) processSymbol(ctx context.Context, sym *DocumentSymbol, dept
 	}
 }
 
-func (c *Collector) updateFunctionInfo(sym *DocumentSymbol, tsyms, ipsyms, opsyms map[int]dependency, ts, is, os []dependency, rsym *dependency) {
+func (c *Collector) updateFunctionInfo(sym *DocumentSymbol, tsyms, ipsyms, opsyms map[int]dependency, ts, is, os []dependency, rsym *dependency, lastToken int) {
 	if _, ok := c.funcs[sym]; !ok {
 		c.funcs[sym] = functionInfo{}
 	}
@@ -645,5 +664,12 @@ func (c *Collector) updateFunctionInfo(sym *DocumentSymbol, tsyms, ipsyms, opsym
 		}
 		f.Method.Receiver = *rsym
 	}
+
+	// ctruncate the function signature text
+	if lastToken >= 0 && lastToken < len(sym.Tokens)-1 {
+		lastPos := sym.Tokens[lastToken+1].Location.Range.Start
+		f.Signature = ChunkHead(sym.Text, sym.Location.Range.Start, lastPos)
+	}
+
 	c.funcs[sym] = f
 }

lang/collect/export.go

@@ -22,7 +22,6 @@ import (
 	"strings"
 
 	"github.com/cloudwego/abcoder/lang/log"
-	"github.com/cloudwego/abcoder/lang/lsp"
 	. "github.com/cloudwego/abcoder/lang/lsp"
 	"github.com/cloudwego/abcoder/lang/uniast"
 )
@@ -44,8 +43,8 @@ func (c *Collector) fileLine(loc Location) uniast.FileLine {
 	return uniast.FileLine{
 		File:        rel,
 		Line:        loc.Range.Start.Line + 1,
-		StartOffset: lsp.PositionOffset(file_uri, text, loc.Range.Start),
-		EndOffset:   lsp.PositionOffset(file_uri, text, loc.Range.End),
+		StartOffset: PositionOffset(file_uri, text, loc.Range.Start),
+		EndOffset:   PositionOffset(file_uri, text, loc.Range.End),
 	}
 }
 
@@ -75,7 +74,7 @@ func (c *Collector) Export(ctx context.Context) (*uniast.Repository, error) {
 	c.filterLocalSymbols()
 
 	// export symbols
-	visited := make(map[*lsp.DocumentSymbol]*uniast.Identity)
+	visited := make(map[*DocumentSymbol]*uniast.Identity)
 	for _, symbol := range c.syms {
 		_, _ = c.exportSymbol(&repo, symbol, "", visited)
 	}
@@ -191,15 +190,15 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 	// map receiver to methods
 	receivers := make(map[*DocumentSymbol][]*DocumentSymbol, len(c.funcs)/4)
 	for method, rec := range c.funcs {
-		if method.Kind == lsp.SKMethod && rec.Method != nil && rec.Method.Receiver.Symbol != nil {
+		if method.Kind == SKMethod && rec.Method != nil && rec.Method.Receiver.Symbol != nil {
 			receivers[rec.Method.Receiver.Symbol] = append(receivers[rec.Method.Receiver.Symbol], method)
 		}
 	}
 
 	switch k := symbol.Kind; k {
 	// Function
-	case lsp.SKFunction, lsp.SKMethod:
-		if p := c.cli.GetParent(symbol); p != nil && p.Kind == lsp.SKInterface {
+	case SKFunction, SKMethod:
+		if p := c.cli.GetParent(symbol); p != nil && p.Kind == SKInterface {
 			// NOTICE: no need collect interface method
 			break
 		}
@@ -209,6 +208,7 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 			Exported: public,
 		}
 		info := c.funcs[symbol]
+		obj.Signature = info.Signature
 		// NOTICE: type parames collect into types
 		if info.TypeParams != nil {
 			for _, input := range info.TypeParamsSorted {
@@ -262,7 +262,7 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 						id.Name = iid.Name + "<" + id.Name + ">"
 					}
 				}
-				if k == lsp.SKFunction {
+				if k == SKFunction {
 					// NOTICE: class static method name is: type::method
 					id.Name += "::" + name
 				} else {
@@ -285,20 +285,20 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 				}
 				pdep := uniast.NewDependency(*depid, c.fileLine(dep.Location))
 				switch dep.Symbol.Kind {
-				case lsp.SKFunction:
+				case SKFunction:
 					obj.FunctionCalls = uniast.InsertDependency(obj.FunctionCalls, pdep)
-				case lsp.SKMethod:
+				case SKMethod:
 					if obj.MethodCalls == nil {
 						obj.MethodCalls = make([]uniast.Dependency, 0, len(deps))
 					}
 					// NOTICE: use loc token as key here, to make it more readable
 					obj.MethodCalls = uniast.InsertDependency(obj.MethodCalls, pdep)
-				case lsp.SKVariable, lsp.SKConstant:
+				case SKVariable, SKConstant:
 					if obj.GlobalVars == nil {
 						obj.GlobalVars = make([]uniast.Dependency, 0, len(deps))
 					}
 					obj.GlobalVars = uniast.InsertDependency(obj.GlobalVars, pdep)
-				case lsp.SKStruct, lsp.SKTypeParameter, lsp.SKInterface, lsp.SKEnum, lsp.SKClass:
+				case SKStruct, SKTypeParameter, SKInterface, SKEnum, SKClass:
 					if obj.Types == nil {
 						obj.Types = make([]uniast.Dependency, 0, len(deps))
 					}
@@ -312,7 +312,7 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 		pkg.Functions[id.Name] = obj
 
 	// Type
-	case lsp.SKStruct, lsp.SKTypeParameter, lsp.SKInterface, lsp.SKEnum, lsp.SKClass:
+	case SKStruct, SKTypeParameter, SKInterface, SKEnum, SKClass:
 		obj := &uniast.Type{
 			FileLine: fileLine,
 			Content:  content,
@@ -328,7 +328,7 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 					continue
 				}
 				switch dep.Symbol.Kind {
-				case lsp.SKStruct, lsp.SKTypeParameter, lsp.SKInterface, lsp.SKEnum, lsp.SKClass:
+				case SKStruct, SKTypeParameter, SKInterface, SKEnum, SKClass:
 					obj.SubStruct = uniast.InsertDependency(obj.SubStruct, uniast.NewDependency(*depid, c.fileLine(dep.Location)))
 				default:
 					log.Error("dep symbol %s not collected for \n", dep.Symbol, id)
@@ -351,12 +351,12 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 		obj.Identity = *id
 		pkg.Types[id.Name] = obj
 	// Vars
-	case lsp.SKConstant, lsp.SKVariable:
+	case SKConstant, SKVariable:
 		obj := &uniast.Var{
 			FileLine:   fileLine,
 			Content:    content,
 			IsExported: public,
-			IsConst:    k == lsp.SKConstant,
+			IsConst:    k == SKConstant,
 		}
 		if ty, ok := c.vars[symbol]; ok {
 			tok, _ := c.cli.Locate(ty.Location)
@@ -374,18 +374,18 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 	return
 }
 
-func mapKind(kind lsp.SymbolKind) uniast.TypeKind {
+func mapKind(kind SymbolKind) uniast.TypeKind {
 	switch kind {
-	case lsp.SKStruct:
+	case SKStruct:
 		return "struct"
 	// XXX: C++ should use class instead of struct
-	case lsp.SKClass:
+	case SKClass:
 		return "struct"
-	case lsp.SKTypeParameter:
+	case SKTypeParameter:
 		return "type-parameter"
-	case lsp.SKInterface:
+	case SKInterface:
 		return "interface"
-	case lsp.SKEnum:
+	case SKEnum:
 		return "enum"
 	default:
 		panic(fmt.Sprintf("unexpected kind %v", kind))

lang/golang/parser/file.go

@@ -455,6 +455,9 @@ func (p *GoParser) parseFunc(ctx *fileContext, funcDecl *ast.FuncDecl) (*Functio
 		ctx.collectFields(funcDecl.Type.Results.List, &results)
 	}
 
+	// collect signature
+	sig := ctx.GetRawContent(funcDecl.Type)
+
 	// collect content
 	content := string(ctx.GetRawContent(funcDecl))
 
@@ -487,6 +490,7 @@ set_func:
 	f.Results = results
 	f.GlobalVars = collects.globalVars
 	f.Types = collects.tys
+	f.Signature = string(sig)
 	return f, false
 }
 
@@ -588,6 +592,7 @@ func (p *GoParser) parseInterface(ctx *fileContext, name *ast.Ident, decl *ast.I
 			fn.FileLine = ctx.FileLine(fieldDecl)
 			fn.IsMethod = true
 			fn.IsInterfaceMethod = true
+			fn.Signature = string(ctx.GetRawContent(fieldDecl))
 		}
 		p.collectTypes(ctx, fieldDecl.Type, st, inlined)
 	}

lang/lsp/utils.go

@@ -31,6 +31,8 @@
 package lsp
 
 import (
+	"strings"
+
 	"github.com/cloudwego/abcoder/lang/log"
 	"github.com/cloudwego/abcoder/lang/utils"
 )
@@ -71,3 +73,94 @@ func PositionOffset(file_uri string, text string, pos Position) int {
 
 	return RelativePostionWithLines(*lines, Position{Line: 0, Character: 0}, pos)
 }
+
+// FindSingle finds the single char's left token index in a text
+// start and end is the limit range of tokens
+func FindSingle(text string, lines []int, textPos Position, tokens []Token, sep string, start int, end int) int {
+	if start < 0 {
+		start = 0
+	}
+	if end >= len(tokens) {
+		end = len(tokens) - 1
+	}
+	if start >= len(tokens) {
+		return -1
+	}
+	sPos := RelativePostionWithLines(lines, textPos, tokens[start].Location.Range.Start)
+	ePos := RelativePostionWithLines(lines, textPos, tokens[end].Location.Range.End)
+	pos := strings.Index(text[sPos:ePos], sep)
+	if pos == -1 {
+		return -1
+	}
+	pos += sPos
+	for i := start; i <= end && i < len(tokens); i++ {
+		rel := RelativePostionWithLines(lines, textPos, tokens[i].Location.Range.Start)
+		if rel > pos {
+			return i - 1
+		}
+	}
+	return -1
+}
+
+// FindPair finds the right token index of lchar and left token index of rchar in a text
+// start and end is the limit range of tokens
+// notAllow is the character that not allow in the range
+func FindPair(text string, lines []int, textPos Position, tokens []Token, lchar rune, rchar rune, start int, end int, notAllow rune) (int, int) {
+	if start < 0 {
+		start = 0
+	}
+	if end >= len(tokens) {
+		end = len(tokens) - 1
+	}
+	if start >= len(tokens) {
+		return -1, -1
+	}
+
+	startIndex := RelativePostionWithLines(lines, textPos, tokens[start].Location.Range.Start)
+
+	lArrow := -1
+	lCount := 0
+	rArrow := -1
+	notAllowCount := 0
+	ctext := text[startIndex:]
+	for i, c := range ctext {
+		if c == notAllow && lCount == 0 {
+			return -1, -1
+		} else if c == lchar && notAllowCount == 0 {
+			lCount++
+			if lCount == 1 {
+				lArrow = i
+			}
+		} else if c == rchar && notAllowCount == 0 {
+			if rchar == '>' && ctext[i-1] == '-' {
+				// notice: -> is not a pair in Rust
+				continue
+			}
+			lCount--
+			if lCount == 0 {
+				rArrow = i
+				break
+			}
+		}
+	}
+	if lArrow == -1 || rArrow == -1 {
+		return -1, -1
+	}
+	lArrow += startIndex
+	rArrow += startIndex
+
+	s := -1
+	e := -1
+	for i := start; i <= end && i < len(tokens); i++ {
+		rel := RelativePostionWithLines(lines, textPos, tokens[i].Location.Range.Start)
+		if rel >= lArrow && s == -1 {
+			s = i
+		}
+		if rel > rArrow {
+			e = i - 1
+			break
+		}
+	}
+
+	return s, e
+}