fix: remove cycle reference to marshal json (cloudwego#16)

Author: AsterDY

src/compress/golang/plugin/file.go

@@ -38,7 +38,7 @@ type Function struct {
 	IsMethod         bool    // If the function is a method
 	Name             string  // Name of the function
 	PkgPath                  // import path to the package where the function is defined
-	FilePath         string  // File where the function is defined, empty if the function declaration is not scanned
+	FilePath         string  `json:"-"` // File where the function is defined, empty if the function declaration is not scanned
 	Content          string  // Content of the function, including functiion signature and body
 	AssociatedStruct *Struct // Method receiver
 
@@ -302,7 +302,7 @@ type Struct struct {
 	IsInterface bool   //maybe a interface type decl
 	Name        string // Name of the struct
 	PkgPath            // Path to the package where the struct is defined
-	FilePath    string // File where the struct is defined
+	FilePath    string `json:"-"` // File where the struct is defined
 	Content     string // struct declaration content
 
 	// related local structs in fields, key is {{pkgName.typName}} or {{typeName}}, val is declaration of the struct

src/compress/golang/plugin/go_ast.go

@@ -142,40 +142,14 @@ type MainStream struct {
 	RelatedStruct []SingleStruct
 }
 
-func (m *MainStream) Dedup() {
-	fs := map[string]string{}
-	for _, f := range m.RelatedFunctions {
-		fs[f.CallName] = f.Content
-	}
-	m.RelatedFunctions = m.RelatedFunctions[:len(fs)]
-	i := 0
-	for k, v := range fs {
-		m.RelatedFunctions[i].CallName = k
-		m.RelatedFunctions[i].Content = v
-		i++
-	}
-
-	fs = map[string]string{}
-	for _, f := range m.RelatedStruct {
-		fs[f.Name] = f.Content
-	}
-	m.RelatedStruct = m.RelatedStruct[:len(fs)]
-	i = 0
-	for k, v := range fs {
-		m.RelatedStruct[i].Name = k
-		m.RelatedStruct[i].Content = v
-		i++
-	}
-}
-
 type SingleFunction struct {
 	CallName string
 	Content  string
 }
 
 type SingleStruct struct {
-	Name    string
-	Content string
+	CallName string
+	Content  string
 }
 
 func (p *goParser) getMain(depth int) (*MainStream, *Function) {
@@ -194,61 +168,70 @@ func (p *goParser) getMain(depth int) (*MainStream, *Function) {
 			}
 		}
 	}
-	visited := map[string]map[string]bool{}
+	visited := cache(map[interface{}]bool{})
 	p.fillRelatedContent(depth, mainFunc, &m.RelatedFunctions, &m.RelatedStruct, visited)
 	return m, mainFunc
 }
 
-func (p *goParser) fillRelatedContent(depth int, f *Function, fl *[]SingleFunction, sl *[]SingleStruct, visited map[string]map[string]bool) {
+func (p *goParser) fillRelatedContent(depth int, f *Function, fl *[]SingleFunction, sl *[]SingleStruct, visited cache) {
 	if depth == 0 {
 		return
 	}
-	if f == nil || (visited[f.PkgPath] != nil && visited[f.PkgPath][f.Name]) {
+	if f == nil {
 		return
-	} else {
-		if visited[f.PkgPath] == nil {
-			visited[f.PkgPath] = map[string]bool{}
-		}
-		visited[f.PkgPath][f.Name] = true
 	}
-	for call, ff := range f.InternalFunctionCalls {
+
+	//BFS
+	var next []Function
+
+	for call, v := range f.InternalFunctionCalls {
+		if visited.Visited(v) {
+			continue
+		}
+		ff := *v
 		s := SingleFunction{
 			CallName: call,
-			Content:  ff.Content,
+			// Name:     ff.PkgPath + "." + ff.Name,
+			Content: ff.Content,
 		}
 		*fl = append(*fl, s)
-		p.fillRelatedContent(depth-1, ff, fl, sl, visited)
+		next = append(next, ff)
 	}
 
-	for call, ff := range f.InternalMethodCalls {
+	for call, v := range f.InternalMethodCalls {
+		if visited.Visited(v) {
+			continue
+		}
+		ff := *v
 		content := ff.Content
 		if ff.AssociatedStruct != nil && ff.AssociatedStruct.IsInterface {
 			content = ff.AssociatedStruct.Content
 		}
 		s := SingleFunction{
 			CallName: call,
-			Content:  content,
+			// Name:     ff.PkgPath + "." + ff.Name,
+			Content: content,
 		}
 		*fl = append(*fl, s)
-		p.fillRelatedContent(depth-1, ff, fl, sl, visited)
+		next = append(next, ff)
 
 		// for method which has been associated with struct, push the struct
 		if ff.AssociatedStruct != nil && ff.AssociatedStruct.Content != "" {
 			st := ff.AssociatedStruct
-			if visited[st.PkgPath] != nil && visited[st.PkgPath][st.Name] {
+			if visited.Visited(st) {
 				continue
-			} else if visited[st.PkgPath] == nil {
-				visited[st.PkgPath] = map[string]bool{}
 			}
-			visited[st.PkgPath][st.Name] = true
 			ss := SingleStruct{
-				Name:    ff.PkgPath + "." + st.Name,
+				CallName: call,
+				// Name:     (*st).PkgPath + "." + st.Name,
 				Content: st.Content,
 			}
 			*sl = append(*sl, ss)
 		}
+	}
 
-		p.fillRelatedContent(depth-1, ff, fl, sl, visited)
+	for _, ff := range next {
+		p.fillRelatedContent(depth-1, &ff, fl, sl, visited)
 	}
 }
 
@@ -294,8 +277,7 @@ func main() {
 	}
 
 	// p.generateStruct()
-	m, _ := p.getMain(100)
-	m.Dedup()
+	m, _ := p.getMain(-1)
 
 	out := bytes.NewBuffer(nil)
 	encoder := json.NewEncoder(out)

src/compress/golang/plugin/go_ast_test.go

@@ -40,11 +40,17 @@ func Test_goParser_ParseTilTheEnd(t *testing.T) {
 			if fun.Name != "main" {
 				t.Fail()
 			}
-			out.Dedup()
-			if out, err := json.MarshalIndent(out, "", "  "); err != nil {
+			fun.RemoveCycle()
+			if out, err := json.Marshal(fun); err != nil {
+				t.Fatal(err)
+			} else {
+				println("func size:", len(out), string(out))
+			}
+			if out, err := json.Marshal(out); err != nil {
 				t.Fatalf("json.Marshal() error = %v", err)
 			} else {
-				println("size:", len(out))
+				println("stream size:", len(out))
+				println(string(out))
 			}
 		})
 	}

src/compress/golang/plugin/helper.go

@@ -0,0 +1,90 @@
+/**
+ * Copyright 2024 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+type cache map[interface{}]bool
+
+func (c cache) Visited(val interface{}) bool {
+	ok := c[val]
+	if !ok {
+		c[val] = true
+	}
+	return ok
+}
+
+// RemoveCycle remove cycle reference to repeated pointer, in case of marshalling fail
+func (f *Function) RemoveCycle() {
+	var visited = cache(map[interface{}]bool{})
+	f.deLoop(visited)
+	if f.AssociatedStruct != nil {
+		f.AssociatedStruct.deLoop(visited)
+	}
+}
+
+func (f *Function) deLoop(visited cache) {
+	for k, ff := range f.InternalFunctionCalls {
+		if visited.Visited(ff) {
+			var nf = new(Function)
+			nf.PkgPath = ff.PkgPath
+			nf.Name = ff.Name
+			f.InternalFunctionCalls[k] = nf
+		} else {
+			ff.deLoop(visited)
+		}
+	}
+	for k, ff := range f.InternalMethodCalls {
+		if visited.Visited(ff) {
+			var nf = new(Function)
+			nf.PkgPath = ff.PkgPath
+			nf.Name = ff.Name
+			nf.IsMethod = ff.IsMethod
+			f.InternalMethodCalls[k] = nf
+		} else {
+			ff.deLoop(visited)
+			if ff.AssociatedStruct != nil {
+				ff.AssociatedStruct.deLoop(visited)
+			}
+		}
+	}
+}
+
+func (f *Struct) deLoop(visited cache) {
+	for k, ff := range f.InternalStructs {
+		if visited.Visited(ff) {
+			var nf = new(Struct)
+			nf.PkgPath = ff.PkgPath
+			nf.Name = ff.Name
+			f.InternalStructs[k] = nf
+		} else {
+			ff.deLoop(visited)
+		}
+	}
+	for k, ff := range f.Methods {
+		if visited.Visited(ff) {
+			var nf = new(Function)
+			nf.PkgPath = ff.PkgPath
+			nf.Name = ff.Name
+			nf.IsMethod = ff.IsMethod
+			f.Methods[k] = nf
+		} else {
+			ff.deLoop(visited)
+			if ff.AssociatedStruct != nil {
+				ff.AssociatedStruct.deLoop(visited)
+			}
+		}
+	}
+}