Skip to content

Commit 56430f0

Browse files
Merge pull request #9 from oasdiff/perf/origin-tree-apply
perf: compact __origin__ format — 4× faster loading for large specs
2 parents c5cbf48 + 1bf705c commit 56430f0

5 files changed

Lines changed: 271 additions & 12 deletions

File tree

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/go-openapi/jsonpointer v0.21.0
77
github.com/gorilla/mux v1.8.0
88
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
9-
github.com/oasdiff/yaml v0.0.4
9+
github.com/oasdiff/yaml v0.0.5-beta.1
1010
github.com/oasdiff/yaml3 v0.0.4
1111
github.com/perimeterx/marshmallow v1.1.5
1212
github.com/stretchr/testify v1.9.0

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0
1818
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
1919
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
2020
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
21-
github.com/oasdiff/yaml v0.0.4 h1:airPco4LbUoK4nbVwu+wwkRg2WarLC96cgBhgN93fsE=
22-
github.com/oasdiff/yaml v0.0.4/go.mod h1:EaJ6/lcrRLK+syawtvtrHdbrrln4/SUmQw6aBTIlaMs=
21+
github.com/oasdiff/yaml v0.0.5-beta.1 h1:xCTUYEOtU5qqX3rYm2T0W5Wf60VVEvWw6RUYR2W54NI=
22+
github.com/oasdiff/yaml v0.0.5-beta.1/go.mod h1:EaJ6/lcrRLK+syawtvtrHdbrrln4/SUmQw6aBTIlaMs=
2323
github.com/oasdiff/yaml3 v0.0.4 h1:U5RTQZpBmsbcyCFlzPVuMctk6Jme6lOrbl0jJoOovMw=
2424
github.com/oasdiff/yaml3 v0.0.4/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o=
2525
github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s=

openapi3/loader.go

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ import (
1515
"strings"
1616
)
1717

18-
// IncludeOrigin specifies whether to include the origin of the OpenAPI elements
19-
// Set this to true before loading a spec to include the origin of the OpenAPI elements
20-
// Note it is global and affects all loaders
18+
// IncludeOrigin specifies whether to include the origin of the OpenAPI elements.
19+
// Deprecated: set Loader.IncludeOrigin instead. This global is read by NewLoader
20+
// for backward compatibility but is not safe for concurrent use.
2121
var IncludeOrigin = false
2222

2323
func foundUnresolvedRef(ref string) error {
@@ -33,6 +33,11 @@ type Loader struct {
3333
// IsExternalRefsAllowed enables visiting other files
3434
IsExternalRefsAllowed bool
3535

36+
// IncludeOrigin enables recording the file/line/column of each OpenAPI element.
37+
// Prefer this over the package-level IncludeOrigin global, which is not safe for
38+
// concurrent use.
39+
IncludeOrigin bool
40+
3641
// ReadFromURIFunc allows overriding the any file/URL reading func
3742
ReadFromURIFunc ReadFromURIFunc
3843

@@ -53,7 +58,8 @@ type Loader struct {
5358
// NewLoader returns an empty Loader
5459
func NewLoader() *Loader {
5560
return &Loader{
56-
Context: context.Background(),
61+
Context: context.Background(),
62+
IncludeOrigin: IncludeOrigin,
5763
}
5864
}
5965

@@ -108,7 +114,7 @@ func (loader *Loader) loadSingleElementFromURI(ref string, rootPath *url.URL, el
108114
if err != nil {
109115
return nil, err
110116
}
111-
if err := unmarshal(data, element, IncludeOrigin, resolvedPath); err != nil {
117+
if err := unmarshal(data, element, loader.IncludeOrigin || IncludeOrigin, resolvedPath); err != nil {
112118
return nil, err
113119
}
114120

@@ -144,7 +150,7 @@ func (loader *Loader) LoadFromIoReader(reader io.Reader) (*T, error) {
144150
func (loader *Loader) LoadFromData(data []byte) (*T, error) {
145151
loader.resetVisitedPathItemRefs()
146152
doc := &T{}
147-
if err := unmarshal(data, doc, IncludeOrigin, nil); err != nil {
153+
if err := unmarshal(data, doc, loader.IncludeOrigin || IncludeOrigin, nil); err != nil {
148154
return nil, err
149155
}
150156
if err := loader.ResolveRefsIn(doc, nil); err != nil {
@@ -173,7 +179,7 @@ func (loader *Loader) loadFromDataWithPathInternal(data []byte, location *url.UR
173179
doc := &T{}
174180
loader.visitedDocuments[uri] = doc
175181

176-
if err := unmarshal(data, doc, IncludeOrigin, location); err != nil {
182+
if err := unmarshal(data, doc, loader.IncludeOrigin || IncludeOrigin, location); err != nil {
177183
return nil, err
178184
}
179185

@@ -427,7 +433,7 @@ func (loader *Loader) resolveComponent(doc *T, ref string, path *url.URL, resolv
427433
if err2 != nil {
428434
return nil, nil, err
429435
}
430-
if err2 = unmarshal(data, &cursor, IncludeOrigin, path); err2 != nil {
436+
if err2 = unmarshal(data, &cursor, loader.IncludeOrigin || IncludeOrigin, path); err2 != nil {
431437
return nil, nil, err
432438
}
433439
if cursor, err2 = drill(cursor); err2 != nil || cursor == nil {

openapi3/marsh.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,13 @@ func unmarshal(data []byte, v any, includeOrigin bool, location *url.URL) error
3030
if location != nil {
3131
file = location.Path
3232
}
33-
if yamlErr = yaml.UnmarshalWithOrigin(data, v, yaml.OriginOpt{Enabled: includeOrigin, File: file}); yamlErr == nil {
33+
originOpt := yaml.OriginOpt{Enabled: includeOrigin, File: file}
34+
tree, err := yaml.UnmarshalWithOriginTree(data, v, originOpt)
35+
if err == nil {
36+
applyOrigins(v, tree)
3437
return nil
3538
}
39+
yamlErr = err
3640

3741
// If both unmarshaling attempts fail, return a new error that includes both errors
3842
return fmt.Errorf("failed to unmarshal data: json error: %v, yaml error: %v", jsonErr, yamlErr)

openapi3/origin.go

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,17 @@
11
package openapi3
22

3+
import (
4+
"encoding/json"
5+
"reflect"
6+
"strings"
7+
8+
"github.com/oasdiff/yaml"
9+
)
10+
311
const originKey = "__origin__"
412

13+
var originPtrType = reflect.TypeOf((*Origin)(nil))
14+
515
// Origin contains the origin of a collection.
616
// Key is the location of the collection itself.
717
// Fields is a map of the location of each scalar field in the collection.
@@ -20,6 +30,245 @@ type Location struct {
2030
Name string `json:"name,omitempty" yaml:"name,omitempty"`
2131
}
2232

33+
// originFromSeq parses the compact []any sequence produced by yaml3's addOrigin.
34+
//
35+
// Format: [file, key_name, key_line, key_col, nf, f1_name, f1_delta, f1_col, ..., ns, s1_name, s1_count, s1_l0_delta, s1_c0, ...]
36+
func originFromSeq(s []any) *Origin {
37+
// Need at least: file, key_name, key_line, key_col, nf, ns
38+
if len(s) < 6 {
39+
return nil
40+
}
41+
file, _ := s[0].(string)
42+
keyName, _ := s[1].(string)
43+
keyLine := toInt(s[2])
44+
keyCol := toInt(s[3])
45+
46+
o := &Origin{
47+
Key: &Location{
48+
File: file,
49+
Line: keyLine,
50+
Column: keyCol,
51+
Name: keyName,
52+
},
53+
}
54+
55+
idx := 4
56+
nf := toInt(s[idx])
57+
idx++
58+
if nf > 0 && idx+nf*3 <= len(s) {
59+
o.Fields = make(map[string]Location, nf)
60+
for i := 0; i < nf; i++ {
61+
fname, _ := s[idx].(string)
62+
delta := toInt(s[idx+1])
63+
col := toInt(s[idx+2])
64+
o.Fields[fname] = Location{
65+
File: file,
66+
Line: keyLine + delta,
67+
Column: col,
68+
Name: fname,
69+
}
70+
idx += 3
71+
}
72+
}
73+
74+
if idx >= len(s) {
75+
return o
76+
}
77+
ns := toInt(s[idx])
78+
idx++
79+
if ns > 0 {
80+
o.Sequences = make(map[string][]Location, ns)
81+
for i := 0; i < ns; i++ {
82+
if idx >= len(s) {
83+
break
84+
}
85+
sname, _ := s[idx].(string)
86+
idx++
87+
count := toInt(s[idx])
88+
idx++
89+
locs := make([]Location, count)
90+
for j := 0; j < count && idx+1 < len(s); j++ {
91+
delta := toInt(s[idx])
92+
col := toInt(s[idx+1])
93+
locs[j] = Location{File: file, Line: keyLine + delta, Column: col}
94+
idx += 2
95+
}
96+
o.Sequences[sname] = locs
97+
}
98+
}
99+
return o
100+
}
101+
102+
// UnmarshalJSON parses the compact []any sequence produced by yaml3's addOrigin.
103+
// This allows __origin__ to be decoded directly during JSON unmarshaling without
104+
// a separate applyOrigins pass when the caller does not use UnmarshalWithOriginTree.
105+
func (o *Origin) UnmarshalJSON(data []byte) error {
106+
var seq []any
107+
if err := json.Unmarshal(data, &seq); err != nil {
108+
return err
109+
}
110+
if parsed := originFromSeq(seq); parsed != nil {
111+
*o = *parsed
112+
}
113+
return nil
114+
}
115+
116+
// toInt converts numeric types to int. Handles int/uint64 from YAML decoding
117+
// and float64 from JSON decoding of []any sequences.
118+
func toInt(v any) int {
119+
switch n := v.(type) {
120+
case int:
121+
return n
122+
case uint64:
123+
return int(n)
124+
case float64:
125+
return int(n)
126+
}
127+
return 0
128+
}
129+
130+
// applyOrigins walks a Go struct tree and a parallel OriginTree, setting
131+
// Origin fields on each struct from the extracted origin data.
132+
func applyOrigins(v any, tree *yaml.OriginTree) {
133+
if tree == nil {
134+
return
135+
}
136+
applyOriginsToValue(reflect.ValueOf(v), tree)
137+
}
138+
139+
func applyOriginsToValue(val reflect.Value, tree *yaml.OriginTree) {
140+
// Keep track of the last pointer so we can pass it to struct handlers
141+
// (needed for calling methods like Map() on maplike types).
142+
var ptr reflect.Value
143+
for val.Kind() == reflect.Ptr || val.Kind() == reflect.Interface {
144+
if val.IsNil() {
145+
return
146+
}
147+
if val.Kind() == reflect.Ptr {
148+
ptr = val
149+
}
150+
val = val.Elem()
151+
}
152+
153+
switch val.Kind() {
154+
case reflect.Struct:
155+
applyOriginsToStruct(val, ptr, tree)
156+
case reflect.Map:
157+
applyOriginsToMap(val, tree)
158+
case reflect.Slice:
159+
applyOriginsToSlice(val, tree)
160+
}
161+
}
162+
163+
func applyOriginsToStruct(val reflect.Value, ptr reflect.Value, tree *yaml.OriginTree) {
164+
typ := val.Type()
165+
166+
// Set Origin field if tagged with json:"__origin__" or json:"-" (maplike types).
167+
// Skip *Ref types which have Origin with no json tag — those pass through to Value.
168+
if tree.Origin != nil {
169+
if sf, ok := typ.FieldByName("Origin"); ok && sf.Type == originPtrType {
170+
tag := sf.Tag.Get("json")
171+
if strings.Contains(tag, originKey) || tag == "-" {
172+
if s, ok := tree.Origin.([]any); ok {
173+
val.FieldByName("Origin").Set(reflect.ValueOf(originFromSeq(s)))
174+
}
175+
}
176+
}
177+
}
178+
179+
// Recurse into exported struct fields using json tags
180+
for i := 0; i < typ.NumField(); i++ {
181+
sf := typ.Field(i)
182+
if !sf.IsExported() {
183+
continue
184+
}
185+
tag := jsonTagName(sf)
186+
if tag == "" || tag == "-" {
187+
continue
188+
}
189+
childTree := tree.Fields[tag]
190+
if childTree != nil {
191+
applyOriginsToValue(val.Field(i), childTree)
192+
}
193+
}
194+
195+
// Handle wrapper types whose inner struct has no json tag:
196+
// - *Ref types (e.g. SchemaRef, ResponseRef) have a "Value" field
197+
// - AdditionalProperties has a "Schema" field
198+
// The origin tree data applies to the inner struct, not a sub-key.
199+
for _, fieldName := range []string{"Value", "Schema"} {
200+
vf := val.FieldByName(fieldName)
201+
if !vf.IsValid() || vf.Kind() != reflect.Ptr || vf.IsNil() {
202+
continue
203+
}
204+
sf, _ := typ.FieldByName(fieldName)
205+
if sf.Tag.Get("json") == "" {
206+
applyOriginsToValue(vf, tree)
207+
}
208+
}
209+
210+
// Handle "maplike" types (Paths, Responses, Callback) whose items are
211+
// stored in an unexported map accessible via a Map() method.
212+
// Use the original pointer (if available) since dereferenced values
213+
// are not addressable.
214+
receiver := val
215+
if ptr.IsValid() {
216+
receiver = ptr
217+
} else if val.CanAddr() {
218+
receiver = val.Addr()
219+
}
220+
if receiver.Kind() == reflect.Ptr {
221+
if mapMethod := receiver.MethodByName("Map"); mapMethod.IsValid() {
222+
results := mapMethod.Call(nil)
223+
if len(results) == 1 {
224+
applyOriginsToMap(results[0], tree)
225+
}
226+
}
227+
}
228+
}
229+
230+
func applyOriginsToMap(val reflect.Value, tree *yaml.OriginTree) {
231+
if tree.Fields == nil {
232+
return
233+
}
234+
for _, key := range val.MapKeys() {
235+
childTree := tree.Fields[key.String()]
236+
if childTree == nil {
237+
continue
238+
}
239+
elem := val.MapIndex(key)
240+
// Map values are not addressable. For pointer-typed values we can
241+
// recurse directly. For value types we must copy, apply, and set back.
242+
if elem.Kind() == reflect.Ptr || elem.Kind() == reflect.Interface {
243+
applyOriginsToValue(elem, childTree)
244+
} else if elem.Kind() == reflect.Struct {
245+
// Copy to a settable value
246+
cp := reflect.New(elem.Type()).Elem()
247+
cp.Set(elem)
248+
applyOriginsToStruct(cp, reflect.Value{}, childTree)
249+
val.SetMapIndex(key, cp)
250+
}
251+
}
252+
}
253+
254+
func applyOriginsToSlice(val reflect.Value, tree *yaml.OriginTree) {
255+
for i := 0; i < val.Len() && i < len(tree.Items); i++ {
256+
if tree.Items[i] != nil {
257+
applyOriginsToValue(val.Index(i), tree.Items[i])
258+
}
259+
}
260+
}
261+
262+
// jsonTagName returns the JSON field name from a struct field's json tag.
263+
func jsonTagName(f reflect.StructField) string {
264+
tag := f.Tag.Get("json")
265+
if tag == "" {
266+
return ""
267+
}
268+
name, _, _ := strings.Cut(tag, ",")
269+
return name
270+
}
271+
23272
// stripOriginFromAny recursively removes the __origin__ key from any
24273
// map[string]any value. This is needed for interface{}/any-typed fields
25274
// (e.g. Schema.Enum, Schema.Default, Parameter.Example) that have no

0 commit comments

Comments
 (0)