From 630e2b4f8ae0a64eb2d5b56639c808fa9c18ab9a Mon Sep 17 00:00:00 2001 From: quobix Date: Wed, 1 Apr 2026 10:05:30 -0400 Subject: [PATCH 1/9] refactored index no breaking changes or changes to the public contract, but a significant clean up, refactor, breakdown and tuneup. --- index/doc.go | 36 + index/extract_refs.go | 1039 +----------------- index/extract_refs_inline.go | 164 +++ index/extract_refs_lookup.go | 235 ++++ index/extract_refs_metadata.go | 204 ++++ index/extract_refs_ref.go | 332 ++++++ index/extract_refs_test.go | 227 ++++ index/extract_refs_walk.go | 165 +++ index/find_component.go | 279 ----- index/find_component_build.go | 106 ++ index/find_component_direct.go | 77 ++ index/find_component_entry.go | 117 ++ index/find_component_external.go | 106 ++ index/find_component_test.go | 125 +++ index/index_model.go | 56 +- index/index_model_test.go | 67 +- index/map_index_nodes.go | 2 +- index/map_index_nodes_test.go | 27 + index/resolver.go | 1051 ------------------ index/resolver_circular.go | 122 +++ index/resolver_entry.go | 216 ++++ index/resolver_mutation.go | 76 ++ index/resolver_paths.go | 119 ++ index/resolver_polymorphic.go | 93 ++ index/resolver_relatives.go | 263 +++++ index/resolver_test.go | 195 +++- index/resolver_visit.go | 170 +++ index/resolver_walk.go | 40 + index/rolodex.go | 387 +++---- index/rolodex_file_loader.go | 3 +- index/rolodex_remote_loader.go | 364 +++--- index/rolodex_remote_loader_test.go | 137 +++ index/rolodex_test.go | 257 +++++ index/search_index.go | 4 +- index/spec_index.go | 1524 -------------------------- index/spec_index_accessors.go | 335 ++++++ index/spec_index_build.go | 164 +++ index/spec_index_counts.go | 699 ++++++++++++ index/spec_index_test.go | 391 +++++++ index/utility_methods.go | 6 +- index/utility_methods_buffer_test.go | 15 + index/utility_methods_test.go | 59 + 42 files changed, 5763 insertions(+), 4291 deletions(-) create mode 100644 index/doc.go create mode 100644 index/extract_refs_inline.go create mode 100644 index/extract_refs_lookup.go create mode 100644 index/extract_refs_metadata.go create mode 100644 index/extract_refs_ref.go create mode 100644 index/extract_refs_walk.go delete mode 100644 index/find_component.go create mode 100644 index/find_component_build.go create mode 100644 index/find_component_direct.go create mode 100644 index/find_component_entry.go create mode 100644 index/find_component_external.go delete mode 100644 index/resolver.go create mode 100644 index/resolver_circular.go create mode 100644 index/resolver_entry.go create mode 100644 index/resolver_mutation.go create mode 100644 index/resolver_paths.go create mode 100644 index/resolver_polymorphic.go create mode 100644 index/resolver_relatives.go create mode 100644 index/resolver_visit.go create mode 100644 index/resolver_walk.go delete mode 100644 index/spec_index.go create mode 100644 index/spec_index_accessors.go create mode 100644 index/spec_index_build.go create mode 100644 index/spec_index_counts.go diff --git a/index/doc.go b/index/doc.go new file mode 100644 index 000000000..ed59ed156 --- /dev/null +++ b/index/doc.go @@ -0,0 +1,36 @@ +// Copyright 2026 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +/* +Package index builds and queries reference indexes for OpenAPI and JSON Schema documents. + +# Internal layout + +The package is organized around a few cooperating subsystems: + + - SpecIndex owns a single parsed document, its discovered references, derived component maps, + counters, and runtime caches. + - reference extraction walks YAML nodes and records references, inline definitions, and + component metadata used later by lookup and resolution. + - lookup resolves local, external, and schema-id based references. Exact component references + use direct map lookups first, with JSONPath-based traversal retained as a fallback. + - Rolodex coordinates local and remote file systems, external document indexing, and shared + reference lookup across multiple indexed documents. + - Resolver performs circular reference analysis and, when requested, destructive in-place + resolution of relative references. + +Key invariants + + - Public behavior is preserved through staged internal refactors. Helper extraction and + subsystem boundaries should not change external APIs. + - SpecIndex.Release and Rolodex.Release are responsible for clearing owned runtime state so + long-lived processes do not retain unnecessary memory. + - Common exact component references should avoid JSONPath parsing on the hot path. + - External lookup must remain safe under concurrent indexing and must not leak locks, response + bodies, or in-flight wait state. + +When editing this package, prefer extending the existing subsystem seams instead of adding more +responsibility to the top-level orchestration files for indexing, rolodex loading, or resolver +entry points. +*/ +package index diff --git a/index/extract_refs.go b/index/extract_refs.go index 154e02803..8f48ba0bc 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -5,32 +5,10 @@ package index import ( "context" - "errors" - "fmt" - "net/url" - "os" - "path/filepath" - "runtime" - "sort" - "strconv" - "strings" - "sync" - "github.com/pb33f/libopenapi/utils" "go.yaml.in/yaml/v4" - "golang.org/x/sync/singleflight" ) -// preserveLegacyRefOrder allows opt-out of deterministic ordering if issues arise. -// Set LIBOPENAPI_LEGACY_REF_ORDER=true to use the old non-deterministic ordering. -var preserveLegacyRefOrder = os.Getenv("LIBOPENAPI_LEGACY_REF_ORDER") == "true" - -// indexedRef pairs a resolved reference with its original input position for deterministic ordering. -type indexedRef struct { - ref *Reference - pos int -} - func isSchemaContainingNode(v string) bool { switch v { case "schema", "items", "additionalProperties", "contains", "not", @@ -77,1021 +55,8 @@ func (index *SpecIndex) ExtractRefs(ctx context.Context, node, parent *yaml.Node return nil } - // Initialize $id scope if not present (uses document base as initial scope) - scope := GetSchemaIdScope(ctx) - if scope == nil { - scope = NewSchemaIdScope(index.specAbsolutePath) - ctx = WithSchemaIdScope(ctx, scope) - } - - // Capture the parent's base URI BEFORE any $id in this node is processed - // This is used for registering any $id found in this node - parentBaseUri := scope.BaseUri - - // Check if THIS node has a $id and update scope for processing children - // This must happen before iterating children so they see the updated scope - if node.Kind == yaml.MappingNode && !underOpenAPIExamplePath(seenPath) { - if nodeId := FindSchemaIdInNode(node); nodeId != "" { - resolvedNodeId, _ := ResolveSchemaId(nodeId, parentBaseUri) - if resolvedNodeId == "" { - resolvedNodeId = nodeId - } - // Update scope for children of this node - scope = scope.Copy() - scope.PushId(resolvedNodeId) - ctx = WithSchemaIdScope(ctx, scope) - } - } - - var found []*Reference - if len(node.Content) > 0 { - var prev, polyName string - for i, n := range node.Content { - if utils.IsNodeMap(n) || utils.IsNodeArray(n) { - level++ - // check if we're using polymorphic values. These tend to create rabbit warrens of circular - // references if every single link is followed. We don't resolve polymorphic values. - isPoly, _ := index.checkPolymorphicNode(prev) - polyName = pName - if isPoly { - poly = true - if prev != "" { - polyName = prev - } - } - - found = append(found, index.ExtractRefs(ctx, n, node, seenPath, level, poly, polyName)...) - } - - // check if we're dealing with an inline schema definition, that isn't part of an array - // (which means it's being used as a value in an array, and it's not a label) - // https://github.com/pb33f/libopenapi/issues/76 - if i%2 == 0 && isSchemaContainingNode(n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) { - - var jsonPath, definitionPath, fullDefinitionPath string - - if len(seenPath) > 0 || n.Value != "" { - loc := append(seenPath, n.Value) - // create definition and full definition paths - locPath := strings.Join(loc, "/") - definitionPath = "#/" + locPath - fullDefinitionPath = index.specAbsolutePath + "#/" + locPath - _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) - } - - ref := &Reference{ - ParentNode: parent, - FullDefinition: fullDefinitionPath, - Definition: definitionPath, - Node: node.Content[i+1], - KeyNode: node.Content[i], - Path: jsonPath, - Index: index, - } - - isRef, _, _ := utils.IsNodeRefValue(node.Content[i+1]) - if isRef { - // record this reference - index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref) - continue - } - - if n.Value == "additionalProperties" || n.Value == "unevaluatedProperties" { - if utils.IsNodeBoolValue(node.Content[i+1]) { - continue - } - } - - index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref) - - // check if the schema is an object or an array, - // and if so, add it to the list of inline schema object definitions. - k, v := utils.FindKeyNodeTop("type", node.Content[i+1].Content) - if k != nil && v != nil { - if v.Value == "object" || v.Value == "array" { - index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref) - } - } - } - - // Perform the same check for all maps of schemas like properties and patternProperties - // https://github.com/pb33f/libopenapi/issues/76 - if i%2 == 0 && isMapOfSchemaContainingNode(n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) { - - // if 'examples' or 'example' exists in the seenPath, skip this 'properties' node. - // https://github.com/pb33f/libopenapi/issues/160 - if len(seenPath) > 0 { - skip := false - - // iterate through the path and look for an OpenAPI example/examples keyword or extension - for j, p := range seenPath { - if p == "examples" || p == "example" { - if j == 0 || (seenPath[j-1] != "properties" && seenPath[j-1] != "patternProperties") { - skip = true - break - } - } - // look for any extension in the path and ignore it - if strings.HasPrefix(p, "x-") { - skip = true - break - } - } - if skip { - continue - } - } - - // for each property add it to our schema definitions - label := "" - for h, prop := range node.Content[i+1].Content { - - if h%2 == 0 { - label = prop.Value - continue - } - var jsonPath, definitionPath, fullDefinitionPath string - if len(seenPath) > 0 || n.Value != "" && label != "" { - loc := append(seenPath, n.Value, label) - locPath := strings.Join(loc, "/") - definitionPath = "#/" + locPath - fullDefinitionPath = index.specAbsolutePath + "#/" + locPath - _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) - } - ref := &Reference{ - ParentNode: parent, - FullDefinition: fullDefinitionPath, - Definition: definitionPath, - Node: prop, - KeyNode: node.Content[i], - Path: jsonPath, - Index: index, - } - - isRef, _, _ := utils.IsNodeRefValue(prop) - if isRef { - // record this reference - index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref) - continue - } - - index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref) - - // check if the schema is an object or an array, - // and if so, add it to the list of inline schema object definitions. - k, v := utils.FindKeyNodeTop("type", prop.Content) - if k != nil && v != nil { - if v.Value == "object" || v.Value == "array" { - index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref) - } - } - } - } - - // Perform the same check for all arrays of schemas like allOf, anyOf, oneOf - if i%2 == 0 && isArrayOfSchemaContainingNode(n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) { - // for each element in the array, add it to our schema definitions - for h, element := range node.Content[i+1].Content { - - var jsonPath, definitionPath, fullDefinitionPath string - if len(seenPath) > 0 { - loc := append(seenPath, n.Value, strconv.Itoa(h)) - locPath := strings.Join(loc, "/") - definitionPath = "#/" + locPath - fullDefinitionPath = index.specAbsolutePath + "#/" + locPath - _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) - } else { - definitionPath = "#/" + n.Value - fullDefinitionPath = index.specAbsolutePath + "#/" + n.Value - _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) - } - - ref := &Reference{ - ParentNode: parent, - FullDefinition: fullDefinitionPath, - Definition: definitionPath, - Node: element, - KeyNode: node.Content[i], - Path: jsonPath, - Index: index, - } - - isRef, _, _ := utils.IsNodeRefValue(element) - if isRef { // record this reference - index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref) - continue - } - index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref) - - // check if the schema is an object or an array, - // and if so, add it to the list of inline schema object definitions. - k, v := utils.FindKeyNodeTop("type", element.Content) - if k != nil && v != nil { - if v.Value == "object" || v.Value == "array" { - index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref) - } - } - } - } - - if i%2 == 0 && n.Value == "$ref" { - - // Check if this reference is under an extension path (x-* field). - // Always compute this so we can mark refs with IsExtensionRef. - isExtensionPath := false - for _, spi := range seenPath { - if strings.HasPrefix(spi, "x-") { - isExtensionPath = true - break - } - } - - // If configured to exclude extension refs, skip it entirely. - if index.config.ExcludeExtensionRefs && isExtensionPath { - continue - } - - // only look at scalar values, not maps (looking at you k8s) - if len(node.Content) > i+1 { - if !utils.IsNodeStringValue(node.Content[i+1]) { - continue - } - // issue #481, don't look at refs in arrays, the next node isn't the value. - if utils.IsNodeArray(node) { - continue - } - } - - index.linesWithRefs[n.Line] = true - - fp := make([]string, len(seenPath)) - copy(fp, seenPath) - - if len(node.Content) > i+1 { - - value := node.Content[i+1].Value - schemaIdBase := "" - if scope != nil && len(scope.Chain) > 0 { - schemaIdBase = scope.BaseUri - } - // extract last path segment without allocating a full slice - lastSlash := strings.LastIndexByte(value, '/') - var name string - if lastSlash >= 0 { - name = value[lastSlash+1:] - } else { - name = value - } - uri := strings.Split(value, "#/") - - // determine absolute path to this definition - var defRoot string - if strings.HasPrefix(index.specAbsolutePath, "http") { - defRoot = index.specAbsolutePath - } else { - defRoot = filepath.Dir(index.specAbsolutePath) - } - - var componentName string - var fullDefinitionPath string - if len(uri) == 2 { - // Check if we are dealing with a ref to a local definition. - if uri[0] == "" { - fullDefinitionPath = fmt.Sprintf("%s#/%s", index.specAbsolutePath, uri[1]) - componentName = value - } else { - if strings.HasPrefix(uri[0], "http") { - fullDefinitionPath = value - componentName = fmt.Sprintf("#/%s", uri[1]) - } else { - if filepath.IsAbs(uri[0]) { - fullDefinitionPath = value - componentName = fmt.Sprintf("#/%s", uri[1]) - } else { - // if the index has a base URL, use that to resolve the path. - if index.config.BaseURL != nil && !filepath.IsAbs(defRoot) { - var u url.URL - if strings.HasPrefix(defRoot, "http") { - up, _ := url.Parse(defRoot) - up.Path = utils.ReplaceWindowsDriveWithLinuxPath(filepath.Dir(up.Path)) - u = *up - } else { - u = *index.config.BaseURL - } - // abs, _ := filepath.Abs(filepath.Join(u.Path, uri[0])) - // abs, _ := filepath.Abs(utils.CheckPathOverlap(u.Path, uri[0], string(os.PathSeparator))) - abs := utils.CheckPathOverlap(u.Path, uri[0], string(os.PathSeparator)) - u.Path = utils.ReplaceWindowsDriveWithLinuxPath(abs) - fullDefinitionPath = fmt.Sprintf("%s#/%s", u.String(), uri[1]) - componentName = fmt.Sprintf("#/%s", uri[1]) - - } else { - abs := index.resolveRelativeFilePath(defRoot, uri[0]) - fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1]) - componentName = fmt.Sprintf("#/%s", uri[1]) - } - } - } - } - } else { - if strings.HasPrefix(uri[0], "http") { - fullDefinitionPath = value - } else { - // is it a relative file include? - if !strings.Contains(uri[0], "#") { - if strings.HasPrefix(defRoot, "http") { - if !filepath.IsAbs(uri[0]) { - u, _ := url.Parse(defRoot) - pathDir := filepath.Dir(u.Path) - // pathAbs, _ := filepath.Abs(filepath.Join(pathDir, uri[0])) - pathAbs, _ := filepath.Abs(utils.CheckPathOverlap(pathDir, uri[0], string(os.PathSeparator))) - pathAbs = utils.ReplaceWindowsDriveWithLinuxPath(pathAbs) - u.Path = pathAbs - fullDefinitionPath = u.String() - } - } else { - if !filepath.IsAbs(uri[0]) { - // if the index has a base URL, use that to resolve the path. - if index.config.BaseURL != nil { - - u := *index.config.BaseURL - abs := utils.CheckPathOverlap(u.Path, uri[0], string(os.PathSeparator)) - abs = utils.ReplaceWindowsDriveWithLinuxPath(abs) - u.Path = abs - fullDefinitionPath = u.String() - componentName = uri[0] - } else { - abs := index.resolveRelativeFilePath(defRoot, uri[0]) - fullDefinitionPath = abs - componentName = uri[0] - } - } - } - } - } - } - - if fullDefinitionPath == "" && value != "" { - fullDefinitionPath = value - } - if componentName == "" { - componentName = value - } - - _, p := utils.ConvertComponentIdIntoFriendlyPathSearch(componentName) - - // check for sibling properties - siblingProps := make(map[string]*yaml.Node) - var siblingKeys []*yaml.Node - hasSiblings := len(node.Content) > 2 - - if hasSiblings { - for j := 0; j < len(node.Content); j += 2 { - if j+1 < len(node.Content) && node.Content[j].Value != "$ref" { - siblingProps[node.Content[j].Value] = node.Content[j+1] - siblingKeys = append(siblingKeys, node.Content[j]) - } - } - } - - ref := &Reference{ - ParentNode: parent, - FullDefinition: fullDefinitionPath, - Definition: componentName, - RawRef: value, - SchemaIdBase: schemaIdBase, - Name: name, - Node: node, - KeyNode: node.Content[i+1], - Path: p, - Index: index, - IsExtensionRef: isExtensionPath, - HasSiblingProperties: len(siblingProps) > 0, - SiblingProperties: siblingProps, - SiblingKeys: siblingKeys, - } - - // add to raw sequenced refs - index.rawSequencedRefs = append(index.rawSequencedRefs, ref) - - // add ref by line number - refNameIndex := strings.LastIndex(value, "/") - refName := value[refNameIndex+1:] - if len(index.refsByLine[refName]) > 0 { - index.refsByLine[refName][n.Line] = true - } else { - v := make(map[int]bool) - v[n.Line] = true - index.refsByLine[refName] = v - } - - // if this ref value has any siblings (node.Content is larger than two elements) - // then add to refs with siblings - if len(node.Content) > 2 { - copiedNode := *node - - // extract sibling properties and keys - siblingProps := make(map[string]*yaml.Node) - var siblingKeys []*yaml.Node - - for j := 0; j < len(node.Content); j += 2 { - if j+1 < len(node.Content) && node.Content[j].Value != "$ref" { - siblingProps[node.Content[j].Value] = node.Content[j+1] - siblingKeys = append(siblingKeys, node.Content[j]) - } - } - - copied := Reference{ - ParentNode: parent, - FullDefinition: fullDefinitionPath, - Definition: ref.Definition, - RawRef: ref.RawRef, - SchemaIdBase: ref.SchemaIdBase, - Name: ref.Name, - Node: &copiedNode, - KeyNode: node.Content[i], - Path: p, - Index: index, - IsExtensionRef: isExtensionPath, - HasSiblingProperties: len(siblingProps) > 0, - SiblingProperties: siblingProps, - SiblingKeys: siblingKeys, - } - // protect this data using a copy, prevent the resolver from destroying things. - index.refsWithSiblings[value] = copied - } - - // if this is a polymorphic reference, we're going to leave it out - // allRefs. We don't ever want these resolved, so instead of polluting - // the timeline, we will keep each poly ref in its own collection for later - // analysis. - if poly { - index.polymorphicRefs[value] = ref - - // index each type - switch pName { - case "anyOf": - index.polymorphicAnyOfRefs = append(index.polymorphicAnyOfRefs, ref) - case "allOf": - index.polymorphicAllOfRefs = append(index.polymorphicAllOfRefs, ref) - case "oneOf": - index.polymorphicOneOfRefs = append(index.polymorphicOneOfRefs, ref) - } - continue - } - - // check if this is a dupe, if so, skip it, we don't care now. - if index.allRefs[value] != nil { // seen before, skip. - continue - } - - if value == "" { - - completedPath := fmt.Sprintf("$.%s", strings.Join(fp, ".")) - c := node.Content[i] - if len(node.Content) > i+1 { // if the next node exists, use that. - c = node.Content[i+1] - } - - indexError := &IndexingError{ - Err: errors.New("schema reference is empty and cannot be processed"), - Node: c, - KeyNode: node.Content[i], - Path: completedPath, - } - - index.refErrors = append(index.refErrors, indexError) - continue - } - - // This sets the ref in the path using the full URL and sub-path. - index.allRefs[fullDefinitionPath] = ref - found = append(found, ref) - } - } - - // Detect and register JSON Schema 2020-12 $id declarations - if i%2 == 0 && n.Value == "$id" { - if underOpenAPIExamplePath(seenPath) { - continue - } - if len(node.Content) > i+1 && utils.IsNodeStringValue(node.Content[i+1]) { - idValue := node.Content[i+1].Value - idNode := node.Content[i+1] - - // Build the definition path for this schema - var definitionPath string - if len(seenPath) > 0 { - definitionPath = "#/" + strings.Join(seenPath, "/") - } else { - definitionPath = "#" - } - - // Validate the $id (must not contain fragment) - if err := ValidateSchemaId(idValue); err != nil { - index.errorLock.Lock() - index.refErrors = append(index.refErrors, &IndexingError{ - Err: fmt.Errorf("invalid $id value '%s': %w", idValue, err), - Node: idNode, - KeyNode: node.Content[i], - Path: definitionPath, - }) - index.errorLock.Unlock() - continue - } - - // Resolve the $id against the PARENT scope's base URI (nearest ancestor $id) - // This implements JSON Schema 2020-12 hierarchical $id resolution - // We use parentBaseUri which was captured before this node's $id was pushed - baseUri := parentBaseUri - if baseUri == "" { - baseUri = index.specAbsolutePath - } - resolvedUri, resolveErr := ResolveSchemaId(idValue, baseUri) - if resolveErr != nil { - if index.logger != nil { - index.logger.Warn("failed to resolve $id", - "id", idValue, - "base", baseUri, - "definitionPath", definitionPath, - "error", resolveErr.Error(), - "line", idNode.Line) - } - resolvedUri = idValue // Use original as fallback - } - - // Create and register the schema ID entry - // ParentId is the parent scope's base URI (if it differs from document base) - parentId := "" - if parentBaseUri != index.specAbsolutePath && parentBaseUri != "" { - parentId = parentBaseUri - } - entry := &SchemaIdEntry{ - Id: idValue, - ResolvedUri: resolvedUri, - SchemaNode: node, - ParentId: parentId, - Index: index, - DefinitionPath: definitionPath, - Line: idNode.Line, - Column: idNode.Column, - } - - // Register in the index (validation already done above) - _ = index.RegisterSchemaId(entry) - } - } - - // Skip $ref and $id from path building - they are keywords, not schema properties - if i%2 == 0 && n.Value != "$ref" && n.Value != "$id" && n.Value != "" { - - v := n.Value - if strings.HasPrefix(v, "/") { - v = strings.Replace(v, "/", "~1", 1) - } - - // Lazily compute jsonPath — only needed for description, summary, security, enum, properties. - var jsonPath string - var jsonPathComputed bool - computeJsonPath := func() string { - if !jsonPathComputed { - loc := append(seenPath, v) - definitionPath := "#/" + strings.Join(loc, "/") - _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) - jsonPathComputed = true - } - return jsonPath - } - - // capture descriptions and summaries - if n.Value == "description" { - - // if the parent is a sequence, ignore. - if utils.IsNodeArray(node) { - continue - } - // Skip if "description" is a property name inside schema properties - // We check if the previous element in seenPath is "properties" and this is at an even index - // (property names are at even indices, values at odd) - if len(seenPath) > 0 && (seenPath[len(seenPath)-1] == "properties" || seenPath[len(seenPath)-1] == "patternProperties") { - // This means "description" is a property name, not a description field, skip extraction - seenPath = append(seenPath, strings.ReplaceAll(n.Value, "/", "~1")) - prev = n.Value - continue - } - if !underOpenAPIExamplePath(seenPath) { - ref := &DescriptionReference{ - ParentNode: parent, - Content: node.Content[i+1].Value, - Path: computeJsonPath(), - Node: node.Content[i+1], - KeyNode: node.Content[i], - IsSummary: false, - } - - if !utils.IsNodeMap(ref.Node) { - index.allDescriptions = append(index.allDescriptions, ref) - index.descriptionCount++ - } - } - } - - if n.Value == "summary" { - - // Skip if "summary" is a property name inside schema properties - // We check if the previous element in seenPath is "properties" and this is at an even index - // (property names are at even indices, values at odd) - if len(seenPath) > 0 && (seenPath[len(seenPath)-1] == "properties" || seenPath[len(seenPath)-1] == "patternProperties") { - // This means "summary" is a property name, not a summary field, skip extraction - seenPath = append(seenPath, strings.ReplaceAll(n.Value, "/", "~1")) - prev = n.Value - continue - } - - if underOpenAPIExamplePath(seenPath) { - continue - } - - var b *yaml.Node - if len(node.Content) == i+1 { - b = node.Content[i] - } else { - b = node.Content[i+1] - } - ref := &DescriptionReference{ - ParentNode: parent, - Content: b.Value, - Path: computeJsonPath(), - Node: b, - KeyNode: n, - IsSummary: true, - } - - index.allSummaries = append(index.allSummaries, ref) - index.summaryCount++ - } - - // capture security requirement references (these are not traditional references, but they - // are used as a look-up. This is the only exception to the design. - if n.Value == "security" { - var b *yaml.Node - if len(node.Content) == i+1 { - b = node.Content[i] - } else { - b = node.Content[i+1] - } - if utils.IsNodeArray(b) { - var secKey string - for k := range b.Content { - if utils.IsNodeMap(b.Content[k]) { - for g := range b.Content[k].Content { - if g%2 == 0 { - secKey = b.Content[k].Content[g].Value - continue - } - if utils.IsNodeArray(b.Content[k].Content[g]) { - var refMap map[string][]*Reference - if index.securityRequirementRefs[secKey] == nil { - index.securityRequirementRefs[secKey] = make(map[string][]*Reference) - refMap = index.securityRequirementRefs[secKey] - } else { - refMap = index.securityRequirementRefs[secKey] - } - for r := range b.Content[k].Content[g].Content { - var refs []*Reference - if refMap[b.Content[k].Content[g].Content[r].Value] != nil { - refs = refMap[b.Content[k].Content[g].Content[r].Value] - } - - refs = append(refs, &Reference{ - Definition: b.Content[k].Content[g].Content[r].Value, - Path: fmt.Sprintf("%s.security[%d].%s[%d]", computeJsonPath(), k, secKey, r), - Node: b.Content[k].Content[g].Content[r], - KeyNode: b.Content[k].Content[g], - }) - - index.securityRequirementRefs[secKey][b.Content[k].Content[g].Content[r].Value] = refs - } - } - } - } - } - } - } - // capture enums - if n.Value == "enum" { - - if len(seenPath) > 0 { - lastItem := seenPath[len(seenPath)-1] - if lastItem == "properties" { - seenPath = append(seenPath, strings.ReplaceAll(n.Value, "/", "~1")) - prev = n.Value - continue - } - } - - // all enums need to have a type, extract the type from the node where the enum was found. - _, enumKeyValueNode := utils.FindKeyNodeTop("type", node.Content) - - if enumKeyValueNode != nil { - ref := &EnumReference{ - ParentNode: parent, - Path: computeJsonPath(), - Node: node.Content[i+1], - KeyNode: node.Content[i], - Type: enumKeyValueNode, - SchemaNode: node, - } - - index.allEnums = append(index.allEnums, ref) - index.enumCount++ - } - } - // capture all objects with properties - if n.Value == "properties" { - _, typeKeyValueNode := utils.FindKeyNodeTop("type", node.Content) - - if typeKeyValueNode != nil { - isObject := false - - if typeKeyValueNode.Value == "object" { - isObject = true - } - - for _, v := range typeKeyValueNode.Content { - if v.Value == "object" { - isObject = true - } - } - - if isObject { - index.allObjectsWithProperties = append(index.allObjectsWithProperties, &ObjectReference{ - Path: computeJsonPath(), - Node: node, - KeyNode: n, - ParentNode: parent, - }) - } - } - } - - seenPath = append(seenPath, strings.ReplaceAll(n.Value, "/", "~1")) - // seenPath = append(seenPath, n.Value) - prev = n.Value - } - - // if next node is map, don't add segment. - if i < len(node.Content)-1 { - next := node.Content[i+1] - - if i%2 != 0 && next != nil && !utils.IsNodeArray(next) && !utils.IsNodeMap(next) && len(seenPath) > 0 { - seenPath = seenPath[:len(seenPath)-1] - } - } - } - } - + state := index.initializeExtractRefsState(ctx, node, seenPath, level, poly, pName) + found := index.walkExtractRefs(node, parent, &state) index.refCount = len(index.allRefs) - - return found -} - -// ExtractComponentsFromRefs returns located components from references. The returned nodes from here -// can be used for resolving as they contain the actual object properties. -// -// This function uses singleflight to deduplicate concurrent lookups for the same reference, -// channel-based collection to avoid mutex contention during resolution, and sorts results -// by input position for deterministic ordering. - -// isExternalReference checks whether a Reference originated from an external $ref. -// ref.Definition may have been transformed (e.g., HTTP URL with fragment becomes "#/fragment"), -// so we also check the original raw ref value. -func isExternalReference(ref *Reference) bool { - if ref == nil { - return false - } - return utils.IsExternalRef(ref.Definition) || utils.IsExternalRef(ref.RawRef) -} - -func (index *SpecIndex) ExtractComponentsFromRefs(ctx context.Context, refs []*Reference) []*Reference { - if len(refs) == 0 { - return nil - } - - refsToCheck := refs - mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck)) - - // Sequential mode: process refs one at a time (used for bundling) - if index.config.ExtractRefsSequentially { - found := make([]*Reference, 0, len(refsToCheck)) - for i, ref := range refsToCheck { - located := index.locateRef(ctx, ref) - if located != nil { - index.refLock.Lock() - if index.allMappedRefs[located.FullDefinition] == nil { - index.allMappedRefs[located.FullDefinition] = located - found = append(found, located) - } - mappedRefsInSequence[i] = &ReferenceMapped{ - OriginalReference: ref, - Reference: located, - Definition: located.Definition, - FullDefinition: located.FullDefinition, - } - index.refLock.Unlock() - } else { - // If SkipExternalRefResolution is enabled, don't record errors for external refs - if index.config != nil && index.config.SkipExternalRefResolution && isExternalReference(ref) { - continue - } - // Record error for definitive failure - _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition) - index.errorLock.Lock() - index.refErrors = append(index.refErrors, &IndexingError{ - Err: fmt.Errorf("component `%s` does not exist in the specification", ref.Definition), - Node: ref.Node, - Path: path, - KeyNode: ref.KeyNode, - }) - index.errorLock.Unlock() - } - } - // Collect sequenced results - for _, rm := range mappedRefsInSequence { - if rm != nil { - index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, rm) - } - } - return found - } - - // Async mode: use singleflight for deduplication and channel-based collection - var wg sync.WaitGroup - var sfGroup singleflight.Group // Local to this call - no cross-index coupling - - // Channel-based collection - no mutex needed during resolution - resultsChan := make(chan indexedRef, len(refsToCheck)) - - // Concurrency control - maxConcurrency := runtime.GOMAXPROCS(0) - if maxConcurrency < 4 { - maxConcurrency = 4 - } - sem := make(chan struct{}, maxConcurrency) - - for i, ref := range refsToCheck { - i, ref := i, ref // capture loop variables - wg.Add(1) - - go func() { - sem <- struct{}{} - defer func() { <-sem }() - defer wg.Done() - - // Singleflight deduplication - one lookup per FullDefinition - result, _, _ := sfGroup.Do(ref.FullDefinition, func() (interface{}, error) { - // Fast path: already mapped - index.refLock.RLock() - if existing := index.allMappedRefs[ref.FullDefinition]; existing != nil { - index.refLock.RUnlock() - return existing, nil - } - index.refLock.RUnlock() - - // Do the actual lookup (only one goroutine per FullDefinition) - return index.locateRef(ctx, ref), nil - }) - - // Type assert and check for nil - interface containing nil pointer is not nil - located := result.(*Reference) - if located != nil { - resultsChan <- indexedRef{ref: located, pos: i} - } else { - resultsChan <- indexedRef{ref: nil, pos: i} // Track failures for reconciliation - } - }() - } - - // Close channel after all goroutines complete - go func() { - wg.Wait() - close(resultsChan) - }() - - // Collect results - single consumer, no lock needed - collected := make([]indexedRef, 0, len(refsToCheck)) - for r := range resultsChan { - collected = append(collected, r) - } - - // Sort by input position for deterministic ordering - if !preserveLegacyRefOrder { - sort.Slice(collected, func(i, j int) bool { - return collected[i].pos < collected[j].pos - }) - } - - // RECONCILIATION PHASE: Build final results with minimal locking - found := make([]*Reference, 0, len(collected)) - - for _, c := range collected { - ref := refsToCheck[c.pos] - located := c.ref - - // Reconcile nil results - check if another goroutine succeeded. - // We use ref.FullDefinition here because that's the singleflight key, - // and located.FullDefinition should match ref.FullDefinition for the - // same reference (FindComponent returns the component at that definition). - if located == nil { - index.refLock.RLock() - located = index.allMappedRefs[ref.FullDefinition] - index.refLock.RUnlock() - } - - if located != nil { - // Add to allMappedRefs if not present - index.refLock.Lock() - if index.allMappedRefs[located.FullDefinition] == nil { - index.allMappedRefs[located.FullDefinition] = located - found = append(found, located) - } - mappedRefsInSequence[c.pos] = &ReferenceMapped{ - OriginalReference: ref, - Reference: located, - Definition: located.Definition, - FullDefinition: located.FullDefinition, - } - index.refLock.Unlock() - } else { - // If SkipExternalRefResolution is enabled, don't record errors for external refs - if index.config != nil && index.config.SkipExternalRefResolution && isExternalReference(ref) { - continue - } - // Definitive failure - record error - _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition) - index.errorLock.Lock() - index.refErrors = append(index.refErrors, &IndexingError{ - Err: fmt.Errorf("component `%s` does not exist in the specification", ref.Definition), - Node: ref.Node, - Path: path, - KeyNode: ref.KeyNode, - }) - index.errorLock.Unlock() - } - } - - // Collect sequenced results in input order - for _, rm := range mappedRefsInSequence { - if rm != nil { - index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, rm) - } - } - return found } - -// locateRef finds a component for a reference, including KeyNode extraction. -// This is a helper used by ExtractComponentsFromRefs to isolate the lookup logic. -func (index *SpecIndex) locateRef(ctx context.Context, ref *Reference) *Reference { - // External references require a full Lock (not RLock) during FindComponent because - // FindComponent may trigger rolodex file loading which mutates index state. - // Internal references can proceed without locking since they only read from - // already-populated data structures. - uri := strings.Split(ref.FullDefinition, "#/") - isExternalRef := len(uri) == 2 && len(uri[0]) > 0 - if isExternalRef { - index.refLock.Lock() - } - located := index.FindComponent(ctx, ref.FullDefinition) - if isExternalRef { - index.refLock.Unlock() - } - - if located == nil { - rawRef := ref.RawRef - if rawRef == "" { - rawRef = ref.FullDefinition - } - normalizedRef := resolveRefWithSchemaBase(rawRef, ref.SchemaIdBase) - if resolved := index.ResolveRefViaSchemaId(normalizedRef); resolved != nil { - located = resolved - } else { - return nil - } - } - - // Extract KeyNode - yamlpath API returns subnodes only, so we need to - // rollback in the nodemap a line (if we can) to extract the keynode. - if located.Node != nil { - index.nodeMapLock.RLock() - if located.Node.Line > 1 && len(index.nodeMap[located.Node.Line-1]) > 0 { - for _, v := range index.nodeMap[located.Node.Line-1] { - located.KeyNode = v - break - } - } - index.nodeMapLock.RUnlock() - } - - return located -} diff --git a/index/extract_refs_inline.go b/index/extract_refs_inline.go new file mode 100644 index 000000000..3be039539 --- /dev/null +++ b/index/extract_refs_inline.go @@ -0,0 +1,164 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "strconv" + "strings" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +func (index *SpecIndex) collectInlineSchemaDefinition(parent, node *yaml.Node, seenPath []string, keyIndex int) { + if keyIndex+1 >= len(node.Content) { + return + } + + keyNode := node.Content[keyIndex] + valueNode := node.Content[keyIndex+1] + + var jsonPath, definitionPath, fullDefinitionPath string + if len(seenPath) > 0 || keyNode.Value != "" { + loc := append(seenPath, keyNode.Value) + locPath := strings.Join(loc, "/") + definitionPath = "#/" + locPath + fullDefinitionPath = index.specAbsolutePath + "#/" + locPath + _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) + } + + ref := &Reference{ + ParentNode: parent, + FullDefinition: fullDefinitionPath, + Definition: definitionPath, + Node: valueNode, + KeyNode: keyNode, + Path: jsonPath, + Index: index, + } + + isRef, _, _ := utils.IsNodeRefValue(valueNode) + if isRef { + index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref) + return + } + + if (keyNode.Value == "additionalProperties" || keyNode.Value == "unevaluatedProperties") && utils.IsNodeBoolValue(valueNode) { + return + } + + index.appendInlineSchemaDefinition(ref) +} + +func (index *SpecIndex) collectMapSchemaDefinitions(parent, node *yaml.Node, seenPath []string, keyIndex int) { + if keyIndex+1 >= len(node.Content) { + return + } + + keyNode := node.Content[keyIndex] + propertiesNode := node.Content[keyIndex+1] + + if len(seenPath) > 0 { + for _, p := range seenPath { + if p == "examples" || p == "example" || strings.HasPrefix(p, "x-") { + return + } + } + } + + label := "" + for h, prop := range propertiesNode.Content { + if h%2 == 0 { + label = prop.Value + continue + } + + var jsonPath, definitionPath, fullDefinitionPath string + if len(seenPath) > 0 || keyNode.Value != "" && label != "" { + loc := append(seenPath, keyNode.Value, label) + locPath := strings.Join(loc, "/") + definitionPath = "#/" + locPath + fullDefinitionPath = index.specAbsolutePath + "#/" + locPath + _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) + } + + ref := &Reference{ + ParentNode: parent, + FullDefinition: fullDefinitionPath, + Definition: definitionPath, + Node: prop, + KeyNode: keyNode, + Path: jsonPath, + Index: index, + } + + isRef, _, _ := utils.IsNodeRefValue(prop) + if isRef { + index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref) + continue + } + + index.appendInlineSchemaDefinition(ref) + } +} + +func (index *SpecIndex) collectArraySchemaDefinitions(parent, node *yaml.Node, seenPath []string, keyIndex int) { + if keyIndex+1 >= len(node.Content) { + return + } + + keyNode := node.Content[keyIndex] + arrayNode := node.Content[keyIndex+1] + + for h, element := range arrayNode.Content { + var jsonPath, definitionPath, fullDefinitionPath string + if len(seenPath) > 0 { + loc := append(seenPath, keyNode.Value, strconv.Itoa(h)) + locPath := strings.Join(loc, "/") + definitionPath = "#/" + locPath + fullDefinitionPath = index.specAbsolutePath + "#/" + locPath + _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) + } else { + definitionPath = "#/" + keyNode.Value + fullDefinitionPath = index.specAbsolutePath + "#/" + keyNode.Value + _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) + } + + ref := &Reference{ + ParentNode: parent, + FullDefinition: fullDefinitionPath, + Definition: definitionPath, + Node: element, + KeyNode: keyNode, + Path: jsonPath, + Index: index, + } + + isRef, _, _ := utils.IsNodeRefValue(element) + if isRef { + index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref) + continue + } + + index.appendInlineSchemaDefinition(ref) + } +} + +func (index *SpecIndex) appendInlineSchemaDefinition(ref *Reference) { + index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref) + if inlineSchemaIsObjectOrArray(ref.Node) { + index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref) + } +} + +func inlineSchemaIsObjectOrArray(node *yaml.Node) bool { + if node == nil { + return false + } + k, v := utils.FindKeyNodeTop("type", node.Content) + if k == nil || v == nil { + return false + } + return v.Value == "object" || v.Value == "array" +} diff --git a/index/extract_refs_lookup.go b/index/extract_refs_lookup.go new file mode 100644 index 000000000..49a0756cf --- /dev/null +++ b/index/extract_refs_lookup.go @@ -0,0 +1,235 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "context" + "fmt" + "os" + "runtime" + "sort" + "strings" + "sync" + + "github.com/pb33f/libopenapi/utils" + "golang.org/x/sync/singleflight" +) + +// preserveLegacyRefOrder allows opt-out of deterministic ordering if issues arise. +// Set LIBOPENAPI_LEGACY_REF_ORDER=true to use the old non-deterministic ordering. +var preserveLegacyRefOrder = os.Getenv("LIBOPENAPI_LEGACY_REF_ORDER") == "true" + +// indexedRef pairs a resolved reference with its original input position for deterministic ordering. +type indexedRef struct { + ref *Reference + pos int +} + +// ExtractComponentsFromRefs returns located components from references. The returned nodes from here +// can be used for resolving as they contain the actual object properties. +// +// This function uses singleflight to deduplicate concurrent lookups for the same reference, +// channel-based collection to avoid mutex contention during resolution, and sorts results +// by input position for deterministic ordering. + +// isExternalReference checks whether a Reference originated from an external $ref. +// ref.Definition may have been transformed (e.g., HTTP URL with fragment becomes "#/fragment"), +// so we also check the original raw ref value. +func isExternalReference(ref *Reference) bool { + if ref == nil { + return false + } + return utils.IsExternalRef(ref.Definition) || utils.IsExternalRef(ref.RawRef) +} + +func (index *SpecIndex) ExtractComponentsFromRefs(ctx context.Context, refs []*Reference) []*Reference { + if len(refs) == 0 { + return nil + } + + refsToCheck := refs + mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck)) + + if index.config.ExtractRefsSequentially { + found := make([]*Reference, 0, len(refsToCheck)) + for i, ref := range refsToCheck { + located := index.locateRef(ctx, ref) + if located != nil { + index.refLock.Lock() + if index.allMappedRefs[located.FullDefinition] == nil { + index.allMappedRefs[located.FullDefinition] = located + found = append(found, located) + } + mappedRefsInSequence[i] = &ReferenceMapped{ + OriginalReference: ref, + Reference: located, + Definition: located.Definition, + FullDefinition: located.FullDefinition, + } + index.refLock.Unlock() + } else { + if index.config != nil && index.config.SkipExternalRefResolution && isExternalReference(ref) { + continue + } + _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition) + index.errorLock.Lock() + index.refErrors = append(index.refErrors, &IndexingError{ + Err: fmt.Errorf("component `%s` does not exist in the specification", ref.Definition), + Node: ref.Node, + Path: path, + KeyNode: ref.KeyNode, + }) + index.errorLock.Unlock() + } + } + for _, rm := range mappedRefsInSequence { + if rm != nil { + index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, rm) + } + } + return found + } + + var wg sync.WaitGroup + var sfGroup singleflight.Group + resultsChan := make(chan indexedRef, len(refsToCheck)) + + maxConcurrency := runtime.GOMAXPROCS(0) + if maxConcurrency < 4 { + maxConcurrency = 4 + } + sem := make(chan struct{}, maxConcurrency) + + for i, ref := range refsToCheck { + i, ref := i, ref + wg.Add(1) + + go func() { + sem <- struct{}{} + defer func() { <-sem }() + defer wg.Done() + + result, _, _ := sfGroup.Do(ref.FullDefinition, func() (interface{}, error) { + index.refLock.RLock() + if existing := index.allMappedRefs[ref.FullDefinition]; existing != nil { + index.refLock.RUnlock() + return existing, nil + } + index.refLock.RUnlock() + + return index.locateRef(ctx, ref), nil + }) + + located := result.(*Reference) + if located != nil { + resultsChan <- indexedRef{ref: located, pos: i} + } else { + resultsChan <- indexedRef{ref: nil, pos: i} + } + }() + } + + go func() { + wg.Wait() + close(resultsChan) + }() + + collected := make([]indexedRef, 0, len(refsToCheck)) + for r := range resultsChan { + collected = append(collected, r) + } + + if !preserveLegacyRefOrder { + sort.Slice(collected, func(i, j int) bool { + return collected[i].pos < collected[j].pos + }) + } + + found := make([]*Reference, 0, len(collected)) + + for _, c := range collected { + ref := refsToCheck[c.pos] + located := c.ref + + if located == nil { + index.refLock.RLock() + located = index.allMappedRefs[ref.FullDefinition] + index.refLock.RUnlock() + } + + if located != nil { + index.refLock.Lock() + if index.allMappedRefs[located.FullDefinition] == nil { + index.allMappedRefs[located.FullDefinition] = located + found = append(found, located) + } + mappedRefsInSequence[c.pos] = &ReferenceMapped{ + OriginalReference: ref, + Reference: located, + Definition: located.Definition, + FullDefinition: located.FullDefinition, + } + index.refLock.Unlock() + } else { + if index.config != nil && index.config.SkipExternalRefResolution && isExternalReference(ref) { + continue + } + _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition) + index.errorLock.Lock() + index.refErrors = append(index.refErrors, &IndexingError{ + Err: fmt.Errorf("component `%s` does not exist in the specification", ref.Definition), + Node: ref.Node, + Path: path, + KeyNode: ref.KeyNode, + }) + index.errorLock.Unlock() + } + } + + for _, rm := range mappedRefsInSequence { + if rm != nil { + index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, rm) + } + } + + return found +} + +func (index *SpecIndex) locateRef(ctx context.Context, ref *Reference) *Reference { + uri := strings.Split(ref.FullDefinition, "#/") + isExternalRef := len(uri) == 2 && len(uri[0]) > 0 + if isExternalRef { + index.refLock.Lock() + } + located := index.FindComponent(ctx, ref.FullDefinition) + if isExternalRef { + index.refLock.Unlock() + } + + if located == nil { + rawRef := ref.RawRef + if rawRef == "" { + rawRef = ref.FullDefinition + } + normalizedRef := resolveRefWithSchemaBase(rawRef, ref.SchemaIdBase) + if resolved := index.ResolveRefViaSchemaId(normalizedRef); resolved != nil { + located = resolved + } else { + return nil + } + } + + if located.Node != nil { + index.nodeMapLock.RLock() + if located.Node.Line > 1 && len(index.nodeMap[located.Node.Line-1]) > 0 { + for _, v := range index.nodeMap[located.Node.Line-1] { + located.KeyNode = v + break + } + } + index.nodeMapLock.RUnlock() + } + + return located +} diff --git a/index/extract_refs_metadata.go b/index/extract_refs_metadata.go new file mode 100644 index 000000000..06d582f61 --- /dev/null +++ b/index/extract_refs_metadata.go @@ -0,0 +1,204 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "fmt" + "strings" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +type metadataPathAction struct { + appendSegment bool + stop bool +} + +func (index *SpecIndex) extractNodeMetadata(node, parent *yaml.Node, seenPath []string, keyIndex int) metadataPathAction { + keyNode := node.Content[keyIndex] + if keyNode == nil || keyNode.Value == "" || keyNode.Value == "$ref" || keyNode.Value == "$id" { + return metadataPathAction{} + } + + segment := keyNode.Value + if strings.HasPrefix(segment, "/") { + segment = strings.Replace(segment, "/", "~1", 1) + } + + var jsonPath string + var jsonPathComputed bool + computeJSONPath := func() string { + if !jsonPathComputed { + loc := append(seenPath, segment) + definitionPath := "#/" + strings.Join(loc, "/") + _, jsonPath = utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath) + jsonPathComputed = true + } + return jsonPath + } + + switch keyNode.Value { + case "description": + if utils.IsNodeArray(node) { + return metadataPathAction{stop: true} + } + if isMetadataPropertyNamePath(seenPath) { + return metadataPathAction{appendSegment: true, stop: true} + } + if !metadataPathContainsExamples(seenPath) { + refNode := metadataValueNode(node, keyIndex) + ref := &DescriptionReference{ + ParentNode: parent, + Content: refNode.Value, + Path: computeJSONPath(), + Node: refNode, + KeyNode: keyNode, + IsSummary: false, + } + if !utils.IsNodeMap(ref.Node) { + index.allDescriptions = append(index.allDescriptions, ref) + index.descriptionCount++ + } + } + case "summary": + if isMetadataPropertyNamePath(seenPath) { + return metadataPathAction{appendSegment: true, stop: true} + } + if metadataPathContainsExamples(seenPath) { + return metadataPathAction{stop: true} + } + refNode := metadataValueNode(node, keyIndex) + index.allSummaries = append(index.allSummaries, &DescriptionReference{ + ParentNode: parent, + Content: refNode.Value, + Path: computeJSONPath(), + Node: refNode, + KeyNode: keyNode, + IsSummary: true, + }) + index.summaryCount++ + case "security": + index.collectSecurityRequirementMetadata(node, keyIndex, computeJSONPath()) + case "enum": + if len(seenPath) > 0 && seenPath[len(seenPath)-1] == "properties" { + return metadataPathAction{appendSegment: true, stop: true} + } + index.collectEnumMetadata(node, parent, keyIndex, computeJSONPath()) + case "properties": + index.collectObjectWithPropertiesMetadata(node, parent, keyNode, computeJSONPath()) + } + + return metadataPathAction{appendSegment: true} +} + +func metadataValueNode(node *yaml.Node, keyIndex int) *yaml.Node { + if len(node.Content) == keyIndex+1 { + return node.Content[keyIndex] + } + return node.Content[keyIndex+1] +} + +func isMetadataPropertyNamePath(seenPath []string) bool { + if len(seenPath) == 0 { + return false + } + last := seenPath[len(seenPath)-1] + return last == "properties" || last == "patternProperties" +} + +func metadataPathContainsExamples(seenPath []string) bool { + return underOpenAPIExamplePath(seenPath) +} + +func (index *SpecIndex) collectSecurityRequirementMetadata(node *yaml.Node, keyIndex int, basePath string) { + if index.securityRequirementRefs == nil { + index.securityRequirementRefs = make(map[string]map[string][]*Reference) + } + securityNode := metadataValueNode(node, keyIndex) + if securityNode == nil || !utils.IsNodeArray(securityNode) { + return + } + var secKey string + for k := range securityNode.Content { + if !utils.IsNodeMap(securityNode.Content[k]) { + continue + } + for g := range securityNode.Content[k].Content { + if g%2 == 0 { + secKey = securityNode.Content[k].Content[g].Value + continue + } + if !utils.IsNodeArray(securityNode.Content[k].Content[g]) { + continue + } + var refMap map[string][]*Reference + if index.securityRequirementRefs[secKey] == nil { + index.securityRequirementRefs[secKey] = make(map[string][]*Reference) + refMap = index.securityRequirementRefs[secKey] + } else { + refMap = index.securityRequirementRefs[secKey] + } + for r := range securityNode.Content[k].Content[g].Content { + valueNode := securityNode.Content[k].Content[g].Content[r] + var refs []*Reference + if refMap[valueNode.Value] != nil { + refs = refMap[valueNode.Value] + } + refs = append(refs, &Reference{ + Definition: valueNode.Value, + Path: fmt.Sprintf("%s.security[%d].%s[%d]", basePath, k, secKey, r), + Node: valueNode, + KeyNode: securityNode.Content[k].Content[g], + }) + index.securityRequirementRefs[secKey][valueNode.Value] = refs + } + } + } +} + +func (index *SpecIndex) collectEnumMetadata(node, parent *yaml.Node, keyIndex int, jsonPath string) { + if keyIndex+1 >= len(node.Content) { + return + } + _, enumTypeNode := utils.FindKeyNodeTop("type", node.Content) + if enumTypeNode == nil { + return + } + index.allEnums = append(index.allEnums, &EnumReference{ + ParentNode: parent, + Path: jsonPath, + Node: node.Content[keyIndex+1], + KeyNode: node.Content[keyIndex], + Type: enumTypeNode, + SchemaNode: node, + }) + index.enumCount++ +} + +func (index *SpecIndex) collectObjectWithPropertiesMetadata(node, parent, keyNode *yaml.Node, jsonPath string) { + _, typeKeyValueNode := utils.FindKeyNodeTop("type", node.Content) + if typeKeyValueNode == nil { + return + } + + isObject := typeKeyValueNode.Value == "object" + if !isObject { + for _, valueNode := range typeKeyValueNode.Content { + if valueNode.Value == "object" { + isObject = true + break + } + } + } + + if isObject { + index.allObjectsWithProperties = append(index.allObjectsWithProperties, &ObjectReference{ + Path: jsonPath, + Node: node, + KeyNode: keyNode, + ParentNode: parent, + }) + } +} diff --git a/index/extract_refs_ref.go b/index/extract_refs_ref.go new file mode 100644 index 000000000..45037678e --- /dev/null +++ b/index/extract_refs_ref.go @@ -0,0 +1,332 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +func (index *SpecIndex) extractReferenceAt( + node, parent *yaml.Node, + keyIndex int, + seenPath []string, + scope *SchemaIdScope, + poly bool, + pName string, +) *Reference { + keyNode := node.Content[keyIndex] + if len(node.Content) <= keyIndex+1 { + return nil + } + + isExtensionPath := false + for _, spi := range seenPath { + if strings.HasPrefix(spi, "x-") { + isExtensionPath = true + break + } + } + + if index.config.ExcludeExtensionRefs && isExtensionPath { + return nil + } + + if len(node.Content) > keyIndex+1 { + if !utils.IsNodeStringValue(node.Content[keyIndex+1]) { + return nil + } + if utils.IsNodeArray(node) { + return nil + } + } + + index.linesWithRefs[keyNode.Line] = true + + value := node.Content[keyIndex+1].Value + schemaIdBase := "" + if scope != nil && len(scope.Chain) > 0 { + schemaIdBase = scope.BaseUri + } + + lastSlash := strings.LastIndexByte(value, '/') + name := value + if lastSlash >= 0 { + name = value[lastSlash+1:] + } + + fullDefinitionPath, componentName := index.resolveReferenceTarget(value) + _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(componentName) + + siblingProps, siblingKeys := extractSiblingRefProperties(node) + ref := &Reference{ + ParentNode: parent, + FullDefinition: fullDefinitionPath, + Definition: componentName, + RawRef: value, + SchemaIdBase: schemaIdBase, + Name: name, + Node: node, + KeyNode: node.Content[keyIndex+1], + Path: path, + Index: index, + IsExtensionRef: isExtensionPath, + HasSiblingProperties: len(siblingProps) > 0, + SiblingProperties: siblingProps, + SiblingKeys: siblingKeys, + } + + index.rawSequencedRefs = append(index.rawSequencedRefs, ref) + index.recordReferenceByLine(value, keyNode.Line) + + if len(node.Content) > 2 { + index.storeReferenceWithSiblings(node, parent, keyIndex, ref, isExtensionPath, path, value) + } + + if poly { + index.polymorphicRefs[value] = ref + switch pName { + case "anyOf": + index.polymorphicAnyOfRefs = append(index.polymorphicAnyOfRefs, ref) + case "allOf": + index.polymorphicAllOfRefs = append(index.polymorphicAllOfRefs, ref) + case "oneOf": + index.polymorphicOneOfRefs = append(index.polymorphicOneOfRefs, ref) + } + return nil + } + + if index.allRefs[value] != nil { + return nil + } + + if value == "" { + fp := make([]string, len(seenPath)) + copy(fp, seenPath) + + completedPath := fmt.Sprintf("$.%s", strings.Join(fp, ".")) + c := keyNode + if len(node.Content) > keyIndex+1 { + c = node.Content[keyIndex+1] + } + + index.refErrors = append(index.refErrors, &IndexingError{ + Err: errors.New("schema reference is empty and cannot be processed"), + Node: c, + KeyNode: keyNode, + Path: completedPath, + }) + return nil + } + + index.allRefs[fullDefinitionPath] = ref + return ref +} + +func (index *SpecIndex) registerSchemaIDAt(node *yaml.Node, keyIndex int, seenPath []string, parentBaseUri string) { + if underOpenAPIExamplePath(seenPath) { + return + } + if len(node.Content) <= keyIndex+1 || !utils.IsNodeStringValue(node.Content[keyIndex+1]) { + return + } + + idValue := node.Content[keyIndex+1].Value + idNode := node.Content[keyIndex+1] + + definitionPath := "#" + if len(seenPath) > 0 { + definitionPath = "#/" + strings.Join(seenPath, "/") + } + + if err := ValidateSchemaId(idValue); err != nil { + index.errorLock.Lock() + index.refErrors = append(index.refErrors, &IndexingError{ + Err: fmt.Errorf("invalid $id value '%s': %w", idValue, err), + Node: idNode, + KeyNode: node.Content[keyIndex], + Path: definitionPath, + }) + index.errorLock.Unlock() + return + } + + baseUri := parentBaseUri + if baseUri == "" { + baseUri = index.specAbsolutePath + } + resolvedUri, resolveErr := ResolveSchemaId(idValue, baseUri) + if resolveErr != nil { + if index.logger != nil { + index.logger.Warn("failed to resolve $id", + "id", idValue, + "base", baseUri, + "definitionPath", definitionPath, + "error", resolveErr.Error(), + "line", idNode.Line) + } + resolvedUri = idValue + } + + parentId := "" + if parentBaseUri != index.specAbsolutePath && parentBaseUri != "" { + parentId = parentBaseUri + } + entry := &SchemaIdEntry{ + Id: idValue, + ResolvedUri: resolvedUri, + SchemaNode: node, + ParentId: parentId, + Index: index, + DefinitionPath: definitionPath, + Line: idNode.Line, + Column: idNode.Column, + } + + _ = index.RegisterSchemaId(entry) +} + +func (index *SpecIndex) resolveReferenceTarget(value string) (string, string) { + uri := strings.Split(value, "#/") + + var defRoot string + if strings.HasPrefix(index.specAbsolutePath, "http") { + defRoot = index.specAbsolutePath + } else { + defRoot = filepath.Dir(index.specAbsolutePath) + } + + var componentName string + var fullDefinitionPath string + if len(uri) == 2 { + if uri[0] == "" { + fullDefinitionPath = fmt.Sprintf("%s#/%s", index.specAbsolutePath, uri[1]) + componentName = value + } else { + if strings.HasPrefix(uri[0], "http") { + fullDefinitionPath = value + componentName = fmt.Sprintf("#/%s", uri[1]) + } else if filepath.IsAbs(uri[0]) { + fullDefinitionPath = value + componentName = fmt.Sprintf("#/%s", uri[1]) + } else if index.config.BaseURL != nil && !filepath.IsAbs(defRoot) { + var u url.URL + if strings.HasPrefix(defRoot, "http") { + up, _ := url.Parse(defRoot) + up.Path = utils.ReplaceWindowsDriveWithLinuxPath(filepath.Dir(up.Path)) + u = *up + } else { + u = *index.config.BaseURL + } + abs := utils.CheckPathOverlap(u.Path, uri[0], string(os.PathSeparator)) + u.Path = utils.ReplaceWindowsDriveWithLinuxPath(abs) + fullDefinitionPath = fmt.Sprintf("%s#/%s", u.String(), uri[1]) + componentName = fmt.Sprintf("#/%s", uri[1]) + } else { + abs := index.resolveRelativeFilePath(defRoot, uri[0]) + fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1]) + componentName = fmt.Sprintf("#/%s", uri[1]) + } + } + } else if strings.HasPrefix(uri[0], "http") { + fullDefinitionPath = value + } else if !strings.Contains(uri[0], "#") { + if strings.HasPrefix(defRoot, "http") { + if !filepath.IsAbs(uri[0]) { + u, _ := url.Parse(defRoot) + pathDir := filepath.Dir(u.Path) + pathAbs, _ := filepath.Abs(utils.CheckPathOverlap(pathDir, uri[0], string(os.PathSeparator))) + pathAbs = utils.ReplaceWindowsDriveWithLinuxPath(pathAbs) + u.Path = pathAbs + fullDefinitionPath = u.String() + } + } else if !filepath.IsAbs(uri[0]) { + if index.config.BaseURL != nil { + u := *index.config.BaseURL + abs := utils.CheckPathOverlap(u.Path, uri[0], string(os.PathSeparator)) + abs = utils.ReplaceWindowsDriveWithLinuxPath(abs) + u.Path = abs + fullDefinitionPath = u.String() + componentName = uri[0] + } else { + abs := index.resolveRelativeFilePath(defRoot, uri[0]) + fullDefinitionPath = abs + componentName = uri[0] + } + } + } + + if fullDefinitionPath == "" && value != "" { + fullDefinitionPath = value + } + if componentName == "" { + componentName = value + } + + return fullDefinitionPath, componentName +} + +func (index *SpecIndex) recordReferenceByLine(value string, line int) { + refNameIndex := strings.LastIndex(value, "/") + refName := value[refNameIndex+1:] + if len(index.refsByLine[refName]) > 0 { + index.refsByLine[refName][line] = true + return + } + v := make(map[int]bool) + v[line] = true + index.refsByLine[refName] = v +} + +func extractSiblingRefProperties(node *yaml.Node) (map[string]*yaml.Node, []*yaml.Node) { + siblingProps := make(map[string]*yaml.Node) + var siblingKeys []*yaml.Node + if len(node.Content) <= 2 { + return siblingProps, siblingKeys + } + for j := 0; j < len(node.Content); j += 2 { + if j+1 < len(node.Content) && node.Content[j].Value != "$ref" { + siblingProps[node.Content[j].Value] = node.Content[j+1] + siblingKeys = append(siblingKeys, node.Content[j]) + } + } + return siblingProps, siblingKeys +} + +func (index *SpecIndex) storeReferenceWithSiblings( + node, parent *yaml.Node, + keyIndex int, + ref *Reference, + isExtensionPath bool, + path, value string, +) { + copiedNode := *node + siblingProps, siblingKeys := extractSiblingRefProperties(node) + + copied := Reference{ + ParentNode: parent, + FullDefinition: ref.FullDefinition, + Definition: ref.Definition, + RawRef: ref.RawRef, + SchemaIdBase: ref.SchemaIdBase, + Name: ref.Name, + Node: &copiedNode, + KeyNode: node.Content[keyIndex], + Path: path, + Index: index, + IsExtensionRef: isExtensionPath, + HasSiblingProperties: len(siblingProps) > 0, + SiblingProperties: siblingProps, + SiblingKeys: siblingKeys, + } + + index.refsWithSiblings[value] = copied +} diff --git a/index/extract_refs_test.go b/index/extract_refs_test.go index 44db84477..6ed9b44fe 100644 --- a/index/extract_refs_test.go +++ b/index/extract_refs_test.go @@ -4,6 +4,7 @@ package index import ( + "context" "runtime" "strings" "testing" @@ -735,3 +736,229 @@ func TestUnderOpenAPIExamplePath(t *testing.T) { }) } } + +func TestExtractRefs_InlineSchemaHelpers(t *testing.T) { + idx := NewTestSpecIndex().Load().(*SpecIndex) + idx.specAbsolutePath = "test.yaml" + + var inlineNode yaml.Node + _ = yaml.Unmarshal([]byte(`additionalProperties: true`), &inlineNode) + idx.collectInlineSchemaDefinition(nil, inlineNode.Content[0], []string{"components", "schemas", "Pet"}, 0) + assert.Empty(t, idx.allInlineSchemaDefinitions) + + var refNode yaml.Node + _ = yaml.Unmarshal([]byte(`schema: + $ref: '#/components/schemas/Pet'`), &refNode) + idx.collectInlineSchemaDefinition(nil, refNode.Content[0], []string{"paths", "/pets"}, 0) + assert.Len(t, idx.allRefSchemaDefinitions, 1) + + before := len(idx.allInlineSchemaDefinitions) + idx.collectInlineSchemaDefinition(nil, refNode.Content[0], []string{"paths"}, 1) + assert.Len(t, idx.allInlineSchemaDefinitions, before) +} + +func TestExtractRefs_MapAndArraySchemaHelpers(t *testing.T) { + idx := NewTestSpecIndex().Load().(*SpecIndex) + idx.specAbsolutePath = "test.yaml" + + var propsNode yaml.Node + _ = yaml.Unmarshal([]byte(`properties: + foo: + $ref: '#/components/schemas/Pet' + bar: + type: object`), &propsNode) + idx.collectMapSchemaDefinitions(nil, propsNode.Content[0], []string{"components", "schemas", "Thing"}, 0) + assert.Len(t, idx.allRefSchemaDefinitions, 1) + assert.Len(t, idx.allInlineSchemaDefinitions, 1) + assert.Len(t, idx.allInlineSchemaObjectDefinitions, 1) + + inlineBefore := len(idx.allInlineSchemaDefinitions) + idx.collectMapSchemaDefinitions(nil, propsNode.Content[0], []string{"examples"}, 0) + assert.Len(t, idx.allInlineSchemaDefinitions, inlineBefore) + idx.collectMapSchemaDefinitions(nil, propsNode.Content[0], []string{"x-test"}, 0) + assert.Len(t, idx.allInlineSchemaDefinitions, inlineBefore) + + var arrayNode yaml.Node + _ = yaml.Unmarshal([]byte(`oneOf: + - $ref: '#/components/schemas/Pet' + - type: string`), &arrayNode) + idx.collectArraySchemaDefinitions(nil, arrayNode.Content[0], nil, 0) + assert.Len(t, idx.allRefSchemaDefinitions, 2) + assert.Len(t, idx.allInlineSchemaDefinitions, 2) + + idx.collectArraySchemaDefinitions(nil, arrayNode.Content[0], nil, 1) + assert.Len(t, idx.allRefSchemaDefinitions, 2) + idx.collectMapSchemaDefinitions(nil, propsNode.Content[0], nil, 1) + idx.collectArraySchemaDefinitions(nil, arrayNode.Content[0], nil, 2) +} + +func TestInlineSchemaIsObjectOrArray(t *testing.T) { + assert.False(t, inlineSchemaIsObjectOrArray(nil)) + + var noType yaml.Node + _ = yaml.Unmarshal([]byte(`description: nope`), &noType) + assert.False(t, inlineSchemaIsObjectOrArray(noType.Content[0])) + + var objectNode yaml.Node + _ = yaml.Unmarshal([]byte(`type: object`), &objectNode) + assert.True(t, inlineSchemaIsObjectOrArray(objectNode.Content[0])) + + var arrayNode yaml.Node + _ = yaml.Unmarshal([]byte(`type: array`), &arrayNode) + assert.True(t, inlineSchemaIsObjectOrArray(arrayNode.Content[0])) +} + +func TestRegisterSchemaIDAt_HelperBranches(t *testing.T) { + idx := NewTestSpecIndex().Load().(*SpecIndex) + idx.specAbsolutePath = "test.yaml" + + var invalidNode yaml.Node + _ = yaml.Unmarshal([]byte(`$id: '#/bad'`), &invalidNode) + idx.registerSchemaIDAt(invalidNode.Content[0], 0, []string{"components", "schemas", "Pet"}, "test.yaml") + assert.NotEmpty(t, idx.refErrors) + + var nonStringNode yaml.Node + _ = yaml.Unmarshal([]byte(`$id: + type: string`), &nonStringNode) + errorsBefore := len(idx.refErrors) + idx.registerSchemaIDAt(nonStringNode.Content[0], 0, nil, "test.yaml") + assert.Len(t, idx.refErrors, errorsBefore) + + var fallbackNode yaml.Node + _ = yaml.Unmarshal([]byte(`$id: schema.json`), &fallbackNode) + idx.registerSchemaIDAt(fallbackNode.Content[0], 0, []string{"components", "schemas", "Pet"}, "://bad-base") + entry := idx.schemaIdRegistry["schema.json"] + assert.NotNil(t, entry) + assert.Equal(t, "schema.json", entry.ResolvedUri) + assert.Equal(t, "://bad-base", entry.ParentId) +} + +func TestExtractRefs_MetadataHelpers(t *testing.T) { + idx := NewTestSpecIndex().Load().(*SpecIndex) + idx.specAbsolutePath = "test.yaml" + + var emptyNode yaml.Node + _ = yaml.Unmarshal([]byte(`$ref: '#/components/schemas/Pet'`), &emptyNode) + action := idx.extractNodeMetadata(emptyNode.Content[0], nil, nil, 0) + assert.False(t, action.appendSegment) + assert.False(t, action.stop) + + seqNode := &yaml.Node{ + Kind: yaml.SequenceNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "description"}, + }, + } + action = idx.extractNodeMetadata(seqNode, nil, nil, 0) + assert.True(t, action.stop) + assert.False(t, action.appendSegment) + + var descNode yaml.Node + _ = yaml.Unmarshal([]byte(`description: hello`), &descNode) + action = idx.extractNodeMetadata(descNode.Content[0], nil, []string{"properties"}, 0) + assert.True(t, action.stop) + assert.True(t, action.appendSegment) + + var summaryNode yaml.Node + _ = yaml.Unmarshal([]byte(`summary: hello`), &summaryNode) + action = idx.extractNodeMetadata(summaryNode.Content[0], nil, []string{"examples"}, 0) + assert.True(t, action.stop) + assert.False(t, action.appendSegment) + + var securityScalar yaml.Node + _ = yaml.Unmarshal([]byte(`security: nope`), &securityScalar) + action = idx.extractNodeMetadata(securityScalar.Content[0], nil, nil, 0) + assert.True(t, action.appendSegment) + assert.Empty(t, idx.securityRequirementRefs) + + var securityNode yaml.Node + _ = yaml.Unmarshal([]byte(`security: + - apiKey: + - read + - write + oauth: + - admin`), &securityNode) + action = idx.extractNodeMetadata(securityNode.Content[0], nil, []string{"paths", "/pets"}, 0) + assert.True(t, action.appendSegment) + assert.Len(t, idx.securityRequirementRefs["apiKey"]["read"], 1) + assert.Len(t, idx.securityRequirementRefs["apiKey"]["write"], 1) + assert.Len(t, idx.securityRequirementRefs["oauth"]["admin"], 1) + + var securityAppendNode yaml.Node + _ = yaml.Unmarshal([]byte(`security: + - apiKey: + - read`), &securityAppendNode) + idx.collectSecurityRequirementMetadata(securityAppendNode.Content[0], 0, "$.paths./pets.security") + assert.Len(t, idx.securityRequirementRefs["apiKey"]["read"], 2) + + var securitySkipNode yaml.Node + _ = yaml.Unmarshal([]byte(`security: + - skip-me + - apiKey: read + - apiKey: + - admin`), &securitySkipNode) + idx.collectSecurityRequirementMetadata(securitySkipNode.Content[0], 0, "$.paths./pets.security") + assert.Len(t, idx.securityRequirementRefs["apiKey"]["admin"], 1) + + var enumPropertyNode yaml.Node + _ = yaml.Unmarshal([]byte(`type: string +enum: + - one`), &enumPropertyNode) + action = idx.extractNodeMetadata(enumPropertyNode.Content[0], nil, []string{"properties"}, 2) + assert.True(t, action.stop) + assert.True(t, action.appendSegment) + + var enumNoType yaml.Node + _ = yaml.Unmarshal([]byte(`enum: + - one`), &enumNoType) + idx.collectEnumMetadata(enumNoType.Content[0], nil, 0, "$.enum") + assert.Empty(t, idx.allEnums) + idx.collectEnumMetadata(enumNoType.Content[0], nil, 1, "$.enum") + + var enumWithType yaml.Node + _ = yaml.Unmarshal([]byte(`type: string +enum: + - one`), &enumWithType) + idx.collectEnumMetadata(enumWithType.Content[0], nil, 2, "$.enum") + assert.Len(t, idx.allEnums, 1) + + var objectProps yaml.Node + _ = yaml.Unmarshal([]byte(`type: + - string + - object +properties: + name: + type: string`), &objectProps) + action = idx.extractNodeMetadata(objectProps.Content[0], nil, []string{"components", "schemas", "Pet"}, 2) + assert.True(t, action.appendSegment) + assert.Len(t, idx.allObjectsWithProperties, 1) + + metadataFallbackNode := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "summary"}, + }, + } + assert.Equal(t, "summary", metadataValueNode(metadataFallbackNode, 0).Value) +} + +func TestExtractRefs_WalkHelpers(t *testing.T) { + idx := NewTestSpecIndex().Load().(*SpecIndex) + state := idx.initializeExtractRefsState(context.Background(), nil, nil, 0, false, "") + + node := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + nil, + }, + } + var found []*Reference + assert.False(t, idx.handleExtractRefsKey(node, nil, &state, 0, &found)) + assert.Empty(t, found) + + assert.False(t, shouldSkipMapSchemaCollection(nil)) + assert.True(t, shouldSkipMapSchemaCollection([]string{"example"})) + assert.False(t, shouldSkipMapSchemaCollection([]string{"properties", "example"})) + assert.False(t, shouldSkipMapSchemaCollection([]string{"patternProperties", "example"})) + assert.True(t, shouldSkipMapSchemaCollection([]string{"x-test"})) +} diff --git a/index/extract_refs_walk.go b/index/extract_refs_walk.go new file mode 100644 index 000000000..9db301b2f --- /dev/null +++ b/index/extract_refs_walk.go @@ -0,0 +1,165 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "context" + "strings" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +type extractRefsState struct { + ctx context.Context + scope *SchemaIdScope + parentBaseURI string + seenPath []string + level int + poly bool + polyName string + prev string +} + +func (index *SpecIndex) initializeExtractRefsState( + ctx context.Context, + node *yaml.Node, + seenPath []string, + level int, + poly bool, + pName string, +) extractRefsState { + scope := GetSchemaIdScope(ctx) + if scope == nil { + scope = NewSchemaIdScope(index.specAbsolutePath) + ctx = WithSchemaIdScope(ctx, scope) + } + + parentBaseURI := scope.BaseUri + if node != nil && node.Kind == yaml.MappingNode && !underOpenAPIExamplePath(seenPath) { + if nodeID := FindSchemaIdInNode(node); nodeID != "" { + resolvedNodeID, _ := ResolveSchemaId(nodeID, parentBaseURI) + if resolvedNodeID == "" { + resolvedNodeID = nodeID + } + scope = scope.Copy() + scope.PushId(resolvedNodeID) + ctx = WithSchemaIdScope(ctx, scope) + } + } + + return extractRefsState{ + ctx: ctx, + scope: scope, + parentBaseURI: parentBaseURI, + seenPath: seenPath, + level: level, + poly: poly, + polyName: pName, + } +} + +func (index *SpecIndex) walkExtractRefs(node, parent *yaml.Node, state *extractRefsState) []*Reference { + if node == nil || len(node.Content) == 0 { + return nil + } + + var found []*Reference + for i, n := range node.Content { + if utils.IsNodeMap(n) || utils.IsNodeArray(n) { + found = append(found, index.walkChildExtractRefs(n, node, state)...) + } + + if i%2 == 0 { + if stop := index.handleExtractRefsKey(node, parent, state, i, &found); stop { + continue + } + } + + index.unwindExtractRefsPath(node, state, i) + } + + return found +} + +func (index *SpecIndex) walkChildExtractRefs(node, parent *yaml.Node, state *extractRefsState) []*Reference { + state.level++ + if isPoly, _ := index.checkPolymorphicNode(state.prev); isPoly { + state.poly = true + if state.prev != "" { + state.polyName = state.prev + } + } + return index.ExtractRefs(state.ctx, node, parent, state.seenPath, state.level, state.poly, state.polyName) +} + +func (index *SpecIndex) handleExtractRefsKey( + node, parent *yaml.Node, + state *extractRefsState, + keyIndex int, + found *[]*Reference, +) bool { + keyNode := node.Content[keyIndex] + if keyNode == nil { + return false + } + + if isSchemaContainingNode(keyNode.Value) && !utils.IsNodeArray(node) && keyIndex+1 < len(node.Content) { + index.collectInlineSchemaDefinition(parent, node, state.seenPath, keyIndex) + } + + if isMapOfSchemaContainingNode(keyNode.Value) && !utils.IsNodeArray(node) && keyIndex+1 < len(node.Content) { + if shouldSkipMapSchemaCollection(state.seenPath) { + return true + } + index.collectMapSchemaDefinitions(parent, node, state.seenPath, keyIndex) + } + + if isArrayOfSchemaContainingNode(keyNode.Value) && !utils.IsNodeArray(node) && keyIndex+1 < len(node.Content) { + index.collectArraySchemaDefinitions(parent, node, state.seenPath, keyIndex) + } + + if keyNode.Value == "$ref" { + if ref := index.extractReferenceAt(node, parent, keyIndex, state.seenPath, state.scope, state.poly, state.polyName); ref != nil { + *found = append(*found, ref) + } + } + + if keyNode.Value == "$id" { + index.registerSchemaIDAt(node, keyIndex, state.seenPath, state.parentBaseURI) + } + + if keyNode.Value != "$ref" && keyNode.Value != "$id" && keyNode.Value != "" { + action := index.extractNodeMetadata(node, parent, state.seenPath, keyIndex) + if action.appendSegment { + state.seenPath = append(state.seenPath, strings.ReplaceAll(keyNode.Value, "/", "~1")) + state.prev = keyNode.Value + } + return action.stop + } + + return false +} + +func shouldSkipMapSchemaCollection(seenPath []string) bool { + if len(seenPath) == 0 { + return false + } + for _, p := range seenPath { + if strings.HasPrefix(p, "x-") { + return true + } + } + return underOpenAPIExamplePath(seenPath) +} + +func (index *SpecIndex) unwindExtractRefsPath(node *yaml.Node, state *extractRefsState, currentIndex int) { + if currentIndex >= len(node.Content)-1 { + return + } + next := node.Content[currentIndex+1] + if currentIndex%2 != 0 && next != nil && !utils.IsNodeArray(next) && !utils.IsNodeMap(next) && len(state.seenPath) > 0 { + state.seenPath = state.seenPath[:len(state.seenPath)-1] + } +} diff --git a/index/find_component.go b/index/find_component.go deleted file mode 100644 index cd499414c..000000000 --- a/index/find_component.go +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley -// SPDX-License-Identifier: MIT - -package index - -import ( - "context" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - - jsonpathconfig "github.com/pb33f/jsonpath/pkg/jsonpath/config" - - "github.com/pb33f/jsonpath/pkg/jsonpath" - "github.com/pb33f/libopenapi/utils" - "go.yaml.in/yaml/v4" -) - -// FindComponent will locate a component by its reference, returns nil if nothing is found. -// This method will recurse through remote, local and file references. For each new external reference -// a new index will be created. These indexes can then be traversed recursively. -func (index *SpecIndex) FindComponent(ctx context.Context, componentId string) *Reference { - if index.root == nil { - return nil - } - - if resolved := index.ResolveRefViaSchemaId(componentId); resolved != nil { - return resolved - } - - if strings.HasPrefix(componentId, "/") { - baseUri, fragment := SplitRefFragment(componentId) - if resolved := index.resolveRefViaSchemaIdPath(baseUri); resolved != nil { - if fragment != "" && resolved.Node != nil { - if fragmentNode := navigateToFragment(resolved.Node, fragment); fragmentNode != nil { - resolved.Node = fragmentNode - } - } - return resolved - } - } - - uri := strings.Split(componentId, "#/") - if len(uri) == 2 { - if uri[0] != "" { - if index.specAbsolutePath == uri[0] { - return index.FindComponentInRoot(ctx, fmt.Sprintf("#/%s", uri[1])) - } else { - return index.lookupRolodex(ctx, uri) - } - } else { - return index.FindComponentInRoot(ctx, fmt.Sprintf("#/%s", uri[1])) - } - } else { - - // does it contain a file extension? - fileExt := filepath.Ext(componentId) - if fileExt != "" { - - // check if the context has a root index set, if so this is a deep search that has moved through multiple - // indexes and we need to adjust the URI to reflect the location of the root index. - // - // the below code has been commended out due to being handled in the index. Keeping it for legacy and for - // future bugs. - // - //if ctx.Value(RootIndexKey) != nil { - // rootIndex := ctx.Value(RootIndexKey).(*SpecIndex) - // if rootIndex != nil && rootIndex.specAbsolutePath != "" { - // dir := filepath.Dir(rootIndex.specAbsolutePath) - // // create an absolute path to the file. - // absoluteFilePath := filepath.Join(dir, componentId) - // // split into a URI. - // uri = []string{absoluteFilePath} - // } - //} - - return index.lookupRolodex(ctx, uri) - } - - // root search - return index.FindComponentInRoot(ctx, componentId) - } -} - -func FindComponent(_ context.Context, root *yaml.Node, componentId, absoluteFilePath string, index *SpecIndex) *Reference { - // check component for url encoding. - if strings.Contains(componentId, "%") { - // decode the url. - componentId, _ = url.QueryUnescape(componentId) - } - - name, friendlySearch := utils.ConvertComponentIdIntoFriendlyPathSearch(componentId) - if friendlySearch == "$." { - friendlySearch = "$" - } - path, err := jsonpath.NewPath(friendlySearch, jsonpathconfig.WithPropertyNameExtension(), jsonpathconfig.WithLazyContextTracking()) - if path == nil || err != nil || root == nil { - return nil // no component found - } - res := path.Query(root) - - if len(res) == 1 { - resNode := res[0] - fullDef := fmt.Sprintf("%s%s", absoluteFilePath, componentId) - // extract properties - - // check if we have already seen this reference and there is a parent, use it - var parentNode *yaml.Node - if index.allRefs[componentId] != nil { - parentNode = index.allRefs[componentId].ParentNode - } - if index.allRefs[fullDef] != nil { - parentNode = index.allRefs[fullDef].ParentNode - } - - ref := &Reference{ - FullDefinition: fullDef, - Definition: componentId, - Name: name, - Node: resNode, - Path: friendlySearch, - RemoteLocation: absoluteFilePath, - ParentNode: parentNode, - Index: index, - RequiredRefProperties: extractDefinitionRequiredRefProperties(resNode, map[string][]string{}, fullDef, index), - } - return ref - } - return nil -} - -func (index *SpecIndex) FindComponentInRoot(ctx context.Context, componentId string) *Reference { - if index.root != nil { - - componentId = utils.ReplaceWindowsDriveWithLinuxPath(componentId) - if !strings.HasPrefix(componentId, "#/") { - spl := strings.Split(componentId, "#/") - if len(spl) == 2 { - if spl[0] != "" { - componentId = fmt.Sprintf("#/%s", spl[1]) - } - } - } - - return FindComponent(ctx, index.root, componentId, index.specAbsolutePath, index) - } - return nil -} - -func (index *SpecIndex) lookupRolodex(ctx context.Context, uri []string) *Reference { - if index.rolodex == nil { - return nil - } - - // If SkipExternalRefResolution is enabled, don't attempt rolodex lookups - if index.config != nil && index.config.SkipExternalRefResolution { - return nil - } - - if len(uri) > 0 { - - // split string to remove file reference - file := strings.ReplaceAll(uri[0], "file:", "") - - var absoluteFileLocation, fileName string - fileName = filepath.Base(file) - absoluteFileLocation = file - if !filepath.IsAbs(file) && !strings.HasPrefix(file, "http") { - // When resolving relative references, use the directory of the current index's spec file - // if available, not the root BasePath. This ensures that external files can resolve - // relative paths correctly (e.g., "./models/foo.yaml" from an external file should - // resolve relative to that external file's directory, not the root spec directory). - basePath := index.config.BasePath - if index.specAbsolutePath != "" { - basePath = filepath.Dir(index.specAbsolutePath) - } - absoluteFileLocation, _ = filepath.Abs(utils.CheckPathOverlap(basePath, file, string(os.PathSeparator))) - } - - // if the absolute file location has no file ext, then get the rolodex root. - ext := filepath.Ext(absoluteFileLocation) - var parsedDocument *yaml.Node - idx := index - if ext != "" { - // extract the document from the rolodex. - rFile, rError := index.rolodex.OpenWithContext(ctx, absoluteFileLocation) - - if rError != nil { - index.logger.Error("unable to open the rolodex file, check specification references and base path", - "file", absoluteFileLocation, "error", rError) - return nil - } - - if rFile == nil { - index.logger.Error("cannot locate file in the rolodex, check specification references and base path", - "file", absoluteFileLocation) - return nil - } - // Check if the index is already available (handles recursive lookups within same goroutine). - // Only wait for indexing if the index isn't already set - this prevents deadlocks - // in recursive scenarios where A->B->A would otherwise wait forever. - if rFile.GetIndex() == nil { - // Check if this file is being indexed in the current call chain. - // If so, we have a circular dependency and should NOT wait (would deadlock). - // Instead, proceed without the index - the component lookup will still work - // using the parsed YAML content, and circular references will be detected later. - if !IsFileBeingIndexed(ctx, absoluteFileLocation) { - // Wait for the file's index to be ready before using it. - // This handles the case where another goroutine is still indexing the file. - rFile.WaitForIndexing() - } - } - if rFile.GetIndex() != nil { - idx = rFile.GetIndex() - } - - parsedDocument, _ = rFile.GetContentAsYAMLNode() - - } else { - parsedDocument = index.root - } - - wholeFile := false - query := "" - if len(uri) < 2 { - wholeFile = true - } else { - query = fmt.Sprintf("#/%s", uri[1]) - } - - // check if there is a component we want to suck in, or if the - // entire root needs to come in. - var foundRef *Reference - if wholeFile { - - if parsedDocument != nil { - if parsedDocument.Kind == yaml.DocumentNode { - parsedDocument = parsedDocument.Content[0] - } - } - - var parentNode *yaml.Node - if index.allRefs[absoluteFileLocation] != nil { - parentNode = index.allRefs[absoluteFileLocation].ParentNode - } - - foundRef = &Reference{ - ParentNode: parentNode, - FullDefinition: absoluteFileLocation, - Definition: fileName, - Name: fileName, - Index: idx, - Node: parsedDocument, - IsRemote: true, - RemoteLocation: absoluteFileLocation, - Path: "$", - RequiredRefProperties: extractDefinitionRequiredRefProperties(parsedDocument, map[string][]string{}, absoluteFileLocation, index), - } - return foundRef - } else { - foundRef = FindComponent(ctx, parsedDocument, query, absoluteFileLocation, index) - if foundRef != nil { - foundRef.IsRemote = true - foundRef.RemoteLocation = absoluteFileLocation - return foundRef - } else { - // Debug: log when FindComponent returns nil - index.logger.Debug("[lookupRolodex] FindComponent returned nil", - "absoluteFileLocation", absoluteFileLocation, - "query", query, - "parsedDocument_nil", parsedDocument == nil, - "idx_nil", idx == nil) - } - } - } - return nil -} diff --git a/index/find_component_build.go b/index/find_component_build.go new file mode 100644 index 000000000..14eb727b9 --- /dev/null +++ b/index/find_component_build.go @@ -0,0 +1,106 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "fmt" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +func cloneFoundComponentReference(index *SpecIndex, found *Reference, componentID, absoluteFilePath string) *Reference { + return buildResolvedComponentReference(index, found, componentID, absoluteFilePath, found.Name, found.Path, found.Node) +} + +func buildResolvedComponentReference( + index *SpecIndex, + source *Reference, + componentID, absoluteFilePath, name, path string, + node *yaml.Node, +) *Reference { + fullDef := fmt.Sprintf("%s%s", absoluteFilePath, componentID) + if path == "" { + _, path = utils.ConvertComponentIdIntoFriendlyPathSearch(componentID) + if path == "$." { + path = "$" + } + } + + var ref Reference + if source != nil { + ref = Reference{ + RawRef: source.RawRef, + SchemaIdBase: source.SchemaIdBase, + KeyNode: source.KeyNode, + ParentNodeSchemaType: source.ParentNodeSchemaType, + Resolved: source.Resolved, + Circular: source.Circular, + Seen: source.Seen, + IsRemote: source.IsRemote, + IsExtensionRef: source.IsExtensionRef, + HasSiblingProperties: source.HasSiblingProperties, + In: source.In, + } + ref.ParentNodeTypes = append([]string(nil), source.ParentNodeTypes...) + ref.SiblingKeys = append([]*yaml.Node(nil), source.SiblingKeys...) + ref.SiblingProperties = cloneSiblingProperties(source.SiblingProperties) + ref.RequiredRefProperties = cloneRequiredRefProperties(source.RequiredRefProperties) + } + + ref.FullDefinition = fullDef + ref.Definition = componentID + ref.Name = name + ref.Node = node + ref.Path = path + ref.RemoteLocation = absoluteFilePath + ref.Index = index + + if source != nil && source.ParentNode != nil { + ref.ParentNode = source.ParentNode + } else { + ref.ParentNode = lookupComponentParentNode(index, componentID, fullDef) + } + + if ref.RequiredRefProperties == nil && node != nil { + ref.RequiredRefProperties = extractDefinitionRequiredRefProperties(node, map[string][]string{}, fullDef, index) + } + + return &ref +} + +func lookupComponentParentNode(index *SpecIndex, componentID, fullDef string) *yaml.Node { + if index == nil { + return nil + } + if ref := index.allRefs[componentID]; ref != nil { + return ref.ParentNode + } + if ref := index.allRefs[fullDef]; ref != nil { + return ref.ParentNode + } + return nil +} + +func cloneRequiredRefProperties(source map[string][]string) map[string][]string { + if source == nil { + return nil + } + cloned := make(map[string][]string, len(source)) + for key, values := range source { + cloned[key] = append([]string(nil), values...) + } + return cloned +} + +func cloneSiblingProperties(source map[string]*yaml.Node) map[string]*yaml.Node { + if source == nil { + return nil + } + cloned := make(map[string]*yaml.Node, len(source)) + for key, node := range source { + cloned[key] = node + } + return cloned +} diff --git a/index/find_component_direct.go b/index/find_component_direct.go new file mode 100644 index 000000000..41e32b687 --- /dev/null +++ b/index/find_component_direct.go @@ -0,0 +1,77 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "net/url" + "strings" + "sync" +) + +func findDirectComponent(index *SpecIndex, componentID, absoluteFilePath string) *Reference { + if index == nil || !strings.HasPrefix(componentID, "#/") { + return nil + } + + normalizedComponentID := normalizeComponentLookupID(componentID) + + var found *Reference + switch { + case strings.HasPrefix(normalizedComponentID, "#/components/schemas/"), + strings.HasPrefix(normalizedComponentID, "#/definitions/"): + found = loadSyncMapReference(index.allComponentSchemaDefinitions, normalizedComponentID) + case strings.HasPrefix(normalizedComponentID, "#/components/securitySchemes/"): + found = loadSyncMapReference(index.allSecuritySchemes, normalizedComponentID) + case strings.HasPrefix(normalizedComponentID, "#/components/parameters/"): + found = index.allParameters[normalizedComponentID] + case strings.HasPrefix(normalizedComponentID, "#/components/requestBodies/"): + found = index.allRequestBodies[normalizedComponentID] + case strings.HasPrefix(normalizedComponentID, "#/components/responses/"): + found = index.allResponses[normalizedComponentID] + case strings.HasPrefix(normalizedComponentID, "#/components/headers/"): + found = index.allHeaders[normalizedComponentID] + case strings.HasPrefix(normalizedComponentID, "#/components/examples/"): + found = index.allExamples[normalizedComponentID] + case strings.HasPrefix(normalizedComponentID, "#/components/links/"): + found = index.allLinks[normalizedComponentID] + case strings.HasPrefix(normalizedComponentID, "#/components/callbacks/"): + found = index.allCallbacks[normalizedComponentID] + case strings.HasPrefix(normalizedComponentID, "#/components/pathItems/"): + found = index.allComponentPathItems[normalizedComponentID] + } + + if found == nil { + return nil + } + return cloneFoundComponentReference(index, found, componentID, absoluteFilePath) +} + +func normalizeComponentLookupID(componentID string) string { + if componentID == "" { + return "" + } + segs := strings.Split(componentID, "/") + for i, seg := range segs { + if i == 0 { + continue + } + seg = strings.ReplaceAll(seg, "~1", "/") + seg = strings.ReplaceAll(seg, "~0", "~") + if strings.ContainsRune(seg, '%') { + seg, _ = url.QueryUnescape(seg) + } + segs[i] = seg + } + return strings.Join(segs, "/") +} + +func loadSyncMapReference(collection *sync.Map, key string) *Reference { + if collection == nil { + return nil + } + if found, ok := collection.Load(key); ok { + return found.(*Reference) + } + return nil +} diff --git a/index/find_component_entry.go b/index/find_component_entry.go new file mode 100644 index 000000000..346ad2bb6 --- /dev/null +++ b/index/find_component_entry.go @@ -0,0 +1,117 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "context" + "fmt" + "net/url" + "path/filepath" + "strings" + + "github.com/pb33f/jsonpath/pkg/jsonpath" + jsonpathconfig "github.com/pb33f/jsonpath/pkg/jsonpath/config" + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +// FindComponent locates a component in the index by reference. +// +// It resolves local references directly from the current document first, then recurses through +// rolodex-backed file and remote references as needed. It returns nil when the target cannot be found. +func (index *SpecIndex) FindComponent(ctx context.Context, componentId string) *Reference { + if index.root == nil { + return nil + } + + if resolved := index.ResolveRefViaSchemaId(componentId); resolved != nil { + return resolved + } + + if strings.HasPrefix(componentId, "/") { + baseURI, fragment := SplitRefFragment(componentId) + if resolved := index.resolveRefViaSchemaIdPath(baseURI); resolved != nil { + if fragment != "" && resolved.Node != nil { + if fragmentNode := navigateToFragment(resolved.Node, fragment); fragmentNode != nil { + resolved.Node = fragmentNode + } + } + return resolved + } + } + + uri := strings.Split(componentId, "#/") + if len(uri) == 2 { + if uri[0] != "" { + if index.specAbsolutePath == uri[0] { + return index.FindComponentInRoot(ctx, fmt.Sprintf("#/%s", uri[1])) + } + return index.lookupRolodex(ctx, uri) + } + return index.FindComponentInRoot(ctx, fmt.Sprintf("#/%s", uri[1])) + } + + fileExt := filepath.Ext(componentId) + if fileExt != "" { + return index.lookupRolodex(ctx, uri) + } + + return index.FindComponentInRoot(ctx, componentId) +} + +// FindComponent locates a component within a specific root YAML node. +// +// The lookup prefers direct fragment navigation and direct component maps first, and falls back to +// JSONPath traversal for legacy or non-direct component identifiers. +func FindComponent(_ context.Context, root *yaml.Node, componentID, absoluteFilePath string, index *SpecIndex) *Reference { + if strings.Contains(componentID, "%") { + componentID, _ = url.QueryUnescape(componentID) + } + + if fastRef := findDirectComponent(index, componentID, absoluteFilePath); fastRef != nil { + return fastRef + } + + if strings.HasPrefix(componentID, "#/") { + if node := navigateToFragment(root, componentID); node != nil { + name, friendlySearch := utils.ConvertComponentIdIntoFriendlyPathSearch(componentID) + if friendlySearch == "$." { + friendlySearch = "$" + } + return buildResolvedComponentReference(index, nil, componentID, absoluteFilePath, name, friendlySearch, node) + } + } + + name, friendlySearch := utils.ConvertComponentIdIntoFriendlyPathSearch(componentID) + if friendlySearch == "$." { + friendlySearch = "$" + } + path, err := jsonpath.NewPath(friendlySearch, jsonpathconfig.WithPropertyNameExtension(), jsonpathconfig.WithLazyContextTracking()) + if path == nil || err != nil || root == nil { + return nil + } + res := path.Query(root) + if len(res) == 1 { + return buildResolvedComponentReference(index, nil, componentID, absoluteFilePath, name, friendlySearch, res[0]) + } + return nil +} + +// FindComponentInRoot locates a component reference in the current root document only. +// +// It normalizes file-prefixed local references back to root-document fragments before delegating +// to FindComponent. +func (index *SpecIndex) FindComponentInRoot(ctx context.Context, componentID string) *Reference { + if index.root != nil { + componentID = utils.ReplaceWindowsDriveWithLinuxPath(componentID) + if !strings.HasPrefix(componentID, "#/") { + spl := strings.Split(componentID, "#/") + if len(spl) == 2 && spl[0] != "" { + componentID = fmt.Sprintf("#/%s", spl[1]) + } + } + return FindComponent(ctx, index.root, componentID, index.specAbsolutePath, index) + } + return nil +} diff --git a/index/find_component_external.go b/index/find_component_external.go new file mode 100644 index 000000000..2aab515c8 --- /dev/null +++ b/index/find_component_external.go @@ -0,0 +1,106 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +func (index *SpecIndex) lookupRolodex(ctx context.Context, uri []string) *Reference { + if index.rolodex == nil { + return nil + } + if index.config != nil && index.config.SkipExternalRefResolution { + return nil + } + + if len(uri) == 0 { + return nil + } + + file := strings.ReplaceAll(uri[0], "file:", "") + fileName := filepath.Base(file) + absoluteFileLocation := file + if !filepath.IsAbs(file) && !strings.HasPrefix(file, "http") { + basePath := index.config.BasePath + if index.specAbsolutePath != "" { + basePath = filepath.Dir(index.specAbsolutePath) + } + absoluteFileLocation, _ = filepath.Abs(utils.CheckPathOverlap(basePath, file, string(os.PathSeparator))) + } + + ext := filepath.Ext(absoluteFileLocation) + var parsedDocument *yaml.Node + idx := index + if ext != "" { + rFile, rError := index.rolodex.OpenWithContext(ctx, absoluteFileLocation) + if rError != nil { + index.logger.Error("unable to open the rolodex file, check specification references and base path", + "file", absoluteFileLocation, "error", rError) + return nil + } + if rFile == nil { + index.logger.Error("cannot locate file in the rolodex, check specification references and base path", + "file", absoluteFileLocation) + return nil + } + if rFile.GetIndex() == nil && !IsFileBeingIndexed(ctx, absoluteFileLocation) { + rFile.WaitForIndexing() + } + if rFile.GetIndex() != nil { + idx = rFile.GetIndex() + } + parsedDocument, _ = rFile.GetContentAsYAMLNode() + } else { + parsedDocument = index.root + } + + wholeFile := len(uri) < 2 + query := "" + if !wholeFile { + query = fmt.Sprintf("#/%s", uri[1]) + } + + if wholeFile { + if parsedDocument != nil && parsedDocument.Kind == yaml.DocumentNode { + parsedDocument = parsedDocument.Content[0] + } + var parentNode *yaml.Node + if index.allRefs[absoluteFileLocation] != nil { + parentNode = index.allRefs[absoluteFileLocation].ParentNode + } + return &Reference{ + ParentNode: parentNode, + FullDefinition: absoluteFileLocation, + Definition: fileName, + Name: fileName, + Index: idx, + Node: parsedDocument, + IsRemote: true, + RemoteLocation: absoluteFileLocation, + Path: "$", + RequiredRefProperties: extractDefinitionRequiredRefProperties(parsedDocument, map[string][]string{}, absoluteFileLocation, index), + } + } + + foundRef := FindComponent(ctx, parsedDocument, query, absoluteFileLocation, index) + if foundRef != nil { + foundRef.IsRemote = true + foundRef.RemoteLocation = absoluteFileLocation + return foundRef + } + index.logger.Debug("[lookupRolodex] FindComponent returned nil", + "absoluteFileLocation", absoluteFileLocation, + "query", query, + "parsedDocument_nil", parsedDocument == nil, + "idx_nil", idx == nil) + return nil +} diff --git a/index/find_component_test.go b/index/find_component_test.go index 5b534fc46..0f889f879 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -6,8 +6,10 @@ package index import ( "bytes" "context" + "io/fs" "log/slog" "os" + "path/filepath" "strings" "testing" "time" @@ -418,6 +420,25 @@ components: assert.Nil(t, n) } +func TestFindComponent_LookupRolodex_NoRolodex(t *testing.T) { + index := NewTestSpecIndex().Load().(*SpecIndex) + index.rolodex = nil + assert.Nil(t, index.lookupRolodex(context.Background(), []string{"pet.yaml"})) +} + +func TestFindComponent_LookupRolodex_MissingWholeFileReturnsNil(t *testing.T) { + cfg := CreateOpenAPIIndexConfig() + rolo := NewRolodex(cfg) + cfg.Rolodex = rolo + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte("openapi: 3.0.2"), &rootNode) + index := NewSpecIndexWithConfig(&rootNode, cfg) + index.rolodex = rolo + + assert.Nil(t, index.lookupRolodex(context.Background(), []string{"/tmp/does-not-exist.yaml"})) +} + func TestFindComponent_LookupRolodex_InvalidFile_NoBypass(t *testing.T) { spec := `i:am : not a yaml file:` @@ -574,3 +595,107 @@ components: n = index.lookupRolodex(context.Background(), []string{"https://example.com/schemas/pet.yaml"}) assert.Nil(t, n, "lookupRolodex should return nil for remote refs when SkipExternalRefResolution is enabled") } + +func TestFindComponent_LookupRolodex_WholeFileLocalDocument(t *testing.T) { + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "schema.yaml") + err := os.WriteFile(filePath, []byte("type: object\nproperties:\n name:\n type: string\n"), 0o644) + assert.NoError(t, err) + + rootSpec := `openapi: 3.0.2 +info: + title: Test + version: 1.0.0` + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(rootSpec), &rootNode) + + cfg := CreateOpenAPIIndexConfig() + cfg.BasePath = tempDir + rolo := NewRolodex(cfg) + rolo.SetRootNode(&rootNode) + cfg.Rolodex = rolo + + fileFS, fsErr := NewLocalFSWithConfig(&LocalFSConfig{ + BaseDirectory: tempDir, + DirFS: os.DirFS(tempDir), + IndexConfig: cfg, + }) + assert.NoError(t, fsErr) + rolo.AddLocalFS(tempDir, fileFS) + + index := NewSpecIndexWithConfig(&rootNode, cfg) + index.rolodex = rolo + + ref := index.lookupRolodex(context.Background(), []string{filePath}) + assert.NotNil(t, ref) + assert.Equal(t, "$", ref.Path) + assert.True(t, ref.IsRemote) + assert.Equal(t, filePath, ref.FullDefinition) +} + +func TestFindComponent_FastFragmentAndNilRoot(t *testing.T) { + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(`components: + schemas: + Pet: + type: string`), &rootNode) + + ref := FindComponent(context.Background(), &rootNode, "#/components/schemas/Pet", "test.yaml", nil) + assert.NotNil(t, ref) + assert.Equal(t, "#/components/schemas/Pet", ref.Definition) + + assert.Nil(t, FindComponent(context.Background(), nil, "#/components/schemas/Pet", "test.yaml", nil)) + assert.Nil(t, FindComponent(context.Background(), &rootNode, "[", "test.yaml", nil)) +} + +func TestFindComponent_UnescapesEncodedComponentId(t *testing.T) { + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(`components: + schemas: + thing name: + type: string`), &rootNode) + + ref := FindComponent(context.Background(), &rootNode, "#/components/schemas/thing%20name", "test.yaml", nil) + assert.NotNil(t, ref) + assert.Equal(t, "#/components/schemas/thing name", ref.Definition) + assert.Equal(t, "$.components.schemas['thing name']", ref.Path) +} + +func TestFindComponent_FallbackRootPointerJSONPath(t *testing.T) { + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(`openapi: 3.1.0 +info: + title: test`), &rootNode) + + ref := FindComponent(context.Background(), &rootNode, "", "test.yaml", nil) + assert.NotNil(t, ref) + assert.Equal(t, "$", ref.Path) + assert.Equal(t, "", ref.Definition) + assert.Equal(t, rootNode.Content[0], ref.Node) +} + +type emptyRemoteFS struct{} + +func (e *emptyRemoteFS) Open(name string) (fs.File, error) { + return &testFile{content: ""}, nil +} + +func TestFindComponent_LookupRolodex_NilRolodexFileLogsAndReturnsNil(t *testing.T) { + var logBuf bytes.Buffer + + cfg := CreateOpenAPIIndexConfig() + cfg.AllowRemoteLookup = true + cfg.Logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + + index := NewTestSpecIndex().Load().(*SpecIndex) + index.config = cfg + index.logger = cfg.Logger + + rolo := NewRolodex(cfg) + rolo.AddRemoteFS("http://example.com", &emptyRemoteFS{}) + index.rolodex = rolo + + ref := index.lookupRolodex(context.Background(), []string{"http://example.com/spec.yaml"}) + assert.Nil(t, ref) + assert.Contains(t, logBuf.String(), "cannot locate file in the rolodex") +} diff --git a/index/index_model.go b/index/index_model.go index c346f5ada..65117fb7a 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -408,7 +408,7 @@ type SpecIndex struct { componentIndexChan chan struct{} polyComponentIndexChan chan struct{} resolver *Resolver - resolverLock sync.Mutex + resolverLock sync.RWMutex cache *sync.Map built bool uri []string @@ -423,10 +423,14 @@ type SpecIndex struct { // GetResolver returns the resolver for this index. func (index *SpecIndex) GetResolver() *Resolver { + index.resolverLock.RLock() + defer index.resolverLock.RUnlock() return index.resolver } func (index *SpecIndex) SetResolver(resolver *Resolver) { + index.resolverLock.Lock() + defer index.resolverLock.Unlock() index.resolver = resolver } @@ -451,8 +455,15 @@ func (index *SpecIndex) Release() { if index == nil { return } + index.releaseDocumentNodes() + index.releaseReferenceIndexes() + index.releaseComponentIndexes() + index.releaseDerivedState() + index.releaseOwnedResources() + index.resetRuntimeState() +} - // yaml.Node tree +func (index *SpecIndex) releaseDocumentNodes() { index.root = nil index.pathsNode = nil index.tagsNode = nil @@ -468,8 +479,9 @@ func (index *SpecIndex) Release() { index.pathItemsNode = nil index.rootServersNode = nil index.rootSecurityNode = nil +} - // reference maps (all hold *Reference with *yaml.Node pointers) +func (index *SpecIndex) releaseReferenceIndexes() { index.allRefs = nil index.rawSequencedRefs = nil index.linesWithRefs = nil @@ -503,8 +515,9 @@ func (index *SpecIndex) Release() { index.externalDocumentsRef = nil index.rootSecurity = nil index.refsWithSiblings = nil +} - // schema / component collections +func (index *SpecIndex) releaseComponentIndexes() { index.allRefSchemaDefinitions = nil index.allInlineSchemaDefinitions = nil index.allInlineSchemaObjectDefinitions = nil @@ -521,8 +534,9 @@ func (index *SpecIndex) Release() { index.allComponentPathItems = nil index.allExternalDocuments = nil index.externalSpecIndex = nil +} - // line/col -> *yaml.Node map +func (index *SpecIndex) releaseDerivedState() { index.nodeMap = nil index.allDescriptions = nil index.allSummaries = nil @@ -540,26 +554,52 @@ func (index *SpecIndex) Release() { index.pendingResolve = nil index.uri = nil index.logger = nil +} - // Break circular SpecIndex <-> Resolver reference. +func (index *SpecIndex) releaseOwnedResources() { + index.resolverLock.Lock() if index.resolver != nil { index.resolver.Release() index.resolver = nil } + index.resolverLock.Unlock() - // Rolodex holds rootNode and child indexes. if index.rolodex != nil { index.rolodex.Release() index.rolodex = nil } - // Config holds SpecInfo which holds RootNode. if index.config != nil { index.config.SpecInfo.Release() index.config = nil } } +func (index *SpecIndex) resetRuntimeState() { + index.externalDocumentsCount = 0 + index.operationTagsCount = 0 + index.globalTagsCount = 0 + index.totalTagsCount = 0 + index.globalLinksCount = 0 + index.globalCallbacksCount = 0 + index.pathCount = 0 + index.operationCount = 0 + index.operationParamCount = 0 + index.componentParamCount = 0 + index.componentsInlineParamUniqueCount = 0 + index.componentsInlineParamDuplicateCount = 0 + index.schemaCount = 0 + index.refCount = 0 + index.enumCount = 0 + index.descriptionCount = 0 + index.summaryCount = 0 + index.allowCircularReferences = false + index.built = false + index.componentIndexChan = nil + index.polyComponentIndexChan = nil + index.nodeMapCompleted = nil +} + // SetAbsolutePath sets the absolute path to the spec file for the index. Will be absolute, either as a http link or a file. func (index *SpecIndex) SetAbsolutePath(absolutePath string) { index.specAbsolutePath = absolutePath diff --git a/index/index_model_test.go b/index/index_model_test.go index 47becbce7..2dafd301e 100644 --- a/index/index_model_test.go +++ b/index/index_model_test.go @@ -135,27 +135,36 @@ func TestSpecIndex_Release(t *testing.T) { rolodex.rootNode = &yaml.Node{Value: "rolodex-root"} idx := &SpecIndex{ - config: cfg, - root: rootNode, - pathsNode: &yaml.Node{}, - tagsNode: &yaml.Node{}, - schemasNode: &yaml.Node{}, - allRefs: map[string]*Reference{"ref": {}}, - rawSequencedRefs: []*Reference{{}}, - allMappedRefs: map[string]*Reference{"mapped": {}}, - allMappedRefsSequenced: []*ReferenceMapped{{}}, - nodeMap: map[int]map[int]*yaml.Node{1: {1: &yaml.Node{}}}, - allDescriptions: []*DescriptionReference{{}}, - allEnums: []*EnumReference{{}}, - circularReferences: []*CircularReferenceResult{{}}, - refErrors: []error{nil}, - resolver: resolver, - rolodex: rolodex, - allComponentSchemas: map[string]*Reference{"schema": {}}, - allExternalDocuments: map[string]*Reference{"ext": {}}, - externalSpecIndex: map[string]*SpecIndex{"ext": {}}, - schemaIdRegistry: map[string]*SchemaIdEntry{"id": {}}, - uri: []string{"test"}, + config: cfg, + root: rootNode, + pathsNode: &yaml.Node{}, + tagsNode: &yaml.Node{}, + schemasNode: &yaml.Node{}, + allRefs: map[string]*Reference{"ref": {}}, + rawSequencedRefs: []*Reference{{}}, + allMappedRefs: map[string]*Reference{"mapped": {}}, + allMappedRefsSequenced: []*ReferenceMapped{{}}, + nodeMap: map[int]map[int]*yaml.Node{1: {1: &yaml.Node{}}}, + allDescriptions: []*DescriptionReference{{}}, + allEnums: []*EnumReference{{}}, + circularReferences: []*CircularReferenceResult{{}}, + refErrors: []error{nil}, + resolver: resolver, + rolodex: rolodex, + allComponentSchemas: map[string]*Reference{"schema": {}}, + allExternalDocuments: map[string]*Reference{"ext": {}}, + externalSpecIndex: map[string]*SpecIndex{"ext": {}}, + schemaIdRegistry: map[string]*SchemaIdEntry{"id": {}}, + uri: []string{"test"}, + globalLinksCount: 2, + globalCallbacksCount: 3, + pathCount: 4, + operationCount: 5, + componentIndexChan: make(chan struct{}), + polyComponentIndexChan: make(chan struct{}), + nodeMapCompleted: make(chan struct{}), + built: true, + allowCircularReferences: true, } idx.Release() @@ -192,6 +201,15 @@ func TestSpecIndex_Release(t *testing.T) { assert.Nil(t, idx.schemaIdRegistry) assert.Nil(t, idx.uri) assert.Nil(t, idx.logger) + assert.Zero(t, idx.globalLinksCount) + assert.Zero(t, idx.globalCallbacksCount) + assert.Zero(t, idx.pathCount) + assert.Zero(t, idx.operationCount) + assert.False(t, idx.built) + assert.False(t, idx.allowCircularReferences) + assert.Nil(t, idx.componentIndexChan) + assert.Nil(t, idx.polyComponentIndexChan) + assert.Nil(t, idx.nodeMapCompleted) // resolver released and niled assert.Nil(t, idx.resolver) @@ -227,6 +245,13 @@ func TestSpecIndex_Release_Idempotent(t *testing.T) { assert.Nil(t, idx.rolodex) } +func TestSpecIndex_GetSetResolver_UsesLock(t *testing.T) { + idx := &SpecIndex{} + resolver := &Resolver{} + idx.SetResolver(resolver) + assert.Same(t, resolver, idx.GetResolver()) +} + func TestSpecIndex_Release_NilConfig(t *testing.T) { idx := &SpecIndex{root: &yaml.Node{}} idx.Release() // config is nil, must not panic diff --git a/index/map_index_nodes.go b/index/map_index_nodes.go index bd2607cda..caa77c537 100644 --- a/index/map_index_nodes.go +++ b/index/map_index_nodes.go @@ -45,11 +45,11 @@ type NodeOrigin struct { // if the node was found, false if not. func (index *SpecIndex) GetNode(line int, column int) (*yaml.Node, bool) { index.nodeMapLock.RLock() + defer index.nodeMapLock.RUnlock() if index.nodeMap[line] == nil { return nil, false } node := index.nodeMap[line][column] - index.nodeMapLock.RUnlock() return node, node != nil } diff --git a/index/map_index_nodes_test.go b/index/map_index_nodes_test.go index e2b820ed6..1e9693d45 100644 --- a/index/map_index_nodes_test.go +++ b/index/map_index_nodes_test.go @@ -7,6 +7,7 @@ import ( "os" "reflect" "testing" + "time" "github.com/pb33f/jsonpath/pkg/jsonpath" "github.com/pb33f/libopenapi/utils" @@ -55,6 +56,32 @@ func TestSpecIndex_MapNodes(t *testing.T) { assert.Nil(t, mappedKeyNode) } +func TestSpecIndex_GetNode_MissDoesNotLeakReadLock(t *testing.T) { + index := NewSpecIndexWithConfig(&yaml.Node{}, CreateOpenAPIIndexConfig()) + index.nodeMap = map[int]map[int]*yaml.Node{ + 1: {1: {Value: "ok"}}, + 2: nil, + } + + node, ok := index.GetNode(2, 1) + assert.False(t, ok) + assert.Nil(t, node) + + locked := make(chan struct{}) + go func() { + index.nodeMapLock.Lock() + index.nodeMap[3] = map[int]*yaml.Node{} + index.nodeMapLock.Unlock() + close(locked) + }() + + select { + case <-locked: + case <-time.After(250 * time.Millisecond): + t.Fatal("writer lock blocked after GetNode miss") + } +} + func BenchmarkSpecIndex_MapNodes(b *testing.B) { petstore, _ := os.ReadFile("../test_specs/petstorev3.json") var rootNode yaml.Node diff --git a/index/resolver.go b/index/resolver.go deleted file mode 100644 index 669f25fc3..000000000 --- a/index/resolver.go +++ /dev/null @@ -1,1051 +0,0 @@ -// Copyright 2022 Dave Shanley / Quobix -// SPDX-License-Identifier: MIT - -package index - -import ( - "context" - "errors" - "fmt" - "net/url" - "path" - "path/filepath" - "slices" - "sort" - "strings" - - "github.com/pb33f/libopenapi/utils" - "go.yaml.in/yaml/v4" -) - -// ResolvingError represents an issue the resolver had trying to stitch the tree together. -type ResolvingError struct { - // ErrorRef is the error thrown by the resolver - ErrorRef error - - // Node is the *yaml.Node reference that contains the resolving error - Node *yaml.Node - - // Path is the shortened journey taken by the resolver - Path string - - // CircularReference is set if the error is a reference to the circular reference. - CircularReference *CircularReferenceResult -} - -func (r *ResolvingError) Error() string { - errs := utils.UnwrapErrors(r.ErrorRef) - var msgs []string - for _, e := range errs { - var idxErr *IndexingError - if errors.As(e, &idxErr) { - msgs = append(msgs, fmt.Sprintf("%s: %s [%d:%d]", idxErr.Error(), - idxErr.Path, idxErr.Node.Line, idxErr.Node.Column)) - } else { - var l, c int - if r.Node != nil { - l = r.Node.Line - c = r.Node.Column - } - msgs = append(msgs, fmt.Sprintf("%s: %s [%d:%d]", e.Error(), - r.Path, l, c)) - } - } - return strings.Join(msgs, "\n") -} - -// Resolver will use a *index.SpecIndex to stitch together a resolved root tree using all the discovered -// references in the doc. -type Resolver struct { - specIndex *SpecIndex - resolvedRoot *yaml.Node - resolvingErrors []*ResolvingError - circularReferences []*CircularReferenceResult - ignoredPolyReferences []*CircularReferenceResult - ignoredArrayReferences []*CircularReferenceResult - referencesVisited int - indexesVisited int - journeysTaken int - relativesSeen int - IgnorePoly bool - IgnoreArray bool - circChecked bool -} - -// Release nils all fields that can pin YAML node trees or SpecIndex references in -// memory. Call this once all consumers of the resolver are finished. -func (resolver *Resolver) Release() { - if resolver == nil { - return - } - resolver.specIndex = nil - resolver.resolvedRoot = nil - resolver.resolvingErrors = nil - resolver.circularReferences = nil - resolver.ignoredPolyReferences = nil - resolver.ignoredArrayReferences = nil -} - -// NewResolver will create a new resolver from a *index.SpecIndex -func NewResolver(index *SpecIndex) *Resolver { - if index == nil { - return nil - } - r := &Resolver{ - specIndex: index, - resolvedRoot: index.GetRootNode(), - } - index.resolver = r - return r -} - -// GetIgnoredCircularPolyReferences returns all ignored circular references that are polymorphic -func (resolver *Resolver) GetIgnoredCircularPolyReferences() []*CircularReferenceResult { - return resolver.ignoredPolyReferences -} - -// GetIgnoredCircularArrayReferences returns all ignored circular references that are arrays -func (resolver *Resolver) GetIgnoredCircularArrayReferences() []*CircularReferenceResult { - return resolver.ignoredArrayReferences -} - -// GetResolvingErrors returns all errors found during resolving -func (resolver *Resolver) GetResolvingErrors() []*ResolvingError { - return resolver.resolvingErrors -} - -func (resolver *Resolver) GetCircularReferences() []*CircularReferenceResult { - return resolver.GetSafeCircularReferences() -} - -// GetSafeCircularReferences returns all circular reference errors found. -func (resolver *Resolver) GetSafeCircularReferences() []*CircularReferenceResult { - var refs []*CircularReferenceResult - for _, ref := range resolver.circularReferences { - if !ref.IsInfiniteLoop { - refs = append(refs, ref) - } - } - return refs -} - -// GetInfiniteCircularReferences returns all circular reference errors found that are infinite / unrecoverable -func (resolver *Resolver) GetInfiniteCircularReferences() []*CircularReferenceResult { - var refs []*CircularReferenceResult - for _, ref := range resolver.circularReferences { - if ref.IsInfiniteLoop { - refs = append(refs, ref) - } - } - return refs -} - -// GetPolymorphicCircularErrors returns all circular errors that stem from polymorphism -func (resolver *Resolver) GetPolymorphicCircularErrors() []*CircularReferenceResult { - var res []*CircularReferenceResult - for i := range resolver.circularReferences { - if !resolver.circularReferences[i].IsInfiniteLoop { - continue - } - if !resolver.circularReferences[i].IsPolymorphicResult { - continue - } - res = append(res, resolver.circularReferences[i]) - } - return res -} - -// GetNonPolymorphicCircularErrors returns all circular errors that DO NOT stem from polymorphism -func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*CircularReferenceResult { - var res []*CircularReferenceResult - for i := range resolver.circularReferences { - if !resolver.circularReferences[i].IsInfiniteLoop { - continue - } - - if !resolver.circularReferences[i].IsPolymorphicResult { - res = append(res, resolver.circularReferences[i]) - } - } - return res -} - -// IgnorePolymorphicCircularReferences will ignore any circular references that are polymorphic (oneOf, anyOf, allOf) -// This must be set before any resolving is done. -func (resolver *Resolver) IgnorePolymorphicCircularReferences() { - resolver.IgnorePoly = true -} - -// IgnoreArrayCircularReferences will ignore any circular references that stem from arrays. This must be set before -// any resolving is done. -func (resolver *Resolver) IgnoreArrayCircularReferences() { - resolver.IgnoreArray = true -} - -// GetJourneysTaken returns the number of journeys taken by the resolver -func (resolver *Resolver) GetJourneysTaken() int { - return resolver.journeysTaken -} - -// GetReferenceVisited returns the number of references visited by the resolver -func (resolver *Resolver) GetReferenceVisited() int { - return resolver.referencesVisited -} - -// GetIndexesVisited returns the number of indexes visited by the resolver -func (resolver *Resolver) GetIndexesVisited() int { - return resolver.indexesVisited -} - -// GetRelativesSeen returns the number of siblings (nodes at the same level) seen for each reference found. -func (resolver *Resolver) GetRelativesSeen() int { - return resolver.relativesSeen -} - -// Resolve will resolve the specification, everything that is not polymorphic and not circular, will be resolved. -// this data can get big, it results in a massive duplication of data. This is a destructive method and will permanently -// re-organize the node tree. Make sure you have copied your original tree before running this (if you want to preserve -// original data) -func (resolver *Resolver) Resolve() []*ResolvingError { - visitIndex(resolver, resolver.specIndex) - - for _, circRef := range resolver.circularReferences { - // If the circular reference is not required, we can ignore it, as it's a terminable loop rather than an infinite one - if !circRef.IsInfiniteLoop { - continue - } - - if !resolver.circChecked { - resolver.resolvingErrors = append(resolver.resolvingErrors, &ResolvingError{ - ErrorRef: fmt.Errorf("infinite circular reference detected: %s", circRef.Start.Definition), - Node: circRef.ParentNode, - Path: circRef.GenerateJourneyPath(), - CircularReference: circRef, - }) - } - } - resolver.specIndex.SetCircularReferences(resolver.circularReferences) - resolver.specIndex.SetIgnoredArrayCircularReferences(resolver.ignoredArrayReferences) - resolver.specIndex.SetIgnoredPolymorphicCircularReferences(resolver.ignoredPolyReferences) - resolver.circChecked = true - return resolver.resolvingErrors -} - -// CheckForCircularReferences Check for circular references, without resolving, a non-destructive run. -func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError { - visitIndexWithoutDamagingIt(resolver, resolver.specIndex) - for _, circRef := range resolver.circularReferences { - // If the circular reference is not required, we can ignore it, as it's a terminable loop rather than an infinite one - if !circRef.IsInfiniteLoop { - continue - } - if !resolver.circChecked { - resolver.resolvingErrors = append(resolver.resolvingErrors, &ResolvingError{ - ErrorRef: fmt.Errorf("infinite circular reference detected: %s", circRef.Start.Name), - Node: circRef.ParentNode, - Path: circRef.GenerateJourneyPath(), - CircularReference: circRef, - }) - } - } - // update our index with any circular refs we found. - resolver.specIndex.SetCircularReferences(resolver.circularReferences) - resolver.specIndex.SetIgnoredArrayCircularReferences(resolver.ignoredArrayReferences) - resolver.specIndex.SetIgnoredPolymorphicCircularReferences(resolver.ignoredPolyReferences) - resolver.circChecked = true - return resolver.resolvingErrors -} - -func visitIndexWithoutDamagingIt(res *Resolver, idx *SpecIndex) { - mapped := idx.GetMappedReferencesSequenced() - mappedIndex := idx.GetMappedReferences() - res.indexesVisited++ - for _, ref := range mapped { - seenReferences := make(map[string]bool) - var journey []*Reference - res.journeysTaken++ - res.VisitReference(ref.Reference, seenReferences, journey, false) - } - - // Sort schema keys for deterministic iteration order - schemas := idx.GetAllComponentSchemas() - schemaKeys := make([]string, 0, len(schemas)) - for k := range schemas { - schemaKeys = append(schemaKeys, k) - } - sort.Strings(schemaKeys) - - for _, s := range schemaKeys { - schemaRef := schemas[s] - if mappedIndex[s] == nil { - seenReferences := make(map[string]bool) - var journey []*Reference - res.journeysTaken++ - res.VisitReference(schemaRef, seenReferences, journey, false) - } - } -} - -type refMap struct { - ref *Reference - nodes []*yaml.Node -} - -func visitIndex(res *Resolver, idx *SpecIndex) { - mapped := idx.GetMappedReferencesSequenced() - mappedIndex := idx.GetMappedReferences() - res.indexesVisited++ - - var refs []refMap - for _, ref := range mapped { - seenReferences := make(map[string]bool) - var journey []*Reference - res.journeysTaken++ - if ref != nil && ref.Reference != nil { - n := res.VisitReference(ref.Reference, seenReferences, journey, true) - if !ref.Reference.Circular { - // make a note of the reference and map the original ref after we're done - if ok, _, _ := utils.IsNodeRefValue(ref.OriginalReference.Node); ok { - refs = append(refs, refMap{ - ref: ref.OriginalReference, - nodes: n, - }) - } - } - } - } - idx.pendingResolve = refs - - // Sort schema keys for deterministic iteration order - schemas := idx.GetAllComponentSchemas() - schemaKeys := make([]string, 0, len(schemas)) - for k := range schemas { - schemaKeys = append(schemaKeys, k) - } - sort.Strings(schemaKeys) - - for _, s := range schemaKeys { - schemaRef := schemas[s] - if mappedIndex[s] == nil { - seenReferences := make(map[string]bool) - var journey []*Reference - res.journeysTaken++ - schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true) - } - } - - // Sort security scheme keys for deterministic iteration order - securitySchemes := idx.GetAllSecuritySchemes() - securityKeys := make([]string, 0, len(securitySchemes)) - for k := range securitySchemes { - securityKeys = append(securityKeys, k) - } - sort.Strings(securityKeys) - - for _, s := range securityKeys { - schemaRef := securitySchemes[s] - if mappedIndex[s] == nil { - seenReferences := make(map[string]bool) - var journey []*Reference - res.journeysTaken++ - schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true) - } - } - - // map everything - for _, sequenced := range idx.GetAllSequencedReferences() { - locatedDef := mappedIndex[sequenced.FullDefinition] - if locatedDef != nil { - if !locatedDef.Circular { - sequenced.Node.Content = locatedDef.Node.Content - } - } - } -} - -// searchReferenceWithContext resolves a reference using document context when enabled in the config. -func (resolver *Resolver) searchReferenceWithContext(sourceRef, searchRef *Reference) (*Reference, *SpecIndex, context.Context) { - if resolver.specIndex == nil || resolver.specIndex.config == nil || !resolver.specIndex.config.ResolveNestedRefsWithDocumentContext { - ref, idx := resolver.specIndex.SearchIndexForReferenceByReference(searchRef) - return ref, idx, context.Background() - } - - searchIndex := resolver.specIndex - if searchRef != nil && searchRef.Index != nil { - searchIndex = searchRef.Index - } else if sourceRef != nil && sourceRef.Index != nil { - searchIndex = sourceRef.Index - } - - ctx := context.Background() - currentPath := "" - if sourceRef != nil { - currentPath = sourceRef.RemoteLocation - } - if currentPath == "" && searchIndex != nil { - currentPath = searchIndex.specAbsolutePath - } - if currentPath != "" { - ctx = context.WithValue(ctx, CurrentPathKey, currentPath) - } - if searchRef != nil || sourceRef != nil { - base := "" - if searchRef != nil && searchRef.SchemaIdBase != "" { - base = searchRef.SchemaIdBase - } else if sourceRef != nil && sourceRef.SchemaIdBase != "" { - base = sourceRef.SchemaIdBase - } - if base != "" { - scope := NewSchemaIdScope(base) - scope.PushId(base) - ctx = WithSchemaIdScope(ctx, scope) - } - } - - return searchIndex.SearchIndexForReferenceByReferenceWithContext(ctx, searchRef) -} - -// VisitReference will visit a reference as part of a journey and will return resolved nodes. -func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, journey []*Reference, resolve bool) []*yaml.Node { - resolver.referencesVisited++ - if resolve && ref != nil && ref.Seen { - if ref.Resolved { - return ref.Node.Content - } - } - if !resolve && ref != nil && ref.Seen { - return ref.Node.Content - } - if ref != nil { - journey = append(journey, ref) - seenRelatives := make(map[int]bool) - base := resolver.resolveSchemaIdBase(ref.SchemaIdBase, ref.Node) - relatives := resolver.extractRelatives(ref, ref.Node, nil, seen, journey, seenRelatives, resolve, 0, base) - - seen = make(map[string]bool) - - seen[ref.FullDefinition] = true - for _, r := range relatives { - // check if we have seen this on the journey before, if so! it's circular - skip := false - for i, j := range journey { - if j.FullDefinition == r.FullDefinition { - - var foundDup *Reference - foundRef, _, _ := resolver.searchReferenceWithContext(ref, r) - if foundRef != nil { - foundDup = foundRef - } - - var circRef *CircularReferenceResult - if !foundDup.Circular { - loop := append(journey, foundDup) - - visitedDefinitions := make(map[string]bool) - isInfiniteLoop, _ := resolver.isInfiniteCircularDependency(foundDup, - visitedDefinitions, nil) - - isArray := false - if r.ParentNodeSchemaType == "array" || slices.Contains(r.ParentNodeTypes, "array") { - isArray = true - } - circRef = &CircularReferenceResult{ - ParentNode: foundDup.ParentNode, - Journey: loop, - Start: foundDup, - LoopIndex: i, - LoopPoint: foundDup, - IsArrayResult: isArray, - IsInfiniteLoop: isInfiniteLoop, - } - - if resolver.IgnorePoly && !isArray { - resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) - } else if resolver.IgnoreArray && isArray { - resolver.ignoredArrayReferences = append(resolver.ignoredArrayReferences, circRef) - } else { - if !resolver.circChecked { - resolver.circularReferences = append(resolver.circularReferences, circRef) - } - } - r.Seen = true - r.Circular = true - foundDup.Seen = true - foundDup.Circular = true - } - skip = true - } - } - - if !skip { - var original *Reference - foundRef, _, _ := resolver.searchReferenceWithContext(ref, r) - if foundRef != nil { - original = foundRef - } - resolved := resolver.VisitReference(original, seen, journey, resolve) - if resolve && !original.Circular { - ref.Resolved = true - r.Resolved = true - r.Node.Content = resolved // this is where we perform the actual resolving. - } - r.Seen = true - ref.Seen = true - } - } - - ref.Seen = true - - if ref.Node != nil { - return ref.Node.Content - } - } - return nil -} - -func (resolver *Resolver) isInfiniteCircularDependency(ref *Reference, visitedDefinitions map[string]bool, - initialRef *Reference, -) (bool, map[string]bool) { - if ref == nil { - return false, visitedDefinitions - } - for refDefinition := range ref.RequiredRefProperties { - r, _ := resolver.specIndex.SearchIndexForReference(refDefinition) - if initialRef != nil && initialRef.FullDefinition == r.FullDefinition { - return true, visitedDefinitions - } - if len(visitedDefinitions) > 0 && ref.FullDefinition == r.FullDefinition { - return true, visitedDefinitions - } - - if visitedDefinitions[r.FullDefinition] { - continue - } - - visitedDefinitions[r.FullDefinition] = true - - ir := initialRef - if ir == nil { - ir = ref - } - - var isChildICD bool - - isChildICD, visitedDefinitions = resolver.isInfiniteCircularDependency(r, visitedDefinitions, ir) - if isChildICD { - return true, visitedDefinitions - } - } - - return false, visitedDefinitions -} - -func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.Node, - foundRelatives map[string]bool, - journey []*Reference, seen map[int]bool, resolve bool, depth int, schemaIdBase string, -) []*Reference { - if len(journey) > 100 { - return nil - } - - // this is a safety check to prevent a stack overflow. - if depth > 500 { - def := "unknown" - if ref != nil { - def = ref.FullDefinition - } - if resolver.specIndex != nil && resolver.specIndex.logger != nil { - resolver.specIndex.logger.Warn("libopenapi resolver: relative depth exceeded 100 levels, "+ - "check for circular references - resolving may be incomplete", - "reference", def) - } - - loop := append(journey, ref) - circRef := &CircularReferenceResult{ - Journey: loop, - Start: ref, - LoopIndex: depth, - LoopPoint: ref, - IsInfiniteLoop: true, - } - if !resolver.circChecked { - resolver.circularReferences = append(resolver.circularReferences, circRef) - ref.Circular = true - } - return nil - } - - currentBase := resolver.resolveSchemaIdBase(schemaIdBase, node) - var found []*Reference - - if node != nil && len(node.Content) > 0 { - skip := false - for i, n := range node.Content { - if skip { - skip = false - continue - } - if utils.IsNodeMap(n) || utils.IsNodeArray(n) { - depth++ - - var foundRef *Reference - foundRef, _, _ = resolver.searchReferenceWithContext(ref, ref) - if foundRef != nil && !foundRef.Circular { - found = append(found, resolver.extractRelatives(foundRef, n, node, foundRelatives, journey, seen, resolve, depth, currentBase)...) - depth-- - } - if foundRef == nil { - found = append(found, resolver.extractRelatives(ref, n, node, foundRelatives, journey, seen, resolve, depth, currentBase)...) - depth-- - } - - } - - if i%2 == 0 && n.Value == "$ref" && len(node.Content) > i%2+1 { - - if !utils.IsNodeStringValue(node.Content[i+1]) { - continue - } - // issue #481 cannot look at an array value, the next not is not the value! - if utils.IsNodeArray(node) { - continue - } - - value := node.Content[i+1].Value - value = strings.ReplaceAll(value, "\\\\", "\\") - - // If SkipExternalRefResolution is enabled, skip external refs entirely - if resolver.specIndex != nil && resolver.specIndex.config != nil && - resolver.specIndex.config.SkipExternalRefResolution && utils.IsExternalRef(value) { - skip = true - continue - } - - var locatedRef *Reference - var fullDef string - var definition string - - // explode value - exp := strings.Split(value, "#/") - if len(exp) == 2 { - definition = fmt.Sprintf("#/%s", exp[1]) - if exp[0] != "" { - if strings.HasPrefix(exp[0], "http") { - fullDef = value - } else { - if strings.HasPrefix(ref.FullDefinition, "http") { - - // split the http URI into parts - httpExp := strings.Split(ref.FullDefinition, "#/") - - u, _ := url.Parse(httpExp[0]) - abs, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), exp[0], string(filepath.Separator))) - u.Path = utils.ReplaceWindowsDriveWithLinuxPath(abs) - u.Fragment = "" - fullDef = fmt.Sprintf("%s#/%s", u.String(), exp[1]) - - } else { - - // split the referring ref full def into parts - fileDef := strings.Split(ref.FullDefinition, "#/") - - // extract the location of the ref and build a full def path. - abs := resolver.resolveLocalRefPath(filepath.Dir(fileDef[0]), exp[0]) - // abs = utils.ReplaceWindowsDriveWithLinuxPath(abs) - fullDef = fmt.Sprintf("%s#/%s", abs, exp[1]) - - } - } - } else { - // local component, full def is based on passed in ref - baseLocation := ref.FullDefinition - if ref.RemoteLocation != "" { - baseLocation = ref.RemoteLocation - } - if strings.HasPrefix(baseLocation, "http") { - - // split the http URI into parts - httpExp := strings.Split(baseLocation, "#/") - - // parse a URL from the full def - u, _ := url.Parse(httpExp[0]) - - // extract the location of the ref and build a full def path. - fullDef = fmt.Sprintf("%s#/%s", u.String(), exp[1]) - - } else { - // split the full def into parts - fileDef := strings.Split(baseLocation, "#/") - fullDef = fmt.Sprintf("%s#/%s", fileDef[0], exp[1]) - } - } - } else { - - definition = value - - // if the reference is a http link - if strings.HasPrefix(value, "http") { - fullDef = value - } else { - - // split the full def into parts - baseLocation := ref.FullDefinition - if ref.RemoteLocation != "" { - baseLocation = ref.RemoteLocation - } - fileDef := strings.Split(baseLocation, "#/") - - // is the file def a http link? - if strings.HasPrefix(fileDef[0], "http") { - u, _ := url.Parse(fileDef[0]) - absPath, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), exp[0], string(filepath.Separator))) - u.Path = utils.ReplaceWindowsDriveWithLinuxPath(absPath) - fullDef = u.String() - - } else { - fullDef = resolver.resolveLocalRefPath(filepath.Dir(fileDef[0]), exp[0]) - } - - } - } - - if currentBase != "" { - fullDef = resolveRefWithSchemaBase(value, currentBase) - } - - searchRef := &Reference{ - Definition: definition, - FullDefinition: fullDef, - RawRef: value, - SchemaIdBase: currentBase, - RemoteLocation: ref.RemoteLocation, - IsRemote: true, - Index: ref.Index, - } - - locatedRef, _, _ = resolver.searchReferenceWithContext(ref, searchRef) - - if locatedRef == nil { - _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value) - err := &ResolvingError{ - ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value), - Node: n, - Path: path, - } - resolver.resolvingErrors = append(resolver.resolvingErrors, err) - continue - } - - if resolve { - // if this is a reference also, we want to resolve it. - if ok, _, _ := utils.IsNodeRefValue(ref.Node); ok { - ref.Node.Content = locatedRef.Node.Content - ref.Resolved = true - } - } - - schemaType := "" - if parent != nil { - _, arrayTypevn := utils.FindKeyNodeTop("type", parent.Content) - if arrayTypevn != nil { - if arrayTypevn.Value == "array" { - schemaType = "array" - } - } - } - if ref.ParentNodeSchemaType != "" { - locatedRef.ParentNodeTypes = append(locatedRef.ParentNodeTypes, ref.ParentNodeSchemaType) - } - locatedRef.ParentNodeSchemaType = schemaType - found = append(found, locatedRef) - foundRelatives[value] = true - } - - if i%2 == 0 && n.Value != "$ref" && n.Value != "" { - // Check if we're inside a properties object - isInsideProperties := false - if parent != nil { - for j := 0; j < len(parent.Content); j += 2 { - if j < len(parent.Content) && parent.Content[j].Value == "properties" { - isInsideProperties = true - break - } - } - } - - // Only treat as polymorphic keywords if not inside properties - if !isInsideProperties && (n.Value == "allOf" || n.Value == "oneOf" || n.Value == "anyOf") { - - // if this is a polymorphic link, we want to follow it and see if it becomes circular - if i+1 < len(node.Content) && utils.IsNodeMap(node.Content[i+1]) { // check for nested items - // check if items is present, to indicate an array - if k, v := utils.FindKeyNodeTop("items", node.Content[i+1].Content); v != nil { - if utils.IsNodeMap(v) { - if d, _, l := utils.IsNodeRefValue(v); d { - - // create full definition lookup based on ref. - def := resolver.buildDefPathWithSchemaBase(ref, l, currentBase) - - mappedRefs, _ := resolver.specIndex.SearchIndexForReference(def) - if mappedRefs != nil && !mappedRefs.Circular { - circ := false - for f := range journey { - if journey[f].FullDefinition == mappedRefs.FullDefinition { - circ = true - break - } - } - if !circ { - resolver.VisitReference(mappedRefs, foundRelatives, journey, resolve) - } else { - loop := append(journey, mappedRefs) - circRef := &CircularReferenceResult{ - ParentNode: k, - Journey: loop, - Start: mappedRefs, - LoopIndex: i, - LoopPoint: mappedRefs, - PolymorphicType: n.Value, - IsPolymorphicResult: true, - } - - mappedRefs.Seen = true - mappedRefs.Circular = true - if resolver.IgnorePoly { - resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) - } else { - if !resolver.circChecked { - resolver.circularReferences = append(resolver.circularReferences, circRef) - } - } - } - } - } - } - } else { - // no items discovered, continue on and investigate anyway. - v := node.Content[i+1] - if utils.IsNodeMap(v) { - if d, _, l := utils.IsNodeRefValue(v); d { - - // create full definition lookup based on ref. - def := resolver.buildDefPathWithSchemaBase(ref, l, currentBase) - - mappedRefs, _ := resolver.specIndex.SearchIndexForReference(def) - if mappedRefs != nil && !mappedRefs.Circular { - circ := false - for f := range journey { - if journey[f].FullDefinition == mappedRefs.FullDefinition { - circ = true - break - } - } - if !circ { - resolver.VisitReference(mappedRefs, foundRelatives, journey, resolve) - } else { - loop := append(journey, mappedRefs) - circRef := &CircularReferenceResult{ - ParentNode: node.Content[i], - Journey: loop, - Start: mappedRefs, - LoopIndex: i, - LoopPoint: mappedRefs, - PolymorphicType: n.Value, - IsPolymorphicResult: true, - } - - mappedRefs.Seen = true - mappedRefs.Circular = true - if resolver.IgnorePoly { - resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) - } else { - if !resolver.circChecked { - resolver.circularReferences = append(resolver.circularReferences, circRef) - } - } - } - } - } - } - } - } - // for array based polymorphic items - if i+1 < len(node.Content) && utils.IsNodeArray(node.Content[i+1]) { // check for nested items - for q := range node.Content[i+1].Content { - v := node.Content[i+1].Content[q] - if utils.IsNodeMap(v) { - if d, _, l := utils.IsNodeRefValue(v); d { - def := resolver.buildDefPathWithSchemaBase(ref, l, currentBase) - mappedRefs, _ := resolver.specIndex.SearchIndexForReference(def) - if mappedRefs != nil && !mappedRefs.Circular { - circ := false - for f := range journey { - if journey[f].FullDefinition == mappedRefs.FullDefinition { - circ = true - break - } - } - if !circ { - resolver.VisitReference(mappedRefs, foundRelatives, journey, resolve) - } else { - loop := append(journey, mappedRefs) - - circRef := &CircularReferenceResult{ - ParentNode: node.Content[i], - Journey: loop, - Start: mappedRefs, - LoopIndex: i, - LoopPoint: mappedRefs, - PolymorphicType: n.Value, - IsPolymorphicResult: true, - } - - mappedRefs.Seen = true - mappedRefs.Circular = true - if resolver.IgnorePoly { - resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) - } else { - if !resolver.circChecked { - resolver.circularReferences = append(resolver.circularReferences, circRef) - } - } - } - } - } else { - depth++ - found = append(found, resolver.extractRelatives(ref, v, n, - foundRelatives, journey, seen, resolve, depth, currentBase)...) - } - } - } - } - skip = true - continue - } - } - } - } - resolver.relativesSeen += len(found) - return found -} - -func (resolver *Resolver) buildDefPath(ref *Reference, l string) string { - def := "" - exp := strings.Split(l, "#/") - if len(exp) == 2 { - if exp[0] != "" { - if !strings.HasPrefix(exp[0], "http") { - if !filepath.IsAbs(exp[0]) { - if strings.HasPrefix(ref.FullDefinition, "http") { - - u, _ := url.Parse(ref.FullDefinition) - p, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), exp[0], string(filepath.Separator))) - u.Path = utils.ReplaceWindowsDriveWithLinuxPath(p) - def = fmt.Sprintf("%s#/%s", u.String(), exp[1]) - - } else { - z := strings.Split(ref.FullDefinition, "#/") - if len(z) == 2 { - if len(z[0]) > 0 { - abs := resolver.resolveLocalRefPath(filepath.Dir(z[0]), exp[0]) - def = fmt.Sprintf("%s#/%s", abs, exp[1]) - } else { - abs, _ := filepath.Abs(exp[0]) - def = fmt.Sprintf("%s#/%s", abs, exp[1]) - } - } else { - abs := resolver.resolveLocalRefPath(filepath.Dir(ref.FullDefinition), exp[0]) - def = fmt.Sprintf("%s#/%s", abs, exp[1]) - } - } - } - } else { - if len(exp[1]) > 0 { - def = l - } else { - def = exp[0] - } - } - } else { - if strings.HasPrefix(ref.FullDefinition, "http") { - u, _ := url.Parse(ref.FullDefinition) - u.Fragment = "" - def = fmt.Sprintf("%s#/%s", u.String(), exp[1]) - - } else { - if strings.HasPrefix(ref.FullDefinition, "#/") { - def = fmt.Sprintf("#/%s", exp[1]) - } else { - fdexp := strings.Split(ref.FullDefinition, "#/") - def = fmt.Sprintf("%s#/%s", fdexp[0], exp[1]) - } - } - } - } else { - if strings.HasPrefix(l, "http") { - def = l - } else { - // check if were dealing with a remote file - if strings.HasPrefix(ref.FullDefinition, "http") { - - // split the url. - u, _ := url.Parse(ref.FullDefinition) - abs, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), l, string(filepath.Separator))) - u.Path = utils.ReplaceWindowsDriveWithLinuxPath(abs) - u.Fragment = "" - def = u.String() - } else { - lookupRef := strings.Split(ref.FullDefinition, "#/") - abs := resolver.resolveLocalRefPath(filepath.Dir(lookupRef[0]), l) - def = abs - } - } - } - - return def -} - -func (resolver *Resolver) resolveLocalRefPath(base, ref string) string { - if resolver != nil && resolver.specIndex != nil { - return resolver.specIndex.ResolveRelativeFilePath(base, ref) - } - abs, _ := filepath.Abs(utils.CheckPathOverlap(base, ref, string(filepath.Separator))) - return abs -} - -func (resolver *Resolver) buildDefPathWithSchemaBase(ref *Reference, l string, schemaIdBase string) string { - if schemaIdBase != "" { - normalized := resolveRefWithSchemaBase(l, schemaIdBase) - if normalized != l { - return normalized - } - } - return resolver.buildDefPath(ref, l) -} - -func (resolver *Resolver) resolveSchemaIdBase(parentBase string, node *yaml.Node) string { - if node == nil { - return parentBase - } - idValue := FindSchemaIdInNode(node) - if idValue == "" { - return parentBase - } - base := parentBase - if base == "" && resolver.specIndex != nil { - base = resolver.specIndex.specAbsolutePath - } - resolved, err := ResolveSchemaId(idValue, base) - if err != nil || resolved == "" { - return idValue - } - return resolved -} - -func (resolver *Resolver) ResolvePendingNodes() { - // map everything afterwards - for _, r := range resolver.specIndex.pendingResolve { - // r.Node.Content = refs[r].nodes - r.ref.Node.Content = r.nodes - } -} diff --git a/index/resolver_circular.go b/index/resolver_circular.go new file mode 100644 index 000000000..52bdfd315 --- /dev/null +++ b/index/resolver_circular.go @@ -0,0 +1,122 @@ +// Copyright 2022 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +func (resolver *Resolver) handleCircularJourneyRelative(ref, relative *Reference, journey []*Reference) bool { + for loopIndex, journeyRef := range journey { + if journeyRef.FullDefinition != relative.FullDefinition { + continue + } + + foundDup, _, _ := resolver.searchReferenceWithContext(ref, relative) + if foundDup == nil { + return true + } + if foundDup.Circular { + return true + } + + circRef := resolver.buildCircularReferenceResult(foundDup, relative, journey, loopIndex) + resolver.recordCircularReferenceResult(circRef) + resolver.markReferencesCircular(relative, foundDup) + return true + } + return false +} + +func (resolver *Resolver) buildCircularReferenceResult( + foundDup, relative *Reference, + journey []*Reference, + loopIndex int, +) *CircularReferenceResult { + loop := append(journey, foundDup) + visitedDefinitions := make(map[string]bool) + isInfiniteLoop, _ := resolver.isInfiniteCircularDependency(foundDup, visitedDefinitions, nil) + + return &CircularReferenceResult{ + ParentNode: foundDup.ParentNode, + Journey: loop, + Start: foundDup, + LoopIndex: loopIndex, + LoopPoint: foundDup, + IsArrayResult: resolver.relativeIsArrayResult(relative), + IsInfiniteLoop: isInfiniteLoop, + } +} + +func (resolver *Resolver) recordCircularReferenceResult(circRef *CircularReferenceResult) { + if circRef == nil { + return + } + if resolver.IgnorePoly && !circRef.IsArrayResult { + resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) + return + } + if resolver.IgnoreArray && circRef.IsArrayResult { + resolver.ignoredArrayReferences = append(resolver.ignoredArrayReferences, circRef) + return + } + if !resolver.circChecked { + resolver.circularReferences = append(resolver.circularReferences, circRef) + } +} + +func (resolver *Resolver) markReferencesCircular(relative, duplicate *Reference) { + if relative != nil { + relative.Seen = true + relative.Circular = true + } + if duplicate != nil { + duplicate.Seen = true + duplicate.Circular = true + } +} + +func (resolver *Resolver) relativeIsArrayResult(relative *Reference) bool { + if relative == nil { + return false + } + if relative.ParentNodeSchemaType == "array" { + return true + } + for _, nodeType := range relative.ParentNodeTypes { + if nodeType == "array" { + return true + } + } + return false +} + +func (resolver *Resolver) isInfiniteCircularDependency( + ref *Reference, visitedDefinitions map[string]bool, initialRef *Reference, +) (bool, map[string]bool) { + if ref == nil { + return false, visitedDefinitions + } + for refDefinition := range ref.RequiredRefProperties { + r, _ := resolver.specIndex.SearchIndexForReference(refDefinition) + if initialRef != nil && initialRef.FullDefinition == r.FullDefinition { + return true, visitedDefinitions + } + if len(visitedDefinitions) > 0 && ref.FullDefinition == r.FullDefinition { + return true, visitedDefinitions + } + if visitedDefinitions[r.FullDefinition] { + continue + } + + visitedDefinitions[r.FullDefinition] = true + ir := initialRef + if ir == nil { + ir = ref + } + + isChildICD, visitedDefinitions := resolver.isInfiniteCircularDependency(r, visitedDefinitions, ir) + if isChildICD { + return true, visitedDefinitions + } + } + + return false, visitedDefinitions +} diff --git a/index/resolver_entry.go b/index/resolver_entry.go new file mode 100644 index 000000000..24bb5c7e0 --- /dev/null +++ b/index/resolver_entry.go @@ -0,0 +1,216 @@ +// Copyright 2022 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import ( + "errors" + "fmt" + "strings" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +// ResolvingError represents an issue the resolver had trying to stitch the tree together. +type ResolvingError struct { + ErrorRef error + Node *yaml.Node + Path string + + // CircularReference is set if the error is a reference to the circular reference. + CircularReference *CircularReferenceResult +} + +func (r *ResolvingError) Error() string { + errs := utils.UnwrapErrors(r.ErrorRef) + var msgs []string + for _, e := range errs { + var idxErr *IndexingError + if errors.As(e, &idxErr) { + msgs = append(msgs, fmt.Sprintf("%s: %s [%d:%d]", idxErr.Error(), + idxErr.Path, idxErr.Node.Line, idxErr.Node.Column)) + } else { + var l, c int + if r.Node != nil { + l = r.Node.Line + c = r.Node.Column + } + msgs = append(msgs, fmt.Sprintf("%s: %s [%d:%d]", e.Error(), r.Path, l, c)) + } + } + return strings.Join(msgs, "\n") +} + +// Resolver will use a *index.SpecIndex to stitch together a resolved root tree using all the discovered +// references in the doc. +type Resolver struct { + specIndex *SpecIndex + resolvedRoot *yaml.Node + resolvingErrors []*ResolvingError + circularReferences []*CircularReferenceResult + ignoredPolyReferences []*CircularReferenceResult + ignoredArrayReferences []*CircularReferenceResult + referencesVisited int + indexesVisited int + journeysTaken int + relativesSeen int + IgnorePoly bool + IgnoreArray bool + circChecked bool +} + +func (resolver *Resolver) Release() { + if resolver == nil { + return + } + resolver.specIndex = nil + resolver.resolvedRoot = nil + resolver.resolvingErrors = nil + resolver.circularReferences = nil + resolver.ignoredPolyReferences = nil + resolver.ignoredArrayReferences = nil +} + +func NewResolver(index *SpecIndex) *Resolver { + if index == nil { + return nil + } + r := &Resolver{ + specIndex: index, + resolvedRoot: index.GetRootNode(), + } + index.SetResolver(r) + return r +} + +func (resolver *Resolver) GetIgnoredCircularPolyReferences() []*CircularReferenceResult { + return resolver.ignoredPolyReferences +} + +func (resolver *Resolver) GetIgnoredCircularArrayReferences() []*CircularReferenceResult { + return resolver.ignoredArrayReferences +} + +func (resolver *Resolver) GetResolvingErrors() []*ResolvingError { + return resolver.resolvingErrors +} + +func (resolver *Resolver) GetCircularReferences() []*CircularReferenceResult { + return resolver.GetSafeCircularReferences() +} + +func (resolver *Resolver) GetSafeCircularReferences() []*CircularReferenceResult { + var refs []*CircularReferenceResult + for _, ref := range resolver.circularReferences { + if !ref.IsInfiniteLoop { + refs = append(refs, ref) + } + } + return refs +} + +func (resolver *Resolver) GetInfiniteCircularReferences() []*CircularReferenceResult { + var refs []*CircularReferenceResult + for _, ref := range resolver.circularReferences { + if ref.IsInfiniteLoop { + refs = append(refs, ref) + } + } + return refs +} + +func (resolver *Resolver) GetPolymorphicCircularErrors() []*CircularReferenceResult { + var res []*CircularReferenceResult + for i := range resolver.circularReferences { + if !resolver.circularReferences[i].IsInfiniteLoop { + continue + } + if !resolver.circularReferences[i].IsPolymorphicResult { + continue + } + res = append(res, resolver.circularReferences[i]) + } + return res +} + +func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*CircularReferenceResult { + var res []*CircularReferenceResult + for i := range resolver.circularReferences { + if !resolver.circularReferences[i].IsInfiniteLoop { + continue + } + if !resolver.circularReferences[i].IsPolymorphicResult { + res = append(res, resolver.circularReferences[i]) + } + } + return res +} + +func (resolver *Resolver) IgnorePolymorphicCircularReferences() { + resolver.IgnorePoly = true +} + +func (resolver *Resolver) IgnoreArrayCircularReferences() { + resolver.IgnoreArray = true +} + +func (resolver *Resolver) GetJourneysTaken() int { + return resolver.journeysTaken +} + +func (resolver *Resolver) GetReferenceVisited() int { + return resolver.referencesVisited +} + +func (resolver *Resolver) GetIndexesVisited() int { + return resolver.indexesVisited +} + +func (resolver *Resolver) GetRelativesSeen() int { + return resolver.relativesSeen +} + +func (resolver *Resolver) Resolve() []*ResolvingError { + visitIndex(resolver, resolver.specIndex) + for _, circRef := range resolver.circularReferences { + if !circRef.IsInfiniteLoop { + continue + } + if !resolver.circChecked { + resolver.resolvingErrors = append(resolver.resolvingErrors, &ResolvingError{ + ErrorRef: fmt.Errorf("infinite circular reference detected: %s", circRef.Start.Definition), + Node: circRef.ParentNode, + Path: circRef.GenerateJourneyPath(), + CircularReference: circRef, + }) + } + } + resolver.specIndex.SetCircularReferences(resolver.circularReferences) + resolver.specIndex.SetIgnoredArrayCircularReferences(resolver.ignoredArrayReferences) + resolver.specIndex.SetIgnoredPolymorphicCircularReferences(resolver.ignoredPolyReferences) + resolver.circChecked = true + return resolver.resolvingErrors +} + +func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError { + visitIndexWithoutDamagingIt(resolver, resolver.specIndex) + for _, circRef := range resolver.circularReferences { + if !circRef.IsInfiniteLoop { + continue + } + if !resolver.circChecked { + resolver.resolvingErrors = append(resolver.resolvingErrors, &ResolvingError{ + ErrorRef: fmt.Errorf("infinite circular reference detected: %s", circRef.Start.Name), + Node: circRef.ParentNode, + Path: circRef.GenerateJourneyPath(), + CircularReference: circRef, + }) + } + } + resolver.specIndex.SetCircularReferences(resolver.circularReferences) + resolver.specIndex.SetIgnoredArrayCircularReferences(resolver.ignoredArrayReferences) + resolver.specIndex.SetIgnoredPolymorphicCircularReferences(resolver.ignoredPolyReferences) + resolver.circChecked = true + return resolver.resolvingErrors +} diff --git a/index/resolver_mutation.go b/index/resolver_mutation.go new file mode 100644 index 000000000..5185286eb --- /dev/null +++ b/index/resolver_mutation.go @@ -0,0 +1,76 @@ +// Copyright 2022 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import "go.yaml.in/yaml/v4" + +func (resolver *Resolver) visitReferenceShortCircuit(ref *Reference, resolve bool) ([]*yaml.Node, bool) { + if ref == nil { + return nil, true + } + if resolve && ref.Seen { + if ref.Resolved { + if ref.Node != nil { + return ref.Node.Content, true + } + return nil, true + } + } + if !resolve && ref.Seen { + if ref.Node != nil { + return ref.Node.Content, true + } + return nil, true + } + return nil, false +} + +func (resolver *Resolver) collectReferenceRelatives( + ref *Reference, + seen map[string]bool, + journey []*Reference, + resolve bool, +) []*Reference { + base := resolver.resolveSchemaIdBase(ref.SchemaIdBase, ref.Node) + return resolver.extractRelatives(ref, ref.Node, nil, seen, journey, resolve, 0, base) +} + +func (resolver *Resolver) visitReferenceRelatives( + ref *Reference, + relatives []*Reference, + seen map[string]bool, + journey []*Reference, + resolve bool, +) { + for _, relative := range relatives { + if resolver.handleCircularJourneyRelative(ref, relative, journey) { + continue + } + resolver.resolveRelativeReference(ref, relative, seen, journey, resolve) + } +} + +func (resolver *Resolver) resolveRelativeReference( + ref, relative *Reference, + seen map[string]bool, + journey []*Reference, + resolve bool, +) { + original := relative + foundRef, _, _ := resolver.searchReferenceWithContext(ref, relative) + if foundRef != nil { + original = foundRef + } + + resolved := resolver.VisitReference(original, seen, journey, resolve) + if resolve && original != nil && !original.Circular { + ref.Resolved = true + relative.Resolved = true + if relative.Node != nil { + relative.Node.Content = resolved + } + } + relative.Seen = true + ref.Seen = true +} diff --git a/index/resolver_paths.go b/index/resolver_paths.go new file mode 100644 index 000000000..4f27919d2 --- /dev/null +++ b/index/resolver_paths.go @@ -0,0 +1,119 @@ +// Copyright 2022 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import ( + "net/url" + "path" + "path/filepath" + "strings" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +func (resolver *Resolver) buildDefPath(ref *Reference, l string) string { + def := "" + exp := strings.Split(l, "#/") + if len(exp) == 2 { + if exp[0] != "" { + if !strings.HasPrefix(exp[0], "http") { + if !filepath.IsAbs(exp[0]) { + if strings.HasPrefix(ref.FullDefinition, "http") { + u, _ := url.Parse(ref.FullDefinition) + p, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), exp[0], string(filepath.Separator))) + u.Path = utils.ReplaceWindowsDriveWithLinuxPath(p) + def = l + if len(exp[1]) > 0 { + def = u.String() + "#/" + exp[1] + } + } else { + z := strings.Split(ref.FullDefinition, "#/") + if len(z) == 2 { + if len(z[0]) > 0 { + abs := resolver.resolveLocalRefPath(filepath.Dir(z[0]), exp[0]) + def = abs + "#/" + exp[1] + } else { + abs, _ := filepath.Abs(exp[0]) + def = abs + "#/" + exp[1] + } + } else { + abs := resolver.resolveLocalRefPath(filepath.Dir(ref.FullDefinition), exp[0]) + def = abs + "#/" + exp[1] + } + } + } + } else if len(exp[1]) > 0 { + def = l + } else { + def = exp[0] + } + } else if strings.HasPrefix(ref.FullDefinition, "http") { + u, _ := url.Parse(ref.FullDefinition) + u.Fragment = "" + def = u.String() + "#/" + exp[1] + } else if strings.HasPrefix(ref.FullDefinition, "#/") { + def = "#/" + exp[1] + } else { + fdexp := strings.Split(ref.FullDefinition, "#/") + def = fdexp[0] + "#/" + exp[1] + } + } else if strings.HasPrefix(l, "http") { + def = l + } else if strings.HasPrefix(ref.FullDefinition, "http") { + u, _ := url.Parse(ref.FullDefinition) + abs, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), l, string(filepath.Separator))) + u.Path = utils.ReplaceWindowsDriveWithLinuxPath(abs) + u.Fragment = "" + def = u.String() + } else { + lookupRef := strings.Split(ref.FullDefinition, "#/") + def = resolver.resolveLocalRefPath(filepath.Dir(lookupRef[0]), l) + } + + return def +} + +func (resolver *Resolver) resolveLocalRefPath(base, ref string) string { + if resolver != nil && resolver.specIndex != nil { + return resolver.specIndex.ResolveRelativeFilePath(base, ref) + } + abs, _ := filepath.Abs(utils.CheckPathOverlap(base, ref, string(filepath.Separator))) + return abs +} + +func (resolver *Resolver) buildDefPathWithSchemaBase(ref *Reference, l string, schemaIDBase string) string { + if schemaIDBase != "" { + normalized := resolveRefWithSchemaBase(l, schemaIDBase) + if normalized != l { + return normalized + } + } + return resolver.buildDefPath(ref, l) +} + +func (resolver *Resolver) resolveSchemaIdBase(parentBase string, node *yaml.Node) string { + if node == nil { + return parentBase + } + idValue := FindSchemaIdInNode(node) + if idValue == "" { + return parentBase + } + base := parentBase + if base == "" && resolver.specIndex != nil { + base = resolver.specIndex.specAbsolutePath + } + resolved, err := ResolveSchemaId(idValue, base) + if err != nil || resolved == "" { + return idValue + } + return resolved +} + +func (resolver *Resolver) ResolvePendingNodes() { + for _, r := range resolver.specIndex.pendingResolve { + r.ref.Node.Content = r.nodes + } +} diff --git a/index/resolver_polymorphic.go b/index/resolver_polymorphic.go new file mode 100644 index 000000000..104dabbb7 --- /dev/null +++ b/index/resolver_polymorphic.go @@ -0,0 +1,93 @@ +// Copyright 2022 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import ( + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +func (resolver *Resolver) extractPolymorphicRelatives( + ref *Reference, + node, keywordNode *yaml.Node, + state relativeWalkState, + index int, +) []*Reference { + var found []*Reference + + if index+1 < len(node.Content) && utils.IsNodeMap(node.Content[index+1]) { + if k, v := utils.FindKeyNodeTop("items", node.Content[index+1].Content); v != nil { + if utils.IsNodeMap(v) { + if d, _, l := utils.IsNodeRefValue(v); d { + resolver.visitPolymorphicReference(ref, keywordNode.Value, k, l, state, index) + } + } + } else { + v := node.Content[index+1] + if utils.IsNodeMap(v) { + if d, _, l := utils.IsNodeRefValue(v); d { + resolver.visitPolymorphicReference(ref, keywordNode.Value, node.Content[index], l, state, index) + } + } + } + } + + if index+1 < len(node.Content) && utils.IsNodeArray(node.Content[index+1]) { + for q := range node.Content[index+1].Content { + v := node.Content[index+1].Content[q] + if utils.IsNodeMap(v) { + if d, _, l := utils.IsNodeRefValue(v); d { + resolver.visitPolymorphicReference(ref, keywordNode.Value, node.Content[index], l, state, index) + } else { + found = append(found, resolver.extractRelativesWithState(ref, v, keywordNode, state.descend())...) + } + } + } + } + return found +} + +func (resolver *Resolver) visitPolymorphicReference( + ref *Reference, + polymorphicType string, + parentNode *yaml.Node, + lookup string, + state relativeWalkState, + loopIndex int, +) { + def := resolver.buildDefPathWithSchemaBase(ref, lookup, state.schemaIDBase) + mappedRefs, _ := resolver.specIndex.SearchIndexForReference(def) + if mappedRefs == nil || mappedRefs.Circular { + return + } + circ := false + for f := range state.journey { + if state.journey[f].FullDefinition == mappedRefs.FullDefinition { + circ = true + break + } + } + if !circ { + resolver.VisitReference(mappedRefs, state.foundRelatives, state.journey, state.resolve) + return + } + + loop := append(state.journey, mappedRefs) + circRef := &CircularReferenceResult{ + ParentNode: parentNode, + Journey: loop, + Start: mappedRefs, + LoopIndex: loopIndex, + LoopPoint: mappedRefs, + PolymorphicType: polymorphicType, + IsPolymorphicResult: true, + } + mappedRefs.Seen = true + mappedRefs.Circular = true + if resolver.IgnorePoly { + resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) + } else if !resolver.circChecked { + resolver.circularReferences = append(resolver.circularReferences, circRef) + } +} diff --git a/index/resolver_relatives.go b/index/resolver_relatives.go new file mode 100644 index 000000000..9d0acf7af --- /dev/null +++ b/index/resolver_relatives.go @@ -0,0 +1,263 @@ +// Copyright 2022 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import ( + "fmt" + "net/url" + "path" + "path/filepath" + "strings" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +func (resolver *Resolver) extractRelatives( + ref *Reference, + node, parent *yaml.Node, + foundRelatives map[string]bool, + journey []*Reference, + resolve bool, + depth int, + schemaIDBase string, +) []*Reference { + state := newRelativeWalkState(foundRelatives, journey, resolve, depth, schemaIDBase) + return resolver.extractRelativesWithState(ref, node, parent, state) +} + +func (resolver *Resolver) extractRelativesWithState( + ref *Reference, + node, parent *yaml.Node, + state relativeWalkState, +) []*Reference { + if len(state.journey) > 100 { + return nil + } + if state.depth > 500 { + return resolver.handleRelativeDepthLimit(ref, state) + } + + state = state.withNodeBase(resolver, node) + var found []*Reference + + if node != nil && len(node.Content) > 0 { + skip := false + for i, n := range node.Content { + if skip { + skip = false + continue + } + + if utils.IsNodeMap(n) || utils.IsNodeArray(n) { + found = append(found, resolver.extractNestedRelatives(ref, node, n, state)...) + } + + if i%2 == 0 && n.Value == "$ref" && len(node.Content) > i%2+1 { + if relative, handled, skipNext := resolver.extractRelativeReference(ref, node, parent, n, i, state); handled { + if relative != nil { + found = append(found, relative) + } + skip = skipNext + continue + } + } + + if i%2 == 0 && shouldExtractPolymorphicRelatives(parent, n) { + found = append(found, resolver.extractPolymorphicRelatives(ref, node, n, state, i)...) + skip = true + } + } + } + + resolver.relativesSeen += len(found) + return found +} + +func (resolver *Resolver) handleRelativeDepthLimit(ref *Reference, state relativeWalkState) []*Reference { + def := "unknown" + if ref != nil { + def = ref.FullDefinition + } + if resolver.specIndex != nil && resolver.specIndex.logger != nil { + resolver.specIndex.logger.Warn("libopenapi resolver: relative depth exceeded 100 levels, "+ + "check for circular references - resolving may be incomplete", + "reference", def) + } + + loop := append(state.journey, ref) + circRef := &CircularReferenceResult{ + Journey: loop, + Start: ref, + LoopIndex: state.depth, + LoopPoint: ref, + IsInfiniteLoop: true, + } + if !resolver.circChecked { + resolver.circularReferences = append(resolver.circularReferences, circRef) + ref.Circular = true + } + return nil +} + +func (resolver *Resolver) extractNestedRelatives( + ref *Reference, + parent, node *yaml.Node, + state relativeWalkState, +) []*Reference { + childState := state.descend() + foundRef, _, _ := resolver.searchReferenceWithContext(ref, ref) + if foundRef != nil { + if foundRef.Circular { + return nil + } + return resolver.extractRelativesWithState(foundRef, node, parent, childState) + } + return resolver.extractRelativesWithState(ref, node, parent, childState) +} + +func (resolver *Resolver) extractRelativeReference( + ref *Reference, + node, parent, keyNode *yaml.Node, + keyIndex int, + state relativeWalkState, +) (*Reference, bool, bool) { + if !utils.IsNodeStringValue(node.Content[keyIndex+1]) || utils.IsNodeArray(node) { + return nil, false, false + } + + value := strings.ReplaceAll(node.Content[keyIndex+1].Value, "\\\\", "\\") + if resolver.specIndex != nil && resolver.specIndex.config != nil && + resolver.specIndex.config.SkipExternalRefResolution && utils.IsExternalRef(value) { + return nil, true, true + } + + definition, fullDef := resolver.buildRelativeLookupDefinitions(ref, value, state.schemaIDBase) + searchRef := &Reference{ + Definition: definition, + FullDefinition: fullDef, + RawRef: value, + SchemaIdBase: state.schemaIDBase, + RemoteLocation: ref.RemoteLocation, + IsRemote: true, + Index: ref.Index, + } + + locatedRef, _, _ := resolver.searchReferenceWithContext(ref, searchRef) + if locatedRef == nil { + _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value) + resolver.resolvingErrors = append(resolver.resolvingErrors, &ResolvingError{ + ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value), + Node: keyNode, + Path: path, + }) + return nil, true, false + } + + if state.resolve { + if ok, _, _ := utils.IsNodeRefValue(ref.Node); ok { + ref.Node.Content = locatedRef.Node.Content + ref.Resolved = true + } + } + + if ref.ParentNodeSchemaType != "" { + locatedRef.ParentNodeTypes = append(locatedRef.ParentNodeTypes, ref.ParentNodeSchemaType) + } + locatedRef.ParentNodeSchemaType = parentArraySchemaType(parent) + state.foundRelatives[value] = true + return locatedRef, true, false +} + +func parentArraySchemaType(parent *yaml.Node) string { + if parent == nil { + return "" + } + _, arrayTypeNode := utils.FindKeyNodeTop("type", parent.Content) + if arrayTypeNode != nil && arrayTypeNode.Value == "array" { + return "array" + } + return "" +} + +func shouldExtractPolymorphicRelatives(parent, keyNode *yaml.Node) bool { + if keyNode == nil || keyNode.Value == "" || keyNode.Value == "$ref" { + return false + } + if keyNode.Value != "allOf" && keyNode.Value != "oneOf" && keyNode.Value != "anyOf" { + return false + } + return !isInsidePropertiesNode(parent) +} + +func isInsidePropertiesNode(parent *yaml.Node) bool { + if parent == nil { + return false + } + for j := 0; j < len(parent.Content); j += 2 { + if j < len(parent.Content) && parent.Content[j].Value == "properties" { + return true + } + } + return false +} + +func (resolver *Resolver) buildRelativeLookupDefinitions(ref *Reference, value, currentBase string) (string, string) { + definition := value + fullDef := "" + exp := strings.Split(value, "#/") + if len(exp) == 2 { + definition = fmt.Sprintf("#/%s", exp[1]) + if exp[0] != "" { + if strings.HasPrefix(exp[0], "http") { + fullDef = value + } else if strings.HasPrefix(ref.FullDefinition, "http") { + httpExp := strings.Split(ref.FullDefinition, "#/") + u, _ := url.Parse(httpExp[0]) + abs, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), exp[0], string(filepath.Separator))) + u.Path = utils.ReplaceWindowsDriveWithLinuxPath(abs) + u.Fragment = "" + fullDef = fmt.Sprintf("%s#/%s", u.String(), exp[1]) + } else { + fileDef := strings.Split(ref.FullDefinition, "#/") + abs := resolver.resolveLocalRefPath(filepath.Dir(fileDef[0]), exp[0]) + fullDef = fmt.Sprintf("%s#/%s", abs, exp[1]) + } + } else { + baseLocation := ref.FullDefinition + if ref.RemoteLocation != "" { + baseLocation = ref.RemoteLocation + } + if strings.HasPrefix(baseLocation, "http") { + httpExp := strings.Split(baseLocation, "#/") + u, _ := url.Parse(httpExp[0]) + fullDef = fmt.Sprintf("%s#/%s", u.String(), exp[1]) + } else { + fileDef := strings.Split(baseLocation, "#/") + fullDef = fmt.Sprintf("%s#/%s", fileDef[0], exp[1]) + } + } + } else if strings.HasPrefix(value, "http") { + fullDef = value + } else { + baseLocation := ref.FullDefinition + if ref.RemoteLocation != "" { + baseLocation = ref.RemoteLocation + } + fileDef := strings.Split(baseLocation, "#/") + if strings.HasPrefix(fileDef[0], "http") { + u, _ := url.Parse(fileDef[0]) + absPath, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), exp[0], string(filepath.Separator))) + u.Path = utils.ReplaceWindowsDriveWithLinuxPath(absPath) + fullDef = u.String() + } else { + fullDef = resolver.resolveLocalRefPath(filepath.Dir(fileDef[0]), exp[0]) + } + } + + if currentBase != "" { + fullDef = resolveRefWithSchemaBase(value, currentBase) + } + return definition, fullDef +} diff --git a/index/resolver_test.go b/index/resolver_test.go index 8c56be6cd..9edf813dd 100644 --- a/index/resolver_test.go +++ b/index/resolver_test.go @@ -448,7 +448,7 @@ func TestResolver_DeepJourney(t *testing.T) { } idx := NewSpecIndexWithConfig(nil, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) - assert.Nil(t, resolver.extractRelatives(nil, nil, nil, nil, journey, nil, false, 0, "")) + assert.Nil(t, resolver.extractRelatives(nil, nil, nil, nil, journey, false, 0, "")) } func TestResolver_DeepDepth(t *testing.T) { @@ -481,7 +481,7 @@ func TestResolver_DeepDepth(t *testing.T) { ref := &Reference{ FullDefinition: "#/components/schemas/A", } - found := resolver.extractRelatives(ref, refA, nil, nil, nil, nil, false, 0, "") + found := resolver.extractRelatives(ref, refA, nil, nil, nil, false, 0, "") assert.Nil(t, found) assert.Contains(t, buf.String(), "libopenapi resolver: relative depth exceeded 100 levels") @@ -911,7 +911,7 @@ func TestResolver_ExtractRelatives_HttpFullDefinition(t *testing.T) { resolver := NewResolver(idx) ref.Index = idx - _ = resolver.extractRelatives(ref, targetNode, nil, map[string]bool{}, []*Reference{}, map[int]bool{}, false, 0, "") + _ = resolver.extractRelatives(ref, targetNode, nil, map[string]bool{}, []*Reference{}, false, 0, "") assert.NotEmpty(t, resolver.GetResolvingErrors()) } @@ -1781,6 +1781,46 @@ func TestVisitReference_Nil(t *testing.T) { assert.Nil(t, n) } +func TestVisitReference_SeenShortCircuit(t *testing.T) { + node := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "type"}, + {Kind: yaml.ScalarNode, Value: "string"}, + }, + } + resolver := &Resolver{} + + resolved := resolver.VisitReference(&Reference{Seen: true, Resolved: true, Node: node}, nil, nil, true) + assert.Equal(t, node.Content, resolved) + + unresolved := resolver.VisitReference(&Reference{Seen: true, Node: node}, nil, nil, false) + assert.Equal(t, node.Content, unresolved) +} + +func TestVisitReference_SeenButUnresolvedReturnsNodeAtEnd(t *testing.T) { + node := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "type"}, + {Kind: yaml.ScalarNode, Value: "string"}, + }, + } + resolver := &Resolver{} + ref := &Reference{Seen: true, Resolved: false, Node: node} + + result := resolver.VisitReference(ref, nil, nil, true) + assert.Equal(t, node.Content, result) +} + +func TestVisitReference_UnresolvedNilNodeReturnsNil(t *testing.T) { + resolver := &Resolver{} + ref := &Reference{FullDefinition: "#/components/schemas/missing"} + + result := resolver.VisitReference(ref, nil, nil, false) + assert.Nil(t, result) +} + func TestResolver_SkipExternalRefResolution(t *testing.T) { // Spec with external $ref that cannot be resolved yml := `openapi: 3.0.0 @@ -1850,4 +1890,151 @@ func TestResolver_Release_Idempotent(t *testing.T) { assert.Nil(t, resolver.resolvedRoot) } -// func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, journey []*Reference, resolve bool) []*yaml.Node { +func TestResolver_VisitReferenceShortCircuit(t *testing.T) { + resolver := &Resolver{} + contentNode := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "type"}, + {Kind: yaml.ScalarNode, Value: "string"}, + }, + } + + content, done := resolver.visitReferenceShortCircuit(nil, true) + assert.True(t, done) + assert.Nil(t, content) + + resolvedRef := &Reference{Seen: true, Resolved: true, Node: contentNode} + content, done = resolver.visitReferenceShortCircuit(resolvedRef, true) + assert.True(t, done) + assert.Equal(t, contentNode.Content, content) + + resolvedRef.Node = nil + content, done = resolver.visitReferenceShortCircuit(resolvedRef, true) + assert.True(t, done) + assert.Nil(t, content) + + seenRef := &Reference{Seen: true, Node: contentNode} + content, done = resolver.visitReferenceShortCircuit(seenRef, false) + assert.True(t, done) + assert.Equal(t, contentNode.Content, content) + + seenRef.Node = nil + content, done = resolver.visitReferenceShortCircuit(seenRef, false) + assert.True(t, done) + assert.Nil(t, content) + + content, done = resolver.visitReferenceShortCircuit(&Reference{}, false) + assert.False(t, done) + assert.Nil(t, content) +} + +func TestResolver_CircularHelperMethods(t *testing.T) { + resolver := &Resolver{} + + assert.False(t, resolver.relativeIsArrayResult(nil)) + assert.True(t, resolver.relativeIsArrayResult(&Reference{ParentNodeSchemaType: "array"})) + assert.True(t, resolver.relativeIsArrayResult(&Reference{ParentNodeTypes: []string{"object", "array"}})) + assert.False(t, resolver.relativeIsArrayResult(&Reference{ParentNodeTypes: []string{"object"}})) + + resolver.recordCircularReferenceResult(nil) + assert.Empty(t, resolver.circularReferences) + + poly := &CircularReferenceResult{} + resolver.IgnorePoly = true + resolver.recordCircularReferenceResult(poly) + assert.Len(t, resolver.ignoredPolyReferences, 1) + + array := &CircularReferenceResult{IsArrayResult: true} + resolver.IgnorePoly = false + resolver.IgnoreArray = true + resolver.recordCircularReferenceResult(array) + assert.Len(t, resolver.ignoredArrayReferences, 1) + + recorded := &CircularReferenceResult{} + resolver.IgnoreArray = false + resolver.recordCircularReferenceResult(recorded) + assert.Len(t, resolver.circularReferences, 1) + + resolver.circChecked = true + resolver.recordCircularReferenceResult(&CircularReferenceResult{}) + assert.Len(t, resolver.circularReferences, 1) + + relative := &Reference{} + duplicate := &Reference{} + resolver.markReferencesCircular(relative, duplicate) + assert.True(t, relative.Seen) + assert.True(t, relative.Circular) + assert.True(t, duplicate.Seen) + assert.True(t, duplicate.Circular) +} + +func TestResolver_HandleCircularJourneyRelative(t *testing.T) { + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte("openapi: 3.1.0\ncomponents:\n schemas: {}\n"), &rootNode) + idx := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + resolver := &Resolver{specIndex: idx} + + relative := &Reference{FullDefinition: "#/components/schemas/Loop"} + assert.True(t, resolver.handleCircularJourneyRelative(nil, relative, []*Reference{relative})) + assert.Empty(t, resolver.circularReferences) + + cachedCircular := &Reference{FullDefinition: relative.FullDefinition, Circular: true} + idx.cache.Store(relative.FullDefinition, cachedCircular) + assert.True(t, resolver.handleCircularJourneyRelative(nil, relative, []*Reference{relative})) + assert.Empty(t, resolver.circularReferences) +} + +func TestResolver_ResolveRelativeReference(t *testing.T) { + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte("openapi: 3.1.0\ncomponents:\n schemas: {}\n"), &rootNode) + idx := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + resolver := &Resolver{specIndex: idx} + + relativeNode := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "type"}, + {Kind: yaml.ScalarNode, Value: "string"}, + }, + } + ref := &Reference{} + relative := &Reference{ + FullDefinition: "#/components/schemas/Pet", + Node: relativeNode, + Seen: true, + } + + resolver.resolveRelativeReference(ref, relative, map[string]bool{}, nil, false) + assert.True(t, ref.Seen) + assert.True(t, relative.Seen) + assert.False(t, ref.Resolved) + + foundNode := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "type"}, + {Kind: yaml.ScalarNode, Value: "integer"}, + }, + } + found := &Reference{ + FullDefinition: relative.FullDefinition, + Node: foundNode, + Seen: true, + Resolved: true, + } + idx.cache.Store(relative.FullDefinition, found) + + relative.Node = &yaml.Node{Kind: yaml.MappingNode} + relative.Seen = false + relative.Resolved = false + ref.Seen = false + ref.Resolved = false + + resolver.resolveRelativeReference(ref, relative, map[string]bool{}, nil, true) + assert.True(t, ref.Seen) + assert.True(t, ref.Resolved) + assert.True(t, relative.Seen) + assert.True(t, relative.Resolved) + assert.Equal(t, foundNode.Content, relative.Node.Content) +} diff --git a/index/resolver_visit.go b/index/resolver_visit.go new file mode 100644 index 000000000..41a399317 --- /dev/null +++ b/index/resolver_visit.go @@ -0,0 +1,170 @@ +// Copyright 2022 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import ( + "context" + "sort" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +func visitIndexWithoutDamagingIt(res *Resolver, idx *SpecIndex) { + mapped := idx.GetMappedReferencesSequenced() + mappedIndex := idx.GetMappedReferences() + res.indexesVisited++ + for _, ref := range mapped { + seenReferences := make(map[string]bool) + var journey []*Reference + res.journeysTaken++ + res.VisitReference(ref.Reference, seenReferences, journey, false) + } + + schemas := idx.GetAllComponentSchemas() + schemaKeys := make([]string, 0, len(schemas)) + for k := range schemas { + schemaKeys = append(schemaKeys, k) + } + sort.Strings(schemaKeys) + + for _, s := range schemaKeys { + schemaRef := schemas[s] + if mappedIndex[s] == nil { + seenReferences := make(map[string]bool) + var journey []*Reference + res.journeysTaken++ + res.VisitReference(schemaRef, seenReferences, journey, false) + } + } +} + +type refMap struct { + ref *Reference + nodes []*yaml.Node +} + +func visitIndex(res *Resolver, idx *SpecIndex) { + mapped := idx.GetMappedReferencesSequenced() + mappedIndex := idx.GetMappedReferences() + res.indexesVisited++ + + var refs []refMap + for _, ref := range mapped { + seenReferences := make(map[string]bool) + var journey []*Reference + res.journeysTaken++ + if ref != nil && ref.Reference != nil { + n := res.VisitReference(ref.Reference, seenReferences, journey, true) + if !ref.Reference.Circular { + if ok, _, _ := utils.IsNodeRefValue(ref.OriginalReference.Node); ok { + refs = append(refs, refMap{ref: ref.OriginalReference, nodes: n}) + } + } + } + } + idx.pendingResolve = refs + + schemas := idx.GetAllComponentSchemas() + schemaKeys := make([]string, 0, len(schemas)) + for k := range schemas { + schemaKeys = append(schemaKeys, k) + } + sort.Strings(schemaKeys) + + for _, s := range schemaKeys { + schemaRef := schemas[s] + if mappedIndex[s] == nil { + seenReferences := make(map[string]bool) + var journey []*Reference + res.journeysTaken++ + schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true) + } + } + + securitySchemes := idx.GetAllSecuritySchemes() + securityKeys := make([]string, 0, len(securitySchemes)) + for k := range securitySchemes { + securityKeys = append(securityKeys, k) + } + sort.Strings(securityKeys) + + for _, s := range securityKeys { + schemaRef := securitySchemes[s] + if mappedIndex[s] == nil { + seenReferences := make(map[string]bool) + var journey []*Reference + res.journeysTaken++ + schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true) + } + } + + for _, sequenced := range idx.GetAllSequencedReferences() { + locatedDef := mappedIndex[sequenced.FullDefinition] + if locatedDef != nil && !locatedDef.Circular { + sequenced.Node.Content = locatedDef.Node.Content + } + } +} + +func (resolver *Resolver) searchReferenceWithContext(sourceRef, searchRef *Reference) (*Reference, *SpecIndex, context.Context) { + if resolver.specIndex == nil || resolver.specIndex.config == nil || !resolver.specIndex.config.ResolveNestedRefsWithDocumentContext { + ref, idx := resolver.specIndex.SearchIndexForReferenceByReference(searchRef) + return ref, idx, context.Background() + } + + searchIndex := resolver.specIndex + if searchRef != nil && searchRef.Index != nil { + searchIndex = searchRef.Index + } else if sourceRef != nil && sourceRef.Index != nil { + searchIndex = sourceRef.Index + } + + ctx := context.Background() + currentPath := "" + if sourceRef != nil { + currentPath = sourceRef.RemoteLocation + } + if currentPath == "" && searchIndex != nil { + currentPath = searchIndex.specAbsolutePath + } + if currentPath != "" { + ctx = context.WithValue(ctx, CurrentPathKey, currentPath) + } + if searchRef != nil || sourceRef != nil { + base := "" + if searchRef != nil && searchRef.SchemaIdBase != "" { + base = searchRef.SchemaIdBase + } else if sourceRef != nil && sourceRef.SchemaIdBase != "" { + base = sourceRef.SchemaIdBase + } + if base != "" { + scope := NewSchemaIdScope(base) + scope.PushId(base) + ctx = WithSchemaIdScope(ctx, scope) + } + } + + return searchIndex.SearchIndexForReferenceByReferenceWithContext(ctx, searchRef) +} + +func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, journey []*Reference, resolve bool) []*yaml.Node { + resolver.referencesVisited++ + if content, done := resolver.visitReferenceShortCircuit(ref, resolve); done { + return content + } + + journey = append(journey, ref) + relatives := resolver.collectReferenceRelatives(ref, seen, journey, resolve) + + seen = make(map[string]bool) + seen[ref.FullDefinition] = true + resolver.visitReferenceRelatives(ref, relatives, seen, journey, resolve) + + ref.Seen = true + if ref.Node != nil { + return ref.Node.Content + } + return nil +} diff --git a/index/resolver_walk.go b/index/resolver_walk.go new file mode 100644 index 000000000..ce431c7de --- /dev/null +++ b/index/resolver_walk.go @@ -0,0 +1,40 @@ +// Copyright 2022 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import "go.yaml.in/yaml/v4" + +type relativeWalkState struct { + foundRelatives map[string]bool + journey []*Reference + resolve bool + depth int + schemaIDBase string +} + +func newRelativeWalkState( + foundRelatives map[string]bool, + journey []*Reference, + resolve bool, + depth int, + schemaIDBase string, +) relativeWalkState { + return relativeWalkState{ + foundRelatives: foundRelatives, + journey: journey, + resolve: resolve, + depth: depth, + schemaIDBase: schemaIDBase, + } +} + +func (state relativeWalkState) withNodeBase(resolver *Resolver, node *yaml.Node) relativeWalkState { + state.schemaIDBase = resolver.resolveSchemaIdBase(state.schemaIDBase, node) + return state +} + +func (state relativeWalkState) descend() relativeWalkState { + state.depth++ + return state +} diff --git a/index/rolodex.go b/index/rolodex.go index 4effb714b..2f1ed6d49 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -99,6 +99,8 @@ func (r *Rolodex) Release() { return } r.indexLock.Lock() + r.localFS = nil + r.remoteFS = nil r.indexes = nil r.indexMap = nil r.rootIndex = nil @@ -115,6 +117,15 @@ func (r *Rolodex) Release() { r.infiniteCircularReferences = nil r.ignoredCircularReferences = nil r.globalSchemaIdRegistry = nil + r.indexConfig = nil + r.indexingDuration = 0 + r.indexed = false + r.built = false + r.manualBuilt = false + r.resolved = false + r.circChecked = false + r.logger = nil + r.id = "" } // NewRolodex creates a new rolodex with the provided index configuration. @@ -577,28 +588,9 @@ func (r *Rolodex) CheckForCircularReferences() { // Resolve resolves references in the rolodex. func (r *Rolodex) Resolve() { - var resolvers []*Resolver - if r.rootIndex != nil && r.rootIndex.resolver != nil { - resolvers = append(resolvers, r.rootIndex.resolver) - } - for _, idx := range r.indexes { - if idx.resolver != nil { - resolvers = append(resolvers, idx.resolver) - } - } + resolvers := r.collectResolvers() for _, res := range resolvers { - resolvingErrors := res.Resolve() - for e := range resolvingErrors { - r.caughtErrors = append(r.caughtErrors, resolvingErrors[e]) - } - if r.rootIndex != nil && len(r.rootIndex.resolver.ignoredPolyReferences) > 0 { - r.ignoredCircularReferences = append(r.ignoredCircularReferences, res.ignoredPolyReferences...) - } - if r.rootIndex != nil && len(r.rootIndex.resolver.ignoredArrayReferences) > 0 { - r.ignoredCircularReferences = append(r.ignoredCircularReferences, res.ignoredArrayReferences...) - } - r.safeCircularReferences = append(r.safeCircularReferences, res.GetSafeCircularReferences()...) - r.infiniteCircularReferences = append(r.infiniteCircularReferences, res.GetInfiniteCircularReferences()...) + r.mergeResolverResults(res) } // resolve pending nodes @@ -611,6 +603,36 @@ func (r *Rolodex) Resolve() { r.debouncedIgnoredCircRefs = nil } +func (r *Rolodex) collectResolvers() []*Resolver { + var resolvers []*Resolver + if r.rootIndex != nil { + if resolver := r.rootIndex.GetResolver(); resolver != nil { + resolvers = append(resolvers, resolver) + } + } + for _, idx := range r.indexes { + if resolver := idx.GetResolver(); resolver != nil { + resolvers = append(resolvers, resolver) + } + } + return resolvers +} + +func (r *Rolodex) mergeResolverResults(res *Resolver) { + resolvingErrors := res.Resolve() + for e := range resolvingErrors { + r.caughtErrors = append(r.caughtErrors, resolvingErrors[e]) + } + if len(res.ignoredPolyReferences) > 0 { + r.ignoredCircularReferences = append(r.ignoredCircularReferences, res.ignoredPolyReferences...) + } + if len(res.ignoredArrayReferences) > 0 { + r.ignoredCircularReferences = append(r.ignoredCircularReferences, res.ignoredArrayReferences...) + } + r.safeCircularReferences = append(r.safeCircularReferences, res.GetSafeCircularReferences()...) + r.infiniteCircularReferences = append(r.infiniteCircularReferences, res.GetInfiniteCircularReferences()...) +} + // BuildIndexes builds the indexes in the rolodex, this is generally not required unless manually building a rolodex. func (r *Rolodex) BuildIndexes() { if r.manualBuilt { @@ -668,8 +690,6 @@ func (r *Rolodex) OpenWithContext(ctx context.Context, location string) (Rolodex } var errorStack []error - var localFile *LocalFile - var remoteFile *RemoteFile fileLookup := location isUrl := false if strings.HasPrefix(location, "http") { @@ -683,109 +703,10 @@ func (r *Rolodex) OpenWithContext(ctx context.Context, location string) (Rolodex "the rolodex has no local file systems configured, cannot open local file '%s'", location, ) } - - for k, v := range r.localFS { - - // check if this is a URL or an abs/rel reference. - if !filepath.IsAbs(location) { - fileLookup, _ = filepath.Abs(utils.CheckPathOverlap(k, location, string(os.PathSeparator))) - } - - // For generic fs.FS implementations, we need to use relative paths - // The fs.FS interface requires paths to be relative and slash-separated. - // This ensures compatibility with standard Go file systems like embed.FS, - // fstest.MapFS, afero.NewIOFS, and others that strictly follow the fs.FS contract. - var pathForOpen string - - // Check if this is our custom LocalFS which supports absolute paths - if _, isLocalFS := v.(*LocalFS); isLocalFS { - // LocalFS can handle absolute paths directly for backward compatibility - pathForOpen = fileLookup - } else { - // For standard fs.FS implementations, convert to relative path - // Calculate relative path from base directory k to target fileLookup - relPath, _ := filepath.Rel(k, fileLookup) - pathForOpen = filepath.ToSlash(relPath) - } - - f, err := openFile(ctx, pathForOpen, v) - - if err != nil { - // If the first attempt failed and we haven't already tried with the original location, - // try a lookup with the original location path - if pathForOpen != location { - f, err = openFile(ctx, location, v) - } - - if err != nil { - errorStack = append(errorStack, err) - continue - } - } - // check if this is a native rolodex FS, then the work is done. - if lf, ko := interface{}(f).(*LocalFile); ko { - localFile = lf - break - } - - if lf, ko := f.(RolodexFile); ko { - var atm atomic.Value - atm.Store(lf.GetIndex()) - var parsed *yaml.Node - var parseErrors []error - if p, e := lf.GetContentAsYAMLNode(); e == nil { - parsed = p - } else { - parseErrors = append(parseErrors, e) - } - parseErrors = append(parseErrors, lf.GetErrors()...) - - localFile = &LocalFile{ - filename: lf.Name(), - name: lf.Name(), - extension: ExtractFileType(lf.Name()), - data: []byte(lf.GetContent()), - fullPath: lf.GetFullPath(), - lastModified: lf.ModTime(), - index: atm, - readingErrors: parseErrors, - parsed: parsed, - } - // If there were errors processing the file content, we should return them - if len(parseErrors) > 0 { - errorStack = append(errorStack, parseErrors...) - } - break - } - - // not a native FS, so we need to read the file and create a local file. - bytes, rErr := io.ReadAll(f) - if rErr != nil { - errorStack = append(errorStack, rErr) - continue - } - s, sErr := f.Stat() - if sErr != nil { - errorStack = append(errorStack, sErr) - continue - } - if len(bytes) > 0 { - var atm atomic.Value - idx := r.rootIndex - atm.Store(idx) - - localFile = &LocalFile{ - filename: filepath.Base(fileLookup), - name: filepath.Base(fileLookup), - extension: ExtractFileType(fileLookup), - data: bytes, - fullPath: fileLookup, - lastModified: s.ModTime(), - index: atm, - } - break - } - + localFile, errs := r.openLocalLocation(ctx, location) + errorStack = append(errorStack, errs...) + if localFile != nil { + return r.wrapLocalRolodexFile(localFile) } } else { @@ -796,78 +717,170 @@ func (r *Rolodex) OpenWithContext(ctx context.Context, location string) (Rolodex "AllowRemoteLookup to true", fileLookup, ) } + remoteFile, errs := r.openRemoteLocation(ctx, fileLookup) + errorStack = append(errorStack, errs...) + if remoteFile != nil { + return r.wrapRemoteRolodexFile(remoteFile) + } + } - for _, v := range r.remoteFS { + return nil, errors.Join(errorStack...) +} - var f fs.File - var err error - if fscw, ok := v.(RolodexFSWithContext); ok { - f, err = fscw.OpenWithContext(ctx, fileLookup) - } else { - f, err = v.Open(fileLookup) - } +func (r *Rolodex) openLocalLocation(ctx context.Context, location string) (*LocalFile, []error) { + var errorStack []error + for baseDir, fileSystem := range r.localFS { + fileLookup := location + if !filepath.IsAbs(location) { + fileLookup, _ = filepath.Abs(utils.CheckPathOverlap(baseDir, location, string(os.PathSeparator))) + } - if err != nil { - r.logger.Warn("[rolodex] errors opening remote file", "location", fileLookup, "error", err) - } - if f != nil { - if rf, ok := interface{}(f).(*RemoteFile); ok { - remoteFile = rf - break - } else { + pathForOpen := r.localPathForOpen(baseDir, fileLookup, fileSystem) + file, err := openFile(ctx, pathForOpen, fileSystem) + if err != nil && pathForOpen != location { + file, err = openFile(ctx, location, fileSystem) + } + if err != nil { + errorStack = append(errorStack, err) + continue + } + localFile, errs := r.asLocalFile(file, fileLookup) + errorStack = append(errorStack, errs...) + if localFile != nil { + return localFile, errorStack + } + } + return nil, errorStack +} - bytes, rErr := io.ReadAll(f) - if rErr != nil { - errorStack = append(errorStack, rErr) - continue - } - s, sErr := f.Stat() - if sErr != nil { - errorStack = append(errorStack, sErr) - continue - } - if len(bytes) > 0 { - var atm atomic.Value - atm.Store(r.rootIndex) - remoteFile = &RemoteFile{ - filename: filepath.Base(fileLookup), - name: filepath.Base(fileLookup), - extension: ExtractFileType(fileLookup), - data: bytes, - fullPath: fileLookup, - lastModified: s.ModTime(), - index: atm, - } - break - } - } - } +func (r *Rolodex) localPathForOpen(baseDir, fileLookup string, fileSystem fs.FS) string { + if _, isLocalFS := fileSystem.(*LocalFS); isLocalFS { + return fileLookup + } + relPath, _ := filepath.Rel(baseDir, fileLookup) + return filepath.ToSlash(relPath) +} + +func (r *Rolodex) asLocalFile(file fs.File, fileLookup string) (*LocalFile, []error) { + var errorStack []error + if localFile, ok := file.(*LocalFile); ok { + return localFile, nil + } + if existing, ok := file.(RolodexFile); ok { + wrapped, errs := wrapExistingRolodexFile(existing) + return wrapped, errs + } + + bytes, readErr := io.ReadAll(file) + if readErr != nil { + return nil, append(errorStack, readErr) + } + stat, statErr := file.Stat() + if statErr != nil { + return nil, append(errorStack, statErr) + } + if len(bytes) == 0 { + return nil, nil + } + var atm atomic.Value + atm.Store(r.rootIndex) + return &LocalFile{ + filename: filepath.Base(fileLookup), + name: filepath.Base(fileLookup), + extension: ExtractFileType(fileLookup), + data: bytes, + fullPath: fileLookup, + lastModified: stat.ModTime(), + index: atm, + }, nil +} + +func wrapExistingRolodexFile(file RolodexFile) (*LocalFile, []error) { + var atm atomic.Value + atm.Store(file.GetIndex()) + var parsed *yaml.Node + var parseErrors []error + if p, err := file.GetContentAsYAMLNode(); err == nil { + parsed = p + } else { + parseErrors = append(parseErrors, err) + } + parseErrors = append(parseErrors, file.GetErrors()...) + + return &LocalFile{ + filename: file.Name(), + name: file.Name(), + extension: ExtractFileType(file.Name()), + data: []byte(file.GetContent()), + fullPath: file.GetFullPath(), + lastModified: file.ModTime(), + index: atm, + readingErrors: parseErrors, + parsed: parsed, + }, parseErrors +} + +func (r *Rolodex) openRemoteLocation(ctx context.Context, location string) (*RemoteFile, []error) { + var errorStack []error + for _, fileSystem := range r.remoteFS { + file, err := openFile(ctx, location, fileSystem) + if err != nil { + r.logger.Warn("[rolodex] errors opening remote file", "location", location, "error", err) + errorStack = append(errorStack, err) + continue + } + remoteFile, errs := r.asRemoteFile(file, location) + errorStack = append(errorStack, errs...) + if remoteFile != nil { + return remoteFile, errorStack } } + return nil, errorStack +} - if localFile != nil { - // Check if the localFile has any reading errors that should be returned - var fileErrors []error - fileErrors = localFile.readingErrors - return &rolodexFile{ - rolodex: r, - location: localFile.fullPath, - localFile: localFile, - }, errors.Join(fileErrors...) +func (r *Rolodex) asRemoteFile(file fs.File, location string) (*RemoteFile, []error) { + if remoteFile, ok := file.(*RemoteFile); ok { + return remoteFile, nil } - if remoteFile != nil { - // Check if the remoteFile has any seeking errors that should be returned - var fileErrors []error - fileErrors = remoteFile.seekingErrors - return &rolodexFile{ - rolodex: r, - location: remoteFile.fullPath, - remoteFile: remoteFile, - }, errors.Join(fileErrors...) + bytes, readErr := io.ReadAll(file) + if readErr != nil { + return nil, []error{readErr} + } + stat, statErr := file.Stat() + if statErr != nil { + return nil, []error{statErr} } + if len(bytes) == 0 { + return nil, nil + } + var atm atomic.Value + atm.Store(r.rootIndex) + return &RemoteFile{ + filename: filepath.Base(location), + name: filepath.Base(location), + extension: ExtractFileType(location), + data: bytes, + fullPath: location, + lastModified: stat.ModTime(), + index: atm, + }, nil +} - return nil, errors.Join(errorStack...) +func (r *Rolodex) wrapLocalRolodexFile(localFile *LocalFile) (RolodexFile, error) { + return &rolodexFile{ + rolodex: r, + location: localFile.fullPath, + localFile: localFile, + }, errors.Join(localFile.readingErrors...) +} + +func (r *Rolodex) wrapRemoteRolodexFile(remoteFile *RemoteFile) (RolodexFile, error) { + return &rolodexFile{ + rolodex: r, + location: remoteFile.fullPath, + remoteFile: remoteFile, + }, errors.Join(remoteFile.seekingErrors...) } func openFile(ctx context.Context, location string, v fs.FS) (fs.File, error) { diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index 9de011667..16c17bfc8 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -163,8 +163,7 @@ func (l *LocalFS) OpenWithContext(ctx context.Context, name string) (fs.File, er } if idx != nil { - resolver := NewResolver(idx) - idx.resolver = resolver + NewResolver(idx) idx.BuildIndex() } if len(extractedFile.data) > 0 { diff --git a/index/rolodex_remote_loader.go b/index/rolodex_remote_loader.go index 3e3c6e7ae..37659bb1d 100644 --- a/index/rolodex_remote_loader.go +++ b/index/rolodex_remote_loader.go @@ -292,20 +292,24 @@ func (f *RemoteFile) GetContent() string { // GetContentAsYAMLNode returns the content of the file as a yaml.Node. func (f *RemoteFile) GetContentAsYAMLNode() (*yaml.Node, error) { f.contentLock.Lock() + defer f.contentLock.Unlock() idx := f.GetIndex() if idx != nil && idx.root != nil { - f.contentLock.Unlock() return idx.GetRootNode(), nil } + if f.parsed != nil { + if idx != nil && idx.root == nil { + idx.root = f.parsed + } + return f.parsed, nil + } if f.data == nil { - f.contentLock.Unlock() return nil, fmt.Errorf("no data to parse for file: %s", f.fullPath) } var root yaml.Node err := yaml.Unmarshal(f.data, &root) if err != nil { - f.contentLock.Unlock() return nil, err } if idx != nil && idx.root == nil { @@ -314,7 +318,6 @@ func (f *RemoteFile) GetContentAsYAMLNode() (*yaml.Node, error) { if f.parsed == nil { f.parsed = &root } - f.contentLock.Unlock() return &root, nil } @@ -524,7 +527,9 @@ func (i *RemoteFS) GetFiles() map[string]RolodexFile { // GetErrors returns any errors that occurred during the indexing process. func (i *RemoteFS) GetErrors() []error { - return i.remoteErrors + i.errMutex.Lock() + defer i.errMutex.Unlock() + return append([]error(nil), i.remoteErrors...) } type waiterRemote struct { @@ -536,6 +541,15 @@ type waiterRemote struct { mu sync.Mutex } +func remoteLookupCacheKey(u *url.URL) string { + if u == nil { + return "" + } + cloned := *u + cloned.Fragment = "" + return cloned.String() +} + func (i *RemoteFS) OpenWithContext(ctx context.Context, remoteURL string) (fs.File, error) { if i.indexConfig != nil && !i.indexConfig.AllowRemoteLookup { return nil, fmt.Errorf("remote lookup for '%s' is not allowed, please set "+ @@ -554,96 +568,25 @@ func (i *RemoteFS) OpenWithContext(ctx context.Context, remoteURL string) (fs.Fi return nil, err } remoteParsedURLOriginal, _ := url.Parse(remoteURL) + i.normalizeRemoteURL(remoteParsedURL) + cacheKey := remoteLookupCacheKey(remoteParsedURL) - // try path first - if r, ok := i.Files.Load(remoteParsedURL.Path); ok { - return r.(*RemoteFile), nil - } - - fileExt := ExtractFileType(remoteParsedURL.Path) - - // Handle unsupported file extensions with content detection if enabled - if fileExt == UNSUPPORTED { - if i.indexConfig != nil && i.indexConfig.AllowUnknownExtensionContentDetection { - if i.logger != nil { - i.logger.Debug("[rolodex remote loader] attempting content detection for unknown file extension", "url", remoteParsedURL.String()) - } - // Attempt to detect content type - fileExt = detectRemoteContentType(remoteParsedURL.String(), i.RemoteHandlerFunc, i.logger) - if fileExt == UNSUPPORTED { - // Clear cache entry on completion to keep memory usage low - defer func() { - contentDetectionMutex.Lock() - delete(contentDetectionCache, remoteParsedURL.String()) - contentDetectionMutex.Unlock() - }() - - i.remoteErrors = append(i.remoteErrors, fs.ErrInvalid) - if i.logger != nil { - i.logger.Warn("[rolodex remote loader] content detection failed, unsupported content type", "url", remoteParsedURL.String()) - } - return nil, &fs.PathError{Op: "open", Path: remoteURL, Err: fs.ErrInvalid} - } - if i.logger != nil { - typeStr := "UNSUPPORTED" - if fileExt == JSON { - typeStr = "JSON" - } else if fileExt == YAML { - typeStr = "YAML" - } - i.logger.Debug("[rolodex remote loader] content detection successful", "url", remoteParsedURL.String(), "detectedType", typeStr) - } - } else { - // Content detection disabled, treat as unsupported - i.remoteErrors = append(i.remoteErrors, fs.ErrInvalid) - if i.logger != nil { - i.logger.Warn("[rolodex remote loader] unknown file extension and content detection disabled", "file", remoteURL, "remoteURL", remoteParsedURL.String()) - } - return nil, &fs.PathError{Op: "open", Path: remoteURL, Err: fs.ErrInvalid} - } + if cached := i.loadCachedRemoteFile(cacheKey, remoteParsedURL.Path); cached != nil { + return cached, nil } - // Use LoadOrStore to atomically check if someone is already processing this file. - // This prevents the race condition where two goroutines both see "not processing" - // and both start processing the same file. - processingWaiter := &waiterRemote{f: remoteParsedURL.Path} - processingWaiter.mu.Lock() - - if existing, loaded := i.ProcessingFiles.LoadOrStore(remoteParsedURL.Path, processingWaiter); loaded { - // Someone else is already processing this file, wait for them - processingWaiter.mu.Unlock() // Release our unused waiter's lock - wait := existing.(*waiterRemote) - - wait.mu.Lock() - i.logger.Debug("[rolodex remote loader] waiting for existing fetch to complete", "file", remoteURL, - "remoteURL", remoteParsedURL.String()) - f := wait.file - e := wait.error - i.logger.Debug("[rolodex remote loader]: waiting done, remote completed, returning file", "file", - remoteParsedURL.String(), "listeners", wait.listeners) - wait.mu.Unlock() - return f, e + fileExt, err := i.detectRemoteFileType(remoteURL, remoteParsedURL) + if err != nil { + return nil, err } - // We successfully stored our waiter, so we're responsible for processing this file - - // if the remote URL is absolute (http:// or https://), and we have a rootURL defined, we need to override - // the host being defined by this URL, and use the rootURL instead, but keep the path. - if i.rootURLParsed != nil { - remoteParsedURL.Host = i.rootURLParsed.Host - remoteParsedURL.Scheme = i.rootURLParsed.Scheme - // this has been disabled, because I don't think it has value, it causes more problems than it solves currently. - // if !strings.HasPrefix(remoteParsedURL.Path, "/") { - // remoteParsedURL.Path = filepath.Join(i.rootURLParsed.Path, remoteParsedURL.Path) - // remoteParsedURL.Path = strings.ReplaceAll(remoteParsedURL.Path, "\\", "/") - // } + processingWaiter, inFlightFile, inFlightErr := i.acquireRemoteProcessingWaiter(cacheKey, remoteParsedURL.Path, remoteURL, remoteParsedURL) + if processingWaiter == nil { + return inFlightFile, inFlightErr } if remoteParsedURL.Scheme == "" { - - processingWaiter.done = true - i.ProcessingFiles.Delete(remoteParsedURL.Path) - processingWaiter.mu.Unlock() + i.releaseRemoteProcessingWaiter(processingWaiter, cacheKey, nil, nil) return nil, nil // not a remote file, nothing wrong with that - just we can't keep looking here partner. } @@ -651,16 +594,11 @@ func (i *RemoteFS) OpenWithContext(ctx context.Context, remoteURL string) (fs.Fi response, clientErr := i.RemoteHandlerFunc(remoteParsedURL.String()) if clientErr != nil { - - i.errMutex.Lock() - i.remoteErrors = append(i.remoteErrors, clientErr) - i.errMutex.Unlock() - - // remove from processing - processingWaiter.done = true - i.ProcessingFiles.Delete(remoteParsedURL.Path) - processingWaiter.mu.Unlock() - + i.appendRemoteError(clientErr) + i.releaseRemoteProcessingWaiter(processingWaiter, cacheKey, nil, nil) + if response != nil && response.Body != nil { + _ = response.Body.Close() + } if response != nil { i.logger.Error("client error", "error", clientErr, "status", response.StatusCode) } else { @@ -669,54 +607,164 @@ func (i *RemoteFS) OpenWithContext(ctx context.Context, remoteURL string) (fs.Fi return nil, clientErr } if response == nil { - // remove from processing - processingWaiter.done = true - i.ProcessingFiles.Delete(remoteParsedURL.Path) - processingWaiter.mu.Unlock() + i.releaseRemoteProcessingWaiter(processingWaiter, cacheKey, nil, nil) return nil, fmt.Errorf("empty response from remote URL: %s", remoteParsedURL.String()) } + defer func() { + if response.Body != nil { + _ = response.Body.Close() + } + }() responseBytes, readError := io.ReadAll(response.Body) if readError != nil { - - // remove from processing - processingWaiter.error = readError - processingWaiter.done = true - i.ProcessingFiles.Delete(remoteParsedURL.Path) - processingWaiter.mu.Unlock() + i.releaseRemoteProcessingWaiter(processingWaiter, cacheKey, nil, readError) return nil, fmt.Errorf("error reading bytes from remote file '%s': [%s]", remoteParsedURL.String(), readError.Error()) } if response.StatusCode >= 400 { - - // remove from processing - processingWaiter.error = fmt.Errorf("remote file '%s' returned status code %d", remoteParsedURL.String(), response.StatusCode) - processingWaiter.done = true - i.ProcessingFiles.Delete(remoteParsedURL.Path) + waitErr := fmt.Errorf("remote file '%s' returned status code %d", remoteParsedURL.String(), response.StatusCode) + i.releaseRemoteProcessingWaiter(processingWaiter, cacheKey, nil, waitErr) i.logger.Error("unable to fetch remote document", "file", remoteParsedURL.Path, "status", response.StatusCode, "resp", string(responseBytes)) - processingWaiter.mu.Unlock() return nil, fmt.Errorf("unable to fetch remote document '%s' (error %d)", remoteParsedURL.String(), response.StatusCode) } - absolutePath := remoteParsedURL.Path + remoteFile := i.createRemoteFile(remoteParsedURL, fileExt, responseBytes, response.Header) + copiedCfg := i.createRemoteIndexConfig(remoteParsedURL, remoteParsedURLOriginal) - // extract last modified from response - lastModified := response.Header.Get("Last-Modified") + if len(remoteFile.data) > 0 { + i.logger.Debug("[rolodex remote loaded] successfully loaded file", "file", remoteParsedURL.Path) + } - // parse the last modified date into a time object - lastModifiedTime, parseErr := time.Parse(time.RFC1123, lastModified) + i.Files.Store(cacheKey, remoteFile) + i.releaseRemoteProcessingWaiter(processingWaiter, cacheKey, remoteFile, nil) - if parseErr != nil { - // can't extract last modified, so use now - lastModifiedTime = time.Now() + i.indexRemoteFile(ctx, remoteFile, copiedCfg, remoteParsedURL, remoteParsedURLOriginal) + + return remoteFile, errors.Join(i.remoteErrors...) +} + +func (i *RemoteFS) normalizeRemoteURL(remoteParsedURL *url.URL) { + if i.rootURLParsed == nil || remoteParsedURL == nil { + return } + remoteParsedURL.Host = i.rootURLParsed.Host + remoteParsedURL.Scheme = i.rootURLParsed.Scheme +} - filename := filepath.Base(remoteParsedURL.Path) +func (i *RemoteFS) loadCachedRemoteFile(cacheKey, legacyPath string) *RemoteFile { + if r, ok := i.Files.Load(cacheKey); ok { + return r.(*RemoteFile) + } + if cacheKey != legacyPath { + if legacy, ok := i.Files.Load(legacyPath); ok { + if legacyFile, ok := legacy.(*RemoteFile); ok && legacyFile.URL == nil { + return legacyFile + } + } + } + return nil +} + +func (i *RemoteFS) detectRemoteFileType(remoteURL string, remoteParsedURL *url.URL) (FileExtension, error) { + fileExt := ExtractFileType(remoteParsedURL.Path) + if fileExt != UNSUPPORTED { + return fileExt, nil + } + if i.indexConfig != nil && i.indexConfig.AllowUnknownExtensionContentDetection { + if i.logger != nil { + i.logger.Debug("[rolodex remote loader] attempting content detection for unknown file extension", "url", remoteParsedURL.String()) + } + fileExt = detectRemoteContentType(remoteParsedURL.String(), i.RemoteHandlerFunc, i.logger) + if fileExt == UNSUPPORTED { + defer func() { + contentDetectionMutex.Lock() + delete(contentDetectionCache, remoteParsedURL.String()) + contentDetectionMutex.Unlock() + }() + i.appendRemoteError(fs.ErrInvalid) + if i.logger != nil { + i.logger.Warn("[rolodex remote loader] content detection failed, unsupported content type", "url", remoteParsedURL.String()) + } + return UNSUPPORTED, &fs.PathError{Op: "open", Path: remoteURL, Err: fs.ErrInvalid} + } + if i.logger != nil { + typeStr := "UNSUPPORTED" + if fileExt == JSON { + typeStr = "JSON" + } else if fileExt == YAML { + typeStr = "YAML" + } + i.logger.Debug("[rolodex remote loader] content detection successful", "url", remoteParsedURL.String(), "detectedType", typeStr) + } + return fileExt, nil + } + i.appendRemoteError(fs.ErrInvalid) + if i.logger != nil { + i.logger.Warn("[rolodex remote loader] unknown file extension and content detection disabled", "file", remoteURL, "remoteURL", remoteParsedURL.String()) + } + return UNSUPPORTED, &fs.PathError{Op: "open", Path: remoteURL, Err: fs.ErrInvalid} +} + +func (i *RemoteFS) acquireRemoteProcessingWaiter(cacheKey, legacyPath, remoteURL string, remoteParsedURL *url.URL) (*waiterRemote, fs.File, error) { + processingWaiter := &waiterRemote{f: cacheKey} + processingWaiter.mu.Lock() + + if cacheKey != legacyPath { + if existing, ok := i.ProcessingFiles.Load(legacyPath); ok { + processingWaiter.mu.Unlock() + file, err := i.waitForRemoteProcessing(existing.(*waiterRemote), remoteURL, remoteParsedURL, true) + return nil, file, err + } + } - remoteFile := &RemoteFile{ - filename: filename, + if existing, loaded := i.ProcessingFiles.LoadOrStore(cacheKey, processingWaiter); loaded { + processingWaiter.mu.Unlock() + file, err := i.waitForRemoteProcessing(existing.(*waiterRemote), remoteURL, remoteParsedURL, false) + return nil, file, err + } + + return processingWaiter, nil, nil +} + +func (i *RemoteFS) waitForRemoteProcessing(wait *waiterRemote, remoteURL string, remoteParsedURL *url.URL, legacy bool) (fs.File, error) { + wait.mu.Lock() + defer wait.mu.Unlock() + if legacy { + i.logger.Debug("[rolodex remote loader] waiting for legacy in-flight fetch to complete", "file", remoteURL, + "remoteURL", remoteParsedURL.String()) + } else { + i.logger.Debug("[rolodex remote loader] waiting for existing fetch to complete", "file", remoteURL, + "remoteURL", remoteParsedURL.String()) + i.logger.Debug("[rolodex remote loader]: waiting done, remote completed, returning file", "file", + remoteParsedURL.String(), "listeners", wait.listeners) + } + return wait.file, wait.error +} + +func (i *RemoteFS) releaseRemoteProcessingWaiter(waiter *waiterRemote, cacheKey string, file *RemoteFile, err error) { + waiter.file = file + waiter.error = err + waiter.done = true + i.ProcessingFiles.Delete(cacheKey) + waiter.mu.Unlock() +} + +func (i *RemoteFS) appendRemoteError(err error) { + i.errMutex.Lock() + i.remoteErrors = append(i.remoteErrors, err) + i.errMutex.Unlock() +} + +func (i *RemoteFS) createRemoteFile(remoteParsedURL *url.URL, fileExt FileExtension, responseBytes []byte, headers http.Header) *RemoteFile { + lastModifiedTime, parseErr := time.Parse(time.RFC1123, headers.Get("Last-Modified")) + if parseErr != nil { + lastModifiedTime = time.Now() + } + return &RemoteFile{ + filename: filepath.Base(remoteParsedURL.Path), name: remoteParsedURL.Path, extension: fileExt, data: responseBytes, @@ -725,71 +773,43 @@ func (i *RemoteFS) OpenWithContext(ctx context.Context, remoteURL string) (fs.Fi lastModified: lastModifiedTime, indexingComplete: make(chan struct{}), } +} +func (i *RemoteFS) createRemoteIndexConfig(remoteParsedURL, remoteParsedURLOriginal *url.URL) *SpecIndexConfig { copiedCfg := *i.indexConfig - newBase := fmt.Sprintf("%s://%s%s", remoteParsedURLOriginal.Scheme, remoteParsedURLOriginal.Host, filepath.Dir(remoteParsedURL.Path)) newBaseURL, _ := url.Parse(newBase) - if newBaseURL != nil { copiedCfg.BaseURL = newBaseURL } copiedCfg.SpecAbsolutePath = remoteParsedURL.String() - // Force sequential extraction for child indexes to avoid cascading async issues. - // When the main index uses async mode, spawning async operations for each remote - // file creates complex timing dependencies that can result in missing references. copiedCfg.ExtractRefsSequentially = true + return &copiedCfg +} - if len(remoteFile.data) > 0 { - i.logger.Debug("[rolodex remote loaded] successfully loaded file", "file", absolutePath) - } - - // Store in Files and release the waiter BEFORE indexing to prevent deadlocks. - // If file A needs file B and file B needs file A, holding the lock during indexing - // would cause a deadlock. The indexOnce in Index handles concurrent access safely. - i.Files.Store(absolutePath, remoteFile) - - // remove from processing - processingWaiter.file = remoteFile - processingWaiter.done = true - i.ProcessingFiles.Delete(remoteParsedURL.Path) - processingWaiter.mu.Unlock() - - // Add this file to the context's indexing set to prevent deadlocks - // when circular references cause the same file to be looked up recursively. - // We add multiple forms of the URL to handle different lookup patterns: - // - absolutePath: the path portion (e.g., /second.yaml) - // - remoteParsedURL.String(): the normalized URL (after host override) - // - remoteParsedURLOriginal.String(): the ORIGINAL URL before host normalization - // The original URL is critical because lookupRolodex uses the original reference URL - // when checking IsFileBeingIndexed, not the normalized one. - indexingCtx := AddIndexingFile(ctx, absolutePath) +func (i *RemoteFS) indexRemoteFile( + ctx context.Context, + remoteFile *RemoteFile, + copiedCfg *SpecIndexConfig, + remoteParsedURL, remoteParsedURLOriginal *url.URL, +) { + indexingCtx := AddIndexingFile(ctx, remoteParsedURL.Path) indexingCtx = AddIndexingFile(indexingCtx, remoteParsedURL.String()) indexingCtx = AddIndexingFile(indexingCtx, remoteParsedURLOriginal.String()) - // Now index the file AFTER releasing the lock - idx, idxError := remoteFile.Index(indexingCtx, &copiedCfg) - + idx, idxError := remoteFile.Index(indexingCtx, copiedCfg) if idxError != nil && idx == nil { - i.errMutex.Lock() - i.remoteErrors = append(i.remoteErrors, idxError) - i.errMutex.Unlock() - } else { - - // for each index, we need a resolver - resolver := NewResolver(idx) - idx.resolver = resolver - idx.BuildIndex() - if i.rolodex != nil { - i.rolodex.AddExternalIndex(idx, remoteParsedURL.String()) - } + i.appendRemoteError(idxError) + remoteFile.signalIndexingComplete() + return + } + NewResolver(idx) + idx.BuildIndex() + if i.rolodex != nil { + i.rolodex.AddExternalIndex(idx, remoteParsedURL.String()) } - - // Signal that indexing is complete - other goroutines waiting for this file can proceed remoteFile.signalIndexingComplete() - - return remoteFile, errors.Join(i.remoteErrors...) } // Open opens a file, returning it or an error. If the file is not found, the error is of type *PathError. diff --git a/index/rolodex_remote_loader_test.go b/index/rolodex_remote_loader_test.go index 23a8b925a..5c408a0ac 100644 --- a/index/rolodex_remote_loader_test.go +++ b/index/rolodex_remote_loader_test.go @@ -12,11 +12,13 @@ import ( "net/http/httptest" "net/url" "strings" + "sync/atomic" "testing" "time" "context" "github.com/stretchr/testify/assert" + "go.yaml.in/yaml/v4" ) var test_httpClient = &http.Client{Timeout: time.Duration(60) * time.Second} @@ -231,6 +233,19 @@ func TestRemoteFile_GoodContent(t *testing.T) { assert.Error(t, err) } +func TestRemoteFile_GetContentAsYAMLNode_UsesParsedFastPath(t *testing.T) { + parsed := &yaml.Node{Kind: yaml.MappingNode} + rf := &RemoteFile{ + data: []byte("not: valid: yaml"), + index: *NewTestSpecIndex(), + parsed: parsed, + } + + node, err := rf.GetContentAsYAMLNode() + assert.NoError(t, err) + assert.Same(t, parsed, node) +} + func TestRemoteFile_Index_AlreadySet(t *testing.T) { rf := &RemoteFile{data: []byte("good: data"), index: *NewTestSpecIndex()} x, y := rf.Index(context.Background(), &SpecIndexConfig{}) @@ -444,6 +459,128 @@ func TestNewRemoteFS_BadURL(t *testing.T) { assert.Error(t, y) } +type trackedReadCloser struct { + io.Reader + closed atomic.Bool +} + +func (t *trackedReadCloser) Close() error { + t.closed.Store(true) + return nil +} + +func TestRemoteFS_OpenWithContext_ClosesResponseBody(t *testing.T) { + config := CreateOpenAPIIndexConfig() + rfs, err := NewRemoteFSWithConfig(config) + assert.NoError(t, err) + + body := &trackedReadCloser{Reader: strings.NewReader("openapi: 3.1.0")} + rfs.RemoteHandlerFunc = func(url string) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: body, + Header: make(http.Header), + }, nil + } + + file, err := rfs.OpenWithContext(context.Background(), "http://example.com/spec.yaml") + assert.NoError(t, err) + assert.NotNil(t, file) + assert.True(t, body.closed.Load()) +} + +func TestRemoteFS_OpenWithContext_ClosesResponseBodyOnClientError(t *testing.T) { + config := CreateOpenAPIIndexConfig() + rfs, err := NewRemoteFSWithConfig(config) + assert.NoError(t, err) + + body := &trackedReadCloser{Reader: strings.NewReader("boom")} + rfs.RemoteHandlerFunc = func(url string) (*http.Response, error) { + return &http.Response{ + StatusCode: 502, + Body: body, + Header: make(http.Header), + }, errors.New("client failed") + } + + file, err := rfs.OpenWithContext(context.Background(), "http://example.com/spec.yaml") + assert.Nil(t, file) + assert.Error(t, err) + assert.True(t, body.closed.Load()) +} + +func TestRemoteFS_OpenWithContext_CacheKeyIncludesHost(t *testing.T) { + config := CreateOpenAPIIndexConfig() + rfs, err := NewRemoteFSWithConfig(config) + assert.NoError(t, err) + + rfs.RemoteHandlerFunc = func(rawURL string) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader(fmt.Sprintf("source: %s", rawURL))), + Header: make(http.Header), + }, nil + } + + first, err := rfs.OpenWithContext(context.Background(), "http://one.example/shared.yaml") + assert.NoError(t, err) + second, err := rfs.OpenWithContext(context.Background(), "http://two.example/shared.yaml") + assert.NoError(t, err) + + firstBytes, err := io.ReadAll(first) + assert.NoError(t, err) + secondBytes, err := io.ReadAll(second) + assert.NoError(t, err) + assert.NotEqual(t, string(firstBytes), string(secondBytes)) +} + +func TestRemoteLookupCacheKey(t *testing.T) { + assert.Equal(t, "", remoteLookupCacheKey(nil)) + + u, err := url.Parse("https://example.com/spec.yaml#/components/schemas/Pet") + assert.NoError(t, err) + assert.Equal(t, "https://example.com/spec.yaml", remoteLookupCacheKey(u)) +} + +func TestRemoteFS_NormalizeAndLoadCachedHelpers(t *testing.T) { + config := CreateOpenAPIIndexConfig() + config.BaseURL, _ = url.Parse("https://root.example/base") + rfs, err := NewRemoteFSWithConfig(config) + assert.NoError(t, err) + + target, err := url.Parse("http://other.example/spec.yaml") + assert.NoError(t, err) + rfs.normalizeRemoteURL(target) + assert.Equal(t, "https", target.Scheme) + assert.Equal(t, "root.example", target.Host) + + legacyFile := &RemoteFile{filename: "spec.yaml"} + rfs.Files.Store("/spec.yaml", legacyFile) + assert.Same(t, legacyFile, rfs.loadCachedRemoteFile("https://root.example/spec.yaml", "/spec.yaml")) +} + +func TestRemoteFS_CreateRemoteHelpers(t *testing.T) { + config := CreateOpenAPIIndexConfig() + rfs, err := NewRemoteFSWithConfig(config) + assert.NoError(t, err) + + current, err := url.Parse("https://root.example/spec.yaml") + assert.NoError(t, err) + original, err := url.Parse("http://source.example/spec.yaml") + assert.NoError(t, err) + + remoteFile := rfs.createRemoteFile(current, YAML, []byte("openapi: 3.1.0"), http.Header{ + "Last-Modified": []string{"Wed, 21 Oct 2015 07:28:00 GMT"}, + }) + assert.Equal(t, "spec.yaml", remoteFile.GetFileName()) + assert.Equal(t, YAML, remoteFile.GetFileExtension()) + + cfg := rfs.createRemoteIndexConfig(current, original) + assert.Equal(t, "https://root.example/spec.yaml", cfg.SpecAbsolutePath) + assert.True(t, cfg.ExtractRefsSequentially) + assert.Equal(t, "http://source.example", cfg.BaseURL.Scheme+"://"+cfg.BaseURL.Host) +} + func TestRemoteFile_SignalIndexingComplete_DoubleClose(t *testing.T) { // Test that calling signalIndexingComplete twice doesn't panic // (i.e., the closed channel check works correctly) diff --git a/index/rolodex_test.go b/index/rolodex_test.go index 1bff32f9c..56b0aad99 100644 --- a/index/rolodex_test.go +++ b/index/rolodex_test.go @@ -1763,6 +1763,70 @@ components: assert.Len(t, rolo.GetCaughtErrors(), 0) } +func TestRolodex_Resolve_AggregatesIgnoredCircularRefsFromExternalIndexes(t *testing.T) { + cfg := CreateOpenAPIIndexConfig() + rolodex := NewRolodex(cfg) + + rootIndex := NewTestSpecIndex().Load().(*SpecIndex) + rootIndex.root = &yaml.Node{Kind: yaml.MappingNode} + rootIndex.SetResolver(NewResolver(rootIndex)) + rolodex.rootIndex = rootIndex + + externalIndex := NewTestSpecIndex().Load().(*SpecIndex) + externalIndex.root = &yaml.Node{Kind: yaml.MappingNode} + externalIndex.SetResolver(NewResolver(externalIndex)) + externalIndex.GetResolver().ignoredPolyReferences = []*CircularReferenceResult{{ + IsPolymorphicResult: true, + LoopPoint: &Reference{FullDefinition: "#/components/schemas/External"}, + }} + rolodex.indexes = []*SpecIndex{externalIndex} + + rolodex.Resolve() + + assert.Len(t, rolodex.GetIgnoredCircularReferences(), 1) +} + +func TestRolodex_collectResolvers(t *testing.T) { + rolodex := NewRolodex(CreateOpenAPIIndexConfig()) + rootIndex := NewTestSpecIndex().Load().(*SpecIndex) + rootIndex.SetResolver(NewResolver(rootIndex)) + rolodex.rootIndex = rootIndex + + externalIndex := NewTestSpecIndex().Load().(*SpecIndex) + externalIndex.SetResolver(NewResolver(externalIndex)) + rolodex.indexes = []*SpecIndex{externalIndex, {}} + + resolvers := rolodex.collectResolvers() + + assert.Len(t, resolvers, 2) +} + +func TestRolodex_mergeResolverResults(t *testing.T) { + rolodex := NewRolodex(CreateOpenAPIIndexConfig()) + idx := NewTestSpecIndex().Load().(*SpecIndex) + idx.root = &yaml.Node{Kind: yaml.MappingNode} + resolver := NewResolver(idx) + resolver.resolvingErrors = []*ResolvingError{{ErrorRef: fmt.Errorf("boom")}} + resolver.ignoredPolyReferences = []*CircularReferenceResult{{ + LoopPoint: &Reference{FullDefinition: "#/poly"}, + }} + resolver.ignoredArrayReferences = []*CircularReferenceResult{{ + LoopPoint: &Reference{FullDefinition: "#/array"}, + }} + resolver.circularReferences = []*CircularReferenceResult{ + {IsInfiniteLoop: false}, + {IsInfiniteLoop: true, Start: &Reference{Definition: "#/loop"}}, + } + resolver.circChecked = true + + rolodex.mergeResolverResults(resolver) + + assert.Len(t, rolodex.caughtErrors, 1) + assert.Len(t, rolodex.ignoredCircularReferences, 2) + assert.Len(t, rolodex.safeCircularReferences, 1) + assert.Len(t, rolodex.infiniteCircularReferences, 1) +} + func TestRolodex_CircularReferencesArrayIgnored_PostCheck(t *testing.T) { d := `openapi: 3.1.0 components: @@ -2139,6 +2203,151 @@ func TestRolodex_RemoteFileSeekingErrors(t *testing.T) { assert.Contains(t, err.Error(), "seeking error 2") } +func TestRolodex_LocalPathForOpen(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + localFS := &LocalFS{} + + assert.Equal(t, "/tmp/spec.yaml", rolo.localPathForOpen("/tmp", "/tmp/spec.yaml", localFS)) + assert.Equal(t, "nested/spec.yaml", rolo.localPathForOpen("/tmp", "/tmp/nested/spec.yaml", os.DirFS("/tmp"))) +} + +func TestRolodex_OpenLocalLocation_EmptyFileReturnsNilWithoutErrors(t *testing.T) { + tempDir := t.TempDir() + specPath := filepath.Join(tempDir, "spec.yaml") + err := os.WriteFile(specPath, nil, 0o644) + assert.NoError(t, err) + + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + rolo.AddLocalFS(tempDir, os.DirFS(tempDir)) + + localFile, errs := rolo.openLocalLocation(context.Background(), specPath) + assert.Nil(t, localFile) + assert.Empty(t, errs) +} + +func TestRolodex_OpenLocalLocation_UsesFallbackPath(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + testFS := &fallbackFS{failOnCalculatedPath: true} + rolo.AddLocalFS("/some/base/path", testFS) + + localFile, errs := rolo.openLocalLocation(context.Background(), "spec.yaml") + assert.NotNil(t, localFile) + assert.Empty(t, errs) + assert.True(t, testFS.usedFallback) +} + +func TestRolodex_OpenLocalLocation_UsesFallbackPathForRelativeBaseDir(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + testFS := &fallbackFS{failOnCalculatedPath: true} + rolo.AddLocalFS("some/base/path", testFS) + + localFile, errs := rolo.openLocalLocation(context.Background(), "spec.yaml") + assert.NotNil(t, localFile) + assert.Empty(t, errs) + assert.True(t, testFS.usedFallback) +} + +func TestRolodex_OpenLocalLocation_ReturnsErrorsWhenMissing(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + rolo.AddLocalFS("", fstest.MapFS{}) + + localFile, errs := rolo.openLocalLocation(context.Background(), "missing.yaml") + assert.Nil(t, localFile) + assert.NotEmpty(t, errs) +} + +func TestRolodex_OpenLocalLocation_ReturnsFallbackErrorWhenBothLookupsFail(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + testFS := &fallbackFS{failOnCalculatedPath: true} + rolo.AddLocalFS("/some/base/path", testFS) + + localFile, errs := rolo.openLocalLocation(context.Background(), "missing.yaml") + assert.Nil(t, localFile) + assert.NotEmpty(t, errs) +} + +func TestRolodex_OpenLocalLocation_FallsBackWhenComputedPathDiffers(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + location := "/tmp/openapi/spec.yaml" + testFS := &absoluteFallbackFS{fallbackLocation: location} + rolo.AddLocalFS("/tmp", testFS) + + localFile, errs := rolo.openLocalLocation(context.Background(), location) + assert.NotNil(t, localFile) + assert.Empty(t, errs) + assert.True(t, testFS.usedFallback) +} + +func TestRolodex_AsLocalFile_GenericAndExistingRolodexFile(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + rolo.rootIndex = NewTestSpecIndex().Load().(*SpecIndex) + + localFile, errs := rolo.asLocalFile(&testRolodexFile{}, "/tmp/spec.yaml") + assert.NotNil(t, localFile) + assert.Empty(t, errs) + assert.Equal(t, "/test/path/spec.yaml", localFile.fullPath) + + localFile, errs = rolo.asLocalFile(&testFile{content: "hello"}, "/tmp/spec.yaml") + assert.NotNil(t, localFile) + assert.Empty(t, errs) + assert.Equal(t, "/tmp/spec.yaml", localFile.fullPath) + + localFile, errs = rolo.asLocalFile(&testRolodexFile{errorYaml: true}, "/tmp/spec.yaml") + assert.NotNil(t, localFile) + assert.Error(t, errors.Join(errs...)) + + localFile, errs = rolo.asLocalFile(&testFile{content: ""}, "/tmp/spec.yaml") + assert.Nil(t, localFile) + assert.Empty(t, errs) +} + +func TestRolodex_AsRemoteFile_AndWrappers(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + rolo.rootIndex = NewTestSpecIndex().Load().(*SpecIndex) + + remoteFile, errs := rolo.asRemoteFile(&testFile{content: "hello"}, "http://example.com/spec.yaml") + assert.NotNil(t, remoteFile) + assert.Empty(t, errs) + + wrappedRemote, err := rolo.wrapRemoteRolodexFile(&RemoteFile{ + fullPath: "http://example.com/spec.yaml", + seekingErrors: []error{fmt.Errorf("seek failed")}, + }) + assert.NotNil(t, wrappedRemote) + assert.Error(t, err) + + wrappedLocal, err := rolo.wrapLocalRolodexFile(&LocalFile{ + fullPath: "/tmp/spec.yaml", + readingErrors: []error{fmt.Errorf("read failed")}, + }) + assert.NotNil(t, wrappedLocal) + assert.Error(t, err) +} + +func TestRolodex_AsFileHelperErrors(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + + localFile, errs := rolo.asLocalFile(&errorReadFile{}, "/tmp/spec.yaml") + assert.Nil(t, localFile) + assert.Len(t, errs, 1) + + localFile, errs = rolo.asLocalFile(&errorStatFile{testFile: testFile{content: "hello"}}, "/tmp/spec.yaml") + assert.Nil(t, localFile) + assert.Len(t, errs, 1) + + remoteFile, errs := rolo.asRemoteFile(&errorReadFile{}, "http://example.com/spec.yaml") + assert.Nil(t, remoteFile) + assert.Len(t, errs, 1) + + remoteFile, errs = rolo.asRemoteFile(&errorStatFile{testFile: testFile{content: "hello"}}, "http://example.com/spec.yaml") + assert.Nil(t, remoteFile) + assert.Len(t, errs, 1) + + remoteFile, errs = rolo.asRemoteFile(&testFile{content: ""}, "http://example.com/spec.yaml") + assert.Nil(t, remoteFile) + assert.Empty(t, errs) +} + func TestRolodex_NilCheck(t *testing.T) { var r *Rolodex _, err := r.OpenWithContext(context.Background(), "spec.yaml") @@ -2182,6 +2391,19 @@ func (f *fallbackFS) Open(name string) (fs.File, error) { return &testFile{content: "test content"}, nil } +type absoluteFallbackFS struct { + fallbackLocation string + usedFallback bool +} + +func (f *absoluteFallbackFS) Open(name string) (fs.File, error) { + if name == f.fallbackLocation { + f.usedFallback = true + return &testFile{content: "test content"}, nil + } + return nil, fs.ErrNotExist +} + func (f *fallbackFS) GetFiles() map[string]RolodexFile { if f.files != nil { return f.files @@ -2215,6 +2437,20 @@ type testFile struct { offset int64 } +type errorReadFile struct{} + +func (e *errorReadFile) Read(_ []byte) (int, error) { return 0, fmt.Errorf("read failed") } +func (e *errorReadFile) Close() error { return nil } +func (e *errorReadFile) Stat() (fs.FileInfo, error) { + return &testFileInfo{name: "bad.yaml", size: 0}, nil +} + +type errorStatFile struct { + testFile +} + +func (e *errorStatFile) Stat() (fs.FileInfo, error) { return nil, fmt.Errorf("stat failed") } + func (tf *testFile) Read(p []byte) (n int, err error) { if tf.offset >= int64(len(tf.content)) { return 0, io.EOF @@ -2631,6 +2867,8 @@ func TestRolodex_Release(t *testing.T) { rolodex := NewRolodex(cfg) idx := &SpecIndex{config: cfg} rolodex.AddIndex(idx) + rolodex.localFS = map[string]fs.FS{"local": os.DirFS(".")} + rolodex.remoteFS = map[string]fs.FS{"remote": os.DirFS(".")} rolodex.rootNode = &yaml.Node{Value: "root"} rolodex.rootIndex = idx rolodex.caughtErrors = []error{fmt.Errorf("test")} @@ -2640,9 +2878,19 @@ func TestRolodex_Release(t *testing.T) { rolodex.debouncedSafeCircRefs = []*CircularReferenceResult{{}} rolodex.debouncedIgnoredCircRefs = []*CircularReferenceResult{{}} rolodex.globalSchemaIdRegistry = map[string]*SchemaIdEntry{"test": {}} + rolodex.indexed = true + rolodex.built = true + rolodex.manualBuilt = true + rolodex.resolved = true + rolodex.circChecked = true + rolodex.indexingDuration = time.Second + rolodex.logger = slog.Default() + rolodex.id = "test-id" rolodex.Release() + assert.Nil(t, rolodex.localFS) + assert.Nil(t, rolodex.remoteFS) assert.Nil(t, rolodex.indexes) assert.Nil(t, rolodex.indexMap) assert.Nil(t, rolodex.rootIndex) @@ -2654,6 +2902,15 @@ func TestRolodex_Release(t *testing.T) { assert.Nil(t, rolodex.debouncedSafeCircRefs) assert.Nil(t, rolodex.debouncedIgnoredCircRefs) assert.Nil(t, rolodex.globalSchemaIdRegistry) + assert.Nil(t, rolodex.indexConfig) + assert.Zero(t, rolodex.indexingDuration) + assert.False(t, rolodex.indexed) + assert.False(t, rolodex.built) + assert.False(t, rolodex.manualBuilt) + assert.False(t, rolodex.resolved) + assert.False(t, rolodex.circChecked) + assert.Nil(t, rolodex.logger) + assert.Empty(t, rolodex.id) } func TestRolodex_Release_Nil(t *testing.T) { diff --git a/index/search_index.go b/index/search_index.go index 5d70dd3f1..a21439951 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -303,9 +303,9 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex } idx := rFile.GetIndex() - if index.resolver != nil { + if resolver := index.GetResolver(); resolver != nil { index.resolverLock.Lock() - index.resolver.indexesVisited++ + resolver.indexesVisited++ index.resolverLock.Unlock() } if idx != nil { diff --git a/index/spec_index.go b/index/spec_index.go deleted file mode 100644 index e16009429..000000000 --- a/index/spec_index.go +++ /dev/null @@ -1,1524 +0,0 @@ -// Copyright 2022-2033 Dave Shanley / Quobix -// SPDX-License-Identifier: MIT - -// Package index contains an OpenAPI indexer that will very quickly scan through an OpenAPI specification (all versions) -// and extract references to all the important nodes you might want to look up, as well as counts on total objects. -// -// When extracting references, the index can determine if the reference is local to the file (recommended) or the -// reference is located in another local file, or a remote file. The index will then attempt to load in those remote -// files and look up the references there, or continue following the chain. -// -// When the index loads in a local or remote file, it will also index that remote spec as well. This means everything -// is indexed and stored as a tree, depending on how deep the remote references go. -package index - -import ( - "context" - "fmt" - "log/slog" - "os" - "path/filepath" - "sort" - "strings" - "sync" - - "github.com/pb33f/jsonpath/pkg/jsonpath" - jsonpathconfig "github.com/pb33f/jsonpath/pkg/jsonpath/config" - - "github.com/pb33f/libopenapi/utils" - - "go.yaml.in/yaml/v4" -) - -const ( - // theoreticalRoot is the name of the theoretical spec file used when a root spec file does not exist - theoreticalRoot = "root.yaml" -) - -func NewSpecIndexWithConfigAndContext(ctx context.Context, rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex { - index := new(SpecIndex) - boostrapIndexCollections(index) - index.InitHighCache() - index.config = config - index.rolodex = config.Rolodex - index.uri = config.uri - index.specAbsolutePath = config.SpecAbsolutePath - if config.Logger != nil { - index.logger = config.Logger - } else { - index.logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ - Level: slog.LevelError, - })) - } - if rootNode == nil || len(rootNode.Content) <= 0 { - return index - } - index.root = rootNode - return createNewIndex(ctx, rootNode, index, config.AvoidBuildIndex) -} - -// NewSpecIndexWithConfig will create a new index of an OpenAPI or Swagger spec. It uses the same logic as NewSpecIndex -// except it sets a base URL for resolving relative references, except it also allows for granular control over -// how the index is set up. -func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex { - return NewSpecIndexWithConfigAndContext(context.Background(), rootNode, config) -} - -// NewSpecIndex will create a new index of an OpenAPI or Swagger spec. It's not resolved or converted into anything -// other than a raw index of every node for every content type in the specification. This process runs as fast as -// possible so dependencies looking through the tree, don't need to walk the entire thing over, and over. -// -// This creates a new index using a default 'open' configuration. This means if a BaseURL or BasePath are supplied -// the rolodex will automatically read those files or open those h -func NewSpecIndex(rootNode *yaml.Node) *SpecIndex { - index := new(SpecIndex) - index.InitHighCache() - index.config = CreateOpenAPIIndexConfig() - index.root = rootNode - boostrapIndexCollections(index) - return createNewIndex(context.Background(), rootNode, index, false) -} - -func createNewIndex(ctx context.Context, rootNode *yaml.Node, index *SpecIndex, avoidBuildOut bool) *SpecIndex { - // there is no node! return an empty index. - if rootNode == nil { - return index - } - index.nodeMapCompleted = make(chan struct{}) - index.nodeMap = make(map[int]map[int]*yaml.Node) - go index.MapNodes(rootNode) // this can run async. - - index.cache = new(sync.Map) - - // boot index. - results := index.ExtractRefs(ctx, index.root.Content[0], index.root, []string{}, 0, false, "") - - // dedupe refs - dd := make(map[string]struct{}) - var dedupedResults []*Reference - for _, ref := range results { - if _, ok := dd[ref.FullDefinition]; !ok { - dd[ref.FullDefinition] = struct{}{} - dedupedResults = append(dedupedResults, ref) - } - } - - // map poly refs - sort keys for deterministic ordering - polyKeys := make([]string, 0, len(index.polymorphicRefs)) - for k := range index.polymorphicRefs { - polyKeys = append(polyKeys, k) - } - sort.Strings(polyKeys) - poly := make([]*Reference, len(index.polymorphicRefs)) - for i, k := range polyKeys { - poly[i] = index.polymorphicRefs[k] - } - - // pull out references - if len(dedupedResults) > 0 { - index.ExtractComponentsFromRefs(ctx, dedupedResults) - } - if len(poly) > 0 { - index.ExtractComponentsFromRefs(ctx, poly) - } - - index.ExtractExternalDocuments(index.root) - index.GetPathCount() - - // build out the index. - if !avoidBuildOut { - index.BuildIndex() - } - <-index.nodeMapCompleted - return index -} - -// BuildIndex will run all the count operations required to build up maps of everything. It's what makes the index -// useful for looking up things, the count operations are all run in parallel and then the final calculations are run -// the index is ready. -func (index *SpecIndex) BuildIndex() { - if index.built { - return - } - countFuncs := []func() int{ - index.GetOperationCount, - index.GetComponentSchemaCount, - index.GetGlobalTagsCount, - index.GetComponentParameterCount, - index.GetOperationsParameterCount, - } - - var wg sync.WaitGroup - wg.Add(len(countFuncs)) - runIndexFunction(countFuncs, &wg) // run as fast as we can. - wg.Wait() - - // these functions are aggregate and can only run once the rest of the datamodel is ready - countFuncs = []func() int{ - index.GetInlineUniqueParamCount, - index.GetOperationTagsCount, - index.GetGlobalLinksCount, - index.GetGlobalCallbacksCount, - } - wg.Add(len(countFuncs)) - runIndexFunction(countFuncs, &wg) // run as fast as we can. - wg.Wait() - - // these have final calculation dependencies - index.GetInlineDuplicateParamCount() - index.GetAllDescriptionsCount() - index.GetTotalTagsCount() - index.built = true -} - -func (index *SpecIndex) GetLogger() *slog.Logger { - return index.logger -} - -// GetRootNode returns document root node. -func (index *SpecIndex) GetRootNode() *yaml.Node { - return index.root -} - -// SetRootNode will override the root node with a supplied one. Be careful with this! -func (index *SpecIndex) SetRootNode(node *yaml.Node) { - index.root = node -} - -func (index *SpecIndex) GetRolodex() *Rolodex { - return index.rolodex -} - -func (index *SpecIndex) SetRolodex(rolodex *Rolodex) { - index.rolodex = rolodex -} - -// GetSpecFileName returns the root spec filename, if it exists, otherwise returns the theoretical root spec -func (index *SpecIndex) GetSpecFileName() string { - if index == nil || index.rolodex == nil || index.rolodex.indexConfig == nil || index.rolodex.indexConfig.SpecFilePath == "" { - return theoreticalRoot - } - return filepath.Base(index.rolodex.indexConfig.SpecFilePath) -} - -// GetGlobalTagsNode returns document root tags node. -func (index *SpecIndex) GetGlobalTagsNode() *yaml.Node { - return index.tagsNode -} - -// SetCircularReferences is a convenience method for the resolver to pass in circular references -// if the resolver is used. -func (index *SpecIndex) SetCircularReferences(refs []*CircularReferenceResult) { - index.circularReferences = refs -} - -// GetCircularReferences will return any circular reference results that were found by the resolver. -func (index *SpecIndex) GetCircularReferences() []*CircularReferenceResult { - return index.circularReferences -} - -// GetTagCircularReferences will return any circular reference results found in tag parent-child relationships. -// This is used for OpenAPI 3.2+ tag hierarchies where a tag can reference another tag as its parent. -func (index *SpecIndex) GetTagCircularReferences() []*CircularReferenceResult { - return index.tagCircularReferences -} - -// SetIgnoredPolymorphicCircularReferences passes on any ignored poly circular refs captured using -// `IgnorePolymorphicCircularReferences` -func (index *SpecIndex) SetIgnoredPolymorphicCircularReferences(refs []*CircularReferenceResult) { - index.polyCircularReferences = refs -} - -func (index *SpecIndex) SetIgnoredArrayCircularReferences(refs []*CircularReferenceResult) { - index.arrayCircularReferences = refs -} - -// GetIgnoredPolymorphicCircularReferences will return any polymorphic circular references that were 'ignored' by -// using the `IgnorePolymorphicCircularReferences` configuration option. -func (index *SpecIndex) GetIgnoredPolymorphicCircularReferences() []*CircularReferenceResult { - return index.polyCircularReferences -} - -// GetIgnoredArrayCircularReferences will return any array based circular references that were 'ignored' by -// using the `IgnoreArrayCircularReferences` configuration option. -func (index *SpecIndex) GetIgnoredArrayCircularReferences() []*CircularReferenceResult { - return index.arrayCircularReferences -} - -// GetPathsNode returns document root node. -func (index *SpecIndex) GetPathsNode() *yaml.Node { - return index.pathsNode -} - -// GetDiscoveredReferences will return all unique references found in the spec -func (index *SpecIndex) GetDiscoveredReferences() map[string]*Reference { - return index.allRefs -} - -// GetPolyReferences will return every polymorphic reference in the doc -func (index *SpecIndex) GetPolyReferences() map[string]*Reference { - return index.polymorphicRefs -} - -// GetPolyAllOfReferences will return every 'allOf' polymorphic reference in the doc -func (index *SpecIndex) GetPolyAllOfReferences() []*Reference { - return index.polymorphicAllOfRefs -} - -// GetPolyAnyOfReferences will return every 'anyOf' polymorphic reference in the doc -func (index *SpecIndex) GetPolyAnyOfReferences() []*Reference { - return index.polymorphicAnyOfRefs -} - -// GetPolyOneOfReferences will return every 'allOf' polymorphic reference in the doc -func (index *SpecIndex) GetPolyOneOfReferences() []*Reference { - return index.polymorphicOneOfRefs -} - -// GetAllCombinedReferences will return the number of unique and polymorphic references discovered. -func (index *SpecIndex) GetAllCombinedReferences() map[string]*Reference { - combined := make(map[string]*Reference) - for k, ref := range index.allRefs { - combined[k] = ref - } - for k, ref := range index.polymorphicRefs { - combined[k] = ref - } - return combined -} - -// GetRefsByLine will return all references and the lines at which they were found. -func (index *SpecIndex) GetRefsByLine() map[string]map[int]bool { - return index.refsByLine -} - -// GetLinesWithReferences will return a map of lines that have a $ref -func (index *SpecIndex) GetLinesWithReferences() map[int]bool { - return index.linesWithRefs -} - -// GetMappedReferences will return all references that were mapped successfully to actual property nodes. -// this collection is completely unsorted, traversing it may produce random results when resolving it and -// encountering circular references can change results depending on where in the collection the resolver started -// its journey through the index. -func (index *SpecIndex) GetMappedReferences() map[string]*Reference { - return index.allMappedRefs -} - -// SetMappedReferences will set the mapped references to the index. Not something you need every day unless you're -// doing some kind of index hacking. -func (index *SpecIndex) SetMappedReferences(mappedRefs map[string]*Reference) { - index.allMappedRefs = mappedRefs -} - -// GetRawReferencesSequenced returns a slice of every single reference found in the document, extracted raw from the doc -// returned in the exact order they were found in the document. -func (index *SpecIndex) GetRawReferencesSequenced() []*Reference { - return index.rawSequencedRefs -} - -// GetExtensionRefsSequenced returns all references that are under extension paths (x-* fields), -// in the order they were found in the document. -func (index *SpecIndex) GetExtensionRefsSequenced() []*Reference { - var extensionRefs []*Reference - for _, ref := range index.rawSequencedRefs { - if ref.IsExtensionRef { - extensionRefs = append(extensionRefs, ref) - } - } - return extensionRefs -} - -// GetMappedReferencesSequenced will return all references that were mapped successfully to nodes, performed in sequence -// as they were read in from the document. -func (index *SpecIndex) GetMappedReferencesSequenced() []*ReferenceMapped { - return index.allMappedRefsSequenced -} - -// GetOperationParameterReferences will return all references to operation parameters -func (index *SpecIndex) GetOperationParameterReferences() map[string]map[string]map[string][]*Reference { - return index.paramOpRefs -} - -// GetAllSchemas returns references to ALL schemas found in the document: -// - Inline schemas (defined directly in operations, parameters, etc.) -// - Component schemas (defined under components/schemas or definitions) -// - Reference schemas ($ref pointers to schemas) -// -// Results are sorted by line number in the source document. -// -// Note: This is the only GetAll* function that returns inline and $ref variants. -// Other GetAll* functions (GetAllRequestBodies, GetAllResponses, etc.) only return -// items defined in the components section. Use GetAllInlineSchemas, GetAllComponentSchemas, -// and GetAllReferenceSchemas for more granular access. -func (index *SpecIndex) GetAllSchemas() []*Reference { - componentSchemas := index.GetAllComponentSchemas() - inlineSchemas := index.GetAllInlineSchemas() - refSchemas := index.GetAllReferenceSchemas() - combined := make([]*Reference, len(inlineSchemas)+len(componentSchemas)+len(refSchemas)) - i := 0 - for x := range inlineSchemas { - combined[i] = inlineSchemas[x] - i++ - } - for x := range componentSchemas { - combined[i] = componentSchemas[x] - i++ - } - for x := range refSchemas { - combined[i] = refSchemas[x] - i++ - } - sort.Slice(combined, func(i, j int) bool { - return combined[i].Node.Line < combined[j].Node.Line - }) - return combined -} - -// GetAllInlineSchemaObjects will return all schemas that are inline (not inside components) and that are also typed -// as 'object' or 'array' (not primitives). -func (index *SpecIndex) GetAllInlineSchemaObjects() []*Reference { - return index.allInlineSchemaObjectDefinitions -} - -// GetAllInlineSchemas will return all schemas defined in the components section of the document. -func (index *SpecIndex) GetAllInlineSchemas() []*Reference { - return index.allInlineSchemaDefinitions -} - -// GetAllReferenceSchemas will return all schemas that are not inline, but $ref'd from somewhere. -func (index *SpecIndex) GetAllReferenceSchemas() []*Reference { - return index.allRefSchemaDefinitions -} - -// GetAllComponentSchemas will return all schemas defined in the components section of the document. -func (index *SpecIndex) GetAllComponentSchemas() map[string]*Reference { - if index == nil { - return nil - } - - // Acquire read lock - index.allComponentSchemasLock.RLock() - if index.allComponentSchemas != nil { - defer index.allComponentSchemasLock.RUnlock() - return index.allComponentSchemas - } - // Release the read lock before acquiring write lock - index.allComponentSchemasLock.RUnlock() - - // Acquire write lock to initialize the map - index.allComponentSchemasLock.Lock() - defer index.allComponentSchemasLock.Unlock() - - // Double-check if another goroutine initialized it - if index.allComponentSchemas == nil { - schemaMap := syncMapToMap[string, *Reference](index.allComponentSchemaDefinitions) - index.allComponentSchemas = schemaMap - } - - return index.allComponentSchemas -} - -// GetAllSecuritySchemes returns all security schemes defined in the components section -// (components/securitySchemes in OpenAPI 3.x, or securityDefinitions in Swagger 2.0). -func (index *SpecIndex) GetAllSecuritySchemes() map[string]*Reference { - return syncMapToMap[string, *Reference](index.allSecuritySchemes) -} - -// GetAllHeaders returns all headers defined in the components section (components/headers). -// This does not include inline headers defined directly in operations or $ref pointers. -func (index *SpecIndex) GetAllHeaders() map[string]*Reference { - return index.allHeaders -} - -// GetAllExternalDocuments will return all external documents found -func (index *SpecIndex) GetAllExternalDocuments() map[string]*Reference { - return index.allExternalDocuments -} - -// GetAllExamples returns all examples defined in the components section (components/examples). -// This does not include inline examples defined directly in operations or $ref pointers. -func (index *SpecIndex) GetAllExamples() map[string]*Reference { - return index.allExamples -} - -// GetAllDescriptions will return all descriptions found in the document -func (index *SpecIndex) GetAllDescriptions() []*DescriptionReference { - return index.allDescriptions -} - -// GetAllEnums will return all enums found in the document -func (index *SpecIndex) GetAllEnums() []*EnumReference { - return index.allEnums -} - -// GetAllObjectsWithProperties will return all objects with properties found in the document -func (index *SpecIndex) GetAllObjectsWithProperties() []*ObjectReference { - return index.allObjectsWithProperties -} - -// GetAllSummaries will return all summaries found in the document -func (index *SpecIndex) GetAllSummaries() []*DescriptionReference { - return index.allSummaries -} - -// GetAllRequestBodies returns all request bodies defined in the components section (components/requestBodies). -// This does not include inline request bodies defined directly in operations or $ref pointers. -func (index *SpecIndex) GetAllRequestBodies() map[string]*Reference { - return index.allRequestBodies -} - -// GetAllLinks returns all links defined in the components section (components/links). -// This does not include inline links defined directly in responses or $ref pointers. -func (index *SpecIndex) GetAllLinks() map[string]*Reference { - return index.allLinks -} - -// GetAllParameters returns all parameters defined in the components section (components/parameters). -// This does not include inline parameters defined directly in operations or path items. -// For operation-level parameters, use GetOperationParameterReferences. -func (index *SpecIndex) GetAllParameters() map[string]*Reference { - return index.allParameters -} - -// GetAllResponses returns all responses defined in the components section (components/responses). -// This does not include inline responses defined directly in operations or $ref pointers. -func (index *SpecIndex) GetAllResponses() map[string]*Reference { - return index.allResponses -} - -// GetAllCallbacks returns all callbacks defined in the components section (components/callbacks). -// This does not include inline callbacks defined directly in operations or $ref pointers. -func (index *SpecIndex) GetAllCallbacks() map[string]*Reference { - return index.allCallbacks -} - -// GetAllComponentPathItems returns all path items defined in the components section (components/pathItems). -// This does not include path items defined directly under the paths object or $ref pointers. -// For paths-level path items, use GetAllPaths. -func (index *SpecIndex) GetAllComponentPathItems() map[string]*Reference { - return index.allComponentPathItems -} - -// GetInlineOperationDuplicateParameters will return a map of duplicates located in operation parameters. -func (index *SpecIndex) GetInlineOperationDuplicateParameters() map[string][]*Reference { - return index.paramInlineDuplicateNames -} - -// GetReferencesWithSiblings will return a map of all the references with sibling nodes (illegal) -func (index *SpecIndex) GetReferencesWithSiblings() map[string]Reference { - return index.refsWithSiblings -} - -// GetAllReferences will return every reference found in the spec, after being de-duplicated. -func (index *SpecIndex) GetAllReferences() map[string]*Reference { - return index.allRefs -} - -// GetAllSequencedReferences will return every reference (in sequence) that was found (non-polymorphic) -func (index *SpecIndex) GetAllSequencedReferences() []*Reference { - return index.rawSequencedRefs -} - -// GetSchemasNode will return the schema's node found in the spec -func (index *SpecIndex) GetSchemasNode() *yaml.Node { - return index.schemasNode -} - -// GetParametersNode will return the schema's node found in the spec -func (index *SpecIndex) GetParametersNode() *yaml.Node { - return index.parametersNode -} - -// GetReferenceIndexErrors will return any errors that occurred when indexing references -func (index *SpecIndex) GetReferenceIndexErrors() []error { - return index.refErrors -} - -// GetOperationParametersIndexErrors any errors that occurred when indexing operation parameters -func (index *SpecIndex) GetOperationParametersIndexErrors() []error { - return index.operationParamErrors -} - -// GetAllPaths will return all paths indexed in the document -func (index *SpecIndex) GetAllPaths() map[string]map[string]*Reference { - return index.pathRefs -} - -// GetOperationTags will return all references to all tags found in operations. -func (index *SpecIndex) GetOperationTags() map[string]map[string][]*Reference { - return index.operationTagsRefs -} - -// GetAllParametersFromOperations will return all paths indexed in the document -func (index *SpecIndex) GetAllParametersFromOperations() map[string]map[string]map[string][]*Reference { - return index.paramOpRefs -} - -// GetRootSecurityReferences will return all root security settings -func (index *SpecIndex) GetRootSecurityReferences() []*Reference { - return index.rootSecurity -} - -// GetSecurityRequirementReferences will return all security requirement definitions found in the document -func (index *SpecIndex) GetSecurityRequirementReferences() map[string]map[string][]*Reference { - return index.securityRequirementRefs -} - -// GetRootSecurityNode will return the root security node -func (index *SpecIndex) GetRootSecurityNode() *yaml.Node { - return index.rootSecurityNode -} - -// GetRootServersNode will return the root servers node -func (index *SpecIndex) GetRootServersNode() *yaml.Node { - return index.rootServersNode -} - -// GetAllRootServers will return all root servers defined -func (index *SpecIndex) GetAllRootServers() []*Reference { - return index.serversRefs -} - -// GetAllOperationsServers will return all operation overrides for servers. -func (index *SpecIndex) GetAllOperationsServers() map[string]map[string][]*Reference { - return index.opServersRefs -} - -// SetAllowCircularReferenceResolving will flip a bit that can be used by any consumers to determine if they want -// to allow or disallow circular references to be resolved or visited -func (index *SpecIndex) SetAllowCircularReferenceResolving(allow bool) { - index.allowCircularReferences = allow -} - -// AllowCircularReferenceResolving will return a bit that allows developers to determine what to do with circular refs. -func (index *SpecIndex) AllowCircularReferenceResolving() bool { - return index.allowCircularReferences -} - -func (index *SpecIndex) checkPolymorphicNode(name string) (bool, string) { - switch name { - case "anyOf": - return true, "anyOf" - case "allOf": - return true, "allOf" - case "oneOf": - return true, "oneOf" - } - return false, "" -} - -// GetPathCount will return the number of paths found in the spec -func (index *SpecIndex) GetPathCount() int { - if index.root == nil { - return -1 - } - - if index.pathCount > 0 { - return index.pathCount - } - pc := 0 - for i, n := range index.root.Content[0].Content { - if i%2 == 0 { - if n.Value == "paths" { - pn := index.root.Content[0].Content[i+1].Content - index.pathsNode = index.root.Content[0].Content[i+1] - pc = len(pn) / 2 - } - } - } - index.pathCount = pc - return pc -} - -// ExtractExternalDocuments will extract the number of externalDocs nodes found in the document. -func (index *SpecIndex) ExtractExternalDocuments(node *yaml.Node) []*Reference { - if node == nil { - return nil - } - var found []*Reference - if len(node.Content) > 0 { - for i, n := range node.Content { - if utils.IsNodeMap(n) || utils.IsNodeArray(n) { - found = append(found, index.ExtractExternalDocuments(n)...) - } - - if i%2 == 0 && n.Value == "externalDocs" { - docNode := node.Content[i+1] - _, urlNode := utils.FindKeyNode("url", docNode.Content) - if urlNode != nil { - ref := &Reference{ - Definition: urlNode.Value, - Name: urlNode.Value, - Node: docNode, - } - index.externalDocumentsRef = append(index.externalDocumentsRef, ref) - } - } - } - } - index.externalDocumentsCount = len(index.externalDocumentsRef) - return found -} - -// GetGlobalTagsCount will return the number of tags found in the top level 'tags' node of the document. -func (index *SpecIndex) GetGlobalTagsCount() int { - if index.root == nil { - return -1 - } - - if index.globalTagsCount > 0 { - return index.globalTagsCount - } - - for i, n := range index.root.Content[0].Content { - if i%2 == 0 { - if n.Value == "tags" { - tagsNode := index.root.Content[0].Content[i+1] - if tagsNode != nil { - index.tagsNode = tagsNode - index.globalTagsCount = len(tagsNode.Content) // tags is an array, don't divide by 2. - for x, tagNode := range index.tagsNode.Content { - - _, name := utils.FindKeyNode("name", tagNode.Content) - _, description := utils.FindKeyNode("description", tagNode.Content) - - var desc string - if description == nil { - desc = "" - } - if name != nil { - ref := &Reference{ - Definition: desc, - Name: name.Value, - Node: tagNode, - Path: fmt.Sprintf("$.tags[%d]", x), - } - index.globalTagRefs[name.Value] = ref - } - } - - // Check for tag circular references (OpenAPI 3.2+) - index.checkTagCircularReferences() - } - } - } - } - return index.globalTagsCount -} - -// checkTagCircularReferences performs circular reference detection for OpenAPI 3.2+ tag parent-child relationships. -// It builds a parent-child map and then uses depth-first search to detect cycles. -func (index *SpecIndex) checkTagCircularReferences() { - if index.tagsNode == nil { - return - } - - // Build parent-child mapping from tag nodes - tagParentMap := make(map[string]string) // tagName -> parentName - tagRefs := make(map[string]*Reference) // tagName -> Reference - tagNodes := make(map[string]*yaml.Node) // tagName -> yaml.Node - - for x, tagNode := range index.tagsNode.Content { - _, nameNode := utils.FindKeyNode("name", tagNode.Content) - _, parentNode := utils.FindKeyNode("parent", tagNode.Content) - - if nameNode != nil { - tagName := nameNode.Value - tagNodes[tagName] = tagNode - tagRefs[tagName] = &Reference{ - Name: tagName, - Node: tagNode, - Path: fmt.Sprintf("$.tags[%d]", x), - } - - if parentNode != nil { - parentName := parentNode.Value - tagParentMap[tagName] = parentName - } - } - } - - // Perform circular reference detection using depth-first search - visited := make(map[string]bool) - recStack := make(map[string]bool) // recursion stack to detect cycles - - for tagName := range tagRefs { - if !visited[tagName] { - // Only check tags that have parents - no point checking orphans - if _, hasParent := tagParentMap[tagName]; hasParent { - if path := index.detectTagCircularHelper(tagName, tagParentMap, tagRefs, visited, recStack, []string{}); len(path) > 0 { - // Circular reference detected, create CircularReferenceResult - journey := make([]*Reference, len(path)) - for i, name := range path { - journey[i] = tagRefs[name] - } - - loopIndex := -1 - loopStart := path[len(path)-1] // The repeated tag name - for i, name := range path { - if name == loopStart { - loopIndex = i - break - } - } - - circRef := &CircularReferenceResult{ - Journey: journey, - Start: tagRefs[path[0]], - LoopIndex: loopIndex, - LoopPoint: tagRefs[loopStart], - ParentNode: tagNodes[loopStart], - IsArrayResult: false, - IsPolymorphicResult: false, - IsInfiniteLoop: true, // Tag parent cycles are always problematic - } - - index.tagCircularReferences = append(index.tagCircularReferences, circRef) - } - } - } - } -} - -// detectTagCircularHelper is a recursive helper function for detecting circular references in tag hierarchies. -// Returns the path to the circular reference if found, empty slice otherwise. -func (index *SpecIndex) detectTagCircularHelper(tagName string, parentMap map[string]string, tagRefs map[string]*Reference, visited map[string]bool, recStack map[string]bool, path []string) []string { - // Check if this tag even exists - if not, we can't have a circular reference - if _, exists := tagRefs[tagName]; !exists { - return []string{} - } - - visited[tagName] = true - recStack[tagName] = true - path = append(path, tagName) - - // Check if this tag has a parent - if parentName, hasParent := parentMap[tagName]; hasParent { - // Validate that parent exists as a defined tag - if _, parentExists := tagRefs[parentName]; !parentExists { - // Parent doesn't exist - this is a validation error but not a circular reference - // Remove from recursion stack before returning - recStack[tagName] = false - return []string{} - } - - // If parent is already in recursion stack, we found a cycle - if recStack[parentName] { - return append(path, parentName) // Return path including the cycle - } - - // If parent not visited, recursively check it - if !visited[parentName] { - if cyclePath := index.detectTagCircularHelper(parentName, parentMap, tagRefs, visited, recStack, path); len(cyclePath) > 0 { - return cyclePath - } - } - } - - // Remove from recursion stack when backtracking - recStack[tagName] = false - return []string{} -} - -// GetOperationTagsCount will return the number of operation tags found (tags referenced in operations) -func (index *SpecIndex) GetOperationTagsCount() int { - if index.root == nil { - return -1 - } - - if index.operationTagsCount > 0 { - return index.operationTagsCount - } - - // this is an aggregate count function that can only be run after operations - // have been calculated. - seen := make(map[string]bool) - count := 0 - for _, path := range index.operationTagsRefs { - for _, method := range path { - for _, tag := range method { - if !seen[tag.Name] { - seen[tag.Name] = true - count++ - } - } - } - } - index.operationTagsCount = count - return index.operationTagsCount -} - -// GetTotalTagsCount will return the number of global and operation tags found that are unique. -func (index *SpecIndex) GetTotalTagsCount() int { - if index.root == nil { - return -1 - } - if index.totalTagsCount > 0 { - return index.totalTagsCount - } - - seen := make(map[string]bool) - count := 0 - - for _, gt := range index.globalTagRefs { - // TODO: do we still need this? - if !seen[gt.Name] { - seen[gt.Name] = true - count++ - } - } - for _, ot := range index.operationTagsRefs { - for _, m := range ot { - for _, t := range m { - if !seen[t.Name] { - seen[t.Name] = true - count++ - } - } - } - } - index.totalTagsCount = count - return index.totalTagsCount -} - -// GetGlobalCallbacksCount for each response of each operation method, multiple callbacks can be defined -func (index *SpecIndex) GetGlobalCallbacksCount() int { - if index.root == nil { - return -1 - } - - if index.globalCallbacksCount > 0 { - return index.globalCallbacksCount - } - - index.pathRefsLock.RLock() - for path, p := range index.pathRefs { - for _, m := range p { - - // look through method for callbacks - callbacks, _ := jsonpath.NewPath("$..callbacks", jsonpathconfig.WithPropertyNameExtension(), jsonpathconfig.WithLazyContextTracking()) - var res []*yaml.Node - res = callbacks.Query(m.Node) - if len(res) > 0 { - for _, callback := range res[0].Content { - if utils.IsNodeMap(callback) { - - ref := &Reference{ - Definition: m.Name, - Name: m.Name, - Node: callback, - } - - if index.callbacksRefs[path] == nil { - index.callbacksRefs[path] = make(map[string][]*Reference) - } - if len(index.callbacksRefs[path][m.Name]) > 0 { - index.callbacksRefs[path][m.Name] = append(index.callbacksRefs[path][m.Name], ref) - } else { - index.callbacksRefs[path][m.Name] = []*Reference{ref} - } - index.globalCallbacksCount++ - } - } - } - } - } - index.pathRefsLock.RUnlock() - return index.globalCallbacksCount -} - -// GetGlobalLinksCount for each response of each operation method, multiple callbacks can be defined -func (index *SpecIndex) GetGlobalLinksCount() int { - if index.root == nil { - return -1 - } - - if index.globalLinksCount > 0 { - return index.globalLinksCount - } - - // index.pathRefsLock.Lock() - for path, p := range index.pathRefs { - for _, m := range p { - - // look through method for links - links, _ := jsonpath.NewPath("$..links", jsonpathconfig.WithPropertyNameExtension(), jsonpathconfig.WithLazyContextTracking()) - var res []*yaml.Node - - res = links.Query(m.Node) - - if len(res) > 0 { - for _, link := range res[0].Content { - if utils.IsNodeMap(link) { - - ref := &Reference{ - Definition: m.Name, - Name: m.Name, - Node: link, - } - if index.linksRefs[path] == nil { - index.linksRefs[path] = make(map[string][]*Reference) - } - if len(index.linksRefs[path][m.Name]) > 0 { - index.linksRefs[path][m.Name] = append(index.linksRefs[path][m.Name], ref) - } - index.linksRefs[path][m.Name] = []*Reference{ref} - index.globalLinksCount++ - } - } - } - } - } - // index.pathRefsLock.Unlock() - return index.globalLinksCount -} - -// GetRawReferenceCount will return the number of raw references located in the document. -func (index *SpecIndex) GetRawReferenceCount() int { - return len(index.rawSequencedRefs) -} - -// GetComponentSchemaCount will return the number of schemas located in the 'components' or 'definitions' node. -func (index *SpecIndex) GetComponentSchemaCount() int { - if index.root == nil || len(index.root.Content) == 0 { - return -1 - } - - if index.schemaCount > 0 { - return index.schemaCount - } - - for i, n := range index.root.Content[0].Content { - if i%2 == 0 { - - // servers - if n.Value == "servers" { - index.rootServersNode = index.root.Content[0].Content[i+1] - if i+1 < len(index.root.Content[0].Content) { - serverDefinitions := index.root.Content[0].Content[i+1] - for x, def := range serverDefinitions.Content { - ref := &Reference{ - Definition: "servers", - Name: "server", - Node: def, - Path: fmt.Sprintf("$.servers[%d]", x), - ParentNode: index.rootServersNode, - } - index.serversRefs = append(index.serversRefs, ref) - } - } - } - - // root security definitions - if n.Value == "security" { - index.rootSecurityNode = index.root.Content[0].Content[i+1] - if i+1 < len(index.root.Content[0].Content) { - securityDefinitions := index.root.Content[0].Content[i+1] - for x, def := range securityDefinitions.Content { - if len(def.Content) > 0 { - name := def.Content[0] - ref := &Reference{ - Definition: name.Value, - Name: name.Value, - Node: def, - Path: fmt.Sprintf("$.security[%d]", x), - } - index.rootSecurity = append(index.rootSecurity, ref) - } - } - } - } - - if n.Value == "components" { - _, schemasNode := utils.FindKeyNode("schemas", index.root.Content[0].Content[i+1].Content) - - // while we are here, go ahead and extract everything in components. - _, parametersNode := utils.FindKeyNode("parameters", index.root.Content[0].Content[i+1].Content) - _, requestBodiesNode := utils.FindKeyNode("requestBodies", index.root.Content[0].Content[i+1].Content) - _, responsesNode := utils.FindKeyNode("responses", index.root.Content[0].Content[i+1].Content) - _, securitySchemesNode := utils.FindKeyNode("securitySchemes", index.root.Content[0].Content[i+1].Content) - _, headersNode := utils.FindKeyNode("headers", index.root.Content[0].Content[i+1].Content) - _, examplesNode := utils.FindKeyNode("examples", index.root.Content[0].Content[i+1].Content) - _, linksNode := utils.FindKeyNode("links", index.root.Content[0].Content[i+1].Content) - _, callbacksNode := utils.FindKeyNode("callbacks", index.root.Content[0].Content[i+1].Content) - _, pathItemsNode := utils.FindKeyNode("pathItems", index.root.Content[0].Content[i+1].Content) - - // extract schemas - if schemasNode != nil { - index.extractDefinitionsAndSchemas(schemasNode, "#/components/schemas/") - index.schemasNode = schemasNode - index.schemaCount = len(schemasNode.Content) / 2 - } - - // extract parameters - if parametersNode != nil { - index.extractComponentParameters(parametersNode, "#/components/parameters/") - index.componentLock.Lock() - index.parametersNode = parametersNode - index.componentLock.Unlock() - } - - // extract requestBodies - if requestBodiesNode != nil { - index.extractComponentRequestBodies(requestBodiesNode, "#/components/requestBodies/") - index.requestBodiesNode = requestBodiesNode - } - - // extract responses - if responsesNode != nil { - index.extractComponentResponses(responsesNode, "#/components/responses/") - index.responsesNode = responsesNode - } - - // extract security schemes - if securitySchemesNode != nil { - index.extractComponentSecuritySchemes(securitySchemesNode, "#/components/securitySchemes/") - index.securitySchemesNode = securitySchemesNode - } - - // extract headers - if headersNode != nil { - index.extractComponentHeaders(headersNode, "#/components/headers/") - index.headersNode = headersNode - } - - // extract examples - if examplesNode != nil { - index.extractComponentExamples(examplesNode, "#/components/examples/") - index.examplesNode = examplesNode - } - - // extract links - if linksNode != nil { - index.extractComponentLinks(linksNode, "#/components/links/") - index.linksNode = linksNode - } - - // extract callbacks - if callbacksNode != nil { - index.extractComponentCallbacks(callbacksNode, "#/components/callbacks/") - index.callbacksNode = callbacksNode - } - - // extract pathItems - if pathItemsNode != nil { - index.extractComponentPathItems(pathItemsNode, "#/components/pathItems/") - index.pathItemsNode = pathItemsNode - } - - } - - // swagger - if n.Value == "definitions" { - schemasNode := index.root.Content[0].Content[i+1] - if schemasNode != nil { - - // extract schemas - index.extractDefinitionsAndSchemas(schemasNode, "#/definitions/") - index.schemasNode = schemasNode - index.schemaCount = len(schemasNode.Content) / 2 - } - } - - // swagger - if n.Value == "parameters" { - parametersNode := index.root.Content[0].Content[i+1] - if parametersNode != nil { - // extract params - index.extractComponentParameters(parametersNode, "#/parameters/") - index.componentLock.Lock() - index.parametersNode = parametersNode - index.componentLock.Unlock() - } - } - - if n.Value == "responses" { - responsesNode := index.root.Content[0].Content[i+1] - if responsesNode != nil { - - // extract responses - index.extractComponentResponses(responsesNode, "#/responses/") - index.responsesNode = responsesNode - } - } - - if n.Value == "securityDefinitions" { - securityDefinitionsNode := index.root.Content[0].Content[i+1] - if securityDefinitionsNode != nil { - - // extract security definitions. - index.extractComponentSecuritySchemes(securityDefinitionsNode, "#/securityDefinitions/") - index.securitySchemesNode = securityDefinitionsNode - } - } - - } - } - return index.schemaCount -} - -// GetComponentParameterCount returns the number of parameter components defined -func (index *SpecIndex) GetComponentParameterCount() int { - if index.root == nil { - return -1 - } - - if index.componentParamCount > 0 { - return index.componentParamCount - } - - for i, n := range index.root.Content[0].Content { - if i%2 == 0 { - // openapi 3 - if n.Value == "components" { - _, parametersNode := utils.FindKeyNode("parameters", index.root.Content[0].Content[i+1].Content) - if parametersNode != nil { - index.componentLock.Lock() - index.parametersNode = parametersNode - index.componentParamCount = len(parametersNode.Content) / 2 - index.componentLock.Unlock() - } - } - // openapi 2 - if n.Value == "parameters" { - parametersNode := index.root.Content[0].Content[i+1] - if parametersNode != nil { - index.componentLock.Lock() - index.parametersNode = parametersNode - index.componentParamCount = len(parametersNode.Content) / 2 - index.componentLock.Unlock() - } - } - } - } - return index.componentParamCount -} - -// GetOperationCount returns the number of operations (for all paths) located in the document -func (index *SpecIndex) GetOperationCount() int { - if index.root == nil { - return -1 - } - - if index.pathsNode == nil { - return -1 - } - - if index.operationCount > 0 { - return index.operationCount - } - - opCount := 0 - - locatedPathRefs := make(map[string]map[string]*Reference) - - for x, p := range index.pathsNode.Content { - if x%2 == 0 { - - var method *yaml.Node - if utils.IsNodeArray(index.pathsNode) { - method = index.pathsNode.Content[x] - } else { - method = index.pathsNode.Content[x+1] - } - - // is the path a ref? - if isRef, _, ref := utils.IsNodeRefValue(method); isRef { - ctx := context.WithValue(context.Background(), CurrentPathKey, index.specAbsolutePath) - ctx = context.WithValue(ctx, RootIndexKey, index) - pNode := seekRefEnd(ctx, index, ref) - if pNode != nil { - method = pNode.Node - } - } - - // extract methods for later use. - for y, m := range method.Content { - if y%2 == 0 { - - // check node is a valid method - valid := false - for _, methodType := range methodTypes { - if m.Value == methodType { - valid = true - } - } - if valid { - ref := &Reference{ - Definition: m.Value, - Name: m.Value, - Node: method.Content[y+1], - Path: fmt.Sprintf("$.paths['%s'].%s", p.Value, m.Value), - ParentNode: m, - } - if locatedPathRefs[p.Value] == nil { - locatedPathRefs[p.Value] = make(map[string]*Reference) - } - locatedPathRefs[p.Value][ref.Name] = ref - // update - opCount++ - } - } - } - } - } - for k, v := range locatedPathRefs { - index.pathRefs[k] = v - } - index.operationCount = opCount - return opCount -} - -// GetOperationsParameterCount returns the number of parameters defined in paths and operations. -// this method looks in top level (path level) and inside each operation (get, post etc.). Parameters can -// be hiding within multiple places. -func (index *SpecIndex) GetOperationsParameterCount() int { - if index.root == nil { - return -1 - } - - if index.pathsNode == nil { - return -1 - } - - if index.operationParamCount > 0 { - return index.operationParamCount - } - - // parameters are sneaky, they can be in paths, in path operations or in components. - // sometimes they are refs, sometimes they are inline definitions, just for fun. - // some authors just LOVE to mix and match them all up. - // check paths first - for x, pathItemNode := range index.pathsNode.Content { - if x%2 == 0 { - - var pathPropertyNode *yaml.Node - if utils.IsNodeArray(index.pathsNode) { - pathPropertyNode = index.pathsNode.Content[x] - } else { - pathPropertyNode = index.pathsNode.Content[x+1] - } - - // is the path a ref? - if isRef, _, ref := utils.IsNodeRefValue(pathPropertyNode); isRef { - ctx := context.WithValue(context.Background(), CurrentPathKey, index.specAbsolutePath) - ctx = context.WithValue(ctx, RootIndexKey, index) - pNode := seekRefEnd(ctx, index, ref) - if pNode != nil { - pathPropertyNode = pNode.Node - } - } - - // extract methods for later use. - for y, prop := range pathPropertyNode.Content { - if y%2 == 0 { - - // while we're here, lets extract any top level servers - if prop.Value == "servers" { - serversNode := pathPropertyNode.Content[y+1] - if index.opServersRefs[pathItemNode.Value] == nil { - index.opServersRefs[pathItemNode.Value] = make(map[string][]*Reference) - } - var serverRefs []*Reference - for i, serverRef := range serversNode.Content { - ref := &Reference{ - Definition: serverRef.Value, - Name: serverRef.Value, - Node: serverRef, - ParentNode: prop, - Path: fmt.Sprintf("$.paths['%s'].servers[%d]", pathItemNode.Value, i), - } - serverRefs = append(serverRefs, ref) - } - index.opServersRefs[pathItemNode.Value]["top"] = serverRefs - } - - // top level params - if prop.Value == "parameters" { - - // let's look at params, check if they are refs or inline. - params := pathPropertyNode.Content[y+1].Content - index.scanOperationParams(params, pathPropertyNode.Content[y], pathItemNode, "top") - } - - // method level params. - if isHttpMethod(prop.Value) { - for z, httpMethodProp := range pathPropertyNode.Content[y+1].Content { - if z%2 == 0 { - if httpMethodProp.Value == "parameters" { - params := pathPropertyNode.Content[y+1].Content[z+1].Content - index.scanOperationParams(params, pathPropertyNode.Content[y+1].Content[z], pathItemNode, prop.Value) - } - - // extract operation tags if set. - if httpMethodProp.Value == "tags" { - tags := pathPropertyNode.Content[y+1].Content[z+1] - - if index.operationTagsRefs[pathItemNode.Value] == nil { - index.operationTagsRefs[pathItemNode.Value] = make(map[string][]*Reference) - } - - var tagRefs []*Reference - for _, tagRef := range tags.Content { - ref := &Reference{ - Definition: tagRef.Value, - Name: tagRef.Value, - Node: tagRef, - } - tagRefs = append(tagRefs, ref) - } - index.operationTagsRefs[pathItemNode.Value][prop.Value] = tagRefs - } - - // extract description and summaries - if httpMethodProp.Value == "description" { - desc := pathPropertyNode.Content[y+1].Content[z+1].Value - ref := &Reference{ - Definition: desc, - Name: "description", - Node: pathPropertyNode.Content[y+1].Content[z+1], - } - if index.operationDescriptionRefs[pathItemNode.Value] == nil { - index.operationDescriptionRefs[pathItemNode.Value] = make(map[string]*Reference) - } - - index.operationDescriptionRefs[pathItemNode.Value][prop.Value] = ref - } - if httpMethodProp.Value == "summary" { - summary := pathPropertyNode.Content[y+1].Content[z+1].Value - ref := &Reference{ - Definition: summary, - Name: "summary", - Node: pathPropertyNode.Content[y+1].Content[z+1], - } - - if index.operationSummaryRefs[pathItemNode.Value] == nil { - index.operationSummaryRefs[pathItemNode.Value] = make(map[string]*Reference) - } - - index.operationSummaryRefs[pathItemNode.Value][prop.Value] = ref - } - - // extract servers from method operation. - if httpMethodProp.Value == "servers" { - serversNode := pathPropertyNode.Content[y+1].Content[z+1] - - var serverRefs []*Reference - for i, serverRef := range serversNode.Content { - ref := &Reference{ - Definition: "servers", - Name: "servers", - Node: serverRef, - ParentNode: httpMethodProp, - Path: fmt.Sprintf("$.paths['%s'].%s.servers[%d]", pathItemNode.Value, prop.Value, i), - } - serverRefs = append(serverRefs, ref) - } - - if index.opServersRefs[pathItemNode.Value] == nil { - index.opServersRefs[pathItemNode.Value] = make(map[string][]*Reference) - } - - index.opServersRefs[pathItemNode.Value][prop.Value] = serverRefs - } - - } - } - } - } - } - } - } - - // Now that all the paths and operations are processed, lets pick out everything from our pre - // mapped refs and populate our ready to roll index of component params. - for key, component := range index.allMappedRefs { - if strings.Contains(key, "/parameters/") { - index.paramCompRefs[key] = component - index.paramAllRefs[key] = component - } - } - - // now build main index of all params by combining comp refs with inline params from operations. - // use the namespace path:::param for inline params to identify them as inline. - for path, params := range index.paramOpRefs { - for mName, mValue := range params { - for pName, pValue := range mValue { - if !strings.HasPrefix(pName, "#") { - index.paramInlineDuplicateNames[pName] = append(index.paramInlineDuplicateNames[pName], pValue...) - for i := range pValue { - if pValue[i] != nil { - _, in := utils.FindKeyNodeTop("in", pValue[i].Node.Content) - if in != nil { - index.paramAllRefs[fmt.Sprintf("%s:::%s:::%s", path, mName, in.Value)] = pValue[i] - } else { - index.paramAllRefs[fmt.Sprintf("%s:::%s", path, mName)] = pValue[i] - } - } - } - } - } - } - } - - index.operationParamCount = len(index.paramCompRefs) + len(index.paramInlineDuplicateNames) - return index.operationParamCount -} - -// GetInlineDuplicateParamCount returns the number of inline duplicate parameters (operation params) -func (index *SpecIndex) GetInlineDuplicateParamCount() int { - if index.componentsInlineParamDuplicateCount > 0 { - return index.componentsInlineParamDuplicateCount - } - dCount := len(index.paramInlineDuplicateNames) - index.countUniqueInlineDuplicates() - index.componentsInlineParamDuplicateCount = dCount - return dCount -} - -// GetInlineUniqueParamCount returns the number of unique inline parameters (operation params) -func (index *SpecIndex) GetInlineUniqueParamCount() int { - return index.countUniqueInlineDuplicates() -} - -// GetAllDescriptionsCount will collect together every single description found in the document -func (index *SpecIndex) GetAllDescriptionsCount() int { - return len(index.allDescriptions) -} - -// GetAllSummariesCount will collect together every single summary found in the document -func (index *SpecIndex) GetAllSummariesCount() int { - return len(index.allSummaries) -} - -// RegisterSchemaId registers a schema by its $id in this index. -// Returns an error if the $id is invalid (e.g., contains a fragment). -func (index *SpecIndex) RegisterSchemaId(entry *SchemaIdEntry) error { - index.schemaIdRegistryLock.Lock() - defer index.schemaIdRegistryLock.Unlock() - - if index.schemaIdRegistry == nil { - index.schemaIdRegistry = make(map[string]*SchemaIdEntry) - } - - _, err := registerSchemaIdToRegistry(index.schemaIdRegistry, entry, index.logger, "local index") - return err -} - -// GetSchemaById looks up a schema by its $id URI. -func (index *SpecIndex) GetSchemaById(uri string) *SchemaIdEntry { - index.schemaIdRegistryLock.RLock() - defer index.schemaIdRegistryLock.RUnlock() - - if index.schemaIdRegistry == nil { - return nil - } - return index.schemaIdRegistry[uri] -} - -// GetAllSchemaIds returns a copy of all registered $id entries in this index. -func (index *SpecIndex) GetAllSchemaIds() map[string]*SchemaIdEntry { - index.schemaIdRegistryLock.RLock() - defer index.schemaIdRegistryLock.RUnlock() - return copySchemaIdRegistry(index.schemaIdRegistry) -} diff --git a/index/spec_index_accessors.go b/index/spec_index_accessors.go new file mode 100644 index 000000000..c3cad9085 --- /dev/null +++ b/index/spec_index_accessors.go @@ -0,0 +1,335 @@ +// Copyright 2022-2033 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import ( + "sort" + + "go.yaml.in/yaml/v4" +) + +func (index *SpecIndex) SetCircularReferences(refs []*CircularReferenceResult) { + index.circularReferences = refs +} + +func (index *SpecIndex) GetCircularReferences() []*CircularReferenceResult { + return index.circularReferences +} + +func (index *SpecIndex) GetTagCircularReferences() []*CircularReferenceResult { + return index.tagCircularReferences +} + +func (index *SpecIndex) SetIgnoredPolymorphicCircularReferences(refs []*CircularReferenceResult) { + index.polyCircularReferences = refs +} + +func (index *SpecIndex) SetIgnoredArrayCircularReferences(refs []*CircularReferenceResult) { + index.arrayCircularReferences = refs +} + +func (index *SpecIndex) GetIgnoredPolymorphicCircularReferences() []*CircularReferenceResult { + return index.polyCircularReferences +} + +func (index *SpecIndex) GetIgnoredArrayCircularReferences() []*CircularReferenceResult { + return index.arrayCircularReferences +} + +func (index *SpecIndex) GetPathsNode() *yaml.Node { + return index.pathsNode +} + +func (index *SpecIndex) GetDiscoveredReferences() map[string]*Reference { + return index.allRefs +} + +func (index *SpecIndex) GetPolyReferences() map[string]*Reference { + return index.polymorphicRefs +} + +func (index *SpecIndex) GetPolyAllOfReferences() []*Reference { + return index.polymorphicAllOfRefs +} + +func (index *SpecIndex) GetPolyAnyOfReferences() []*Reference { + return index.polymorphicAnyOfRefs +} + +func (index *SpecIndex) GetPolyOneOfReferences() []*Reference { + return index.polymorphicOneOfRefs +} + +func (index *SpecIndex) GetAllCombinedReferences() map[string]*Reference { + combined := make(map[string]*Reference) + for k, ref := range index.allRefs { + combined[k] = ref + } + for k, ref := range index.polymorphicRefs { + combined[k] = ref + } + return combined +} + +func (index *SpecIndex) GetRefsByLine() map[string]map[int]bool { + return index.refsByLine +} + +func (index *SpecIndex) GetLinesWithReferences() map[int]bool { + return index.linesWithRefs +} + +func (index *SpecIndex) GetMappedReferences() map[string]*Reference { + return index.allMappedRefs +} + +func (index *SpecIndex) SetMappedReferences(mappedRefs map[string]*Reference) { + index.allMappedRefs = mappedRefs +} + +func (index *SpecIndex) GetRawReferencesSequenced() []*Reference { + return index.rawSequencedRefs +} + +func (index *SpecIndex) GetExtensionRefsSequenced() []*Reference { + var extensionRefs []*Reference + for _, ref := range index.rawSequencedRefs { + if ref.IsExtensionRef { + extensionRefs = append(extensionRefs, ref) + } + } + return extensionRefs +} + +func (index *SpecIndex) GetMappedReferencesSequenced() []*ReferenceMapped { + return index.allMappedRefsSequenced +} + +func (index *SpecIndex) GetOperationParameterReferences() map[string]map[string]map[string][]*Reference { + return index.paramOpRefs +} + +func (index *SpecIndex) GetAllSchemas() []*Reference { + componentSchemas := index.GetAllComponentSchemas() + inlineSchemas := index.GetAllInlineSchemas() + refSchemas := index.GetAllReferenceSchemas() + combined := make([]*Reference, len(inlineSchemas)+len(componentSchemas)+len(refSchemas)) + i := 0 + for x := range inlineSchemas { + combined[i] = inlineSchemas[x] + i++ + } + for x := range componentSchemas { + combined[i] = componentSchemas[x] + i++ + } + for x := range refSchemas { + combined[i] = refSchemas[x] + i++ + } + sort.Slice(combined, func(i, j int) bool { + return combined[i].Node.Line < combined[j].Node.Line + }) + return combined +} + +func (index *SpecIndex) GetAllInlineSchemaObjects() []*Reference { + return index.allInlineSchemaObjectDefinitions +} + +func (index *SpecIndex) GetAllInlineSchemas() []*Reference { + return index.allInlineSchemaDefinitions +} + +func (index *SpecIndex) GetAllReferenceSchemas() []*Reference { + return index.allRefSchemaDefinitions +} + +func (index *SpecIndex) GetAllComponentSchemas() map[string]*Reference { + if index == nil { + return nil + } + index.allComponentSchemasLock.RLock() + if index.allComponentSchemas != nil { + defer index.allComponentSchemasLock.RUnlock() + return index.allComponentSchemas + } + index.allComponentSchemasLock.RUnlock() + + index.allComponentSchemasLock.Lock() + defer index.allComponentSchemasLock.Unlock() + if index.allComponentSchemas == nil { + index.allComponentSchemas = syncMapToMap[string, *Reference](index.allComponentSchemaDefinitions) + } + return index.allComponentSchemas +} + +func (index *SpecIndex) GetAllSecuritySchemes() map[string]*Reference { + return syncMapToMap[string, *Reference](index.allSecuritySchemes) +} + +func (index *SpecIndex) GetAllHeaders() map[string]*Reference { + return index.allHeaders +} + +func (index *SpecIndex) GetAllExternalDocuments() map[string]*Reference { + return index.allExternalDocuments +} + +func (index *SpecIndex) GetAllExamples() map[string]*Reference { + return index.allExamples +} + +func (index *SpecIndex) GetAllDescriptions() []*DescriptionReference { + return index.allDescriptions +} + +func (index *SpecIndex) GetAllEnums() []*EnumReference { + return index.allEnums +} + +func (index *SpecIndex) GetAllObjectsWithProperties() []*ObjectReference { + return index.allObjectsWithProperties +} + +func (index *SpecIndex) GetAllSummaries() []*DescriptionReference { + return index.allSummaries +} + +func (index *SpecIndex) GetAllRequestBodies() map[string]*Reference { + return index.allRequestBodies +} + +func (index *SpecIndex) GetAllLinks() map[string]*Reference { + return index.allLinks +} + +func (index *SpecIndex) GetAllParameters() map[string]*Reference { + return index.allParameters +} + +func (index *SpecIndex) GetAllResponses() map[string]*Reference { + return index.allResponses +} + +func (index *SpecIndex) GetAllCallbacks() map[string]*Reference { + return index.allCallbacks +} + +func (index *SpecIndex) GetAllComponentPathItems() map[string]*Reference { + return index.allComponentPathItems +} + +func (index *SpecIndex) GetInlineOperationDuplicateParameters() map[string][]*Reference { + return index.paramInlineDuplicateNames +} + +func (index *SpecIndex) GetReferencesWithSiblings() map[string]Reference { + return index.refsWithSiblings +} + +func (index *SpecIndex) GetAllReferences() map[string]*Reference { + return index.allRefs +} + +func (index *SpecIndex) GetAllSequencedReferences() []*Reference { + return index.rawSequencedRefs +} + +func (index *SpecIndex) GetSchemasNode() *yaml.Node { + return index.schemasNode +} + +func (index *SpecIndex) GetParametersNode() *yaml.Node { + return index.parametersNode +} + +func (index *SpecIndex) GetReferenceIndexErrors() []error { + return index.refErrors +} + +func (index *SpecIndex) GetOperationParametersIndexErrors() []error { + return index.operationParamErrors +} + +func (index *SpecIndex) GetAllPaths() map[string]map[string]*Reference { + return index.pathRefs +} + +func (index *SpecIndex) GetOperationTags() map[string]map[string][]*Reference { + return index.operationTagsRefs +} + +func (index *SpecIndex) GetAllParametersFromOperations() map[string]map[string]map[string][]*Reference { + return index.paramOpRefs +} + +func (index *SpecIndex) GetRootSecurityReferences() []*Reference { + return index.rootSecurity +} + +func (index *SpecIndex) GetSecurityRequirementReferences() map[string]map[string][]*Reference { + return index.securityRequirementRefs +} + +func (index *SpecIndex) GetRootSecurityNode() *yaml.Node { + return index.rootSecurityNode +} + +func (index *SpecIndex) GetRootServersNode() *yaml.Node { + return index.rootServersNode +} + +func (index *SpecIndex) GetAllRootServers() []*Reference { + return index.serversRefs +} + +func (index *SpecIndex) GetAllOperationsServers() map[string]map[string][]*Reference { + return index.opServersRefs +} + +func (index *SpecIndex) SetAllowCircularReferenceResolving(allow bool) { + index.allowCircularReferences = allow +} + +func (index *SpecIndex) AllowCircularReferenceResolving() bool { + return index.allowCircularReferences +} + +func (index *SpecIndex) checkPolymorphicNode(name string) (bool, string) { + switch name { + case "anyOf": + return true, "anyOf" + case "allOf": + return true, "allOf" + case "oneOf": + return true, "oneOf" + } + return false, "" +} + +func (index *SpecIndex) RegisterSchemaId(entry *SchemaIdEntry) error { + index.schemaIdRegistryLock.Lock() + defer index.schemaIdRegistryLock.Unlock() + if index.schemaIdRegistry == nil { + index.schemaIdRegistry = make(map[string]*SchemaIdEntry) + } + _, err := registerSchemaIdToRegistry(index.schemaIdRegistry, entry, index.logger, "local index") + return err +} + +func (index *SpecIndex) GetSchemaById(uri string) *SchemaIdEntry { + index.schemaIdRegistryLock.RLock() + defer index.schemaIdRegistryLock.RUnlock() + if index.schemaIdRegistry == nil { + return nil + } + return index.schemaIdRegistry[uri] +} + +func (index *SpecIndex) GetAllSchemaIds() map[string]*SchemaIdEntry { + index.schemaIdRegistryLock.RLock() + defer index.schemaIdRegistryLock.RUnlock() + return copySchemaIdRegistry(index.schemaIdRegistry) +} diff --git a/index/spec_index_build.go b/index/spec_index_build.go new file mode 100644 index 000000000..f6f8e1f40 --- /dev/null +++ b/index/spec_index_build.go @@ -0,0 +1,164 @@ +// Copyright 2022-2033 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "sort" + "sync" + + "go.yaml.in/yaml/v4" +) + +const ( + // theoreticalRoot is the name of the theoretical spec file used when a root spec file does not exist + theoreticalRoot = "root.yaml" +) + +func NewSpecIndexWithConfigAndContext(ctx context.Context, rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex { + index := new(SpecIndex) + boostrapIndexCollections(index) + index.InitHighCache() + index.config = config + index.rolodex = config.Rolodex + index.uri = config.uri + index.specAbsolutePath = config.SpecAbsolutePath + if config.Logger != nil { + index.logger = config.Logger + } else { + index.logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelError})) + } + if rootNode == nil || len(rootNode.Content) <= 0 { + return index + } + index.root = rootNode + return createNewIndex(ctx, rootNode, index, config.AvoidBuildIndex) +} + +func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex { + return NewSpecIndexWithConfigAndContext(context.Background(), rootNode, config) +} + +func NewSpecIndex(rootNode *yaml.Node) *SpecIndex { + index := new(SpecIndex) + index.InitHighCache() + index.config = CreateOpenAPIIndexConfig() + index.root = rootNode + boostrapIndexCollections(index) + return createNewIndex(context.Background(), rootNode, index, false) +} + +func createNewIndex(ctx context.Context, rootNode *yaml.Node, index *SpecIndex, avoidBuildOut bool) *SpecIndex { + if rootNode == nil { + return index + } + index.nodeMapCompleted = make(chan struct{}) + index.nodeMap = make(map[int]map[int]*yaml.Node) + go index.MapNodes(rootNode) + + index.cache = new(sync.Map) + results := index.ExtractRefs(ctx, index.root.Content[0], index.root, []string{}, 0, false, "") + + dd := make(map[string]struct{}) + var dedupedResults []*Reference + for _, ref := range results { + if _, ok := dd[ref.FullDefinition]; !ok { + dd[ref.FullDefinition] = struct{}{} + dedupedResults = append(dedupedResults, ref) + } + } + + polyKeys := make([]string, 0, len(index.polymorphicRefs)) + for k := range index.polymorphicRefs { + polyKeys = append(polyKeys, k) + } + sort.Strings(polyKeys) + poly := make([]*Reference, len(index.polymorphicRefs)) + for i, k := range polyKeys { + poly[i] = index.polymorphicRefs[k] + } + + if len(dedupedResults) > 0 { + index.ExtractComponentsFromRefs(ctx, dedupedResults) + } + if len(poly) > 0 { + index.ExtractComponentsFromRefs(ctx, poly) + } + + index.ExtractExternalDocuments(index.root) + index.GetPathCount() + + if !avoidBuildOut { + index.BuildIndex() + } + <-index.nodeMapCompleted + return index +} + +func (index *SpecIndex) BuildIndex() { + if index.built { + return + } + countFuncs := []func() int{ + index.GetOperationCount, + index.GetComponentSchemaCount, + index.GetGlobalTagsCount, + index.GetComponentParameterCount, + index.GetOperationsParameterCount, + } + + var wg sync.WaitGroup + wg.Add(len(countFuncs)) + runIndexFunction(countFuncs, &wg) + wg.Wait() + + countFuncs = []func() int{ + index.GetInlineUniqueParamCount, + index.GetOperationTagsCount, + index.GetGlobalLinksCount, + index.GetGlobalCallbacksCount, + } + wg.Add(len(countFuncs)) + runIndexFunction(countFuncs, &wg) + wg.Wait() + + index.GetInlineDuplicateParamCount() + index.GetAllDescriptionsCount() + index.GetTotalTagsCount() + index.built = true +} + +func (index *SpecIndex) GetLogger() *slog.Logger { + return index.logger +} + +func (index *SpecIndex) GetRootNode() *yaml.Node { + return index.root +} + +func (index *SpecIndex) SetRootNode(node *yaml.Node) { + index.root = node +} + +func (index *SpecIndex) GetRolodex() *Rolodex { + return index.rolodex +} + +func (index *SpecIndex) SetRolodex(rolodex *Rolodex) { + index.rolodex = rolodex +} + +func (index *SpecIndex) GetSpecFileName() string { + if index == nil || index.rolodex == nil || index.rolodex.indexConfig == nil || index.rolodex.indexConfig.SpecFilePath == "" { + return theoreticalRoot + } + return filepath.Base(index.rolodex.indexConfig.SpecFilePath) +} + +func (index *SpecIndex) GetGlobalTagsNode() *yaml.Node { + return index.tagsNode +} diff --git a/index/spec_index_counts.go b/index/spec_index_counts.go new file mode 100644 index 000000000..6538b8556 --- /dev/null +++ b/index/spec_index_counts.go @@ -0,0 +1,699 @@ +// Copyright 2022-2033 Dave Shanley / Quobix +// SPDX-License-Identifier: MIT + +package index + +import ( + "context" + "fmt" + "strings" + + "github.com/pb33f/libopenapi/utils" + "go.yaml.in/yaml/v4" +) + +func (index *SpecIndex) GetPathCount() int { + if index.root == nil { + return -1 + } + if index.pathCount > 0 { + return index.pathCount + } + pc := 0 + for i, n := range index.root.Content[0].Content { + if i%2 == 0 && n.Value == "paths" { + pn := index.root.Content[0].Content[i+1].Content + index.pathsNode = index.root.Content[0].Content[i+1] + pc = len(pn) / 2 + } + } + index.pathCount = pc + return pc +} + +func (index *SpecIndex) ExtractExternalDocuments(node *yaml.Node) []*Reference { + if node == nil { + return nil + } + var found []*Reference + if len(node.Content) > 0 { + for i, n := range node.Content { + if utils.IsNodeMap(n) || utils.IsNodeArray(n) { + found = append(found, index.ExtractExternalDocuments(n)...) + } + if i%2 == 0 && n.Value == "externalDocs" { + docNode := node.Content[i+1] + _, urlNode := utils.FindKeyNode("url", docNode.Content) + if urlNode != nil { + ref := &Reference{Definition: urlNode.Value, Name: urlNode.Value, Node: docNode} + index.externalDocumentsRef = append(index.externalDocumentsRef, ref) + found = append(found, ref) + } + } + } + } + index.externalDocumentsCount = len(index.externalDocumentsRef) + return found +} + +func (index *SpecIndex) GetGlobalTagsCount() int { + if index.root == nil { + return -1 + } + if index.globalTagsCount > 0 { + return index.globalTagsCount + } + for i, n := range index.root.Content[0].Content { + if i%2 == 0 && n.Value == "tags" { + tagsNode := index.root.Content[0].Content[i+1] + if tagsNode != nil { + index.tagsNode = tagsNode + index.globalTagsCount = len(tagsNode.Content) + for x, tagNode := range index.tagsNode.Content { + _, name := utils.FindKeyNode("name", tagNode.Content) + _, description := utils.FindKeyNode("description", tagNode.Content) + desc := "" + if description == nil { + desc = "" + } + if name != nil { + index.globalTagRefs[name.Value] = &Reference{ + Definition: desc, + Name: name.Value, + Node: tagNode, + Path: fmt.Sprintf("$.tags[%d]", x), + } + } + } + index.checkTagCircularReferences() + } + } + } + return index.globalTagsCount +} + +func (index *SpecIndex) checkTagCircularReferences() { + if index.tagsNode == nil { + return + } + + tagParentMap := make(map[string]string) + tagRefs := make(map[string]*Reference) + tagNodes := make(map[string]*yaml.Node) + + for x, tagNode := range index.tagsNode.Content { + _, nameNode := utils.FindKeyNode("name", tagNode.Content) + _, parentNode := utils.FindKeyNode("parent", tagNode.Content) + if nameNode != nil { + tagName := nameNode.Value + tagNodes[tagName] = tagNode + tagRefs[tagName] = &Reference{Name: tagName, Node: tagNode, Path: fmt.Sprintf("$.tags[%d]", x)} + if parentNode != nil { + tagParentMap[tagName] = parentNode.Value + } + } + } + + visited := make(map[string]bool) + recStack := make(map[string]bool) + for tagName := range tagRefs { + if !visited[tagName] { + if _, hasParent := tagParentMap[tagName]; hasParent { + if path := index.detectTagCircularHelper(tagName, tagParentMap, tagRefs, visited, recStack, []string{}); len(path) > 0 { + journey := make([]*Reference, len(path)) + for i, name := range path { + journey[i] = tagRefs[name] + } + loopIndex := -1 + loopStart := path[len(path)-1] + for i, name := range path { + if name == loopStart { + loopIndex = i + break + } + } + index.tagCircularReferences = append(index.tagCircularReferences, &CircularReferenceResult{ + Journey: journey, + Start: tagRefs[path[0]], + LoopIndex: loopIndex, + LoopPoint: tagRefs[loopStart], + ParentNode: tagNodes[loopStart], + IsInfiniteLoop: true, + }) + } + } + } + } +} + +func (index *SpecIndex) detectTagCircularHelper(tagName string, parentMap map[string]string, tagRefs map[string]*Reference, visited map[string]bool, recStack map[string]bool, path []string) []string { + if _, exists := tagRefs[tagName]; !exists { + return []string{} + } + visited[tagName] = true + recStack[tagName] = true + path = append(path, tagName) + + if parentName, hasParent := parentMap[tagName]; hasParent { + if _, parentExists := tagRefs[parentName]; !parentExists { + recStack[tagName] = false + return []string{} + } + if recStack[parentName] { + return append(path, parentName) + } + if !visited[parentName] { + if cyclePath := index.detectTagCircularHelper(parentName, parentMap, tagRefs, visited, recStack, path); len(cyclePath) > 0 { + return cyclePath + } + } + } + + recStack[tagName] = false + return []string{} +} + +func (index *SpecIndex) GetOperationTagsCount() int { + if index.root == nil { + return -1 + } + if index.operationTagsCount > 0 { + return index.operationTagsCount + } + seen := make(map[string]bool) + count := 0 + for _, path := range index.operationTagsRefs { + for _, method := range path { + for _, tag := range method { + if !seen[tag.Name] { + seen[tag.Name] = true + count++ + } + } + } + } + index.operationTagsCount = count + return count +} + +func (index *SpecIndex) GetTotalTagsCount() int { + if index.root == nil { + return -1 + } + if index.totalTagsCount > 0 { + return index.totalTagsCount + } + seen := make(map[string]bool) + count := 0 + for _, gt := range index.globalTagRefs { + if !seen[gt.Name] { + seen[gt.Name] = true + count++ + } + } + for _, ot := range index.operationTagsRefs { + for _, m := range ot { + for _, t := range m { + if !seen[t.Name] { + seen[t.Name] = true + count++ + } + } + } + } + index.totalTagsCount = count + return count +} + +func (index *SpecIndex) GetGlobalCallbacksCount() int { + if index.root == nil { + return -1 + } + if index.globalCallbacksCount > 0 { + return index.globalCallbacksCount + } + index.pathRefsLock.RLock() + for path, p := range index.pathRefs { + for _, m := range p { + index.globalCallbacksCount += index.collectOperationObjectReferences(path, m, "callbacks", index.callbacksRefs) + } + } + index.pathRefsLock.RUnlock() + return index.globalCallbacksCount +} + +func (index *SpecIndex) GetGlobalLinksCount() int { + if index.root == nil { + return -1 + } + if index.globalLinksCount > 0 { + return index.globalLinksCount + } + for path, p := range index.pathRefs { + for _, m := range p { + index.globalLinksCount += index.collectOperationObjectReferences(path, m, "links", index.linksRefs) + } + } + return index.globalLinksCount +} + +func (index *SpecIndex) collectOperationObjectReferences(path string, operation *Reference, key string, target map[string]map[string][]*Reference) int { + var count int + for _, container := range findNestedObjectContainers(operation.Node, key) { + for _, node := range container.Content { + if !utils.IsNodeMap(node) { + continue + } + if target[path] == nil { + target[path] = make(map[string][]*Reference) + } + target[path][operation.Name] = append(target[path][operation.Name], &Reference{ + Definition: operation.Name, + Name: operation.Name, + Node: node, + }) + count++ + } + } + return count +} + +func findNestedObjectContainers(node *yaml.Node, key string) []*yaml.Node { + if node == nil { + return nil + } + + var found []*yaml.Node + var visit func(*yaml.Node) + visit = func(current *yaml.Node) { + if current == nil { + return + } + switch current.Kind { + case yaml.DocumentNode: + for _, child := range current.Content { + visit(child) + } + case yaml.MappingNode: + for i := 0; i < len(current.Content)-1; i += 2 { + k := current.Content[i] + v := current.Content[i+1] + if k != nil && k.Value == key && v != nil && v.Kind == yaml.MappingNode { + found = append(found, v) + } + visit(v) + } + case yaml.SequenceNode: + for _, child := range current.Content { + visit(child) + } + } + } + + visit(node) + return found +} + +func (index *SpecIndex) GetRawReferenceCount() int { return len(index.rawSequencedRefs) } + +func (index *SpecIndex) GetComponentSchemaCount() int { + if index.root == nil || len(index.root.Content) == 0 { + return -1 + } + if index.schemaCount > 0 { + return index.schemaCount + } + + for i, n := range index.root.Content[0].Content { + if i%2 != 0 { + continue + } + if n.Value == "servers" { + index.rootServersNode = index.root.Content[0].Content[i+1] + if i+1 < len(index.root.Content[0].Content) { + serverDefinitions := index.root.Content[0].Content[i+1] + for x, def := range serverDefinitions.Content { + index.serversRefs = append(index.serversRefs, &Reference{ + Definition: "servers", + Name: "server", + Node: def, + Path: fmt.Sprintf("$.servers[%d]", x), + ParentNode: index.rootServersNode, + }) + } + } + } + if n.Value == "security" { + index.rootSecurityNode = index.root.Content[0].Content[i+1] + if i+1 < len(index.root.Content[0].Content) { + securityDefinitions := index.root.Content[0].Content[i+1] + for x, def := range securityDefinitions.Content { + if len(def.Content) > 0 { + name := def.Content[0] + index.rootSecurity = append(index.rootSecurity, &Reference{ + Definition: name.Value, + Name: name.Value, + Node: def, + Path: fmt.Sprintf("$.security[%d]", x), + }) + } + } + } + } + if n.Value == "components" { + _, schemasNode := utils.FindKeyNode("schemas", index.root.Content[0].Content[i+1].Content) + _, parametersNode := utils.FindKeyNode("parameters", index.root.Content[0].Content[i+1].Content) + _, requestBodiesNode := utils.FindKeyNode("requestBodies", index.root.Content[0].Content[i+1].Content) + _, responsesNode := utils.FindKeyNode("responses", index.root.Content[0].Content[i+1].Content) + _, securitySchemesNode := utils.FindKeyNode("securitySchemes", index.root.Content[0].Content[i+1].Content) + _, headersNode := utils.FindKeyNode("headers", index.root.Content[0].Content[i+1].Content) + _, examplesNode := utils.FindKeyNode("examples", index.root.Content[0].Content[i+1].Content) + _, linksNode := utils.FindKeyNode("links", index.root.Content[0].Content[i+1].Content) + _, callbacksNode := utils.FindKeyNode("callbacks", index.root.Content[0].Content[i+1].Content) + _, pathItemsNode := utils.FindKeyNode("pathItems", index.root.Content[0].Content[i+1].Content) + + if schemasNode != nil { + index.extractDefinitionsAndSchemas(schemasNode, "#/components/schemas/") + index.schemasNode = schemasNode + index.schemaCount = len(schemasNode.Content) / 2 + } + if parametersNode != nil { + index.extractComponentParameters(parametersNode, "#/components/parameters/") + index.componentLock.Lock() + index.parametersNode = parametersNode + index.componentLock.Unlock() + } + if requestBodiesNode != nil { + index.extractComponentRequestBodies(requestBodiesNode, "#/components/requestBodies/") + index.requestBodiesNode = requestBodiesNode + } + if responsesNode != nil { + index.extractComponentResponses(responsesNode, "#/components/responses/") + index.responsesNode = responsesNode + } + if securitySchemesNode != nil { + index.extractComponentSecuritySchemes(securitySchemesNode, "#/components/securitySchemes/") + index.securitySchemesNode = securitySchemesNode + } + if headersNode != nil { + index.extractComponentHeaders(headersNode, "#/components/headers/") + index.headersNode = headersNode + } + if examplesNode != nil { + index.extractComponentExamples(examplesNode, "#/components/examples/") + index.examplesNode = examplesNode + } + if linksNode != nil { + index.extractComponentLinks(linksNode, "#/components/links/") + index.linksNode = linksNode + } + if callbacksNode != nil { + index.extractComponentCallbacks(callbacksNode, "#/components/callbacks/") + index.callbacksNode = callbacksNode + } + if pathItemsNode != nil { + index.extractComponentPathItems(pathItemsNode, "#/components/pathItems/") + index.pathItemsNode = pathItemsNode + } + } + if n.Value == "definitions" { + schemasNode := index.root.Content[0].Content[i+1] + if schemasNode != nil { + index.extractDefinitionsAndSchemas(schemasNode, "#/definitions/") + index.schemasNode = schemasNode + index.schemaCount = len(schemasNode.Content) / 2 + } + } + if n.Value == "parameters" { + parametersNode := index.root.Content[0].Content[i+1] + if parametersNode != nil { + index.extractComponentParameters(parametersNode, "#/parameters/") + index.componentLock.Lock() + index.parametersNode = parametersNode + index.componentLock.Unlock() + } + } + if n.Value == "responses" { + responsesNode := index.root.Content[0].Content[i+1] + if responsesNode != nil { + index.extractComponentResponses(responsesNode, "#/responses/") + index.responsesNode = responsesNode + } + } + if n.Value == "securityDefinitions" { + securityDefinitionsNode := index.root.Content[0].Content[i+1] + if securityDefinitionsNode != nil { + index.extractComponentSecuritySchemes(securityDefinitionsNode, "#/securityDefinitions/") + index.securitySchemesNode = securityDefinitionsNode + } + } + } + return index.schemaCount +} + +func (index *SpecIndex) GetComponentParameterCount() int { + if index.root == nil { + return -1 + } + if index.componentParamCount > 0 { + return index.componentParamCount + } + for i, n := range index.root.Content[0].Content { + if i%2 == 0 { + if n.Value == "components" { + _, parametersNode := utils.FindKeyNode("parameters", index.root.Content[0].Content[i+1].Content) + if parametersNode != nil { + index.componentLock.Lock() + index.parametersNode = parametersNode + index.componentParamCount = len(parametersNode.Content) / 2 + index.componentLock.Unlock() + } + } + if n.Value == "parameters" { + parametersNode := index.root.Content[0].Content[i+1] + if parametersNode != nil { + index.componentLock.Lock() + index.parametersNode = parametersNode + index.componentParamCount = len(parametersNode.Content) / 2 + index.componentLock.Unlock() + } + } + } + } + return index.componentParamCount +} + +func (index *SpecIndex) GetOperationCount() int { + if index.root == nil || index.pathsNode == nil { + return -1 + } + if index.operationCount > 0 { + return index.operationCount + } + opCount := 0 + locatedPathRefs := make(map[string]map[string]*Reference) + for x, p := range index.pathsNode.Content { + if x%2 == 0 { + var method *yaml.Node + if utils.IsNodeArray(index.pathsNode) { + method = index.pathsNode.Content[x] + } else { + method = index.pathsNode.Content[x+1] + } + if isRef, _, ref := utils.IsNodeRefValue(method); isRef { + ctx := context.WithValue(context.Background(), CurrentPathKey, index.specAbsolutePath) + ctx = context.WithValue(ctx, RootIndexKey, index) + pNode := seekRefEnd(ctx, index, ref) + if pNode != nil { + method = pNode.Node + } + } + for y, m := range method.Content { + if y%2 == 0 { + valid := false + for _, methodType := range methodTypes { + if m.Value == methodType { + valid = true + } + } + if valid { + ref := &Reference{ + Definition: m.Value, + Name: m.Value, + Node: method.Content[y+1], + Path: fmt.Sprintf("$.paths['%s'].%s", p.Value, m.Value), + ParentNode: m, + } + if locatedPathRefs[p.Value] == nil { + locatedPathRefs[p.Value] = make(map[string]*Reference) + } + locatedPathRefs[p.Value][ref.Name] = ref + opCount++ + } + } + } + } + } + for k, v := range locatedPathRefs { + index.pathRefs[k] = v + } + index.operationCount = opCount + return opCount +} + +func (index *SpecIndex) GetOperationsParameterCount() int { + if index.root == nil || index.pathsNode == nil { + return -1 + } + if index.operationParamCount > 0 { + return index.operationParamCount + } + for x, pathItemNode := range index.pathsNode.Content { + if x%2 == 0 { + var pathPropertyNode *yaml.Node + if utils.IsNodeArray(index.pathsNode) { + pathPropertyNode = index.pathsNode.Content[x] + } else { + pathPropertyNode = index.pathsNode.Content[x+1] + } + if isRef, _, ref := utils.IsNodeRefValue(pathPropertyNode); isRef { + ctx := context.WithValue(context.Background(), CurrentPathKey, index.specAbsolutePath) + ctx = context.WithValue(ctx, RootIndexKey, index) + pNode := seekRefEnd(ctx, index, ref) + if pNode != nil { + pathPropertyNode = pNode.Node + } + } + for y, prop := range pathPropertyNode.Content { + if y%2 == 0 { + if prop.Value == "servers" { + serversNode := pathPropertyNode.Content[y+1] + if index.opServersRefs[pathItemNode.Value] == nil { + index.opServersRefs[pathItemNode.Value] = make(map[string][]*Reference) + } + var serverRefs []*Reference + for i, serverRef := range serversNode.Content { + serverRefs = append(serverRefs, &Reference{ + Definition: serverRef.Value, + Name: serverRef.Value, + Node: serverRef, + ParentNode: prop, + Path: fmt.Sprintf("$.paths['%s'].servers[%d]", pathItemNode.Value, i), + }) + } + index.opServersRefs[pathItemNode.Value]["top"] = serverRefs + } + if prop.Value == "parameters" { + index.scanOperationParams(pathPropertyNode.Content[y+1].Content, pathPropertyNode.Content[y], pathItemNode, "top") + } + if isHttpMethod(prop.Value) { + for z, httpMethodProp := range pathPropertyNode.Content[y+1].Content { + if z%2 == 0 { + if httpMethodProp.Value == "parameters" { + index.scanOperationParams(pathPropertyNode.Content[y+1].Content[z+1].Content, pathPropertyNode.Content[y+1].Content[z], pathItemNode, prop.Value) + } + if httpMethodProp.Value == "tags" { + tags := pathPropertyNode.Content[y+1].Content[z+1] + if index.operationTagsRefs[pathItemNode.Value] == nil { + index.operationTagsRefs[pathItemNode.Value] = make(map[string][]*Reference) + } + var tagRefs []*Reference + for _, tagRef := range tags.Content { + tagRefs = append(tagRefs, &Reference{Definition: tagRef.Value, Name: tagRef.Value, Node: tagRef}) + } + index.operationTagsRefs[pathItemNode.Value][prop.Value] = tagRefs + } + if httpMethodProp.Value == "description" { + desc := pathPropertyNode.Content[y+1].Content[z+1].Value + if index.operationDescriptionRefs[pathItemNode.Value] == nil { + index.operationDescriptionRefs[pathItemNode.Value] = make(map[string]*Reference) + } + index.operationDescriptionRefs[pathItemNode.Value][prop.Value] = &Reference{ + Definition: desc, + Name: "description", + Node: pathPropertyNode.Content[y+1].Content[z+1], + } + } + if httpMethodProp.Value == "summary" { + summary := pathPropertyNode.Content[y+1].Content[z+1].Value + if index.operationSummaryRefs[pathItemNode.Value] == nil { + index.operationSummaryRefs[pathItemNode.Value] = make(map[string]*Reference) + } + index.operationSummaryRefs[pathItemNode.Value][prop.Value] = &Reference{ + Definition: summary, + Name: "summary", + Node: pathPropertyNode.Content[y+1].Content[z+1], + } + } + if httpMethodProp.Value == "servers" { + serversNode := pathPropertyNode.Content[y+1].Content[z+1] + var serverRefs []*Reference + for i, serverRef := range serversNode.Content { + serverRefs = append(serverRefs, &Reference{ + Definition: "servers", + Name: "servers", + Node: serverRef, + ParentNode: httpMethodProp, + Path: fmt.Sprintf("$.paths['%s'].%s.servers[%d]", pathItemNode.Value, prop.Value, i), + }) + } + if index.opServersRefs[pathItemNode.Value] == nil { + index.opServersRefs[pathItemNode.Value] = make(map[string][]*Reference) + } + index.opServersRefs[pathItemNode.Value][prop.Value] = serverRefs + } + } + } + } + } + } + } + } + + for key, component := range index.allMappedRefs { + if strings.Contains(key, "/parameters/") { + index.paramCompRefs[key] = component + index.paramAllRefs[key] = component + } + } + + for path, params := range index.paramOpRefs { + for mName, mValue := range params { + for pName, pValue := range mValue { + if !strings.HasPrefix(pName, "#") { + index.paramInlineDuplicateNames[pName] = append(index.paramInlineDuplicateNames[pName], pValue...) + for i := range pValue { + if pValue[i] != nil { + _, in := utils.FindKeyNodeTop("in", pValue[i].Node.Content) + if in != nil { + index.paramAllRefs[fmt.Sprintf("%s:::%s:::%s", path, mName, in.Value)] = pValue[i] + } else { + index.paramAllRefs[fmt.Sprintf("%s:::%s", path, mName)] = pValue[i] + } + } + } + } + } + } + } + + index.operationParamCount = len(index.paramCompRefs) + len(index.paramInlineDuplicateNames) + return index.operationParamCount +} + +func (index *SpecIndex) GetInlineDuplicateParamCount() int { + if index.componentsInlineParamDuplicateCount > 0 { + return index.componentsInlineParamDuplicateCount + } + dCount := len(index.paramInlineDuplicateNames) - index.countUniqueInlineDuplicates() + index.componentsInlineParamDuplicateCount = dCount + return dCount +} + +func (index *SpecIndex) GetInlineUniqueParamCount() int { + return index.countUniqueInlineDuplicates() +} + +func (index *SpecIndex) GetAllDescriptionsCount() int { return len(index.allDescriptions) } + +func (index *SpecIndex) GetAllSummariesCount() int { return len(index.allSummaries) } diff --git a/index/spec_index_test.go b/index/spec_index_test.go index 2eb15e898..db3b994fa 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -19,6 +19,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -203,6 +204,9 @@ func TestSpecIndex_DigitalOcean(t *testing.T) { // index the rolodex. indexedErr := rolo.IndexTheRolodex(context.Background()) + if indexedErr != nil && strings.Contains(indexedErr.Error(), "429") { + t.Skipf("skipping due to GitHub rate limit: %v", indexedErr) + } assert.NoError(t, indexedErr) // get all the files! @@ -211,8 +215,14 @@ func TestSpecIndex_DigitalOcean(t *testing.T) { // if windows if runtime.GOOS != "windows" { + if fileLen != 1660 && hasRateLimitedRemoteErrors(remoteFS.GetErrors()) { + t.Skipf("skipping due to GitHub rate limit; fetched %d remote files", fileLen) + } assert.Equal(t, 1660, fileLen) } + if hasRateLimitedRemoteErrors(remoteFS.GetErrors()) { + t.Skip("skipping due to GitHub rate limit in remote filesystem fetches") + } assert.Len(t, remoteFS.GetErrors(), 0) // check circular references @@ -221,6 +231,15 @@ func TestSpecIndex_DigitalOcean(t *testing.T) { assert.Len(t, rolo.GetIgnoredCircularReferences(), 0) } +func hasRateLimitedRemoteErrors(errs []error) bool { + for _, err := range errs { + if err != nil && strings.Contains(err.Error(), "429") { + return true + } + } + return false +} + func TestSpecIndex_Redocly(t *testing.T) { do, _ := os.ReadFile("../test_specs/redocly-starter.yaml") var rootNode yaml.Node @@ -817,6 +836,28 @@ func TestSpecIndex_NoRoot(t *testing.T) { assert.Equal(t, -1, index.GetGlobalLinksCount()) } +func TestSpecIndex_ExtractExternalDocuments_ReturnsFoundReferences(t *testing.T) { + yml := `openapi: 3.1.0 +info: + title: test + version: 1.0.0 +externalDocs: + url: https://example.com/top +paths: + /test: + get: + externalDocs: + url: https://example.com/op` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewTestSpecIndex().Load().(*SpecIndex) + docs := index.ExtractExternalDocuments(&rootNode) + assert.Len(t, docs, 2) + assert.Len(t, index.externalDocumentsRef, 2) +} + func test_buildMixedRefServer() *httptest.Server { bs, _ := os.ReadFile("../test_specs/burgershop.openapi.yaml") return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { @@ -963,6 +1004,28 @@ func TestGlobalCallbacksNoIndexTest(t *testing.T) { assert.Equal(t, -1, idx.GetGlobalCallbacksCount()) } +func TestGlobalCallbacksCount_MultipleMatchesWithinOperation(t *testing.T) { + yml := `paths: + /events: + post: + callbacks: + outer: + '{$request.query.url}': + post: + callbacks: + inner: + '{$request.query.url}': + post: + description: nested` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Equal(t, 2, index.GetGlobalCallbacksCount()) + assert.Len(t, index.callbacksRefs["/events"]["post"], 2) +} + func TestMultipleCallbacksPerOperationVerb(t *testing.T) { yml := `components: callbacks: @@ -1000,6 +1063,106 @@ paths: assert.Equal(t, 4, index.GetGlobalCallbacksCount()) } +func TestGlobalLinksCount_CollectsAllMatchesAndPreservesSlice(t *testing.T) { + yml := `paths: + /orders: + get: + responses: + '200': + description: ok + links: + first: + operationId: one + second: + operationId: two + '201': + description: created + links: + third: + operationId: three` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Equal(t, 3, index.GetGlobalLinksCount()) + assert.Len(t, index.linksRefs["/orders"]["get"], 3) +} + +func TestCollectOperationObjectReferences(t *testing.T) { + yml := `get: + responses: + '200': + description: ok + links: + first: + operationId: one` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewTestSpecIndex().Load().(*SpecIndex) + target := map[string]map[string][]*Reference{} + operation := &Reference{Name: "get", Node: rootNode.Content[0]} + + count := index.collectOperationObjectReferences("/orders", operation, "links", target) + + assert.Equal(t, 1, count) + assert.Len(t, target["/orders"]["get"], 1) +} + +func TestFindNestedObjectContainers(t *testing.T) { + yml := `post: + callbacks: + outer: + '{$request.query.url}': + post: + callbacks: + inner: + '{$request.query.url}': + post: + description: nested` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + callbacks := findNestedObjectContainers(rootNode.Content[0], "callbacks") + assert.Len(t, callbacks, 2) + assert.Nil(t, findNestedObjectContainers(nil, "links")) +} + +func TestFindNestedObjectContainers_DocumentAndSequenceTraversal(t *testing.T) { + yml := `items: + - links: + one: + operationId: listPets + - callbacks: + cb: + '{$request.query.url}': + post: + description: nested` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + links := findNestedObjectContainers(&rootNode, "links") + callbacks := findNestedObjectContainers(&rootNode, "callbacks") + missing := findNestedObjectContainers(rootNode.Content[0], "missing") + + assert.Len(t, links, 1) + assert.Len(t, callbacks, 1) + assert.Empty(t, missing) +} + +func TestFindNestedObjectContainers_NilChildTraversal(t *testing.T) { + root := &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{nil}, + } + + assert.Empty(t, findNestedObjectContainers(root, "links")) +} + func TestSpecIndex_ExtractComponentsFromRefs(t *testing.T) { yml := `components: schemas: @@ -1374,6 +1537,234 @@ func TestSpecIndex_FindComponent(t *testing.T) { assert.Nil(t, index.FindComponent(context.Background(), "I-do-not-exist")) } +func TestSpecIndex_FindComponent_DirectComponentFastPath(t *testing.T) { + yml := `components: + schemas: + pizza: + type: object + required: + - topping + properties: + topping: + $ref: '#/components/schemas/topping' + topping: + type: string` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + ref := index.FindComponent(context.Background(), "#/components/schemas/pizza") + assert.NotNil(t, ref) + assert.Equal(t, "#/components/schemas/pizza", ref.Definition) + assert.Len(t, ref.RequiredRefProperties, 1) + for _, properties := range ref.RequiredRefProperties { + assert.Equal(t, []string{"topping"}, properties) + } +} + +func TestSpecIndex_FindComponent_DirectComponentFastPath_DecodesPointerTokens(t *testing.T) { + yml := `components: + schemas: + thing/one: + type: string` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + ref := index.FindComponent(context.Background(), "#/components/schemas/thing~1one") + assert.NotNil(t, ref) + assert.Equal(t, "#/components/schemas/thing~1one", ref.Definition) + assert.Equal(t, "thing/one", ref.Name) +} + +func TestFindComponentDirectHelpers(t *testing.T) { + assert.Equal(t, "", normalizeComponentLookupID("")) + assert.Equal(t, "#/components/schemas/thing/one", normalizeComponentLookupID("#/components/schemas/thing~1one")) + assert.Equal(t, "#/components/schemas/thing~two name", normalizeComponentLookupID("#/components/schemas/thing~0two%20name")) + assert.Nil(t, loadSyncMapReference(nil, "#/components/schemas/thing")) + + var sm sync.Map + source := &Reference{ + Name: "thing/one", + Node: &yaml.Node{Kind: yaml.ScalarNode, Value: "x"}, + RequiredRefProperties: map[string][]string{"test": []string{"value"}}, + } + sm.Store("#/components/schemas/thing/one", source) + assert.Same(t, source, loadSyncMapReference(&sm, "#/components/schemas/thing/one")) + + index := NewTestSpecIndex().Load().(*SpecIndex) + index.allComponentSchemaDefinitions = &sm + index.allRefs = map[string]*Reference{ + "#/components/schemas/thing~1one": {ParentNode: &yaml.Node{Kind: yaml.MappingNode}}, + } + + ref := findDirectComponent(index, "#/components/schemas/thing~1one", "test.yaml") + assert.NotNil(t, ref) + assert.Equal(t, "test.yaml#/components/schemas/thing~1one", ref.FullDefinition) + assert.NotNil(t, ref.ParentNode) + assert.Equal(t, source.RequiredRefProperties, ref.RequiredRefProperties) + + index.allRefs = map[string]*Reference{ + "test.yaml#/components/schemas/thing~1one": {ParentNode: &yaml.Node{Kind: yaml.SequenceNode}}, + } + ref = cloneFoundComponentReference(index, &Reference{Name: "thing/one"}, "#/components/schemas/thing~1one", "test.yaml") + assert.Equal(t, yaml.SequenceNode, ref.ParentNode.Kind) + + assert.Nil(t, findDirectComponent(nil, "#/components/schemas/thing", "test.yaml")) + assert.Nil(t, findDirectComponent(index, "thing", "test.yaml")) +} + +func TestCloneFoundComponentReference_PreservesPathAndUsesFullDefinitionParent(t *testing.T) { + index := NewTestSpecIndex().Load().(*SpecIndex) + index.allRefs = map[string]*Reference{ + "test.yaml#/components/schemas/pizza": {ParentNode: &yaml.Node{Kind: yaml.MappingNode}}, + } + + var schemaNode yaml.Node + _ = yaml.Unmarshal([]byte(`type: object +required: + - topping +properties: + topping: + $ref: '#/components/schemas/topping'`), &schemaNode) + + ref := cloneFoundComponentReference(index, &Reference{ + Name: "pizza", + Path: "$.custom", + Node: schemaNode.Content[0], + }, "#/components/schemas/pizza", "test.yaml") + + assert.NotNil(t, ref) + assert.Equal(t, "$.custom", ref.Path) + assert.Equal(t, yaml.MappingNode, ref.ParentNode.Kind) + assert.NotNil(t, ref.RequiredRefProperties) + for _, properties := range ref.RequiredRefProperties { + assert.Equal(t, []string{"topping"}, properties) + } +} + +func TestFindComponentReferenceHelpers(t *testing.T) { + index := NewTestSpecIndex().Load().(*SpecIndex) + componentParent := &yaml.Node{Kind: yaml.MappingNode} + fullDefinitionParent := &yaml.Node{Kind: yaml.SequenceNode} + index.allRefs = map[string]*Reference{ + "#/components/schemas/pizza": {ParentNode: componentParent}, + "test.yaml#/components/schemas/pizza": {ParentNode: fullDefinitionParent}, + } + + assert.Nil(t, lookupComponentParentNode(nil, "#/components/schemas/pizza", "test.yaml#/components/schemas/pizza")) + assert.Same(t, componentParent, lookupComponentParentNode(index, "#/components/schemas/pizza", "test.yaml#/components/schemas/pizza")) + + index.allRefs = map[string]*Reference{ + "test.yaml#/components/schemas/pizza": {ParentNode: fullDefinitionParent}, + } + assert.Same(t, fullDefinitionParent, lookupComponentParentNode(index, "#/components/schemas/pizza", "test.yaml#/components/schemas/pizza")) + + assert.Nil(t, cloneSiblingProperties(nil)) + clonedSiblings := cloneSiblingProperties(map[string]*yaml.Node{"x": {Kind: yaml.ScalarNode, Value: "y"}}) + assert.Len(t, clonedSiblings, 1) + assert.Equal(t, "y", clonedSiblings["x"].Value) +} + +func TestFindDirectComponent_CategoryBranches(t *testing.T) { + index := NewTestSpecIndex().Load().(*SpecIndex) + index.allComponentSchemaDefinitions = &sync.Map{} + index.allComponentSchemaDefinitions.Store("#/definitions/Model", &Reference{Name: "Model"}) + index.allParameters = map[string]*Reference{"#/components/parameters/p": {Name: "p"}} + index.allRequestBodies = map[string]*Reference{"#/components/requestBodies/rb": {Name: "rb"}} + index.allResponses = map[string]*Reference{"#/components/responses/r": {Name: "r"}} + index.allHeaders = map[string]*Reference{"#/components/headers/h": {Name: "h"}} + index.allExamples = map[string]*Reference{"#/components/examples/e": {Name: "e"}} + index.allLinks = map[string]*Reference{"#/components/links/l": {Name: "l"}} + index.allCallbacks = map[string]*Reference{"#/components/callbacks/cb": {Name: "cb"}} + index.allComponentPathItems = map[string]*Reference{"#/components/pathItems/pi": {Name: "pi"}} + index.allSecuritySchemes = &sync.Map{} + index.allSecuritySchemes.Store("#/components/securitySchemes/sec", &Reference{Name: "sec"}) + + tests := []string{ + "#/definitions/Model", + "#/components/securitySchemes/sec", + "#/components/parameters/p", + "#/components/requestBodies/rb", + "#/components/responses/r", + "#/components/headers/h", + "#/components/examples/e", + "#/components/links/l", + "#/components/callbacks/cb", + "#/components/pathItems/pi", + } + + for _, componentID := range tests { + assert.NotNil(t, findDirectComponent(index, componentID, "test.yaml"), componentID) + } + assert.Nil(t, findDirectComponent(index, "#/components/unknown/nope", "test.yaml")) +} + +func TestSpecIndex_FindComponentInRoot_NoRoot(t *testing.T) { + assert.Nil(t, (&SpecIndex{}).FindComponentInRoot(context.Background(), "#/components/schemas/pizza")) +} + +func TestSpecIndex_FindComponentInRoot_NormalizesPrefixedReference(t *testing.T) { + yml := `components: + schemas: + pizza: + type: string + thing name: + type: string` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + + ref := index.FindComponentInRoot(context.Background(), "test.yaml#/components/schemas/pizza") + assert.NotNil(t, ref) + assert.Equal(t, "#/components/schemas/pizza", ref.Definition) + + ref = index.FindComponent(context.Background(), "#/components/schemas/thing%20name") + assert.NotNil(t, ref) + assert.Equal(t, "thing name", ref.Name) +} + +func TestFindComponent_FunctionFallbackEdges(t *testing.T) { + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte("pizza: pie"), &rootNode) + + index := NewTestSpecIndex().Load().(*SpecIndex) + index.allRefs = map[string]*Reference{} + + assert.Nil(t, FindComponent(context.Background(), nil, "#/missing", "", index)) + assert.Nil(t, FindComponent(context.Background(), &rootNode, "#/missing", "", index)) + + rootRef := FindComponent(context.Background(), &rootNode, "#/", "", index) + assert.NotNil(t, rootRef) + assert.Equal(t, "$", rootRef.Path) + + cloned := cloneFoundComponentReference(index, &Reference{}, "#/", "") + assert.Equal(t, "$", cloned.Path) +} + +func BenchmarkFindComponent_DirectComponentFastPath(b *testing.B) { + yml := `components: + schemas: + pizza: + type: object + topping: + type: string` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + var sink atomic.Pointer[Reference] + b.ResetTimer() + for i := 0; i < b.N; i++ { + sink.Store(index.FindComponent(context.Background(), "#/components/schemas/pizza")) + } +} + func TestSpecIndex_TestPathsNodeAsArray(t *testing.T) { yml := `components: schemas: diff --git a/index/utility_methods.go b/index/utility_methods.go index ebb84323c..cf6c46778 100644 --- a/index/utility_methods.go +++ b/index/utility_methods.go @@ -39,7 +39,7 @@ func (index *SpecIndex) extractDefinitionsAndSchemas(schemasNode *yaml.Node, pat Node: schema, Path: fmt.Sprintf("$.components.schemas['%s']", name), ParentNode: schemasNode, - RequiredRefProperties: extractDefinitionRequiredRefProperties(schemasNode, map[string][]string{}, fullDef, index), + RequiredRefProperties: extractDefinitionRequiredRefProperties(schema, map[string][]string{}, fullDef, index), } index.allComponentSchemaDefinitions.Store(def, ref) } @@ -67,7 +67,7 @@ func extractDefinitionRequiredRefProperties(schemaNode *yaml.Node, reqRefProps m _, propertiesMapNode := utils.FindKeyNodeTop("properties", schemaNode.Content) if propertiesMapNode == nil { - // TODO: Log a warning on the resolver, because if you have required properties, but no actual properties, something is wrong + // A schema with required properties but no properties map contributes no required ref edges. return reqRefProps } @@ -383,7 +383,7 @@ func (index *SpecIndex) extractComponentSecuritySchemes(securitySchemesNode *yam KeyNode: keyNode, Path: fmt.Sprintf("$.components.securitySchemes.%s", name), ParentNode: securitySchemesNode, - RequiredRefProperties: extractDefinitionRequiredRefProperties(securitySchemesNode, map[string][]string{}, fullDef, index), + RequiredRefProperties: extractDefinitionRequiredRefProperties(schema, map[string][]string{}, fullDef, index), } index.allSecuritySchemes.Store(def, ref) } diff --git a/index/utility_methods_buffer_test.go b/index/utility_methods_buffer_test.go index 8d911c39e..44b046cd9 100644 --- a/index/utility_methods_buffer_test.go +++ b/index/utility_methods_buffer_test.go @@ -558,6 +558,21 @@ func TestHashNode_InternalNilHandling(t *testing.T) { assert.Equal(t, hash, hash2) } +func TestHashNode_ContentWithNilChild(t *testing.T) { + rootNode := &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + Content: []*yaml.Node{ + nil, + {Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"}, + }, + } + + hash := HashNode(rootNode) + assert.NotEmpty(t, hash) + assert.Equal(t, hash, HashNode(rootNode)) +} + // Test extreme depth scenarios to hit the depth limit checks func TestHashNode_ExtremeDepthLimits(t *testing.T) { // Create a node structure that will definitely hit the >1000 depth limit diff --git a/index/utility_methods_test.go b/index/utility_methods_test.go index c69010783..b6939a3e3 100644 --- a/index/utility_methods_test.go +++ b/index/utility_methods_test.go @@ -8,6 +8,7 @@ import ( "hash/maphash" "net/url" "runtime" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -271,6 +272,64 @@ func Test_extractDefinitionRequiredRefProperties_nil(t *testing.T) { assert.Nil(t, extractDefinitionRequiredRefProperties(nil, nil, "", nil)) } +func TestExtractDefinitionsAndSchemas_UsesSchemaNodeForRequiredRefs(t *testing.T) { + d := `Pet: + type: object + required: + - owner + properties: + owner: + $ref: '#/components/schemas/Owner' +Owner: + type: object` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(d), &rootNode) + + idx := NewTestSpecIndex().Load().(*SpecIndex) + idx.specAbsolutePath = "test.yaml" + idx.allComponentSchemaDefinitions = &sync.Map{} + + idx.extractDefinitionsAndSchemas(rootNode.Content[0], "#/components/schemas/") + + refValue, ok := idx.allComponentSchemaDefinitions.Load("#/components/schemas/Pet") + assert.True(t, ok) + ref := refValue.(*Reference) + assert.Len(t, ref.RequiredRefProperties, 1) + for _, properties := range ref.RequiredRefProperties { + assert.Equal(t, []string{"owner"}, properties) + } +} + +func TestExtractComponentSecuritySchemes_UsesSchemeNodeForRequiredRefs(t *testing.T) { + d := `oauth: + type: object + required: + - token + properties: + token: + $ref: '#/components/securitySchemes/Token' +Token: + type: object` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(d), &rootNode) + + idx := NewTestSpecIndex().Load().(*SpecIndex) + idx.specAbsolutePath = "test.yaml" + idx.allSecuritySchemes = &sync.Map{} + + idx.extractComponentSecuritySchemes(rootNode.Content[0], "#/components/securitySchemes/") + + refValue, ok := idx.allSecuritySchemes.Load("#/components/securitySchemes/oauth") + assert.True(t, ok) + ref := refValue.(*Reference) + assert.Len(t, ref.RequiredRefProperties, 1) + for _, properties := range ref.RequiredRefProperties { + assert.Equal(t, []string{"token"}, properties) + } +} + func TestSyncMapToMap_Nil(t *testing.T) { assert.Nil(t, syncMapToMap[string, string](nil)) } From 39f05e750fbbae8c95d5058e2130fc6ecbde4701 Mon Sep 17 00:00:00 2001 From: quobix Date: Wed, 1 Apr 2026 10:44:11 -0400 Subject: [PATCH 2/9] updated copyright years --- index/extract_refs_inline.go | 2 +- index/extract_refs_lookup.go | 2 +- index/extract_refs_metadata.go | 2 +- index/extract_refs_ref.go | 2 +- index/extract_refs_walk.go | 2 +- index/find_component_build.go | 2 +- index/find_component_direct.go | 2 +- index/find_component_entry.go | 2 +- index/find_component_external.go | 2 +- index/resolver_circular.go | 2 +- index/resolver_entry.go | 2 +- index/resolver_mutation.go | 2 +- index/resolver_paths.go | 2 +- index/resolver_polymorphic.go | 2 +- index/resolver_relatives.go | 2 +- index/resolver_visit.go | 2 +- index/resolver_walk.go | 2 +- index/spec_index_accessors.go | 2 +- index/spec_index_build.go | 2 +- index/spec_index_counts.go | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/index/extract_refs_inline.go b/index/extract_refs_inline.go index 3be039539..3f80fe033 100644 --- a/index/extract_refs_inline.go +++ b/index/extract_refs_inline.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_lookup.go b/index/extract_refs_lookup.go index 49a0756cf..4062daca8 100644 --- a/index/extract_refs_lookup.go +++ b/index/extract_refs_lookup.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_metadata.go b/index/extract_refs_metadata.go index 06d582f61..8349c1f1e 100644 --- a/index/extract_refs_metadata.go +++ b/index/extract_refs_metadata.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_ref.go b/index/extract_refs_ref.go index 45037678e..cc8e070f5 100644 --- a/index/extract_refs_ref.go +++ b/index/extract_refs_ref.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_walk.go b/index/extract_refs_walk.go index 9db301b2f..534657ca3 100644 --- a/index/extract_refs_walk.go +++ b/index/extract_refs_walk.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_build.go b/index/find_component_build.go index 14eb727b9..ff7dfe95b 100644 --- a/index/find_component_build.go +++ b/index/find_component_build.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_direct.go b/index/find_component_direct.go index 41e32b687..000c28367 100644 --- a/index/find_component_direct.go +++ b/index/find_component_direct.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_entry.go b/index/find_component_entry.go index 346ad2bb6..12da39084 100644 --- a/index/find_component_entry.go +++ b/index/find_component_entry.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_external.go b/index/find_component_external.go index 2aab515c8..cf0727b35 100644 --- a/index/find_component_external.go +++ b/index/find_component_external.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/resolver_circular.go b/index/resolver_circular.go index 52bdfd315..8ee923bc5 100644 --- a/index/resolver_circular.go +++ b/index/resolver_circular.go @@ -1,4 +1,4 @@ -// Copyright 2022 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/resolver_entry.go b/index/resolver_entry.go index 24bb5c7e0..9d0df03ae 100644 --- a/index/resolver_entry.go +++ b/index/resolver_entry.go @@ -1,4 +1,4 @@ -// Copyright 2022 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/resolver_mutation.go b/index/resolver_mutation.go index 5185286eb..c5f28e8b7 100644 --- a/index/resolver_mutation.go +++ b/index/resolver_mutation.go @@ -1,4 +1,4 @@ -// Copyright 2022 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/resolver_paths.go b/index/resolver_paths.go index 4f27919d2..01bc0afa3 100644 --- a/index/resolver_paths.go +++ b/index/resolver_paths.go @@ -1,4 +1,4 @@ -// Copyright 2022 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/resolver_polymorphic.go b/index/resolver_polymorphic.go index 104dabbb7..530d83620 100644 --- a/index/resolver_polymorphic.go +++ b/index/resolver_polymorphic.go @@ -1,4 +1,4 @@ -// Copyright 2022 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/resolver_relatives.go b/index/resolver_relatives.go index 9d0acf7af..f3c861339 100644 --- a/index/resolver_relatives.go +++ b/index/resolver_relatives.go @@ -1,4 +1,4 @@ -// Copyright 2022 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/resolver_visit.go b/index/resolver_visit.go index 41a399317..82a63fd65 100644 --- a/index/resolver_visit.go +++ b/index/resolver_visit.go @@ -1,4 +1,4 @@ -// Copyright 2022 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/resolver_walk.go b/index/resolver_walk.go index ce431c7de..b70636fb1 100644 --- a/index/resolver_walk.go +++ b/index/resolver_walk.go @@ -1,4 +1,4 @@ -// Copyright 2022 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/spec_index_accessors.go b/index/spec_index_accessors.go index c3cad9085..9cf1765b2 100644 --- a/index/spec_index_accessors.go +++ b/index/spec_index_accessors.go @@ -1,4 +1,4 @@ -// Copyright 2022-2033 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/spec_index_build.go b/index/spec_index_build.go index f6f8e1f40..1e51d282f 100644 --- a/index/spec_index_build.go +++ b/index/spec_index_build.go @@ -1,4 +1,4 @@ -// Copyright 2022-2033 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index diff --git a/index/spec_index_counts.go b/index/spec_index_counts.go index 6538b8556..8c822e721 100644 --- a/index/spec_index_counts.go +++ b/index/spec_index_counts.go @@ -1,4 +1,4 @@ -// Copyright 2022-2033 Dave Shanley / Quobix +// Copyright 2022-2026 Dave Shanley / Quobix // SPDX-License-Identifier: MIT package index From 1df5376279fd8e8d272c681146abbbf7fa8c493f Mon Sep 17 00:00:00 2001 From: quobix Date: Wed, 1 Apr 2026 10:54:16 -0400 Subject: [PATCH 3/9] updating docs and comments and fixing the main build. builds are not triggering. --- index/extract_refs_inline.go | 2 +- index/extract_refs_lookup.go | 2 +- index/extract_refs_metadata.go | 9 +++- index/extract_refs_ref.go | 12 +++++- index/extract_refs_walk.go | 4 +- index/find_component_build.go | 6 ++- index/find_component_direct.go | 2 +- index/find_component_entry.go | 2 +- index/find_component_external.go | 2 +- index/index_model.go | 28 +++++++++---- index/index_utils.go | 2 +- index/map_index_nodes.go | 4 +- index/resolve_refs_node.go | 6 +++ index/resolver_circular.go | 7 ++++ index/resolver_entry.go | 8 ++-- index/resolver_paths.go | 14 +++++++ index/resolver_relatives.go | 10 +++++ index/resolver_visit.go | 3 ++ index/rolodex.go | 2 +- index/rolodex_file_loader.go | 4 +- index/rolodex_ref_extractor.go | 4 ++ index/rolodex_remote_loader.go | 2 +- index/search_index.go | 23 +++++++++-- index/spec_index_accessors.go | 71 ++++++++++++++++++++++++++++++++ index/spec_index_build.go | 19 ++++++++- index/spec_index_counts.go | 23 +++++++++++ index/utility_methods.go | 2 + 27 files changed, 240 insertions(+), 33 deletions(-) diff --git a/index/extract_refs_inline.go b/index/extract_refs_inline.go index 3f80fe033..3be039539 100644 --- a/index/extract_refs_inline.go +++ b/index/extract_refs_inline.go @@ -1,4 +1,4 @@ -// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_lookup.go b/index/extract_refs_lookup.go index 4062daca8..49a0756cf 100644 --- a/index/extract_refs_lookup.go +++ b/index/extract_refs_lookup.go @@ -1,4 +1,4 @@ -// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_metadata.go b/index/extract_refs_metadata.go index 8349c1f1e..19f2026b1 100644 --- a/index/extract_refs_metadata.go +++ b/index/extract_refs_metadata.go @@ -1,4 +1,4 @@ -// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index @@ -116,16 +116,23 @@ func (index *SpecIndex) collectSecurityRequirementMetadata(node *yaml.Node, keyI if index.securityRequirementRefs == nil { index.securityRequirementRefs = make(map[string]map[string][]*Reference) } + // Security requirements are an array of objects. Each object maps a security scheme + // name (key) to an array of required scopes (value). For example: + // security: + // - oauth2: ["read", "write"] <-- k=0, scheme="oauth2", scopes=["read","write"] + // apiKey: [] <-- same k, scheme="apiKey", scopes=[] securityNode := metadataValueNode(node, keyIndex) if securityNode == nil || !utils.IsNodeArray(securityNode) { return } var secKey string for k := range securityNode.Content { + // Outer loop: each security requirement object in the array. if !utils.IsNodeMap(securityNode.Content[k]) { continue } for g := range securityNode.Content[k].Content { + // Inner loop: key-value pairs within a single requirement object. if g%2 == 0 { secKey = securityNode.Content[k].Content[g].Value continue diff --git a/index/extract_refs_ref.go b/index/extract_refs_ref.go index cc8e070f5..a60cd45fa 100644 --- a/index/extract_refs_ref.go +++ b/index/extract_refs_ref.go @@ -1,4 +1,4 @@ -// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index @@ -207,17 +207,22 @@ func (index *SpecIndex) resolveReferenceTarget(value string) (string, string) { var componentName string var fullDefinitionPath string if len(uri) == 2 { + // Reference contains a fragment (e.g. "file.yaml#/components/schemas/Foo" or "#/definitions/Bar"). if uri[0] == "" { + // Fragment-only local ref — prefix with the spec's absolute path. fullDefinitionPath = fmt.Sprintf("%s#/%s", index.specAbsolutePath, uri[1]) componentName = value } else { if strings.HasPrefix(uri[0], "http") { + // Absolute HTTP URL with fragment — use as-is. fullDefinitionPath = value componentName = fmt.Sprintf("#/%s", uri[1]) } else if filepath.IsAbs(uri[0]) { + // Absolute local file path with fragment — use as-is. fullDefinitionPath = value componentName = fmt.Sprintf("#/%s", uri[1]) } else if index.config.BaseURL != nil && !filepath.IsAbs(defRoot) { + // Relative path with a configured BaseURL — resolve against the base URL. var u url.URL if strings.HasPrefix(defRoot, "http") { up, _ := url.Parse(defRoot) @@ -231,15 +236,19 @@ func (index *SpecIndex) resolveReferenceTarget(value string) (string, string) { fullDefinitionPath = fmt.Sprintf("%s#/%s", u.String(), uri[1]) componentName = fmt.Sprintf("#/%s", uri[1]) } else { + // Relative local file path — resolve against the spec's directory. abs := index.resolveRelativeFilePath(defRoot, uri[0]) fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1]) componentName = fmt.Sprintf("#/%s", uri[1]) } } } else if strings.HasPrefix(uri[0], "http") { + // No fragment, absolute HTTP URL — use as-is. fullDefinitionPath = value } else if !strings.Contains(uri[0], "#") { + // No fragment, not a bare anchor — whole-file reference or relative path. if strings.HasPrefix(defRoot, "http") { + // Spec root is remote — resolve the relative path against the remote URL. if !filepath.IsAbs(uri[0]) { u, _ := url.Parse(defRoot) pathDir := filepath.Dir(u.Path) @@ -249,6 +258,7 @@ func (index *SpecIndex) resolveReferenceTarget(value string) (string, string) { fullDefinitionPath = u.String() } } else if !filepath.IsAbs(uri[0]) { + // Relative local file path — resolve against BaseURL if configured, else the spec's directory. if index.config.BaseURL != nil { u := *index.config.BaseURL abs := utils.CheckPathOverlap(u.Path, uri[0], string(os.PathSeparator)) diff --git a/index/extract_refs_walk.go b/index/extract_refs_walk.go index 534657ca3..31554c6d8 100644 --- a/index/extract_refs_walk.go +++ b/index/extract_refs_walk.go @@ -1,4 +1,4 @@ -// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index @@ -71,6 +71,8 @@ func (index *SpecIndex) walkExtractRefs(node, parent *yaml.Node, state *extractR found = append(found, index.walkChildExtractRefs(n, node, state)...) } + // In YAML mapping nodes, Content alternates key-value: even indices (0, 2, 4...) + // are keys, odd indices (1, 3, 5...) are values. if i%2 == 0 { if stop := index.handleExtractRefsKey(node, parent, state, i, &found); stop { continue diff --git a/index/find_component_build.go b/index/find_component_build.go index ff7dfe95b..7be916f6e 100644 --- a/index/find_component_build.go +++ b/index/find_component_build.go @@ -1,4 +1,4 @@ -// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index @@ -14,6 +14,10 @@ func cloneFoundComponentReference(index *SpecIndex, found *Reference, componentI return buildResolvedComponentReference(index, found, componentID, absoluteFilePath, found.Name, found.Path, found.Node) } +// buildResolvedComponentReference constructs a fully resolved Reference for a component. +// source is the original unresolved ref (may be nil for fresh lookups); componentID is the +// JSON Pointer fragment (e.g. "#/components/schemas/Pet"); absoluteFilePath is the file +// or URL where the component lives. func buildResolvedComponentReference( index *SpecIndex, source *Reference, diff --git a/index/find_component_direct.go b/index/find_component_direct.go index 000c28367..41e32b687 100644 --- a/index/find_component_direct.go +++ b/index/find_component_direct.go @@ -1,4 +1,4 @@ -// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_entry.go b/index/find_component_entry.go index 12da39084..346ad2bb6 100644 --- a/index/find_component_entry.go +++ b/index/find_component_entry.go @@ -1,4 +1,4 @@ -// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_external.go b/index/find_component_external.go index cf0727b35..2aab515c8 100644 --- a/index/find_component_external.go +++ b/index/find_component_external.go @@ -1,4 +1,4 @@ -// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/index_model.go b/index/index_model.go index 65117fb7a..46a5c386f 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -18,8 +18,9 @@ import ( "go.yaml.in/yaml/v4" ) -// Reference is a wrapper around *yaml.Node results to make things more manageable when performing -// algorithms on data models. the *yaml.Node def is just a bit too low level for tracking state. +// Reference is a wrapper around *yaml.Node that tracks a single $ref usage in a specification. +// It captures the full definition path, the resolved node, parent context, circular reference state, +// and sibling properties. Used throughout the index for reference resolution and change detection. type Reference struct { FullDefinition string `json:"fullDefinition,omitempty"` Definition string `json:"definition,omitempty"` @@ -46,7 +47,8 @@ type Reference struct { In string `json:"-"` // parameter location (path, query, header, cookie) - cached for performance } -// ReferenceMapped is a helper struct for mapped references put into sequence (we lose the key) +// ReferenceMapped is a helper struct that pairs a mapped reference with its original definition key, +// preserving insertion order when references are sequenced from a map. type ReferenceMapped struct { OriginalReference *Reference `json:"originalReference,omitempty"` Reference *Reference `json:"reference,omitempty"` @@ -99,7 +101,7 @@ type SpecIndexConfig struct { // If resolving remotely, the RemoteURLHandler will be used to fetch the remote document. // If not set, the default http client will be used. // Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132 - // deprecated: Use the Rolodex instead + // Deprecated: Use the Rolodex instead. RemoteURLHandler func(url string) (*http.Response, error) // FSHandler is an entity that implements the `fs.FS` interface that will be used to fetch local or remote documents. @@ -110,11 +112,11 @@ type SpecIndexConfig struct { // the document. This is really useful if your application has a custom file system or uses a database for storing // documents. // - // Is the FSHandler is set, it will be used for all lookups, regardless of whether they are local or remote. - // it also overrides the RemoteURLHandler if set. + // If the FSHandler is set, it will be used for all lookups, regardless of whether they are local or remote. + // It also overrides the RemoteURLHandler if set. // - // Resolves[#85] https://github.com/pb33f/libopenapi/issues/85 - // deprecated: Use the Rolodex instead + // Resolves [#85]: https://github.com/pb33f/libopenapi/issues/85 + // Deprecated: Use the Rolodex instead. FSHandler fs.FS // If resolving locally, the BasePath will be the root from which relative references will be resolved from @@ -428,6 +430,7 @@ func (index *SpecIndex) GetResolver() *Resolver { return index.resolver } +// SetResolver sets the resolver for this index. func (index *SpecIndex) SetResolver(resolver *Resolver) { index.resolverLock.Lock() defer index.resolverLock.Unlock() @@ -439,10 +442,12 @@ func (index *SpecIndex) GetConfig() *SpecIndexConfig { return index.config } +// GetNodeMap returns the line-to-column-to-node map built during indexing. func (index *SpecIndex) GetNodeMap() map[int]map[int]*yaml.Node { return index.nodeMap } +// GetCache returns the reference lookup cache used during resolution. func (index *SpecIndex) GetCache() *sync.Map { return index.cache } @@ -614,7 +619,8 @@ func (index *SpecIndex) GetSpecAbsolutePath() string { // URI based document. Decides if the reference is local, remote or in a file. type ExternalLookupFunction func(id string) (foundNode *yaml.Node, rootNode *yaml.Node, lookupError error) -// IndexingError holds data about something that went wrong during indexing. +// IndexingError holds data about something that went wrong during indexing, including the +// offending node and its path within the specification. type IndexingError struct { Err error Node *yaml.Node @@ -622,6 +628,7 @@ type IndexingError struct { Path string } +// Error returns the underlying error message. func (i *IndexingError) Error() string { return i.Err.Error() } @@ -636,6 +643,8 @@ type DescriptionReference struct { IsSummary bool } +// EnumReference holds data about an enum definition found during indexing, including its +// type, schema node, and location path within the specification. type EnumReference struct { Node *yaml.Node KeyNode *yaml.Node @@ -645,6 +654,7 @@ type EnumReference struct { ParentNode *yaml.Node } +// ObjectReference holds data about an object with properties found during indexing. type ObjectReference struct { Node *yaml.Node KeyNode *yaml.Node diff --git a/index/index_utils.go b/index/index_utils.go index 12b209c50..84fdbeeca 100644 --- a/index/index_utils.go +++ b/index/index_utils.go @@ -28,7 +28,7 @@ func isHttpMethod(val string) bool { return false } -func boostrapIndexCollections(index *SpecIndex) { +func bootstrapIndexCollections(index *SpecIndex) { index.allRefs = make(map[string]*Reference) index.allMappedRefs = make(map[string]*Reference) index.refsByLine = make(map[string]map[int]bool) diff --git a/index/map_index_nodes.go b/index/map_index_nodes.go index caa77c537..4f8ecde50 100644 --- a/index/map_index_nodes.go +++ b/index/map_index_nodes.go @@ -17,7 +17,7 @@ type NodeOrigin struct { // ValueNode is the value node of the node in question, if has a different origin ValueNode *yaml.Node `json:"-"` - // Line is yhe original line of where the node was found in the original file + // Line is the original line of where the node was found in the original file Line int `json:"line" yaml:"line"` // Column is the original column of where the node was found in the original file @@ -26,7 +26,7 @@ type NodeOrigin struct { // LineValue is the line of the value (if the origin of the key and value are different) LineValue int `json:"lineValue,omitempty" yaml:"lineValue,omitempty"` - // ColumnValue is the line of the value (if the origin of the key and value are different) + // ColumnValue is the column of the value (if the origin of the key and value are different) ColumnValue int `json:"columnKey,omitempty" yaml:"columnKey,omitempty"` // AbsoluteLocation is the absolute path to the reference was extracted from. diff --git a/index/resolve_refs_node.go b/index/resolve_refs_node.go index eb0724160..e1d241a83 100644 --- a/index/resolve_refs_node.go +++ b/index/resolve_refs_node.go @@ -36,6 +36,7 @@ func resolveRefsInNode(node *yaml.Node, idx *SpecIndex, seen map[string]struct{} } } +// resolveRefsInMappingNode handles $ref resolution for a single mapping node, including sibling merging. func resolveRefsInMappingNode(node *yaml.Node, idx *SpecIndex, seen map[string]struct{}) *yaml.Node { ref, hasRef := findRefInMappingNode(node) if !hasRef { @@ -81,6 +82,7 @@ func resolveRefsInMappingNode(node *yaml.Node, idx *SpecIndex, seen map[string]s return cloneMappingNodeWithResolvedChildren(node, idx, seen) } +// hasNonRefSiblings returns true if the mapping node contains keys other than "$ref". func hasNonRefSiblings(node *yaml.Node) bool { for i := 0; i+1 < len(node.Content); i += 2 { key := node.Content[i] @@ -91,6 +93,7 @@ func hasNonRefSiblings(node *yaml.Node) bool { return false } +// findRefInMappingNode extracts the "$ref" value from a mapping node, if present. func findRefInMappingNode(node *yaml.Node) (string, bool) { for i := 0; i+1 < len(node.Content); i += 2 { key := node.Content[i] @@ -102,6 +105,7 @@ func findRefInMappingNode(node *yaml.Node) (string, bool) { return "", false } +// extractResolvedSiblingPairs collects all non-$ref key-value pairs from a mapping, resolving their values. func extractResolvedSiblingPairs(node *yaml.Node, idx *SpecIndex, seen map[string]struct{}) []*yaml.Node { out := make([]*yaml.Node, 0, len(node.Content)) for i := 0; i+1 < len(node.Content); i += 2 { @@ -115,6 +119,7 @@ func extractResolvedSiblingPairs(node *yaml.Node, idx *SpecIndex, seen map[strin return out } +// cloneMappingNodeWithResolvedChildren shallow-clones a mapping node, recursively resolving each child value. func cloneMappingNodeWithResolvedChildren(node *yaml.Node, idx *SpecIndex, seen map[string]struct{}) *yaml.Node { clone := *node clone.Content = make([]*yaml.Node, 0, len(node.Content)) @@ -126,6 +131,7 @@ func cloneMappingNodeWithResolvedChildren(node *yaml.Node, idx *SpecIndex, seen return &clone } +// mergeResolvedMappingWithSiblings combines a resolved mapping with sibling key-value pairs; siblings win on conflict. func mergeResolvedMappingWithSiblings(resolved *yaml.Node, siblings []*yaml.Node) *yaml.Node { merged := *resolved merged.Content = make([]*yaml.Node, 0, len(resolved.Content)+len(siblings)) diff --git a/index/resolver_circular.go b/index/resolver_circular.go index 8ee923bc5..836f4f42b 100644 --- a/index/resolver_circular.go +++ b/index/resolver_circular.go @@ -91,17 +91,24 @@ func (resolver *Resolver) relativeIsArrayResult(relative *Reference) bool { func (resolver *Resolver) isInfiniteCircularDependency( ref *Reference, visitedDefinitions map[string]bool, initialRef *Reference, ) (bool, map[string]bool) { + // Recursive DFS: walks all required $ref properties of ref, tracking visited + // definitions to detect cycles. initialRef anchors the starting point so we + // can recognize when the chain loops back to the origin. if ref == nil { return false, visitedDefinitions } for refDefinition := range ref.RequiredRefProperties { r, _ := resolver.specIndex.SearchIndexForReference(refDefinition) + + // Direct loop back to the original starting reference — infinite cycle. if initialRef != nil && initialRef.FullDefinition == r.FullDefinition { return true, visitedDefinitions } + // Self-reference: ref points back to itself. if len(visitedDefinitions) > 0 && ref.FullDefinition == r.FullDefinition { return true, visitedDefinitions } + // Already visited in this DFS path — skip to avoid re-processing. if visitedDefinitions[r.FullDefinition] { continue } diff --git a/index/resolver_entry.go b/index/resolver_entry.go index 9d0df03ae..c2c90ce43 100644 --- a/index/resolver_entry.go +++ b/index/resolver_entry.go @@ -18,7 +18,7 @@ type ResolvingError struct { Node *yaml.Node Path string - // CircularReference is set if the error is a reference to the circular reference. + // CircularReference is the detected circular reference result, if this error relates to one. CircularReference *CircularReferenceResult } @@ -42,8 +42,8 @@ func (r *ResolvingError) Error() string { return strings.Join(msgs, "\n") } -// Resolver will use a *index.SpecIndex to stitch together a resolved root tree using all the discovered -// references in the doc. +// Resolver uses a SpecIndex to stitch together a resolved root tree from all discovered references, +// detecting circular references and resolving polymorphic relationships along the way. type Resolver struct { specIndex *SpecIndex resolvedRoot *yaml.Node @@ -193,6 +193,8 @@ func (resolver *Resolver) Resolve() []*ResolvingError { return resolver.resolvingErrors } +// CheckForCircularReferences walks all references without resolving them, detecting circular +// reference chains. Returns any resolving errors found, including infinite circular loops. func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError { visitIndexWithoutDamagingIt(resolver, resolver.specIndex) for _, circRef := range resolver.circularReferences { diff --git a/index/resolver_paths.go b/index/resolver_paths.go index 01bc0afa3..dc7d49e7a 100644 --- a/index/resolver_paths.go +++ b/index/resolver_paths.go @@ -17,10 +17,14 @@ func (resolver *Resolver) buildDefPath(ref *Reference, l string) string { def := "" exp := strings.Split(l, "#/") if len(exp) == 2 { + // Reference contains a fragment (e.g. "file.yaml#/components/schemas/Foo" or "#/definitions/Bar"). if exp[0] != "" { + // Has a file/URL portion before the fragment. if !strings.HasPrefix(exp[0], "http") { if !filepath.IsAbs(exp[0]) { + // Relative file path — resolve against the parent ref's location. if strings.HasPrefix(ref.FullDefinition, "http") { + // Parent is a remote URL: resolve relative path against the URL's directory. u, _ := url.Parse(ref.FullDefinition) p, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), exp[0], string(filepath.Separator))) u.Path = utils.ReplaceWindowsDriveWithLinuxPath(p) @@ -29,6 +33,7 @@ func (resolver *Resolver) buildDefPath(ref *Reference, l string) string { def = u.String() + "#/" + exp[1] } } else { + // Parent is a local file path: resolve relative to the parent's directory. z := strings.Split(ref.FullDefinition, "#/") if len(z) == 2 { if len(z[0]) > 0 { @@ -45,29 +50,37 @@ func (resolver *Resolver) buildDefPath(ref *Reference, l string) string { } } } else if len(exp[1]) > 0 { + // Absolute HTTP URL with a fragment — use as-is. def = l } else { + // HTTP URL with no fragment content — use just the URL part. def = exp[0] } } else if strings.HasPrefix(ref.FullDefinition, "http") { + // Fragment-only ref (e.g. "#/components/schemas/Foo") with a remote parent. u, _ := url.Parse(ref.FullDefinition) u.Fragment = "" def = u.String() + "#/" + exp[1] } else if strings.HasPrefix(ref.FullDefinition, "#/") { + // Fragment-only ref with a fragment-only parent — keep as local fragment. def = "#/" + exp[1] } else { + // Fragment-only ref with a local file parent — prepend the file portion. fdexp := strings.Split(ref.FullDefinition, "#/") def = fdexp[0] + "#/" + exp[1] } } else if strings.HasPrefix(l, "http") { + // No fragment, absolute HTTP URL — use as-is. def = l } else if strings.HasPrefix(ref.FullDefinition, "http") { + // No fragment, relative path with a remote parent — resolve against the URL. u, _ := url.Parse(ref.FullDefinition) abs, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), l, string(filepath.Separator))) u.Path = utils.ReplaceWindowsDriveWithLinuxPath(abs) u.Fragment = "" def = u.String() } else { + // No fragment, local relative path — resolve against the parent's directory. lookupRef := strings.Split(ref.FullDefinition, "#/") def = resolver.resolveLocalRefPath(filepath.Dir(lookupRef[0]), l) } @@ -112,6 +125,7 @@ func (resolver *Resolver) resolveSchemaIdBase(parentBase string, node *yaml.Node return resolved } +// ResolvePendingNodes applies deferred node content replacements that were collected during resolution. func (resolver *Resolver) ResolvePendingNodes() { for _, r := range resolver.specIndex.pendingResolve { r.ref.Node.Content = r.nodes diff --git a/index/resolver_relatives.go b/index/resolver_relatives.go index f3c861339..ca33a60d9 100644 --- a/index/resolver_relatives.go +++ b/index/resolver_relatives.go @@ -32,6 +32,8 @@ func (resolver *Resolver) extractRelativesWithState( node, parent *yaml.Node, state relativeWalkState, ) []*Reference { + // Guard against stack overflow from deeply nested or circular specs. + // Journey tracks the reference chain (100 max); depth tracks recursive calls (500 max). if len(state.journey) > 100 { return nil } @@ -208,11 +210,15 @@ func (resolver *Resolver) buildRelativeLookupDefinitions(ref *Reference, value, fullDef := "" exp := strings.Split(value, "#/") if len(exp) == 2 { + // Reference contains a fragment (e.g. "other.yaml#/components/schemas/Foo"). definition = fmt.Sprintf("#/%s", exp[1]) if exp[0] != "" { + // Has a file/URL prefix before the fragment. if strings.HasPrefix(exp[0], "http") { + // Absolute HTTP URL — use as-is. fullDef = value } else if strings.HasPrefix(ref.FullDefinition, "http") { + // Relative file path, but the parent ref is remote — resolve against the URL. httpExp := strings.Split(ref.FullDefinition, "#/") u, _ := url.Parse(httpExp[0]) abs, _ := filepath.Abs(utils.CheckPathOverlap(path.Dir(u.Path), exp[0], string(filepath.Separator))) @@ -220,11 +226,13 @@ func (resolver *Resolver) buildRelativeLookupDefinitions(ref *Reference, value, u.Fragment = "" fullDef = fmt.Sprintf("%s#/%s", u.String(), exp[1]) } else { + // Relative file path with a local parent — resolve against the parent's directory. fileDef := strings.Split(ref.FullDefinition, "#/") abs := resolver.resolveLocalRefPath(filepath.Dir(fileDef[0]), exp[0]) fullDef = fmt.Sprintf("%s#/%s", abs, exp[1]) } } else { + // Fragment-only ref (e.g. "#/definitions/Bar") — resolve against the parent's base location. baseLocation := ref.FullDefinition if ref.RemoteLocation != "" { baseLocation = ref.RemoteLocation @@ -239,8 +247,10 @@ func (resolver *Resolver) buildRelativeLookupDefinitions(ref *Reference, value, } } } else if strings.HasPrefix(value, "http") { + // No fragment, absolute HTTP URL — use as-is. fullDef = value } else { + // No fragment, relative file path — resolve against the parent's base location. baseLocation := ref.FullDefinition if ref.RemoteLocation != "" { baseLocation = ref.RemoteLocation diff --git a/index/resolver_visit.go b/index/resolver_visit.go index 82a63fd65..3b0916bcd 100644 --- a/index/resolver_visit.go +++ b/index/resolver_visit.go @@ -149,6 +149,9 @@ func (resolver *Resolver) searchReferenceWithContext(sourceRef, searchRef *Refer return searchIndex.SearchIndexForReferenceByReferenceWithContext(ctx, searchRef) } +// VisitReference visits a single reference, collecting its relatives (dependencies) and recursively +// visiting them. The seen map prevents infinite loops, journey tracks the path for circular detection, +// and resolve controls whether nodes are actually resolved or just visited for analysis. func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, journey []*Reference, resolve bool) []*yaml.Node { resolver.referencesVisited++ if content, done := resolver.visitReferenceShortCircuit(ref, resolve); done { diff --git a/index/rolodex.go b/index/rolodex.go index 2f1ed6d49..12523b1ca 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -257,7 +257,7 @@ func (r *Rolodex) GetConfig() *SpecIndexConfig { return r.indexConfig } -// GetRootNode returns the root index of the rolodex (the entry point, the main document) +// GetRootNode returns the root node of the rolodex (the entry point, the main document) func (r *Rolodex) GetRootNode() *yaml.Node { return r.rootNode } diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index 16c17bfc8..79a94b560 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -242,7 +242,7 @@ func (l *LocalFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { return l.IndexWithContext(context.Background(), config) } -// // IndexWithContext returns the *SpecIndex for the file. If the index has not been created, it will be created (indexed), also supplied context +// IndexWithContext returns the *SpecIndex for the file. If the index has not been created, it will be created (indexed), also supplied context func (l *LocalFile) IndexWithContext(ctx context.Context, config *SpecIndexConfig) (*SpecIndex, error) { var result *SpecIndex var resultErr error @@ -438,7 +438,7 @@ func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) { return err } - // we don't care about directories, or errors, just read everything we can. + // skip non-matching directories, process all readable files. if d.IsDir() { if d.Name() != config.BaseDirectory { return nil diff --git a/index/rolodex_ref_extractor.go b/index/rolodex_ref_extractor.go index 3ff9882b7..a11b97d31 100644 --- a/index/rolodex_ref_extractor.go +++ b/index/rolodex_ref_extractor.go @@ -8,14 +8,18 @@ import ( "strings" ) +// RefType identifies where a reference points to: within the same file (Local), +// to another file on disk (File), or to a remote URL (HTTP). const ( Local RefType = iota File HTTP ) +// RefType is an enum identifying the location type of a reference. type RefType int +// ExtractedRef represents a parsed reference with its resolved location and type. type ExtractedRef struct { Location string Type RefType diff --git a/index/rolodex_remote_loader.go b/index/rolodex_remote_loader.go index 37659bb1d..595246ebe 100644 --- a/index/rolodex_remote_loader.go +++ b/index/rolodex_remote_loader.go @@ -587,7 +587,7 @@ func (i *RemoteFS) OpenWithContext(ctx context.Context, remoteURL string) (fs.Fi if remoteParsedURL.Scheme == "" { i.releaseRemoteProcessingWaiter(processingWaiter, cacheKey, nil, nil) - return nil, nil // not a remote file, nothing wrong with that - just we can't keep looking here partner. + return nil, nil // not a remote file — scheme is empty, skip processing. } i.logger.Debug("[rolodex remote loader] loading remote file", "file", remoteURL, "remoteURL", remoteParsedURL.String()) diff --git a/index/search_index.go b/index/search_index.go index a21439951..9e0c92d17 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -78,6 +78,7 @@ func IsFileBeingIndexed(ctx context.Context, filePath string) bool { return false } +// SearchIndexForReferenceByReference searches the index for a matching reference using a background context. func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) (*Reference, *SpecIndex) { r, idx, _ := index.SearchIndexForReferenceByReferenceWithContext(context.Background(), fullRef) return r, idx @@ -90,6 +91,7 @@ func (index *SpecIndex) SearchIndexForReference(ref string) (*Reference, *SpecIn return index.SearchIndexForReferenceByReference(&Reference{FullDefinition: ref}) } +// SearchIndexForReferenceWithContext searches the index for a reference string with context for schema ID tracking. func (index *SpecIndex) SearchIndexForReferenceWithContext(ctx context.Context, ref string) (*Reference, *SpecIndex, context.Context) { return index.SearchIndexForReferenceByReferenceWithContext(ctx, &Reference{FullDefinition: ref}) } @@ -102,6 +104,10 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex } } + // --- Step 1: JSON Schema $id resolution --- + // Resolve the ref against JSON Schema $id values first. Specs using JSON Schema 2020-12 + // register schemas by their $id URI (e.g. "$id: https://example.com/a.json"), so a bare + // ref like "a.json" can match by normalizing it against the current $id base URI scope. schemaIdBase := searchRef.SchemaIdBase if schemaIdBase == "" { if scope := GetSchemaIdScope(ctx); scope != nil && scope.BaseUri != "" { @@ -122,8 +128,7 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex } } - // Try to resolve via JSON Schema 2020-12 $id registry first - // This handles refs like "a.json" resolving to schemas with $id: "https://example.com/a.json" + // Try the $id registry for an exact match, then fall back to path-only matching. if resolved := index.ResolveRefViaSchemaId(normalizedRef); resolved != nil { if index.cache != nil { index.cache.Store(searchRef.FullDefinition, resolved) @@ -151,6 +156,11 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex } } + // --- Step 2: Parse the ref into URI components and build lookup paths --- + // Split the ref on "#/" to separate the file path (uri[0]) from the JSON Pointer + // fragment (uri[1]). Depending on whether the ref is absolute, relative, or HTTP, + // construct `roloLookup` (the file path for rolodex search), `ref` (the primary + // lookup key), and `refAlt` (an alternate absolute-path form of the key). ref := normalizedRef refAlt := ref absPath := index.specAbsolutePath @@ -209,6 +219,9 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex refAlt, _ = url.QueryUnescape(refAlt) } + // --- Step 3: Local index lookup --- + // Search the current index's mapped refs, component schema definitions, and security + // schemes using both the primary key (`ref`) and the alternate absolute form (`refAlt`). if r, ok := index.allMappedRefs[ref]; ok { idx := index.extractIndex(r) index.cache.Store(ref, r) @@ -238,7 +251,11 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex } } - // check the rolodex for the reference. + // --- Step 4: Rolodex / external file lookup --- + // Open the target file via the rolodex (the multi-file filesystem abstraction), then + // search through that file's index for the ref. Handles self-references back to the + // current spec, relative path normalization, inline/ref schema scanning, and + // component-tree walking inside the remote file. if roloLookup != "" { if strings.Contains(roloLookup, "#") { diff --git a/index/spec_index_accessors.go b/index/spec_index_accessors.go index 9cf1765b2..cbce124db 100644 --- a/index/spec_index_accessors.go +++ b/index/spec_index_accessors.go @@ -9,58 +9,77 @@ import ( "go.yaml.in/yaml/v4" ) +// SetCircularReferences sets the circular reference results for this index. func (index *SpecIndex) SetCircularReferences(refs []*CircularReferenceResult) { index.circularReferences = refs } +// GetCircularReferences returns all circular references found during resolution. func (index *SpecIndex) GetCircularReferences() []*CircularReferenceResult { return index.circularReferences } +// GetTagCircularReferences returns circular references found in tag parent hierarchies. func (index *SpecIndex) GetTagCircularReferences() []*CircularReferenceResult { return index.tagCircularReferences } +// SetIgnoredPolymorphicCircularReferences sets circular references that were ignored because they +// involve polymorphic keywords (allOf, oneOf, anyOf). func (index *SpecIndex) SetIgnoredPolymorphicCircularReferences(refs []*CircularReferenceResult) { index.polyCircularReferences = refs } +// SetIgnoredArrayCircularReferences sets circular references that were ignored because they +// involve array items. func (index *SpecIndex) SetIgnoredArrayCircularReferences(refs []*CircularReferenceResult) { index.arrayCircularReferences = refs } +// GetIgnoredPolymorphicCircularReferences returns circular references that were ignored because +// they involve polymorphic keywords. func (index *SpecIndex) GetIgnoredPolymorphicCircularReferences() []*CircularReferenceResult { return index.polyCircularReferences } +// GetIgnoredArrayCircularReferences returns circular references that were ignored because they +// involve array items. func (index *SpecIndex) GetIgnoredArrayCircularReferences() []*CircularReferenceResult { return index.arrayCircularReferences } +// GetPathsNode returns the raw YAML node for the top-level "paths" object. func (index *SpecIndex) GetPathsNode() *yaml.Node { return index.pathsNode } +// GetDiscoveredReferences returns all deduplicated references found during extraction, +// keyed by their full definition path. func (index *SpecIndex) GetDiscoveredReferences() map[string]*Reference { return index.allRefs } +// GetPolyReferences returns all polymorphic references (allOf, oneOf, anyOf) keyed by definition. func (index *SpecIndex) GetPolyReferences() map[string]*Reference { return index.polymorphicRefs } +// GetPolyAllOfReferences returns all references found under allOf keywords. func (index *SpecIndex) GetPolyAllOfReferences() []*Reference { return index.polymorphicAllOfRefs } +// GetPolyAnyOfReferences returns all references found under anyOf keywords. func (index *SpecIndex) GetPolyAnyOfReferences() []*Reference { return index.polymorphicAnyOfRefs } +// GetPolyOneOfReferences returns all references found under oneOf keywords. func (index *SpecIndex) GetPolyOneOfReferences() []*Reference { return index.polymorphicOneOfRefs } +// GetAllCombinedReferences returns a merged map of all standard and polymorphic references. func (index *SpecIndex) GetAllCombinedReferences() map[string]*Reference { combined := make(map[string]*Reference) for k, ref := range index.allRefs { @@ -72,26 +91,33 @@ func (index *SpecIndex) GetAllCombinedReferences() map[string]*Reference { return combined } +// GetRefsByLine returns a map of reference definition to the set of line numbers where it appears. func (index *SpecIndex) GetRefsByLine() map[string]map[int]bool { return index.refsByLine } +// GetLinesWithReferences returns a set of line numbers that contain at least one reference. func (index *SpecIndex) GetLinesWithReferences() map[int]bool { return index.linesWithRefs } +// GetMappedReferences returns all resolved component references keyed by definition path. func (index *SpecIndex) GetMappedReferences() map[string]*Reference { return index.allMappedRefs } +// SetMappedReferences replaces the mapped references for this index. func (index *SpecIndex) SetMappedReferences(mappedRefs map[string]*Reference) { index.allMappedRefs = mappedRefs } +// GetRawReferencesSequenced returns all raw references in the order they were scanned. func (index *SpecIndex) GetRawReferencesSequenced() []*Reference { return index.rawSequencedRefs } +// GetExtensionRefsSequenced returns only references that appear under x-* extension paths, +// in scan order. func (index *SpecIndex) GetExtensionRefsSequenced() []*Reference { var extensionRefs []*Reference for _, ref := range index.rawSequencedRefs { @@ -102,14 +128,17 @@ func (index *SpecIndex) GetExtensionRefsSequenced() []*Reference { return extensionRefs } +// GetMappedReferencesSequenced returns all resolved component references in deterministic order. func (index *SpecIndex) GetMappedReferencesSequenced() []*ReferenceMapped { return index.allMappedRefsSequenced } +// GetOperationParameterReferences returns parameters keyed by path, then HTTP method, then parameter name. func (index *SpecIndex) GetOperationParameterReferences() map[string]map[string]map[string][]*Reference { return index.paramOpRefs } +// GetAllSchemas returns all schemas (component, inline, and reference) sorted by line number. func (index *SpecIndex) GetAllSchemas() []*Reference { componentSchemas := index.GetAllComponentSchemas() inlineSchemas := index.GetAllInlineSchemas() @@ -134,18 +163,23 @@ func (index *SpecIndex) GetAllSchemas() []*Reference { return combined } +// GetAllInlineSchemaObjects returns all inline schema definitions that are objects. func (index *SpecIndex) GetAllInlineSchemaObjects() []*Reference { return index.allInlineSchemaObjectDefinitions } +// GetAllInlineSchemas returns all inline schema definitions found during extraction. func (index *SpecIndex) GetAllInlineSchemas() []*Reference { return index.allInlineSchemaDefinitions } +// GetAllReferenceSchemas returns all schema definitions that are $ref references. func (index *SpecIndex) GetAllReferenceSchemas() []*Reference { return index.allRefSchemaDefinitions } +// GetAllComponentSchemas returns all component schema definitions, converting from the +// internal sync.Map on first access and caching the result. func (index *SpecIndex) GetAllComponentSchemas() map[string]*Reference { if index == nil { return nil @@ -165,134 +199,168 @@ func (index *SpecIndex) GetAllComponentSchemas() map[string]*Reference { return index.allComponentSchemas } +// GetAllSecuritySchemes returns all security scheme definitions from the components section. func (index *SpecIndex) GetAllSecuritySchemes() map[string]*Reference { return syncMapToMap[string, *Reference](index.allSecuritySchemes) } +// GetAllHeaders returns all header definitions from the components section. func (index *SpecIndex) GetAllHeaders() map[string]*Reference { return index.allHeaders } +// GetAllExternalDocuments returns all external document references found in the specification. func (index *SpecIndex) GetAllExternalDocuments() map[string]*Reference { return index.allExternalDocuments } +// GetAllExamples returns all example definitions from the components section. func (index *SpecIndex) GetAllExamples() map[string]*Reference { return index.allExamples } +// GetAllDescriptions returns all description nodes found during indexing. func (index *SpecIndex) GetAllDescriptions() []*DescriptionReference { return index.allDescriptions } +// GetAllEnums returns all enum definitions found during indexing. func (index *SpecIndex) GetAllEnums() []*EnumReference { return index.allEnums } +// GetAllObjectsWithProperties returns all objects that have a "properties" keyword. func (index *SpecIndex) GetAllObjectsWithProperties() []*ObjectReference { return index.allObjectsWithProperties } +// GetAllSummaries returns all summary nodes found during indexing. func (index *SpecIndex) GetAllSummaries() []*DescriptionReference { return index.allSummaries } +// GetAllRequestBodies returns all request body definitions from the components section. func (index *SpecIndex) GetAllRequestBodies() map[string]*Reference { return index.allRequestBodies } +// GetAllLinks returns all link definitions from the components section. func (index *SpecIndex) GetAllLinks() map[string]*Reference { return index.allLinks } +// GetAllParameters returns all parameter definitions from the components section. func (index *SpecIndex) GetAllParameters() map[string]*Reference { return index.allParameters } +// GetAllResponses returns all response definitions from the components section. func (index *SpecIndex) GetAllResponses() map[string]*Reference { return index.allResponses } +// GetAllCallbacks returns all callback definitions from the components section. func (index *SpecIndex) GetAllCallbacks() map[string]*Reference { return index.allCallbacks } +// GetAllComponentPathItems returns all path item definitions from the components section. func (index *SpecIndex) GetAllComponentPathItems() map[string]*Reference { return index.allComponentPathItems } +// GetInlineOperationDuplicateParameters returns parameters with duplicate names found inline in operations. func (index *SpecIndex) GetInlineOperationDuplicateParameters() map[string][]*Reference { return index.paramInlineDuplicateNames } +// GetReferencesWithSiblings returns references that have sibling properties alongside the $ref keyword. func (index *SpecIndex) GetReferencesWithSiblings() map[string]Reference { return index.refsWithSiblings } +// GetAllReferences returns all deduplicated references found during extraction. func (index *SpecIndex) GetAllReferences() map[string]*Reference { return index.allRefs } +// GetAllSequencedReferences returns all raw references in scan order. func (index *SpecIndex) GetAllSequencedReferences() []*Reference { return index.rawSequencedRefs } +// GetSchemasNode returns the raw YAML node for the components/schemas (or definitions) section. func (index *SpecIndex) GetSchemasNode() *yaml.Node { return index.schemasNode } +// GetParametersNode returns the raw YAML node for the components/parameters section. func (index *SpecIndex) GetParametersNode() *yaml.Node { return index.parametersNode } +// GetReferenceIndexErrors returns any errors that occurred during reference extraction. func (index *SpecIndex) GetReferenceIndexErrors() []error { return index.refErrors } +// GetOperationParametersIndexErrors returns any errors found when scanning operation parameters. func (index *SpecIndex) GetOperationParametersIndexErrors() []error { return index.operationParamErrors } +// GetAllPaths returns all path items keyed by path, then HTTP method. func (index *SpecIndex) GetAllPaths() map[string]map[string]*Reference { return index.pathRefs } +// GetOperationTags returns tags keyed by path, then HTTP method. func (index *SpecIndex) GetOperationTags() map[string]map[string][]*Reference { return index.operationTagsRefs } +// GetAllParametersFromOperations returns all parameters keyed by path, HTTP method, then parameter name. func (index *SpecIndex) GetAllParametersFromOperations() map[string]map[string]map[string][]*Reference { return index.paramOpRefs } +// GetRootSecurityReferences returns references from the top-level security requirement array. func (index *SpecIndex) GetRootSecurityReferences() []*Reference { return index.rootSecurity } +// GetSecurityRequirementReferences returns security requirements keyed by security scheme name. func (index *SpecIndex) GetSecurityRequirementReferences() map[string]map[string][]*Reference { return index.securityRequirementRefs } +// GetRootSecurityNode returns the raw YAML node for the top-level "security" array. func (index *SpecIndex) GetRootSecurityNode() *yaml.Node { return index.rootSecurityNode } +// GetRootServersNode returns the raw YAML node for the top-level "servers" array. func (index *SpecIndex) GetRootServersNode() *yaml.Node { return index.rootServersNode } +// GetAllRootServers returns all server references from the top-level "servers" array. func (index *SpecIndex) GetAllRootServers() []*Reference { return index.serversRefs } +// GetAllOperationsServers returns server references keyed by path, then HTTP method. func (index *SpecIndex) GetAllOperationsServers() map[string]map[string][]*Reference { return index.opServersRefs } +// SetAllowCircularReferenceResolving sets whether circular references should be resolved +// instead of returning an error. func (index *SpecIndex) SetAllowCircularReferenceResolving(allow bool) { index.allowCircularReferences = allow } +// AllowCircularReferenceResolving returns whether circular reference resolving is enabled. func (index *SpecIndex) AllowCircularReferenceResolving() bool { return index.allowCircularReferences } @@ -309,6 +377,7 @@ func (index *SpecIndex) checkPolymorphicNode(name string) (bool, string) { return false, "" } +// RegisterSchemaId registers a JSON Schema $id entry in this index's local registry. func (index *SpecIndex) RegisterSchemaId(entry *SchemaIdEntry) error { index.schemaIdRegistryLock.Lock() defer index.schemaIdRegistryLock.Unlock() @@ -319,6 +388,7 @@ func (index *SpecIndex) RegisterSchemaId(entry *SchemaIdEntry) error { return err } +// GetSchemaById looks up a schema by its resolved $id URI in this index's local registry. func (index *SpecIndex) GetSchemaById(uri string) *SchemaIdEntry { index.schemaIdRegistryLock.RLock() defer index.schemaIdRegistryLock.RUnlock() @@ -328,6 +398,7 @@ func (index *SpecIndex) GetSchemaById(uri string) *SchemaIdEntry { return index.schemaIdRegistry[uri] } +// GetAllSchemaIds returns a copy of all $id entries registered in this index. func (index *SpecIndex) GetAllSchemaIds() map[string]*SchemaIdEntry { index.schemaIdRegistryLock.RLock() defer index.schemaIdRegistryLock.RUnlock() diff --git a/index/spec_index_build.go b/index/spec_index_build.go index 1e51d282f..9df192ecd 100644 --- a/index/spec_index_build.go +++ b/index/spec_index_build.go @@ -19,9 +19,11 @@ const ( theoreticalRoot = "root.yaml" ) +// NewSpecIndexWithConfigAndContext creates a new SpecIndex from the given root YAML node and configuration. +// The context is passed through to reference extraction for schema ID scope tracking. func NewSpecIndexWithConfigAndContext(ctx context.Context, rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex { index := new(SpecIndex) - boostrapIndexCollections(index) + bootstrapIndexCollections(index) index.InitHighCache() index.config = config index.rolodex = config.Rolodex @@ -39,16 +41,19 @@ func NewSpecIndexWithConfigAndContext(ctx context.Context, rootNode *yaml.Node, return createNewIndex(ctx, rootNode, index, config.AvoidBuildIndex) } +// NewSpecIndexWithConfig creates a new SpecIndex from the given root YAML node and configuration, +// using a background context. func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex { return NewSpecIndexWithConfigAndContext(context.Background(), rootNode, config) } +// NewSpecIndex creates a new SpecIndex with default configuration from the given root YAML node. func NewSpecIndex(rootNode *yaml.Node) *SpecIndex { index := new(SpecIndex) index.InitHighCache() index.config = CreateOpenAPIIndexConfig() index.root = rootNode - boostrapIndexCollections(index) + bootstrapIndexCollections(index) return createNewIndex(context.Background(), rootNode, index, false) } @@ -99,6 +104,8 @@ func createNewIndex(ctx context.Context, rootNode *yaml.Node, index *SpecIndex, return index } +// BuildIndex runs all count and extraction functions concurrently to populate the index. +// This is called automatically during construction unless AvoidBuildIndex is set in the config. func (index *SpecIndex) BuildIndex() { if index.built { return @@ -132,26 +139,33 @@ func (index *SpecIndex) BuildIndex() { index.built = true } +// GetLogger returns the structured logger used by this index. func (index *SpecIndex) GetLogger() *slog.Logger { return index.logger } +// GetRootNode returns the root YAML node of the specification document. func (index *SpecIndex) GetRootNode() *yaml.Node { return index.root } +// SetRootNode sets the root YAML node for this index. func (index *SpecIndex) SetRootNode(node *yaml.Node) { index.root = node } +// GetRolodex returns the Rolodex file system abstraction associated with this index. func (index *SpecIndex) GetRolodex() *Rolodex { return index.rolodex } +// SetRolodex sets the Rolodex file system abstraction for this index. func (index *SpecIndex) SetRolodex(rolodex *Rolodex) { index.rolodex = rolodex } +// GetSpecFileName returns the base filename of the specification (e.g. "openapi.yaml"). +// Falls back to "root.yaml" if no file path is configured. func (index *SpecIndex) GetSpecFileName() string { if index == nil || index.rolodex == nil || index.rolodex.indexConfig == nil || index.rolodex.indexConfig.SpecFilePath == "" { return theoreticalRoot @@ -159,6 +173,7 @@ func (index *SpecIndex) GetSpecFileName() string { return filepath.Base(index.rolodex.indexConfig.SpecFilePath) } +// GetGlobalTagsNode returns the raw YAML node for the top-level "tags" array. func (index *SpecIndex) GetGlobalTagsNode() *yaml.Node { return index.tagsNode } diff --git a/index/spec_index_counts.go b/index/spec_index_counts.go index 8c822e721..37e1280e6 100644 --- a/index/spec_index_counts.go +++ b/index/spec_index_counts.go @@ -12,6 +12,7 @@ import ( "go.yaml.in/yaml/v4" ) +// GetPathCount returns the number of paths defined in the specification. Returns -1 if root is nil. func (index *SpecIndex) GetPathCount() int { if index.root == nil { return -1 @@ -31,6 +32,8 @@ func (index *SpecIndex) GetPathCount() int { return pc } +// ExtractExternalDocuments recursively searches the YAML tree for externalDocs objects and returns +// references to each one found. func (index *SpecIndex) ExtractExternalDocuments(node *yaml.Node) []*Reference { if node == nil { return nil @@ -56,6 +59,8 @@ func (index *SpecIndex) ExtractExternalDocuments(node *yaml.Node) []*Reference { return found } +// GetGlobalTagsCount returns the number of top-level tags and also extracts tag references +// and checks for circular parent references. Returns -1 if root is nil. func (index *SpecIndex) GetGlobalTagsCount() int { if index.root == nil { return -1 @@ -173,6 +178,7 @@ func (index *SpecIndex) detectTagCircularHelper(tagName string, parentMap map[st return []string{} } +// GetOperationTagsCount returns the number of unique tags referenced across all operations. func (index *SpecIndex) GetOperationTagsCount() int { if index.root == nil { return -1 @@ -196,6 +202,7 @@ func (index *SpecIndex) GetOperationTagsCount() int { return count } +// GetTotalTagsCount returns the combined count of unique global and operation tags. func (index *SpecIndex) GetTotalTagsCount() int { if index.root == nil { return -1 @@ -225,6 +232,7 @@ func (index *SpecIndex) GetTotalTagsCount() int { return count } +// GetGlobalCallbacksCount returns the total number of callback objects found across all operations. func (index *SpecIndex) GetGlobalCallbacksCount() int { if index.root == nil { return -1 @@ -242,6 +250,7 @@ func (index *SpecIndex) GetGlobalCallbacksCount() int { return index.globalCallbacksCount } +// GetGlobalLinksCount returns the total number of link objects found across all operations. func (index *SpecIndex) GetGlobalLinksCount() int { if index.root == nil { return -1 @@ -314,8 +323,12 @@ func findNestedObjectContainers(node *yaml.Node, key string) []*yaml.Node { return found } +// GetRawReferenceCount returns the total number of raw (non-deduplicated) references found. func (index *SpecIndex) GetRawReferenceCount() int { return len(index.rawSequencedRefs) } +// GetComponentSchemaCount extracts and counts all component schemas, parameters, request bodies, +// responses, security schemes, headers, examples, links, callbacks, and path items from the +// specification. Also handles Swagger 2.0 "definitions" and "securityDefinitions" sections. func (index *SpecIndex) GetComponentSchemaCount() int { if index.root == nil || len(index.root.Content) == 0 { return -1 @@ -451,6 +464,7 @@ func (index *SpecIndex) GetComponentSchemaCount() int { return index.schemaCount } +// GetComponentParameterCount returns the number of component-level parameter definitions. func (index *SpecIndex) GetComponentParameterCount() int { if index.root == nil { return -1 @@ -483,6 +497,8 @@ func (index *SpecIndex) GetComponentParameterCount() int { return index.componentParamCount } +// GetOperationCount returns the total number of operations across all paths and extracts +// path-level and operation-level references (methods, tags, descriptions, summaries, servers). func (index *SpecIndex) GetOperationCount() int { if index.root == nil || index.pathsNode == nil { return -1 @@ -541,6 +557,9 @@ func (index *SpecIndex) GetOperationCount() int { return opCount } +// GetOperationsParameterCount scans all path items and operations to count parameters, +// extract tags, descriptions, summaries, and servers. Also builds the inline parameter +// deduplication maps. func (index *SpecIndex) GetOperationsParameterCount() int { if index.root == nil || index.pathsNode == nil { return -1 @@ -681,6 +700,7 @@ func (index *SpecIndex) GetOperationsParameterCount() int { return index.operationParamCount } +// GetInlineDuplicateParamCount returns the number of inline parameters that have duplicate names. func (index *SpecIndex) GetInlineDuplicateParamCount() int { if index.componentsInlineParamDuplicateCount > 0 { return index.componentsInlineParamDuplicateCount @@ -690,10 +710,13 @@ func (index *SpecIndex) GetInlineDuplicateParamCount() int { return dCount } +// GetInlineUniqueParamCount returns the number of unique inline parameter names. func (index *SpecIndex) GetInlineUniqueParamCount() int { return index.countUniqueInlineDuplicates() } +// GetAllDescriptionsCount returns the total number of description nodes found during indexing. func (index *SpecIndex) GetAllDescriptionsCount() int { return len(index.allDescriptions) } +// GetAllSummariesCount returns the total number of summary nodes found during indexing. func (index *SpecIndex) GetAllSummariesCount() int { return len(index.allSummaries) } diff --git a/index/utility_methods.go b/index/utility_methods.go index cf6c46778..1e8fc191f 100644 --- a/index/utility_methods.go +++ b/index/utility_methods.go @@ -580,6 +580,8 @@ func runIndexFunction(funcs []func() int, wg *sync.WaitGroup) { } } +// GenerateCleanSpecConfigBaseURL builds a cleaned base URL by merging the baseURL path with dir, +// removing duplicate segments. If includeFile is true, the last path segment is preserved. func GenerateCleanSpecConfigBaseURL(baseURL *url.URL, dir string, includeFile bool) string { cleanedPath := baseURL.Path // not cleaned yet! From 6f07f53b710bf935ce66b5ce3400a291a5decae2 Mon Sep 17 00:00:00 2001 From: quobix Date: Wed, 1 Apr 2026 10:55:32 -0400 Subject: [PATCH 4/9] more header changes. --- index/extract_refs_inline.go | 2 +- index/extract_refs_lookup.go | 2 +- index/extract_refs_metadata.go | 2 +- index/extract_refs_ref.go | 2 +- index/extract_refs_walk.go | 2 +- index/find_component_build.go | 2 +- index/find_component_direct.go | 2 +- index/find_component_entry.go | 2 +- index/find_component_external.go | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/index/extract_refs_inline.go b/index/extract_refs_inline.go index 3be039539..3f80fe033 100644 --- a/index/extract_refs_inline.go +++ b/index/extract_refs_inline.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_lookup.go b/index/extract_refs_lookup.go index 49a0756cf..4062daca8 100644 --- a/index/extract_refs_lookup.go +++ b/index/extract_refs_lookup.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_metadata.go b/index/extract_refs_metadata.go index 19f2026b1..ce9b82f78 100644 --- a/index/extract_refs_metadata.go +++ b/index/extract_refs_metadata.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_ref.go b/index/extract_refs_ref.go index a60cd45fa..21317c170 100644 --- a/index/extract_refs_ref.go +++ b/index/extract_refs_ref.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/extract_refs_walk.go b/index/extract_refs_walk.go index 31554c6d8..694311fc5 100644 --- a/index/extract_refs_walk.go +++ b/index/extract_refs_walk.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_build.go b/index/find_component_build.go index 7be916f6e..5c027c28f 100644 --- a/index/find_component_build.go +++ b/index/find_component_build.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_direct.go b/index/find_component_direct.go index 41e32b687..000c28367 100644 --- a/index/find_component_direct.go +++ b/index/find_component_direct.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_entry.go b/index/find_component_entry.go index 346ad2bb6..12da39084 100644 --- a/index/find_component_entry.go +++ b/index/find_component_entry.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index diff --git a/index/find_component_external.go b/index/find_component_external.go index 2aab515c8..cf0727b35 100644 --- a/index/find_component_external.go +++ b/index/find_component_external.go @@ -1,4 +1,4 @@ -// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// Copyright 2023-2026 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT package index From 29472b3bcace15895321cc3c21a30cf5d9039545 Mon Sep 17 00:00:00 2001 From: quobix Date: Wed, 1 Apr 2026 11:01:06 -0400 Subject: [PATCH 5/9] fixed go vet issue --- index/find_component_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index/find_component_test.go b/index/find_component_test.go index 0f889f879..8c8d4ecdc 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -241,7 +241,8 @@ paths: // add remote filesystem rolo.AddRemoteFS("", remoteFS) - ctx, _ := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second)) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second)) + defer cancel() var idx *SpecIndex done := make(chan struct{}) go func() { From 9b731bc093fc32968d65c7840c685b9a19281c9e Mon Sep 17 00:00:00 2001 From: quobix Date: Wed, 1 Apr 2026 12:32:17 -0400 Subject: [PATCH 6/9] fixing windows issues. --- index/rolodex.go | 42 ++++++++++++++++++++++------------ index/rolodex_remote_loader.go | 6 ++--- index/rolodex_test.go | 32 ++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/index/rolodex.go b/index/rolodex.go index 12523b1ca..bd955a836 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -771,13 +771,9 @@ func (r *Rolodex) asLocalFile(file fs.File, fileLookup string) (*LocalFile, []er return wrapped, errs } - bytes, readErr := io.ReadAll(file) - if readErr != nil { - return nil, append(errorStack, readErr) - } - stat, statErr := file.Stat() - if statErr != nil { - return nil, append(errorStack, statErr) + bytes, stat, errs := consumeAdaptedFile(file) + if len(errs) > 0 { + return nil, append(errorStack, errs...) } if len(bytes) == 0 { return nil, nil @@ -843,13 +839,9 @@ func (r *Rolodex) asRemoteFile(file fs.File, location string) (*RemoteFile, []er return remoteFile, nil } - bytes, readErr := io.ReadAll(file) - if readErr != nil { - return nil, []error{readErr} - } - stat, statErr := file.Stat() - if statErr != nil { - return nil, []error{statErr} + bytes, stat, errs := consumeAdaptedFile(file) + if len(errs) > 0 { + return nil, errs } if len(bytes) == 0 { return nil, nil @@ -867,6 +859,28 @@ func (r *Rolodex) asRemoteFile(file fs.File, location string) (*RemoteFile, []er }, nil } +func consumeAdaptedFile(file fs.File) ([]byte, fs.FileInfo, []error) { + var errorStack []error + + bytes, readErr := io.ReadAll(file) + if readErr != nil { + errorStack = append(errorStack, readErr) + _ = file.Close() + return nil, nil, errorStack + } + + stat, statErr := file.Stat() + if statErr != nil { + errorStack = append(errorStack, statErr) + _ = file.Close() + return nil, nil, errorStack + } + + _ = file.Close() + + return bytes, stat, nil +} + func (r *Rolodex) wrapLocalRolodexFile(localFile *LocalFile) (RolodexFile, error) { return &rolodexFile{ rolodex: r, diff --git a/index/rolodex_remote_loader.go b/index/rolodex_remote_loader.go index 595246ebe..3a81f2354 100644 --- a/index/rolodex_remote_loader.go +++ b/index/rolodex_remote_loader.go @@ -14,7 +14,7 @@ import ( "net/http" "net/url" "os" - "path/filepath" + "path" "strings" "sync" "sync/atomic" @@ -764,7 +764,7 @@ func (i *RemoteFS) createRemoteFile(remoteParsedURL *url.URL, fileExt FileExtens lastModifiedTime = time.Now() } return &RemoteFile{ - filename: filepath.Base(remoteParsedURL.Path), + filename: path.Base(remoteParsedURL.Path), name: remoteParsedURL.Path, extension: fileExt, data: responseBytes, @@ -778,7 +778,7 @@ func (i *RemoteFS) createRemoteFile(remoteParsedURL *url.URL, fileExt FileExtens func (i *RemoteFS) createRemoteIndexConfig(remoteParsedURL, remoteParsedURLOriginal *url.URL) *SpecIndexConfig { copiedCfg := *i.indexConfig newBase := fmt.Sprintf("%s://%s%s", remoteParsedURLOriginal.Scheme, remoteParsedURLOriginal.Host, - filepath.Dir(remoteParsedURL.Path)) + path.Dir(remoteParsedURL.Path)) newBaseURL, _ := url.Parse(newBase) if newBaseURL != nil { copiedCfg.BaseURL = newBaseURL diff --git a/index/rolodex_test.go b/index/rolodex_test.go index 56b0aad99..b7b664aad 100644 --- a/index/rolodex_test.go +++ b/index/rolodex_test.go @@ -2324,6 +2324,28 @@ func TestRolodex_AsRemoteFile_AndWrappers(t *testing.T) { assert.Error(t, err) } +func TestRolodex_AsLocalFile_ClosesAdaptedFile(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + rolo.rootIndex = NewTestSpecIndex().Load().(*SpecIndex) + + file := &closeTrackingFile{testFile: testFile{content: "hello"}} + localFile, errs := rolo.asLocalFile(file, "/tmp/spec.yaml") + assert.NotNil(t, localFile) + assert.Empty(t, errs) + assert.True(t, file.closed) +} + +func TestRolodex_AsRemoteFile_ClosesAdaptedFile(t *testing.T) { + rolo := NewRolodex(CreateOpenAPIIndexConfig()) + rolo.rootIndex = NewTestSpecIndex().Load().(*SpecIndex) + + file := &closeTrackingFile{testFile: testFile{content: "hello"}} + remoteFile, errs := rolo.asRemoteFile(file, "http://example.com/spec.yaml") + assert.NotNil(t, remoteFile) + assert.Empty(t, errs) + assert.True(t, file.closed) +} + func TestRolodex_AsFileHelperErrors(t *testing.T) { rolo := NewRolodex(CreateOpenAPIIndexConfig()) @@ -2437,6 +2459,11 @@ type testFile struct { offset int64 } +type closeTrackingFile struct { + testFile + closed bool +} + type errorReadFile struct{} func (e *errorReadFile) Read(_ []byte) (int, error) { return 0, fmt.Errorf("read failed") } @@ -2462,6 +2489,11 @@ func (tf *testFile) Read(p []byte) (n int, err error) { func (tf *testFile) Close() error { return nil } +func (tf *closeTrackingFile) Close() error { + tf.closed = true + return nil +} + func (tf *testFile) Stat() (fs.FileInfo, error) { return &testFileInfo{name: "test.yaml", size: int64(len(tf.content))}, nil } From 3051fd193ff41b6ad8aea5251b5071d64d108534 Mon Sep 17 00:00:00 2001 From: quobix Date: Wed, 1 Apr 2026 13:12:48 -0400 Subject: [PATCH 7/9] OH WINDOWS. PLEASE. STOP BEING STRANGE --- index/rolodex_file_loader.go | 8 ++++++-- index/rolodex_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index 79a94b560..3a2547ec7 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -494,7 +494,11 @@ func (l *LocalFS) extractFile(p string) (*LocalFile, error) { var file fs.File if config != nil && config.DirFS != nil { l.logger.Debug("[rolodex file loader]: collecting file from dirFS", "file", extension, "location", abs) - file, _ = config.DirFS.Open(p) + var fileError error + file, fileError = config.DirFS.Open(p) + if fileError != nil { + return nil, fileError + } } else { l.logger.Debug("[rolodex file loader]: reading local file from OS", "file", extension, "location", abs) var fileError error @@ -503,8 +507,8 @@ func (l *LocalFS) extractFile(p string) (*LocalFile, error) { if fileError != nil { return nil, fileError } - defer file.Close() } + defer file.Close() modTime := time.Now() stat, _ := file.Stat() diff --git a/index/rolodex_test.go b/index/rolodex_test.go index b7b664aad..0f8239310 100644 --- a/index/rolodex_test.go +++ b/index/rolodex_test.go @@ -2346,6 +2346,21 @@ func TestRolodex_AsRemoteFile_ClosesAdaptedFile(t *testing.T) { assert.True(t, file.closed) } +func TestLocalFS_ExtractFile_ClosesDirFSFile(t *testing.T) { + lfs, err := NewLocalFSWithConfig(&LocalFSConfig{ + BaseDirectory: "/tmp", + DirFS: &closeTrackingDirFS{ + file: &closeTrackingFile{testFile: testFile{content: "hello"}}, + }, + }) + assert.NoError(t, err) + + localFile, extractErr := lfs.extractFile("spec.yaml") + assert.NoError(t, extractErr) + assert.NotNil(t, localFile) + assert.True(t, lfs.fsConfig.DirFS.(*closeTrackingDirFS).file.closed) +} + func TestRolodex_AsFileHelperErrors(t *testing.T) { rolo := NewRolodex(CreateOpenAPIIndexConfig()) @@ -2464,6 +2479,10 @@ type closeTrackingFile struct { closed bool } +type closeTrackingDirFS struct { + file *closeTrackingFile +} + type errorReadFile struct{} func (e *errorReadFile) Read(_ []byte) (int, error) { return 0, fmt.Errorf("read failed") } @@ -2494,6 +2513,12 @@ func (tf *closeTrackingFile) Close() error { return nil } +func (f *closeTrackingDirFS) Open(name string) (fs.File, error) { + f.file.offset = 0 + f.file.closed = false + return f.file, nil +} + func (tf *testFile) Stat() (fs.FileInfo, error) { return &testFileInfo{name: "test.yaml", size: int64(len(tf.content))}, nil } From 5e2cb234bfab061fc440cccbd489cf641c66a7d4 Mon Sep 17 00:00:00 2001 From: quobix Date: Thu, 2 Apr 2026 07:09:39 -0400 Subject: [PATCH 8/9] address coverage issues --- index/rolodex_file_loader_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/index/rolodex_file_loader_test.go b/index/rolodex_file_loader_test.go index 20387998a..1f67f5dd4 100644 --- a/index/rolodex_file_loader_test.go +++ b/index/rolodex_file_loader_test.go @@ -4,6 +4,7 @@ package index import ( + "errors" "io" "io/fs" "log/slog" @@ -232,6 +233,26 @@ func TestRolodexLocalFile_TestBadFS(t *testing.T) { assert.Nil(t, fileFS) } +type openErrorDirFS struct{} + +func (f *openErrorDirFS) Open(name string) (fs.File, error) { + return nil, errors.New("open failed") +} + +func TestRolodexLocalFS_ExtractFile_DirFSOpenError(t *testing.T) { + lfs := &LocalFS{ + fsConfig: &LocalFSConfig{ + BaseDirectory: ".", + DirFS: &openErrorDirFS{}, + }, + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + f, extractErr := lfs.extractFile("spec.yaml") + assert.Nil(t, f) + assert.EqualError(t, extractErr, "open failed") +} + func TestNewRolodexLocalFile_BadOffset(t *testing.T) { lf := &LocalFile{offset: -1} z, y := io.ReadAll(lf) From e743716783e3df5128cd74d8f3fa2702c46a096d Mon Sep 17 00:00:00 2001 From: quobix Date: Thu, 2 Apr 2026 11:40:00 -0400 Subject: [PATCH 9/9] rebased and ported luke's work --- index/spec_index_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index/spec_index_test.go b/index/spec_index_test.go index db3b994fa..c51319699 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -204,7 +204,7 @@ func TestSpecIndex_DigitalOcean(t *testing.T) { // index the rolodex. indexedErr := rolo.IndexTheRolodex(context.Background()) - if indexedErr != nil && strings.Contains(indexedErr.Error(), "429") { + if indexedErr != nil && (strings.Contains(indexedErr.Error(), "429") || hasRateLimitedRemoteErrors(remoteFS.GetErrors())) { t.Skipf("skipping due to GitHub rate limit: %v", indexedErr) } assert.NoError(t, indexedErr)