config

Author: KSemenenko

src/MarkdownLd.Kb/Pipeline/TiktokenKnowledgeGraphExtractor.cs

@@ -30,7 +30,9 @@ public TokenizedKnowledgeExtractionResult Extract(IReadOnlyList<MarkdownDocument
         var segments = candidates.Select(candidate => CreateSegment(candidate, vectorSpace)).ToArray();
         var topics = _topicExtractor.Extract(candidates);
         var entityHints = _entityHintExtractor.Extract(documents);
-        var relations = BuildRelations(segments).ToArray();
+        var relations = _options.BuildAutoRelatedSegmentRelations
+            ? BuildRelations(segments).ToArray()
+            : [];
         var facts = TokenizedKnowledgeFactFactory.Build(sections, segments, topics, entityHints, relations);
         return new TokenizedKnowledgeExtractionResult(facts, segments, vectorSpace);
     }

src/MarkdownLd.Kb/Pipeline/TiktokenKnowledgeGraphOptions.cs

@@ -8,6 +8,8 @@ public sealed record TiktokenKnowledgeGraphOptions
 
     public TokenVectorWeighting Weighting { get; init; } = DefaultTokenVectorWeighting;
 
+    public bool BuildAutoRelatedSegmentRelations { get; init; } = true;
+
     public int MaxRelatedSegments { get; init; } = DefaultMaxRelatedTokenSegments;
 
     public int MinimumTokenCount { get; init; } = DefaultMinimumTokenCount;

tests/MarkdownLd.Kb.Tests/Integration/TiktokenKnowledgeGraphFlowTests.cs

@@ -224,13 +224,32 @@ public async Task Tiktoken_mode_builds_token_distance_graph_and_shows_language_d
         ukrainianToEnglishHits.ShouldBeLessThanOrEqualTo(ExpectedCrossLanguageMaximumHits);
     }
 
-    private static Task<MarkdownKnowledgeBuildResult> BuildTokenGraphAsync(string path, string markdown)
+    [Test]
+    public async Task Tiktoken_mode_can_disable_auto_related_segment_relations_and_keep_token_distance_search()
+    {
+        var result = await BuildTokenGraphAsync(
+            EnglishPath,
+            EnglishMarkdown,
+            buildAutoRelatedSegmentRelations: false);
+
+        result.ExtractionMode.ShouldBe(MarkdownKnowledgeExtractionMode.Tiktoken);
+        result.Graph.CanSearchByTokenDistance.ShouldBeTrue();
+
+        var hits = await CountTopHitsAsync(result.Graph, EnglishQueries);
+        hits.ShouldBeGreaterThanOrEqualTo(ExpectedSameLanguageMinimumHits);
+    }
+
+    private static Task<MarkdownKnowledgeBuildResult> BuildTokenGraphAsync(
+        string path,
+        string markdown,
+        bool buildAutoRelatedSegmentRelations = true)
     {
         var pipeline = new MarkdownKnowledgePipeline(
             BaseUri,
             extractionMode: MarkdownKnowledgeExtractionMode.Tiktoken,
             tiktokenOptions: new TiktokenKnowledgeGraphOptions
             {
+                BuildAutoRelatedSegmentRelations = buildAutoRelatedSegmentRelations,
                 MaxRelatedSegments = 2,
             });