Skip to content

Commit dcf30c3

Browse files
iceljciceljc
authored
Merge pull request SciSharp#1341 from iceljc/features/refine-knowledge-service
Features/refine knowledge service
2 parents 16a1da4 + 40b0141 commit dcf30c3

115 files changed

Lines changed: 2005 additions & 1690 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ public class EntityAnalysisOptions
77
/// </summary>
88
[JsonPropertyName("data_providers")]
99
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
10-
public List<string>? DataProviders { get; set; }
10+
public IEnumerable<string>? DataProviders { get; set; }
1111

1212
/// <summary>
1313
/// Maximum n-gram size

‎src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -62,17 +62,9 @@ public interface IFileStorageService
6262
#endregion
6363

6464
#region Knowledge
65-
bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData);
66-
67-
/// <summary>
68-
/// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection.
69-
/// </summary>
70-
/// <param name="collectionName"></param>
71-
/// <param name="vectorStoreProvider"></param>
72-
/// <param name="fileId"></param>
73-
/// <returns></returns>
74-
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
75-
string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
76-
BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
65+
bool SaveKnowledgeBaseFile(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName, BinaryData fileData);
66+
bool DeleteKnowledgeFile(string collectionName, string knowledgebaseProvider, Guid? fileId = null);
67+
string GetKnowledgeBaseFileUrl(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName);
68+
BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string knowledgebaseProvider, Guid fileId, string fileName);
7769
#endregion
7870
}

‎src/Infrastructure/BotSharp.Abstraction/Graph/IGraphKnowledgeService.cs‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionType.cs renamed to src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeBaseType.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
namespace BotSharp.Abstraction.Knowledges.Enums;
22

3-
public static class KnowledgeCollectionType
3+
public static class KnowledgeBaseType
44
{
55
public static string QuestionAnswer = "question-answer";
66
public static string Document = "document";
7+
public static string Taxonomy = "taxonomy";
8+
public static string SemanticGraph = "semantic-graph";
79
}

‎src/Infrastructure/BotSharp.Abstraction/Knowledges/Filters/KnowledgeFileFilter.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,10 @@ namespace BotSharp.Abstraction.Knowledges.Filters;
22

33
public class KnowledgeFileFilter : Pagination
44
{
5+
public string? DbProvider { get; set; }
56
public IEnumerable<Guid>? FileIds { get; set; }
6-
77
public IEnumerable<string>? FileNames { get; set; }
8-
98
public IEnumerable<string>? ContentTypes { get; set; }
10-
119
public IEnumerable<string>? FileSources { get; set; }
1210

1311
public KnowledgeFileFilter()

‎src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeFileOrchestrator.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using BotSharp.Abstraction.Knowledges.Filters;
2+
using BotSharp.Abstraction.Knowledges.Responses;
3+
4+
namespace BotSharp.Abstraction.Knowledges;
5+
6+
public interface IKnowledgeFileOrchestrator
7+
{
8+
string Provider { get; }
9+
10+
/// <summary>
11+
/// Save files and their contents to knowledgebase
12+
/// </summary>
13+
/// <param name="collectionName"></param>
14+
/// <param name="files"></param>
15+
/// <param name="options"></param>
16+
/// <returns></returns>
17+
Task<UploadKnowledgeResponse> UploadFilesToKnowledge(string collectionName,
18+
IEnumerable<ExternalFileModel> files, KnowledgeFileHandleOptions? options = null);
19+
20+
/// <summary>
21+
/// Import file content to knowledgebase without saving the file
22+
/// </summary>
23+
/// <param name="collectionName"></param>
24+
/// <param name="fileName"></param>
25+
/// <param name="fileSource"></param>
26+
/// <param name="contents"></param>
27+
/// <param name="options"></param>
28+
/// <returns></returns>
29+
Task<bool> ImportFileContentToKnowledge(string collectionName, string fileName, string fileSource,
30+
IEnumerable<string> contents, ImportKnowledgeFileOptions? options = null);
31+
32+
/// <summary>
33+
/// Delete one file and its related knowledge in the collection
34+
/// </summary>
35+
/// <param name="collectionName"></param>
36+
/// <param name="fileId"></param>
37+
/// <param name="options"></param>
38+
/// <returns></returns>
39+
Task<bool> DeleteKnowledgeFile(string collectionName, Guid fileId, KnowledgeFileOptions? options = null);
40+
41+
/// <summary>
42+
/// Delete all files and their related knowledge in the collection
43+
/// </summary>
44+
/// <param name="collectionName"></param>
45+
/// <param name="filter"></param>
46+
/// <returns></returns>
47+
Task<bool> DeleteKnowledgeFiles(string collectionName, KnowledgeFileFilter filter);
48+
49+
/// <summary>
50+
/// Get knowledge files by pagination
51+
/// </summary>
52+
/// <param name="collectionName"></param>
53+
/// <param name="filter"></param>
54+
/// <returns></returns>
55+
Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeFiles(string collectionName, KnowledgeFileFilter filter);
56+
57+
/// <summary>
58+
/// Get knowledge file binary data
59+
/// </summary>
60+
/// <param name="collectionName"></param>
61+
/// <param name="fileId"></param>
62+
/// <param name="options"></param>
63+
/// <returns></returns>
64+
Task<FileBinaryDataModel> GetKnowledgeFileBinaryData(string collectionName, Guid fileId, KnowledgeFileOptions? options = null);
65+
}

‎src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs‎

Lines changed: 45 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,83 +1,61 @@
1-
using BotSharp.Abstraction.Graph.Models;
2-
using BotSharp.Abstraction.Graph.Options;
3-
using BotSharp.Abstraction.Knowledges.Filters;
1+
using BotSharp.Abstraction.Knowledges.Models;
42
using BotSharp.Abstraction.Knowledges.Options;
5-
using BotSharp.Abstraction.Knowledges.Responses;
6-
using BotSharp.Abstraction.VectorStorage.Models;
7-
using BotSharp.Abstraction.VectorStorage.Options;
83

94
namespace BotSharp.Abstraction.Knowledges;
105

116
public interface IKnowledgeService
127
{
13-
#region Vector
14-
Task<bool> ExistVectorCollection(string collectionName);
15-
Task<bool> CreateVectorCollection(string collectionName, string collectionType, VectorCollectionCreateOptions options);
16-
Task<bool> DeleteVectorCollection(string collectionName);
17-
Task<IEnumerable<VectorCollectionConfig>> GetVectorCollections(string? type = null);
18-
Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName);
19-
Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
20-
Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
21-
Task<IEnumerable<VectorCollectionData>> GetVectorCollectionData(string collectionName, IEnumerable<string> ids, VectorQueryOptions? options = null);
22-
Task<bool> DeleteVectorCollectionData(string collectionName, string id);
23-
Task<bool> DeleteVectorCollectionAllData(string collectionName);
24-
Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
25-
Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update);
26-
Task<bool> UpsertVectorCollectionData(string collectionName, VectorUpdateModel update);
27-
#endregion
8+
string KnowledgeType { get; }
289

29-
#region Document
30-
/// <summary>
31-
/// Save documents and their contents to knowledgebase
32-
/// </summary>
33-
/// <param name="collectionName"></param>
34-
/// <param name="files"></param>
35-
/// <param name="option"></param>
36-
/// <returns></returns>
37-
Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files, KnowledgeDocOptions? options = null);
38-
/// <summary>
39-
/// Save document content to knowledgebase without saving the document
40-
/// </summary>
41-
/// <param name="collectionName"></param>
42-
/// <param name="fileName"></param>
43-
/// <param name="fileSource"></param>
44-
/// <param name="contents"></param>
45-
/// <param name="refData"></param>
46-
/// <returns></returns>
47-
Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable<string> contents,
48-
DocMetaRefData? refData = null, Dictionary<string, VectorPayloadValue>? payload = null);
49-
/// <summary>
50-
/// Delete one document and its related knowledge in the collection
51-
/// </summary>
52-
/// <param name="collectionName"></param>
53-
/// <param name="fileId"></param>
54-
/// <returns></returns>
55-
Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
56-
/// <summary>
57-
/// Delete all documents and their related knowledge in the collection
58-
/// </summary>
59-
/// <param name="collectionName"></param>
60-
/// <param name="filter"></param>
61-
/// <returns></returns>
62-
Task<bool> DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
63-
Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
64-
Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
10+
#region Collection
11+
Task<bool> ExistCollection(string collectionName, KnowledgeCollectionOptions options)
12+
=> Task.FromResult(false);
13+
Task<bool> CreateCollection(string collectionName, CollectionCreateOptions options)
14+
=> Task.FromResult(false);
15+
Task<bool> DeleteCollection(string collectionName, KnowledgeCollectionOptions options)
16+
=> Task.FromResult(false);
17+
Task<IEnumerable<KnowledgeCollectionConfig>> GetCollections(KnowledgeCollectionOptions options)
18+
=> Task.FromResult(Enumerable.Empty<KnowledgeCollectionConfig>());
19+
Task<KnowledgeCollectionDetails?> GetCollectionDetails(string collectionName, KnowledgeCollectionOptions options)
20+
=> Task.FromResult<KnowledgeCollectionDetails?>(null);
6521
#endregion
6622

67-
#region Snapshot
68-
Task<IEnumerable<VectorCollectionSnapshot>> GetVectorCollectionSnapshots(string collectionName);
69-
Task<VectorCollectionSnapshot?> CreateVectorCollectionSnapshot(string collectionName);
70-
Task<BinaryData> DownloadVectorCollectionSnapshot(string collectionName, string snapshotFileName);
71-
Task<bool> RecoverVectorCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData);
72-
Task<bool> DeleteVectorCollectionSnapshot(string collectionName, string snapshotName);
23+
#region Data
24+
Task<IEnumerable<KnowledgeExecuteResult>> ExecuteQuery(string query, string collectionName, KnowledgeExecuteOptions options)
25+
=> Task.FromResult(Enumerable.Empty<KnowledgeExecuteResult>());
26+
Task<StringIdPagedItems<KnowledgeCollectionData>> GetPagedCollectionData(string collectionName, KnowledgeFilter filter)
27+
=> Task.FromResult(new StringIdPagedItems<KnowledgeCollectionData>());
28+
Task<IEnumerable<KnowledgeCollectionData>> GetCollectionData(string collectionName, IEnumerable<string> ids, KnowledgeQueryOptions? options = null)
29+
=> Task.FromResult(Enumerable.Empty<KnowledgeCollectionData>());
30+
Task<bool> DeleteCollectionData(string collectionName, string id, KnowledgeCollectionOptions? options)
31+
=> Task.FromResult(false);
32+
Task<bool> DeleteCollectionData(string collectionName, KnowledgeCollectionOptions? options)
33+
=> Task.FromResult(false);
34+
Task<bool> CreateCollectionData(string collectionName, KnowledgeCreateModel create)
35+
=> Task.FromResult(false);
36+
Task<bool> UpdateCollectionData(string collectionName, KnowledgeUpdateModel update)
37+
=> Task.FromResult(false);
38+
Task<bool> UpsertCollectionData(string collectionName, KnowledgeUpdateModel update)
39+
=> Task.FromResult(false);
7340
#endregion
7441

7542
#region Index
76-
Task<SuccessFailResponse<string>> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable<CreateVectorCollectionIndexOptions> options);
77-
Task<SuccessFailResponse<string>> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable<DeleteVectorCollectionIndexOptions> options);
43+
Task<SuccessFailResponse<string>> CreateIndexes(string collectionName, KnowledgeIndexOptions options)
44+
=> Task.FromResult(new SuccessFailResponse<string>());
45+
Task<SuccessFailResponse<string>> DeleteIndexes(string collectionName, KnowledgeIndexOptions options)
46+
=> Task.FromResult(new SuccessFailResponse<string>());
7847
#endregion
7948

80-
#region Common
81-
Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs);
49+
#region Snapshot
50+
Task<IEnumerable<KnowledgeCollectionSnapshot>> GetCollectionSnapshots(string collectionName, KnowledgeSnapshotOptions? options = null)
51+
=> Task.FromResult(Enumerable.Empty<KnowledgeCollectionSnapshot>());
52+
Task<KnowledgeCollectionSnapshot?> CreateCollectionSnapshot(string collectionName, KnowledgeSnapshotOptions? options = null)
53+
=> Task.FromResult<KnowledgeCollectionSnapshot?>(null);
54+
Task<BinaryData> DownloadCollectionSnapshot(string collectionName, string snapshotFileName, KnowledgeSnapshotOptions? options = null)
55+
=> Task.FromResult(new BinaryData(Array.Empty<byte>()));
56+
Task<bool> RecoverCollectionFromSnapshot(string collectionName, string snapshotFileName, BinaryData snapshotData, KnowledgeSnapshotOptions? options = null)
57+
=> Task.FromResult(false);
58+
Task<bool> DeleteCollectionSnapshot(string collectionName, string snapshotName, KnowledgeSnapshotOptions? options = null)
59+
=> Task.FromResult(false);
8260
#endregion
8361
}

‎src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionConfigModel.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
using BotSharp.Abstraction.VectorStorage.Models;
2+
3+
namespace BotSharp.Abstraction.Knowledges.Models;
4+
5+
public class KnowledgeCollectionConfig : VectorCollectionConfig
6+
{
7+
}

‎src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using BotSharp.Abstraction.VectorStorage.Models;
2+
3+
namespace BotSharp.Abstraction.Knowledges.Models;
4+
5+
public class KnowledgeCollectionData
6+
{
7+
public string Id { get; set; }
8+
public Dictionary<string, VectorPayloadValue> Payload { get; set; } = new();
9+
public Dictionary<string, object> Data => Payload?.ToDictionary(x => x.Key, x => x.Value.DataValue) ?? [];
10+
public double? Score { get; set; }
11+
12+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
13+
public float[]? Vector { get; set; }
14+
}

‎src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionDetails.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
using BotSharp.Abstraction.VectorStorage.Models;
2+
3+
namespace BotSharp.Abstraction.Knowledges.Models;
4+
5+
public class KnowledgeCollectionDetails
6+
{
7+
[JsonPropertyName("status")]
8+
public string Status { get; set; }
9+
10+
[JsonPropertyName("payload_schema")]
11+
public List<PayloadSchemaDetail> PayloadSchema { get; set; } = [];
12+
}

0 commit comments

Comments
 (0)