Merge pull request #1315 from hchen2020/master

BuildRetryPolicy for reliable membase

Author: Oceania2018

src/Plugins/BotSharp.Plugin.Membase/BotSharp.Plugin.Membase.csproj

@@ -10,6 +10,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Polly" />
     <PackageReference Include="Refit" />
     <PackageReference Include="Refit.HttpClientFactory" />
   </ItemGroup>

src/Plugins/BotSharp.Plugin.Membase/GraphDb/MembaseGraphDb.cs

@@ -1,12 +1,6 @@
-using BotSharp.Abstraction.Graph;
-using BotSharp.Abstraction.Graph.Models;
-using BotSharp.Abstraction.Graph.Options;
-using BotSharp.Abstraction.Models;
-using BotSharp.Abstraction.Options;
-using BotSharp.Abstraction.Utilities;
-using BotSharp.Plugin.Membase.Interfaces;
-using Microsoft.Extensions.Logging;
-using System.Text.Json;
+using Polly;
+using Polly.Timeout;
+using Refit;
 
 namespace BotSharp.Plugin.Membase.GraphDb;
 
@@ -28,6 +22,33 @@ public MembaseGraphDb(
 
     public string Provider => "membase";
 
+    private const int RetryCount = 3;
+
+    private AsyncPolicy BuildRetryPolicy()
+    {
+        var settings = _services.GetRequiredService<MembaseSettings>();
+        var timeoutSeconds = (double)settings.TimeoutSecond / RetryCount;
+
+        var timeoutPolicy = Policy.TimeoutAsync(TimeSpan.FromSeconds(timeoutSeconds));
+
+        var retryPolicy = Policy
+            .Handle<HttpRequestException>()
+            .Or<TaskCanceledException>()
+            .Or<TimeoutRejectedException>()
+            .Or<ApiException>(ex => ex.StatusCode == HttpStatusCode.ServiceUnavailable)
+            .WaitAndRetryAsync(
+                retryCount: RetryCount,
+                sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
+                onRetry: (ex, timespan, retryAttempt, _) =>
+                {
+                    _logger.LogWarning(ex,
+                        "CypherQueryAsync retry {RetryAttempt}/{MaxRetries} after {Delay}s. Exception: {Message}",
+                        retryAttempt, RetryCount, timespan.TotalSeconds, ex.Message);
+                });
+
+        return Policy.WrapAsync(retryPolicy, timeoutPolicy);
+    }
+
     public async Task<GraphQueryResult> ExecuteQueryAsync(string query, GraphQueryExecuteOptions? options = null)
     {
         if (string.IsNullOrEmpty(options?.GraphId))
@@ -36,14 +57,17 @@ public async Task<GraphQueryResult> ExecuteQueryAsync(string query, GraphQueryEx
         }
 
         var args = options?.Arguments ?? new();
+        var argLogs = JsonSerializer.Serialize(args, BotSharpOptions.defaultJsonOptions);
 
         try
         {
-            var response = await _membaseApi.CypherQueryAsync(options!.GraphId, new CypherQueryRequest
-            {
-                Query = query,
-                Parameters = args
-            });
+            var retryPolicy = BuildRetryPolicy();
+            var response = await retryPolicy.ExecuteAsync(() =>
+                _membaseApi.CypherQueryAsync(options!.GraphId, new CypherQueryRequest
+                {
+                    Query = query,
+                    Parameters = args
+                }));
 
             return new GraphQueryResult
             {
@@ -52,10 +76,14 @@ public async Task<GraphQueryResult> ExecuteQueryAsync(string query, GraphQueryEx
                 Result = JsonSerializer.Serialize(response.Data)
             };
         }
+        catch (ApiException ex)
+        {
+            _logger.LogError($"Error when executing query in {Provider} graph db:\r\n{ex.Content}\r\n{query}\r\n{argLogs}");
+            throw;
+        }
         catch (Exception ex)
         {
-            var argLogs = args.Select(x => (new KeyValue(x.Key, x.Value.ConvertToString(BotSharpOptions.defaultJsonOptions))).ToString());
-            _logger.LogError(ex, $"Error when executing query in {Provider} graph db. (Query: {query}), (Argments: \r\n{string.Join("\r\n", argLogs)})");
+            _logger.LogError(ex, $"Error when executing query in {Provider} graph db. (Query: {query}), (Argments: \r\n{argLogs})");
             throw;
         }
     }

src/Plugins/BotSharp.Plugin.Membase/MembasePlugin.cs

@@ -33,7 +33,8 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
                 .ConfigureHttpClient(c =>
                 {
                     c.BaseAddress = new Uri(settings.Host);
-                    c.Timeout = TimeSpan.FromSeconds(settings.TimeoutSecond);
+                    // Timeout is set by MembaseGrapbDb internally, but we set it here as well to ensure that the Refit client does not timeout before the graph db does.
+                    c.Timeout = TimeSpan.FromSeconds(90);
                 });
 
         services.AddScoped<IGraphDb, MembaseGraphDb>();

src/Plugins/BotSharp.Plugin.Membase/Settings/MembaseSettings.cs

@@ -7,6 +7,6 @@ public class MembaseSettings
     public string Host { get; set; } = "localhost";
     public string ProjectId { get; set; } = string.Empty;
     public string ApiKey { get; set; } = string.Empty;
-    public int TimeoutSecond { get; set; } = 10;
+    public int TimeoutSecond { get; set; } = 30;
     public GraphInstance[] GraphInstances { get; set; } = [];
 }

src/Plugins/BotSharp.Plugin.Membase/Using.cs

@@ -2,11 +2,15 @@
 global using System.Collections.Generic;
 global using System.Linq;
 global using System.Threading.Tasks;
+global using System.Net;
+global using System.Net.Http;
+global using System.Text.Json;
 
 global using Microsoft.AspNetCore.Authorization;
 global using Microsoft.AspNetCore.Mvc;
 global using Microsoft.Extensions.Configuration;
 global using Microsoft.Extensions.DependencyInjection;
+global using Microsoft.Extensions.Logging;
 
 global using BotSharp.Abstraction.Users;
 global using BotSharp.Abstraction.Knowledges;
@@ -15,3 +19,8 @@
 global using BotSharp.Plugin.Membase.Models;
 global using BotSharp.Plugin.Membase.Services;
 global using BotSharp.Plugin.Membase.Settings;
+global using BotSharp.Abstraction.Graph;
+global using BotSharp.Abstraction.Graph.Models;
+global using BotSharp.Abstraction.Graph.Options;
+global using BotSharp.Abstraction.Options;
+global using BotSharp.Plugin.Membase.Interfaces;