Skip to content

Commit 066cf97

Browse files
KSemenenkoKSemenenko
committed
Fix live runtime graph telemetry
1 parent ed85f9d commit 066cf97

28 files changed

Lines changed: 902 additions & 222 deletions

‎AGENTS.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ Update guidelines:
3434
- **Runtime Graph Telemetry**: For live graph features, have filters report observed calls to stateless worker aggregators that periodically flush to an in-memory grain; do not rely on per-request `CallHistory` alone for a global runtime graph.
3535
- **Runtime Graph Identity**: Live graph nodes must never silently fall back to the Orleans base `Grain` type or any guessed identity; use a concrete grain implementation class or a real grain interface, and fail fast if neither can be resolved.
3636
- **Runtime Graph API Shape**: Expose live telemetry as a graph model with explicit `Vertices` and `Edges`; Mermaid output is only a renderer for that same graph, while methods/counts/timestamps belong on edges.
37+
- **No Silent Fallbacks**: In call tracking and runtime graph code, never substitute wildcard methods, base `Grain`, reflection guesses, or history scans when exact caller identity is required; fail fast so identity bugs are visible immediately.
38+
- **Filter Hot Path**: Keep Orleans call filters O(1) on request context data; avoid reflection, interface scanning, or call-history searching in the hot path so graph tracking stays cheap.
3739
- **Telemetry Filtering**: Do not track Orleans.Graph internal telemetry calls by default; expose a configuration switch to include them, and test both filtered and full-tracking modes.
3840
- **Testing Scope**: Exercise new behavior through the Orleans-hosted integration tests in `ManagedCode.Orleans.Graph.Tests`, covering both positive and negative paths to mirror real cluster flows.
3941
- **Cluster-to-Cluster Tests**: For grain-only runtime graph scenarios, add coverage that starts from a silo-side `IGrainFactory` instead of `IClusterClient`, so client-origin edges cannot hide cluster-to-cluster behavior.

‎ManagedCode.Orleans.Graph.Tests/AttributeClusterTests.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,13 @@ public void AttributeGraph_LiveDiagramHighlightsUsage()
5858
var manager = _fixture.Cluster.Client.ServiceProvider.GetRequiredService<GrainTransitionManager>();
5959

6060
var history = new CallHistory();
61-
history.Push(new OutCall(null, null, typeof(IAttributeClusterGrainA).FullName!, typeof(IAttributeClusterGrainB).FullName!, nameof(IAttributeClusterGrainA.CallB)));
61+
history.Push(new OutCall(
62+
null,
63+
null,
64+
typeof(IAttributeClusterGrainA).FullName!,
65+
typeof(IAttributeClusterGrainB).FullName!,
66+
nameof(IAttributeClusterGrainA.CallB),
67+
nameof(IAttributeClusterGrainA.CallB)));
6268
history.Push(new InCall(null, null, typeof(IAttributeClusterGrainB).FullName!, nameof(IAttributeClusterGrainB.MethodB)));
6369

6470
var diagram = manager.GenerateLiveMermaidDiagram(history);

‎ManagedCode.Orleans.Graph.Tests/AttributeGraphConfiguratorTests.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,18 @@ public void AttributeConfigurator_BuildsExpectedGraph()
1616

1717
var clientHistory = new CallHistory();
1818
clientHistory.Push(new InCall(null, null, typeof(IAttributeGrainA).FullName!, nameof(IAttributeGrainA.MethodA1)));
19-
clientHistory.Push(new OutCall(null, null, Constants.ClientCallerId, typeof(IAttributeGrainA).FullName!, nameof(IAttributeGrainA.MethodA1)));
19+
clientHistory.Push(new OutCall(null, null, Constants.ClientCallerId, typeof(IAttributeGrainA).FullName!, nameof(IAttributeGrainA.MethodA1), Constants.AnyMethod));
2020

2121
manager.IsTransitionAllowed(clientHistory).ShouldBeTrue();
2222

2323
var reentrantHistory = new CallHistory();
24-
reentrantHistory.Push(new OutCall(null, null, typeof(IAttributeGrainB).FullName!, typeof(IAttributeGrainB).FullName!, nameof(IAttributeGrainB.MethodB1)));
24+
reentrantHistory.Push(new OutCall(
25+
null,
26+
null,
27+
typeof(IAttributeGrainB).FullName!,
28+
typeof(IAttributeGrainB).FullName!,
29+
nameof(IAttributeGrainB.MethodB1),
30+
nameof(IAttributeGrainB.MethodB1)));
2531
reentrantHistory.Push(new InCall(null, null, typeof(IAttributeGrainB).FullName!, nameof(IAttributeGrainB.MethodB1)));
2632

2733
manager.IsTransitionAllowed(reentrantHistory).ShouldBeTrue();
@@ -73,7 +79,7 @@ public void AttributeConfigurator_ClientAttributeWithTargetType_AllowsClientCall
7379

7480
var history = new CallHistory();
7581
history.Push(new InCall(null, null, typeof(IAttributeGrainB).FullName!, nameof(IAttributeGrainB.MethodB1)));
76-
history.Push(new OutCall(null, null, Constants.ClientCallerId, typeof(IAttributeGrainB).FullName!, nameof(IAttributeGrainB.MethodB1)));
82+
history.Push(new OutCall(null, null, Constants.ClientCallerId, typeof(IAttributeGrainB).FullName!, nameof(IAttributeGrainB.MethodB1), Constants.AnyMethod));
7783

7884
manager.IsTransitionAllowed(history).ShouldBeTrue();
7985
}

‎ManagedCode.Orleans.Graph.Tests/GrainTransitionManagerTests.cs‎

Lines changed: 184 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using ManagedCode.Orleans.Graph.Interfaces;
22
using ManagedCode.Orleans.Graph.Models;
3-
using ManagedCode.Orleans.Graph.Tests.Cluster.Grains;
43
using ManagedCode.Orleans.Graph.Tests.Cluster.Grains.Interfaces;
54

65
namespace ManagedCode.Orleans.Graph.Tests;
@@ -135,22 +134,94 @@ public void IsTransitionAllowed_UsesSourceIncomingMethodForOutgoingGrainCall()
135134
var grainBId = GrainId.Create("grainb", "source-method");
136135

137136
var callHistory = new CallHistory();
138-
callHistory.Push(new OutCall(null, grainAId, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB2)));
137+
callHistory.Push(new OutCall(null, grainAId, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB2), Constants.AnyMethod));
139138
callHistory.Push(new InCall(null, grainAId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB2)));
140139
callHistory.Push(new OutCall(grainAId, grainBId, typeof(IGrainA).FullName!, typeof(IGrainB).FullName!, nameof(IGrainB.MethodC2), nameof(IGrainA.MethodB2)));
141140
callHistory.Push(new InCall(grainAId, grainBId, typeof(IGrainB).FullName!, nameof(IGrainB.MethodC2)));
142141

143142
graph.IsTransitionAllowed(callHistory).ShouldBeTrue();
144143
}
145144

145+
[Test]
146+
public void IsLatestTransitionAllowed_ClientOutgoingWithoutIncoming_DoesNotEnforceClientSideGraph()
147+
{
148+
var graph = GrainCallsBuilder.Create()
149+
.Build();
150+
151+
var callHistory = new CallHistory();
152+
callHistory.Push(new OutCall(
153+
null,
154+
GrainId.Create("graina", "client-side"),
155+
Constants.ClientCallerId,
156+
typeof(IGrainA).FullName!,
157+
nameof(IGrainA.MethodB1),
158+
Constants.AnyMethod));
159+
160+
graph.IsLatestTransitionAllowed(callHistory).ShouldBeTrue();
161+
}
162+
163+
[Test]
164+
public void IsLatestTransitionAllowed_GrainOutgoing_EnforcesLatestTransition()
165+
{
166+
var graph = GrainCallsBuilder.Create()
167+
.From<IGrainA>()
168+
.To<IGrainB>()
169+
.MethodByName(nameof(IGrainA.MethodB1), nameof(IGrainB.MethodB1))
170+
.And()
171+
.Build();
172+
173+
var callHistory = new CallHistory();
174+
callHistory.Push(new InCall(
175+
null,
176+
GrainId.Create("graina", "latest-valid"),
177+
typeof(IGrainA).FullName!,
178+
nameof(IGrainA.MethodB1)));
179+
callHistory.Push(new OutCall(
180+
GrainId.Create("graina", "latest-valid"),
181+
GrainId.Create("grainb", "latest-valid"),
182+
typeof(IGrainA).FullName!,
183+
typeof(IGrainB).FullName!,
184+
nameof(IGrainB.MethodB1),
185+
nameof(IGrainA.MethodB1)));
186+
187+
graph.IsLatestTransitionAllowed(callHistory).ShouldBeTrue();
188+
}
189+
190+
[Test]
191+
public void IsLatestTransitionAllowed_GrainOutgoing_RejectsMissingLatestTransition()
192+
{
193+
var graph = GrainCallsBuilder.Create()
194+
.From<IGrainA>()
195+
.To<IGrainB>()
196+
.MethodByName(nameof(IGrainA.MethodB1), nameof(IGrainB.MethodB1))
197+
.And()
198+
.Build();
199+
200+
var callHistory = new CallHistory();
201+
callHistory.Push(new InCall(
202+
null,
203+
GrainId.Create("graina", "latest-invalid"),
204+
typeof(IGrainA).FullName!,
205+
nameof(IGrainA.MethodB2)));
206+
callHistory.Push(new OutCall(
207+
GrainId.Create("graina", "latest-invalid"),
208+
GrainId.Create("grainb", "latest-invalid"),
209+
typeof(IGrainA).FullName!,
210+
typeof(IGrainB).FullName!,
211+
nameof(IGrainB.MethodC2),
212+
nameof(IGrainA.MethodB2)));
213+
214+
graph.IsLatestTransitionAllowed(callHistory).ShouldBeFalse();
215+
}
216+
146217
[Test]
147218
public void GetObservedGraph_ReturnsVerticesAndClientAndNestedGrainEdges()
148219
{
149220
var grainAId = GrainId.Create("graina", "observed");
150221
var grainBId = GrainId.Create("grainb", "observed");
151222

152223
var callHistory = new CallHistory();
153-
callHistory.Push(new OutCall(null, grainAId, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1)));
224+
callHistory.Push(new OutCall(null, grainAId, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1), Constants.AnyMethod));
154225
callHistory.Push(new InCall(null, grainAId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1)));
155226
callHistory.Push(new OutCall(grainAId, grainBId, typeof(IGrainA).FullName!, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1), nameof(IGrainA.MethodB1)));
156227
callHistory.Push(new InCall(grainAId, grainBId, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1)));
@@ -184,7 +255,7 @@ public void GetLatestObservedCall_ReturnsCurrentIncomingPair()
184255
var grainBId = GrainId.Create("grainb", "latest");
185256

186257
var callHistory = new CallHistory();
187-
callHistory.Push(new OutCall(null, grainAId, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1)));
258+
callHistory.Push(new OutCall(null, grainAId, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1), Constants.AnyMethod));
188259
callHistory.Push(new InCall(null, grainAId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1)));
189260
callHistory.Push(new OutCall(grainAId, grainBId, typeof(IGrainA).FullName!, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1), nameof(IGrainA.MethodB1)));
190261
callHistory.Push(new InCall(grainAId, grainBId, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1)));
@@ -198,6 +269,28 @@ public void GetLatestObservedCall_ReturnsCurrentIncomingPair()
198269
edge.TargetMethod.ShouldBe(nameof(IGrainB.MethodB1));
199270
}
200271

272+
[Test]
273+
public void GetObservedGraph_UsesExplicitCallerMethodForNonClientCallWithoutSourceId()
274+
{
275+
var callHistory = new CallHistory();
276+
callHistory.Push(new OutCall(
277+
null,
278+
null,
279+
typeof(IGrainA).FullName!,
280+
typeof(IGrainB).FullName!,
281+
nameof(IGrainB.MethodC2),
282+
nameof(IGrainA.MethodB2)));
283+
callHistory.Push(new InCall(null, null, typeof(IGrainB).FullName!, nameof(IGrainB.MethodC2)));
284+
285+
var graph = GrainTransitionManager.GetObservedGraph(callHistory);
286+
287+
graph.Edges.ShouldContain(edge =>
288+
edge.Source == typeof(IGrainA).FullName &&
289+
edge.Target == typeof(IGrainB).FullName &&
290+
edge.SourceMethod == nameof(IGrainA.MethodB2) &&
291+
edge.TargetMethod == nameof(IGrainB.MethodC2));
292+
}
293+
201294
[Test]
202295
public void GetObservedGraph_RejectsBaseGrainIdentity()
203296
{
@@ -209,7 +302,8 @@ public void GetObservedGraph_RejectsBaseGrainIdentity()
209302
grainAId,
210303
Constants.ClientCallerId,
211304
typeof(Grain).FullName!,
212-
nameof(IGrainA.MethodB1)));
305+
nameof(IGrainA.MethodB1),
306+
Constants.AnyMethod));
213307
callHistory.Push(new InCall(
214308
null,
215309
grainAId,
@@ -431,12 +525,85 @@ public void DetectDeadlocks_IgnoresReentrantTransitions()
431525
var grainId = GrainId.Create("test", nameof(IGrainA));
432526

433527
var callHistory = new CallHistory();
434-
callHistory.Push(new OutCall(grainId, grainId, typeof(IGrainA).FullName!, typeof(IGrainA).FullName!, nameof(IGrainA.MethodA1)));
528+
callHistory.Push(new OutCall(
529+
grainId,
530+
grainId,
531+
typeof(IGrainA).FullName!,
532+
typeof(IGrainA).FullName!,
533+
nameof(IGrainA.MethodA1),
534+
nameof(IGrainA.MethodA1)));
435535
callHistory.Push(new InCall(grainId, grainId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodA1)));
436536

437537
graph.DetectDeadlocks(callHistory).ShouldBeFalse();
438538
}
439539

540+
[Test]
541+
public void DetectLatestDeadlock_DetectsCycleClosedByLatestOutgoingCall()
542+
{
543+
var graph = GrainCallsBuilder.Create()
544+
.AllowAll()
545+
.Build();
546+
547+
var grainAId = GrainId.Create("graina", "latest-deadlock");
548+
var grainBId = GrainId.Create("grainb", "latest-deadlock");
549+
var grainCId = GrainId.Create("grainc", "latest-deadlock");
550+
551+
var callHistory = new CallHistory();
552+
callHistory.Push(new OutCall(
553+
grainAId,
554+
grainBId,
555+
typeof(IGrainA).FullName!,
556+
typeof(IGrainB).FullName!,
557+
nameof(IGrainB.MethodB1),
558+
nameof(IGrainA.MethodB1)));
559+
callHistory.Push(new OutCall(
560+
grainBId,
561+
grainCId,
562+
typeof(IGrainB).FullName!,
563+
typeof(IGrainC).FullName!,
564+
nameof(IGrainC.MethodC1),
565+
nameof(IGrainB.MethodB1)));
566+
callHistory.Push(new OutCall(
567+
grainCId,
568+
grainAId,
569+
typeof(IGrainC).FullName!,
570+
typeof(IGrainA).FullName!,
571+
nameof(IGrainA.MethodA1),
572+
nameof(IGrainC.MethodC1)));
573+
574+
graph.DetectLatestDeadlock(callHistory).ShouldBeTrue();
575+
}
576+
577+
[Test]
578+
public void DetectLatestDeadlock_ReturnsFalseWhenLatestOutgoingDoesNotCloseCycle()
579+
{
580+
var graph = GrainCallsBuilder.Create()
581+
.AllowAll()
582+
.Build();
583+
584+
var grainAId = GrainId.Create("graina", "latest-no-deadlock");
585+
var grainBId = GrainId.Create("grainb", "latest-no-deadlock");
586+
var grainCId = GrainId.Create("grainc", "latest-no-deadlock");
587+
588+
var callHistory = new CallHistory();
589+
callHistory.Push(new OutCall(
590+
grainAId,
591+
grainBId,
592+
typeof(IGrainA).FullName!,
593+
typeof(IGrainB).FullName!,
594+
nameof(IGrainB.MethodB1),
595+
nameof(IGrainA.MethodB1)));
596+
callHistory.Push(new OutCall(
597+
grainBId,
598+
grainCId,
599+
typeof(IGrainB).FullName!,
600+
typeof(IGrainC).FullName!,
601+
nameof(IGrainC.MethodC1),
602+
nameof(IGrainB.MethodB1)));
603+
604+
graph.DetectLatestDeadlock(callHistory).ShouldBeFalse();
605+
}
606+
440607
[Test]
441608
public void GeneratePolicyMermaidDiagram_ProducesEdges()
442609
{
@@ -494,9 +661,15 @@ public void GenerateLiveMermaidDiagram_HighlightsActiveEdges()
494661
var grainBId = GrainId.Create("grainb", "live-policy");
495662

496663
var callHistory = new CallHistory();
497-
callHistory.Push(new OutCall(null, grainAId, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1)));
664+
callHistory.Push(new OutCall(null, grainAId, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1), Constants.AnyMethod));
498665
callHistory.Push(new InCall(null, grainAId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1)));
499-
callHistory.Push(new OutCall(grainAId, grainBId, typeof(IGrainA).FullName!, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1)));
666+
callHistory.Push(new OutCall(
667+
grainAId,
668+
grainBId,
669+
typeof(IGrainA).FullName!,
670+
typeof(IGrainB).FullName!,
671+
nameof(IGrainB.MethodB1),
672+
nameof(IGrainA.MethodB1)));
500673
callHistory.Push(new InCall(grainAId, grainBId, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1)));
501674

502675
var diagram = graph.GenerateLiveMermaidDiagram(callHistory);
@@ -514,7 +687,7 @@ public void GenerateLiveMermaidDiagram_RendersObservedCallsWithoutPolicyEdges()
514687
.Build();
515688

516689
var callHistory = new CallHistory();
517-
callHistory.Push(new OutCall(null, null, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1)));
690+
callHistory.Push(new OutCall(null, null, Constants.ClientCallerId, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1), Constants.AnyMethod));
518691
callHistory.Push(new InCall(null, null, typeof(IGrainA).FullName!, nameof(IGrainA.MethodB1)));
519692

520693
var diagram = graph.GenerateLiveMermaidDiagram(callHistory);
@@ -595,9 +768,9 @@ public void GenerateLiveMermaidDiagram_AppendsUsageCounts()
595768
.Build();
596769

597770
var callHistory = new CallHistory();
598-
callHistory.Push(new OutCall(null, null, typeof(IGrainA).FullName!, typeof(IGrainB).FullName!, nameof(IGrainA.MethodB1)));
771+
callHistory.Push(new OutCall(null, null, typeof(IGrainA).FullName!, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1), nameof(IGrainA.MethodB1)));
599772
callHistory.Push(new InCall(null, null, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1)));
600-
callHistory.Push(new OutCall(null, null, typeof(IGrainA).FullName!, typeof(IGrainB).FullName!, nameof(IGrainA.MethodB1)));
773+
callHistory.Push(new OutCall(null, null, typeof(IGrainA).FullName!, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1), nameof(IGrainA.MethodB1)));
601774
callHistory.Push(new InCall(null, null, typeof(IGrainB).FullName!, nameof(IGrainB.MethodB1)));
602775

603776
var diagram = graph.GenerateLiveMermaidDiagram(callHistory);

‎ManagedCode.Orleans.Graph.Tests/RuntimeGraphInternalTests.cs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using ManagedCode.Orleans.Graph.Interfaces;
22
using ManagedCode.Orleans.Graph.Models;
3-
using ManagedCode.Orleans.Graph.Telemetry;
43
using ManagedCode.Orleans.Graph.Tests.Cluster.Grains.Interfaces;
54
using ManagedCode.Orleans.Graph.Tests.RuntimeGraphCluster;
65

@@ -28,14 +27,19 @@ await _fixture.Cluster.Client
2827
{
2928
var graph = await telemetry.GetObservedGraphAsync();
3029
edges = graph.Edges;
31-
if (edges.Any(edge => edge.Target == typeof(OrleansGraphTelemetryWorker).FullName))
30+
if (edges.Any(edge => edge.Target == typeof(IOrleansGraphTelemetryWorker).FullName))
3231
{
3332
break;
3433
}
3534

3635
await Task.Delay(100);
3736
}
3837

39-
edges.ShouldContain(edge => edge.Target == typeof(OrleansGraphTelemetryWorker).FullName);
38+
edges.ShouldContain(edge => edge.Target == typeof(IOrleansGraphTelemetryWorker).FullName);
39+
edges.ShouldContain(edge =>
40+
edge.Source == typeof(IOrleansGraphTelemetryWorker).FullName &&
41+
edge.Target == typeof(IOrleansGraphTelemetryGrain).FullName &&
42+
edge.SourceMethod == nameof(IOrleansGraphTelemetryWorker.FlushAsync) &&
43+
edge.TargetMethod == nameof(IOrleansGraphTelemetryGrain.MergeObservedCallsAsync));
4044
}
4145
}

0 commit comments

Comments
 (0)