Skip to content

Commit c4087c3

Browse files
Lucki2gLucki2g
authored
Merge pull request #86 from delegateas/fix/query-pagination-wrapper
Fix/query pagination wrapper
2 parents d08fa5a + 8538f0a commit c4087c3

12 files changed

Lines changed: 95 additions & 41 deletions

‎Generator/ClientExtensions.cs‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,66 @@
11
using Microsoft.PowerPlatform.Dataverse.Client;
2+
using Microsoft.Xrm.Sdk;
23
using Microsoft.Xrm.Sdk.Messages;
34
using Microsoft.Xrm.Sdk.Metadata;
5+
using Microsoft.Xrm.Sdk.Query;
46

57
namespace Generator;
68

79
public static class ClientExtensions
810
{
11+
/// <summary>
12+
/// Retrieves all pages of records matching the query, automatically following Dataverse pagination.
13+
/// Dataverse returns at most 5000 records per page by default; this wrapper collects all pages.
14+
/// </summary>
15+
public static async Task<List<Entity>> RetrieveAllAsync(this ServiceClient client, QueryExpression query, CancellationToken cancellationToken = default)
16+
{
17+
var results = new List<Entity>();
18+
query.PageInfo = new PagingInfo
19+
{
20+
Count = 5000,
21+
PageNumber = 1,
22+
PagingCookie = null,
23+
ReturnTotalRecordCount = false
24+
};
25+
26+
EntityCollection response;
27+
do
28+
{
29+
response = await client.RetrieveMultipleAsync(query, cancellationToken);
30+
results.AddRange(response.Entities);
31+
query.PageInfo.PageNumber++;
32+
query.PageInfo.PagingCookie = response.PagingCookie;
33+
} while (response.MoreRecords);
34+
35+
return results;
36+
}
37+
38+
/// <summary>
39+
/// Synchronous variant of RetrieveAllAsync for use in non-async contexts.
40+
/// </summary>
41+
public static List<Entity> RetrieveAll(this ServiceClient client, QueryExpression query)
42+
{
43+
var results = new List<Entity>();
44+
query.PageInfo = new PagingInfo
45+
{
46+
Count = 5000,
47+
PageNumber = 1,
48+
PagingCookie = null,
49+
ReturnTotalRecordCount = false
50+
};
51+
52+
EntityCollection response;
53+
do
54+
{
55+
response = client.RetrieveMultiple(query);
56+
results.AddRange(response.Entities);
57+
query.PageInfo.PageNumber++;
58+
query.PageInfo.PagingCookie = response.PagingCookie;
59+
} while (response.MoreRecords);
60+
61+
return results;
62+
}
63+
964
public static async Task<EntityMetadata> RetrieveEntityAsync(this ServiceClient client, Guid objectId, CancellationToken token)
1065
{
1166
var resp = await client.ExecuteAsync(new RetrieveEntityRequest()

‎Generator/DataverseService.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,8 @@ public DataverseService(
8282
IEnumerable<ComponentInfo> solutionComponents;
8383
try
8484
{
85-
logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Calling solutionComponentService.GetAllSolutionComponents()");
86-
solutionComponents = solutionComponentService.GetAllSolutionComponents(solutionIds);
85+
logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Calling solutionComponentService.GetAllSolutionComponentsAsync()");
86+
solutionComponents = await solutionComponentService.GetAllSolutionComponentsAsync(solutionIds);
8787
logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieved {solutionComponents.Count()} solution components");
8888
}
8989
catch (Exception ex)
@@ -219,7 +219,7 @@ public DataverseService(
219219

220220
// Get workflow dependencies for attributes (returns attribute ObjectId -> list of workflow ObjectIds)
221221
var explicitComponentsList = solutionComponents.ToList();
222-
var workflowDependencyMap = solutionComponentService.GetWorkflowDependenciesForAttributes(
222+
var workflowDependencyMap = await solutionComponentService.GetWorkflowDependenciesForAttributesAsync(
223223
explicitComponentsList.Where(c => c.ComponentType == 2).Select(c => new SolutionComponentInfo(
224224
c.ObjectId,
225225
c.SolutionComponentId ?? Guid.Empty,

‎Generator/Queries/PluginQueries.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,9 @@ public static async Task<IEnumerable<SDKStep>> GetSDKMessageProcessingStepsAsync
7171

7272

7373
//if (solutionIds is not null) query.Criteria.Conditions.Add(new ConditionExpression("solutionid", ConditionOperator.In, solutionIds));
74-
var result = await service.RetrieveMultipleAsync(query);
74+
var result = await service.RetrieveAllAsync(query);
7575

76-
var steps = result.Entities.Select(e =>
76+
var steps = result.Select(e =>
7777
{
7878
var sdkMessageId = e.GetAttributeValue<AliasedValue>("step.sdkmessageid")?.Value as EntityReference;
7979
var sdkStepName = e.GetAttributeValue<AliasedValue>("step.name")?.Value as string;

‎Generator/Queries/PowerAutomateQueries.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@ public static async Task<IEnumerable<PowerAutomateFlow>> GetPowerAutomateFlowsAs
4242
}
4343
};
4444

45-
var result = await service.RetrieveMultipleAsync(query);
45+
var result = await service.RetrieveAllAsync(query);
4646

47-
var flows = result.Entities.Select(e =>
47+
var flows = result.Select(e =>
4848
{
4949
return new PowerAutomateFlow(
5050
e.GetAttributeValue<AliasedValue>("flow.workflowid").Value as string,

‎Generator/Queries/WebResourceQueries.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public static async Task<IEnumerable<WebResource>> GetWebResourcesAsync(this Ser
4343
}
4444
};
4545

46-
var results = (await service.RetrieveMultipleAsync(query)).Entities;
46+
var results = await service.RetrieveAllAsync(query);
4747

4848
var webResources = results.Select(e =>
4949
{

‎Generator/Services/EntityIconService.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ public async Task<Dictionary<string, string>> GetEntityIconMap(IEnumerable<Entit
4343
}
4444
};
4545

46-
var webresources = await client.RetrieveMultipleAsync(query);
47-
iconNameToSvg = webresources.Entities.ToDictionary(x => x.GetAttributeValue<string>("name"), x => x.GetAttributeValue<string>("content"));
46+
var webresources = await client.RetrieveAllAsync(query);
47+
iconNameToSvg = webresources.ToDictionary(x => x.GetAttributeValue<string>("name"), x => x.GetAttributeValue<string>("content"));
4848
}
4949

5050
var logicalNameToSvg =

‎Generator/Services/SecurityRoleService.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,9 @@ public async Task<Dictionary<string, List<SecurityRole>>> GetSecurityRoles(
6464
}
6565
};
6666

67-
var roles = await client.RetrieveMultipleAsync(query);
67+
var roles = await client.RetrieveAllAsync(query);
6868

69-
var rolePrivileges = roles.Entities.Select(e =>
69+
var rolePrivileges = roles.Select(e =>
7070
{
7171
var name = e.GetAttributeValue<string>("name");
7272
var depth = (PrivilegeDepth)e.GetAttributeValue<AliasedValue>("rolepriv.privilegedepthmask").Value;

‎Generator/Services/SolutionComponentExtractor.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,8 @@ private async Task<List<RawComponentInfo>> QuerySolutionComponentsAsync(List<Gui
170170

171171
try
172172
{
173-
var response = await _client.RetrieveMultipleAsync(query);
174-
foreach (var entity in response.Entities)
173+
var entities = await _client.RetrieveAllAsync(query);
174+
foreach (var entity in entities)
175175
{
176176
var componentType = entity.GetAttributeValue<OptionSetValue>("componenttype")?.Value ?? 0;
177177
var objectId = entity.GetAttributeValue<Guid>("objectid");
@@ -315,8 +315,8 @@ private async Task<List<RawComponentInfo>> QuerySolutionComponentsAsync(List<Gui
315315
}
316316
};
317317

318-
var response = await _client.RetrieveMultipleAsync(query);
319-
foreach (var entity in response.Entities)
318+
var entities = await _client.RetrieveAllAsync(query);
319+
foreach (var entity in entities)
320320
{
321321
var id = entity.GetAttributeValue<Guid>(primaryKey);
322322
var name = entity.GetAttributeValue<string>(nameColumn) ?? id.ToString();

‎Generator/Services/SolutionComponentService.cs‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public SolutionComponentService(ServiceClient client, IConfiguration configurati
6565
_logger = logger;
6666
}
6767

68-
public IEnumerable<ComponentInfo> GetAllSolutionComponents(List<Guid> solutionIds)
68+
public async Task<IEnumerable<ComponentInfo>> GetAllSolutionComponentsAsync(List<Guid> solutionIds)
6969
{
7070
_logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Getting all solution components for solutions: {string.Join(", ", solutionIds)}");
7171
var allComponents = new HashSet<ComponentInfo>();
@@ -74,7 +74,7 @@ public IEnumerable<ComponentInfo> GetAllSolutionComponents(List<Guid> solutionId
7474
IEnumerable<SolutionComponentInfo> explicitComponents;
7575
try
7676
{
77-
explicitComponents = GetExplicitComponents(solutionIds);
77+
explicitComponents = await GetExplicitComponentsAsync(solutionIds);
7878
_logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Found {explicitComponents.Count()} explicit components");
7979
}
8080
catch (Exception ex)
@@ -118,7 +118,7 @@ public IEnumerable<ComponentInfo> GetAllSolutionComponents(List<Guid> solutionId
118118
_logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieving component information for {requiredNodeIds.Count} dependency nodes");
119119

120120
// Retrieve component node information
121-
var componentNodes = GetComponentNodes(requiredNodeIds);
121+
var componentNodes = await GetComponentNodesAsync(requiredNodeIds);
122122
_logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieved {componentNodes.Count} component nodes");
123123

124124
// Add required components as implicit
@@ -152,7 +152,7 @@ public IEnumerable<ComponentInfo> GetAllSolutionComponents(List<Guid> solutionId
152152
return allComponents;
153153
}
154154

155-
private IEnumerable<SolutionComponentInfo> GetExplicitComponents(List<Guid> solutionIds)
155+
private async Task<IEnumerable<SolutionComponentInfo>> GetExplicitComponentsAsync(List<Guid> solutionIds)
156156
{
157157
var query = new QueryExpression("solutioncomponent")
158158
{
@@ -167,16 +167,16 @@ private IEnumerable<SolutionComponentInfo> GetExplicitComponents(List<Guid> solu
167167
}
168168
};
169169

170-
return _client.RetrieveMultiple(query).Entities
171-
.Select(e => new SolutionComponentInfo(
172-
e.GetAttributeValue<Guid>("objectid"),
173-
e.GetAttributeValue<Guid>("solutioncomponentid"),
174-
e.GetAttributeValue<OptionSetValue>("componenttype").Value,
175-
e.Contains("rootcomponentbehavior") ? e.GetAttributeValue<OptionSetValue>("rootcomponentbehavior").Value : -1,
176-
e.GetAttributeValue<EntityReference>("solutionid")));
170+
var entities = await _client.RetrieveAllAsync(query);
171+
return entities.Select(e => new SolutionComponentInfo(
172+
e.GetAttributeValue<Guid>("objectid"),
173+
e.GetAttributeValue<Guid>("solutioncomponentid"),
174+
e.GetAttributeValue<OptionSetValue>("componenttype").Value,
175+
e.Contains("rootcomponentbehavior") ? e.GetAttributeValue<OptionSetValue>("rootcomponentbehavior").Value : -1,
176+
e.GetAttributeValue<EntityReference>("solutionid")));
177177
}
178178

179-
private List<ComponentNodeInfo> GetComponentNodes(List<Guid> nodeIds)
179+
private async Task<List<ComponentNodeInfo>> GetComponentNodesAsync(List<Guid> nodeIds)
180180
{
181181
if (!nodeIds.Any())
182182
{
@@ -198,8 +198,8 @@ private List<ComponentNodeInfo> GetComponentNodes(List<Guid> nodeIds)
198198

199199
try
200200
{
201-
var response = _client.RetrieveMultiple(query);
202-
foreach (var entity in response.Entities)
201+
var entities = await _client.RetrieveAllAsync(query);
202+
foreach (var entity in entities)
203203
{
204204
results.Add(new ComponentNodeInfo(
205205
entity.GetAttributeValue<Guid>("dependencynodeid"),
@@ -374,7 +374,7 @@ private HashSet<DependencyInfo> GetRequiredComponents(IEnumerable<SolutionCompon
374374
/// </summary>
375375
/// <param name="attributeComponents">List of attribute components to check for dependencies</param>
376376
/// <returns>Dictionary mapping attribute ObjectId to list of workflow ObjectIds that depend on it</returns>
377-
public Dictionary<Guid, List<Guid>> GetWorkflowDependenciesForAttributes(IEnumerable<SolutionComponentInfo> attributeComponents)
377+
public async Task<Dictionary<Guid, List<Guid>>> GetWorkflowDependenciesForAttributesAsync(IEnumerable<SolutionComponentInfo> attributeComponents)
378378
{
379379
var workflowDependencies = new Dictionary<Guid, List<Guid>>();
380380

@@ -406,7 +406,7 @@ public Dictionary<Guid, List<Guid>> GetWorkflowDependenciesForAttributes(IEnumer
406406
return workflowDependencies;
407407

408408
// Retrieve component node information for all nodes
409-
var allNodes = GetComponentNodes(allNodeIds);
409+
var allNodes = await GetComponentNodesAsync(allNodeIds);
410410

411411
// Filter to only workflow components (type 29)
412412
var workflowNodes = allNodes.Where(n => n.ComponentType == 29).ToList();

‎Generator/Services/SolutionService.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg
3434
}
3535
var solutionNames = solutionNameArg.Split(",").Select(x => x.Trim().ToLower()).ToList();
3636

37-
var resp = await client.RetrieveMultipleAsync(new QueryExpression("solution")
37+
var entities = await client.RetrieveAllAsync(new QueryExpression("solution")
3838
{
3939
ColumnSet = new ColumnSet("publisherid", "friendlyname", "uniquename", "solutionid"),
4040
Criteria = new FilterExpression(LogicalOperator.And)
@@ -46,7 +46,7 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg
4646
}
4747
});
4848

49-
return (resp.Entities.Select(e => e.GetAttributeValue<Guid>("solutionid")).ToList(), resp.Entities.ToList());
49+
return (entities.Select(e => e.GetAttributeValue<Guid>("solutionid")).ToList(), entities);
5050
}
5151

5252
/// <summary>
@@ -73,8 +73,8 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg
7373
}
7474
};
7575

76-
var publishers = await client.RetrieveMultipleAsync(publisherQuery);
77-
return publishers.Entities.ToDictionary(
76+
var publishers = await client.RetrieveAllAsync(publisherQuery);
77+
return publishers.ToDictionary(
7878
p => p.GetAttributeValue<Guid>("publisherid"),
7979
p => (
8080
Name: p.GetAttributeValue<string>("friendlyname") ?? "Unknown Publisher",

0 commit comments

Comments
 (0)