Skip to content

Commit 396b163

Browse files
imadityaaAditya Abhishek
imadityaa
and
Aditya Abhishek
authored
Modifications in the Script based Executors (#536)
* scriptExecChanges * add doc * nit * resolve comments * deserialize to Metrics --------- Co-authored-by: Aditya Abhishek <adityaa@microsoft.com>
1 parent 7df7120 commit 396b163

6 files changed

Lines changed: 225 additions & 21 deletions

File tree

‎src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/ScriptExecutor/validJsonExample_array.json‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
[
2+
{
3+
"Name": "metric1",
4+
"Value": 0,
5+
"Unit": "unit1",
6+
"MetaData": {
7+
"metadata1": 0.1,
8+
"metadata2": "m2"
9+
}
10+
},
11+
{
12+
"Name": "metric2",
13+
"Value": -1,
14+
"Unit": "unit2",
15+
"MetaData": {
16+
"metadata1": "m3",
17+
"metadata2": true
18+
}
19+
},
20+
{
21+
"Name": "metric3",
22+
"Value": 1.2,
23+
"Unit": "unit3",
24+
"MetaData": {
25+
"metadata1": "m5",
26+
"metadata2": -2
27+
}
28+
},
29+
{
30+
"Name": "metric4",
31+
"Value": 1.0,
32+
"MetaData": {
33+
"metadata1": "m7",
34+
"metadata2": "m8"
35+
}
36+
},
37+
{
38+
"Name": "metric5",
39+
"Value": "1.24",
40+
"Unit": "unit5"
41+
},
42+
{
43+
"Name": "metric6",
44+
"Value": "-5.8"
45+
}
46+
]

‎src/VirtualClient/VirtualClient.Actions.UnitTests/ScriptExecutor/JsonMetricsParserTests.cs‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ class JsonMetricsParserTests
2020
private JsonMetricsParser testParser;
2121

2222
[Test]
23-
public void JsonMetricsParserVerifyMetricsForPassResults()
23+
public void JsonMetricsParserVerifyMetricsForPassResults_Format1()
2424
{
2525
string resultsPath = MockFixture.GetDirectory(typeof(JsonMetricsParserTests), "Examples", "ScriptExecutor", "validJsonExample.json");
2626
string rawText = File.ReadAllText(resultsPath);
@@ -33,6 +33,41 @@ public void JsonMetricsParserVerifyMetricsForPassResults()
3333
MetricAssert.Exists(metrics, "metric3", 1279854282929.09);
3434
}
3535

36+
[Test]
37+
public void JsonMetricsParserVerifyMetricsForPassResults_ArrayFormat()
38+
{
39+
string resultsPath = MockFixture.GetDirectory(typeof(JsonMetricsParserTests), "Examples", "ScriptExecutor", "validJsonExample_array.json");
40+
string rawText = File.ReadAllText(resultsPath);
41+
this.testParser = new JsonMetricsParser(rawText, new InMemoryLogger(), EventContext.None);
42+
IList<Metric> metrics = this.testParser.Parse();
43+
44+
Assert.AreEqual(6, metrics.Count);
45+
MetricAssert.Exists(metrics, "metric1", 0);
46+
MetricAssert.Exists(metrics, "metric2", -1);
47+
MetricAssert.Exists(metrics, "metric3", 1.2);
48+
MetricAssert.Exists(metrics, "metric4", 1);
49+
MetricAssert.Exists(metrics, "metric5", 1.24);
50+
MetricAssert.Exists(metrics, "metric6", -5.8);
51+
}
52+
53+
[Test]
54+
public void JsonMetricsParserThrowsIfTheJsonArrayResultsHaveInvalidMetrics()
55+
{
56+
string rawText = "[\r\n\t{\r\n\t\t\"Name\": \"metric3\",\r\n\t\t\"Value\": \"a1\",\r\n\t\t\"Unit\": \"unit3\",\r\n\t\t\"MetaData\": {\r\n\t\t\t\"metadata1\": \"m5\",\r\n\t\t\t\"metadata2\": \"m6\"\r\n\t\t}\r\n\t},\r\n\t{\r\n\t\t\"Name\": \"metric4\",\r\n\t\t\"Value\": 1.0,\r\n\t\t\"MetaData\": {\r\n\t\t\t\"metadata1\": \"m7\"\r\n\t\t}\r\n\t}\r\n]";
57+
58+
this.testParser = new JsonMetricsParser(rawText, new InMemoryLogger(), EventContext.None);
59+
Assert.Throws<WorkloadResultsException>(() => this.testParser.Parse());
60+
}
61+
62+
[Test]
63+
public void JsonMetricsParserThrowsIfTheJsonArrayResultsHaveMisingMetricName()
64+
{
65+
string rawText = "[\r\n\t{\r\n\t\t\"Value\": 0,\r\n\t\t\"Unit\": \"unit3\",\r\n\t\t\"MetaData\": {\r\n\t\t\t\"metadata1\": \"m5\",\r\n\t\t\t\"metadata2\": \"m6\"\r\n\t\t}\r\n\t},\r\n\t{\r\n\t\t\"Name\": \"metric4\",\r\n\t\t\"Value\": 1.0,\r\n\t\t\"MetaData\": {\r\n\t\t\t\"metadata1\": \"m7\"\r\n\t\t}\r\n\t}\r\n]";
66+
67+
this.testParser = new JsonMetricsParser(rawText, new InMemoryLogger(), EventContext.None);
68+
Assert.Throws<WorkloadResultsException>(() => this.testParser.Parse());
69+
}
70+
3671
[Test]
3772
public void JsonMetricsParserThrowsIfTheJsonResultsHaveInvalidMetrics()
3873
{

‎src/VirtualClient/VirtualClient.Actions/ScriptExecutor/JsonMetricsParser.cs‎

Lines changed: 52 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,7 @@ namespace VirtualClient.Actions
55
{
66
using System;
77
using System.Collections.Generic;
8-
using System.ComponentModel;
98
using System.Linq;
10-
using System.Text.RegularExpressions;
119
using global::VirtualClient;
1210
using global::VirtualClient.Actions;
1311
using global::VirtualClient.Contracts;
@@ -46,20 +44,64 @@ public override IList<Metric> Parse()
4644

4745
try
4846
{
49-
JObject keyValuePairs = JObject.Parse(this.RawText);
50-
foreach (var keyValuePair in keyValuePairs.Properties())
47+
JToken token = JToken.Parse(this.RawText);
48+
49+
if (token.Type == JTokenType.Object)
50+
{
51+
// Format 1: { "metric1": 123, "metric2": 2.5, ... }
52+
JObject keyValuePairs = (JObject)token;
53+
foreach (var keyValuePair in keyValuePairs.Properties())
54+
{
55+
if (double.TryParse(keyValuePair.Value.ToString(), out var value))
56+
{
57+
metrics.Add(new Metric(keyValuePair.Name, value, MetricRelativity.Undefined));
58+
}
59+
else
60+
{
61+
throw new WorkloadResultsException(
62+
$"Invalid JSON metrics content formatting. The metric value for '{keyValuePair.Name}' is not a valid numeric data type. Provided metric value is '{keyValuePair.Value}'",
63+
ErrorReason.InvalidResults);
64+
}
65+
}
66+
}
67+
else if (token.Type == JTokenType.Array)
5168
{
52-
if (double.TryParse(keyValuePair.Value.ToString(), out var value))
69+
/* Format 2:
70+
[
71+
{
72+
"Nmae": "metric1",
73+
"Value": 0,
74+
"Unit": "unit1",
75+
"MetaData": {
76+
"metadata1": "m1",
77+
"metadata2": "m2"
78+
}
79+
}
80+
]
81+
*/
82+
try
5383
{
54-
metrics.Add(new Metric(keyValuePair.Name, value, MetricRelativity.Undefined));
84+
metrics = JsonConvert.DeserializeObject<List<Metric>>(this.RawText);
5585
}
56-
else
86+
catch (Exception exc)
5787
{
5888
throw new WorkloadResultsException(
59-
$"Invalid JSON metrics content formatting. The metric value for '{keyValuePair.Name}' is not a valid numeric data type.",
89+
"Invalid JSON metrics content formatting. Failed to deserialze the Array JSON Contents into Metrics format.", exc, ErrorReason.InvalidResults);
90+
}
91+
92+
if (metrics.Any(m => string.IsNullOrWhiteSpace(m.Name)))
93+
{
94+
throw new WorkloadResultsException(
95+
"Invalid JSON metrics content formatting. 'Name' is a required property for each metric, it can't be null or whitespace.",
6096
ErrorReason.InvalidResults);
6197
}
6298
}
99+
else
100+
{
101+
throw new WorkloadResultsException(
102+
"Invalid JSON metrics content formatting. The root element must be either an object or an array.",
103+
ErrorReason.InvalidResults);
104+
}
63105
}
64106
catch (WorkloadResultsException)
65107
{
@@ -68,8 +110,8 @@ public override IList<Metric> Parse()
68110
catch (Exception exc)
69111
{
70112
throw new WorkloadResultsException(
71-
$"Invalid JSON metrics content formatting. The metrics content must be in a valid JSON key/value pair JSON-format " +
72-
$"(e.g. {{ \"metric1\": 1234, \"metric2\": 987.65, \"metric3\": 32.0023481 }} )",
113+
$"Invalid JSON metrics content formatting. The metrics content must be in a valid JSON key/value pair format " +
114+
$"(e.g. {{ \"metric1\": 1234, \"metric2\": 987.65, \"metric3\": 32.0023481 }} ) or an array of Json objects.",
73115
exc,
74116
ErrorReason.InvalidResults);
75117
}

‎src/VirtualClient/VirtualClient.Actions/ScriptExecutor/ScriptExecutor.cs‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ protected async Task CaptureLogsAsync(CancellationToken cancellationToken)
261261
{
262262
foreach (string logFilePath in this.fileSystem.Directory.GetFiles(fullLogPath, "*", SearchOption.AllDirectories))
263263
{
264-
this.RequestUploadAndMoveToLogsDirectory(logFilePath, destinitionLogsDir, cancellationToken);
264+
this.RequestUploadAndMoveToLogsDirectory(logFilePath, destinitionLogsDir, cancellationToken, sourceRoot: fullLogPath);
265265
}
266266
}
267267

@@ -301,9 +301,13 @@ protected Task RequestUpload(string logPath)
301301
}
302302

303303
/// <summary>
304-
/// Move the log files to central logs directory and Upload to Content Store
304+
/// Move the log files to central logs directory (retaining source directory structure) and Upload to Content Store.
305305
/// </summary>
306-
protected async Task RequestUploadAndMoveToLogsDirectory(string sourcePath, string destinitionDirectory, CancellationToken cancellationToken)
306+
private async Task RequestUploadAndMoveToLogsDirectory(
307+
string sourcePath,
308+
string destinitionDirectory,
309+
CancellationToken cancellationToken,
310+
string sourceRoot = null)
307311
{
308312
if (string.Equals(sourcePath, this.ExecutablePath))
309313
{
@@ -317,13 +321,27 @@ protected async Task RequestUploadAndMoveToLogsDirectory(string sourcePath, stri
317321

318322
await (this.FileOperationsRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(() =>
319323
{
320-
// e.g.
321-
// /logs/anytool/executecustomscript1/2023-06-27T21-13-12-51001Z-CustomScript.sh
322-
// /logs/anytool/executecustomscript1/2023-06-27T21-15-36-12018Z-CustomScript.sh
323324
string fileName = Path.GetFileName(sourcePath);
324-
string destinitionPath = this.Combine(destinitionDirectory, BlobDescriptor.SanitizeBlobPath($"{DateTime.UtcNow.ToString("o").Replace('.', '-')}-{fileName}"));
325-
this.fileSystem.File.Move(sourcePath, destinitionPath, true);
325+
string destPath;
326326

327+
if (!string.IsNullOrEmpty(sourceRoot))
328+
{
329+
// Compute relative path from sourceRoot to sourcePath
330+
string relativePath = this.fileSystem.Path.GetRelativePath(sourceRoot, sourcePath);
331+
string destDir = this.fileSystem.Path.Combine(destinitionDirectory, this.fileSystem.Path.GetDirectoryName(relativePath));
332+
if (!this.fileSystem.Directory.Exists(destDir))
333+
{
334+
this.fileSystem.Directory.CreateDirectory(destDir);
335+
}
336+
337+
destPath = this.fileSystem.Path.Combine(destDir, BlobDescriptor.SanitizeBlobPath($"{DateTime.UtcNow:O}".Replace('.', '-') + "-" + fileName));
338+
}
339+
else
340+
{
341+
destPath = this.Combine(destinitionDirectory, BlobDescriptor.SanitizeBlobPath($"{DateTime.UtcNow:O}".Replace('.', '-') + "-" + fileName));
342+
}
343+
344+
this.fileSystem.File.Move(sourcePath, destPath, true);
327345
return Task.CompletedTask;
328346
});
329347
}

‎src/VirtualClient/VirtualClient.Contracts/Metric.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ namespace VirtualClient.Contracts
88
using System.Diagnostics;
99
using System.Linq;
1010
using System.Text;
11+
using Newtonsoft.Json;
12+
using VirtualClient.Common.Contracts;
1113

1214
/// <summary>
1315
/// Represents the result of a single metric
@@ -18,6 +20,7 @@ public class Metric : IEquatable<Metric>
1820
/// <summary>
1921
/// Creates a metric
2022
/// </summary>
23+
[JsonConstructor]
2124
public Metric(string name, double value)
2225
{
2326
this.Name = name;
@@ -133,6 +136,7 @@ public Metric(string name, double value, string unit, MetricRelativity relativit
133136
/// <summary>
134137
/// Telemetry context for metric.
135138
/// </summary>
139+
[JsonConverter(typeof(ParameterDictionaryJsonConverter))]
136140
public IDictionary<string, IConvertible> Metadata { get; }
137141

138142
/// <summary>

‎website/docs/guides/0221-usage-extensions.md‎

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -471,8 +471,9 @@ and requirements.
471471
* [Geekbench Workload Metrics](https://microsoft.github.io/VirtualClient/docs/workloads/geekbench/)
472472

473473
Virtual Client provides a facility for script-based automation to emit metrics for capture as well. To enable metrics capture, scripts emit the metrics to a single/central file on the file
474-
system. The file should be named ```test-metrics.json``` and should exist in the same directory as the script that generated it. The file contents should be a simple JSON-formatted structure
475-
as illustrated below. Virtual Client will read this file and upload the metrics defined within alongside any out-of-box metrics already captured.
474+
system. The file should be named ```test-metrics.json``` and should exist in the same directory as the script that generated it. There are two acceptable formats for test-metrics.json.
475+
476+
* The file contents are a simple JSON-formatted structure as illustrated below. Virtual Client will read this file and upload the metrics defined within, alongside any out-of-box metrics already captured.
476477

477478
``` json
478479
# Example contents of the 'test-metrics.json' file. Simple key/value pairs. This file should
@@ -489,6 +490,64 @@ and requirements.
489490
}
490491
```
491492

493+
* The file contents are formatted as a JSON array as illustrated below. Virtual Client will read this file and upload the metrics defined within, alongside any out-of-box metrics already captured.
494+
495+
``` json
496+
# Example contents of the 'test-metrics.json' file. JSON Array based structure. This file should
497+
# be written to the same directory where the script that generated it exists. Here, 'metricName' and 'metricValue' are mandatory fields, while 'metricUnit' and 'metricMetadata' are optional. The field 'metricMetadata' can be used to provide additional information about the metric, such as the source of the metric or any other relevant context. It is a key-value pair structure.
498+
#
499+
# e.g.
500+
# Given a script /any.script.extensions.1.0.0/linux-x64/install.py, the file should be
501+
# written to /any.script.extensions.1.0.0/linux-x64/test-metrics.json
502+
503+
[
504+
{
505+
"Name": "metric1",
506+
"Value": 0,
507+
"Unit": "unit1",
508+
"MetaData": {
509+
"metadata1": "m1",
510+
"metadata2": "m2"
511+
}
512+
},
513+
{
514+
"Name": "metric2",
515+
"Value": -1,
516+
"Unit": "unit2",
517+
"MetaData": {
518+
"metadata1": "m3",
519+
"metadata2": "m4"
520+
}
521+
},
522+
{
523+
"Name": "metric3",
524+
"Value": 1.2,
525+
"Unit": "unit3",
526+
"MetaData": {
527+
"metadata1": "m5",
528+
"metadata2": "m6"
529+
}
530+
},
531+
{
532+
"Name": "metric4",
533+
"Value": 1.0,
534+
"MetaData": {
535+
"metadata1": "m7",
536+
"metadata2": "m8"
537+
}
538+
},
539+
{
540+
"Name": "metric5",
541+
"Value": "1.24",
542+
"Unit": "unit5"
543+
},
544+
{
545+
"Name": "metric6",
546+
"Value": "-5.8"
547+
}
548+
]
549+
```
550+
492551
### Out-of-Box Components for Execution of Script-Based Automation
493552
The following section provides examples of how scripts can be integrated into the Virtual Client using out-of-box features to integrate script-based
494553
extensions.

0 commit comments

Comments
 (0)