Skip to content

Commit 52f9c76

Browse files
ChristopherHXChristopherHX
authored
Move some classes out of MessageController (#569)
* secretprovider * template context creation
1 parent c14e037 commit 52f9c76

12 files changed

Lines changed: 426 additions & 367 deletions

‎src/Runner.Server/Controllers/MessageController.cs‎

Lines changed: 2 additions & 366 deletions
Large diffs are not rendered by default.

‎src/Runner.Server/Services/AzurePipelinesVariablesProvider.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
using System.Collections.Generic;
2+
3+
namespace Runner.Server.Services
4+
{
5+
6+
public class AzurePipelinesVariablesProvider : IVariablesProvider
7+
{
8+
private readonly ISecretsProvider parent;
9+
private readonly IDictionary<string, string> rootVars;
10+
11+
public AzurePipelinesVariablesProvider(ISecretsProvider parent, IDictionary<string, string> rootVars){
12+
this.parent = parent;
13+
this.rootVars = rootVars;
14+
}
15+
16+
public IDictionary<string, string> GetVariablesForEnvironment(string name = null)
17+
{
18+
if(string.IsNullOrEmpty(name)) {
19+
return rootVars;
20+
}
21+
return parent.GetVariablesForEnvironment(name);
22+
}
23+
}
24+
}

‎src/Runner.Server/Services/DefaultSecretsProvider.cs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using Microsoft.Extensions.Configuration;
5+
6+
namespace Runner.Server.Services
7+
{
8+
public class DefaultSecretsProvider : ISecretsProvider
9+
{
10+
private Dictionary<string, Dictionary<string, string>> SecretsEnvironments { get; set; }
11+
private Dictionary<string, Dictionary<string, string>> VarEnvironments { get; set; }
12+
public DefaultSecretsProvider(IConfiguration configuration) {
13+
var secenvs = (configuration.GetSection("Runner.Server:Environments").Get<Dictionary<string, Dictionary<string, string>>>() ?? new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase)).ToArray();
14+
SecretsEnvironments = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
15+
foreach(var secenv in secenvs) {
16+
SecretsEnvironments[secenv.Key] = new Dictionary<string, string>(secenv.Value, StringComparer.OrdinalIgnoreCase);
17+
}
18+
var globalSecrets = configuration.GetSection("Runner.Server:Secrets")?.Get<List<Secret>>() ?? new List<Secret>();
19+
if(!SecretsEnvironments.ContainsKey("")) {
20+
SecretsEnvironments[""] = globalSecrets.ToDictionary(sec => sec.Name, sec => sec.Value, StringComparer.OrdinalIgnoreCase);
21+
}
22+
var varenvs = (configuration.GetSection("Runner.Server:VarEnvironments").Get<Dictionary<string, Dictionary<string, string>>>() ?? new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase)).ToArray();
23+
VarEnvironments = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
24+
foreach(var varenv in varenvs) {
25+
VarEnvironments[varenv.Key] = new Dictionary<string, string>(varenv.Value, StringComparer.OrdinalIgnoreCase);
26+
}
27+
28+
}
29+
public IDictionary<string, string> GetSecretsForEnvironment(GitHub.DistributedTask.ObjectTemplating.ITraceWriter traceWriter, string name = null)
30+
{
31+
return (SecretsEnvironments.TryGetValue("", out var def) ? def : null).Merge(name != null && SecretsEnvironments.TryGetValue(name, out var val) ? val : null);
32+
}
33+
34+
public IDictionary<string, string> GetReservedSecrets()
35+
{
36+
return SecretHelper.WithReservedSecrets(null, SecretsEnvironments.TryGetValue("", out var def) ? def : null);
37+
}
38+
39+
public IDictionary<string, string> GetVariablesForEnvironment(string name = null)
40+
{
41+
return (VarEnvironments.TryGetValue("", out var def) ? def : null).Merge(name != null && VarEnvironments.TryGetValue(name, out var val) ? val : null);
42+
}
43+
}
44+
}

‎src/Runner.Server/Services/ISecretsProvider.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
using System.Collections.Generic;
2+
3+
namespace Runner.Server.Services
4+
{
5+
public interface ISecretsProvider {
6+
IDictionary<string, string> GetSecretsForEnvironment(GitHub.DistributedTask.ObjectTemplating.ITraceWriter traceWriter, string name = null);
7+
IDictionary<string, string> GetVariablesForEnvironment(string name = null);
8+
IDictionary<string, string> GetReservedSecrets();
9+
}
10+
}

‎src/Runner.Server/Services/ReusableWorkflowSecretsProvider.cs‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Text.RegularExpressions;
4+
using GitHub.DistributedTask.Logging;
5+
using GitHub.DistributedTask.ObjectTemplating.Tokens;
6+
using GitHub.DistributedTask.Pipelines.ContextData;
7+
8+
namespace Runner.Server.Services
9+
{
10+
public class ReusableWorkflowSecretsProvider : ISecretsProvider
11+
{
12+
private ISecretsProvider parent;
13+
private TemplateToken secretsMapping;
14+
private DictionaryContextData contextData;
15+
private string jobName;
16+
private WorkflowContext workflowContext;
17+
private List<TemplateToken> environment;
18+
19+
public ReusableWorkflowSecretsProvider(string jobName, ISecretsProvider parent, TemplateToken secretsMapping, DictionaryContextData contextData, WorkflowContext workflowContext, List<TemplateToken> environment){
20+
this.parent = parent;
21+
this.secretsMapping = secretsMapping;
22+
this.contextData = contextData;
23+
this.jobName = jobName;
24+
this.workflowContext = workflowContext;
25+
this.environment = environment;
26+
}
27+
public IDictionary<string, string> GetSecretsForEnvironment(GitHub.DistributedTask.ObjectTemplating.ITraceWriter traceWriter, string name = null)
28+
{
29+
var parentSecrets = parent.GetSecretsForEnvironment(traceWriter, name);
30+
traceWriter.Info("{0}", $"Evaluating Secrets of {jobName} for environment '{name?.Replace("'","''")}'");
31+
SecretMasker masker = new SecretMasker();
32+
var linesplitter = new Regex("\r?\n");
33+
foreach(var variable in parentSecrets) {
34+
if(!string.IsNullOrEmpty(variable.Value)) {
35+
masker.AddValue(variable.Value);
36+
if(variable.Value.Contains('\r') || variable.Value.Contains('\n')) {
37+
foreach(var line in linesplitter.Split(variable.Value)) {
38+
masker.AddValue(line);
39+
}
40+
}
41+
}
42+
}
43+
masker.AddValueEncoder(ValueEncoders.Base64StringEscape);
44+
masker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift1);
45+
masker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift2);
46+
masker.AddValueEncoder(ValueEncoders.CommandLineArgumentEscape);
47+
masker.AddValueEncoder(ValueEncoders.ExpressionStringEscape);
48+
masker.AddValueEncoder(ValueEncoders.JsonStringEscape);
49+
masker.AddValueEncoder(ValueEncoders.UriDataEscape);
50+
masker.AddValueEncoder(ValueEncoders.XmlDataEscape);
51+
masker.AddValueEncoder(ValueEncoders.TrimDoubleQuotes);
52+
masker.AddValueEncoder(ValueEncoders.PowerShellPreAmpersandEscape);
53+
masker.AddValueEncoder(ValueEncoders.PowerShellPostAmpersandEscape);
54+
var secureTraceWriter = new TraceWriter2(line => {
55+
traceWriter.Info("{0}", masker.MaskSecrets(line));
56+
});
57+
var result = new DictionaryContextData();
58+
foreach (var variable in parentSecrets) {
59+
result[variable.Key] = new StringContextData(variable.Value);
60+
}
61+
var jobEnvCtx = new DictionaryContextData();
62+
foreach(var envBlock in environment) {
63+
var envTemplateContext = SecretHelper.CreateTemplateContext(secureTraceWriter, workflowContext, contextData);
64+
envTemplateContext.ExpressionValues["env"] = jobEnvCtx;
65+
envTemplateContext.ExpressionValues["secrets"] = result;
66+
var cEnv = GitHub.DistributedTask.ObjectTemplating.TemplateEvaluator.Evaluate(envTemplateContext, "job-env", envBlock, 0, null, true);
67+
// Best effort, don't check for errors
68+
// templateContext.Errors.Check();
69+
// Best effort, make global env available this is not available on github actions
70+
if(cEnv is MappingToken genvToken) {
71+
foreach(var kv in genvToken) {
72+
if(kv.Key is StringToken key && kv.Value is StringToken val) {
73+
jobEnvCtx[key.Value] = new StringContextData(val.Value);
74+
}
75+
}
76+
}
77+
}
78+
var templateContext = SecretHelper.CreateTemplateContext(secureTraceWriter, workflowContext, contextData);
79+
templateContext.ExpressionValues["env"] = jobEnvCtx;
80+
templateContext.ExpressionValues["secrets"] = result;
81+
var evalSec = secretsMapping != null ? GitHub.DistributedTask.ObjectTemplating.TemplateEvaluator.Evaluate(templateContext, templateContext.Schema.Definitions.ContainsKey("job-secrets") ? "job-secrets" : "workflow-job-secrets-mapping", secretsMapping, 0, null, true)?.AssertMapping($"jobs.{name}.secrets") : null;
82+
templateContext.Errors.Check();
83+
IDictionary<string, string> ret = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
84+
if(evalSec != null) {
85+
foreach(var entry in evalSec) {
86+
ret[entry.Key.AssertString($"jobs.{jobName}.secrets mapping key").Value] = entry.Value.AssertString($"jobs.{jobName}.secrets mapping value").Value;
87+
}
88+
}
89+
return SecretHelper.WithReservedSecrets(ret, parentSecrets);
90+
}
91+
92+
public IDictionary<string, string> GetReservedSecrets()
93+
{
94+
return parent.GetReservedSecrets();
95+
}
96+
97+
public IDictionary<string, string> GetVariablesForEnvironment(string name = null)
98+
{
99+
return parent.GetVariablesForEnvironment(name);
100+
}
101+
}
102+
}

‎src/Runner.Server/Services/ScheduleSecretsProvider.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
using System.Collections.Generic;
2+
3+
namespace Runner.Server.Services
4+
{
5+
public class ScheduleSecretsProvider : ISecretsProvider
6+
{
7+
public IDictionary<string, IDictionary<string, string>> SecretsEnvironments { get; set; }
8+
public IDictionary<string, IDictionary<string, string>> VarEnvironments { get; set; }
9+
public IDictionary<string, string> GetSecretsForEnvironment(GitHub.DistributedTask.ObjectTemplating.ITraceWriter traceWriter, string name = null)
10+
{
11+
return (SecretsEnvironments.TryGetValue("", out var def) ? def : null).Merge(name != null && SecretsEnvironments.TryGetValue(name, out var val) ? val : null);
12+
}
13+
14+
public IDictionary<string, string> GetReservedSecrets()
15+
{
16+
return SecretHelper.WithReservedSecrets(null, SecretsEnvironments.TryGetValue("", out var def) ? def : null);
17+
}
18+
19+
public IDictionary<string, string> GetVariablesForEnvironment(string name = null)
20+
{
21+
return (VarEnvironments.TryGetValue("", out var def) ? def : null).Merge(name != null && VarEnvironments.TryGetValue(name, out var val) ? val : null);
22+
}
23+
}
24+
}

‎src/Runner.Server/Services/Secret.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
namespace Runner.Server.Services
2+
{
3+
public class Secret
4+
{
5+
public string Name {get;set;}
6+
public string Value {get;set;}
7+
}
8+
}

‎src/Runner.Server/Services/SecretHelper.cs‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Text.RegularExpressions;
4+
using GitHub.DistributedTask.Expressions2;
5+
using GitHub.DistributedTask.ObjectTemplating;
6+
using GitHub.DistributedTask.ObjectTemplating.Schema;
7+
using GitHub.DistributedTask.Pipelines.ContextData;
8+
using GitHub.DistributedTask.Pipelines.ObjectTemplating;
9+
using Sdk.Pipelines;
10+
11+
namespace Runner.Server.Services
12+
{
13+
public class SecretHelper
14+
{
15+
public static bool IsReservedVariable(string v) {
16+
var pattern = new Regex("^[a-zA-Z_][a-zA-Z_0-9]*$");
17+
return !pattern.IsMatch(v) || v.StartsWith("GITHUB_", StringComparison.OrdinalIgnoreCase);
18+
}
19+
20+
public static bool IsActionsDebugVariable(string v) {
21+
return string.Equals(v, "ACTIONS_STEP_DEBUG", StringComparison.OrdinalIgnoreCase) || string.Equals(v, "ACTIONS_RUNNER_DEBUG", StringComparison.OrdinalIgnoreCase);
22+
}
23+
24+
public static IDictionary<string, string> WithReservedSecrets(IDictionary<string, string> dict, IDictionary<string, string> reservedsecrets) {
25+
var ret = dict == null ? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) : new Dictionary<string, string>(dict, StringComparer.OrdinalIgnoreCase);
26+
if(reservedsecrets != null) {
27+
foreach(var kv in reservedsecrets) {
28+
if(IsReservedVariable(kv.Key) || /* Allow overriding them while calling reusable workflows */ !ret.ContainsKey(kv.Key) && IsActionsDebugVariable(kv.Key)) {
29+
ret[kv.Key] = kv.Value;
30+
}
31+
}
32+
}
33+
return ret;
34+
}
35+
36+
public static TemplateContext CreateTemplateContext(GitHub.DistributedTask.ObjectTemplating.ITraceWriter traceWriter, WorkflowContext context, DictionaryContextData contextData = null, ExecutionContext exctx = null) {
37+
ExpressionFlags flags = ExpressionFlags.None;
38+
if(context.HasFeature("system.runner.server.extendedFunctions")) {
39+
flags |= ExpressionFlags.ExtendedFunctions;
40+
}
41+
if(context.HasFeature("system.runner.server.extendedDirectives")) {
42+
flags |= ExpressionFlags.ExtendedDirectives;
43+
}
44+
if(context.HasFeature("system.runner.server.allowAnyForInsert")) {
45+
flags |= ExpressionFlags.AllowAnyForInsert;
46+
}
47+
if(context.HasFeature("system.runner.server.FixInvalidActionsIfExpression")) {
48+
flags |= ExpressionFlags.FixInvalidActionsIfExpression;
49+
}
50+
if(context.HasFeature("system.runner.server.FailInvalidActionsIfExpression")) {
51+
flags |= ExpressionFlags.FailInvalidActionsIfExpression;
52+
}
53+
// For Gitea Actions
54+
var absoluteActions = context.HasFeature("system.runner.server.absolute_actions");
55+
var templateContext = new TemplateContext() {
56+
Flags = flags,
57+
CancellationToken = System.Threading.CancellationToken.None,
58+
Errors = new TemplateValidationErrors(10, 500),
59+
Memory = new TemplateMemory(
60+
maxDepth: 100,
61+
maxEvents: 1000000,
62+
maxBytes: 10 * 1024 * 1024),
63+
TraceWriter = traceWriter,
64+
Schema = PipelineTemplateSchemaFactory.GetSchema(),
65+
AbsoluteActions = absoluteActions,
66+
};
67+
if(context.FeatureToggles.TryGetValue("system.runner.server.workflow_schema", out var workflow_schema)) {
68+
var objectReader = new JsonObjectReader(null, workflow_schema);
69+
templateContext.Schema = TemplateSchema.Load(objectReader);
70+
}
71+
if(exctx != null) {
72+
templateContext.State[nameof(ExecutionContext)] = exctx;
73+
templateContext.ExpressionFunctions.Add(new FunctionInfo<AlwaysFunction>(PipelineTemplateConstants.Always, 0, 0));
74+
templateContext.ExpressionFunctions.Add(new FunctionInfo<CancelledFunction>(PipelineTemplateConstants.Cancelled, 0, 0));
75+
templateContext.ExpressionFunctions.Add(new FunctionInfo<FailureFunction>(PipelineTemplateConstants.Failure, 0, Int32.MaxValue));
76+
templateContext.ExpressionFunctions.Add(new FunctionInfo<SuccessFunction>(PipelineTemplateConstants.Success, 0, Int32.MaxValue));
77+
}
78+
if(contextData != null) {
79+
foreach (var pair in contextData) {
80+
templateContext.ExpressionValues[pair.Key] = pair.Value;
81+
}
82+
}
83+
if(context.FileTable != null) {
84+
foreach(var fileName in context.FileTable) {
85+
templateContext.GetFileId(fileName);
86+
}
87+
}
88+
return templateContext;
89+
}
90+
91+
}
92+
}

‎src/Runner.Server/Services/TraceWriter2.cs‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
using System;
2+
using System.Text.RegularExpressions;
3+
4+
namespace Runner.Server.Services
5+
{
6+
public class TraceWriter2 : GitHub.DistributedTask.ObjectTemplating.ITraceWriter, GitHub.DistributedTask.Expressions2.ITraceWriter
7+
{
8+
private Action<string> callback;
9+
private Regex regex;
10+
private int verbosity;
11+
public TraceWriter2(Action<string> callback, int verbosity = 0) {
12+
this.callback = callback;
13+
regex = new Regex("\r?\n");
14+
this.verbosity = verbosity;
15+
}
16+
17+
public void Callback(string lines) {
18+
foreach(var line in regex.Split(lines)) {
19+
callback(line);
20+
}
21+
}
22+
23+
public void Error(string format, params object[] args)
24+
{
25+
if(args?.Length == 1 && args[0] is Exception ex) {
26+
Callback(string.Format("{0} {1}", format, ex.Message));
27+
return;
28+
}
29+
try {
30+
Callback(args?.Length > 0 ? string.Format(format, args) : format);
31+
} catch {
32+
Callback(format);
33+
}
34+
}
35+
36+
public void Info(string format, params object[] args)
37+
{
38+
if(verbosity <= 2) {
39+
try {
40+
Callback(args?.Length > 0 ? string.Format(format, args) : format);
41+
} catch {
42+
Callback(format);
43+
}
44+
}
45+
}
46+
47+
public void Info(string message)
48+
{
49+
if(verbosity <= 2) {
50+
Callback(message);
51+
}
52+
}
53+
public void Verbose(string format, params object[] args)
54+
{
55+
if(verbosity <= 1) {
56+
try {
57+
Callback(args?.Length > 0 ? string.Format(format, args) : format);
58+
} catch {
59+
Callback(format);
60+
}
61+
}
62+
}
63+
64+
public void Verbose(string message)
65+
{
66+
if(verbosity <= 1) {
67+
Callback(message);
68+
}
69+
}
70+
71+
public void Trace(string message)
72+
{
73+
if(verbosity <= 0) {
74+
Callback(message);
75+
}
76+
}
77+
}
78+
79+
}

0 commit comments

Comments
 (0)