feat: add improved parsing for labels and support for multiple fallbacks (#4977)

* feat: add improved regex parsing for labels

* feat: parse chain of tokens for labels

* feat: add label tokenizer and tests

* chore: remove unused format type

* fix: address sonarqube issues with label parsing

* fix: address test sonarqube issues

* fix: fix failing test

* fix: update failing test

* feat: add documentation for advanced parsing

* fix: change label token to be record

* docs: add breaking change docs for env and placeholder behavior

* fix: tighten parsing of label formats

* docs: improve fallback documentation

* fix: throw on missing properties and use integer fallbacks

* fix: fix sonarqube issues

* fix: revert global.json changes

Author: robertcoltheart

docs/input/docs/reference/mdsource/configuration.source.md

@@ -504,7 +504,7 @@ of `alpha.foo` with `label: 'alpha.{BranchName}'` and `regex: '^features?[\/-](?
 
 Another example: branch `features/sc-12345/some-description` would become a pre-release label of `sc-12345` with `label: '{StoryNo}'` and `regex: '^features?[\/-](?<StoryNo>sc-\d+)[-/].+'`.
 
-You can also use environment variable placeholders with the `{env:VARIABLE_NAME}` syntax. Environment variable placeholders can also be combined with regex placeholders, for example `{BranchName}-{env:VARIABLE_NAME}`, and support fallback values using the `{env:VARIABLE_NAME ?? "fallback"}` syntax.
+You can also use environment variable placeholders with the `{env:VARIABLE_NAME}` syntax. Environment variable placeholders can also be combined with regex placeholders, for example `{BranchName}-{env:VARIABLE_NAME}`, and support fallback values using the `{env:VARIABLE_NAME ?? "fallback"}` syntax. These can be combined with cascading fallbacks using environment variables and placeholders like this: `{env:VARIABLE_NAME ?? BranchName ?? "fallback"}`.
 
 **Note:** To clear a default use an empty string: `label: ''`
 

src/GitVersion.Configuration.Tests/Configuration/ConfigurationExtensionsTests.cs

@@ -127,7 +127,7 @@ public void EnsureGetBranchSpecificLabelWorksWithoutEnvironmentWhenNoEnvPlacehol
     }
 
     [Test]
-    public void EnsureGetBranchSpecificLabelThrowsWhenThrowIfNotFoundAndEnvVarMissing()
+    public void EnsureGetBranchSpecificLabelThrowsWhenEnvVarMissing()
     {
         var environment = new TestEnvironment();
         // Do not set MISSING_VAR

src/GitVersion.Core.Tests/Extensions/StringFormatWithExtensionTests.cs

@@ -33,9 +33,9 @@ public void FormatWithSingleSimpleToken()
     [Test]
     public void FormatWithMultipleTokensAndVerbatimText()
     {
-        var propertyObject = new { SomeProperty = "SomeValue", AnotherProperty = "Other Value" };
+        var propertyObject = new { SomeProperty = "AValue", AnotherProperty = "Other Value" };
         const string target = "{SomeProperty} some text {AnotherProperty}";
-        const string expected = "SomeValue some text Other Value";
+        const string expected = "AValue some text Other Value";
         var actual = target.FormatWith(propertyObject, this.environment);
         Assert.That(actual, Is.EqualTo(expected));
     }
@@ -56,7 +56,7 @@ public void FormatWithEnvVarTokenWithFallback()
     {
         this.environment.SetEnvironmentVariable("GIT_VERSION_TEST_VAR", "Env Var Value");
         var propertyObject = new { };
-        const string target = "{env:GIT_VERSION_TEST_VAR ?? fallback}";
+        const string target = "{env:GIT_VERSION_TEST_VAR ?? \"fallback\"}";
         const string expected = "Env Var Value";
         var actual = target.FormatWith(propertyObject, this.environment);
         Assert.That(actual, Is.EqualTo(expected));
@@ -67,7 +67,7 @@ public void FormatWithUnsetEnvVarToken_WithFallback()
     {
         this.environment.SetEnvironmentVariable("GIT_VERSION_UNSET_TEST_VAR", null);
         var propertyObject = new { };
-        const string target = "{env:GIT_VERSION_UNSET_TEST_VAR ?? fallback}";
+        const string target = "{env:GIT_VERSION_UNSET_TEST_VAR ?? \"fallback\"}";
         const string expected = "fallback";
         var actual = target.FormatWith(propertyObject, this.environment);
         Assert.That(actual, Is.EqualTo(expected));
@@ -98,44 +98,139 @@ public void FormatWithMultipleEnvVars()
     public void FormatWithMultipleEnvChars()
     {
         var propertyObject = new { };
-        //Test the greediness of the regex in matching env: char
-        const string target = "{env:env:GIT_VERSION_TEST_VAR_1} and {env:DUMMY_VAR ?? fallback}";
-        const string expected = "{env:env:GIT_VERSION_TEST_VAR_1} and fallback";
-        var actual = target.FormatWith(propertyObject, this.environment);
-        Assert.That(actual, Is.EqualTo(expected));
+        const string target = "{env:env:GIT_VERSION_TEST_VAR_1} and {env:DUMMY_VAR ?? \"fallback\"}";
+        Assert.Throws<ArgumentException>(() => target.FormatWith(propertyObject, this.environment));
     }
 
     [Test]
     public void FormatWithMultipleFallbackChars()
     {
         var propertyObject = new { };
-        //Test the greediness of the regex in matching env: and ?? chars
-        const string target = "{env:env:GIT_VERSION_TEST_VAR_1} and {env:DUMMY_VAR ??? fallback}";
-        var actual = target.FormatWith(propertyObject, this.environment);
-        Assert.That(actual, Is.EqualTo(target));
+        const string target = " and {env:DUMMY_VAR ??? \"fallback\"}";
+        Assert.Throws<FormatException>(() => target.FormatWith(propertyObject, this.environment));
     }
 
     [Test]
     public void FormatWithSingleFallbackChar()
     {
-        this.environment.SetEnvironmentVariable("DUMMY_ENV_VAR", "Dummy-Val");
+        this.environment.SetEnvironmentVariable("DUMMY_ENV_VAR", "DummyVal");
         var propertyObject = new { };
-        //Test the sanity of the regex when there is a grammar mismatch
-        const string target = "{en:DUMMY_ENV_VAR} and {env:DUMMY_ENV_VAR??fallback}";
-        var actual = target.FormatWith(propertyObject, this.environment);
-        Assert.That(actual, Is.EqualTo(target));
+        const string target = "{en:DUMMY_ENV_VAR} and {env:DUMMY_ENV_VAR??\"fallback\"}";
+        Assert.Throws<ArgumentException>(() => target.FormatWith(propertyObject, this.environment));
     }
 
     [Test]
-    public void FormatWIthNullPropagationWithMultipleSpaces()
+    public void FormatWithNullPropagationWithMultipleSpaces()
     {
         var propertyObject = new { SomeProperty = "Some Value" };
-        const string target = "{SomeProperty} and {env:DUMMY_ENV_VAR  ??  fallback}";
+        const string target = "{SomeProperty} and {env:DUMMY_ENV_VAR  ??  \"fallback\"}";
         const string expected = "Some Value and fallback";
         var actual = target.FormatWith(propertyObject, this.environment);
         Assert.That(actual, Is.EqualTo(expected));
     }
 
+    [Test]
+    public void FormatWithMissingPropertyAndEnvFallback()
+    {
+        this.environment.SetEnvironmentVariable("DUMMY_ENV_VAR", "Dummy-Value");
+        var propertyObject = new { };
+        const string target = "{SomeProperty ?? env:DUMMY_ENV_VAR}";
+        const string expected = "Dummy-Value";
+        var actual = target.FormatWith(propertyObject, this.environment);
+        Assert.That(actual, Is.EqualTo(expected));
+    }
+
+    [Test]
+    public void FormatWithMissingEnvAndPropertyFallback()
+    {
+        var propertyObject = new { SomeProperty = "Some Value" };
+        const string target = "{env:DUMMY_ENV_VAR ?? SomeProperty}";
+        const string expected = "Some Value";
+        var actual = target.FormatWith(propertyObject, this.environment);
+        Assert.That(actual, Is.EqualTo(expected));
+    }
+
+    [Test]
+    public void FormatWithMultiplePropertiesAndNoFallback()
+    {
+        var propertyObject = new { };
+        const string target = "{SomeProperty ?? SomeOtherProperty ?? MissingProp}";
+        Assert.Throws<ArgumentException>(() => target.FormatWith(propertyObject, this.environment));
+    }
+
+    [Test]
+    public void FormatWithMultiplePropertiesAndQuotedFallback()
+    {
+        var propertyObject = new { };
+        const string target = "{SomeProperty ?? SomeOtherProperty ?? \"fallback\"}";
+        const string expected = "fallback";
+        var actual = target.FormatWith(propertyObject, this.environment);
+        Assert.That(actual, Is.EqualTo(expected));
+    }
+
+    [Test]
+    public void FormatWithMultipleAvailablePropertiesAndFallback()
+    {
+        var propertyObject = new { SomeOtherProperty = "Some-Value" };
+        const string target = "{SomeProperty ?? SomeOtherProperty ?? \"fallback\"}";
+        const string expected = "Some-Value";
+        var actual = target.FormatWith(propertyObject, this.environment);
+        Assert.That(actual, Is.EqualTo(expected));
+    }
+
+    [Test]
+    public void FormatWithMissingPropertiesAndIntegerFallback()
+    {
+        var propertyObject = new { };
+        const string target = "{SomeProperty ?? SomeOtherProperty ?? 47}";
+        const string expected = "47";
+        var actual = target.FormatWith(propertyObject, this.environment);
+        Assert.That(actual, Is.EqualTo(expected));
+    }
+
+    [Test]
+    public void FormatWithPropertyAndEnvAndFormatters()
+    {
+        this.environment.SetEnvironmentVariable("DUMMY_ENV_VAR", "DummyVal");
+        var propertyObject = new { SomeProperty = "TheValue" };
+        const string target = "{SomeProperty:l} and {env:DUMMY_ENV_VAR:l}";
+        const string expected = "thevalue and dummyval";
+        var actual = target.FormatWith(propertyObject, this.environment);
+        Assert.That(actual, Is.EqualTo(expected));
+    }
+
+    [TestCase("{env:VARIABLE ?? \"\"}", null, null, "")]
+    [TestCase("{env:MISSING ?? env:VARIABLE}", "Var", null, "Var")]
+    [TestCase("{Property ?? \"\"}", null, null, "")]
+    [TestCase("{Property ?? 47}", null, null, "47")]
+    [TestCase("{Property ?? env:VARIABLE}", null, "Branch", "Branch")]
+    [TestCase("{Property ?? env:VARIABLE ?? \"\"}", null, null, "")]
+    [TestCase("{Property ?? env:VARIABLE ?? 42}", null, null, "42")]
+    public void FormatWith_EnvVarAndPropertyAndFallback_DoesNotThrow(string input, string? envVar, string? property, string expected)
+    {
+        if (envVar != null)
+        {
+            this.environment.SetEnvironmentVariable("VARIABLE", envVar);
+        }
+
+        object propertyObject = property != null
+            ? new { Property = property }
+            : new { };
+
+        var actual = input.FormatWith(propertyObject, this.environment);
+        Assert.That(actual, Is.EqualTo(expected));
+    }
+
+    [TestCase("{env:VARIABLE}")]
+    [TestCase("{Property}")]
+    [TestCase("{Property ?? env:VARIABLE}")]
+    [TestCase("{Property ?? Property}")]
+    public void FormatWith_MissingEnvVarOrPropertyAndNoFallback_Throws(string input)
+    {
+        object propertyObject = new { };
+        Assert.Throws<ArgumentException>(() => input.FormatWith(propertyObject, this.environment));
+    }
+
     [Test]
     public void FormatEnvVar_WithFallback_QuotedAndEmpty()
     {
@@ -186,7 +281,7 @@ public void FormatProperty_NullInteger()
     public void FormatProperty_String_WithFallback()
     {
         var propertyObject = new { Property = "Value" };
-        const string target = "{Property ?? fallback}";
+        const string target = "{Property ?? \"fallback\"}";
         var actual = target.FormatWith(propertyObject, this.environment);
         Assert.That(actual, Is.EqualTo("Value"));
     }
@@ -195,7 +290,7 @@ public void FormatProperty_String_WithFallback()
     public void FormatProperty_Integer_WithFallback()
     {
         var propertyObject = new { Property = 42 };
-        const string target = "{Property ?? fallback}";
+        const string target = "{Property ?? \"fallback\"}";
         var actual = target.FormatWith(propertyObject, this.environment);
         Assert.That(actual, Is.EqualTo("42"));
     }
@@ -204,7 +299,7 @@ public void FormatProperty_Integer_WithFallback()
     public void FormatProperty_NullObject_WithFallback()
     {
         var propertyObject = new { Property = (object?)null };
-        const string target = "{Property ?? fallback}";
+        const string target = "{Property ?? \"fallback\"}";
         var actual = target.FormatWith(propertyObject, this.environment);
         Assert.That(actual, Is.EqualTo("fallback"));
     }
@@ -213,18 +308,18 @@ public void FormatProperty_NullObject_WithFallback()
     public void FormatProperty_NullInteger_WithFallback()
     {
         var propertyObject = new { Property = (int?)null };
-        const string target = "{Property ?? fallback}";
+        const string target = "{Property ?? 43}";
         var actual = target.FormatWith(propertyObject, this.environment);
-        Assert.That(actual, Is.EqualTo("fallback"));
+        Assert.That(actual, Is.EqualTo("43"));
     }
 
     [Test]
     public void FormatProperty_NullObject_WithFallback_Quoted()
     {
         var propertyObject = new { Property = (object?)null };
-        const string target = "{Property ?? \"fallback\"}";
+        const string target = "{Property ?? \"literal\"}";
         var actual = target.FormatWith(propertyObject, this.environment);
-        Assert.That(actual, Is.EqualTo("fallback"));
+        Assert.That(actual, Is.EqualTo("literal"));
     }
 
     [Test]

src/GitVersion.Core.Tests/Formatting/LabelTokenizerTests.cs

@@ -0,0 +1,65 @@
+using GitVersion.Formatting;
+
+namespace GitVersion.Tests.Formatting;
+
+[TestFixture]
+public class LabelTokenizerTests
+{
+    [TestCase("Pattern", "Pattern")]
+    [TestCase("Pattern  ", "Pattern")]
+    [TestCase("  Pattern", "Pattern")]
+    [TestCase("  Pattern  ", "Pattern")]
+    [TestCase("Pat\\\"tern", "Pat\"tern")]
+    [TestCase("\"Pattern\"", "Pattern")]
+    [TestCase("\"Pat?tern\"", "Pat?tern")]
+    [TestCase("\"Pat tern\"", "Pat tern")]
+    [TestCase("\" Pattern\"", " Pattern")]
+    [TestCase("\"Pattern \"", "Pattern ")]
+    [TestCase("\"Pat\\\"tern\"", "Pat\"tern")]
+    public void ParseTokens_ValidLiterals_ReturnsValid(string input, params string[] expected) => AssertTokens(input, expected);
+
+    [TestCase("Pat?tern")]
+    [TestCase("\"Pattern")]
+    [TestCase("Pattern\"")]
+    [TestCase("Pat\"tern")]
+    public void ParseTokens_InvalidLiterals_Throws(string input) => AssertThrows(input);
+
+    [TestCase("Prop1 ?? Prop2", "Prop1", "Prop2")]
+    [TestCase("Prop1??Prop2", "Prop1", "Prop2")]
+    [TestCase("Prop1??Prop2??42", "Prop1", "Prop2", "42")]
+    [TestCase("Prop1??Prop2??\"42\"", "Prop1", "Prop2", "42")]
+    [TestCase("Prop1 ?? Prop2 ?? \"fallback\"", "Prop1", "Prop2", "fallback")]
+    [TestCase("Prop1 ??Prop2?? \"fallback\"", "Prop1", "Prop2", "fallback")]
+    [TestCase("Prop1 ?? Prop2 ?? 42", "Prop1", "Prop2", "42")]
+    [TestCase("Prop1:format ?? Prop2 ?? \"fallback\"", "Prop1", "Prop2", "fallback")]
+    [TestCase("env:Env1 ?? Prop2 ?? \"fallback\"", "Env1", "Prop2", "fallback")]
+    [TestCase("env:Env1:format ?? \"literal\" ?? \"fallback\"", "Env1", "literal", "fallback")]
+    [TestCase("env:Env1 ?? 42", "Env1", "42")]
+    public void ParseTokens_ValidIdentifiers_ReturnsValid(string input, params string[] expected) => AssertTokens(input, expected);
+
+    [TestCase("Prop ??? literal")]
+    [TestCase("Prop literal")]
+    [TestCase("Prop ?? literal ?? ? fallback")]
+    [TestCase("Prop ? fallback")]
+    [TestCase("Prop ?? fall?back")]
+    [TestCase("Prop ?? fallback ??")]
+    [TestCase("Prop ?? fallback ??   ")]
+    public void ParseTokens_MalformedIdentifiers_Throws(string input) => AssertThrows(input);
+
+    private static void AssertTokens(string input, string[] expected)
+    {
+        var tokenizer = new LabelTokenizer(input);
+        var tokens = tokenizer.ParseTokens()
+            .Select(x => x.Name)
+            .ToArray();
+
+        tokens.ShouldBeEquivalentTo(expected);
+    }
+
+    private static void AssertThrows(string input)
+    {
+        var tokenizer = new LabelTokenizer(input);
+
+        Assert.Throws<FormatException>(() => tokenizer.ParseTokens());
+    }
+}

src/GitVersion.Core/Core/RegexPatterns.cs

@@ -17,27 +17,7 @@ internal static partial class RegexPatterns
     private const string ObscurePasswordRegexPattern = "(https?://)(.+)(:.+@)";
 
     [StringSyntax(StringSyntaxAttribute.Regex)]
-    private const string ExpandTokensRegexPattern =
-        """
-        \{                              # Opening brace
-            (?:                         # Start of either env or member expression
-                env:(?!env:)(?<envvar>[A-Za-z_][A-Za-z0-9_]*)       # Only a single env: prefix, not followed by another env:
-                |                                                   # OR
-                (?<member>[A-Za-z_][A-Za-z0-9_]*)                   # member/property name
-                (?:                                                 # Optional format specifier
-                    :(?<format>[A-Za-z0-9\.\-,]+)                   # Colon followed by format string (no spaces, ?, or }), format cannot contain colon
-                )?                                                  # Format is optional
-            )                                                       # End group for env or member
-            (?:                                                     # Optional fallback group
-                \s*\?\?\s+                                          # '??' operator with optional whitespace: exactly two question marks for fallback
-                (?:                                                 # Fallback value alternatives:
-                    (?<fallback>\w+)                                #   A single word fallback
-                    |                                               # OR
-                    "(?<fallback>[^"]*)"                            #   A quoted string fallback
-                )
-            )?                                                      # Fallback is optional
-        \}
-        """;
+    private const string ExpandTokensRegexPattern = @"\{([^{}]+)\}";
 
     /// <summary>
     ///   Allow alphanumeric, underscore, colon (for custom format specification), hyphen, and dot

src/GitVersion.Core/Extensions/ConfigurationExtensions.cs

@@ -154,9 +154,13 @@ private static Dictionary<string, object> BuildLabelPlaceholders(string? regular
             if (!match.Success)
                 return placeholders;
 
-            foreach (var groupName in regex.GetGroupNames())
+            var namedGroups = regex.GetGroupNames()
+                .Where(name => !int.TryParse(name, out _));
+
+            foreach (var groupName in namedGroups)
             {
                 var groupValue = match.Groups[groupName].Value;
+
                 placeholders[groupName] = groupValue.RegexReplace(RegexPatterns.SanitizeNameRegexPattern, "-");
             }
 

src/GitVersion.Core/Formatting/LabelToken.cs

@@ -0,0 +1,3 @@
+namespace GitVersion.Formatting;
+
+internal record LabelToken(string Name, LabelTokenType Type, string? Format = null);

src/GitVersion.Core/Formatting/LabelTokenType.cs

@@ -0,0 +1,8 @@
+namespace GitVersion.Formatting;
+
+internal enum LabelTokenType
+{
+    Literal,
+    Property,
+    Environment
+}