add more yaml inputs

vmelamed · vmelamed · commit 55fc0d52da71 · 2025-10-02T22:22:58.000-04:00
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -27,6 +27,18 @@ on:
         required: false
         default: "Release"
 
+      define:
+        description: "Define constants to pass to the compiler"
+        type: string
+        default: ""
+        required: false
+
+      verbose:
+        description: "Whether to enable verbose logging"
+        type: boolean
+        default: false
+        required: false
+
       test-project:
         description: "Path to test project"
         type: string
@@ -89,6 +101,8 @@ jobs:
       os: ${{ inputs.os || vars.MATRIX_OS || '["ubuntu-latest"]' }}
       test-project: ${{ inputs.test-project || vars.TEST_PROJECT || './test/UlidType.Tests/UlidType.Tests.csproj' }}
       configuration: ${{ inputs.build-configuration ||vars.BUILD_CONFIGURATION || 'Release' }}
+      define: ${{ inputs.define || vars.DEFINE || '' }}
+      verbose: ${{ inputs.verbose || vars.VERBOSE || false }}
       min_coverage_pct: ${{ inputs.min_coverage_pct || fromJSON(vars.MIN_COVERAGE_PCT) || 75 }}
 
   benchmarks:
@@ -101,5 +115,7 @@ jobs:
       os: ${{ inputs.os || vars.MATRIX_OS || '["ubuntu-latest"]' }}
       benchmark-project: ${{ inputs.benchmark-project || vars.BENCHMARK_PROJECT || './benchmarks/UlidType.Benchmarks/UlidType.Benchmarks.csproj' }}
       configuration: ${{ inputs.build-configuration ||vars.BUILD_CONFIGURATION || 'Release' }}
-      force-new-baseline: ${{ inputs.force-new-baseline || fromJSON(vars.FORCE_NEW_BASELINE) || false }}
+      define: ${{ inputs.define || vars.DEFINE || '' }}
+      verbose: ${{ inputs.verbose || vars.VERBOSE || false }}
       max-regression-pct: ${{ inputs.max-regression-pct || fromJSON(vars.MAX_REGRESSION_PCT) || 10 }}
+      force-new-baseline: ${{ inputs.force-new-baseline || fromJSON(vars.FORCE_NEW_BASELINE) || false }}
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -20,6 +20,18 @@ on:
         required: false
         default: "Release"
 
+      define:
+        description: "Define constants to pass to the compiler"
+        type: string
+        default: ""
+        required: false
+
+      verbose:
+        description: "Whether to enable verbose logging"
+        type: boolean
+        default: false
+        required: false
+
       force-new-baseline:
         description: "Ignore the existing baseline and force a new baseline"
         type: boolean
@@ -47,7 +59,7 @@ env:
   ARTIFACTS_DIR: BmArtifacts
   SUMMARIES_DIR: BmArtifacts/summaries
   BASELINE_DIR: BmArtifacts/baseline
-  FORCE_NEW_BASELINE: false
+  FORCE_NEW_BASELINE: ${{ inputs.force-new-baseline || vars.FORCE_NEW_BASELINE || false }}
 
 jobs:
   benchmarks:
@@ -83,9 +95,12 @@ jobs:
         shell: bash
         env:
           FORCE_NEW_BASELINE: ${{ inputs.force-new-baseline }}
-          CONFIGURATION: ${{ inputs.configuration }}
         run: |
-          ./scripts/bash/run-benchmarks.sh ${{ inputs.benchmark-project }}
+          ./scripts/bash/run-benchmarks.sh ${{ inputs.benchmark-project }} \
+              --verbose "${{ inputs.verbose }}" \
+              --max_regression_pct "${{ inputs.max-regression-pct }}" \
+              --define "${{ inputs.define }}"
+              --configuration "${{ inputs.configuration }}"
 
       - name: Upload benchmark summaries artifact
         uses: actions/upload-artifact@v4
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -7,19 +7,35 @@ on:
         type: string
         default: "9.0.x"
         required: true
+
       test-project:
         description: "Path to the test project"
         type: string
         required: false
+
       configuration:
         description: "The type of build to produce, e.g. Release vs Debug"
         type: string
         required: false
         default: "Release"
+
+      define:
+        description: "Define constants to pass to the compiler"
+        type: string
+        default: ""
+        required: false
+
+      verbose:
+        description: "Whether to enable verbose logging"
+        type: boolean
+        default: false
+        required: false
+
       min_coverage_pct:
         type: number
         required: true
         default: 75
+
       os:
         description: "Runner OS-s"
         type: string
diff --git a/scripts/bash/run-benchmarks-utils.sh b/scripts/bash/run-benchmarks-utils.sh
@@ -52,6 +52,11 @@ Switches:
     --short-run | -s        A shortcut for '--define SHORT_RUN'. See below.
                             Initial value from \$DEFINE.
 
+    --force-new-baseline | -f
+                            When specified, a new baseline will be created even
+                            if a previous baseline already exists.
+                            Initial value from \$FORCE_NEW_BASELINE or 'false'
+
 Options:
     --define | -d           Defines one or more user-defined pre-processor
                             symbols to be used when building the benchmark
@@ -123,6 +128,8 @@ function get_arguments()
 
             --trace|-x ) trace_enabled=true; set -x ;;
 
+            --force-new-baseline|-f ) force_new_baseline=true ;;
+
             --artifacts|-a ) value="$1"; shift; ARTIFACTS_DIR=$(realpath -m "$value") ;;
 
             --max-regression-pct|-r )
@@ -140,18 +147,18 @@ function get_arguments()
                     usage "The specified pre-processor symbol '$value' is not valid."
                     exit 2
                 fi
-                if [[ -z "$DEFINE" ]]; then
-                    DEFINE="$value"
-                elif [[ ! "$DEFINE" =~ (^|;)"$value"($|;) ]]; then
-                    DEFINE="$DEFINE;$value"
+                if [[ -z "$define" ]]; then
+                    define="$value"
+                elif [[ ! "$define" =~ (^|;)"$value"($|;) ]]; then
+                    define="$define;$value"
                 fi
                 ;;
 
             --short-run|-s )
-                if [[ -z "$DEFINE" ]]; then
-                    DEFINE="SHORT_RUN"
-                elif [[ ! "$DEFINE" =~ (^|;)SHORT_RUN($|;) ]]; then
-                    DEFINE="$DEFINE;SHORT_RUN"
+                if [[ -z "$define" ]]; then
+                    define="SHORT_RUN"
+                elif [[ ! "$define" =~ (^|;)SHORT_RUN($|;) ]]; then
+                    define="$define;SHORT_RUN"
                 fi
                 ;;
 
diff --git a/scripts/bash/run-benchmarks.sh b/scripts/bash/run-benchmarks.sh
@@ -17,14 +17,20 @@ bm_project=$(realpath -e "$bm_project")  # ensure it's an absolute path and exis
 declare -x ARTIFACTS_DIR=${ARTIFACTS_DIR:="$solution_dir/BmArtifacts"}
 ARTIFACTS_DIR=$(realpath -m "$ARTIFACTS_DIR")  # ensure it's an absolute path
 
+declare -x force_new_baseline=${FORCE_NEW_BASELINE:-false}
+
 declare configuration=${CONFIGURATION:="Release"}
 
+declare define=${DEFINE:-}
+
 source "$script_dir/_common.sh"
 source "$script_dir/run-benchmarks-utils.sh"
 
 get_arguments "$@"
 declare -r bm_project
 declare -r configuration
+declare -r force_new_baseline
+declare -r define
 
 declare -x results_dir=${results_dir:="$ARTIFACTS_DIR/results"}
 results_dir=$(realpath -m "$results_dir")  # ensure it's an absolute path
@@ -38,12 +44,12 @@ declare -x baseline_dir=${baseline_dir:="$ARTIFACTS_DIR/baseline"}
 baseline_dir=$(realpath -m "$baseline_dir")
 declare -r baseline_dir
 
-renamed_artifacts_dir="$ARTIFACTS_DIR-$(date -u +"%Y%m%dT%H%M%S")"
-declare -r renamed_artifacts_dir
-
 max_regression_pct=${MAX_REGRESSION_PCT:-10}
 declare -ri max_regression_pct
 
+renamed_artifacts_dir="$ARTIFACTS_DIR-$(date -u +"%Y%m%dT%H%M%S")"
+declare -r renamed_artifacts_dir
+
 dump_all_variables
 
 if [[ -d "$ARTIFACTS_DIR" && -n "$(ls -A "$ARTIFACTS_DIR")" ]]; then
@@ -84,7 +90,7 @@ execute mkdir -p "$summaries_dir"
 trace "Running benchmark tests in project '$bm_project' with configuration '$configuration'..."
 execute mkdir -p "$ARTIFACTS_DIR"
 execute dotnet run \
-    /p:DefineConstants="$DEFINE" \
+    /p:DefineConstants="$define" \
     --project "$bm_project" \
     --configuration "$configuration" \
     --filter '*' \
@@ -157,6 +163,12 @@ flush_stdout
 
 if (( pct > max_regression_pct )); then
     echo "Performance regression exceeds threshold" >&2
+    if [[ $force_new_baseline == "true" ]] && is_defined "GITHUB_ENV"; then
+        echo "Significant regression of $pct% over baseline. Updating the baseline."
+        # shellcheck disable=SC2154
+        echo "FORCE_NEW_BASELINE=true" >> "$GITHUB_ENV"
+        exit 0
+    fi
     echo "If this is acceptable, please update the baseline by setting the variable 'FORCE_NEW_BASELINE=true'." >&2
     exit 2
 elif (( pct > 0 )); then
diff --git a/scripts/bash/run-test-utils.sh b/scripts/bash/run-test-utils.sh
@@ -55,16 +55,24 @@ Options:
                             percentage (0-100).
                             Initial value from \$MIN_COVERAGE_PCT or 80
 
-    --configuration | -c    Specifies the build configuration to use ('Debug' or
-                            'Release').
-                            Initial value from \$CONFIGURATION or 'Release'
-
     --artifacts | -a        Specifies the directory where to create the script's
                             artifacts: summary, report files, etc.
                             Initial value from \$ARTIFACTS_DIR or
                             '\$solution_dir/TestArtifacts'
                             ($solution_dir/TestArtifacts)
 
+    --configuration | -c    Specifies the build configuration to use ('Debug' or
+                            'Release').
+                            Initial value from \$CONFIGURATION or 'Release'
+
+    --define | -d           Defines one or more user-defined pre-processor
+                            symbols to be used when building the benchmark
+                            project, e.g. 'SHORT_RUN'. Which generates a shorter
+                            and faster, but less accurate benchmark run. You can
+                            specify this option multiple times to define multiple
+                            symbols.
+                            Initial value from \$DEFINE or ''
+
 "
     if [[ "${#}" -gt 0 && -n "$1" ]]; then
         echo "$1" >&2
@@ -129,6 +137,19 @@ function get_arguments()
                 configuration="${value^}"
                 ;;
 
+            --define|-d )
+                value="$1"; shift
+                if ! [[ "$value" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
+                    usage "The specified pre-processor symbol '$value' is not valid."
+                    exit 2
+                fi
+                if [[ -z "$define" ]]; then
+                    define="$value"
+                elif [[ ! "$define" =~ (^|;)"$value"($|;) ]]; then
+                    define="$define;$value"
+                fi
+                ;;
+
             *)  value="$flag"
                 if ! p=$(realpath -e "$value"); then
                     usage "The specified test project file '$value' does not exist."
diff --git a/scripts/bash/run-tests.sh b/scripts/bash/run-tests.sh
@@ -22,12 +22,16 @@ declare -x COVERAGE_RESULTS_DIR
 declare configuration=${CONFIGURATION:="Release"}
 declare -i min_coverage_pct=${MIN_COVERAGE_PCT:-80}
 
+declare define=${DEFINE:-}
+
 source "$script_dir/_common.sh"
 source "$script_dir/run-test-utils.sh"
 
 get_arguments "$@"
 declare -r test_project
 declare -ri min_coverage_pct
+declare -r configuration
+declare -r define
 
 renamed_results_dir="$ARTIFACTS_DIR-$(date -u +"%Y%m%dT%H%M%S")"
 declare -r renamed_results_dir
@@ -83,6 +87,7 @@ execute mkdir -p "$coverage_summary_dir"
 
 trace "Running tests in project '$test_project' with configuration '$configuration'..."
 execute dotnet test "$test_project" \
+    /p:DefineConstants="$define" \
     --configuration "$configuration" -- \
     --results-directory "$test_results_results_dir" \
     --coverage \
