Update EntryPointCatalog baselines with Objective and Alpha fields. Improved unit tests.

Author: ssperling-wmp

src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs

@@ -142,7 +142,7 @@ public enum RegressionObjective
                 /// Quantile regression. Use <see cref="Alpha"/> to set the target quantile.
                 /// </summary>
                 Quantile
-            }
+            };
 
             /// <summary>
             /// Determines what evaluation metric to use.
@@ -179,6 +179,7 @@ static Options()
                 NameMapping.Add(nameof(EvaluateMetricType.MeanAbsoluteError), "mae");
                 NameMapping.Add(nameof(EvaluateMetricType.RootMeanSquaredError), "rmse");
                 NameMapping.Add(nameof(EvaluateMetricType.MeanSquaredError), "mse");
+                NameMapping.Add(nameof(Objective), "_regression_objective");
             }
 
             internal override Dictionary<string, object> ToDictionary(IHost host)

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -13266,6 +13266,36 @@
           "IsNullable": false,
           "Default": "RootMeanSquaredError"
         },
+        {
+          "Name": "Objective",
+          "Type": {
+            "Kind": "Enum",
+            "Values": [
+              "Regression",
+              "Quantile"
+            ]
+          },
+          "Desc": "Regression objective type. Use 'Quantile' for quantile regression.",
+          "Aliases": [
+            "obj"
+          ],
+          "Required": false,
+          "SortOrder": 150.0,
+          "IsNullable": false,
+          "Default": "Regression"
+        },
+        {
+          "Name": "Alpha",
+          "Type": "Float",
+          "Desc": "The alpha (quantile) value for quantile regression. Must be in (0, 1).",
+          "Aliases": [
+            "qa"
+          ],
+          "Required": false,
+          "SortOrder": 150.0,
+          "IsNullable": false,
+          "Default": 0.5
+        },
         {
           "Name": "MaximumBinCountPerFeature",
           "Type": "Int",

test/BaselineOutput/Common/EntryPoints/netcoreapp/core_manifest.json

@@ -13266,6 +13266,36 @@
           "IsNullable": false,
           "Default": "RootMeanSquaredError"
         },
+        {
+          "Name": "Objective",
+          "Type": {
+            "Kind": "Enum",
+            "Values": [
+              "Regression",
+              "Quantile"
+            ]
+          },
+          "Desc": "Regression objective type. Use 'Quantile' for quantile regression.",
+          "Aliases": [
+            "obj"
+          ],
+          "Required": false,
+          "SortOrder": 150.0,
+          "IsNullable": false,
+          "Default": "Regression"
+        },
+        {
+          "Name": "Alpha",
+          "Type": "Float",
+          "Desc": "The alpha (quantile) value for quantile regression. Must be in (0, 1).",
+          "Aliases": [
+            "qa"
+          ],
+          "Required": false,
+          "SortOrder": 150.0,
+          "IsNullable": false,
+          "Default": 0.5
+        },
         {
           "Name": "MaximumBinCountPerFeature",
           "Type": "Int",

test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs

@@ -262,6 +262,13 @@ public void LightGBMQuantileRegressorEstimator()
 
             TestEstimatorCore(trainer, dataView);
             var model = trainer.Fit(dataView, dataView);
+
+            var gbmParameters = trainer.GetGbmParameters();
+            Assert.True(gbmParameters.ContainsKey("objective"));
+            Assert.Equal("quantile", gbmParameters["objective"]);
+            Assert.True(gbmParameters.ContainsKey("alpha"));
+            Assert.Equal(0.5, gbmParameters["alpha"]);
+
             Done();
         }
 
@@ -281,6 +288,8 @@ public void LightGBMQuantileRegressorPredictionOrdering()
                 Alpha = 0.05,
                 NumberOfIterations = 50,
                 NumberOfLeaves = 10,
+                Seed = 42,
+                Deterministic = true,
             });
 
             // Train model for the 95th percentile
@@ -290,6 +299,8 @@ public void LightGBMQuantileRegressorPredictionOrdering()
                 Alpha = 0.95,
                 NumberOfIterations = 50,
                 NumberOfLeaves = 10,
+                Seed = 42,
+                Deterministic = true,
             });
 
             var modelLow = trainerLow.Fit(dataView);
@@ -304,11 +315,18 @@ public void LightGBMQuantileRegressorPredictionOrdering()
             Assert.Equal(scoresLow.Length, scoresHigh.Length);
             Assert.True(scoresLow.Length > 0);
 
-            // On average, the 95th percentile predictions should exceed the 5th percentile predictions.
-            var avgLow = scoresLow.Average();
-            var avgHigh = scoresHigh.Average();
-            Assert.True(avgHigh > avgLow,
-                $"Expected average 95th percentile ({avgHigh}) > average 5th percentile ({avgLow})");
+            // The 95th percentile predictions should generally be at least as large as the
+            // 5th percentile predictions. Allow a small numerical tolerance and a limited
+            // number of crossings since the models are trained independently.
+            const float tolerance = 1e-4f;
+            var orderedCount = Enumerable.Range(0, scoresLow.Length)
+                .Count(i => scoresHigh[i] + tolerance >= scoresLow[i]);
+            var orderedRatio = (float)orderedCount / scoresLow.Length;
+
+            Assert.True(orderedRatio >= 0.90f,
+                $"Expected the 95th percentile prediction to be >= the 5th percentile prediction for most rows, " +
+                $"but only {orderedCount} of {scoresLow.Length} rows satisfied the condition " +
+                $"({orderedRatio:P2}, tolerance={tolerance}).");
 
             Done();
         }
@@ -327,23 +345,23 @@ public void LightGBMQuantileRegressorInvalidAlpha()
                 Objective = LightGbmRegressionTrainer.Options.RegressionObjective.Quantile,
                 Alpha = 0.0,
             });
-            Assert.ThrowsAny<Exception>(() => trainerZero.Fit(dataView));
+            Assert.Throws<ArgumentOutOfRangeException>(() => trainerZero.Fit(dataView));
 
             // Alpha = 1 should fail
             var trainerOne = ML.Regression.Trainers.LightGbm(new LightGbmRegressionTrainer.Options
             {
                 Objective = LightGbmRegressionTrainer.Options.RegressionObjective.Quantile,
                 Alpha = 1.0,
             });
-            Assert.ThrowsAny<Exception>(() => trainerOne.Fit(dataView));
+            Assert.Throws<ArgumentOutOfRangeException>(() => trainerOne.Fit(dataView));
 
             // Alpha = -0.1 should fail
             var trainerNeg = ML.Regression.Trainers.LightGbm(new LightGbmRegressionTrainer.Options
             {
                 Objective = LightGbmRegressionTrainer.Options.RegressionObjective.Quantile,
                 Alpha = -0.1,
             });
-            Assert.ThrowsAny<Exception>(() => trainerNeg.Fit(dataView));
+            Assert.Throws<ArgumentOutOfRangeException>(() => trainerNeg.Fit(dataView));
 
             Done();
         }