Implements metric normalization in the HV Curve Converter

PiperOrigin-RevId: 720664003

Author: vizier-team

vizier/_src/algorithms/ensemble/ensemble_designer.py

@@ -74,6 +74,7 @@ def curve_generator() -> analyzers.StatefulCurveConverter:
             reference_value=self.reference_value,
             num_vectors=self.num_vectors,
             infer_origin_factor=0.1,
+            disable_metric_normalization=True,
         )
 
     stateful_curve_generator = analyzers.RestartingCurveConverter(

vizier/_src/benchmarks/analyzers/convergence_curve.py

@@ -349,6 +349,7 @@ def __init__(
       reference_value: Optional[np.ndarray] = None,
       num_vectors: int = 10000,
       infer_origin_factor: float = 0.0,
+      disable_metric_normalization: bool = False,
   ):
     """Init.
 
@@ -361,6 +362,7 @@ def __init__(
       num_vectors: Number of vectors from which hypervolume is computed.
       infer_origin_factor: When inferring the reference point, set origin to be
         minimum value - factor * (range).
+      disable_metric_normalization: If True, disable metric normalization.
     """
     if len(metric_informations) < 2:
       raise ValueError(
@@ -386,8 +388,9 @@ def create_metric_converter(mc):
     self._origin_value = reference_value
     # TODO: Speed this up with hypervolume vector tracking.
     self._min_trial_idx = 1
-    self._pareto_frontier = np.empty(shape=(0, len(metric_informations)))
     self._infer_origin_factor = infer_origin_factor
+    self._disable_metric_normalization = disable_metric_normalization
+    self._all_metrics = np.empty(shape=(0, len(metric_informations)))
 
   def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve:
     """Returns ConvergenceCurve with a curve of shape 1 x len(trials)."""
@@ -396,6 +399,28 @@ def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve:
       raise ValueError(f'No trials provided {trials}')
 
     metrics = self._converter.to_labels_array(trials)
+    self._all_metrics = np.vstack([self._all_metrics, metrics])
+    if self._disable_metric_normalization:
+      normalizer = 1.0
+    else:
+      normalizer = np.nanmedian(
+          np.absolute(
+              self._all_metrics
+              - np.nanmedian(self._all_metrics, axis=0, keepdims=True)
+          ),
+          axis=0,
+          keepdims=True,
+      )
+      if np.any(normalizer <= 0):
+        raise ValueError(
+            'Some metric normalizers are zero. This is likely due to some'
+            ' metrics being identical or contain a lot of duplicates across'
+            f' trials. Normalizers: {normalizer}'
+        )
+    metrics = metrics / normalizer
+    # shape is [num_existing_points + num_new_points, num_metrics]
+    all_metrics = self._all_metrics / normalizer
+
     if self._origin_value is None:
       # Set origin to the minimum of finite values.
       origin = np.zeros(shape=(metrics.shape[1],))
@@ -428,10 +453,6 @@ def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve:
           )
         origin = self._origin_value
 
-    # Calculate cumulative hypervolume with the Pareto frontier.
-    all_metrics = np.vstack(
-        [self._pareto_frontier, metrics]
-    )  # shape is [num_pareto_points + num_points, feature dimension]
     front = multimetric.ParetoFrontier(
         points=all_metrics,
         origin=origin,
@@ -440,17 +461,12 @@ def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve:
     )
     all_hv_curve = front.hypervolume(is_cumulative=True)
 
-    # Remove the Pareto frontier add-in and update state.
-    hv_curve = all_hv_curve[len(self._pareto_frontier) :]
+    # Extracts the hv points for the new trials.
+    hv_curve = all_hv_curve[-metrics.shape[0] :]
     xs = np.asarray(
         range(self._min_trial_idx, len(hv_curve) + self._min_trial_idx)
     )
     self._min_trial_idx += len(hv_curve)
-    algo = multimetric.FastParetoOptimalAlgorithm(
-        xla_pareto.JaxParetoOptimalAlgorithm()
-    )
-    pareto_points = algo.is_pareto_optimal(points=all_metrics)
-    self._pareto_frontier = all_metrics[pareto_points]
 
     return ConvergenceCurve(
         xs=xs,

vizier/_src/benchmarks/analyzers/convergence_curve_test.py

@@ -278,25 +278,28 @@ def test_convert_with_origin_reference(self):
     pytrials = []
     pytrials.append(
         pyvizier.Trial().complete(
-            pyvizier.Measurement(metrics={'max': 4.0, 'min': 2.0})
+            pyvizier.Measurement(metrics={'max': 8.0, 'min': 0.0})
         )
     )
     pytrials.append(
         pyvizier.Trial().complete(
-            pyvizier.Measurement(metrics={'max': 3.0, 'min': -1.0})
+            pyvizier.Measurement(metrics={'max': 6.0, 'min': -2.0})
         )
     )
     pytrials.append(
         pyvizier.Trial().complete(
-            pyvizier.Measurement(metrics={'max': 4.0, 'min': -2.0})
+            pyvizier.Measurement(metrics={'max': 4.0, 'min': -4.0})
         )
     )
 
     curve = generator.convert(pytrials)
     np.testing.assert_array_equal(curve.xs, [1, 2, 3])
-    np.testing.assert_array_almost_equal(
-        curve.ys, [[0.0, 3.0, 8.0]], decimal=0.5
-    )
+    # After the sign of the minimization metric is flipped, the three metric
+    # points are (8, 0), (6, 2), (4, 4). After normalization, they become
+    # (4, 0), (3, 1), (2, 2). The origin is (0, 0). The sequence of hypervolume
+    # is expected to be (0, 3.5, 6.0). However, we allow a large error because
+    # the hypervolume computation is approximate.
+    np.testing.assert_array_almost_equal(curve.ys, [[0.0, 3.5, 6.0]], decimal=0)
 
   def test_convert_with_reference(self):
     generator = convergence.HypervolumeCurveConverter(
@@ -313,25 +316,29 @@ def test_convert_with_reference(self):
     pytrials = []
     pytrials.append(
         pyvizier.Trial().complete(
-            pyvizier.Measurement(metrics={'max': 4.0, 'min': 2.0})
+            pyvizier.Measurement(metrics={'max': 8.0, 'min': 0.0})
         )
     )
     pytrials.append(
         pyvizier.Trial().complete(
-            pyvizier.Measurement(metrics={'max': 3.0, 'min': -1.0})
+            pyvizier.Measurement(metrics={'max': 6.0, 'min': -2.0})
         )
     )
     pytrials.append(
         pyvizier.Trial().complete(
-            pyvizier.Measurement(metrics={'max': 4.0, 'min': -2.0})
+            pyvizier.Measurement(metrics={'max': 4.0, 'min': -4.0})
         )
     )
 
     curve = generator.convert(pytrials)
     np.testing.assert_array_equal(curve.xs, [1, 2, 3])
-    np.testing.assert_array_almost_equal(
-        curve.ys, [[0.0, 0.0, 2.0]], decimal=0.5
-    )
+    # After the sign of the minimization metric is flipped, the three metric
+    # points are (8, 0), (6, 2), (4, 4). After normalization, they become
+    # (4, 0), (3, 1), (2, 2). After accounting for the reference point (3, 0),
+    # they are (1, 0), (0, 1), (-1, 2). The sequence of hypervolume
+    # is expected to be (0, 0.5, 0.5). However, we allow a large error
+    # because the hypervolume computation is approximate.
+    np.testing.assert_array_almost_equal(curve.ys, [[0.0, 0.5, 0.5]], decimal=0)
 
   def test_convert_with_none_reference(self):
     generator = convergence.HypervolumeCurveConverter([
@@ -361,9 +368,7 @@ def test_convert_with_none_reference(self):
 
     curve = generator.convert(pytrials)
     np.testing.assert_array_equal(curve.xs, [1, 2, 3])
-    np.testing.assert_array_almost_equal(
-        curve.ys, [[0.0, 0.0, 1.0]], decimal=0.5
-    )
+    np.testing.assert_array_almost_equal(curve.ys, [[0.0, 0.0, 1.0]], decimal=0)
 
   def test_convert_with_inf_none_reference(self):
     generator = convergence.HypervolumeCurveConverter([
@@ -401,6 +406,7 @@ def test_convert_with_state(self):
             ),
         ],
         reference_value=np.array([0.0]),
+        disable_metric_normalization=True,
     )
     pytrials = []
     pytrials.append(
@@ -415,15 +421,13 @@ def test_convert_with_state(self):
     )
     pytrials.append(
         pyvizier.Trial().complete(
-            pyvizier.Measurement(metrics={'max': 4.0, 'min': -2.0})
+            pyvizier.Measurement(metrics={'max': 3.0, 'min': -2.0})
         )
     )
 
     curve = generator.convert(pytrials)
     np.testing.assert_array_equal(curve.xs, [1, 2, 3])
-    np.testing.assert_array_almost_equal(
-        curve.ys, [[0.0, 5.0, 9.0]], decimal=0.5
-    )
+    np.testing.assert_array_almost_equal(curve.ys, [[0.0, 5.0, 9.0]], decimal=0)
 
     pytrials = []
     pytrials.append(
@@ -439,7 +443,7 @@ def test_convert_with_state(self):
 
     curve = generator.convert(pytrials)
     np.testing.assert_array_equal(curve.xs, [4, 5])
-    np.testing.assert_array_almost_equal(curve.ys, [[9.0, 10.0]], decimal=0.5)
+    np.testing.assert_array_almost_equal(curve.ys, [[9.0, 10.0]], decimal=0)
 
   def test_convert_factor_with_inf(self):
     generator = convergence.HypervolumeCurveConverter(
@@ -534,7 +538,7 @@ def test_convert_multiobjective(self):
     pytrials = []
     pytrials.append(
         pyvizier.Trial().complete(
-            pyvizier.Measurement(metrics={'max': 4.0, 'min': -1.0, 'safe': 1.0})
+            pyvizier.Measurement(metrics={'max': 2.0, 'min': 0.0, 'safe': 1.0})
         )
     )
     pytrials.append(
@@ -552,9 +556,7 @@ def test_convert_multiobjective(self):
 
     curve = generator.convert(pytrials)
     np.testing.assert_array_equal(curve.xs, [1, 2, 3])
-    np.testing.assert_array_almost_equal(
-        curve.ys, [[4.0, 4.0, 8.0]], decimal=0.5
-    )
+    np.testing.assert_array_almost_equal(curve.ys, [[0.0, 0.0, 2.0]], decimal=0)
 
 
 class RestartingCurveConverterTest(absltest.TestCase):
@@ -570,6 +572,7 @@ def converter_factory():
                   name='min', goal=pyvizier.ObjectiveMetricGoal.MINIMIZE
               ),
           ],
+          disable_metric_normalization=True,
       )
 
     restart_converter = convergence.RestartingCurveConverter(