diff --git a/sequence/network_management/network_manager.py b/sequence/network_management/network_manager.py index 9c5d38c2..1b1097c5 100644 --- a/sequence/network_management/network_manager.py +++ b/sequence/network_management/network_manager.py @@ -14,7 +14,8 @@ from ..components.memory import MemoryArray from ..message import Message -from ..utils import log +from ..utils import log, metrics +from ..utils.metrics.event_types import EventTypes from .forwarding import ForwardingProtocol from .memory_timecard import MemoryTimeCard from .reservation import Reservation @@ -226,12 +227,38 @@ def pop(self, msg: RSVPMessage): """ reservation: Reservation = msg.reservation if msg.msg_type == RSVPMsgType.APPROVE: + metrics.record( + EventTypes.RESERVATION_APPROVED, + self.owner.name, + identity=reservation.identity, + initiator=reservation.initiator, + responder=reservation.responder, + start_time=reservation.start_time, + end_time=reservation.end_time, + memory_size=reservation.memory_size, + entanglement_number=reservation.entanglement_number, + target_fidelity=reservation.fidelity, + path=list(reservation.path), + ) self.generate_rules(reservation) if reservation.initiator == self.owner.name: self.owner.get_reservation_result(reservation, True) # Deliver the result to the Node elif reservation.responder == self.owner.name: self.owner.get_other_reservation(reservation) elif msg.msg_type == RSVPMsgType.REJECT: + metrics.record( + EventTypes.RESERVATION_REJECTED, + self.owner.name, + identity=reservation.identity, + initiator=reservation.initiator, + responder=reservation.responder, + start_time=reservation.start_time, + end_time=reservation.end_time, + memory_size=reservation.memory_size, + entanglement_number=reservation.entanglement_number, + target_fidelity=reservation.fidelity, + path=[], + ) if reservation.initiator == self.owner.name: self.owner.get_reservation_result(reservation, False) diff --git a/sequence/utils/metrics/__init__.py b/sequence/utils/metrics/__init__.py index 67684c62..5c4a4037 100644 --- a/sequence/utils/metrics/__init__.py +++ b/sequence/utils/metrics/__init__.py @@ -26,18 +26,10 @@ from .event_types import ( EventType, EventTypes, - get_event_type, list_event_types, register_event_type, ) -from .metric_types import ( - CollectContext, - CounterMetric, - DeliveryTimeMetric, - FidelityMetric, - Metric, - RateMetric, -) +from .metric_types import CollectContext, CounterMetric, DeliveryTimeMetric, FidelityMetric, Metric, ThroughputMetric from .registry import ( clear_registry, get_counter, @@ -131,8 +123,6 @@ def collect_trial_metrics( *, delivery_owner: str | None = None, target_pairs: int | None = None, - reservation_start_time: int | None = None, - throughput: float | None = None, ) -> dict[str, Any]: """Collect per-trial metrics for a node from the metrics module. @@ -140,8 +130,6 @@ def collect_trial_metrics( owner_name: Node name to collect counter and fidelity metrics for. delivery_owner: Node name used for delivery-time metrics; defaults to `owner_name`. target_pairs: Number of delivered pairs required to compute delivery time. - reservation_start_time: Simulation time when the reservation started (ps). - throughput: Application throughput to include in collected metrics. Returns: Mapping of metric output keys to per-trial values. @@ -151,8 +139,6 @@ def collect_trial_metrics( storage=storage, delivery_owner=delivery_owner or owner_name, target_pairs=target_pairs, - reservation_start_time=reservation_start_time, - throughput=throughput, ) result: dict[str, Any] = {} for metric in list_metrics(): @@ -160,6 +146,69 @@ def collect_trial_metrics( return result +def collect_reservation_data(owner_name: str | None = None) -> list[list]: + """Collect per-reservation tabular metrics from recorded delivery events.""" + from collections import defaultdict + + records = storage.get_by_event(EventTypes.DELIVERY) + if owner_name is not None: + records = [record for record in records if record["owner_name"] == owner_name] + + groups: dict[tuple[str, int], list[dict[str, Any]]] = defaultdict(list) + for record in records: + groups[(record["owner_name"], record["identity"])].append(record) + + data: list[list] = [] + for (node, _), deliveries in groups.items(): + if not deliveries: + continue + deliveries.sort(key=lambda record: record["sim_time"]) + timestamps = [record["sim_time"] for record in deliveries] + fidelities = [record["fidelity"] for record in deliveries] + + first = deliveries[0] + start_time = first["start_time"] + end_time = first["end_time"] + reserved_time = end_time - start_time + served_pairs = len(deliveries) + throughput = served_pairs / reserved_time * 1e12 if reserved_time > 0 else 0.0 + completion_time = timestamps[-1] + entanglement_number = first["entanglement_number"] + fulfilled = served_pairs == entanglement_number + path = first.get("path", []) + path_length = len(path) + first_pair = timestamps[0] + avg_fidelity = mean(fidelities) if fidelities else 0.0 + std_fidelity = stdev(fidelities) if len(fidelities) > 1 else 0.0 + durations = [n - c for c, n in zip(timestamps, timestamps[1:])] + avg_duration = mean(durations) if durations else 0.0 + std_duration = stdev(durations) if len(durations) > 1 else 0.0 + + data.append( + [ + node, + first["identity"], + first["initiator"], + first["responder"], + start_time, + end_time, + reserved_time, + entanglement_number, + served_pairs, + throughput, + completion_time, + fulfilled, + path_length, + first_pair, + avg_fidelity, + std_fidelity, + avg_duration, + std_duration, + ] + ) + return data + + def _mean_and_std(values: list[float]) -> tuple[float, float]: finite_values = [value for value in values if math.isfinite(value)] if not finite_values: @@ -223,7 +272,6 @@ def aggregate_trial_metrics( # From event_types "EventType", "EventTypes", - "get_event_type", "list_event_types", "register_event_type", # From metric_types @@ -232,7 +280,7 @@ def aggregate_trial_metrics( "DeliveryTimeMetric", "FidelityMetric", "Metric", - "RateMetric", + "ThroughputMetric", # From registry "clear_registry", "get_counter", diff --git a/sequence/utils/metrics/builtins.py b/sequence/utils/metrics/builtins.py index 59361c2e..1077ba53 100644 --- a/sequence/utils/metrics/builtins.py +++ b/sequence/utils/metrics/builtins.py @@ -3,15 +3,10 @@ from __future__ import annotations from .event_types import EventTypes -from .metric_types import ( - CounterMetric, - DeliveryTimeMetric, - FidelityMetric, - Metric, - RateMetric, -) +from .metric_types import CounterMetric, DeliveryTimeMetric, FidelityMetric, Metric, ThroughputMetric from .registry import register_metric +# Entanglement Management Metrics EG_METRIC = CounterMetric( prefix="eg", failure_event=EventTypes.EG_FAILURE, @@ -24,28 +19,43 @@ success_event=EventTypes.EP_SUCCESS, rate_field="ep_success_rate", ) -THROUGHPUT_METRIC = RateMetric(key="app_throughput") -PURIFIED_FIDELITIES_METRIC = FidelityMetric( - key="purified_fidelities", - event=EventTypes.EP_SUCCESS, - field="fidelity", -) -DELIVERY_TIME_METRIC = DeliveryTimeMetric( - key="delivery_time", - delivery_event=EventTypes.DELIVERY, -) ES_METRIC = CounterMetric( prefix="es", failure_event=EventTypes.ES_FAILURE, success_event=EventTypes.ES_SUCCESS, rate_field="es_success_rate", ) +PURIFIED_FIDELITIES_METRIC = FidelityMetric( + key="purified_fidelities", + event=EventTypes.EP_SUCCESS, + field="fidelity", +) SWAPPED_FIDELITIES_METRIC = FidelityMetric( key="swapped_fidelities", event=EventTypes.ES_SUCCESS, field="fidelity", ) +# Network Management Metrics +RESERVATION_APPROVAL_RATE = CounterMetric( + prefix="reservation_approval_rate", + failure_event=EventTypes.RESERVATION_REJECTED, + success_event=EventTypes.RESERVATION_APPROVED, + rate_field="reservation_approval_rate", +) + +# Resource Management Metrics + +# Application Metrics +THROUGHPUT_METRIC = ThroughputMetric( + key="app_throughput", + delivery_event=EventTypes.DELIVERY, +) +DELIVERY_TIME_METRIC = DeliveryTimeMetric( + key="delivery_time", + delivery_event=EventTypes.DELIVERY, +) + def register_builtin_metrics() -> None: """Register all built-in metrics with the global registry. @@ -57,10 +67,10 @@ def register_builtin_metrics() -> None: BUILTIN_METRICS: list[Metric] = [ EG_METRIC, EP_METRIC, + ES_METRIC, THROUGHPUT_METRIC, PURIFIED_FIDELITIES_METRIC, DELIVERY_TIME_METRIC, - ES_METRIC, SWAPPED_FIDELITIES_METRIC, ] diff --git a/sequence/utils/metrics/event_types.py b/sequence/utils/metrics/event_types.py index 89ca6c6c..a3644e8a 100644 --- a/sequence/utils/metrics/event_types.py +++ b/sequence/utils/metrics/event_types.py @@ -34,21 +34,6 @@ def register_event_type(name: str) -> EventType: return event_type -def get_event_type(name: str) -> EventType: - """Return a registered event type by name. - - Args: - name: Name of the event type to look up. - - Returns: - The registered event type. - """ - try: - return _registry[name] - except KeyError as exc: - raise KeyError(f"Event type '{name}' is not registered.") from exc - - def list_event_types() -> list[EventType]: """Return all registered event types. @@ -61,10 +46,30 @@ def list_event_types() -> list[EventType]: class EventTypes: """Namespace for built-in simulation event types.""" + # Entanglement Management Events EG_FAILURE = register_event_type("EG_FAILURE") EG_SUCCESS = register_event_type("EG_SUCCESS") EP_FAILURE = register_event_type("EP_FAILURE") EP_SUCCESS = register_event_type("EP_SUCCESS") - DELIVERY = register_event_type("DELIVERY") ES_FAILURE = register_event_type("ES_FAILURE") ES_SUCCESS = register_event_type("ES_SUCCESS") + + # Network Management Events + RESERVATION_APPROVED = register_event_type("RESERVATION_APPROVED") + RESERVATION_REJECTED = register_event_type("RESERVATION_REJECTED") + RESERVATION_REQUESTED = register_event_type("RESERVATION_REQUESTED") + RESERVATION_HOP_REJECT = register_event_type("RESERVATION_HOP_REJECT") + RESERVATION_REACHED_RESPONDER = register_event_type("RESERVATION_REACHED_RESPONDER") + + # Forwarding + FORWARDING_TABLE_MISS = register_event_type("FORWARDING_TABLE_MISS") + + # Routing + NEIGHBOR_DOWN = register_event_type("NEIGHBOR_DOWN") + NEIGHBOR_FULL = register_event_type("NEIGHBOR_FULL") + ROUTE_RECOMPUTED = register_event_type("ROUTE_RECOMPUTED") + LSA_ORIGINATED = register_event_type("LSA_ORIGINATED") + LSDB_UPDATED = register_event_type("LSDB_UPDATED") + + # Application Events + DELIVERY = register_event_type("DELIVERY") diff --git a/sequence/utils/metrics/metric_types.py b/sequence/utils/metrics/metric_types.py index 94056c30..5037c6cc 100644 --- a/sequence/utils/metrics/metric_types.py +++ b/sequence/utils/metrics/metric_types.py @@ -19,16 +19,12 @@ class CollectContext: storage: In-memory store of recorded events for the trial. delivery_owner: Node name for delivery-time metrics. target_pairs: Number of delivered pairs required to compute delivery time. - reservation_start_time: Simulation time when the reservation started (ps). - throughput: Application throughput supplied at collection time. """ owner_name: str storage: InMemoryStorage delivery_owner: str | None = None target_pairs: int | None = None - reservation_start_time: int | None = None - throughput: float | None = None class Metric(ABC): @@ -189,10 +185,11 @@ def reset(self) -> None: @dataclass -class RateMetric(Metric): - """Collects a rate value supplied at trial collection time (e.g. throughput).""" +class ThroughputMetric(Metric): + """Computes application throughput from recorded deliveries at collection time.""" key: str = "app_throughput" + delivery_event: EventType | None = None @property def event_types(self) -> frozenset[EventType]: @@ -203,17 +200,34 @@ def output_keys(self) -> frozenset[str]: return frozenset({self.key}) def collect(self, ctx: CollectContext) -> dict[str, Any]: - """Return the throughput value supplied at collection time. + """Compute throughput as delivered pairs per second over the reservation window. Args: - ctx: Collection context; uses `throughput` when set. + ctx: Collection context with delivery owner and stored delivery events. Returns: - Mapping with the configured rate key and throughput or NaN. + Mapping with the configured rate key in pairs per second, or NaN if data is insufficient. """ - if ctx.throughput is None: + if self.delivery_event is None: return {self.key: float("nan")} - return {self.key: ctx.throughput} + + delivery_owner = ctx.delivery_owner or ctx.owner_name + delivery_records = [ + record for record in ctx.storage.get_by_owner(delivery_owner) if record["event_type"] == self.delivery_event + ] + if not delivery_records: + return {self.key: float("nan")} + + delivery_records.sort(key=lambda record: record["sim_time"]) + start_time = delivery_records[0].get("start_time") + if start_time is None: + return {self.key: float("nan")} + + elapsed_ps = delivery_records[-1]["sim_time"] - start_time + if elapsed_ps <= 0: + return {self.key: float("nan")} + + return {self.key: len(delivery_records) / elapsed_ps * 1e12} @dataclass @@ -286,9 +300,12 @@ def collect(self, ctx: CollectContext) -> dict[str, Any]: delivery_records.sort(key=lambda record: record["sim_time"]) if ctx.target_pairs is None or len(delivery_records) < ctx.target_pairs: return {self.key: float("nan")} - if ctx.reservation_start_time is None: + + start_time = delivery_records[0].get("start_time") + if start_time is None: return {self.key: float("nan")} + target_time = delivery_records[ctx.target_pairs - 1]["sim_time"] return { - self.key: (target_time - ctx.reservation_start_time) * 1e-12, + self.key: (target_time - start_time) * 1e-12, } diff --git a/tests/utils/test_metrics.py b/tests/utils/test_metrics.py index b04d8eee..eeb1da2a 100644 --- a/tests/utils/test_metrics.py +++ b/tests/utils/test_metrics.py @@ -150,12 +150,39 @@ def test_collect_trial_metrics_returns_node_snapshot(): metrics.record(EventTypes.EG_FAILURE, "e0", fidelity=0.8) metrics.record(EventTypes.EG_SUCCESS, "e0", fidelity=0.8) - trial = metrics.collect_trial_metrics("e0", throughput=12.5) + trial = metrics.collect_trial_metrics("e0") assert trial["eg_failures"] == 1 assert trial["eg_success"] == 1 assert trial["eg_success_rate"] == 0.5 - assert trial["app_throughput"] == 12.5 + assert math.isnan(trial["app_throughput"]) + + +def test_collect_trial_metrics_computes_throughput_from_deliveries(): + timeline = Timeline(int(1e12)) + timeline.time = int(1e12) + metrics.register_time_provider(timeline) + metrics.enable([metrics.DELIVERY_TIME_METRIC]) + + metrics.record( + EventTypes.DELIVERY, + "right", + fidelity=0.9, + pair_number=1, + start_time=int(1e12), + ) + timeline.time = int(2e12) + metrics.record( + EventTypes.DELIVERY, + "right", + fidelity=0.9, + pair_number=2, + start_time=int(1e12), + ) + + trial = metrics.collect_trial_metrics("left", delivery_owner="right") + + assert trial["app_throughput"] == pytest.approx(2.0) def test_collect_trial_metrics_without_throughput_is_nan(): @@ -303,18 +330,19 @@ def now(self) -> int: "right", fidelity=0.7 + pair_number * 0.01, pair_number=pair_number, + start_time=int(1e12), ) trial = metrics.collect_trial_metrics( "left", delivery_owner="right", target_pairs=3, - reservation_start_time=int(1e12), ) assert trial["ep_success"] == 2 assert trial["purified_fidelities"] == [0.7, 0.75] assert trial["delivery_time"] == pytest.approx(0.4) + assert trial["app_throughput"] == pytest.approx(7.5) def test_collect_trial_metrics_delivery_time_nan_when_target_not_reached(): @@ -325,7 +353,6 @@ def test_collect_trial_metrics_delivery_time_nan_when_target_not_reached(): "left", delivery_owner="right", target_pairs=500, - reservation_start_time=int(1e12), ) assert math.isnan(trial["delivery_time"]) @@ -333,12 +360,17 @@ def test_collect_trial_metrics_delivery_time_nan_when_target_not_reached(): def test_collect_trial_metrics_delivery_owner_defaults_to_owner(): metrics.enable([metrics.DELIVERY_TIME_METRIC]) - metrics.record(EventTypes.DELIVERY, "right", fidelity=0.9, pair_number=1) + metrics.record( + EventTypes.DELIVERY, + "right", + fidelity=0.9, + pair_number=1, + start_time=0, + ) trial = metrics.collect_trial_metrics( "right", target_pairs=1, - reservation_start_time=0, ) assert not math.isnan(trial["delivery_time"]) @@ -415,3 +447,76 @@ def test_register_metric_rejects_duplicate_output_keys(): ) with pytest.raises(ValueError, match="already registered"): metrics.register_metric(duplicate) + + +def test_purified_delivery_assigns_pair_index(): + metrics.enable([metrics.DELIVERY_TIME_METRIC]) + kwargs = { + "identity": 7, + "initiator": "n1", + "responder": "n2", + "start_time": 0, + "end_time": int(1e12), + "memory_size": 5, + "entanglement_number": 2, + "target_fidelity": 0.9, + "path": ["n1", "n2"], + "fidelity": 0.91, + } + metrics.record(EventTypes.DELIVERY, "n1", **kwargs) + metrics.record(EventTypes.DELIVERY, "n1", **{**kwargs, "fidelity": 0.92}) + metrics.record(EventTypes.DELIVERY, "n2", **{**kwargs, "fidelity": 0.93}) + + n1_records = metrics.storage.get_by_owner("n1") + n2_records = metrics.storage.get_by_owner("n2") + assert n1_records[0]["pair_index"] == 1 + assert n1_records[1]["pair_index"] == 2 + assert n2_records[0]["pair_index"] == 1 + + +def test_collect_reservation_data_produces_expected_row(): + metrics.enable([metrics.DELIVERY_TIME_METRIC]) + start_time = int(1e12) + end_time = int(2e12) + base = { + "identity": 1, + "initiator": "n1", + "responder": "n2", + "start_time": start_time, + "end_time": end_time, + "memory_size": 5, + "entanglement_number": 2, + "target_fidelity": 0.9, + "path": ["n1", "m1", "n2"], + } + + class StubTimeProvider: + def __init__(self) -> None: + self._time = start_time + int(1e11) + + def now(self) -> int: + current = self._time + self._time += int(2e11) + return current + + metrics.register_time_provider(StubTimeProvider()) + metrics.record(EventTypes.DELIVERY, "n1", fidelity=0.91, **base) + metrics.record(EventTypes.DELIVERY, "n1", fidelity=0.92, **base) + + rows = metrics.collect_reservation_data("n1") + assert len(rows) == 1 + row = rows[0] + assert row[0] == "n1" + assert row[1] == 1 + assert row[2] == "n1" + assert row[3] == "n2" + assert row[4] == start_time + assert row[5] == end_time + assert row[6] == end_time - start_time + assert row[7] == 2 + assert row[8] == 2 + assert row[9] == pytest.approx(2 / (end_time - start_time) * 1e12) + assert row[11] is True + assert row[12] == 3 + assert row[14] == pytest.approx(0.915) + assert row[16] == pytest.approx(int(2e11))