Add inner benchmark metrics component

Author: Deleter-D

benchmarks/format_benchmark_metrics.py

@@ -0,0 +1,115 @@
+"""
+# Copyright (c) 2026  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+import argparse
+import json
+import os
+import sys
+
+
+def read_last_line(file_path: str) -> str:
+    with open(file_path, "rb") as f:
+        f.seek(0, 2)
+        file_size = f.tell()
+        if file_size == 0:
+            return ""
+        pos = file_size - 1
+        while pos > 0:
+            f.seek(pos)
+            char = f.read(1)
+            if char == b"\n" and pos < file_size - 1:
+                break
+            pos -= 1
+        if pos == 0:
+            f.seek(0)
+        return f.read().decode("utf-8").strip()
+
+
+def print_stat_block(data: dict, key: str, metric_name: str, header: str, is_time: bool = True):
+    stats = data.get(key)
+    if not stats:
+        return
+    suffix = "(ms)" if is_time else ""
+    if key == "decode_speed":
+        suffix = "(tok/s)"
+
+    print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
+    print("{:<40} {:<10.2f}".format(f"Mean {metric_name} {suffix}:", stats["mean"]))
+    print("{:<40} {:<10.2f}".format(f"Median {metric_name} {suffix}:", stats["median"]))
+
+    for k, v in stats.items():
+        if k.startswith("p"):
+            label = k.upper()
+            print("{:<40} {:<10.2f}".format(f"{label} {metric_name} {suffix}:", v))
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Read and display benchmark metrics from JSONL file.")
+    parser.add_argument(
+        "--file",
+        type=str,
+        default=None,
+        help="Path to benchmark_metrics.jsonl. Defaults to $FD_LOG_DIR/benchmark_metrics.jsonl",
+    )
+    args = parser.parse_args()
+
+    if args.file:
+        file_path = args.file
+    else:
+        log_dir = os.environ.get("FD_LOG_DIR", "./log")
+        file_path = os.path.join(log_dir, "benchmark_metrics.jsonl")
+
+    if not os.path.exists(file_path):
+        print(f"File not found: {file_path}", file=sys.stderr)
+        sys.exit(1)
+
+    last_line = read_last_line(file_path)
+    if not last_line:
+        print("No data in file.", file=sys.stderr)
+        sys.exit(1)
+
+    data = json.loads(last_line)
+
+    print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
+    print("{:<40} {:<10}".format("Timestamp:", data.get("timestamp", "N/A")))
+    print("{:<40} {:<10}".format("Window size:", data.get("window_size", 0) or "all"))
+    print("{:<40} {:<10}".format("Completed requests:", data.get("completed", 0)))
+    print("{:<40} {:<10}".format("Total input tokens:", data.get("total_input_tokens", 0)))
+    print("{:<40} {:<10}".format("Total output tokens:", data.get("total_output_tokens", 0)))
+
+    if "request_throughput" in data:
+        print("{:<40} {:<10.3f}".format("Request throughput (req/s):", data["request_throughput"]))
+    if "output_throughput" in data:
+        print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):", data["output_throughput"]))
+    if "total_throughput" in data:
+        print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):", data["total_throughput"]))
+
+    print_stat_block(data, "s_decode", "Decode", "解码速度(tok/s)", is_time=False)
+    print_stat_block(data, "ttft_ms", "TTFT", "Time to First Token")
+    print_stat_block(data, "s_ttft_ms", "S_TTFT", "Infer Time to First Token")
+    print_stat_block(data, "tpot_ms", "TPOT", "Time per Output Token (excl. 1st token)")
+    print_stat_block(data, "itl_ms", "S_ITL", "Infer Inter-token Latency")
+    print_stat_block(data, "e2el_ms", "E2EL", "End-to-end Latency")
+    print_stat_block(data, "s_e2el_ms", "S_E2EL", "Infer End-to-end Latency")
+    print_stat_block(data, "input_len", "Cached Tokens", "Cached Tokens", is_time=False)
+    print_stat_block(data, "s_input_len", "Input Length", "Infer Input Length", is_time=False)
+    print_stat_block(data, "output_len", "Output Length", "Output Length", is_time=False)
+
+    print("=" * 50)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/plot_benchmark_metrics.py

@@ -0,0 +1,203 @@
+"""
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+import argparse
+import json
+import os
+import sys
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def load_jsonl(file_path: str) -> list[dict]:
+    records = []
+    with open(file_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+    return records
+
+
+_THROUGHPUT_KEYS = ["request_throughput", "output_throughput", "total_throughput"]
+
+
+def detect_metrics(records: list[dict]) -> list[str]:
+    """Detect metric names from JSONL records (throughput scalars + dict-valued keys)."""
+    throughput_found = set()
+    dict_metrics = []
+
+    for rec in records:
+        for k in _THROUGHPUT_KEYS:
+            if k in rec:
+                throughput_found.add(k)
+        if not dict_metrics:
+            for k, v in rec.items():
+                if isinstance(v, dict):
+                    dict_metrics.append(k)
+        if len(throughput_found) == len(_THROUGHPUT_KEYS) and dict_metrics:
+            break
+
+    metrics = [k for k in _THROUGHPUT_KEYS if k in throughput_found]
+    metrics.extend(dict_metrics)
+    return metrics
+
+
+def plot_metric(records: list[dict], metric: str, output_path: str):
+    """Plot a single metric for a chunk of records."""
+    stat_keys = []
+    for rec in records:
+        val = rec.get(metric)
+        if isinstance(val, dict):
+            stat_keys = list(val.keys())
+            break
+
+    if not stat_keys:
+        values = [rec.get(metric) for rec in records if rec.get(metric) is not None]
+        if not values:
+            return
+        x = np.arange(len(values))
+        plt.figure(figsize=(12, 6))
+        plt.plot(x, values, linewidth=2, label=metric)
+        plt.xlabel("Request Index")
+        plt.ylabel(metric)
+        plt.title(f"{metric} over requests")
+        plt.legend()
+        plt.grid(True, alpha=0.3)
+        plt.tight_layout()
+        plt.savefig(output_path, dpi=150)
+        plt.close()
+        print(f"Saved: {output_path}")
+        return
+
+    plt.figure(figsize=(12, 6))
+
+    for key in stat_keys:
+        values = []
+        for rec in records:
+            val = rec.get(metric)
+            if isinstance(val, dict) and key in val:
+                values.append(val[key])
+            else:
+                values.append(None)
+
+        valid_x = [i for i, v in enumerate(values) if v is not None]
+        valid_v = [v for v in values if v is not None]
+
+        if not valid_v:
+            continue
+
+        linestyle = "-" if key == "mean" else "--"
+        lw = 2 if key == "mean" else 1.5
+        plt.plot(valid_x, valid_v, linewidth=lw, linestyle=linestyle, label=key)
+
+    plt.xlabel("Request Index")
+    unit = ""
+    if metric.endswith("_ms"):
+        unit = " (ms)"
+    elif metric in ("s_decode",):
+        unit = " (tok/s)"
+    elif metric == "request_throughput":
+        unit = " (req/s)"
+    elif metric in ("output_throughput", "total_throughput"):
+        unit = " (tok/s)"
+    plt.ylabel(f"{metric}{unit}")
+    plt.title(f"{metric} over requests")
+    plt.legend()
+    plt.grid(True, alpha=0.3)
+    plt.tight_layout()
+    plt.savefig(output_path, dpi=150)
+    plt.close()
+    print(f"Saved: {output_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Plot benchmark metrics from JSONL file. "
+        "Metrics and window_size are read from the data automatically. "
+        "Records are grouped into window-sized chunks for per-step analysis."
+    )
+    parser.add_argument(
+        "--file",
+        type=str,
+        default=None,
+        help="Path to benchmark_metrics.jsonl. Defaults to $FD_LOG_DIR/benchmark_metrics.jsonl",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=None,
+        help="Directory to save output PNG files. Defaults to $FD_LOG_DIR/benchmark_plots/",
+    )
+    args = parser.parse_args()
+
+    log_dir = os.environ.get("FD_LOG_DIR", "./log")
+
+    if args.file:
+        file_path = args.file
+    else:
+        file_path = os.path.join(log_dir, "benchmark_metrics.jsonl")
+
+    if not os.path.exists(file_path):
+        print(f"File not found: {file_path}", file=sys.stderr)
+        sys.exit(1)
+
+    output_dir = args.output_dir if args.output_dir else os.path.join(log_dir, "benchmark_plots")
+
+    records = load_jsonl(file_path)
+    if not records:
+        print("No data in file.", file=sys.stderr)
+        sys.exit(1)
+
+    window_size = records[0].get("window_size", 0)
+    metrics = detect_metrics(records)
+
+    if not metrics:
+        print("No plottable metrics found in data.", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Loaded {len(records)} records from {file_path}")
+    print(f"Window size: {window_size or 'all'}")
+    print(f"Metrics: {', '.join(metrics)}")
+
+    if window_size > 0:
+        num_chunks = (len(records) + window_size - 1) // window_size
+        chunks = []
+        for i in range(num_chunks):
+            start = i * window_size
+            end = min(start + window_size, len(records))
+            chunks.append(records[start:end])
+        print(f"Split into {num_chunks} step(s) of size {window_size}")
+    else:
+        chunks = [records]
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    for step_idx, chunk in enumerate(chunks):
+        if len(chunks) > 1:
+            step_dir = os.path.join(output_dir, f"step_{step_idx + 1}")
+            os.makedirs(step_dir, exist_ok=True)
+        else:
+            step_dir = output_dir
+
+        for metric in metrics:
+            output_path = os.path.join(step_dir, f"{metric}.png")
+            plot_metric(chunk, metric, output_path)
+
+
+if __name__ == "__main__":
+    main()

fastdeploy/config.py

@@ -1896,6 +1896,59 @@ def __str__(self):
         return self.to_json_string()
 
 
+class BenchmarkMetricsConfig:
+    """Configuration for in-process benchmark metrics logger.
+
+    Args (passed as JSON dict via --benchmark-metrics-config):
+        window_size: Number of recent requests to aggregate. 0 = all requests (cumulative).
+        percentiles: Comma-separated percentile values to compute, e.g. "50,90,95,99".
+        metrics: Comma-separated metric names to report, or "all".
+            Available metrics (aligned with benchmark_serving.py --percentile-metrics):
+                ttft          - Time to First Token (client arrival → first token)
+                s_ttft        - Server TTFT (inference start → first token)
+                tpot          - Time per Output Token (excluding first token)
+                itl           - Inter-token Latency
+                e2el          - End-to-end Latency (client arrival → last token)
+                s_e2el        - Server E2EL (inference start → last token)
+                s_decode      - Decode speed (tokens/s, excluding first token)
+                input_len     - Prefix cache hit token count ("Cached Tokens" in benchmark_serving)
+                s_input_len   - Infer input length (total prompt tokens on inference side)
+                output_len    - Output token length per request
+    """
+
+    _DEFAULTS = {
+        "window_size": 0,
+        "percentiles": "50,90,95,99",
+        "metrics": "all",
+    }
+
+    _ALL_METRICS = [
+        "ttft",  # Time to First Token
+        "s_ttft",  # Server TTFT
+        "tpot",  # Time per Output Token
+        "itl",  # Inter-token Latency
+        "e2el",  # End-to-end Latency
+        "s_e2el",  # Server E2EL
+        "s_decode",  # Decode speed (tok/s)
+        "input_len",  # Prefix cache hit tokens (= "Cached Tokens" in benchmark_serving)
+        "s_input_len",  # Infer input length (total prompt tokens)
+        "output_len",  # Output token length
+    ]
+
+    def __init__(self, args: Optional[dict] = None):
+        for key, value in self._DEFAULTS.items():
+            setattr(self, key, value)
+        if args:
+            for key, value in args.items():
+                if key in self._DEFAULTS:
+                    setattr(self, key, value)
+        self.percentile_values = [float(p.strip()) for p in self.percentiles.split(",") if p.strip()]
+        if self.metrics == "all":
+            self.selected_metrics = set(self._ALL_METRICS)
+        else:
+            self.selected_metrics = {m.strip() for m in self.metrics.split(",") if m.strip()}
+
+
 class FDConfig:
     """
     The configuration class which contains all fastdeploy-related configuration. This
@@ -1930,6 +1983,7 @@ def __init__(
         tool_parser: str = None,
         test_mode=False,
         routing_replay_config: Optional[RoutingReplayConfig] = None,
+        benchmark_metrics_config=None,
         deploy_modality: DeployModality = DeployModality.MIXED,
     ):
         self.model_config: ModelConfig = model_config  # type: ignore
@@ -1947,6 +2001,7 @@ def __init__(
         self.structured_outputs_config: StructuredOutputsConfig = structured_outputs_config
         self.router_config: RouterConfig = router_config
         self.routing_replay_config = routing_replay_config
+        self.benchmark_metrics_config = benchmark_metrics_config
         self.deploy_modality: DeployModality = deploy_modality
 
         # Initialize cuda graph capture list