lint

Author: Gasoonjia

.ci/scripts/cuda_benchmark.py

@@ -721,7 +721,7 @@ def create_metric_stats(
             {"trimmed_runs": len(trimmed_throughput)},
         ),
         prefill_throughput=create_metric_stats(
-            "prefill_throughput(tokens/sec)",
+            "prefill_encoder_throughput(tokens/sec)",
             [r.prefill_tokens_per_sec for r in results],
             "t/s",
         ),

extension/asr/runner/runner.cpp

@@ -242,7 +242,6 @@ Result<std::vector<int64_t>> AsrRunner::transcribe(
   ET_CHECK_OK_OR_RETURN_ERROR(encoder_result.error());
 
   stats_.prompt_eval_end_ms = ::executorch::extension::llm::time_in_ms();
-  stats_.num_prompt_tokens = 0;
 
   auto encoder_outputs = std::move(*encoder_result);
   ET_CHECK_OR_RETURN_ERROR(
@@ -254,6 +253,11 @@ Result<std::vector<int64_t>> AsrRunner::transcribe(
   ::executorch::aten::Tensor encoder_output_tensor =
       std::move(encoder_outputs[0]).toTensor();
 
+  // Use encoder output sequence length as "prompt tokens" for throughput
+  stats_.num_prompt_tokens =
+      static_cast<int64_t>(encoder_output_tensor.size(1));
+  std::move(encoder_outputs[0]).toTensor();
+
   ET_LOG(
       Info,
       "Encoder output shape: [%zu, %zu, %zu]",