Decode LLMs generate one token at a time

tanner-andrulis · tanner-andrulis · commit 5dbec7b75ca2 · 2026-02-24T16:13:54.000-05:00
diff --git a/examples/workloads/gpt3_175B_kv_cache.yaml b/examples/workloads/gpt3_175B_kv_cache.yaml
@@ -2,7 +2,7 @@ workload:
   rank_sizes:
     {% set BATCH_SIZE = BATCH_SIZE | default(1) %}
     {% set N_TOKENS = N_TOKENS | default(8192) %}
-    {% set N_NEW_TOKENS = N_TOKENS | default(8) %}
+    {% set N_NEW_TOKENS = N_TOKENS | default(1) %}
     B: {{BATCH_SIZE}}
     M: {{N_NEW_TOKENS}}
     M_FULL: {{N_TOKENS}}
diff --git a/examples/workloads/gpt3_6.7B_kv_cache.yaml b/examples/workloads/gpt3_6.7B_kv_cache.yaml
@@ -2,7 +2,7 @@ workload:
   rank_sizes:
     {% set BATCH_SIZE = BATCH_SIZE | default(1) %}
     {% set N_TOKENS = N_TOKENS | default(8192) %}
-    {% set N_NEW_TOKENS = N_TOKENS | default(8) %}
+    {% set N_NEW_TOKENS = N_TOKENS | default(1) %}
     B: {{BATCH_SIZE}}
     M: {{N_NEW_TOKENS}}
     M_FULL: {{N_TOKENS}}
