fix inference server's base config

Author: MoringLotus

python/infinilm/base_config.py

@@ -23,6 +23,10 @@ def __init__(self):
         self.enable_graph = self.args.enable_graph
         self.cache_type = self.args.cache_type
         self.enable_paged_attn = self.args.enable_paged_attn
+
+        # When enable_paged_attn is True, automatically set attn to "paged-attn"
+        if self.enable_paged_attn and self.attn == "default":
+            self.attn = "paged-attn"
         self.paged_kv_block_size = self.args.paged_kv_block_size
         self.num_blocks = self.args.num_blocks
         self.block_size = self.args.block_size
@@ -70,6 +74,7 @@ def __init__(self):
         self.port = self.args.port
         self.endpoint = self.args.endpoint
 
+        self.ignore_eos = self.args.ignore_eos
     def _add_common_args(self):
         # --- base configuration ---
         self.parser.add_argument("--model", type=str, required=True)
@@ -79,8 +84,8 @@ def _add_common_args(self):
 
         # --- Infer backend optimization ---
         self.parser.add_argument("--attn", type=str, default="default", choices=["default", "paged-attn", "flash-attn"])
-        self.parser.add_argument("--enable-graph", action="store_true")
-        self.parser.add_argument("--cache-type", type=str, default="paged", choices=["paged",   "static"])
+        self.parser.add_argument("--enable-graph", action="store_false")
+        self.parser.add_argument("--cache-type", type=str, default="paged", choices=["paged", "static"])
         self.parser.add_argument("--enable-paged-attn", action="store_true", help="use paged cache",)
         self.parser.add_argument("--paged-kv-block-size", type=int, default=256)
         self.parser.add_argument("--num-blocks", type=int, default=512, help="number of KV cache blocks")
@@ -131,6 +136,7 @@ def _add_common_args(self):
         self.parser.add_argument("--port", type=int, default=8000, help="server port")
         self.parser.add_argument("--endpoint", type=str, default="/completions", help="API endpoint")
 
+        self.parser.add_argument("--ignore-eos", action="store_true", dest="ignore_eos", default=False, help="Ignore EOS token and continue generation",)
 
     def get_device_str(self, device):
         """Convert device name to backend string (cuda/cpu/musa/mlu)"""

python/infinilm/server/inference_server.py

@@ -550,130 +550,31 @@ def setup_logging(log_level: str = "INFO"):
     )
 
 
-def parse_args():
-    """Parse command line arguments."""
-    parser = argparse.ArgumentParser(description="InfiniLM Inference Server")
-    parser.add_argument(
-        "--model_path", type=str, required=True, help="Path to model directory"
-    )
-    parser.add_argument("--tp", type=int, default=1, help="Tensor parallelism degree")
-    parser.add_argument(
-        "--cache_type",
-        type=str,
-        default="paged",
-        choices=["paged", "static"],
-        help="Cache type: paged or static",
-    )
-    parser.add_argument(
-        "--max_tokens",
-        type=int,
-        default=512,
-        help="Maximum number of tokens to generate",
-    )
-    parser.add_argument(
-        "--max_batch_size",
-        type=int,
-        default=8,
-        help="Maximum batch size (paged cache only)",
-    )
-    parser.add_argument(
-        "--num_blocks",
-        type=int,
-        default=512,
-        help="Number of blocks for KV cache (paged cache only)",
-    )
-    parser.add_argument(
-        "--block_size",
-        type=int,
-        default=256,
-        help="Block size for KV cache (paged cache only)",
-    )
-    parser.add_argument(
-        "--max_cache_len",
-        type=int,
-        default=4096,
-        help="Maximum sequence length (static cache only)",
-    )
-    parser.add_argument(
-        "--dtype",
-        type=str,
-        default="float16",
-        choices=["float32", "float16", "bfloat16"],
-        help="Data type",
-    )
-    parser.add_argument(
-        "--temperature", type=float, default=1.0, help="Sampling temperature"
-    )
-    parser.add_argument(
-        "--top_p", type=float, default=0.8, help="Top-p sampling parameter"
-    )
-    parser.add_argument("--top_k", type=int, default=1, help="Top-k sampling parameter")
-    parser.add_argument("--host", type=str, default="0.0.0.0", help="Server host")
-    parser.add_argument("--port", type=int, default=8000, help="Server port")
-    parser.add_argument("--cpu", action="store_true", help="Use CPU")
-    parser.add_argument("--nvidia", action="store_true", help="Use NVIDIA GPU")
-    parser.add_argument("--qy", action="store_true", help="Use QY GPU")
-    parser.add_argument("--metax", action="store_true", help="Use MetaX device")
-    parser.add_argument("--moore", action="store_true", help="Use Moore device")
-    parser.add_argument("--iluvatar", action="store_true", help="Use Iluvatar device")
-    parser.add_argument("--cambricon", action="store_true", help="Use Cambricon device")
-    parser.add_argument("--ali", action="store_true", help="Use Ali PPU device")
-    parser.add_argument("--hygon", action="store_true", help="Use Hygon DCU device")
-    parser.add_argument(
-        "--enable-graph",
-        action="store_true",
-        help="Enable graph compiling",
-    )
-    parser.add_argument(
-        "--attn",
-        type=str,
-        default="default",
-        choices=["default", "paged-attn", "flash-attn"],
-        help="Attention backend to use: 'default' or 'flash-attn'",
-    )
-    parser.add_argument(
-        "--log_level",
-        type=str,
-        default="INFO",
-        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
-        help="Logging level",
-    )
-    parser.add_argument(
-        "--ignore-eos",
-        action="store_true",
-        dest="ignore_eos",
-        default=False,
-        help="Ignore EOS token and continue generation",
-    )
-
-    return parser.parse_args()
-
 
 def main():
-    # args = parse_args()
     cfg = BaseConfig()
     setup_logging(cfg.log_level)
     device = cfg.get_device_str(cfg.device)
 
     server = InferenceServer(
         model_path=cfg.model,
         device=device,
-        dtype=args.dtype,
-        tensor_parallel_size=args.tp,
-        cache_type=args.cache_type,
-        max_tokens=args.max_tokens,
-        max_batch_size=args.max_batch_size,
-        num_blocks=args.num_blocks,
-        block_size=args.block_size,
-        max_cache_len=args.max_cache_len,
-        temperature=args.temperature,
-        top_p=args.top_p,
-        top_k=args.top_k,
-        host=args.host,
-        port=args.port,
-        enable_graph=args.enable_graph,
-        attn_backend=args.attn,
-        ignore_eos=args.ignore_eos,
+        dtype=cfg.dtype,
+        tensor_parallel_size=cfg.tp,
+        cache_type=cfg.cache_type,
+        max_tokens=cfg.max_tokens,
+        max_batch_size=cfg.max_batch_size,
+        num_blocks=cfg.num_blocks,
+        block_size=cfg.block_size,
+        max_cache_len=cfg.max_cache_len,
+        temperature=cfg.temperature,
+        top_p=cfg.top_p,
+        top_k=cfg.top_k,
+        host=cfg.host,
+        port=cfg.port,
+        enable_graph=cfg.enable_graph,
+        attn_backend=cfg.attn,
+        ignore_eos=cfg.ignore_eos,
     )
     server.start()