Pass torch_dtype="auto" to preserve the model's saved dtype

Author: voltjia

infer.py

@@ -81,7 +81,9 @@
     assert num_warmup_iterations >= 0
 
     tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
-    model = AutoModelForCausalLM.from_pretrained(model_name_or_path).to(device)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name_or_path, torch_dtype="auto"
+    ).to(device)
 
     tokenizer.pad_token = tokenizer.eos_token
     model.generation_config.pad_token_id = tokenizer.pad_token_id