|
| 1 | +--- |
| 2 | +title: Model Preparation and Conversion |
| 3 | +weight: 4 |
| 4 | + |
| 5 | +### FIXED, DO NOT MODIFY |
| 6 | +layout: learningpathall |
| 7 | +--- |
| 8 | + |
| 9 | +To begin working with Llama 3, the pre-trained model parameters can be accessed through Meta’s Llama Downloads page. Users are required to request access by submitting their details and reviewing and accepting the Responsible Use Guide. Upon approval, a license and a download link—valid for 24 hours—are provided. For this exercise, the Llama 3.2 1B Instruct model is utilized; however, the same procedures can be applied to other available variants with only minor modifications. |
| 10 | + |
| 11 | +Convert the model into an ExecuTorch-compatible format optimized for Arm devices |
| 12 | +## Script the Model |
| 13 | + |
| 14 | +```python |
| 15 | +import torch |
| 16 | +from transformers import AutoModelForCausalLM |
| 17 | + |
| 18 | +model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-3.1-8B-Instruct", torch_dtype=torch.float16) |
| 19 | +scripted_model = torch.jit.script(model) |
| 20 | +scripted_model.save("llama_exec.pt") |
| 21 | + |
| 22 | +``` |
| 23 | + |
| 24 | +Install the llama-stack package from pip. |
| 25 | +```python |
| 26 | +pip install llama-stack |
| 27 | +``` |
| 28 | + |
| 29 | +Run the command to download, and paste the download link from the email when prompted. |
| 30 | +```python |
| 31 | +llama model download --source meta --model-id Llama3.2-1B-Instruct |
| 32 | +``` |
| 33 | + |
| 34 | +When the download is finished, the installation path is printed as output. |
| 35 | +```python |
| 36 | +Successfully downloaded model to /<path-to-home>/.llama/checkpoints/Llama3.2-1B-Instruct |
| 37 | +``` |
| 38 | +Verify by viewing the downloaded files under this path: |
| 39 | +``` |
| 40 | +ls $HOME/.llama/checkpoints/Llama3.2-1B-Instruct |
| 41 | +checklist.chk consolidated.00.pth params.json tokenizer.model |
| 42 | +
|
| 43 | +``` |
| 44 | + |
| 45 | +Export the model and generate a .pte file by running the appropriate Python command. This command will export the model and save the resulting file in your current working directory. |
| 46 | +```python |
| 47 | +python3 -m examples.models.llama.export_llama \ |
| 48 | +--checkpoint $HOME/.llama/checkpoints/Llama3.2-1B-Instruct/consolidated.00.pth \ |
| 49 | +--params $HOME/.llama/checkpoints/Llama3.2-1B-Instruct/params.json \ |
| 50 | +-kv --use_sdpa_with_kv_cache -X --xnnpack-extended-ops -qmode 8da4w \ |
| 51 | +--group_size 64 -d fp32 \ |
| 52 | +--metadata '{"get_bos_id":128000, "get_eos_ids":[128009, 128001, 128006, 128007]}' \ |
| 53 | +--embedding-quantize 4,32 \ |
| 54 | +--output_name="llama3_1B_kv_sdpa_xnn_qe_4_64_1024_embedding_4bit.pte" \ |
| 55 | +--max_seq_length 1024 \ |
| 56 | +--max_context_length 1024 |
| 57 | +``` |
| 58 | + |
| 59 | +Because Llama 3 has a larger vocabulary size, it is recommended to quantize the embeddings using the parameter --embedding-quantize 4,32. This helps to further optimize memory usage and reduce the overall model size. |
| 60 | + |
| 61 | + |
| 62 | +###### Load a pre-fine-tuned model (from Hugging Face) |
| 63 | +- Example: meta-llama/Llama-3-8B-Instruct or a customer-support fine-tuned variant |
| 64 | + |
| 65 | +###### Model Optimization for ARM (Understanding Quantization) |
| 66 | +- Reduces model precision (e.g., 32-bit → 8-bit) |
| 67 | +- Decreases memory footprint (~4x reduction) |
| 68 | +- Speeds up inference on CPU |
| 69 | +- Minimal accuracy loss for most tasks |
| 70 | + |
| 71 | +###### Apply Dynamic Quantization |
| 72 | +- Create optimize_model.py |
| 73 | + |
| 74 | +```python |
| 75 | +import torch |
| 76 | +from transformers import AutoModelForCausalLM, AutoTokenizer |
| 77 | +from torch.quantization import quantize_dynamic |
| 78 | +import time |
| 79 | +import os |
| 80 | + |
| 81 | +def load_base_model(model_name): |
| 82 | + """Load the base model""" |
| 83 | + print(f"Loading base model: {model_name}") |
| 84 | + |
| 85 | + tokenizer = AutoTokenizer.from_pretrained(model_name) |
| 86 | + tokenizer.pad_token = tokenizer.eos_token |
| 87 | + |
| 88 | + model = AutoModelForCausalLM.from_pretrained( |
| 89 | + model_name, |
| 90 | + torch_dtype=torch.float32, |
| 91 | + device_map=None, |
| 92 | + low_cpu_mem_usage=True |
| 93 | + ) |
| 94 | + model.eval() |
| 95 | + |
| 96 | + return model, tokenizer |
| 97 | + |
| 98 | +def apply_quantization(model): |
| 99 | + """Apply dynamic quantization""" |
| 100 | + print("Applying dynamic quantization...") |
| 101 | + |
| 102 | + quantized_model = quantize_dynamic( |
| 103 | + model, |
| 104 | + {torch.nn.Linear}, # Quantize linear layers |
| 105 | + dtype=torch.qint8 |
| 106 | + ) |
| 107 | + |
| 108 | + return quantized_model |
| 109 | + |
| 110 | +def test_model(model, tokenizer, prompt): |
| 111 | + """Test model with a sample prompt""" |
| 112 | + inputs = tokenizer(prompt, return_tensors="pt") |
| 113 | + |
| 114 | + start_time = time.time() |
| 115 | + with torch.no_grad(): |
| 116 | + outputs = model.generate( |
| 117 | + inputs.input_ids, |
| 118 | + max_new_tokens=100, |
| 119 | + do_sample=False, |
| 120 | + pad_token_id=tokenizer.eos_token_id |
| 121 | + ) |
| 122 | + inference_time = time.time() - start_time |
| 123 | + |
| 124 | + response = tokenizer.decode(outputs[0], skip_special_tokens=True) |
| 125 | + |
| 126 | + return response, inference_time |
| 127 | + |
| 128 | +def main(): |
| 129 | + model_name = "meta-llama/Meta-Llama-3-8B-Instruct" |
| 130 | + |
| 131 | + # Load base model |
| 132 | + base_model, tokenizer = load_base_model(model_name) |
| 133 | + |
| 134 | + # Test base model |
| 135 | + test_prompt = "How do I track my order?" |
| 136 | + print("\nTesting base model...") |
| 137 | + response, base_time = test_model(base_model, tokenizer, test_prompt) |
| 138 | + print(f"Base model inference time: {base_time:.2f}s") |
| 139 | + |
| 140 | + # Apply quantization |
| 141 | + quantized_model = apply_quantization(base_model) |
| 142 | + |
| 143 | + # Test quantized model |
| 144 | + print("\nTesting quantized model...") |
| 145 | + response, quant_time = test_model(quantized_model, tokenizer, test_prompt) |
| 146 | + print(f"Quantized model inference time: {quant_time:.2f}s") |
| 147 | + print(f"Speedup: {base_time / quant_time:.2f}x") |
| 148 | + |
| 149 | + # Save quantized model |
| 150 | + save_dir = "./models/quantized_llama3" |
| 151 | + os.makedirs(save_dir, exist_ok=True) |
| 152 | + |
| 153 | + torch.save(quantized_model.state_dict(), f"{save_dir}/model.pt") |
| 154 | + tokenizer.save_pretrained(save_dir) |
| 155 | + |
| 156 | + print(f"\nQuantized model saved to: {save_dir}") |
| 157 | + |
| 158 | +if __name__ == "__main__": |
| 159 | + main() |
| 160 | + |
| 161 | +``` |
0 commit comments