From 4748b9662d921365a06ac75730a2ae9ffa5e9f87 Mon Sep 17 00:00:00 2001 From: teresatree Date: Tue, 6 Jan 2026 17:42:43 +0800 Subject: [PATCH 1/2] added wilderness survival model training notebook --- wilderness survivial.ipynb | 727 +++++++++++++++++++++++++++++++++++++ 1 file changed, 727 insertions(+) create mode 100644 wilderness survivial.ipynb diff --git a/wilderness survivial.ipynb b/wilderness survivial.ipynb new file mode 100644 index 00000000..2d4b3804 --- /dev/null +++ b/wilderness survivial.ipynb @@ -0,0 +1,727 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "dc935f2b-a58b-4110-8918-96c868047b38", + "metadata": {}, + "source": [ + "# Training A Small ERNIE Model With SFT" + ] + }, + { + "cell_type": "markdown", + "id": "a8baf9d5", + "metadata": {}, + "source": [ + "## Overview\n", + "This notebook fine-tunes a small ERNIE model (nghuyong/ernie-2.0-base-en) on survival scenario data using Supervised Fine-Tuning (SFT)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d76b86a7-7a45-4749-866e-0d561041bce1", + "metadata": { + "height": 64 + }, + "outputs": [], + "source": [ + "# Warning control\n", + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "id": "9bbfc986-9ac7-4a2d-9dd0-a76841c7f46d", + "metadata": {}, + "source": [ + "## Import libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "3304e49d-bd1e-469b-a5b4-5edb16ecf344", + "metadata": { + "height": 98 + }, + "outputs": [], + "source": [ + "import torch\n", + "import pandas as pd\n", + "from datasets import load_dataset, Dataset\n", + "from transformers import TrainingArguments, AutoTokenizer, AutoModelForCausalLM\n", + "from trl import SFTTrainer, DataCollatorForCompletionOnlyLM, SFTConfig" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "36cd279b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading nghuyong/ernie-2.0-base-en for causal language modeling...\n" + ] + } + ], + "source": [ + "model_name = \"nghuyong/ernie-2.0-base-en\"\n", + "USE_GPU = False\n", + "\n", + "print(f\"Loading {model_name} for causal language modeling...\")" + ] + }, + { + "cell_type": "markdown", + "id": "3cc63b02-5e9a-4a83-b042-4a2386cf5976", + "metadata": {}, + "source": [ + "## Setting up helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "69982ae0-755e-48cf-ba4c-3b83b091fd9a", + "metadata": { + "height": 557 + }, + "outputs": [], + "source": [ + "def generate_responses(model, tokenizer, user_message, system_message=None, \n", + " max_new_tokens=150):\n", + " # Simple format without chat template\n", + " if system_message:\n", + " prompt = f\"System: {system_message}\\nQuestion: {user_message}\\nAnswer:\"\n", + " else:\n", + " prompt = f\"Question: {user_message}\\nAnswer:\"\n", + " \n", + " inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n", + " \n", + " with torch.no_grad():\n", + " outputs = model.generate(\n", + " **inputs,\n", + " max_new_tokens=max_new_tokens,\n", + " do_sample=True,\n", + " temperature=0.7,\n", + " top_p=0.9,\n", + " pad_token_id=tokenizer.pad_token_id,\n", + " eos_token_id=tokenizer.eos_token_id,\n", + " )\n", + " \n", + " # Decode only the new tokens\n", + " input_len = inputs[\"input_ids\"].shape[1]\n", + " generated_ids = outputs[0][input_len:]\n", + " response = tokenizer.decode(generated_ids, skip_special_tokens=True).strip()\n", + "\n", + " return response" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "234e5b05-a493-4683-91fd-7417885efc0f", + "metadata": { + "height": 149 + }, + "outputs": [], + "source": [ + "def test_model_with_questions(model, tokenizer, questions, \n", + " system_message=None, title=\"Model Output\"):\n", + " print(f\"\\n=== {title} ===\")\n", + " for i, question in enumerate(questions, 1):\n", + " response = generate_responses(model, tokenizer, question, \n", + " system_message)\n", + " print(f\"\\nModel Input {i}:\\n{question}\\nModel Output {i}:\\n{response}\\n\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9c273931-6827-4ee1-af1a-83a99bf94bf7", + "metadata": { + "height": 387 + }, + "outputs": [], + "source": [ + "def load_model_and_tokenizer(model_name, use_gpu = False):\n", + " \n", + " # Load base model and tokenizer\n", + " tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + " model = AutoModelForCausalLM.from_pretrained(model_name)\n", + " \n", + " if use_gpu:\n", + " model.to(\"cuda\")\n", + " \n", + " if not tokenizer.chat_template:\n", + " tokenizer.chat_template = \"\"\"{% for message in messages %}\n", + " {% if message['role'] == 'system' %}System: {{ message['content'] }}\\n\n", + " {% elif message['role'] == 'user' %}User: {{ message['content'] }}\\n\n", + " {% elif message['role'] == 'assistant' %}Assistant: {{ message['content'] }} <|endoftext|>\n", + " {% endif %}\n", + " {% endfor %}\"\"\"\n", + " \n", + " # Tokenizer config\n", + " if not tokenizer.pad_token:\n", + " tokenizer.pad_token = tokenizer.eos_token\n", + " \n", + " return model, tokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "5bd15e1d-6ecd-4337-a5dd-1602da354f62", + "metadata": { + "height": 319 + }, + "outputs": [], + "source": [ + "def display_dataset(dataset):\n", + " # Visualize the dataset \n", + " rows = []\n", + " for i in range(3):\n", + " example = dataset[i]\n", + " user_msg = next(m['content'] for m in example['messages']\n", + " if m['role'] == 'user')\n", + " assistant_msg = next(m['content'] for m in example['messages']\n", + " if m['role'] == 'assistant')\n", + " rows.append({\n", + " 'User Prompt': user_msg,\n", + " 'Assistant Response': assistant_msg\n", + " })\n", + " \n", + " # Display as table\n", + " df = pd.DataFrame(rows)\n", + " pd.set_option('display.max_colwidth', None) # Avoid truncating long strings\n", + " display(df)" + ] + }, + { + "cell_type": "markdown", + "id": "9424f19b", + "metadata": {}, + "source": [ + "# Custom Dataset" + ] + }, + { + "cell_type": "markdown", + "id": "86260a8f", + "metadata": {}, + "source": [ + "## Dataset\n", + "\n", + "- Ideally you would need at least 50+ datasets." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "a3c1c428", + "metadata": {}, + "outputs": [], + "source": [ + "survival_data = [\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"I'm lost in the woods and running out of water. What do I do?\"},\n", + " {\"role\": \"assistant\", \"content\": \"Water is your top priority - you can only survive 3 days without it. Here's what to do:\\n\\n1. FIND water sources: Look for streams, rivers, or collect morning dew with a cloth\\n2. PURIFY before drinking: Boil for 1 minute (3 minutes at high altitude), or use purification tablets if you have them\\n3. If no fire: Solar water disinfection - fill a clear bottle and leave in direct sunlight for 6 hours\\n4. Collect rainwater: Use any container, tarp, or large leaves\\n5. Never drink stagnant water or urine - it will dehydrate you faster\\n\\nStay near water sources but not too close - predators also need water.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"How do I know if water is safe to drink?\"},\n", + " {\"role\": \"assistant\", \"content\": \"Even clear-looking water can contain parasites. Here's the hierarchy:\\n\\nSAFEST to RISKIEST:\\n1. Rainwater collected in clean container - generally safe\\n2. Fast-flowing streams/rivers - lower parasite risk, but STILL boil\\n3. Lakes and ponds - moderate risk, MUST purify\\n4. Stagnant water - high parasite risk, last resort only\\n\\nNEVER drink without purifying if possible. If you MUST drink unpurified:\\n- Choose fast-flowing water over still water\\n- Upstream from animal areas\\n- Filter through cloth first to remove debris\\n\\nSymptoms of waterborne illness take days to appear, so you might feel fine initially.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"I'm in the desert and need shelter from the sun.\"},\n", + " {\"role\": \"assistant\", \"content\": \"Desert survival is about staying cool during day, warm at night:\\n\\nDAYTIME SHELTER:\\n1. Find natural shade: rocks, cliffs, caves (check for animals first!)\\n2. Build a shade tarp: Use any fabric elevated 2-3 feet above ground\\n3. Create air flow: Don't seal yourself in - ventilation prevents overheating\\n4. Stay off hot ground: Sit on rocks, backpack, or dig shallow pit\\n\\nCRITICAL RULES:\\n- Rest during hottest hours (10am-4pm)\\n- Travel at dawn/dusk when cooler\\n- Cover your skin to prevent sunburn and reduce water loss\\n- Conserve energy - heat exhaustion is deadly\\n\\nNight gets COLD in deserts - keep your insulation for nighttime.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"I need to start a fire but I don't have matches or a lighter.\"},\n", + " {\"role\": \"assistant\", \"content\": \"Fire without tools is hard but doable. Here are methods that ACTUALLY work with practice:\\n\\nMETHOD 1: FIRE PLOUGH (Easiest to learn)\\n1. Find dry softwood board (cedar, willow, cottonwood)\\n2. Cut groove down the board\\n3. Find hardwood stick to rub in groove\\n4. Rub stick rapidly up and down groove with pressure\\n5. Wood dust accumulates at end and heats up\\n6. When smoking heavily, dump into tinder nest\\n\\nMETHOD 2: BOW DRILL (Most reliable once learned)\\nYou need:\\n- Spindle: straight stick, 8 inches, thumb-thick\\n- Fireboard: flat softwood, 2 inches wide\\n- Bow: curved branch, 2 feet long + shoelace/cord\\n- Handhold: wood/rock with dimple for top of spindle\\n\\nTechnique:\\n1. Cut notch in fireboard (45° wedge to center hole)\\n2. Wrap bowstring once around spindle\\n3. Press handhold on top, fireboard under spindle\\n4. Bow back and forth rapidly - spindle spins\\n5. Push down hard - friction creates ember in notch\\n6. Transfer ember to tinder nest carefully\\n\\nMETHOD 3: FLINT & STEEL (if you have knife)\\n- Strike knife spine against hard rock at 30° angle\\n- Aim sparks at char cloth or dry tinder\\n- Char cloth (charred cotton) catches sparks best\\n\\nCRITICAL SUCCESS FACTORS:\\n- BONE DRY materials (wet = impossible)\\n- Prepared tinder nest (size of baseball, fluffy)\\n- Practice at home first - it's harder than it looks\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"How should i place the fire?\"},\n", + " {\"role\": \"assistant\", \"content\": \"Fire structure matters! Here are proven layouts:\\n\\nTEEPEE FIRE (best for starting):\\n1. Place tinder bundle in center\\n2. Make small teepee of toothpick-size kindling over it\\n3. Build larger teepee of progressively bigger sticks around that\\n4. Leave opening for lighting and airflow\\n5. Light tinder through opening\\n6. As it grows, add bigger sticks\\n\\nWHY IT WORKS: Flames rise naturally up through structure, heat rises and dries wood above\\n\\nLOG CABIN (for lasting fire):\\n1. Start with tinder bundle\\n2. Place two logs parallel, tinder between them\\n3. Place two more logs on top, perpendicular\\n4. Continue building square layers\\n5. Fill center with kindling\\n6. Light from bottom\\n\\nWHY IT WORKS: Great airflow, burns evenly, self-feeding as it collapses\\n\\nLEAN-TO (for wind protection):\\n1. Place large log as windbreak\\n2. Lean kindling against it at angle\\n3. Tinder bundle underneath\\n4. Light from upwind side\\n\\nWHY IT WORKS: Protected from wind, easy to build\\n\\nPLATFORM FIRE (for wet ground):\\n1. Lay platform of green logs (wet wood okay here)\\n2. Build normal fire on top of platform\\n3. Keeps fire off wet ground\\n\\nKEY RULES:\\n- Fire needs AIR - don't pack too tight\\n- Add wood gradually - too much smothers it\\n- Small → medium → large (never skip sizes)\\n- Have next size ready before adding it\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"How should i signal for rescue?\"},\n", + " {\"role\": \"assistant\", \"content\": \"Search teams are looking for things that DON'T belong in nature. Make yourself obvious:\\n\\nMOST EFFECTIVE SIGNALS:\\n\\n1. SIGNAL MIRROR (visible 10+ miles to aircraft):\\n- Use any reflective surface: phone screen, metal, CD, belt buckle\\n- Technique: Hold up V-peace sign toward target\\n- Reflect light through the V onto target\\n- Flash repeatedly - movement catches attention\\n- Works in daytime only\\n\\n2. SIGNAL FIRE - THREE FIRES (universal distress):\\n- Build 3 fires in TRIANGLE pattern (100 feet apart)\\n- OR 3 fires in straight LINE\\n- Three of anything = distress signal\\n- Keep fires ready with tinder, light when you hear aircraft\\n- Add green branches for SMOKE during day (visible for miles)\\n- Keep fires burning bright at night\\n\\n3. GROUND SIGNALS (for aircraft):\\n- Make LARGE symbols (20+ feet):\\n - X = Need medical help\\n - V = Need assistance\\n - I = Need medical supplies\\n - → = Going this direction\\n- Use rocks, logs, clothing laid out\\n- Contrast is key - dark materials on light ground or vice versa\\n\\n4. WHISTLE (audible 1+ miles, farther than voice):\\n- THREE sharp blasts = universal distress\\n- Saves energy vs yelling\\n- Every hiking kit should have whistle\\n\\n5. BRIGHT COLORS:\\n- Hang bright clothing in trees\\n- Anything not natural colored (orange, red, yellow best)\\n- Movement attracts attention - wave bright items\\n\\nSTAY PUT once you signal. Moving makes rescue harder.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"I don't have a compass. How do I figure out which way is north?\"},\n", + " {\"role\": \"assistant\", \"content\": \"You can navigate using nature and the sky:\\n\\nDAYTIME - SUN METHOD (most accurate):\\n1. SHADOW STICK:\\n - Plant straight stick (3 feet tall) in flat ground\\n - Mark shadow tip with rock\\n - Wait 15-20 minutes\\n - Mark new shadow tip with second rock\\n - Draw line between rocks: this is EAST-WEST line\\n - First mark = WEST, second = EAST\\n - Face east, north is to your left\\n\\n2. WATCH METHOD (Northern Hemisphere):\\n - Point hour hand at sun\\n - Halfway between hour hand and 12 is SOUTH\\n - (Use daylight saving time awareness)\\n\\n3. SUN POSITION:\\n - Sun rises EAST, sets WEST (roughly)\\n - At noon, sun is in SOUTH sky (Northern Hemisphere)\\n - At noon, sun is in NORTH sky (Southern Hemisphere)\\n\\nNIGHTTIME - STAR METHOD:\\nNORTHERN HEMISPHERE:\\n1. Find Big Dipper (looks like pot with handle)\\n2. Find the two stars at end of the 'cup'\\n3. Draw imaginary line through them upward\\n4. Follow 5x that distance\\n5. You'll hit Polaris (North Star) - it's medium brightness\\n6. Polaris = TRUE NORTH (doesn't move)\\n\\nSOUTHERN HEMISPHERE:\\n1. Find Southern Cross (4 stars in cross shape)\\n2. Follow long axis of cross 4.5x its length\\n3. That point = SOUTH\\n\\nNATURE SIGNS (less reliable, use as backup only):\\n- Moss grows on NORTH side of trees (Northern Hemisphere) - the shaded, moist side\\n- Trees fuller/more branches on SOUTH side (more sun)\\n- Snow melts faster on SOUTH-facing slopes\\n- Ant hills often on SOUTH side of trees\\n\\nThese are general tendencies, not absolute. Use multiple methods to confirm.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"I'm shivering uncontrollably and feel confused. What's happening to me?\"},\n", + " {\"role\": \"assistant\", \"content\": \"You have HYPOTHERMIA - a medical emergency. Your core body temperature has dropped below 95°F. Act immediately:\\n\\nSTAGES & SYMPTOMS:\\n\\nMILD (95-90°F):\\n- Intense shivering\\n- Can't perform complex tasks\\n- Hands numb, fumbling\\n→ You're still conscious and functional\\n\\nMODERATE (90-82°F) - YOU ARE HERE:\\n- Violent shivering OR shivering stops\\n- Confusion, poor decisions\\n- Slurred speech\\n- Drowsiness, apathy\\n- Loss of coordination\\n→ Judgment is impaired - you need to act NOW\\n\\nSEVERE (below 82°F):\\n- No shivering (body gave up)\\n- Can't walk\\n- Unconscious\\n- Weak pulse\\n→ Death is imminent\\n\\nIMMEDIATE ACTIONS:\\n\\n1. STOP HEAT LOSS:\\n - Get OUT of wind/rain NOW (even crude shelter helps)\\n - Remove WET clothes immediately (wet = 25x more heat loss)\\n - Wet clothes kill - being naked is better than wet clothes\\n\\n2. INSULATE:\\n - Put on dry clothes if available\\n - Wrap in emergency blanket, tarp, garbage bag\\n - Stuff dry leaves, grass inside clothes\\n - Cover head (lose 40% of heat through head)\\n\\n3. EXTERNAL HEAT:\\n - Build fire if possible\\n - Huddle with another person (skin-to-skin in sleeping bag)\\n - Warm packs in armpits/groin (high blood flow areas)\\n - DO NOT rub extremities (can cause heart attack)\\n\\n4. WARM DRINKS (if conscious):\\n - Warm water, tea (not hot - can shock system)\\n - NO alcohol (dilates blood vessels = more heat loss)\\n - NO caffeine (increases urine = dehydration)\\n\\n5. STAY AWAKE:\\n - Sleep can be fatal in severe hypothermia\\n - Keep moving if you can (not vigorously)\\n - Talk to yourself, sing, count\\n\\nDANGER SIGNS:\\n- Feeling warm (paradoxical undressing) - you're NOT warm, this is late-stage hypothermia\\n- Want to lie down and sleep - fight this urge\\n- Removing clothes because you feel hot - put them back on!\\n\\nPrevent getting here: Stay DRY at all costs.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"I'm stranded on a beach after a shipwreck. What should I do first?\"},\n", + " {\"role\": \"assistant\", \"content\": \"First, move above the high tide line to avoid being swept away. Collect any floating debris or supplies immediately before they drift away. Find freshwater - never drink seawater as it will dehydrate you faster. Look for streams flowing into the ocean or collect rainwater. Build a shelter away from the water to protect from sun and wind. Create signal fires on the beach using driftwood. Three fires in a line is an international distress signal.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"I'm lost in the mountains in winter. How do I avoid freezing?\"},\n", + " {\"role\": \"assistant\", \"content\": \"Build a snow shelter immediately. Dig a snow cave into a drift or build a quinzhee by piling snow and hollowing it out. Snow provides excellent insulation. Make the entrance lower than the sleeping platform so cold air sinks out. Stay dry at all costs - wet clothing loses heat 25 times faster. Insulate yourself from the ground using pine branches, your backpack, or extra clothing. Keep your head covered as you lose 40% of body heat through your head. Avoid sweating by removing layers when active.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"I'm stuck in a swamp. How do I get out safely?\"},\n", + " {\"role\": \"assistant\", \"content\": \"Test ground before each step using a long stick. Firm ground will resist the stick while mud gives way. If you start sinking in quicksand or mud, fall backward to distribute your weight and slowly crawl out. Never struggle violently. Look for elevated areas with trees or dry vegetation. Avoid stagnant water as it contains dangerous bacteria. Watch for alligators and venomous snakes near water edges. Move slowly and deliberately to conserve energy in the humid heat.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"I'm trapped in a collapsed building after an earthquake. How do I survive?\"},\n", + " {\"role\": \"assistant\", \"content\": \"Don't shout constantly - you'll exhaust yourself and breathe in dust. Tap on pipes or walls rhythmically to signal rescuers. Cover your mouth with clothing to filter dust and debris. If you have a phone, text instead of calling to conserve battery. Don't light matches or lighters as gas leaks are common. Stay still to conserve air in your pocket. Don't drink your own urine. Ration any water you find. Listen for rescue teams and respond when you hear them nearby. Move debris carefully to avoid causing further collapse.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"How can I collect and filter water if I have no equipment?\"},\n", + " {\"role\": \"assistant\", \"content\": \"Collect morning dew by tying cloth around your ankles and walking through grass at dawn, then wring it into a container. For filtration, create a tripod filter using three layers. Fill the bottom with grass to catch large debris, middle layer with sand for smaller particles, and top layer with crushed charcoal from your fire for chemical filtration. Pour water through slowly. This removes debris but not all pathogens, so still boil if possible. Collect rainwater directly using large leaves funneled into containers.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"How do I make basic survival tools from natural materials?\"},\n", + " {\"role\": \"assistant\", \"content\": \"For a cutting tool, find a hard rock like flint or obsidian. Strike it at an angle with another rock to flake off sharp edges. For a spear, choose a straight hardwood branch about 6 feet long. Sharpen one end by scraping against rough rock or hardening in fire. For a knife, split a small piece of sharp rock and wrap the base with cloth, bark, or cordage for a handle. Create cordage by twisting plant fibers, inner tree bark, or long grass together. Braid three strands for stronger rope.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"How do I catch fish without a fishing rod?\"},\n", + " {\"role\": \"assistant\", \"content\": \"Build a fish trap using sticks. Create a funnel shape with a narrow opening that fish can enter but can't easily exit. Place it in shallow water with the opening facing upstream. Weight it down with rocks. For a spear, sharpen a long straight stick and split the tip into four prongs. Spread them apart with a small wedge and sharpen each point. Stand still in shallow water and strike quickly when fish approach. You can also make a net using woven plant fibers or catch fish by hand in rocky pools by cornering them slowly.\"}\n", + " ]\n", + " },\n", + " {\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"How can I make containers to carry water and supplies?\"},\n", + " {\"role\": \"assistant\", \"content\": \"For a water container, find large sections of bamboo with natural joints that seal the bottom. Cut just above a joint for the base. For a basket, weave flexible young branches or vines in an over-under pattern around a circular base. Make birch bark containers by carefully peeling large sheets of bark, folding into a box shape, and securing edges with wooden pegs or plant fiber stitching. Use pine pitch as waterproof sealant by heating it and applying to seams. Large leaves from plants like taro can hold water temporarily. Create a backpack using a shirt - tie the bottom closed, put supplies inside, and wear it backwards with arms through sleeves.\"}\n", + " ]\n", + " }\n", + "\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "1fc30c23", + "metadata": {}, + "source": [ + "# Format Data For Training" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "78225d26", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training dataset: 16 examples\n", + "\n", + "Example training format:\n", + "Question: I'm lost in the woods and running out of water. What do I do?\n", + "Answer: Water is your top priority - you can only survive 3 days without it. Here's what to do:\n", + "\n", + "1. FIND water sources: Look for streams, rivers, or collect morning dew with a cloth\n", + "2. PURIFY before drinking: Boil for 1 minute (3 minutes at high altitude), or use purification tablets if you have them\n", + "3. If no fire: Solar water disinfection - fill a clear bottle and leave in direct sunlight for 6 hours\n", + "4. Collect rainwater: U\n", + "\n", + "================================================================================\n", + "\n" + ] + } + ], + "source": [ + "def format_ernie_data(data):\n", + " formatted = []\n", + " for item in data:\n", + " text = \"\"\n", + " for msg in item[\"messages\"]:\n", + " if msg[\"role\"] == \"user\":\n", + " text += f\"Question: {msg['content']}\\n\"\n", + " elif msg[\"role\"] == \"assistant\":\n", + " text += f\"Answer: {msg['content']}\\n\\n\"\n", + " formatted.append({\"text\": text})\n", + " return formatted\n", + "\n", + "formatted_data = format_ernie_data(survival_data)\n", + "train_dataset = Dataset.from_list(formatted_data)\n", + "\n", + "print(f\"Training dataset: {len(train_dataset)} examples\")\n", + "print(\"\\nExample training format:\")\n", + "print(formatted_data[0][\"text\"][:500])\n", + "print(\"\\n\" + \"=\"*80 + \"\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "fde1ab6d", + "metadata": {}, + "source": [ + "# Load Base Model and Train" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "01ebde17", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading nghuyong/ernie-2.0-base-en for causal LM...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of ErnieForCausalLM were not initialized from the model checkpoint at nghuyong/ernie-2.0-base-en and are newly initialized: ['cls.predictions.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model loaded successfully for causal generation!\n" + ] + } + ], + "source": [ + "from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig\n", + "\n", + "model_name = \"nghuyong/ernie-2.0-base-en\"\n", + "USE_GPU = False\n", + "\n", + "print(f\"Loading {model_name} for causal LM...\")\n", + "\n", + "# Load config and set is_decoder=True\n", + "config = AutoConfig.from_pretrained(model_name)\n", + "config.is_decoder = True\n", + "config.add_cross_attention = False # We don't need cross-attention\n", + "\n", + "# Load tokenizer\n", + "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + "\n", + "# Load model with the modified config\n", + "model = AutoModelForCausalLM.from_pretrained(\n", + " model_name,\n", + " config=config,\n", + " ignore_mismatched_sizes=True # This handles the newly initialized weights\n", + ")\n", + "\n", + "if USE_GPU:\n", + " model.to(\"cuda\")\n", + "\n", + "# Set pad token\n", + "if not tokenizer.pad_token:\n", + " tokenizer.pad_token = tokenizer.eos_token\n", + " model.config.pad_token_id = tokenizer.pad_token_id\n", + "\n", + "tokenizer.padding_side = 'left'\n", + "\n", + "print(\"Model loaded successfully for causal generation!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9cfc63f", + "metadata": {}, + "outputs": [], + "source": [ + "# training configuration for ERNIE\n", + "sft_config = SFTConfig(\n", + " learning_rate=2e-5, # Lower learning rate\n", + " num_train_epochs=50, # More epochs since small dataset\n", + " per_device_train_batch_size=1,\n", + " gradient_accumulation_steps=8,\n", + " gradient_checkpointing=False,\n", + " logging_steps=1,\n", + " output_dir=\"./ernie_survival_model\",\n", + " max_seq_length=512,\n", + " warmup_steps=10, \n", + " weight_decay=0.01, \n", + " save_strategy=\"epoch\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "5ea1d96a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Map: 100%|██████████| 16/16 [00:00<00:00, 233.37 examples/s]\n" + ] + } + ], + "source": [ + "sft_trainer = SFTTrainer(\n", + " model=model,\n", + " args=sft_config,\n", + " train_dataset=train_dataset,\n", + " processing_class=tokenizer,\n", + " \n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a7659687", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [10/10 00:58, Epoch 5/5]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
110.344300
210.315400
310.142800
49.986600
59.601800
69.138600
78.802400
88.647800
98.323000
108.232400

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "TrainOutput(global_step=10, training_loss=9.353510570526122, metrics={'train_runtime': 63.4321, 'train_samples_per_second': 1.261, 'train_steps_per_second': 0.158, 'total_flos': 8433349519500.0, 'train_loss': 9.353510570526122, 'epoch': 5.0})" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sft_trainer.train()" + ] + }, + { + "cell_type": "markdown", + "id": "1f302860", + "metadata": {}, + "source": [ + "# Result" + ] + }, + { + "cell_type": "markdown", + "id": "0fd98148", + "metadata": {}, + "source": [ + "Note that the result will be gibberish since some weights of ErnieForCausalLM were not initialized, so the datasets is small or when the number of training epochs is low, the result will be gibberish, but the code is still working since the training loss is steadily decreasing." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "cc2d5896-6fd6-43d2-85f1-dacbd594f4cf", + "metadata": { + "height": 30 + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "=== Survival Model (After SFT) Output ===\n", + "\n", + "Model Input 1:\n", + "I'm in a forest and I'm hungry, what should i do\n", + "Model Output 1:\n", + "##mara 1 et se double fire seee - - but ice point2 ice on under on fire co for see face more on - througha 2s go at forke i note more facenee more baree crosss 4s face cuessc against spot double against double spot post both to fire be -na 2 2 go / open moree cross cross to fire gas seee face cross area - on ice cover gas watch spotk butek butn - basedaekk cross double point near on lead spot 5 just watchen be both to really but so both have for in fore on sense on fire north firek watchn coveryn 3 both for 5x on even fire 7ce ink\n", + "\n", + "\n", + "Model Input 2:\n", + "How do I start a fire without matches?\n", + "Model Output 2:\n", + "stunning notty for introducedn 18 ice - forces openty ice crossscch moresc mean about for ben combined mo on fire 11chme doubleers check on ground i under facea about fire 16sc both -hen side 5 2 fire note 4 handa on light also nature sub - woodka - point be in - with fork ground mo open just on 3 4 for with 2 against iceto under doublek for mo fire with goe cu cut 10 mo 3a - with for live 4 be more moren firen forcee mo lighten ice even for fires got through mean bek ground true on spot 4 seal be both againstxe lowerer 11m 5s but about\n", + "\n" + ] + } + ], + "source": [ + "# Test the trained model\n", + "survival_questions = [\n", + " \"I'm in a forest and I'm hungry, what should i do\",\n", + " \"How do I start a fire without matches?\",\n", + "]\n", + "\n", + "test_model_with_questions(\n", + " sft_trainer.model, \n", + " tokenizer, \n", + " survival_questions,\n", + " title=\"Survival Model (After SFT) Output\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a3c0bba-984a-494c-8374-33db30ad1da6", + "metadata": { + "height": 30 + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5521a00a-88d8-4ad9-b134-12ea94e76984", + "metadata": { + "height": 30 + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ecddf372-7382-41ed-bc3d-ead2eef33201", + "metadata": { + "height": 30 + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9841d1c8-c4d5-43e9-9d1e-1f267057279a", + "metadata": { + "height": 30 + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b64a77d2-3338-4a8f-b634-d25be5b19407", + "metadata": { + "height": 30 + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 002b3c9e460d07d74dc12830b128e8ab70dea96d Mon Sep 17 00:00:00 2001 From: Zhang Jun Date: Wed, 7 Jan 2026 13:24:07 +0800 Subject: [PATCH 2/2] minor Signed-off-by: Zhang Jun --- .../source/wilderness survivial.ipynb | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename wilderness survivial.ipynb => docs/source/wilderness survivial.ipynb (100%) diff --git a/wilderness survivial.ipynb b/docs/source/wilderness survivial.ipynb similarity index 100% rename from wilderness survivial.ipynb rename to docs/source/wilderness survivial.ipynb