sdg.yaml

# Global configuration
device: cuda:0               # CUDA device for all GPU operations (vLLM, SentenceTransformer, MinerU)
n_workers: 1                 # Number of parallel workers to accelerate the process (default to 1 as sequential processing)
output_dir: ./outputs/gsm8k  # synthetic dataset output directory
export_format: jsonl         # export dataset format

# Global task configuration
task:
  name: gsm8k
  domain: mathematics        # Keywords of the domain of target dataset
  demo_examples_path: /path/to/your/demo_examples.jsonl  # path for demo examples .jsonl file (Optional)

  # Task instruction used for data generation, define the type, structure, and requirements of the dataset to be generated
  task_instruction: >-
    Generate a grade school math word problem that requires multi-step reasoning.
    The problem should involve basic arithmetic operations and have a clear numerical answer.
  # Input instruction - appended to prompt to guide the format of input field
  input_instruction: >-
    The input should be a math word problem that requires multi-step reasoning.
  # Output instruction - appended to prompt to guide the format of output field
  output_instruction: >-
    The output should contain reasoning process step by step.

  num_samples: 10      # Number of samples to generate
  batch_size: 5        # Batch processing size across pipeline

  # Text modality configuration
  # Output format: {"input": "question", "output": "answer"}
  text:
    # Local task: generate from local documents
    local:
      # Parsing Configuration
      parsing:
        document_dir: /path/to/your/document_dir   # Path to documents directory
        method: mineru                             # default to mineru parser

      # Retrieval Configuration
      retrieval:
        passages_dir: ./dataset/passages   # path to document passages after parsing and chunking
        method: bm25                       # default to bm25 retriever
        top_k: 1000                        # selected top k passage retrieved

      # Generation Configuration
      generation:
        temperature: 1.0       # Temperature for LLM in initial data generation

    # Web task: crawl from HuggingFace
    # web:
    #   huggingface_token: "hf_"        # Optional. This parameter can be set by 'export HUGGINGFACE_TOKEN="YOUR TOKEN"'
    #   dataset_limit: 5                # Number of datasets to crawl per keyword (default: 5)

    # Distill task: pure instruction-based generation
    # distill:
    #   temperature: 1.0         # Temperature for LLM distillation

  # Image modality configuration (comment out 'text' above to use this)
  # Output format: {"input": "question", "output": "answer", "image": "path/to/image.png"}
  # image:
  #   # Local task: generate QA data from local images
  #   local:
  #     # Image sources (at least one required, can combine both)
  #     image_dir: /path/to/your/images        # Directory containing images (jpg, png, etc.)
  #
  #     # PDF parsing for image extraction (optional, uses MinerU)
  #     parsing:
  #       document_dir: /path/to/your/pdfs     # Directory containing PDF documents
  #       method: mineru                       # Parser method (default: mineru)
  #
  #     # Generation Configuration
  #     generation:
  #       temperature: 1.0       # Temperature for VLM in data generation
  #
  #   # Web task: crawl VQA datasets from HuggingFace
  #   web:
  #     huggingface_token: "hf_"        # Optional. Can be set via 'export HUGGINGFACE_TOKEN="YOUR TOKEN"'
  #     dataset_limit: 5                # Number of datasets to crawl per keyword (default: 5)

# Base model for evaluation and inference (vLLM for text, VLM for image modality)
# For image modality, use a VLM model like Qwen/Qwen2-VL-7B-Instruct
base_model:
  path: Qwen/Qwen2.5-7B   # Image: Qwen/Qwen3-VL-7B-Instruct

# LLM/VLM used throughout the pipeline (Create .env file to specify base url and api key)
# For image modality, use a VLM like gpt-4o or claude-3-5-sonnet
llm:
  provider: openai
  model: gpt-4o-mini      # Image: gpt-4o (VLM required)

# Answer extraction configuration (HOW to mark final answer of the question, refer to GSM8K dataset)
answer_extraction:
  tag: "####"
  instruction: "Output your final answer after ####"

# Postprocess for LLMs' responses
postprocess:
  methods: 
  - majority_voting    # default to majority voting
      
  # Majority voting for quality control of LLM's responses
  # Uncomment ONE method below: exact_match / semantic_clustering / llm_judge
  majority_voting:
    n: 8    # Number of voting

    # Exact match settings (mostly used for numerical answer)
    exact_match:
      numeric_tolerance: 1e-3        # Set to 0 or null for strict string matching

    # Semantic similarity settings (better tolerance of answer output)
    # semantic_clustering:
    #   model_path: BAAI/bge-m3      # Semantic model path
    #   similarity_threshold: 0.85   # Similarity threshold to determine majority answer

    # LLM judge settings (best choice but cost)
    # llm_judge:
    #   temperature: 0.3

# Evaluation Configuration
evaluation:
  # Inference parameters for initial binary evaluation (solved vs unsolved)
  inference:
    temperature: 0.0
    max_tokens: 1500
    n: 1

  # Sampling parameters for pass@n scoring
  scoring:
    temperature: 1.2
    n: 8

  # Answer comparison configuration (Used for both evaluation scoring and majority voting)
  # Uncomment ONE method below: exact_match / semantic / llm_judge
  answer_comparison:
    # Exact match settings (mostly used for numerical answer)
    # exact_match:
    #   numeric_tolerance: 1e-3     # Set to 0 or null for strict string matching

    # Semantic similarity settings (better tolerance of base model output answer)
    semantic:
      model_path: BAAI/bge-m3       # Model path for semantic answer comparing
      similarity_threshold: 0.85    # Semantic similarity threshold to determine same answer

    # LLM judge settings (best choice but cost)
    # llm_judge:
    #   temperature: 0.3

# Rewrite Configuration
rewrite:
  difficulty_adjust:
    easier_temperature: 0.9
    harder_temperature: 1.1

# Translation Configuration
translation:
  language: english  # Target language: 'english' (no translation), 'arabic', etc.
  # model_path: hammh0a/Hala-1.2B-EN-AR-Translator    # If language is not 'english', you must specify model_path for the translation model
  max_tokens: 256    # Maximum tokens for translation generation