llama3-1-8b-sft-500-2-with-QA-only.yml

task: llm-sft
base_model: meta-llama/Meta-Llama-3.1-8B-Instruct
project_name: HPCBot-Llama-3-1-8B-Instruct-QA-only
log: wandb
backend: local

data:
  path: /home/btrungvo/workspace/HPCBot/data/train/QA_only
  train_split: train
  valid_split: null
  chat_template: null
  column_mapping:
    text_column: text

params:
  block_size: 1024
  model_max_length: 2048
  epochs: 300
  batch_size: 2
  lr: 1e-5
  peft: true
  quantization: null
  target_modules: all-linear
  padding: right
  optimizer: paged_adamw_8bit
  scheduler: cosine
  gradient_accumulation: 8
  mixed_precision: bf16