{
  "summary": {
    "method": "IO",
    "model": "qwen2.5:32b",
    "task": "gsm8k",
    "samples": 3,
    "correct": 2,
    "accuracy": 0.6667,
    "avg_time_s": 0.5
  },
  "details": [
    {
      "question": "Janet's ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins with four every day. She sells the remaining eggs at the farmers' market daily for $2 per dozen. How much money does she make per day at the farmers' market?",
      "expected": "1.50",
      "predicted": "2",
      "full_response": "$2",
      "correct": false,
      "time_s": 0.5,
      "tokens": 1
    },
    {
      "question": "A robe takes 2 1/2 yards of fabric and 1/8 yard of trim. If you have 10 yards of fabric and 1 yard of trim, how many robes can you make?",
      "expected": "4",
      "predicted": "4",
      "full_response": "4",
      "correct": true,
      "time_s": 0.37,
      "tokens": 1
    },
    {
      "question": "Kyle is a collector of rare coins. One day he found a rare coin online and bought it for $50. The next month, he sold it at a 20% profit. Six months later, he bought the same coin back, but had to pay 10% more than he sold it for. How much did Kyle pay to buy the coin back?",
      "expected": "66",
      "predicted": "66",
      "full_response": "66 dollars",
      "correct": true,
      "time_s": 0.63,
      "tokens": 4
    }
  ]
}