cat
d84f98799b
## Added ### Core framework - src/llm_client.py — Ollama API wrapper for local Docker inference - src/samples.py — Embedded benchmark datasets (GSM8K/MATH/AIME + Chinese) - src/run_all.py — Unified experiment runner ### Phase 1: Baselines - src/baseline/benchmark.py — IO, CoT, CoT-SC, ToT evaluation with Chinese support ### Phase 2: AGoT - src/agot/agot.py — AGoT core algorithm (6 agent types, recursive decomposition) - src/agot/graph_utils.py — DAG graph data structure with cycle detection - src/agot/prompts.py — 6 agent prompt templates (EN + ZH) - src/agot/run.py — AGoT experiment runner ### Graph Analysis - src/graph_analysis/reasoning_graph.py — Graph property computation (cyclicity, diameter, small-world) - src/graph_analysis/visualize.py — Visualization charts ### Documentation - docs/graph_reasoning_principles.md — Comprehensive principles document - docs/next_steps.md — Roadmap for reliable reasoning tool ### Experiment Results - GSM8K/MATH baselines on qwen2.5:32b and qwq:latest - AGoT validation (100% on small sample) - Chinese dataset benchmarks (C-GSM8K, CMATH) - Graph property analysis confirming Topology of Reasoning findings
103 lines
2.7 KiB
JSON
103 lines
2.7 KiB
JSON
{
|
|
"summary": {
|
|
"method": "IO",
|
|
"model": "qwen2.5:32b",
|
|
"task": "math",
|
|
"samples": 10,
|
|
"correct": 8,
|
|
"accuracy": 0.8,
|
|
"avg_time_s": 0.89
|
|
},
|
|
"details": [
|
|
{
|
|
"question": "Find the value of $x$ such that $\\sqrt{4x + 5} = 7$.",
|
|
"expected": "11",
|
|
"predicted": "12",
|
|
"full_response": "12",
|
|
"correct": false,
|
|
"time_s": 0.49,
|
|
"tokens": 1
|
|
},
|
|
{
|
|
"question": "A box contains 5 red balls and 7 blue balls. If two balls are drawn at random without replacement, what is the probability that both balls are red? Express your answer as a fraction in simplest form.",
|
|
"expected": "5/33",
|
|
"predicted": "5/33",
|
|
"full_response": "5/33",
|
|
"correct": true,
|
|
"time_s": 0.65,
|
|
"tokens": 2
|
|
},
|
|
{
|
|
"question": "What is the sum of the first 20 positive integers?",
|
|
"expected": "210",
|
|
"predicted": "210",
|
|
"full_response": "210",
|
|
"correct": true,
|
|
"time_s": 0.85,
|
|
"tokens": 2
|
|
},
|
|
{
|
|
"question": "If $f(x) = 2x^2 + 3x - 5$, find $f(-2)$.",
|
|
"expected": "-3",
|
|
"predicted": "-7",
|
|
"full_response": "-7",
|
|
"correct": false,
|
|
"time_s": 1.0,
|
|
"tokens": 1
|
|
},
|
|
{
|
|
"question": "The sum of two numbers is 20 and their difference is 4. What is the product of the two numbers?",
|
|
"expected": "96",
|
|
"predicted": "96",
|
|
"full_response": "96",
|
|
"correct": true,
|
|
"time_s": 0.95,
|
|
"tokens": 1
|
|
},
|
|
{
|
|
"question": "How many distinguishable permutations can be made from the letters in the word 'MISSISSIPPI'?",
|
|
"expected": "34650",
|
|
"predicted": "34650",
|
|
"full_response": "34650",
|
|
"correct": true,
|
|
"time_s": 1.17,
|
|
"tokens": 2
|
|
},
|
|
{
|
|
"question": "Find the area of a circle with circumference $8\\pi$.",
|
|
"expected": "16\\pi",
|
|
"predicted": "16",
|
|
"full_response": "$16\\pi$",
|
|
"correct": true,
|
|
"time_s": 1.11,
|
|
"tokens": 3
|
|
},
|
|
{
|
|
"question": "If $\\log_2(x) + \\log_2(x-2) = 3$, find $x$.",
|
|
"expected": "4",
|
|
"predicted": "4",
|
|
"full_response": "4",
|
|
"correct": true,
|
|
"time_s": 0.82,
|
|
"tokens": 1
|
|
},
|
|
{
|
|
"question": "A geometric sequence has first term 3 and common ratio 2. What is the 8th term?",
|
|
"expected": "384",
|
|
"predicted": "384",
|
|
"full_response": "384",
|
|
"correct": true,
|
|
"time_s": 1.01,
|
|
"tokens": 2
|
|
},
|
|
{
|
|
"question": "How many ways can 5 people be arranged in a line?",
|
|
"expected": "120",
|
|
"predicted": "120",
|
|
"full_response": "120",
|
|
"correct": true,
|
|
"time_s": 0.85,
|
|
"tokens": 2
|
|
}
|
|
]
|
|
} |