cat
d84f98799b
## Added ### Core framework - src/llm_client.py — Ollama API wrapper for local Docker inference - src/samples.py — Embedded benchmark datasets (GSM8K/MATH/AIME + Chinese) - src/run_all.py — Unified experiment runner ### Phase 1: Baselines - src/baseline/benchmark.py — IO, CoT, CoT-SC, ToT evaluation with Chinese support ### Phase 2: AGoT - src/agot/agot.py — AGoT core algorithm (6 agent types, recursive decomposition) - src/agot/graph_utils.py — DAG graph data structure with cycle detection - src/agot/prompts.py — 6 agent prompt templates (EN + ZH) - src/agot/run.py — AGoT experiment runner ### Graph Analysis - src/graph_analysis/reasoning_graph.py — Graph property computation (cyclicity, diameter, small-world) - src/graph_analysis/visualize.py — Visualization charts ### Documentation - docs/graph_reasoning_principles.md — Comprehensive principles document - docs/next_steps.md — Roadmap for reliable reasoning tool ### Experiment Results - GSM8K/MATH baselines on qwen2.5:32b and qwq:latest - AGoT validation (100% on small sample) - Chinese dataset benchmarks (C-GSM8K, CMATH) - Graph property analysis confirming Topology of Reasoning findings
40 lines
1.4 KiB
JSON
40 lines
1.4 KiB
JSON
{
|
|
"summary": {
|
|
"method": "IO",
|
|
"model": "qwen2.5:32b",
|
|
"task": "gsm8k",
|
|
"samples": 3,
|
|
"correct": 2,
|
|
"accuracy": 0.6667,
|
|
"avg_time_s": 0.51
|
|
},
|
|
"details": [
|
|
{
|
|
"question": "Janet's ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins with four every day. She sells the remaining eggs at the farmers' market daily for $2 per dozen. How much money does she make per day at the farmers' market?",
|
|
"expected": "1.50",
|
|
"predicted": "2",
|
|
"full_response": "$2",
|
|
"correct": false,
|
|
"time_s": 0.5,
|
|
"tokens": 1
|
|
},
|
|
{
|
|
"question": "A robe takes 2 1/2 yards of fabric and 1/8 yard of trim. If you have 10 yards of fabric and 1 yard of trim, how many robes can you make?",
|
|
"expected": "4",
|
|
"predicted": "4",
|
|
"full_response": "4",
|
|
"correct": true,
|
|
"time_s": 0.36,
|
|
"tokens": 1
|
|
},
|
|
{
|
|
"question": "Kyle is a collector of rare coins. One day he found a rare coin online and bought it for $50. The next month, he sold it at a 20% profit. Six months later, he bought the same coin back, but had to pay 10% more than he sold it for. How much did Kyle pay to buy the coin back?",
|
|
"expected": "66",
|
|
"predicted": "66",
|
|
"full_response": "66 dollars",
|
|
"correct": true,
|
|
"time_s": 0.66,
|
|
"tokens": 4
|
|
}
|
|
]
|
|
} |