cat
d84f98799b
## Added ### Core framework - src/llm_client.py — Ollama API wrapper for local Docker inference - src/samples.py — Embedded benchmark datasets (GSM8K/MATH/AIME + Chinese) - src/run_all.py — Unified experiment runner ### Phase 1: Baselines - src/baseline/benchmark.py — IO, CoT, CoT-SC, ToT evaluation with Chinese support ### Phase 2: AGoT - src/agot/agot.py — AGoT core algorithm (6 agent types, recursive decomposition) - src/agot/graph_utils.py — DAG graph data structure with cycle detection - src/agot/prompts.py — 6 agent prompt templates (EN + ZH) - src/agot/run.py — AGoT experiment runner ### Graph Analysis - src/graph_analysis/reasoning_graph.py — Graph property computation (cyclicity, diameter, small-world) - src/graph_analysis/visualize.py — Visualization charts ### Documentation - docs/graph_reasoning_principles.md — Comprehensive principles document - docs/next_steps.md — Roadmap for reliable reasoning tool ### Experiment Results - GSM8K/MATH baselines on qwen2.5:32b and qwq:latest - AGoT validation (100% on small sample) - Chinese dataset benchmarks (C-GSM8K, CMATH) - Graph property analysis confirming Topology of Reasoning findings
40 lines
993 B
JSON
40 lines
993 B
JSON
{
|
||
"summary": {
|
||
"method": "IO",
|
||
"model": "qwen2.5:32b",
|
||
"task": "cmath",
|
||
"samples": 3,
|
||
"correct": 1,
|
||
"accuracy": 0.3333,
|
||
"avg_time_s": 0.51
|
||
},
|
||
"details": [
|
||
{
|
||
"question": "已知函数 $f(x) = 2x^2 - 4x + 1$，求 $f(x)$ 的最小值。",
|
||
"expected": "-1",
|
||
"predicted": "-1",
|
||
"full_response": "-1",
|
||
"correct": true,
|
||
"time_s": 0.5,
|
||
"tokens": 1
|
||
},
|
||
{
|
||
"question": "在等差数列 $\\{a_n\\}$ 中，$a_1 = 3$，$a_3 + a_5 = 22$，求 $a_7$ 的值。",
|
||
"expected": "25",
|
||
"predicted": "15",
|
||
"full_response": "15",
|
||
"correct": false,
|
||
"time_s": 0.47,
|
||
"tokens": 1
|
||
},
|
||
{
|
||
"question": "从 3 名男生和 2 名女生中选出 2 人参加志愿者活动，求选出的两人恰好是一男一女的概率。",
|
||
"expected": "3/5",
|
||
"predicted": "0.6",
|
||
"full_response": "0.6",
|
||
"correct": false,
|
||
"time_s": 0.56,
|
||
"tokens": 2
|
||
}
|
||
]
|
||
} |