cat
d84f98799b
## Added ### Core framework - src/llm_client.py — Ollama API wrapper for local Docker inference - src/samples.py — Embedded benchmark datasets (GSM8K/MATH/AIME + Chinese) - src/run_all.py — Unified experiment runner ### Phase 1: Baselines - src/baseline/benchmark.py — IO, CoT, CoT-SC, ToT evaluation with Chinese support ### Phase 2: AGoT - src/agot/agot.py — AGoT core algorithm (6 agent types, recursive decomposition) - src/agot/graph_utils.py — DAG graph data structure with cycle detection - src/agot/prompts.py — 6 agent prompt templates (EN + ZH) - src/agot/run.py — AGoT experiment runner ### Graph Analysis - src/graph_analysis/reasoning_graph.py — Graph property computation (cyclicity, diameter, small-world) - src/graph_analysis/visualize.py — Visualization charts ### Documentation - docs/graph_reasoning_principles.md — Comprehensive principles document - docs/next_steps.md — Roadmap for reliable reasoning tool ### Experiment Results - GSM8K/MATH baselines on qwen2.5:32b and qwq:latest - AGoT validation (100% on small sample) - Chinese dataset benchmarks (C-GSM8K, CMATH) - Graph property analysis confirming Topology of Reasoning findings
58 lines
1.8 KiB
JSON
58 lines
1.8 KiB
JSON
{
|
|
"summary": {
|
|
"method": "IO",
|
|
"model": "qwen2.5:32b",
|
|
"task": "c_gsm8k",
|
|
"samples": 5,
|
|
"correct": 3,
|
|
"accuracy": 0.6,
|
|
"avg_time_s": 3.23
|
|
},
|
|
"details": [
|
|
{
|
|
"question": "小明有 12 个苹果，他给了小红 5 个，又给了小刚 3 个。请问小明还剩下几个苹果？",
|
|
"expected": "4",
|
|
"predicted": "4",
|
|
"full_response": "4个苹果",
|
|
"correct": true,
|
|
"time_s": 11.31,
|
|
"tokens": 2
|
|
},
|
|
{
|
|
"question": "一个长方形的长是 8 厘米，宽是 5 厘米。如果长增加 2 厘米，宽不变，新的长方形面积比原来的大多少平方厘米？",
|
|
"expected": "10",
|
|
"predicted": "10",
|
|
"full_response": "10 平方厘米",
|
|
"correct": true,
|
|
"time_s": 0.95,
|
|
"tokens": 3
|
|
},
|
|
{
|
|
"question": "超市里一瓶牛奶卖 6 元，一袋面包卖 4 元。小华买了 3 瓶牛奶和 2 袋面包，付了 50 元，请问应找回多少元？",
|
|
"expected": "24",
|
|
"predicted": "28",
|
|
"full_response": "28元",
|
|
"correct": false,
|
|
"time_s": 0.57,
|
|
"tokens": 2
|
|
},
|
|
{
|
|
"question": "一本书有 200 页，小明第一天看了全书的 1/4，第二天看了余下的 1/3。请问小明两天一共看了多少页？",
|
|
"expected": "100",
|
|
"predicted": "100",
|
|
"full_response": "100页",
|
|
"correct": true,
|
|
"time_s": 0.67,
|
|
"tokens": 2
|
|
},
|
|
{
|
|
"question": "一个水池有甲、乙两个进水管。单开甲管 6 小时可以注满水池，单开乙管 8 小时可以注满水池。如果两管同时开放，多少小时可以注满水池？",
|
|
"expected": "24/7",
|
|
"predicted": "3.43",
|
|
"full_response": "3.42857142857小时，约等于3.43小时。",
|
|
"correct": false,
|
|
"time_s": 2.63,
|
|
"tokens": 9
|
|
}
|
|
]
|
|
} |