mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-11-07 06:19:43 -10:00
docs(readme): simplify evaluation overview and refine trend chart (#14093)
Co-authored-by: Qwen-Coder <[email protected]>
This commit is contained in:
pomeloandQwen-Coder authored and GitHub committed
1 parent
b0cee09604
commit
a9cd8aceea
6 files changed
+208
-60
No files matched your search
@@ -1,37 +1,56 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="430" viewBox="0 0 1120 430">
|
||||
<rect width="100%" height="100%" fill="#FFFFFF"/>
|
||||
<g font-family="Arial, sans-serif">
|
||||
<line x1="70" y1="368" x2="1082" y2="368" stroke="#D8E0EA" stroke-width="1"/>
|
||||
<text x="56" y="373" text-anchor="end" font-size="14" fill="#52606D">76.0%</text>
|
||||
<line x1="70" y1="285.5" x2="1082" y2="285.5" stroke="#D8E0EA" stroke-width="1"/>
|
||||
<text x="56" y="290.5" text-anchor="end" font-size="14" fill="#52606D">76.5%</text>
|
||||
<line x1="70" y1="203" x2="1082" y2="203" stroke="#D8E0EA" stroke-width="1"/>
|
||||
<text x="56" y="208" text-anchor="end" font-size="14" fill="#52606D">77.0%</text>
|
||||
<line x1="70" y1="120.5" x2="1082" y2="120.5" stroke="#D8E0EA" stroke-width="1"/>
|
||||
<text x="56" y="125.5" text-anchor="end" font-size="14" fill="#52606D">77.5%</text>
|
||||
<line x1="70" y1="38" x2="1082" y2="38" stroke="#D8E0EA" stroke-width="1"/>
|
||||
<text x="56" y="43" text-anchor="end" font-size="14" fill="#52606D">78.0%</text>
|
||||
<path d="M 70 71 L 238.7 247.6 L 407.3 269 L 576 290.5 L 744.7 92.5 L 913.3 302 L 1082 148.6" fill="none" stroke="#1F6FEB" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<circle cx="70" cy="71" r="6" fill="#1F6FEB" stroke="#FFFFFF" stroke-width="3"/>
|
||||
<text x="70" y="55" text-anchor="middle" font-size="14" font-weight="700" fill="#16324F">77.80%</text>
|
||||
<text x="70" y="407" text-anchor="middle" font-size="14" fill="#344054">0.0.14</text>
|
||||
<circle cx="238.7" cy="247.6" r="6" fill="#1F6FEB" stroke="#FFFFFF" stroke-width="3"/>
|
||||
<text x="238.7" y="231.6" text-anchor="middle" font-size="14" font-weight="700" fill="#16324F">76.73%</text>
|
||||
<text x="238.7" y="407" text-anchor="middle" font-size="14" fill="#344054">0.1.0</text>
|
||||
<circle cx="407.3" cy="269" r="6" fill="#1F6FEB" stroke="#FFFFFF" stroke-width="3"/>
|
||||
<text x="407.3" y="253" text-anchor="middle" font-size="14" font-weight="700" fill="#16324F">76.60%</text>
|
||||
<text x="407.3" y="407" text-anchor="middle" font-size="14" fill="#344054">0.5.0</text>
|
||||
<circle cx="576" cy="290.5" r="6" fill="#1F6FEB" stroke="#FFFFFF" stroke-width="3"/>
|
||||
<text x="576" y="274.5" text-anchor="middle" font-size="14" font-weight="700" fill="#16324F">76.47%</text>
|
||||
<text x="576" y="407" text-anchor="middle" font-size="14" fill="#344054">0.10.0</text>
|
||||
<circle cx="744.7" cy="92.5" r="6" fill="#1F6FEB" stroke="#FFFFFF" stroke-width="3"/>
|
||||
<text x="744.7" y="76.5" text-anchor="middle" font-size="14" font-weight="700" fill="#16324F">77.67%</text>
|
||||
<text x="744.7" y="407" text-anchor="middle" font-size="14" fill="#344054">0.15.0</text>
|
||||
<circle cx="913.3" cy="302" r="6" fill="#1F6FEB" stroke="#FFFFFF" stroke-width="3"/>
|
||||
<text x="913.3" y="286" text-anchor="middle" font-size="14" font-weight="700" fill="#16324F">76.40%</text>
|
||||
<text x="913.3" y="407" text-anchor="middle" font-size="14" fill="#344054">0.19.0</text>
|
||||
<circle cx="1082" cy="148.6" r="6" fill="#1F6FEB" stroke="#FFFFFF" stroke-width="3"/>
|
||||
<text x="1082" y="132.6" text-anchor="middle" font-size="14" font-weight="700" fill="#16324F">77.33%</text>
|
||||
<text x="1082" y="407" text-anchor="middle" font-size="14" fill="#344054">0.22.0</text>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="840" height="220" viewBox="0 0 840 220" role="img" aria-labelledby="chart-title chart-description">
|
||||
<title id="chart-title">Average SWE-bench Verified score by Qwen Code version</title>
|
||||
<desc id="chart-description">Qwen 3.7 Max, 500 tasks, three trials per version. Average scores: 0.0.14, 77.80%; 0.1.0, 76.73%; 0.5.0, 76.60%; 0.10.0, 76.47%; 0.15.0, 77.67%; 0.19.0, 76.40%; 0.22.0, 77.33%. The line connects evaluated versions in release order. The vertical axis spans 70% to 80%.</desc>
|
||||
<style>
|
||||
.background { fill: #ffffff; }
|
||||
.heading, .score { fill: #1f2328; }
|
||||
.label { fill: #59636e; }
|
||||
.grid { stroke: #e5e7eb; }
|
||||
.axis { stroke: #9ca3af; }
|
||||
.trend { stroke: #6b4eff; }
|
||||
.point { fill: #6b4eff; stroke: #ffffff; }
|
||||
@media (prefers-color-scheme: dark) {
|
||||
.background { fill: #0d1117; }
|
||||
.heading, .score { fill: #f0f6fc; }
|
||||
.label { fill: #b1bac4; }
|
||||
.grid { stroke: #30363d; }
|
||||
.axis { stroke: #6e7681; }
|
||||
.trend { stroke: #a78bfa; }
|
||||
.point { fill: #a78bfa; stroke: #0d1117; }
|
||||
}
|
||||
</style>
|
||||
<rect class="background" width="840" height="220"/>
|
||||
<g font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif">
|
||||
<text class="heading" x="24" y="28" font-size="20" font-weight="600">Average score by version</text>
|
||||
<text class="label" x="24" y="51" font-size="14">SWE-bench Verified · Qwen 3.7 Max · y-axis: 70–80%</text>
|
||||
<line class="grid" x1="84" y1="96" x2="816" y2="96" stroke-width="1"/>
|
||||
<text class="label" x="70" y="101" text-anchor="end" font-size="14">80%</text>
|
||||
<line class="grid" x1="84" y1="142" x2="816" y2="142" stroke-width="1"/>
|
||||
<text class="label" x="70" y="147" text-anchor="end" font-size="14">75%</text>
|
||||
<line class="axis" x1="84" y1="188" x2="816" y2="188" stroke-width="1"/>
|
||||
<text class="label" x="70" y="193" text-anchor="end" font-size="14">70%</text>
|
||||
<line class="axis" x1="84" y1="96" x2="84" y2="188" stroke-width="1"/>
|
||||
<path class="trend" d="M 140 116.2400 L 244 126.0840 L 348 127.2800 L 452 128.4760 L 556 117.4360 L 660 129.1200 L 764 120.5640" fill="none" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<circle class="point" cx="140" cy="116.2400" r="4.5" stroke-width="2"/>
|
||||
<text class="score" x="140" y="82" text-anchor="middle" font-size="18" font-weight="600">77.80%</text>
|
||||
<text class="label" x="140" y="214" text-anchor="middle" font-size="16">0.0.14</text>
|
||||
<circle class="point" cx="244" cy="126.0840" r="4.5" stroke-width="2"/>
|
||||
<text class="score" x="244" y="82" text-anchor="middle" font-size="18" font-weight="600">76.73%</text>
|
||||
<text class="label" x="244" y="214" text-anchor="middle" font-size="16">0.1.0</text>
|
||||
<circle class="point" cx="348" cy="127.2800" r="4.5" stroke-width="2"/>
|
||||
<text class="score" x="348" y="82" text-anchor="middle" font-size="18" font-weight="600">76.60%</text>
|
||||
<text class="label" x="348" y="214" text-anchor="middle" font-size="16">0.5.0</text>
|
||||
<circle class="point" cx="452" cy="128.4760" r="4.5" stroke-width="2"/>
|
||||
<text class="score" x="452" y="82" text-anchor="middle" font-size="18" font-weight="600">76.47%</text>
|
||||
<text class="label" x="452" y="214" text-anchor="middle" font-size="16">0.10.0</text>
|
||||
<circle class="point" cx="556" cy="117.4360" r="4.5" stroke-width="2"/>
|
||||
<text class="score" x="556" y="82" text-anchor="middle" font-size="18" font-weight="600">77.67%</text>
|
||||
<text class="label" x="556" y="214" text-anchor="middle" font-size="16">0.15.0</text>
|
||||
<circle class="point" cx="660" cy="129.1200" r="4.5" stroke-width="2"/>
|
||||
<text class="score" x="660" y="82" text-anchor="middle" font-size="18" font-weight="600">76.40%</text>
|
||||
<text class="label" x="660" y="214" text-anchor="middle" font-size="16">0.19.0</text>
|
||||
<circle class="point" cx="764" cy="120.5640" r="4.5" stroke-width="2"/>
|
||||
<text class="score" x="764" y="82" text-anchor="middle" font-size="18" font-weight="600">77.33%</text>
|
||||
<text class="label" x="764" y="214" text-anchor="middle" font-size="16">0.22.0</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.2 KiB After Width: | Height: | Size: 4.1 KiB |
@@ -157,33 +157,11 @@ If you know Claude Code, you already know Qwen Code — and then some. We've put
|
||||
|
||||
## Qwen Code Evaluation
|
||||
|
||||
### Evaluation Configuration
|
||||
On **SWE-bench Verified**, seven Qwen Code versions scored **76.40%–77.80%** on average using the same Qwen 3.7 Max model (500 tasks, three trials per version).
|
||||
|
||||
| Configuration | Value |
|
||||
| ----------------------- | --------------------------------------------------------------------------------------------------- |
|
||||
| Dataset | `princeton-nlp/SWE-bench_Verified`, 500 cases |
|
||||
| Runs | 3 trials per version, 1,500 jobs per version; 7 Qwen Code versions |
|
||||
| Model | `Qwen 3.7 Max` |
|
||||
| Sampling | `temperature=1`, `top_p=0.95` |
|
||||
| Thinking / Reasoning | `interleaved_thinking=true`, `thinking_type=adaptive`, `reasoning_effort=high` |
|
||||
| Context / Output Limits | `context_1m=true`, `model_context_length=1,000,000`, `max_tokens=64,000`, `model_max_tokens=65,535` |
|
||||
| Harness Limits | `max_iterations=500`, `runtime_timeout_sec=7,200`, `agent_timeout_multiplier=3.0` |
|
||||
|
||||
|
||||
### Scores by Qwen Code Version
|
||||
|
||||
| Version | Avg Score | Pass Rate | pass@3 | pass^3 |
|
||||
| ------- | ---------: | --------: | -----: | -----: |
|
||||
| 0.0.14 | **77.80%** | 84.6% | 84.6% | 69.2% |
|
||||
| 0.1.0 | **76.73%** | 84.6% | 84.6% | 68.0% |
|
||||
| 0.5.0 | **76.60%** | 83.0% | 83.0% | 68.4% |
|
||||
| 0.10.0 | **76.47%** | 84.4% | 84.4% | 67.4% |
|
||||
| 0.15.0 | **77.67%** | 84.6% | 84.6% | 68.6% |
|
||||
| 0.19.0 | **76.40%** | 85.0% | 85.0% | 65.8% |
|
||||
| 0.22.0 | **77.33%** | 84.4% | 84.4% | 68.0% |
|
||||
|
||||
### Avg Score Trend
|
||||
|
||||
|
||||
[Full results and evaluation configuration →](./docs/evaluation.md)
|
||||
|
||||
## Ecosystem
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# README Evaluation Presentation
|
||||
|
||||
[English](2026-10-14-readme-evaluation.md) | [简体中文](2026-10-14-readme-evaluation.zh-CN.md)
|
||||
|
||||
## Problem
|
||||
|
||||
The README expands evaluation configuration, a results table, and a large chart
|
||||
before readers reach the ecosystem section. The chart's 76%–78% axis visually
|
||||
amplifies the 1.40 percentage point spread across versions.
|
||||
|
||||
## Design
|
||||
|
||||
- Keep the existing evaluation heading and anchor. Show one sentence of benchmark
|
||||
context, a compact chart, and a link to the full results.
|
||||
- Move all existing configuration and result values to linked English and Chinese
|
||||
evaluation documents under `docs/`.
|
||||
- Use a compact line chart to show the version sequence. Widen the vertical axis
|
||||
to 70%–80% and label that range in the subtitle and ticks. Preserve release
|
||||
order and exact score labels; connect observed points with straight segments
|
||||
without smoothing. Use one Qwen purple accent, quiet grid lines, and readable
|
||||
text. Support light and dark backgrounds.
|
||||
- Retain the existing SVG path so incoming image references continue to work.
|
||||
Include accessible title, description, and Markdown alternative text.
|
||||
- Change presentation only. Do not update benchmark results or claim statistical
|
||||
significance, improvement, or results for untested releases.
|
||||
|
||||
## Validation and Acceptance Criteria
|
||||
|
||||
- The README evaluation section has no expanded tables or third-level headings.
|
||||
- Both evaluation documents preserve every score and configuration value from
|
||||
the original README and have reciprocal language links.
|
||||
- SVG point positions agree with the reported scores and labeled 70%–80% scale.
|
||||
- Inspect rendered desktop and narrow views in light and dark themes; check
|
||||
labels, contrast, spacing, links, and an accessible table fallback.
|
||||
- Check Markdown formatting, SVG syntax, and the full diff in two clean passes.
|
||||
Attempt the repository build and typecheck; report any environment limitation.
|
||||
|
||||
## Scope
|
||||
|
||||
This change affects repository documentation and its existing chart asset only.
|
||||
It does not change CLI behavior and needs no CLI E2E plan or unit test.
|
||||
@@ -0,0 +1,34 @@
|
||||
# README 评测展示
|
||||
|
||||
[English](2026-10-14-readme-evaluation.md) | [简体中文](2026-10-14-readme-evaluation.zh-CN.md)
|
||||
|
||||
## 问题
|
||||
|
||||
README 在生态章节之前完整展开评测配置、成绩表和大幅图表。原图使用 76%–78%
|
||||
的纵轴，把版本间 1.40 个百分点的差异在视觉上放大。
|
||||
|
||||
## 设计
|
||||
|
||||
- 保留既有评测标题和锚点。正文只展示一句评测背景、紧凑图表和完整结果链接。
|
||||
- 将现有全部配置和成绩移到 `docs/` 下互相链接的英文、中文评测文档。
|
||||
- 使用紧凑折线图展示版本序列。将纵轴扩大到 70%–80%，并在副标题和刻度中明确
|
||||
标示。保留版本顺序和精确得分标签，以直线连接观测点，不做平滑处理。使用一种
|
||||
Qwen 紫色、浅色网格和清晰文字，支持浅色与深色背景。
|
||||
- 保留现有 SVG 路径，确保外部图片引用仍然有效。补充无障碍标题、描述和
|
||||
Markdown 图片替代文本。
|
||||
- 只调整展示，不更新评测结果，不宣称统计显著性、能力提升或未测试版本的成绩。
|
||||
|
||||
## 验证与验收标准
|
||||
|
||||
- README 评测部分没有展开的表格或三级标题。
|
||||
- 两份评测文档保留原 README 的所有成绩和配置值，并包含双向语言链接。
|
||||
- SVG 数据点位置与已报告得分和标明的 70%–80% 刻度一致。
|
||||
- 检查桌面与窄屏的浅色、深色实际渲染，确认标签、对比度、间距、链接和可访问的
|
||||
表格替代形式。
|
||||
- 检查 Markdown 格式、SVG 语法和完整 diff，完成两轮无问题审阅。尝试仓库构建
|
||||
和类型检查，如受环境限制则如实记录。
|
||||
|
||||
## 范围
|
||||
|
||||
本次变更只涉及仓库文档及现有图表资源，不改变 CLI 行为，无需 CLI E2E 计划或
|
||||
单元测试。
|
||||
@@ -0,0 +1,39 @@
|
||||
# Qwen Code Evaluation
|
||||
|
||||
[English](evaluation.md) | [简体中文](evaluation.zh-CN.md)
|
||||
|
||||
This evaluation compares seven Qwen Code versions on SWE-bench Verified using
|
||||
the same Qwen 3.7 Max model and configuration. Average scores range from
|
||||
**76.40% to 77.80%**, a spread of **1.40 percentage points**.
|
||||
|
||||
## Scores by Qwen Code Version
|
||||
|
||||
|
||||
|
||||
| Version | Avg Score | Pass Rate | pass@3 | pass^3 |
|
||||
| ------- | --------: | --------: | -----: | -----: |
|
||||
| 0.0.14 | 77.80% | 84.6% | 84.6% | 69.2% |
|
||||
| 0.1.0 | 76.73% | 84.6% | 84.6% | 68.0% |
|
||||
| 0.5.0 | 76.60% | 83.0% | 83.0% | 68.4% |
|
||||
| 0.10.0 | 76.47% | 84.4% | 84.4% | 67.4% |
|
||||
| 0.15.0 | 77.67% | 84.6% | 84.6% | 68.6% |
|
||||
| 0.19.0 | 76.40% | 85.0% | 85.0% | 65.8% |
|
||||
| 0.22.0 | 77.33% | 84.4% | 84.4% | 68.0% |
|
||||
|
||||
The line chart connects versions in release order and uses an explicitly labeled
|
||||
70%–80% scale. The table retains all reported metrics;
|
||||
Pass Rate and pass@3 have identical values in these results.
|
||||
|
||||
## Evaluation Configuration
|
||||
|
||||
| Configuration | Value |
|
||||
| ----------------------- | --------------------------------------------------------------------------------------------------- |
|
||||
| Dataset | `princeton-nlp/SWE-bench_Verified`, 500 cases |
|
||||
| Runs | 3 trials per version, 1,500 jobs per version; 7 Qwen Code versions |
|
||||
| Model | `Qwen 3.7 Max` |
|
||||
| Sampling | `temperature=1`, `top_p=0.95` |
|
||||
| Thinking / Reasoning | `interleaved_thinking=true`, `thinking_type=adaptive`, `reasoning_effort=high` |
|
||||
| Context / Output Limits | `context_1m=true`, `model_context_length=1,000,000`, `max_tokens=64,000`, `model_max_tokens=65,535` |
|
||||
| Harness Limits | `max_iterations=500`, `runtime_timeout_sec=7,200`, `agent_timeout_multiplier=3.0` |
|
||||
|
||||
[Back to README](../README.md#qwen-code-evaluation)
|
||||
@@ -0,0 +1,37 @@
|
||||
# Qwen Code 评测
|
||||
|
||||
[English](evaluation.md) | [简体中文](evaluation.zh-CN.md)
|
||||
|
||||
本次评测使用相同的 Qwen 3.7 Max 模型和配置，在 SWE-bench Verified 上比较七个
|
||||
Qwen Code 版本。平均得分为 **76.40%–77.80%**，最大差值为 **1.40 个百分点**。
|
||||
|
||||
## 各版本得分
|
||||
|
||||
|
||||
|
||||
| 版本 | Avg Score | Pass Rate | pass@3 | pass^3 |
|
||||
| ------ | --------: | --------: | -----: | -----: |
|
||||
| 0.0.14 | 77.80% | 84.6% | 84.6% | 69.2% |
|
||||
| 0.1.0 | 76.73% | 84.6% | 84.6% | 68.0% |
|
||||
| 0.5.0 | 76.60% | 83.0% | 83.0% | 68.4% |
|
||||
| 0.10.0 | 76.47% | 84.4% | 84.4% | 67.4% |
|
||||
| 0.15.0 | 77.67% | 84.6% | 84.6% | 68.6% |
|
||||
| 0.19.0 | 76.40% | 85.0% | 85.0% | 65.8% |
|
||||
| 0.22.0 | 77.33% | 84.4% | 84.4% | 68.0% |
|
||||
|
||||
折线图按发布顺序连接各版本，明确标示 70%–80% 的纵轴范围。表格保留全部已报告指标；本组结果中 Pass Rate 与
|
||||
pass@3 的数值相同。
|
||||
|
||||
## 评测配置
|
||||
|
||||
| 配置项 | 值 |
|
||||
| ------------- | --------------------------------------------------------------------------------------------------- |
|
||||
| 数据集 | `princeton-nlp/SWE-bench_Verified`，500 个任务 |
|
||||
| 运行次数 | 每个版本 3 次，共 1,500 个作业；覆盖 7 个 Qwen Code 版本 |
|
||||
| 模型 | `Qwen 3.7 Max` |
|
||||
| 采样 | `temperature=1`, `top_p=0.95` |
|
||||
| 思考 / 推理 | `interleaved_thinking=true`, `thinking_type=adaptive`, `reasoning_effort=high` |
|
||||
| 上下文 / 输出 | `context_1m=true`, `model_context_length=1,000,000`, `max_tokens=64,000`, `model_max_tokens=65,535` |
|
||||
| Harness 限制 | `max_iterations=500`, `runtime_timeout_sec=7,200`, `agent_timeout_multiplier=3.0` |
|
||||
|
||||
[返回 README](../README.md#qwen-code-evaluation)
|
||||
Reference in new issue
Block a user