Plot every model with both a SWE-bench Verified score and a price on the shared coding-agent session, so the value frontier is visible at a glance: open-weight DeepSeek V4 Pro / Kimi K2.6 reach ~80% (level with Gemini 3.1 Pro) at ~1/10-1/25 the cost, while the 95% ceiling (Fable 5) costs ~46x the cheapest 80% model. - scripts/make_coding_chart.py renders assets/coding-value.png. The cost axis reuses the unit-tested cost_calc engine (no drift from the published tables); the capability axis is the dated swe_bench_verified figure. PIL is lazy-imported so the pure data layer stays importable on the CI runner. - scripts/test_make_coding_chart.py unit-tests build_coding_data and value_headline (filtering, sort order, cost math, peer/value logic). - Embed the chart + caption in BENCHMARKS.md and BENCHMARKS.zh-CN.md 3.3. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
256 lines
9.2 KiB
Python
256 lines
9.2 KiB
Python
#!/usr/bin/env python3
|
||
"""Render the coding capability-vs-cost chart to assets/coding-value.png.
|
||
|
||
This is the *coding* counterpart to make_cost_chart.py. Instead of a single
|
||
scenario's price, it plots each model's **agentic-coding capability**
|
||
(SWE-bench Verified) against the **cost of one coding-agent session**, so the
|
||
"value frontier" is visible at a glance: open-weight models reach ~80% at a
|
||
fraction of flagship cost.
|
||
|
||
Both axes are reproducible. The cost axis reuses the unit-tested cost engine
|
||
(``cost_calc.compute_cost`` on the shared ``coding`` scenario) so the chart can
|
||
never drift from BENCHMARKS.md Part 3.3; the capability axis is the
|
||
``swe_bench_verified`` figure dated in data/models.json. Any model that has
|
||
*both* a published SWE-bench Verified score and a price is plotted — nothing is
|
||
hand-placed.
|
||
|
||
PIL is imported lazily inside ``render()`` so the pure data function
|
||
(``build_coding_data``) stays importable — and unit-testable — on machines
|
||
without Pillow (e.g. the CI runner).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import math
|
||
from pathlib import Path
|
||
|
||
from cost_calc import MODELS_FILE, SCENARIOS, compute_cost, format_cost
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
OUT = ROOT / "assets" / "coding-value.png"
|
||
|
||
|
||
def build_coding_data(models: list[dict]) -> list[dict]:
|
||
"""Rows for every model with both a SWE-bench Verified score and a price.
|
||
|
||
Cost is the shared ``coding`` agent-session scenario, computed by the same
|
||
engine that drives the published cost tables (reasoning models bill their
|
||
thinking tokens as output). Returns rows sorted cheapest-first; the single
|
||
highest-capability row is flagged ``flagship`` so the renderer can mark the
|
||
price ceiling.
|
||
"""
|
||
coding = next(s for s in SCENARIOS if s.id == "coding")
|
||
rows = []
|
||
for m in models:
|
||
swe = (m.get("benchmarks") or {}).get("swe_bench_verified")
|
||
pricing = m.get("pricing_usd_per_mtok")
|
||
if swe is None or not pricing:
|
||
continue
|
||
rows.append(
|
||
{
|
||
"name": m["name"],
|
||
"provider": m["provider"],
|
||
"open": bool(m.get("open")),
|
||
"swe": float(swe), # fraction 0–1
|
||
"cost": compute_cost(pricing, coding, m.get("reasoning", False)),
|
||
"flagship": False,
|
||
}
|
||
)
|
||
rows.sort(key=lambda r: r["cost"])
|
||
if rows:
|
||
max(rows, key=lambda r: r["swe"])["flagship"] = True
|
||
return rows
|
||
|
||
|
||
PEER_TOLERANCE = 0.02 # ±2 SWE-bench points counts as "ties / rivals"
|
||
|
||
|
||
def value_headline(rows: list[dict]) -> dict | None:
|
||
"""The one-line "value" story: best open model vs. the closed peer it rivals.
|
||
|
||
Picks the strongest open-weight model, then the most capable *closed* model
|
||
it still matches on SWE-bench (within ±2 points) while costing less, and
|
||
reports the cost multiple it undercuts that peer by. This is the honest
|
||
framing — open weights don't reach the 95% ceiling, but they tie a pricier
|
||
flagship-tier closed model for a fraction of the cost. ``None`` when there
|
||
is no open model that rivals a pricier closed one.
|
||
"""
|
||
open_rows = [r for r in rows if r["open"]]
|
||
closed_rows = [r for r in rows if not r["open"]]
|
||
if not open_rows or not closed_rows:
|
||
return None
|
||
best_open = max(open_rows, key=lambda r: (r["swe"], -r["cost"]))
|
||
peers = [
|
||
c for c in closed_rows
|
||
if c["cost"] > best_open["cost"] and abs(c["swe"] - best_open["swe"]) <= PEER_TOLERANCE
|
||
]
|
||
if not peers:
|
||
return None
|
||
peer = max(peers, key=lambda r: r["swe"])
|
||
top_swe = max(r["swe"] for r in rows)
|
||
return {
|
||
"name": best_open["name"],
|
||
"peer": peer["name"],
|
||
"swe": best_open["swe"],
|
||
"cost": best_open["cost"],
|
||
"cheaper_x": peer["cost"] / best_open["cost"] if best_open["cost"] else 0.0,
|
||
"gap_pts": (top_swe - best_open["swe"]) * 100,
|
||
}
|
||
|
||
|
||
# ── Rendering (PIL only; not exercised by unit tests) ────────────────────────
|
||
|
||
W, H = 1200, 675
|
||
BG = "#0d1117"
|
||
WHITE = "#f0f6fc"
|
||
GRAY = "#8b949e"
|
||
BLUE = "#58a6ff"
|
||
GREEN = "#3fb950"
|
||
GOLD = "#e3b341"
|
||
TRACK = "#161b22"
|
||
GRID = "#21262d"
|
||
|
||
# x-axis (cost, log scale) and y-axis (SWE-bench %) ranges
|
||
X_MIN, X_MAX = 0.05, 3.6
|
||
Y_MIN, Y_MAX = 70.0, 96.0
|
||
X_TICKS = [0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 3.0]
|
||
Y_TICKS = [70, 75, 80, 85, 90, 95]
|
||
|
||
# Per-model label placement (anchor, dx, dy) to keep the 7 labels legible.
|
||
# anchor: "l" left-aligned right of the dot, "r" right-aligned left of it.
|
||
LABELS = {
|
||
"Claude Fable 5": ("r", -16, -8),
|
||
"GPT-5.5": ("l", 16, -8),
|
||
"Claude Opus 4.8": ("r", -16, -8),
|
||
"Gemini 3.1 Pro": ("c", 0, -34),
|
||
"DeepSeek V4 Pro": ("l", 14, -32),
|
||
"Kimi K2.6": ("l", 14, 16),
|
||
"Claude Haiku 4.5": ("l", 16, -8),
|
||
}
|
||
|
||
PLOT_L, PLOT_R, PLOT_T, PLOT_B = 132, 1140, 185, 560
|
||
|
||
|
||
def _x(cost: float) -> float:
|
||
lo, hi = math.log10(X_MIN), math.log10(X_MAX)
|
||
return PLOT_L + (math.log10(cost) - lo) / (hi - lo) * (PLOT_R - PLOT_L)
|
||
|
||
|
||
def _y(swe_pct: float) -> float:
|
||
return PLOT_B - (swe_pct - Y_MIN) / (Y_MAX - Y_MIN) * (PLOT_B - PLOT_T)
|
||
|
||
|
||
def render(rows: list[dict], out: Path = OUT) -> Path:
|
||
from PIL import Image, ImageDraw, ImageFont
|
||
|
||
fdir = "/System/Library/Fonts/Supplemental/"
|
||
bold = lambda s: ImageFont.truetype(fdir + "Arial Bold.ttf", s)
|
||
reg = lambda s: ImageFont.truetype(fdir + "Arial.ttf", s)
|
||
|
||
img = Image.new("RGB", (W, H), BG)
|
||
d = ImageDraw.Draw(img)
|
||
d.rectangle([0, 0, W, 6], fill=BLUE)
|
||
|
||
d.text((48, 36), "Coding: capability vs. cost", font=bold(38), fill=WHITE)
|
||
d.text(
|
||
(48, 88),
|
||
"SWE-bench Verified (higher = better) against the cost of one coding-agent session.",
|
||
font=reg(21),
|
||
fill=GRAY,
|
||
)
|
||
d.text(
|
||
(48, 116),
|
||
"Same unit-tested engine as the cost tables — open-weight models in green.",
|
||
font=reg(21),
|
||
fill=GRAY,
|
||
)
|
||
|
||
# grid + y ticks
|
||
for t in Y_TICKS:
|
||
y = _y(t)
|
||
d.line([(PLOT_L, y), (PLOT_R, y)], fill=GRID, width=1)
|
||
d.text((PLOT_L - 46, y - 11), f"{t}%", font=bold(17), fill=GRAY)
|
||
# x ticks
|
||
for t in X_TICKS:
|
||
x = _x(t)
|
||
d.line([(x, PLOT_T), (x, PLOT_B)], fill=GRID, width=1)
|
||
lab = f"${t:g}" if t >= 0.1 else f"${t:.2f}"
|
||
bb = d.textbbox((0, 0), lab, font=bold(16))
|
||
d.text((x - (bb[2] - bb[0]) / 2, PLOT_B + 10), lab, font=bold(16), fill=GRAY)
|
||
|
||
# axis captions (no arrow glyphs — Arial lacks them)
|
||
d.text((PLOT_L - 52, PLOT_T - 28), "SWE-bench Verified (higher = better)", font=bold(16), fill=GRAY)
|
||
cap = "cost per coding-agent session (log scale, USD)"
|
||
bb = d.textbbox((0, 0), cap, font=bold(16))
|
||
d.text(((PLOT_L + PLOT_R) / 2 - (bb[2] - bb[0]) / 2, PLOT_B + 34), cap, font=bold(16), fill=GRAY)
|
||
|
||
# points + labels
|
||
for r in rows:
|
||
x, y = _x(r["cost"]), _y(r["swe"] * 100)
|
||
color = GOLD if r["flagship"] else GREEN if r["open"] else BLUE
|
||
d.ellipse([x - 9, y - 9, x + 9, y + 9], fill=color, outline=BG, width=2)
|
||
d.ellipse([x - 3, y - 3, x + 3, y + 3], fill=BG)
|
||
|
||
text = f"{r['name']} {r['swe'] * 100:.1f}% · {format_cost(r['cost'])}"
|
||
anchor, dx, dy = LABELS.get(r["name"], ("l", 14, -8))
|
||
tx, ty = x + dx, y + dy
|
||
bb = d.textbbox((0, 0), text, font=bold(18))
|
||
tw = bb[2] - bb[0]
|
||
if anchor == "r":
|
||
tx -= tw
|
||
elif anchor == "c":
|
||
tx -= tw / 2
|
||
d.text((tx, ty), text, font=bold(18), fill=WHITE)
|
||
|
||
# value callout in the empty top-left zone, with a drawn arrow (not a glyph)
|
||
# pointing down to the open-weight cluster.
|
||
hd = value_headline(rows)
|
||
if hd:
|
||
d.text((PLOT_L + 24, PLOT_T + 30), "Open-weight value zone", font=bold(20), fill=GREEN)
|
||
d.text(
|
||
(PLOT_L + 24, PLOT_T + 58),
|
||
"~80% SWE-bench Verified at a fraction of the flagship cost",
|
||
font=bold(15),
|
||
fill=GREEN,
|
||
)
|
||
# arrow: from below the callout down to the cheapest open dot
|
||
target = min((r for r in rows if r["open"]), key=lambda r: r["cost"])
|
||
ax, ay = _x(target["cost"]) + 6, _y(target["swe"] * 100) - 14
|
||
sx, sy = PLOT_L + 60, PLOT_T + 92
|
||
d.line([(sx, sy), (ax, ay)], fill=GREEN, width=2)
|
||
ang = math.atan2(ay - sy, ax - sx)
|
||
for da in (2.5, -2.5):
|
||
d.line(
|
||
[(ax, ay), (ax - 12 * math.cos(ang + da), ay - 12 * math.sin(ang + da))],
|
||
fill=GREEN,
|
||
width=2,
|
||
)
|
||
|
||
# footer
|
||
note = "github.com/cuihuan/awesome-ai-gateway"
|
||
if hd:
|
||
note = (
|
||
f"{hd['name']} ties {hd['peer']} on SWE-bench at ~{hd['cheaper_x']:.0f}× less · "
|
||
+ note
|
||
)
|
||
d.text((48, H - 30), note, font=bold(19), fill=BLUE)
|
||
|
||
out.parent.mkdir(exist_ok=True)
|
||
img.save(out, "PNG")
|
||
return out
|
||
|
||
|
||
def main() -> int:
|
||
models = json.loads(MODELS_FILE.read_text(encoding="utf-8"))["models"]
|
||
rows = build_coding_data(models)
|
||
out = render(rows)
|
||
hd = value_headline(rows)
|
||
extra = f" — {hd['name']} {hd['cheaper_x']:.0f}× cheaper" if hd else ""
|
||
print(f"wrote {out} ({out.stat().st_size // 1024} KB) — {len(rows)} models{extra}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|