{"family":{"id":"ionwarp-cost","key":"cost-review","agent":null,"name":"Cost Review","slug":"cost"},"snapshot":{"id":"onerow-cost-2026-09-24-r1","captured_at":"2026-09-24T18:23:13.403655+00:00","status":"methodology_review","source_revision_url":"https://github.com/Winding-Labs/ai-code-review-benchmark"},"release":{"id":"2026-10-12-public-v1","date":"2026-10-12"},"publication":{"protocol":"category-scoped-v4-all-golden-match","status":"methodology_review","label":"Provisional scoped-label matching","ranking_allowed":false,"blockers":["operator_labels_not_official_corpus","generalist_output_rescored"],"methodology_url":"https://ionwarp.com/benchmarks/methodology"},"judge":"IonWarp adapted category-scoped judge, Claude Opus 4.5","scope":"cost specialist · fresh local pilot · provisional semantic matching","case_count":20,"issue_count":14,"score_definition":"Reviewer Score = (3 × F1 + latency + $/run) ÷ 5. Each part is this row's place between the worst (0) and best (100) full-cohort row in this family — higher F1, lower latency and lower $/run score higher. Scored once F1, latency and $/run are all measured and at least 3 rows are.","ranking_basis":"Sorted by Reviewer Score (F1 counts 3×, time and cost 1× each).","scenario":{"seats":5,"monthly_reviews":100},"configured_pin":{"model":"z-ai/glm-5.3-flash","reasoning_effort":"medium","retry":null},"warnings":["Local reviewer execution; no hosted queue or GitHub delivery measurement.","One trial per declared configuration and PR; exploratory comparison.","Pinned source issue matching is not independent finding adjudication.","93 cells are admitted unchanged from the experiments that paid for them (identical case identity, engine and output ceiling, on the runtime of their configuration); they are never re-bought.","Contains operator-labelled PRs outside the official Martian corpus. These results are not an official-corpus leaderboard score.","Precision is golden-issue matching precision, not independently adjudicated finding correctness.","Competitor outputs are historical; fresh IonWarp runs use the same PR heads and new semantic grading.","Available-evidence checkpoint: unstarted jobs and completed but ungraded outputs remain explicit missing coverage. Scores use only saved grades under the named protocol.","The saved grade files preserve all source rows. This checkpoint projects only its declared case/configuration identities; unselected rows are not pooled into displayed scores."],"notices":["Provisional: one agent searched for these pull requests by this lens's own defect classes and wrote the goldens from each later fix; a second rater checked every golden against its diff and held 5 of 21 (10 of 10 planted decoys sorted correctly). 11 of 31 PRs (7 positive, 4 control) come from the repository that ships this reviewer. IonWarp models only.","Measured on the reviewer prompt of 2026-09-22; production has changed it since"],"not_measured":["CodeRabbit","Cursor Bugbot","Greptile v4.1","Qodo v2"],"license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"cost_unit":"review_run","case_ids":["vercel-ai-17923","cline-cline-7795","cline-cline-9576","langgenius-dify-4140","portkey-ai-gateway-1373","langchain-ai-langchainjs-6922","danny-avila-librechat-13014","danny-avila-librechat-13683","run-llama-llama_index-20206","mastra-ai-mastra-12465","onyx-dot-app-onyx-10478","open-webui-open-webui-15122","posthog-posthog-60403","posthog-posthog-71860","posthog-posthog-104266","posthog-posthog-python-346","pydantic-pydantic-ai-2771","roocodeinc-roo-code-6864","getsentry-sentry-python-4612","getsentry-sentry-python-5319"],"benchmark_default":null,"benefits":null,"rows":[{"id":"ionwarp-glm-5-3-flash","name":"IonWarp · GLM 5.3 Flash","label":"IonWarp · GLM 5.3 Flash","kind":"ionwarp","model":"z-ai/glm-5.3-flash","model_label":"GLM 5.3 Flash","developer":"z-ai","effort":"low","composite":72.9,"composite_parts":{"f1":72,"latency":47,"cost":100},"f1":0.5517,"f2":0.5634,"precision":0.5333,"recall":0.5714,"tp":8,"fp":7,"fn":6,"expected":14,"prs":20,"coverage":{"graded":20,"cohort":20},"display_f1":0.5517,"latency_p50_seconds":142.8,"latency_p90_seconds":383.1,"display_latency_seconds":142.8,"cost_per_review_usd":0.0137,"cost_basis":"observed_inference","chart_cost":0.0137,"run_dates":["2026-09-22","2026-09-22"],"popovers":{"f1":{"f2":0.5634,"cohort_of":20},"cost":{"estimate":false},"score":{"parts":{"f1":72,"latency":47,"cost":100}},"prs":{"cases":[{"case_id":"vercel-ai-17923","repository":"ai","number":17923,"url":"https://github.com/vercel/ai/pull/17923","status":"judged","expected":0,"caught":0,"seconds":40.359,"review_url":null},{"case_id":"cline-cline-7795","repository":"cline","number":7795,"url":"https://github.com/cline/cline/pull/7795","status":"judged","expected":1,"caught":1,"seconds":414.22,"review_url":null},{"case_id":"cline-cline-9576","repository":"cline","number":9576,"url":"https://github.com/cline/cline/pull/9576","status":"judged","expected":1,"caught":1,"seconds":318.825,"review_url":null},{"case_id":"langgenius-dify-4140","repository":"dify","number":4140,"url":"https://github.com/langgenius/dify/pull/4140","status":"judged","expected":0,"caught":0,"seconds":130.854,"review_url":null},{"case_id":"portkey-ai-gateway-1373","repository":"gateway","number":1373,"url":"https://github.com/Portkey-AI/gateway/pull/1373","status":"judged","expected":0,"caught":0,"seconds":20.25,"review_url":null},{"case_id":"langchain-ai-langchainjs-6922","repository":"langchainjs","number":6922,"url":"https://github.com/langchain-ai/langchainjs/pull/6922","status":"judged","expected":1,"caught":1,"seconds":82.498,"review_url":null},{"case_id":"danny-avila-librechat-13014","repository":"LibreChat","number":13014,"url":"https://github.com/danny-avila/LibreChat/pull/13014","status":"judged","expected":1,"caught":0,"seconds":34.866,"review_url":null},{"case_id":"danny-avila-librechat-13683","repository":"LibreChat","number":13683,"url":"https://github.com/danny-avila/LibreChat/pull/13683","status":"judged","expected":0,"caught":0,"seconds":36.754,"review_url":null},{"case_id":"run-llama-llama_index-20206","repository":"llama_index","number":20206,"url":"https://github.com/run-llama/llama_index/pull/20206","status":"judged","expected":1,"caught":1,"seconds":383.15,"review_url":null},{"case_id":"mastra-ai-mastra-12465","repository":"mastra","number":12465,"url":"https://github.com/mastra-ai/mastra/pull/12465","status":"judged","expected":1,"caught":0,"seconds":176.784,"review_url":null},{"case_id":"onyx-dot-app-onyx-10478","repository":"onyx","number":10478,"url":"https://github.com/onyx-dot-app/onyx/pull/10478","status":"judged","expected":1,"caught":1,"seconds":340.931,"review_url":null},{"case_id":"open-webui-open-webui-15122","repository":"open-webui","number":15122,"url":"https://github.com/open-webui/open-webui/pull/15122","status":"judged","expected":0,"caught":0,"seconds":26.483,"review_url":null},{"case_id":"posthog-posthog-60403","repository":"posthog","number":60403,"url":"https://github.com/PostHog/posthog/pull/60403","status":"judged","expected":1,"caught":0,"seconds":337.383,"review_url":null},{"case_id":"posthog-posthog-71860","repository":"posthog","number":71860,"url":"https://github.com/PostHog/posthog/pull/71860","status":"judged","expected":1,"caught":0,"seconds":154.842,"review_url":null},{"case_id":"posthog-posthog-104266","repository":"posthog","number":104266,"url":"https://github.com/PostHog/posthog/pull/104266","status":"judged","expected":0,"caught":0,"seconds":25.354,"review_url":null},{"case_id":"posthog-posthog-python-346","repository":"posthog-python","number":346,"url":"https://github.com/PostHog/posthog-python/pull/346","status":"judged","expected":1,"caught":1,"seconds":117.035,"review_url":null},{"case_id":"pydantic-pydantic-ai-2771","repository":"pydantic-ai","number":2771,"url":"https://github.com/pydantic/pydantic-ai/pull/2771","status":"judged","expected":1,"caught":1,"seconds":414.646,"review_url":null},{"case_id":"roocodeinc-roo-code-6864","repository":"Roo-Code","number":6864,"url":"https://github.com/RooCodeInc/Roo-Code/pull/6864","status":"judged","expected":1,"caught":1,"seconds":358.392,"review_url":null},{"case_id":"getsentry-sentry-python-4612","repository":"sentry-python","number":4612,"url":"https://github.com/getsentry/sentry-python/pull/4612","status":"judged","expected":1,"caught":0,"seconds":354.71,"review_url":null},{"case_id":"getsentry-sentry-python-5319","repository":"sentry-python","number":5319,"url":"https://github.com/getsentry/sentry-python/pull/5319","status":"judged","expected":1,"caught":0,"seconds":26.741,"review_url":null}]}}},{"id":"ionwarp-deepseek-v4-1-flash","name":"IonWarp · DeepSeek V4.1 Flash","label":"IonWarp · DeepSeek V4.1 Flash","kind":"ionwarp","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","developer":"deepseek","effort":"medium","composite":60,"composite_parts":{"f1":100,"latency":0,"cost":0},"f1":0.5714,"f2":0.6494,"precision":0.4762,"recall":0.7143,"tp":10,"fp":11,"fn":4,"expected":14,"prs":20,"coverage":{"graded":20,"cohort":20},"display_f1":0.5714,"latency_p50_seconds":639.4,"latency_p90_seconds":977.4,"display_latency_seconds":639.4,"cost_per_review_usd":0.0728,"cost_basis":"observed_inference","chart_cost":0.0728,"run_dates":["2026-09-24","2026-09-24"],"popovers":{"f1":{"f2":0.6494,"cohort_of":20},"cost":{"estimate":false},"score":{"parts":{"f1":100,"latency":0,"cost":0}},"prs":{"cases":[{"case_id":"vercel-ai-17923","repository":"ai","number":17923,"url":"https://github.com/vercel/ai/pull/17923","status":"judged","expected":0,"caught":0,"seconds":727.56,"review_url":null},{"case_id":"cline-cline-7795","repository":"cline","number":7795,"url":"https://github.com/cline/cline/pull/7795","status":"judged","expected":1,"caught":1,"seconds":315.174,"review_url":null},{"case_id":"cline-cline-9576","repository":"cline","number":9576,"url":"https://github.com/cline/cline/pull/9576","status":"judged","expected":1,"caught":1,"seconds":357.132,"review_url":null},{"case_id":"langgenius-dify-4140","repository":"dify","number":4140,"url":"https://github.com/langgenius/dify/pull/4140","status":"judged","expected":0,"caught":0,"seconds":578.831,"review_url":null},{"case_id":"portkey-ai-gateway-1373","repository":"gateway","number":1373,"url":"https://github.com/Portkey-AI/gateway/pull/1373","status":"judged","expected":0,"caught":0,"seconds":726.946,"review_url":null},{"case_id":"langchain-ai-langchainjs-6922","repository":"langchainjs","number":6922,"url":"https://github.com/langchain-ai/langchainjs/pull/6922","status":"judged","expected":1,"caught":1,"seconds":541.025,"review_url":null},{"case_id":"danny-avila-librechat-13014","repository":"LibreChat","number":13014,"url":"https://github.com/danny-avila/LibreChat/pull/13014","status":"judged","expected":1,"caught":1,"seconds":977.361,"review_url":null},{"case_id":"danny-avila-librechat-13683","repository":"LibreChat","number":13683,"url":"https://github.com/danny-avila/LibreChat/pull/13683","status":"judged","expected":0,"caught":0,"seconds":765.108,"review_url":null},{"case_id":"run-llama-llama_index-20206","repository":"llama_index","number":20206,"url":"https://github.com/run-llama/llama_index/pull/20206","status":"judged","expected":1,"caught":1,"seconds":560.324,"review_url":null},{"case_id":"mastra-ai-mastra-12465","repository":"mastra","number":12465,"url":"https://github.com/mastra-ai/mastra/pull/12465","status":"judged","expected":1,"caught":1,"seconds":635.855,"review_url":null},{"case_id":"onyx-dot-app-onyx-10478","repository":"onyx","number":10478,"url":"https://github.com/onyx-dot-app/onyx/pull/10478","status":"judged","expected":1,"caught":0,"seconds":1031.135,"review_url":null},{"case_id":"open-webui-open-webui-15122","repository":"open-webui","number":15122,"url":"https://github.com/open-webui/open-webui/pull/15122","status":"judged","expected":0,"caught":0,"seconds":673.415,"review_url":null},{"case_id":"posthog-posthog-60403","repository":"posthog","number":60403,"url":"https://github.com/PostHog/posthog/pull/60403","status":"judged","expected":1,"caught":0,"seconds":302.891,"review_url":null},{"case_id":"posthog-posthog-71860","repository":"posthog","number":71860,"url":"https://github.com/PostHog/posthog/pull/71860","status":"judged","expected":1,"caught":0,"seconds":926.908,"review_url":null},{"case_id":"posthog-posthog-104266","repository":"posthog","number":104266,"url":"https://github.com/PostHog/posthog/pull/104266","status":"judged","expected":0,"caught":0,"seconds":770.649,"review_url":null},{"case_id":"posthog-posthog-python-346","repository":"posthog-python","number":346,"url":"https://github.com/PostHog/posthog-python/pull/346","status":"judged","expected":1,"caught":0,"seconds":506.984,"review_url":null},{"case_id":"pydantic-pydantic-ai-2771","repository":"pydantic-ai","number":2771,"url":"https://github.com/pydantic/pydantic-ai/pull/2771","status":"judged","expected":1,"caught":1,"seconds":642.845,"review_url":null},{"case_id":"roocodeinc-roo-code-6864","repository":"Roo-Code","number":6864,"url":"https://github.com/RooCodeInc/Roo-Code/pull/6864","status":"judged","expected":1,"caught":1,"seconds":333.816,"review_url":null},{"case_id":"getsentry-sentry-python-4612","repository":"sentry-python","number":4612,"url":"https://github.com/getsentry/sentry-python/pull/4612","status":"judged","expected":1,"caught":1,"seconds":430.976,"review_url":null},{"case_id":"getsentry-sentry-python-5319","repository":"sentry-python","number":5319,"url":"https://github.com/getsentry/sentry-python/pull/5319","status":"judged","expected":1,"caught":1,"seconds":984.548,"review_url":null}]}}},{"id":"ionwarp-gpt-6-luna-pro","name":"IonWarp · GPT-6 Luna Pro","label":"IonWarp · GPT-6 Luna Pro","kind":"ionwarp","model":"openai/gpt-6-luna-pro","model_label":"GPT-6 Luna Pro","developer":"openai","effort":"medium","composite":36.2,"composite_parts":{"f1":0,"latency":100,"cost":81},"f1":0.5,"f2":0.4545,"precision":0.6,"recall":0.4286,"tp":6,"fp":4,"fn":8,"expected":14,"prs":20,"coverage":{"graded":20,"cohort":20},"display_f1":0.5,"latency_p50_seconds":27.2,"latency_p90_seconds":329.2,"display_latency_seconds":27.2,"cost_per_review_usd":0.0188,"cost_basis":"observed_inference","chart_cost":0.0188,"run_dates":["2026-09-22","2026-09-22"],"popovers":{"f1":{"f2":0.4545,"cohort_of":20},"cost":{"estimate":false},"score":{"parts":{"f1":0,"latency":100,"cost":81}},"prs":{"cases":[{"case_id":"vercel-ai-17923","repository":"ai","number":17923,"url":"https://github.com/vercel/ai/pull/17923","status":"judged","expected":0,"caught":0,"seconds":113.448,"review_url":null},{"case_id":"cline-cline-7795","repository":"cline","number":7795,"url":"https://github.com/cline/cline/pull/7795","status":"judged","expected":1,"caught":1,"seconds":368.426,"review_url":null},{"case_id":"cline-cline-9576","repository":"cline","number":9576,"url":"https://github.com/cline/cline/pull/9576","status":"judged","expected":1,"caught":1,"seconds":211.597,"review_url":null},{"case_id":"langgenius-dify-4140","repository":"dify","number":4140,"url":"https://github.com/langgenius/dify/pull/4140","status":"judged","expected":0,"caught":0,"seconds":17.769,"review_url":null},{"case_id":"portkey-ai-gateway-1373","repository":"gateway","number":1373,"url":"https://github.com/Portkey-AI/gateway/pull/1373","status":"judged","expected":0,"caught":0,"seconds":26.601,"review_url":null},{"case_id":"langchain-ai-langchainjs-6922","repository":"langchainjs","number":6922,"url":"https://github.com/langchain-ai/langchainjs/pull/6922","status":"judged","expected":1,"caught":1,"seconds":280.25,"review_url":null},{"case_id":"danny-avila-librechat-13014","repository":"LibreChat","number":13014,"url":"https://github.com/danny-avila/LibreChat/pull/13014","status":"judged","expected":1,"caught":0,"seconds":18.21,"review_url":null},{"case_id":"danny-avila-librechat-13683","repository":"LibreChat","number":13683,"url":"https://github.com/danny-avila/LibreChat/pull/13683","status":"judged","expected":0,"caught":0,"seconds":33.637,"review_url":null},{"case_id":"run-llama-llama_index-20206","repository":"llama_index","number":20206,"url":"https://github.com/run-llama/llama_index/pull/20206","status":"judged","expected":1,"caught":0,"seconds":27.782,"review_url":null},{"case_id":"mastra-ai-mastra-12465","repository":"mastra","number":12465,"url":"https://github.com/mastra-ai/mastra/pull/12465","status":"judged","expected":1,"caught":0,"seconds":16.213,"review_url":null},{"case_id":"onyx-dot-app-onyx-10478","repository":"onyx","number":10478,"url":"https://github.com/onyx-dot-app/onyx/pull/10478","status":"judged","expected":1,"caught":0,"seconds":13.839,"review_url":null},{"case_id":"open-webui-open-webui-15122","repository":"open-webui","number":15122,"url":"https://github.com/open-webui/open-webui/pull/15122","status":"judged","expected":0,"caught":0,"seconds":11.916,"review_url":null},{"case_id":"posthog-posthog-60403","repository":"posthog","number":60403,"url":"https://github.com/PostHog/posthog/pull/60403","status":"judged","expected":1,"caught":0,"seconds":15.925,"review_url":null},{"case_id":"posthog-posthog-71860","repository":"posthog","number":71860,"url":"https://github.com/PostHog/posthog/pull/71860","status":"judged","expected":1,"caught":0,"seconds":110.057,"review_url":null},{"case_id":"posthog-posthog-104266","repository":"posthog","number":104266,"url":"https://github.com/PostHog/posthog/pull/104266","status":"judged","expected":0,"caught":0,"seconds":17.452,"review_url":null},{"case_id":"posthog-posthog-python-346","repository":"posthog-python","number":346,"url":"https://github.com/PostHog/posthog-python/pull/346","status":"judged","expected":1,"caught":0,"seconds":21.9,"review_url":null},{"case_id":"pydantic-pydantic-ai-2771","repository":"pydantic-ai","number":2771,"url":"https://github.com/pydantic/pydantic-ai/pull/2771","status":"judged","expected":1,"caught":1,"seconds":329.217,"review_url":null},{"case_id":"roocodeinc-roo-code-6864","repository":"Roo-Code","number":6864,"url":"https://github.com/RooCodeInc/Roo-Code/pull/6864","status":"judged","expected":1,"caught":1,"seconds":341.981,"review_url":null},{"case_id":"getsentry-sentry-python-4612","repository":"sentry-python","number":4612,"url":"https://github.com/getsentry/sentry-python/pull/4612","status":"judged","expected":1,"caught":1,"seconds":152.991,"review_url":null},{"case_id":"getsentry-sentry-python-5319","repository":"sentry-python","number":5319,"url":"https://github.com/getsentry/sentry-python/pull/5319","status":"judged","expected":1,"caught":0,"seconds":15.141,"review_url":null}]}}}]}