{"family":{"id":"ionwarp-docs-review","key":"docs-freshness","agent":"docs","name":"Docs Review","slug":"docs"},"snapshot":{"id":"onerow-docs-2026-09-22-three-full","captured_at":"2026-09-23T02:19:04.285290+00:00","status":"methodology_review","source_revision_url":"https://github.com/Winding-Labs/ai-code-review-benchmark"},"release":{"id":"2026-10-12-public-v1","date":"2026-10-12"},"publication":{"protocol":"category-scoped-v4-all-golden-match","status":"methodology_review","label":"Provisional scoped-label matching","ranking_allowed":false,"blockers":["operator_labels_not_official_corpus","generalist_output_rescored"],"methodology_url":"https://ionwarp.com/benchmarks/methodology"},"judge":"IonWarp adapted category-scoped judge, Claude Opus 4.5","scope":"docs specialist · fresh local pilot · provisional semantic matching","case_count":19,"issue_count":17,"score_definition":"Reviewer Score = (3 × F1 + latency + $/run) ÷ 5. Each part is this row's place between the worst (0) and best (100) full-cohort row in this family — higher F1, lower latency and lower $/run score higher. Scored once F1, latency and $/run are all measured and at least 3 rows are.","ranking_basis":"Sorted by Reviewer Score (F1 counts 3×, time and cost 1× each).","scenario":{"seats":5,"monthly_reviews":100},"configured_pin":{"model":"deepseek/deepseek-v4.1-flash","reasoning_effort":"medium","retry":{"model":"google/gemini-3.8-flash","reasoning_effort":"medium"}},"warnings":["Local reviewer execution; no hosted queue or GitHub delivery measurement.","One trial per declared configuration and PR; exploratory comparison.","Pinned source issue matching is not independent finding adjudication.","65 cells are admitted unchanged from the experiments that paid for them (identical case identity, engine and output ceiling, on the runtime of their configuration); they are never re-bought.","Contains operator-labelled PRs outside the official Martian corpus. These results are not an official-corpus leaderboard score.","Precision is golden-issue matching precision, not independently adjudicated finding correctness.","Competitor outputs are historical; fresh IonWarp runs use the same PR heads and new semantic grading.","The saved grade files preserve all source rows. This checkpoint projects only its declared case/configuration identities; unselected rows are not pooled into displayed scores."],"notices":["Measured on the reviewer prompt of 2026-09-22; production has changed it since"],"not_measured":[],"license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"cost_unit":"review_run","case_ids":["calcom-cal.com-10600","calcom-cal.com-22532","ai-code-review-evaluation-discourse-graphite-10","grafana-grafana-79265","grafana-grafana-97529","grafana-grafana-103633","keycloak-keycloak-36882","keycloak-keycloak-37429","keycloak-keycloak-37634","pandas-dev-pandas-43951","pandas-dev-pandas-45411","pandas-dev-pandas-47633","pandas-dev-pandas-57347","pandas-dev-pandas-61032","pandas-dev-pandas-61264","pandas-dev-pandas-61467","pandas-dev-pandas-63384","getsentry-sentry-80168","getsentry-sentry-95633"],"benchmark_default":null,"benefits":{"row_id":null,"model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","rivals":0,"wins":[],"cost_line":null,"basis":"DeepSeek V4.1 Flash, the model Docs Review runs, is not measured on this lane yet."},"rows":[{"id":"ionwarp-gemini-3-8-flash","name":"IonWarp · Gemini 3.8 Flash","label":"IonWarp · Gemini 3.8 Flash","kind":"ionwarp","model":"google/gemini-3.8-flash","model_label":"Gemini 3.8 Flash","developer":"google","effort":"medium","composite":79.2,"composite_parts":{"f1":100,"latency":84,"cost":12},"f1":0.6857,"f2":0.6977,"precision":0.6667,"recall":0.7059,"tp":12,"fp":6,"fn":5,"expected":17,"prs":19,"coverage":{"graded":19,"cohort":19},"display_f1":0.6857,"latency_p50_seconds":58.6,"latency_p90_seconds":92.9,"display_latency_seconds":58.6,"cost_per_review_usd":0.0839,"cost_basis":"observed_inference","chart_cost":0.0839,"run_dates":["2026-09-21","2026-09-22"],"popovers":{"f1":{"f2":0.6977,"cohort_of":19},"cost":{"estimate":false},"score":{"parts":{"f1":100,"latency":84,"cost":12}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":1,"seconds":47.742,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":35.517,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":69.004,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":0,"seconds":33.845,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":22.556,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":1,"caught":1,"seconds":55.806,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":1,"caught":0,"seconds":64.194,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":2,"caught":1,"seconds":115.608,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":1,"caught":1,"seconds":42.077,"review_url":null},{"case_id":"pandas-dev-pandas-43951","repository":"pandas","number":43951,"url":"https://github.com/pandas-dev/pandas/pull/43951","status":"judged","expected":1,"caught":1,"seconds":92.942,"review_url":null},{"case_id":"pandas-dev-pandas-45411","repository":"pandas","number":45411,"url":"https://github.com/pandas-dev/pandas/pull/45411","status":"judged","expected":1,"caught":1,"seconds":67.266,"review_url":null},{"case_id":"pandas-dev-pandas-47633","repository":"pandas","number":47633,"url":"https://github.com/pandas-dev/pandas/pull/47633","status":"judged","expected":1,"caught":1,"seconds":58.648,"review_url":null},{"case_id":"pandas-dev-pandas-57347","repository":"pandas","number":57347,"url":"https://github.com/pandas-dev/pandas/pull/57347","status":"judged","expected":1,"caught":0,"seconds":78.019,"review_url":null},{"case_id":"pandas-dev-pandas-61032","repository":"pandas","number":61032,"url":"https://github.com/pandas-dev/pandas/pull/61032","status":"judged","expected":1,"caught":1,"seconds":51.737,"review_url":null},{"case_id":"pandas-dev-pandas-61264","repository":"pandas","number":61264,"url":"https://github.com/pandas-dev/pandas/pull/61264","status":"judged","expected":1,"caught":0,"seconds":43.862,"review_url":null},{"case_id":"pandas-dev-pandas-61467","repository":"pandas","number":61467,"url":"https://github.com/pandas-dev/pandas/pull/61467","status":"judged","expected":1,"caught":1,"seconds":56.837,"review_url":null},{"case_id":"pandas-dev-pandas-63384","repository":"pandas","number":63384,"url":"https://github.com/pandas-dev/pandas/pull/63384","status":"judged","expected":1,"caught":1,"seconds":68.495,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":1,"caught":1,"seconds":66.789,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":1,"seconds":87.038,"review_url":null}]}}},{"id":"ionwarp-deepseek-v4-pro","name":"IonWarp · DeepSeek V4 Pro","label":"IonWarp · DeepSeek V4 Pro","kind":"ionwarp","model":"deepseek/deepseek-v4-pro-0813","model_label":"DeepSeek V4 Pro","developer":"deepseek","effort":"medium","composite":57.9,"composite_parts":{"f1":63,"latency":0,"cost":100},"f1":0.6522,"f2":0.7732,"precision":0.5172,"recall":0.8824,"tp":15,"fp":14,"fn":2,"expected":17,"prs":19,"coverage":{"graded":19,"cohort":19},"display_f1":0.6522,"latency_p50_seconds":180.4,"latency_p90_seconds":464.1,"display_latency_seconds":180.4,"cost_per_review_usd":0.0638,"cost_basis":"observed_inference","chart_cost":0.0638,"run_dates":["2026-09-21","2026-09-23"],"popovers":{"f1":{"f2":0.7732,"cohort_of":19},"cost":{"estimate":false},"score":{"parts":{"f1":63,"latency":0,"cost":100}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":1,"seconds":189.409,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":67.261,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":341.411,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":114.995,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":30.043,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":1,"caught":0,"seconds":36.685,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":1,"caught":0,"seconds":291.838,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":2,"caught":2,"seconds":120.33,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":1,"caught":1,"seconds":205.806,"review_url":null},{"case_id":"pandas-dev-pandas-43951","repository":"pandas","number":43951,"url":"https://github.com/pandas-dev/pandas/pull/43951","status":"judged","expected":1,"caught":1,"seconds":1787.212,"review_url":null},{"case_id":"pandas-dev-pandas-45411","repository":"pandas","number":45411,"url":"https://github.com/pandas-dev/pandas/pull/45411","status":"judged","expected":1,"caught":1,"seconds":205.124,"review_url":null},{"case_id":"pandas-dev-pandas-47633","repository":"pandas","number":47633,"url":"https://github.com/pandas-dev/pandas/pull/47633","status":"judged","expected":1,"caught":1,"seconds":131.757,"review_url":null},{"case_id":"pandas-dev-pandas-57347","repository":"pandas","number":57347,"url":"https://github.com/pandas-dev/pandas/pull/57347","status":"judged","expected":1,"caught":1,"seconds":154.468,"review_url":null},{"case_id":"pandas-dev-pandas-61032","repository":"pandas","number":61032,"url":"https://github.com/pandas-dev/pandas/pull/61032","status":"judged","expected":1,"caught":1,"seconds":180.362,"review_url":null},{"case_id":"pandas-dev-pandas-61264","repository":"pandas","number":61264,"url":"https://github.com/pandas-dev/pandas/pull/61264","status":"judged","expected":1,"caught":1,"seconds":292.167,"review_url":null},{"case_id":"pandas-dev-pandas-61467","repository":"pandas","number":61467,"url":"https://github.com/pandas-dev/pandas/pull/61467","status":"judged","expected":1,"caught":1,"seconds":141.941,"review_url":null},{"case_id":"pandas-dev-pandas-63384","repository":"pandas","number":63384,"url":"https://github.com/pandas-dev/pandas/pull/63384","status":"judged","expected":1,"caught":1,"seconds":464.101,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":1,"caught":1,"seconds":82.905,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":1,"seconds":221.652,"review_url":null}]}}},{"id":"ionwarp-muse-spark-1-3","name":"IonWarp · Muse Spark 1.3","label":"IonWarp · Muse Spark 1.3","kind":"ionwarp","model":"meta/muse-spark-1.3","model_label":"Muse Spark 1.3","developer":"meta","effort":"medium","composite":20,"composite_parts":{"f1":0,"latency":100,"cost":0},"f1":0.5946,"f2":0.625,"precision":0.55,"recall":0.6471,"tp":11,"fp":9,"fn":6,"expected":17,"prs":19,"coverage":{"graded":19,"cohort":19},"display_f1":0.5946,"latency_p50_seconds":47.6,"latency_p90_seconds":109.8,"display_latency_seconds":47.6,"cost_per_review_usd":0.0871,"cost_basis":"observed_inference","chart_cost":0.0871,"run_dates":["2026-09-21","2026-09-22"],"popovers":{"f1":{"f2":0.625,"cohort_of":19},"cost":{"estimate":false},"score":{"parts":{"f1":0,"latency":100,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":26.503,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":44.054,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":44.219,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":73.236,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":30.763,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":1,"caught":1,"seconds":52.342,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":1,"caught":0,"seconds":36.64,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":2,"caught":2,"seconds":115.962,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":1,"caught":1,"seconds":61.296,"review_url":null},{"case_id":"pandas-dev-pandas-43951","repository":"pandas","number":43951,"url":"https://github.com/pandas-dev/pandas/pull/43951","status":"judged","expected":1,"caught":1,"seconds":109.835,"review_url":null},{"case_id":"pandas-dev-pandas-45411","repository":"pandas","number":45411,"url":"https://github.com/pandas-dev/pandas/pull/45411","status":"judged","expected":1,"caught":0,"seconds":49.291,"review_url":null},{"case_id":"pandas-dev-pandas-47633","repository":"pandas","number":47633,"url":"https://github.com/pandas-dev/pandas/pull/47633","status":"judged","expected":1,"caught":1,"seconds":39.118,"review_url":null},{"case_id":"pandas-dev-pandas-57347","repository":"pandas","number":57347,"url":"https://github.com/pandas-dev/pandas/pull/57347","status":"judged","expected":1,"caught":0,"seconds":39.506,"review_url":null},{"case_id":"pandas-dev-pandas-61032","repository":"pandas","number":61032,"url":"https://github.com/pandas-dev/pandas/pull/61032","status":"judged","expected":1,"caught":1,"seconds":46.965,"review_url":null},{"case_id":"pandas-dev-pandas-61264","repository":"pandas","number":61264,"url":"https://github.com/pandas-dev/pandas/pull/61264","status":"judged","expected":1,"caught":1,"seconds":47.583,"review_url":null},{"case_id":"pandas-dev-pandas-61467","repository":"pandas","number":61467,"url":"https://github.com/pandas-dev/pandas/pull/61467","status":"judged","expected":1,"caught":0,"seconds":66.449,"review_url":null},{"case_id":"pandas-dev-pandas-63384","repository":"pandas","number":63384,"url":"https://github.com/pandas-dev/pandas/pull/63384","status":"judged","expected":1,"caught":1,"seconds":69.392,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":1,"caught":1,"seconds":34.391,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":70.885,"review_url":null}]}}},{"id":"coderabbit","name":"CodeRabbit","label":"CodeRabbit","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":null,"composite_parts":{"f1":0,"latency":0,"cost":0},"f1":0.5556,"f2":0.5556,"precision":0.5556,"recall":0.5556,"tp":5,"fp":4,"fn":4,"expected":9,"prs":11,"coverage":{"graded":11,"cohort":19},"display_f1":null,"latency_p50_seconds":463,"latency_p90_seconds":610,"display_latency_seconds":null,"cost_per_review_usd":1.5,"cost_basis":"app_list_price_estimate","chart_cost":1.5,"run_dates":["2026-01-23","2026-04-05"],"popovers":{"f1":{"f2":0.5556,"cohort_of":19},"cost":{"estimate":true},"score":{"parts":{"f1":0,"latency":0,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":610,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR10600__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":463,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":566,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":367,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR79265__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":445,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR97529__20260122/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":1,"caught":0,"seconds":358,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR103633__20260122/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":1,"caught":0,"seconds":382,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR36882__20260122/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":2,"caught":2,"seconds":477,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR37429__20260122/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":1,"caught":1,"seconds":574,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR37634__20260122/pull/1"},{"case_id":"pandas-dev-pandas-43951","repository":"pandas","number":43951,"url":"https://github.com/pandas-dev/pandas/pull/43951","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-45411","repository":"pandas","number":45411,"url":"https://github.com/pandas-dev/pandas/pull/45411","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-47633","repository":"pandas","number":47633,"url":"https://github.com/pandas-dev/pandas/pull/47633","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-57347","repository":"pandas","number":57347,"url":"https://github.com/pandas-dev/pandas/pull/57347","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61032","repository":"pandas","number":61032,"url":"https://github.com/pandas-dev/pandas/pull/61032","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61264","repository":"pandas","number":61264,"url":"https://github.com/pandas-dev/pandas/pull/61264","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61467","repository":"pandas","number":61467,"url":"https://github.com/pandas-dev/pandas/pull/61467","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-63384","repository":"pandas","number":63384,"url":"https://github.com/pandas-dev/pandas/pull/63384","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":1,"caught":1,"seconds":308,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR80168__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":699,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR95633__20260122/pull/1"}]}}},{"id":"cursor-bugbot","name":"Cursor Bugbot","label":"Cursor Bugbot","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":null,"composite_parts":{"f1":0,"latency":0,"cost":0},"f1":0.3636,"f2":0.2632,"precision":1,"recall":0.2222,"tp":2,"fp":0,"fn":7,"expected":9,"prs":11,"coverage":{"graded":11,"cohort":19},"display_f1":null,"latency_p50_seconds":592.5,"latency_p90_seconds":729,"display_latency_seconds":null,"cost_per_review_usd":1.25,"cost_basis":"app_list_price_estimate","chart_cost":1.25,"run_dates":["2026-01-28","2026-01-28"],"popovers":{"f1":{"f2":0.2632,"cohort_of":19},"cost":{"estimate":true},"score":{"parts":{"f1":0,"latency":0,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":701,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR10600__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":null,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":314,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":0,"seconds":698,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR79265__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":594,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR97529__20260122/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":1,"caught":0,"seconds":730,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR103633__20260122/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":1,"caught":0,"seconds":591,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR36882__20260122/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":2,"caught":2,"seconds":165,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR37429__20260122/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":1,"caught":0,"seconds":406,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR37634__20260122/pull/1"},{"case_id":"pandas-dev-pandas-43951","repository":"pandas","number":43951,"url":"https://github.com/pandas-dev/pandas/pull/43951","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-45411","repository":"pandas","number":45411,"url":"https://github.com/pandas-dev/pandas/pull/45411","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-47633","repository":"pandas","number":47633,"url":"https://github.com/pandas-dev/pandas/pull/47633","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-57347","repository":"pandas","number":57347,"url":"https://github.com/pandas-dev/pandas/pull/57347","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61032","repository":"pandas","number":61032,"url":"https://github.com/pandas-dev/pandas/pull/61032","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61264","repository":"pandas","number":61264,"url":"https://github.com/pandas-dev/pandas/pull/61264","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61467","repository":"pandas","number":61467,"url":"https://github.com/pandas-dev/pandas/pull/61467","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-63384","repository":"pandas","number":63384,"url":"https://github.com/pandas-dev/pandas/pull/63384","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":1,"caught":0,"seconds":417,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR80168__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":729,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR95633__20260122/pull/1"}]}}},{"id":"greptile-v4-1","name":"Greptile v4.1","label":"Greptile v4.1","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":null,"composite_parts":{"f1":0,"latency":0,"cost":0},"f1":0.2857,"f2":0.2439,"precision":0.4,"recall":0.2222,"tp":2,"fp":3,"fn":7,"expected":9,"prs":11,"coverage":{"graded":11,"cohort":19},"display_f1":null,"latency_p50_seconds":323,"latency_p90_seconds":351,"display_latency_seconds":null,"cost_per_review_usd":1.75,"cost_basis":"app_list_price_estimate","chart_cost":1.75,"run_dates":["2026-04-07","2026-04-07"],"popovers":{"f1":{"f2":0.2439,"cohort_of":19},"cost":{"estimate":true},"score":{"parts":{"f1":0,"latency":0,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":351,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10600__20260406/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":302,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22532__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":344,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR10__20260405/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":267,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR79265__20260405/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":345,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR97529__20260405/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":1,"caught":0,"seconds":228,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR103633__20260405/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":1,"caught":0,"seconds":289,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36882__20260406/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":2,"caught":0,"seconds":725,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37429__20260406/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":1,"caught":0,"seconds":337,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37634__20260406/pull/1"},{"case_id":"pandas-dev-pandas-43951","repository":"pandas","number":43951,"url":"https://github.com/pandas-dev/pandas/pull/43951","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-45411","repository":"pandas","number":45411,"url":"https://github.com/pandas-dev/pandas/pull/45411","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-47633","repository":"pandas","number":47633,"url":"https://github.com/pandas-dev/pandas/pull/47633","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-57347","repository":"pandas","number":57347,"url":"https://github.com/pandas-dev/pandas/pull/57347","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61032","repository":"pandas","number":61032,"url":"https://github.com/pandas-dev/pandas/pull/61032","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61264","repository":"pandas","number":61264,"url":"https://github.com/pandas-dev/pandas/pull/61264","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61467","repository":"pandas","number":61467,"url":"https://github.com/pandas-dev/pandas/pull/61467","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-63384","repository":"pandas","number":63384,"url":"https://github.com/pandas-dev/pandas/pull/63384","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":1,"caught":1,"seconds":323,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80168__20260406/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":309,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR95633__20260406/pull/1"}]}}},{"id":"qodo-v2","name":"Qodo v2","label":"Qodo v2","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":null,"composite_parts":{"f1":0,"latency":0,"cost":0},"f1":0.2667,"f2":0.2381,"precision":0.3333,"recall":0.2222,"tp":2,"fp":4,"fn":7,"expected":9,"prs":11,"coverage":{"graded":11,"cohort":19},"display_f1":null,"latency_p50_seconds":389,"latency_p90_seconds":573,"display_latency_seconds":null,"cost_per_review_usd":1.6667,"cost_basis":"app_list_price_estimate","chart_cost":1.6667,"run_dates":["2026-02-27","2026-02-27"],"popovers":{"f1":{"f2":0.2381,"cohort_of":19},"cost":{"estimate":true},"score":{"parts":{"f1":0,"latency":0,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":436,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR10600__20260227/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":321,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22532__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":449,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR10__20260227/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":389,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR79265__20260227/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":301,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR97529__20260227/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":1,"caught":1,"seconds":410,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR103633__20260227/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":1,"caught":0,"seconds":573,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR36882__20260227/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":2,"caught":0,"seconds":588,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR37429__20260227/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":1,"caught":0,"seconds":318,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR37634__20260227/pull/1"},{"case_id":"pandas-dev-pandas-43951","repository":"pandas","number":43951,"url":"https://github.com/pandas-dev/pandas/pull/43951","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-45411","repository":"pandas","number":45411,"url":"https://github.com/pandas-dev/pandas/pull/45411","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-47633","repository":"pandas","number":47633,"url":"https://github.com/pandas-dev/pandas/pull/47633","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-57347","repository":"pandas","number":57347,"url":"https://github.com/pandas-dev/pandas/pull/57347","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61032","repository":"pandas","number":61032,"url":"https://github.com/pandas-dev/pandas/pull/61032","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61264","repository":"pandas","number":61264,"url":"https://github.com/pandas-dev/pandas/pull/61264","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-61467","repository":"pandas","number":61467,"url":"https://github.com/pandas-dev/pandas/pull/61467","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-63384","repository":"pandas","number":63384,"url":"https://github.com/pandas-dev/pandas/pull/63384","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":1,"caught":0,"seconds":196,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR80168__20260227/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":377,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR95633__20260227/pull/1"}]}}}]}