{"family":{"id":"ionwarp-api-performance","key":"performance-review","agent":"api","name":"API Performance","slug":"performance"},"snapshot":{"id":"onerow-performance-2026-09-24-r1","captured_at":"2026-09-24T18:23:02.023172+00:00","status":"methodology_review","source_revision_url":"https://github.com/Winding-Labs/ai-code-review-benchmark"},"release":{"id":"2026-10-12-public-v1","date":"2026-10-12"},"publication":{"protocol":"category-scoped-v4-all-golden-match","status":"methodology_review","label":"Provisional scoped-label matching","ranking_allowed":false,"blockers":["operator_labels_not_official_corpus","generalist_output_rescored"],"methodology_url":"https://ionwarp.com/benchmarks/methodology"},"judge":"IonWarp adapted category-scoped judge, Claude Opus 4.5","scope":"performance specialist · fresh local pilot · provisional semantic matching","case_count":18,"issue_count":13,"score_definition":"Reviewer Score = (3 × F1 + latency + $/run) ÷ 5. Each part is this row's place between the worst (0) and best (100) full-cohort row in this family — higher F1, lower latency and lower $/run score higher. Scored once F1, latency and $/run are all measured and at least 3 rows are.","ranking_basis":"Sorted by Reviewer Score (F1 counts 3×, time and cost 1× each).","scenario":{"seats":5,"monthly_reviews":100},"configured_pin":{"model":"deepseek/deepseek-v4.1-flash","reasoning_effort":"medium","retry":{"model":"meta/muse-spark-1.3","reasoning_effort":"medium"}},"warnings":["Local reviewer execution; no hosted queue or GitHub delivery measurement.","One trial per declared configuration and PR; exploratory comparison.","Pinned source issue matching is not independent finding adjudication.","108 cells are admitted unchanged from the experiments that paid for them (identical case identity, engine and output ceiling, on the runtime of their configuration); they are never re-bought.","Contains operator-labelled PRs outside the official Martian corpus. These results are not an official-corpus leaderboard score.","Precision is golden-issue matching precision, not independently adjudicated finding correctness.","Competitor outputs are historical; fresh IonWarp runs use the same PR heads and new semantic grading.","Available-evidence checkpoint: unstarted jobs and completed but ungraded outputs remain explicit missing coverage. Scores use only saved grades under the named protocol.","The saved grade files preserve all source rows. This checkpoint projects only its declared case/configuration identities; unselected rows are not pooled into displayed scores."],"notices":["Measured on the reviewer prompt of 2026-09-18; production has changed it since"],"not_measured":[],"license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"cost_unit":"review_run","case_ids":["calcom-cal.com-10600","ai-code-review-evaluation-discourse-graphite-9","ai-code-review-evaluation-discourse-graphite-10","grafana-grafana-80329","grafana-grafana-90939","grafana-grafana-97529","keycloak-keycloak-33832","keycloak-keycloak-36882","keycloak-keycloak-37634","netbox-community-netbox-15141","pandas-dev-pandas-50331","pandas-dev-pandas-65056","getsentry-sentry-11908","getsentry-sentry-40636","getsentry-sentry-45792","getsentry-sentry-80168","getsentry-sentry-93824","getsentry-sentry-95633"],"benchmark_default":"ionwarp-deepseek-v4-1-flash","benefits":{"row_id":"ionwarp-deepseek-v4-1-flash","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","rivals":0,"wins":[],"cost_line":"about 4¢ a review","basis":"No review app was measured on all 18 PRs of this lane, so there is no comparison yet."},"rows":[{"id":"ionwarp-muse-spark-1-3","name":"IonWarp · Muse Spark 1.3","label":"IonWarp · Muse Spark 1.3","kind":"ionwarp","model":"meta/muse-spark-1.3","model_label":"Muse Spark 1.3","developer":"meta","effort":"medium","composite":77.8,"composite_parts":{"f1":100,"latency":84,"cost":5},"f1":0.56,"f2":0.5469,"precision":0.5833,"recall":0.5385,"tp":7,"fp":5,"fn":6,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.56,"latency_p50_seconds":41.8,"latency_p90_seconds":91.2,"display_latency_seconds":41.8,"cost_per_review_usd":0.0627,"cost_basis":"observed_inference","chart_cost":0.0627,"run_dates":["2026-09-19","2026-09-21"],"popovers":{"f1":{"f2":0.5469,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":100,"latency":84,"cost":5}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":32.941,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":0,"seconds":41.506,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":35.485,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":1,"seconds":68.641,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":1,"seconds":43.569,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":0,"seconds":32.316,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":31.854,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":12.297,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":19.927,"review_url":null},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"judged","expected":1,"caught":0,"seconds":151.896,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"judged","expected":1,"caught":1,"seconds":41.152,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"judged","expected":1,"caught":1,"seconds":91.22,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"judged","expected":1,"caught":1,"seconds":50.291,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"judged","expected":1,"caught":1,"seconds":75.511,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"judged","expected":1,"caught":1,"seconds":48.238,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":24.747,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":0,"seconds":42.159,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":63.246,"review_url":null}]}}},{"id":"ionwarp-glm-5-3-flash","name":"IonWarp · GLM 5.3 Flash","label":"IonWarp · GLM 5.3 Flash","kind":"ionwarp","model":"z-ai/glm-5.3-flash","model_label":"GLM 5.3 Flash","developer":"z-ai","effort":"low","composite":63.4,"composite_parts":{"f1":46,"latency":78,"cost":100},"f1":0.4828,"f2":0.5147,"precision":0.4375,"recall":0.5385,"tp":7,"fp":9,"fn":6,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.4828,"latency_p50_seconds":48.7,"latency_p90_seconds":152.3,"display_latency_seconds":48.7,"cost_per_review_usd":0.0049,"cost_basis":"observed_inference","chart_cost":0.0049,"run_dates":["2026-09-19","2026-09-21"],"popovers":{"f1":{"f2":0.5147,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":46,"latency":78,"cost":100}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":175.525,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":0,"seconds":18.281,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":92.605,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":1,"seconds":39.214,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":0,"seconds":12.69,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":82.048,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":22.7,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":14.19,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":32.232,"review_url":null},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"judged","expected":1,"caught":0,"seconds":92.985,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"judged","expected":1,"caught":1,"seconds":44.456,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"judged","expected":1,"caught":1,"seconds":55.808,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"judged","expected":1,"caught":1,"seconds":42.499,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"judged","expected":1,"caught":1,"seconds":49.019,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"judged","expected":1,"caught":1,"seconds":49.731,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":48.373,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":0,"seconds":152.343,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":137.175,"review_url":null}]}}},{"id":"greptile-v4-1","name":"Greptile v4.1","label":"Greptile v4.1","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":null,"composite_parts":{"f1":100,"latency":10,"cost":0},"f1":0.7692,"f2":0.7353,"precision":0.8333,"recall":0.7143,"tp":5,"fp":1,"fn":2,"expected":7,"prs":12,"coverage":{"graded":12,"cohort":18},"display_f1":null,"latency_p50_seconds":299,"latency_p90_seconds":345,"display_latency_seconds":null,"cost_per_review_usd":1.75,"cost_basis":"app_list_price_estimate","chart_cost":1.75,"run_dates":["2026-04-07","2026-04-07"],"popovers":{"f1":{"f2":0.7353,"cohort_of":18},"cost":{"estimate":true},"score":{"parts":{"f1":100,"latency":10,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":1,"seconds":351,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10600__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":0,"seconds":247,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR9__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":344,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR10__20260405/pull/1"},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":1,"seconds":230,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR80329__20260405/pull/1"},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":1,"seconds":144,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR90939__20260405/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":345,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR97529__20260405/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":244,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR33832__20260406/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":289,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36882__20260406/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":337,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37634__20260406/pull/1"},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":323,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80168__20260406/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":1,"seconds":176,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR93824__20260406/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":309,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR95633__20260406/pull/1"}]}}},{"id":"ionwarp-gpt-5-6-luna-pro","name":"IonWarp · GPT 5.6 Luna Pro","label":"IonWarp · GPT 5.6 Luna Pro","kind":"ionwarp","model":"openai/gpt-5.6-luna-pro","model_label":"GPT 5.6 Luna Pro","developer":"openai","effort":"medium","composite":61.3,"composite_parts":{"f1":58,"latency":100,"cost":32},"f1":0.5,"f2":0.4762,"precision":0.5455,"recall":0.4615,"tp":6,"fp":5,"fn":7,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.5,"latency_p50_seconds":27.5,"latency_p90_seconds":43.8,"display_latency_seconds":27.5,"cost_per_review_usd":0.0302,"cost_basis":"observed_inference","chart_cost":0.0302,"run_dates":["2026-09-19","2026-09-22"],"popovers":{"f1":{"f2":0.4762,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":58,"latency":100,"cost":32}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":21.353,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":0,"seconds":22.573,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":27.983,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":1,"seconds":43.755,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":1,"seconds":27.324,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":0,"seconds":25.844,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":24.221,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":13.012,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":19.592,"review_url":null},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"judged","expected":1,"caught":0,"seconds":77.073,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"judged","expected":1,"caught":1,"seconds":27.557,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"judged","expected":1,"caught":0,"seconds":30.608,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"judged","expected":1,"caught":1,"seconds":31.976,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"judged","expected":1,"caught":1,"seconds":38.888,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"judged","expected":1,"caught":1,"seconds":36.519,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":27.517,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":0,"seconds":37.431,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":25.625,"review_url":null}]}}},{"id":"qodo-v2","name":"Qodo v2","label":"Qodo v2","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":null,"composite_parts":{"f1":100,"latency":5,"cost":0},"f1":0.6667,"f2":0.7692,"precision":0.5455,"recall":0.8571,"tp":6,"fp":5,"fn":1,"expected":7,"prs":12,"coverage":{"graded":12,"cohort":18},"display_f1":null,"latency_p50_seconds":343,"latency_p90_seconds":449,"display_latency_seconds":null,"cost_per_review_usd":1.6667,"cost_basis":"app_list_price_estimate","chart_cost":1.6667,"run_dates":["2026-02-27","2026-02-27"],"popovers":{"f1":{"f2":0.7692,"cohort_of":18},"cost":{"estimate":true},"score":{"parts":{"f1":100,"latency":5,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":1,"seconds":436,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR10600__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":1,"seconds":272,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR9__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":449,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR10__20260227/pull/1"},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":1,"seconds":421,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR80329__20260227/pull/1"},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":1,"seconds":195,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR90939__20260227/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":301,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR97529__20260227/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":349,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR33832__20260227/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":573,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR36882__20260227/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":318,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR37634__20260227/pull/1"},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":196,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR80168__20260227/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":1,"seconds":337,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR93824__20260227/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":377,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR95633__20260227/pull/1"}]}}},{"id":"cursor-bugbot","name":"Cursor Bugbot","label":"Cursor Bugbot","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":null,"composite_parts":{"f1":100,"latency":0,"cost":0},"f1":0.7143,"f2":0.7143,"precision":0.7143,"recall":0.7143,"tp":5,"fp":2,"fn":2,"expected":7,"prs":12,"coverage":{"graded":12,"cohort":18},"display_f1":null,"latency_p50_seconds":476.5,"latency_p90_seconds":701,"display_latency_seconds":null,"cost_per_review_usd":1.25,"cost_basis":"app_list_price_estimate","chart_cost":1.25,"run_dates":["2026-01-28","2026-01-28"],"popovers":{"f1":{"f2":0.7143,"cohort_of":18},"cost":{"estimate":true},"score":{"parts":{"f1":100,"latency":0,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":701,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR10600__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":1,"seconds":444,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR9__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":314,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":1,"seconds":509,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR80329__20260122/pull/1"},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":1,"seconds":208,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR90939__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":594,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR97529__20260122/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":349,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR33832__20260122/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":591,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR36882__20260122/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":406,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR37634__20260122/pull/1"},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":417,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR80168__20260122/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":1,"seconds":604,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR93824__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":729,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR95633__20260122/pull/1"}]}}},{"id":"ionwarp-deepseek-v4-pro","name":"IonWarp · DeepSeek V4 Pro","label":"IonWarp · DeepSeek V4 Pro","kind":"ionwarp","model":"deepseek/deepseek-v4-pro-0813","model_label":"DeepSeek V4 Pro","developer":"deepseek","effort":"medium","composite":59.7,"composite_parts":{"f1":73,"latency":59,"cost":20},"f1":0.5217,"f2":0.4839,"precision":0.6,"recall":0.4615,"tp":6,"fp":4,"fn":7,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.5217,"latency_p50_seconds":81.1,"latency_p90_seconds":454.5,"display_latency_seconds":81.1,"cost_per_review_usd":0.0421,"cost_basis":"observed_inference","chart_cost":0.0421,"run_dates":["2026-09-19","2026-09-21"],"popovers":{"f1":{"f2":0.4839,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":73,"latency":59,"cost":20}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":25.803,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":0,"seconds":51.831,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":61.821,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":0,"seconds":78.944,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":1,"seconds":41.556,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":83.237,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":62.712,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":16.787,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":33.792,"review_url":null},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"judged","expected":1,"caught":0,"seconds":454.507,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"judged","expected":1,"caught":1,"seconds":132.845,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"judged","expected":1,"caught":0,"seconds":106.023,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"judged","expected":1,"caught":1,"seconds":480.063,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"judged","expected":1,"caught":1,"seconds":432.219,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"judged","expected":1,"caught":1,"seconds":175.624,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":45.544,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":0,"seconds":104.988,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":126.913,"review_url":null}]}}},{"id":"ionwarp-deepseek-v4-1-flash","name":"IonWarp · DeepSeek V4.1 Flash","label":"IonWarp · DeepSeek V4.1 Flash","kind":"ionwarp","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","developer":"deepseek","effort":"medium","composite":52.6,"composite_parts":{"f1":81,"latency":0,"cost":19},"f1":0.5333,"f2":0.5797,"precision":0.4706,"recall":0.6154,"tp":8,"fp":9,"fn":5,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.5333,"latency_p50_seconds":386.6,"latency_p90_seconds":998,"display_latency_seconds":386.6,"cost_per_review_usd":0.043,"cost_basis":"observed_inference","chart_cost":0.043,"run_dates":["2026-09-24","2026-09-24"],"popovers":{"f1":{"f2":0.5797,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":81,"latency":0,"cost":19}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":292.537,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":0,"seconds":997.96,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":143.642,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":1,"seconds":487.257,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":1,"seconds":108.922,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":418.677,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":123.845,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":133.461,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":730.82,"review_url":null},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"judged","expected":1,"caught":0,"seconds":1089.662,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"judged","expected":1,"caught":1,"seconds":125.904,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"judged","expected":1,"caught":1,"seconds":504.958,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"judged","expected":1,"caught":1,"seconds":237.851,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"judged","expected":1,"caught":1,"seconds":450.162,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"judged","expected":1,"caught":1,"seconds":172.849,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":354.446,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":0,"seconds":625.207,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":622.139,"review_url":null}]}}},{"id":"ionwarp-gemini-3-8-flash","name":"IonWarp · Gemini 3.8 Flash","label":"IonWarp · Gemini 3.8 Flash","kind":"ionwarp","model":"google/gemini-3.8-flash","model_label":"Gemini 3.8 Flash","developer":"google","effort":"medium","composite":15.9,"composite_parts":{"f1":0,"latency":80,"cost":0},"f1":0.4167,"f2":0.3968,"precision":0.4545,"recall":0.3846,"tp":5,"fp":6,"fn":8,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.4167,"latency_p50_seconds":47.2,"latency_p90_seconds":135.3,"display_latency_seconds":47.2,"cost_per_review_usd":0.0712,"cost_basis":"observed_inference","chart_cost":0.0712,"run_dates":["2026-09-21","2026-09-22"],"popovers":{"f1":{"f2":0.3968,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":0,"latency":80,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":20.99,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":0,"seconds":84.986,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":135.317,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":1,"seconds":76.707,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":0,"seconds":41.083,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":0,"seconds":44.665,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":25.694,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":10.207,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":34.818,"review_url":null},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"judged","expected":1,"caught":0,"seconds":178.583,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"judged","expected":1,"caught":1,"seconds":33.364,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"judged","expected":1,"caught":0,"seconds":39.544,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"judged","expected":1,"caught":1,"seconds":49.831,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"judged","expected":1,"caught":1,"seconds":74.451,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"judged","expected":1,"caught":1,"seconds":67.231,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":29.714,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":0,"seconds":109.4,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":123.217,"review_url":null}]}}},{"id":"coderabbit","name":"CodeRabbit","label":"CodeRabbit","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":null,"composite_parts":{"f1":19,"latency":3,"cost":0},"f1":0.4444,"f2":0.3333,"precision":1,"recall":0.2857,"tp":2,"fp":0,"fn":5,"expected":7,"prs":12,"coverage":{"graded":12,"cohort":18},"display_f1":null,"latency_p50_seconds":360,"latency_p90_seconds":610,"display_latency_seconds":null,"cost_per_review_usd":1.5,"cost_basis":"app_list_price_estimate","chart_cost":1.5,"run_dates":["2026-01-23","2026-04-05"],"popovers":{"f1":{"f2":0.3333,"cohort_of":18},"cost":{"estimate":true},"score":{"parts":{"f1":19,"latency":3,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":610,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR10600__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":1,"caught":0,"seconds":338,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR9__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":566,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":1,"caught":0,"seconds":310,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR80329__20260122/pull/1"},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":1,"caught":1,"seconds":320,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR90939__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":445,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR97529__20260122/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":293,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR33832__20260122/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":0,"caught":0,"seconds":382,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR36882__20260122/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":0,"caught":0,"seconds":574,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR37634__20260122/pull/1"},{"case_id":"netbox-community-netbox-15141","repository":"netbox","number":15141,"url":"https://github.com/netbox-community/netbox/pull/15141","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-50331","repository":"pandas","number":50331,"url":"https://github.com/pandas-dev/pandas/pull/50331","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"pandas-dev-pandas-65056","repository":"pandas","number":65056,"url":"https://github.com/pandas-dev/pandas/pull/65056","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-11908","repository":"sentry","number":11908,"url":"https://github.com/getsentry/sentry/pull/11908","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-40636","repository":"sentry","number":40636,"url":"https://github.com/getsentry/sentry/pull/40636","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-45792","repository":"sentry","number":45792,"url":"https://github.com/getsentry/sentry/pull/45792","status":"not_started","expected":1,"caught":null,"seconds":null,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":0,"caught":0,"seconds":308,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR80168__20260122/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":2,"caught":0,"seconds":288,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR93824__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":699,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR95633__20260122/pull/1"}]}}}]}