{"family":{"id":"ionwarp-code-review","key":"code-review","agent":"code","name":"Code Review","slug":"code"},"snapshot":{"id":"onerow4-code-2026-10-06-r1","captured_at":"2026-10-06T19:35:11.983317+00:00","status":"pinned_method_reproduction","source_revision_url":"https://github.com/Winding-Labs/ai-code-review-benchmark"},"release":{"id":"2026-10-12-public-v1","date":"2026-10-12"},"publication":{"protocol":"martian-core-e616e849-v1","status":"pinned_method_reproduction","label":"Pinned Martian Core reproduction","ranking_allowed":true,"blockers":[],"methodology_url":"https://ionwarp.com/benchmarks/methodology"},"judge":"Martian Core, Claude Opus 4.5 judge","scope":"code reviewer · pinned Martian Core issues · not specialist-specific accuracy","case_count":50,"issue_count":158,"score_definition":"Reviewer Score = (3 × F1 + latency + $/run) ÷ 5. Each part is this row's place between the worst (0) and best (100) full-cohort row in this family — higher F1, lower latency and lower $/run score higher. Scored once F1, latency and $/run are all measured and at least 3 rows are.","ranking_basis":"Sorted by Reviewer Score (F1 counts 3×, time and cost 1× each).","scenario":{"seats":5,"monthly_reviews":100},"configured_pin":{"model":"deepseek/deepseek-v4.1-flash","reasoning_effort":"medium","retry":{"model":"meta/muse-spark-1.3","reasoning_effort":"medium"}},"warnings":["Local reviewer execution; no hosted queue or GitHub delivery measurement.","One trial per declared configuration and PR; exploratory comparison.","Pinned source issue matching is not independent finding adjudication.","250 cells are admitted unchanged from the experiments that paid for them (identical case identity, engine and output ceiling, on the runtime of their configuration); they are never re-bought.","Precision is golden-issue matching precision, not independently adjudicated finding correctness.","Competitor outputs are historical; fresh IonWarp runs use the same PR heads and new semantic grading.","Available-evidence checkpoint: unstarted jobs and completed but ungraded outputs remain explicit missing coverage. Scores use only saved grades under the named protocol.","1 provider calls have unavailable billing records. Their full $0.80061680 reservations remain in budget exposure. Reported spend is settled spend only; affected review costs remain null and cannot support cost comparisons or routing recommendations.","The saved grade files preserve all source rows. This checkpoint projects only its declared case/configuration identities; unselected rows are not pooled into displayed scores."],"notices":["Measured on the reviewer prompt of 2026-09-07; production has changed it since"],"not_measured":[],"license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"cost_unit":"review_run","case_ids":["calcom-cal.com-7232","calcom-cal.com-8087","calcom-cal.com-8330","calcom-cal.com-10600","calcom-cal.com-10967","calcom-cal.com-11059","calcom-cal.com-14740","calcom-cal.com-14943","calcom-cal.com-22345","calcom-cal.com-22532","ai-code-review-evaluation-discourse-graphite-1","ai-code-review-evaluation-discourse-graphite-2","ai-code-review-evaluation-discourse-graphite-3","ai-code-review-evaluation-discourse-graphite-4","ai-code-review-evaluation-discourse-graphite-5","ai-code-review-evaluation-discourse-graphite-6","ai-code-review-evaluation-discourse-graphite-7","ai-code-review-evaluation-discourse-graphite-8","ai-code-review-evaluation-discourse-graphite-9","ai-code-review-evaluation-discourse-graphite-10","grafana-grafana-76186","grafana-grafana-79265","grafana-grafana-80329","grafana-grafana-90045","grafana-grafana-90939","grafana-grafana-94942","grafana-grafana-97529","grafana-grafana-103633","grafana-grafana-106778","grafana-grafana-107534","keycloak-keycloak-32918","keycloak-keycloak-33832","keycloak-keycloak-36880","keycloak-keycloak-36882","keycloak-keycloak-37038","keycloak-keycloak-37429","keycloak-keycloak-37634","keycloak-keycloak-38446","keycloak-keycloak-40940","ai-code-review-evaluation-keycloak-greptile-1","getsentry-sentry-67876","getsentry-sentry-77754","getsentry-sentry-80168","getsentry-sentry-80528","getsentry-sentry-93824","getsentry-sentry-95633","ai-code-review-evaluation-sentry-greptile-1","ai-code-review-evaluation-sentry-greptile-2","ai-code-review-evaluation-sentry-greptile-3","ai-code-review-evaluation-sentry-greptile-5"],"benchmark_default":"ionwarp-deepseek-v4-1-flash","benefits":{"row_id":"ionwarp-deepseek-v4-1-flash","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","rivals":4,"wins":[{"metric":"cost","percent":69,"text":"69% cheaper"}],"cost_line":"about 10¢ a review","basis":"DeepSeek V4.1 Flash against the median of 4 review apps on the same 50 PRs. Accuracy is F1, wait is median review time, cost is Pro ($49 a month, 5 seats) at 100 reviews a month against each app's list price for the same team."},"rows":[{"id":"ionwarp-muse-spark-1-3","name":"IonWarp · Muse Spark 1.3","label":"IonWarp · Muse Spark 1.3","kind":"ionwarp","model":"meta/muse-spark-1.3","model_label":"Muse Spark 1.3","developer":"meta","effort":"medium","composite":71.2,"composite_parts":{"f1":89,"latency":58,"cost":31},"f1":0.5663,"f2":0.5246,"precision":0.6529,"recall":0.5,"tp":79,"fp":42,"fn":79,"expected":158,"prs":50,"coverage":{"graded":50,"cohort":50},"display_f1":0.5663,"latency_p50_seconds":198.4,"latency_p90_seconds":451.2,"display_latency_seconds":198.4,"cost_per_review_usd":0.4242,"cost_basis":"observed_inference","chart_cost":0.4242,"run_dates":["2026-09-08","2026-09-11"],"popovers":{"f1":{"f2":0.5246,"cohort_of":50},"cost":{"estimate":false},"score":{"parts":{"f1":89,"latency":58,"cost":31}},"prs":{"cases":[{"case_id":"calcom-cal.com-7232","repository":"cal.com","number":7232,"url":"https://github.com/calcom/cal.com/pull/7232","status":"judged","expected":3,"caught":2,"seconds":287.893,"review_url":null},{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":158.507,"review_url":null},{"case_id":"calcom-cal.com-8330","repository":"cal.com","number":8330,"url":"https://github.com/calcom/cal.com/pull/8330","status":"judged","expected":2,"caught":2,"seconds":153.174,"review_url":null},{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":4,"caught":0,"seconds":412.267,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":5,"caught":2,"seconds":116.648,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":9,"caught":6,"seconds":353.971,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":6,"caught":3,"seconds":204.099,"review_url":null},{"case_id":"calcom-cal.com-14943","repository":"cal.com","number":14943,"url":"https://github.com/calcom/cal.com/pull/14943","status":"judged","expected":2,"caught":2,"seconds":134.072,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":139.135,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":4,"caught":2,"seconds":110.02,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":4,"caught":2,"seconds":194.906,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-2","repository":"discourse-graphite","number":2,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2","status":"judged","expected":1,"caught":1,"seconds":242.452,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":3,"caught":2,"seconds":230.317,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":8,"caught":2,"seconds":201.951,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-5","repository":"discourse-graphite","number":5,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5","status":"judged","expected":3,"caught":3,"seconds":108.126,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-6","repository":"discourse-graphite","number":6,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6","status":"judged","expected":1,"caught":1,"seconds":208.211,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-7","repository":"discourse-graphite","number":7,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7","status":"judged","expected":3,"caught":3,"seconds":170.547,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-8","repository":"discourse-graphite","number":8,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8","status":"judged","expected":6,"caught":3,"seconds":228.079,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":2,"caught":1,"seconds":83.993,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":7,"caught":3,"seconds":164.018,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":451.168,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":5,"caught":2,"seconds":278.303,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":2,"caught":1,"seconds":183.805,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":3,"seconds":292.464,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":2,"caught":1,"seconds":129.669,"review_url":null},{"case_id":"grafana-grafana-94942","repository":"grafana","number":94942,"url":"https://github.com/grafana/grafana/pull/94942","status":"judged","expected":2,"caught":1,"seconds":116.14,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":2,"caught":1,"seconds":98.118,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":3,"caught":0,"seconds":210.861,"review_url":null},{"case_id":"grafana-grafana-106778","repository":"grafana","number":106778,"url":"https://github.com/grafana/grafana/pull/106778","status":"judged","expected":2,"caught":2,"seconds":92.842,"review_url":null},{"case_id":"grafana-grafana-107534","repository":"grafana","number":107534,"url":"https://github.com/grafana/grafana/pull/107534","status":"judged","expected":1,"caught":0,"seconds":736.724,"review_url":null},{"case_id":"keycloak-keycloak-32918","repository":"keycloak","number":32918,"url":"https://github.com/keycloak/keycloak/pull/32918","status":"judged","expected":2,"caught":0,"seconds":408.503,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":3,"caught":2,"seconds":402.644,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":4,"caught":1,"seconds":1191.2,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":2,"caught":0,"seconds":339.052,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":1,"seconds":529.526,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":3,"caught":2,"seconds":500.003,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":4,"caught":2,"seconds":150.386,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":2,"caught":1,"seconds":549.224,"review_url":null},{"case_id":"keycloak-keycloak-40940","repository":"keycloak","number":40940,"url":"https://github.com/keycloak/keycloak/pull/40940","status":"judged","expected":2,"caught":2,"seconds":186.254,"review_url":null},{"case_id":"ai-code-review-evaluation-keycloak-greptile-1","repository":"keycloak-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1","status":"judged","expected":1,"caught":1,"seconds":185.58,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":3,"caught":1,"seconds":277.247,"review_url":null},{"case_id":"getsentry-sentry-77754","repository":"sentry","number":77754,"url":"https://github.com/getsentry/sentry/pull/77754","status":"judged","expected":2,"caught":1,"seconds":126.181,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":3,"caught":1,"seconds":84.408,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":2,"caught":0,"seconds":92.28,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":5,"caught":1,"seconds":178.688,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":432.368,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":2,"seconds":146.49,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":4,"caught":3,"seconds":310.786,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-3","repository":"sentry-greptile","number":3,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3","status":"judged","expected":3,"caught":2,"seconds":208.746,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-5","repository":"sentry-greptile","number":5,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5","status":"judged","expected":6,"caught":3,"seconds":192.709,"review_url":null}]}}},{"id":"qodo-v2","name":"Qodo v2","label":"Qodo v2","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":64.7,"composite_parts":{"f1":100,"latency":22,"cost":1},"f1":0.5851,"f2":0.6057,"precision":0.5537,"recall":0.6203,"tp":98,"fp":79,"fn":60,"expected":158,"prs":50,"coverage":{"graded":50,"cohort":50},"display_f1":0.5851,"latency_p50_seconds":339.5,"latency_p90_seconds":490,"display_latency_seconds":339.5,"cost_per_review_usd":1.6667,"cost_basis":"app_list_price_estimate","chart_cost":1.6667,"run_dates":["2026-02-27","2026-02-27"],"popovers":{"f1":{"f2":0.6057,"cohort_of":50},"cost":{"estimate":true},"score":{"parts":{"f1":100,"latency":22,"cost":1}},"prs":{"cases":[{"case_id":"calcom-cal.com-7232","repository":"cal.com","number":7232,"url":"https://github.com/calcom/cal.com/pull/7232","status":"judged","expected":3,"caught":2,"seconds":272,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR7232__20260227/pull/1"},{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":222,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR8087__20260227/pull/1"},{"case_id":"calcom-cal.com-8330","repository":"cal.com","number":8330,"url":"https://github.com/calcom/cal.com/pull/8330","status":"judged","expected":2,"caught":2,"seconds":342,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR8330__20260227/pull/1"},{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":4,"caught":2,"seconds":436,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR10600__20260227/pull/1"},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":5,"caught":2,"seconds":384,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR10967__20260227/pull/1"},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":9,"caught":8,"seconds":321,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR11059__20260227/pull/1"},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":6,"caught":5,"seconds":274,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR14740__20260227/pull/1"},{"case_id":"calcom-cal.com-14943","repository":"cal.com","number":14943,"url":"https://github.com/calcom/cal.com/pull/14943","status":"judged","expected":2,"caught":1,"seconds":223,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR14943__20260227/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":352,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22345__20260227/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":4,"caught":3,"seconds":321,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22532__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":4,"caught":2,"seconds":294,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR1__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-2","repository":"discourse-graphite","number":2,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2","status":"judged","expected":1,"caught":1,"seconds":304,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR2__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":3,"caught":2,"seconds":382,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR3__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":8,"caught":6,"seconds":462,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR4__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-5","repository":"discourse-graphite","number":5,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5","status":"judged","expected":3,"caught":1,"seconds":276,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR5__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-6","repository":"discourse-graphite","number":6,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6","status":"judged","expected":1,"caught":1,"seconds":286,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR6__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-7","repository":"discourse-graphite","number":7,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7","status":"judged","expected":3,"caught":3,"seconds":225,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR7__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-8","repository":"discourse-graphite","number":8,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8","status":"judged","expected":6,"caught":2,"seconds":648,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR8__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":2,"caught":1,"seconds":272,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR9__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":7,"caught":3,"seconds":449,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR10__20260227/pull/1"},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":2,"seconds":379,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR76186__20260227/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":5,"caught":3,"seconds":389,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR79265__20260227/pull/1"},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":2,"caught":2,"seconds":421,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR80329__20260227/pull/1"},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":2,"seconds":406,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR90045__20260227/pull/1"},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":2,"caught":1,"seconds":195,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR90939__20260227/pull/1"},{"case_id":"grafana-grafana-94942","repository":"grafana","number":94942,"url":"https://github.com/grafana/grafana/pull/94942","status":"judged","expected":2,"caught":2,"seconds":264,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR94942__20260227/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":2,"caught":2,"seconds":301,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR97529__20260227/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":3,"caught":2,"seconds":410,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR103633__20260227/pull/1"},{"case_id":"grafana-grafana-106778","repository":"grafana","number":106778,"url":"https://github.com/grafana/grafana/pull/106778","status":"judged","expected":2,"caught":0,"seconds":465,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR106778__20260227/pull/1"},{"case_id":"grafana-grafana-107534","repository":"grafana","number":107534,"url":"https://github.com/grafana/grafana/pull/107534","status":"judged","expected":1,"caught":0,"seconds":490,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR107534__20260227/pull/1"},{"case_id":"keycloak-keycloak-32918","repository":"keycloak","number":32918,"url":"https://github.com/keycloak/keycloak/pull/32918","status":"judged","expected":2,"caught":1,"seconds":336,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR32918__20260227/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":3,"caught":1,"seconds":349,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR33832__20260227/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":4,"caught":2,"seconds":647,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR36880__20260227/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":2,"caught":1,"seconds":573,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR36882__20260227/pull/1"},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":499,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR37038__20260227/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":3,"caught":0,"seconds":588,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR37429__20260227/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":4,"caught":2,"seconds":318,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR37634__20260227/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":2,"caught":2,"seconds":449,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR38446__20260227/pull/1"},{"case_id":"keycloak-keycloak-40940","repository":"keycloak","number":40940,"url":"https://github.com/keycloak/keycloak/pull/40940","status":"judged","expected":2,"caught":2,"seconds":196,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR40940__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-keycloak-greptile-1","repository":"keycloak-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1","status":"judged","expected":1,"caught":1,"seconds":307,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak-greptile__qodo-v2__PR1__20260227/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":3,"caught":2,"seconds":406,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR67876__20260227/pull/1"},{"case_id":"getsentry-sentry-77754","repository":"sentry","number":77754,"url":"https://github.com/getsentry/sentry/pull/77754","status":"judged","expected":2,"caught":1,"seconds":232,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR77754__20260227/pull/1"},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":3,"caught":1,"seconds":196,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR80168__20260227/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":2,"caught":1,"seconds":218,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR80528__20260227/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":5,"caught":4,"seconds":337,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR93824__20260227/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":377,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR95633__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":3,"seconds":267,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo-v2__PR1__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":4,"caught":4,"seconds":358,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo-v2__PR2__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-3","repository":"sentry-greptile","number":3,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3","status":"judged","expected":3,"caught":2,"seconds":358,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo-v2__PR3__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-5","repository":"sentry-greptile","number":5,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5","status":"judged","expected":6,"caught":2,"seconds":266,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo-v2__PR5__20260227/pull/1"}]}}},{"id":"ionwarp-deepseek-v4-pro","name":"IonWarp · DeepSeek V4 Pro","label":"IonWarp · DeepSeek V4 Pro","kind":"ionwarp","model":"deepseek/deepseek-v4-pro-0813","model_label":"DeepSeek V4 Pro","developer":"deepseek","effort":"medium","composite":48.7,"composite_parts":{"f1":58,"latency":18,"cost":51},"f1":0.5141,"f2":0.4426,"precision":0.7033,"recall":0.4051,"tp":64,"fp":27,"fn":94,"expected":158,"prs":50,"coverage":{"graded":50,"cohort":50},"display_f1":0.5141,"latency_p50_seconds":359.4,"latency_p90_seconds":798.5,"display_latency_seconds":359.4,"cost_per_review_usd":0.1647,"cost_basis":"observed_inference","chart_cost":0.1647,"run_dates":["2026-09-11","2026-09-13"],"popovers":{"f1":{"f2":0.4426,"cohort_of":50},"cost":{"estimate":false},"score":{"parts":{"f1":58,"latency":18,"cost":51}},"prs":{"cases":[{"case_id":"calcom-cal.com-7232","repository":"cal.com","number":7232,"url":"https://github.com/calcom/cal.com/pull/7232","status":"judged","expected":3,"caught":1,"seconds":297.093,"review_url":null},{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":448.694,"review_url":null},{"case_id":"calcom-cal.com-8330","repository":"cal.com","number":8330,"url":"https://github.com/calcom/cal.com/pull/8330","status":"judged","expected":2,"caught":1,"seconds":359.905,"review_url":null},{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":4,"caught":2,"seconds":511.789,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":5,"caught":2,"seconds":275.979,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":9,"caught":5,"seconds":317.878,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":6,"caught":3,"seconds":228.349,"review_url":null},{"case_id":"calcom-cal.com-14943","repository":"cal.com","number":14943,"url":"https://github.com/calcom/cal.com/pull/14943","status":"judged","expected":2,"caught":1,"seconds":299.021,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":383.47,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":4,"caught":1,"seconds":487.241,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":4,"caught":2,"seconds":1166.746,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-2","repository":"discourse-graphite","number":2,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2","status":"judged","expected":1,"caught":1,"seconds":358.796,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":3,"caught":2,"seconds":581.219,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":8,"caught":2,"seconds":194.907,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-5","repository":"discourse-graphite","number":5,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5","status":"judged","expected":3,"caught":0,"seconds":153.957,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-6","repository":"discourse-graphite","number":6,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6","status":"judged","expected":1,"caught":1,"seconds":744.175,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-7","repository":"discourse-graphite","number":7,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7","status":"judged","expected":3,"caught":3,"seconds":381.179,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-8","repository":"discourse-graphite","number":8,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8","status":"judged","expected":6,"caught":2,"seconds":191.404,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":2,"caught":0,"seconds":590.86,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":7,"caught":1,"seconds":305.411,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":0,"seconds":681.937,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":5,"caught":1,"seconds":745.048,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":2,"caught":2,"seconds":331.665,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":2,"seconds":345.918,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":2,"caught":1,"seconds":91.47,"review_url":null},{"case_id":"grafana-grafana-94942","repository":"grafana","number":94942,"url":"https://github.com/grafana/grafana/pull/94942","status":"judged","expected":2,"caught":2,"seconds":189.76,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":2,"caught":1,"seconds":519.643,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":3,"caught":0,"seconds":280.794,"review_url":null},{"case_id":"grafana-grafana-106778","repository":"grafana","number":106778,"url":"https://github.com/grafana/grafana/pull/106778","status":"judged","expected":2,"caught":1,"seconds":197.048,"review_url":null},{"case_id":"grafana-grafana-107534","repository":"grafana","number":107534,"url":"https://github.com/grafana/grafana/pull/107534","status":"judged","expected":1,"caught":0,"seconds":731.546,"review_url":null},{"case_id":"keycloak-keycloak-32918","repository":"keycloak","number":32918,"url":"https://github.com/keycloak/keycloak/pull/32918","status":"judged","expected":2,"caught":1,"seconds":798.501,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":3,"caught":1,"seconds":353.993,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":4,"caught":2,"seconds":1860.724,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":2,"caught":0,"seconds":552.057,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":383.77,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":3,"caught":2,"seconds":1017.488,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":4,"caught":2,"seconds":325.654,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":2,"caught":0,"seconds":866.287,"review_url":null},{"case_id":"keycloak-keycloak-40940","repository":"keycloak","number":40940,"url":"https://github.com/keycloak/keycloak/pull/40940","status":"judged","expected":2,"caught":2,"seconds":454.921,"review_url":null},{"case_id":"ai-code-review-evaluation-keycloak-greptile-1","repository":"keycloak-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1","status":"judged","expected":1,"caught":1,"seconds":927.473,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":3,"caught":1,"seconds":343.498,"review_url":null},{"case_id":"getsentry-sentry-77754","repository":"sentry","number":77754,"url":"https://github.com/getsentry/sentry/pull/77754","status":"judged","expected":2,"caught":1,"seconds":261.572,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":3,"caught":1,"seconds":151.469,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":2,"caught":1,"seconds":108.99,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":5,"caught":1,"seconds":705.221,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":466.963,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":1,"seconds":235.047,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":4,"caught":3,"seconds":230.045,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-3","repository":"sentry-greptile","number":3,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3","status":"judged","expected":3,"caught":0,"seconds":551.076,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-5","repository":"sentry-greptile","number":5,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5","status":"judged","expected":6,"caught":1,"seconds":353.989,"review_url":null}]}}},{"id":"ionwarp-deepseek-v4-1-flash","name":"IonWarp · DeepSeek V4.1 Flash","label":"IonWarp · DeepSeek V4.1 Flash","kind":"ionwarp","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","developer":"deepseek","effort":"medium","composite":41,"composite_parts":{"f1":45,"latency":9,"cost":62},"f1":0.492,"f2":0.4892,"precision":0.4968,"recall":0.4873,"tp":77,"fp":78,"fn":81,"expected":158,"prs":50,"coverage":{"graded":50,"cohort":50},"display_f1":0.492,"latency_p50_seconds":415.6,"latency_p90_seconds":495.2,"display_latency_seconds":415.6,"cost_per_review_usd":0.102,"cost_basis":"observed_inference","chart_cost":0.102,"run_dates":["2026-10-05","2026-10-06"],"popovers":{"f1":{"f2":0.4892,"cohort_of":50},"cost":{"estimate":false},"score":{"parts":{"f1":45,"latency":9,"cost":62}},"prs":{"cases":[{"case_id":"calcom-cal.com-7232","repository":"cal.com","number":7232,"url":"https://github.com/calcom/cal.com/pull/7232","status":"judged","expected":3,"caught":2,"seconds":590.055,"review_url":null},{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":417.41,"review_url":null},{"case_id":"calcom-cal.com-8330","repository":"cal.com","number":8330,"url":"https://github.com/calcom/cal.com/pull/8330","status":"judged","expected":2,"caught":1,"seconds":226.554,"review_url":null},{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":4,"caught":1,"seconds":467.89,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":5,"caught":3,"seconds":460.885,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":9,"caught":5,"seconds":299.512,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":6,"caught":4,"seconds":336.319,"review_url":null},{"case_id":"calcom-cal.com-14943","repository":"cal.com","number":14943,"url":"https://github.com/calcom/cal.com/pull/14943","status":"judged","expected":2,"caught":0,"seconds":446.372,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":588.903,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":4,"caught":0,"seconds":447.711,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":4,"caught":2,"seconds":190.37,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-2","repository":"discourse-graphite","number":2,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2","status":"judged","expected":1,"caught":1,"seconds":527.298,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":3,"caught":2,"seconds":420.244,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":8,"caught":2,"seconds":466.987,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-5","repository":"discourse-graphite","number":5,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5","status":"judged","expected":3,"caught":1,"seconds":334.405,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-6","repository":"discourse-graphite","number":6,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6","status":"judged","expected":1,"caught":1,"seconds":414.935,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-7","repository":"discourse-graphite","number":7,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7","status":"judged","expected":3,"caught":3,"seconds":304.715,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-8","repository":"discourse-graphite","number":8,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8","status":"judged","expected":6,"caught":3,"seconds":476.604,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":2,"caught":0,"seconds":444.935,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":7,"caught":3,"seconds":491.346,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":334.085,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":5,"caught":2,"seconds":330.537,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":2,"caught":2,"seconds":380.853,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":0,"seconds":288.791,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":2,"caught":2,"seconds":251.117,"review_url":null},{"case_id":"grafana-grafana-94942","repository":"grafana","number":94942,"url":"https://github.com/grafana/grafana/pull/94942","status":"judged","expected":2,"caught":2,"seconds":279.684,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":2,"caught":2,"seconds":367.341,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":3,"caught":1,"seconds":402.645,"review_url":null},{"case_id":"grafana-grafana-106778","repository":"grafana","number":106778,"url":"https://github.com/grafana/grafana/pull/106778","status":"judged","expected":2,"caught":2,"seconds":434.32,"review_url":null},{"case_id":"grafana-grafana-107534","repository":"grafana","number":107534,"url":"https://github.com/grafana/grafana/pull/107534","status":"judged","expected":1,"caught":0,"seconds":416.171,"review_url":null},{"case_id":"keycloak-keycloak-32918","repository":"keycloak","number":32918,"url":"https://github.com/keycloak/keycloak/pull/32918","status":"judged","expected":2,"caught":0,"seconds":381.893,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":3,"caught":1,"seconds":570.655,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":4,"caught":2,"seconds":337.802,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":2,"caught":0,"seconds":411.017,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":434.96,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":3,"caught":2,"seconds":495.176,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":4,"caught":1,"seconds":144.994,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":2,"caught":0,"seconds":322.821,"review_url":null},{"case_id":"keycloak-keycloak-40940","repository":"keycloak","number":40940,"url":"https://github.com/keycloak/keycloak/pull/40940","status":"judged","expected":2,"caught":2,"seconds":312.757,"review_url":null},{"case_id":"ai-code-review-evaluation-keycloak-greptile-1","repository":"keycloak-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1","status":"judged","expected":1,"caught":1,"seconds":424.604,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":3,"caught":2,"seconds":430.61,"review_url":null},{"case_id":"getsentry-sentry-77754","repository":"sentry","number":77754,"url":"https://github.com/getsentry/sentry/pull/77754","status":"judged","expected":2,"caught":1,"seconds":558.886,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":3,"caught":1,"seconds":233.213,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":2,"caught":1,"seconds":294.739,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":5,"caught":3,"seconds":433.746,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":483.578,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":2,"seconds":203.484,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":4,"caught":4,"seconds":380.106,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-3","repository":"sentry-greptile","number":3,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3","status":"judged","expected":3,"caught":1,"seconds":447.775,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-5","repository":"sentry-greptile","number":5,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5","status":"judged","expected":6,"caught":2,"seconds":483.799,"review_url":null}]}}},{"id":"ionwarp-glm-5-3-flash","name":"IonWarp · GLM 5.3 Flash","label":"IonWarp · GLM 5.3 Flash","kind":"ionwarp","model":"z-ai/glm-5.3-flash","model_label":"GLM 5.3 Flash","developer":"z-ai","effort":"low","composite":40.8,"composite_parts":{"f1":7,"latency":84,"cost":100},"f1":0.4275,"f2":0.3933,"precision":0.5,"recall":0.3734,"tp":59,"fp":59,"fn":99,"expected":158,"prs":50,"coverage":{"graded":50,"cohort":50},"display_f1":0.4275,"latency_p50_seconds":135.4,"latency_p90_seconds":591.8,"display_latency_seconds":135.4,"cost_per_review_usd":0.0175,"cost_basis":"observed_inference","chart_cost":0.0175,"run_dates":["2026-09-08","2026-09-11"],"popovers":{"f1":{"f2":0.3933,"cohort_of":50},"cost":{"estimate":false},"score":{"parts":{"f1":7,"latency":84,"cost":100}},"prs":{"cases":[{"case_id":"calcom-cal.com-7232","repository":"cal.com","number":7232,"url":"https://github.com/calcom/cal.com/pull/7232","status":"judged","expected":3,"caught":2,"seconds":106.731,"review_url":null},{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":57.884,"review_url":null},{"case_id":"calcom-cal.com-8330","repository":"cal.com","number":8330,"url":"https://github.com/calcom/cal.com/pull/8330","status":"judged","expected":2,"caught":2,"seconds":134.296,"review_url":null},{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":4,"caught":1,"seconds":390.775,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":5,"caught":2,"seconds":24.801,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":9,"caught":4,"seconds":97.201,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":6,"caught":3,"seconds":702.399,"review_url":null},{"case_id":"calcom-cal.com-14943","repository":"cal.com","number":14943,"url":"https://github.com/calcom/cal.com/pull/14943","status":"judged","expected":2,"caught":0,"seconds":931.421,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":241.601,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":4,"caught":1,"seconds":38.515,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":4,"caught":2,"seconds":404.705,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-2","repository":"discourse-graphite","number":2,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2","status":"judged","expected":1,"caught":1,"seconds":66.291,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":3,"caught":2,"seconds":457.047,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":8,"caught":1,"seconds":80.179,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-5","repository":"discourse-graphite","number":5,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5","status":"judged","expected":3,"caught":0,"seconds":35.752,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-6","repository":"discourse-graphite","number":6,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6","status":"judged","expected":1,"caught":0,"seconds":59.333,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-7","repository":"discourse-graphite","number":7,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7","status":"judged","expected":3,"caught":1,"seconds":41.405,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-8","repository":"discourse-graphite","number":8,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8","status":"judged","expected":6,"caught":2,"seconds":136.508,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":2,"caught":0,"seconds":99.04,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":7,"caught":0,"seconds":934.644,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":2,"seconds":176.636,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":5,"caught":1,"seconds":167.273,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":2,"caught":2,"seconds":67.557,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":2,"seconds":120.933,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":2,"caught":1,"seconds":591.772,"review_url":null},{"case_id":"grafana-grafana-94942","repository":"grafana","number":94942,"url":"https://github.com/grafana/grafana/pull/94942","status":"judged","expected":2,"caught":2,"seconds":396.067,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":2,"caught":1,"seconds":116.312,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":3,"caught":1,"seconds":109.639,"review_url":null},{"case_id":"grafana-grafana-106778","repository":"grafana","number":106778,"url":"https://github.com/grafana/grafana/pull/106778","status":"judged","expected":2,"caught":2,"seconds":62.675,"review_url":null},{"case_id":"grafana-grafana-107534","repository":"grafana","number":107534,"url":"https://github.com/grafana/grafana/pull/107534","status":"judged","expected":1,"caught":0,"seconds":254.79,"review_url":null},{"case_id":"keycloak-keycloak-32918","repository":"keycloak","number":32918,"url":"https://github.com/keycloak/keycloak/pull/32918","status":"judged","expected":2,"caught":0,"seconds":64.383,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":3,"caught":1,"seconds":81.518,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":4,"caught":0,"seconds":230.583,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":2,"caught":0,"seconds":235.988,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":345.966,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":3,"caught":0,"seconds":223.061,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":4,"caught":2,"seconds":1498.725,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":2,"caught":1,"seconds":181.271,"review_url":null},{"case_id":"keycloak-keycloak-40940","repository":"keycloak","number":40940,"url":"https://github.com/keycloak/keycloak/pull/40940","status":"judged","expected":2,"caught":2,"seconds":995.79,"review_url":null},{"case_id":"ai-code-review-evaluation-keycloak-greptile-1","repository":"keycloak-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1","status":"judged","expected":1,"caught":1,"seconds":109.085,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":3,"caught":1,"seconds":139.805,"review_url":null},{"case_id":"getsentry-sentry-77754","repository":"sentry","number":77754,"url":"https://github.com/getsentry/sentry/pull/77754","status":"judged","expected":2,"caught":1,"seconds":36.27,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":3,"caught":1,"seconds":424.096,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":2,"caught":0,"seconds":108.113,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":5,"caught":1,"seconds":59.389,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":70.525,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":2,"seconds":74.185,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":4,"caught":3,"seconds":166.035,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-3","repository":"sentry-greptile","number":3,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3","status":"judged","expected":3,"caught":0,"seconds":172.753,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-5","repository":"sentry-greptile","number":5,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5","status":"judged","expected":6,"caught":2,"seconds":235.155,"review_url":null}]}}},{"id":"greptile-v4-1","name":"Greptile v4.1","label":"Greptile v4.1","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":39.9,"composite_parts":{"f1":56,"latency":31,"cost":0},"f1":0.511,"f2":0.512,"precision":0.5094,"recall":0.5127,"tp":81,"fp":78,"fn":77,"expected":158,"prs":50,"coverage":{"graded":50,"cohort":50},"display_f1":0.511,"latency_p50_seconds":298,"latency_p90_seconds":429,"display_latency_seconds":298,"cost_per_review_usd":1.75,"cost_basis":"app_list_price_estimate","chart_cost":1.75,"run_dates":["2026-04-07","2026-04-07"],"popovers":{"f1":{"f2":0.512,"cohort_of":50},"cost":{"estimate":true},"score":{"parts":{"f1":56,"latency":31,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-7232","repository":"cal.com","number":7232,"url":"https://github.com/calcom/cal.com/pull/7232","status":"judged","expected":3,"caught":1,"seconds":570,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR7232__20260406/pull/1"},{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":312,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR8087__20260406/pull/1"},{"case_id":"calcom-cal.com-8330","repository":"cal.com","number":8330,"url":"https://github.com/calcom/cal.com/pull/8330","status":"judged","expected":2,"caught":2,"seconds":322,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR8330__20260406/pull/1"},{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":4,"caught":2,"seconds":351,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10600__20260406/pull/1"},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":5,"caught":2,"seconds":214,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10967__20260406/pull/1"},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":9,"caught":5,"seconds":409,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR11059__20260406/pull/1"},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":6,"caught":3,"seconds":290,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR14740__20260406/pull/1"},{"case_id":"calcom-cal.com-14943","repository":"cal.com","number":14943,"url":"https://github.com/calcom/cal.com/pull/14943","status":"judged","expected":2,"caught":1,"seconds":159,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR14943__20260406/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":402,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22345__20260406/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":4,"caught":3,"seconds":302,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22532__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":4,"caught":3,"seconds":247,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR1__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-2","repository":"discourse-graphite","number":2,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2","status":"judged","expected":1,"caught":1,"seconds":279,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR2__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":3,"caught":2,"seconds":255,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR3__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":8,"caught":4,"seconds":254,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR4__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-5","repository":"discourse-graphite","number":5,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5","status":"judged","expected":3,"caught":1,"seconds":199,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR5__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-6","repository":"discourse-graphite","number":6,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6","status":"judged","expected":1,"caught":1,"seconds":223,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR6__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-7","repository":"discourse-graphite","number":7,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7","status":"judged","expected":3,"caught":2,"seconds":184,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR7__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-8","repository":"discourse-graphite","number":8,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8","status":"judged","expected":6,"caught":4,"seconds":350,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR8__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":2,"caught":0,"seconds":247,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR9__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":7,"caught":2,"seconds":344,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR10__20260405/pull/1"},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":176,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR76186__20260405/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":5,"caught":4,"seconds":267,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR79265__20260405/pull/1"},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":2,"caught":2,"seconds":230,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR80329__20260405/pull/1"},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":3,"seconds":223,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR90045__20260405/pull/1"},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":2,"caught":1,"seconds":144,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR90939__20260405/pull/1"},{"case_id":"grafana-grafana-94942","repository":"grafana","number":94942,"url":"https://github.com/grafana/grafana/pull/94942","status":"judged","expected":2,"caught":1,"seconds":171,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR94942__20260405/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":2,"caught":2,"seconds":345,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR97529__20260405/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":3,"caught":1,"seconds":228,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR103633__20260405/pull/1"},{"case_id":"grafana-grafana-106778","repository":"grafana","number":106778,"url":"https://github.com/grafana/grafana/pull/106778","status":"judged","expected":2,"caught":1,"seconds":376,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR106778__20260405/pull/1"},{"case_id":"grafana-grafana-107534","repository":"grafana","number":107534,"url":"https://github.com/grafana/grafana/pull/107534","status":"judged","expected":1,"caught":0,"seconds":517,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR107534__20260405/pull/1"},{"case_id":"keycloak-keycloak-32918","repository":"keycloak","number":32918,"url":"https://github.com/keycloak/keycloak/pull/32918","status":"judged","expected":2,"caught":0,"seconds":487,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR32918__20260406/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":3,"caught":1,"seconds":244,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR33832__20260406/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":4,"caught":1,"seconds":418,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36880__20260406/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":2,"caught":0,"seconds":289,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36882__20260406/pull/1"},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":534,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37038__20260406/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":3,"caught":0,"seconds":725,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37429__20260406/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":4,"caught":2,"seconds":337,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37634__20260406/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":2,"caught":2,"seconds":306,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR38446__20260406/pull/1"},{"case_id":"keycloak-keycloak-40940","repository":"keycloak","number":40940,"url":"https://github.com/keycloak/keycloak/pull/40940","status":"judged","expected":2,"caught":1,"seconds":173,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR40940__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-keycloak-greptile-1","repository":"keycloak-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1","status":"judged","expected":1,"caught":1,"seconds":340,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak-greptile__greptile-v4-1__PR1__20260406/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":3,"caught":1,"seconds":294,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR67876__20260406/pull/1"},{"case_id":"getsentry-sentry-77754","repository":"sentry","number":77754,"url":"https://github.com/getsentry/sentry/pull/77754","status":"judged","expected":2,"caught":1,"seconds":273,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR77754__20260406/pull/1"},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":3,"caught":2,"seconds":323,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80168__20260406/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":2,"caught":1,"seconds":180,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80528__20260406/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":5,"caught":1,"seconds":176,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR93824__20260406/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":309,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR95633__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":3,"seconds":429,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR1__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":4,"caught":3,"seconds":317,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR2__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-3","repository":"sentry-greptile","number":3,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3","status":"judged","expected":3,"caught":2,"seconds":315,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR3__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-5","repository":"sentry-greptile","number":5,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5","status":"judged","expected":6,"caught":1,"seconds":320,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR5__20260406/pull/1"}]}}},{"id":"cursor-bugbot","name":"Cursor Bugbot","label":"Cursor Bugbot","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":36.2,"composite_parts":{"f1":58,"latency":0,"cost":7},"f1":0.5139,"f2":0.4856,"precision":0.5692,"recall":0.4684,"tp":74,"fp":56,"fn":84,"expected":158,"prs":50,"coverage":{"graded":50,"cohort":50},"display_f1":0.5139,"latency_p50_seconds":472,"latency_p90_seconds":701,"display_latency_seconds":472,"cost_per_review_usd":1.25,"cost_basis":"app_list_price_estimate","chart_cost":1.25,"run_dates":["2026-01-28","2026-01-28"],"popovers":{"f1":{"f2":0.4856,"cohort_of":50},"cost":{"estimate":true},"score":{"parts":{"f1":58,"latency":0,"cost":7}},"prs":{"cases":[{"case_id":"calcom-cal.com-7232","repository":"cal.com","number":7232,"url":"https://github.com/calcom/cal.com/pull/7232","status":"judged","expected":3,"caught":1,"seconds":592,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR7232__20260122/pull/1"},{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":431,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR8087__20260122/pull/1"},{"case_id":"calcom-cal.com-8330","repository":"cal.com","number":8330,"url":"https://github.com/calcom/cal.com/pull/8330","status":"judged","expected":2,"caught":2,"seconds":538,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR8330__20260122/pull/1"},{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":4,"caught":1,"seconds":701,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR10600__20260122/pull/1"},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":5,"caught":3,"seconds":444,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR10967__20260122/pull/1"},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":9,"caught":6,"seconds":528,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR11059__20260122/pull/1"},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":6,"caught":3,"seconds":301,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR14740__20260122/pull/1"},{"case_id":"calcom-cal.com-14943","repository":"cal.com","number":14943,"url":"https://github.com/calcom/cal.com/pull/14943","status":"judged","expected":2,"caught":1,"seconds":253,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR14943__20260122/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":1,"seconds":757,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR22345__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":4,"caught":0,"seconds":null,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":4,"caught":2,"seconds":360,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR1__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-2","repository":"discourse-graphite","number":2,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2","status":"judged","expected":1,"caught":1,"seconds":403,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR2__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":3,"caught":1,"seconds":517,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR3__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":8,"caught":2,"seconds":310,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR4__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-5","repository":"discourse-graphite","number":5,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5","status":"judged","expected":3,"caught":1,"seconds":472,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR5__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-6","repository":"discourse-graphite","number":6,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6","status":"judged","expected":1,"caught":1,"seconds":262,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR6__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-7","repository":"discourse-graphite","number":7,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7","status":"judged","expected":3,"caught":3,"seconds":189,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR7__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-8","repository":"discourse-graphite","number":8,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8","status":"judged","expected":6,"caught":2,"seconds":489,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR8__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":2,"caught":1,"seconds":444,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR9__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":7,"caught":5,"seconds":314,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":464,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR76186__20260122/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":5,"caught":1,"seconds":698,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR79265__20260122/pull/1"},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":2,"caught":2,"seconds":509,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR80329__20260122/pull/1"},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":2,"seconds":329,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR90045__20260122/pull/1"},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":2,"caught":1,"seconds":208,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR90939__20260122/pull/1"},{"case_id":"grafana-grafana-94942","repository":"grafana","number":94942,"url":"https://github.com/grafana/grafana/pull/94942","status":"judged","expected":2,"caught":1,"seconds":76,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR94942__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":2,"caught":1,"seconds":594,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR97529__20260122/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":3,"caught":0,"seconds":730,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR103633__20260122/pull/1"},{"case_id":"grafana-grafana-106778","repository":"grafana","number":106778,"url":"https://github.com/grafana/grafana/pull/106778","status":"judged","expected":2,"caught":2,"seconds":447,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR106778__20260122/pull/1"},{"case_id":"grafana-grafana-107534","repository":"grafana","number":107534,"url":"https://github.com/grafana/grafana/pull/107534","status":"judged","expected":1,"caught":0,"seconds":728,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR107534__20260122/pull/1"},{"case_id":"keycloak-keycloak-32918","repository":"keycloak","number":32918,"url":"https://github.com/keycloak/keycloak/pull/32918","status":"judged","expected":2,"caught":1,"seconds":520,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR32918__20260122/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":3,"caught":1,"seconds":349,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR33832__20260122/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":4,"caught":3,"seconds":488,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR36880__20260122/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":2,"caught":0,"seconds":591,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR36882__20260122/pull/1"},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":597,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR37038__20260122/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":3,"caught":2,"seconds":165,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR37429__20260122/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":4,"caught":2,"seconds":406,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR37634__20260122/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":2,"caught":0,"seconds":663,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR38446__20260122/pull/1"},{"case_id":"keycloak-keycloak-40940","repository":"keycloak","number":40940,"url":"https://github.com/keycloak/keycloak/pull/40940","status":"judged","expected":2,"caught":1,"seconds":394,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR40940__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-keycloak-greptile-1","repository":"keycloak-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1","status":"judged","expected":1,"caught":0,"seconds":521,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak-greptile__bugbot__PR1__20260122/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":3,"caught":1,"seconds":397,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR67876__20260122/pull/1"},{"case_id":"getsentry-sentry-77754","repository":"sentry","number":77754,"url":"https://github.com/getsentry/sentry/pull/77754","status":"judged","expected":2,"caught":1,"seconds":443,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR77754__20260122/pull/1"},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":3,"caught":1,"seconds":417,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR80168__20260122/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":2,"caught":0,"seconds":326,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR80528__20260122/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":5,"caught":1,"seconds":604,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR93824__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":729,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR95633__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":3,"seconds":562,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR1__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":4,"caught":4,"seconds":579,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR2__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-3","repository":"sentry-greptile","number":3,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3","status":"judged","expected":3,"caught":0,"seconds":512,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR3__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-5","repository":"sentry-greptile","number":5,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5","status":"judged","expected":6,"caught":2,"seconds":595,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR5__20260122/pull/1"}]}}},{"id":"ionwarp-gpt-5-6-luna-pro","name":"IonWarp · GPT 5.6 Luna Pro","label":"IonWarp · GPT 5.6 Luna Pro","kind":"ionwarp","model":"openai/gpt-5.6-luna-pro","model_label":"GPT 5.6 Luna Pro","developer":"openai","effort":"medium","composite":31.8,"composite_parts":{"f1":0,"latency":100,"cost":59},"f1":0.4163,"f2":0.3309,"precision":0.7302,"recall":0.2911,"tp":46,"fp":17,"fn":112,"expected":158,"prs":50,"coverage":{"graded":50,"cohort":50},"display_f1":0.4163,"latency_p50_seconds":107.1,"latency_p90_seconds":164.2,"display_latency_seconds":107.1,"cost_per_review_usd":0.1158,"cost_basis":"observed_inference","chart_cost":0.1158,"run_dates":["2026-09-11","2026-09-13"],"popovers":{"f1":{"f2":0.3309,"cohort_of":50},"cost":{"estimate":false},"score":{"parts":{"f1":0,"latency":100,"cost":59}},"prs":{"cases":[{"case_id":"calcom-cal.com-7232","repository":"cal.com","number":7232,"url":"https://github.com/calcom/cal.com/pull/7232","status":"judged","expected":3,"caught":0,"seconds":97.3,"review_url":null},{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":87.881,"review_url":null},{"case_id":"calcom-cal.com-8330","repository":"cal.com","number":8330,"url":"https://github.com/calcom/cal.com/pull/8330","status":"judged","expected":2,"caught":0,"seconds":87.139,"review_url":null},{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":4,"caught":1,"seconds":108.259,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":5,"caught":2,"seconds":198.197,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":9,"caught":3,"seconds":98.572,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":6,"caught":2,"seconds":137.125,"review_url":null},{"case_id":"calcom-cal.com-14943","repository":"cal.com","number":14943,"url":"https://github.com/calcom/cal.com/pull/14943","status":"judged","expected":2,"caught":1,"seconds":76.87,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":172.307,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":4,"caught":1,"seconds":63.807,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":4,"caught":2,"seconds":144.127,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-2","repository":"discourse-graphite","number":2,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2","status":"judged","expected":1,"caught":0,"seconds":89.354,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":3,"caught":1,"seconds":106.239,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":8,"caught":1,"seconds":124.707,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-5","repository":"discourse-graphite","number":5,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5","status":"judged","expected":3,"caught":0,"seconds":81.967,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-6","repository":"discourse-graphite","number":6,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6","status":"judged","expected":1,"caught":0,"seconds":115.858,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-7","repository":"discourse-graphite","number":7,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7","status":"judged","expected":3,"caught":3,"seconds":114.329,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-8","repository":"discourse-graphite","number":8,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8","status":"judged","expected":6,"caught":2,"seconds":77.466,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":2,"caught":0,"seconds":145.879,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":7,"caught":2,"seconds":139.988,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":0,"seconds":182.452,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":5,"caught":2,"seconds":87.371,"review_url":null},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":2,"caught":1,"seconds":86.008,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":1,"seconds":107.873,"review_url":null},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":2,"caught":1,"seconds":44.408,"review_url":null},{"case_id":"grafana-grafana-94942","repository":"grafana","number":94942,"url":"https://github.com/grafana/grafana/pull/94942","status":"judged","expected":2,"caught":2,"seconds":70.733,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":2,"caught":1,"seconds":90.368,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":3,"caught":1,"seconds":123.869,"review_url":null},{"case_id":"grafana-grafana-106778","repository":"grafana","number":106778,"url":"https://github.com/grafana/grafana/pull/106778","status":"judged","expected":2,"caught":2,"seconds":120.979,"review_url":null},{"case_id":"grafana-grafana-107534","repository":"grafana","number":107534,"url":"https://github.com/grafana/grafana/pull/107534","status":"judged","expected":1,"caught":0,"seconds":65.661,"review_url":null},{"case_id":"keycloak-keycloak-32918","repository":"keycloak","number":32918,"url":"https://github.com/keycloak/keycloak/pull/32918","status":"judged","expected":2,"caught":0,"seconds":164.247,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":3,"caught":0,"seconds":100.616,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":4,"caught":0,"seconds":90.363,"review_url":null},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":2,"caught":0,"seconds":76.12,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":125.971,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":3,"caught":1,"seconds":141.888,"review_url":null},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":4,"caught":1,"seconds":121.395,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":2,"caught":0,"seconds":122.865,"review_url":null},{"case_id":"keycloak-keycloak-40940","repository":"keycloak","number":40940,"url":"https://github.com/keycloak/keycloak/pull/40940","status":"judged","expected":2,"caught":1,"seconds":42.499,"review_url":null},{"case_id":"ai-code-review-evaluation-keycloak-greptile-1","repository":"keycloak-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1","status":"judged","expected":1,"caught":1,"seconds":41.187,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":3,"caught":1,"seconds":138.433,"review_url":null},{"case_id":"getsentry-sentry-77754","repository":"sentry","number":77754,"url":"https://github.com/getsentry/sentry/pull/77754","status":"judged","expected":2,"caught":0,"seconds":82.414,"review_url":null},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":3,"caught":1,"seconds":43.194,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":2,"caught":0,"seconds":72.98,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":5,"caught":0,"seconds":109.826,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":150.569,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":2,"seconds":117.273,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":4,"caught":1,"seconds":164.32,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-3","repository":"sentry-greptile","number":3,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3","status":"judged","expected":3,"caught":1,"seconds":95.743,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-5","repository":"sentry-greptile","number":5,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5","status":"judged","expected":6,"caught":1,"seconds":169.221,"review_url":null}]}}},{"id":"coderabbit","name":"CodeRabbit","label":"CodeRabbit","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":3.1,"composite_parts":{"f1":3,"latency":3,"cost":3},"f1":0.4215,"f2":0.5109,"precision":0.3264,"recall":0.5949,"tp":94,"fp":194,"fn":64,"expected":158,"prs":50,"coverage":{"graded":50,"cohort":50},"display_f1":0.4215,"latency_p50_seconds":454,"latency_p90_seconds":730,"display_latency_seconds":454,"cost_per_review_usd":1.5,"cost_basis":"app_list_price_estimate","chart_cost":1.5,"run_dates":["2026-01-23","2026-04-05"],"popovers":{"f1":{"f2":0.5109,"cohort_of":50},"cost":{"estimate":true},"score":{"parts":{"f1":3,"latency":3,"cost":3}},"prs":{"cases":[{"case_id":"calcom-cal.com-7232","repository":"cal.com","number":7232,"url":"https://github.com/calcom/cal.com/pull/7232","status":"judged","expected":3,"caught":2,"seconds":417,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR7232__20260122/pull/1"},{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":513,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR8087__20260122/pull/1"},{"case_id":"calcom-cal.com-8330","repository":"cal.com","number":8330,"url":"https://github.com/calcom/cal.com/pull/8330","status":"judged","expected":2,"caught":2,"seconds":478,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR8330__20260122/pull/1"},{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":4,"caught":1,"seconds":610,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR10600__20260122/pull/1"},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":5,"caught":5,"seconds":680,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR10967__20260122/pull/1"},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":9,"caught":9,"seconds":677,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR11059__20260122/pull/1"},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":6,"caught":5,"seconds":589,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR14740__20260122/pull/1"},{"case_id":"calcom-cal.com-14943","repository":"cal.com","number":14943,"url":"https://github.com/calcom/cal.com/pull/14943","status":"judged","expected":2,"caught":2,"seconds":552,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR14943__20260122/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":730,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR22345__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":4,"caught":2,"seconds":463,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":4,"caught":3,"seconds":309,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR1__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-2","repository":"discourse-graphite","number":2,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2","status":"judged","expected":1,"caught":1,"seconds":595,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR2__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":3,"caught":2,"seconds":641,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR3__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":8,"caught":6,"seconds":527,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR4__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-5","repository":"discourse-graphite","number":5,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5","status":"judged","expected":3,"caught":1,"seconds":326,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR5__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-6","repository":"discourse-graphite","number":6,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6","status":"judged","expected":1,"caught":1,"seconds":440,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR6__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-7","repository":"discourse-graphite","number":7,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7","status":"judged","expected":3,"caught":1,"seconds":159,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR7__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-8","repository":"discourse-graphite","number":8,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8","status":"judged","expected":6,"caught":4,"seconds":618,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR8__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-9","repository":"discourse-graphite","number":9,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9","status":"judged","expected":2,"caught":0,"seconds":338,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR9__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":7,"caught":3,"seconds":566,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":277,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR76186__20260122/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":5,"caught":3,"seconds":367,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR79265__20260122/pull/1"},{"case_id":"grafana-grafana-80329","repository":"grafana","number":80329,"url":"https://github.com/grafana/grafana/pull/80329","status":"judged","expected":2,"caught":1,"seconds":310,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR80329__20260122/pull/1"},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":3,"seconds":693,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR90045__20260122/pull/1"},{"case_id":"grafana-grafana-90939","repository":"grafana","number":90939,"url":"https://github.com/grafana/grafana/pull/90939","status":"judged","expected":2,"caught":1,"seconds":320,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR90939__20260122/pull/1"},{"case_id":"grafana-grafana-94942","repository":"grafana","number":94942,"url":"https://github.com/grafana/grafana/pull/94942","status":"judged","expected":2,"caught":1,"seconds":232,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR94942__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":2,"caught":1,"seconds":445,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR97529__20260122/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":3,"caught":1,"seconds":358,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR103633__20260122/pull/1"},{"case_id":"grafana-grafana-106778","repository":"grafana","number":106778,"url":"https://github.com/grafana/grafana/pull/106778","status":"judged","expected":2,"caught":1,"seconds":765,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR106778__20260122/pull/1"},{"case_id":"grafana-grafana-107534","repository":"grafana","number":107534,"url":"https://github.com/grafana/grafana/pull/107534","status":"judged","expected":1,"caught":0,"seconds":337,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR107534__20260122/pull/1"},{"case_id":"keycloak-keycloak-32918","repository":"keycloak","number":32918,"url":"https://github.com/keycloak/keycloak/pull/32918","status":"judged","expected":2,"caught":1,"seconds":481,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR32918__20260122/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":3,"caught":2,"seconds":293,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR33832__20260122/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":4,"caught":1,"seconds":378,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR36880__20260122/pull/1"},{"case_id":"keycloak-keycloak-36882","repository":"keycloak","number":36882,"url":"https://github.com/keycloak/keycloak/pull/36882","status":"judged","expected":2,"caught":0,"seconds":382,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR36882__20260122/pull/1"},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":817,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR37038__20260122/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":3,"caught":2,"seconds":477,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR37429__20260122/pull/1"},{"case_id":"keycloak-keycloak-37634","repository":"keycloak","number":37634,"url":"https://github.com/keycloak/keycloak/pull/37634","status":"judged","expected":4,"caught":3,"seconds":574,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR37634__20260122/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":2,"caught":2,"seconds":913,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR38446__20260122/pull/1"},{"case_id":"keycloak-keycloak-40940","repository":"keycloak","number":40940,"url":"https://github.com/keycloak/keycloak/pull/40940","status":"judged","expected":2,"caught":1,"seconds":415,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR40940__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-keycloak-greptile-1","repository":"keycloak-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1","status":"judged","expected":1,"caught":1,"seconds":432,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak-greptile__coderabbit__PR1__20260122/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":3,"caught":2,"seconds":1051,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR67876__20260122/pull/1"},{"case_id":"getsentry-sentry-77754","repository":"sentry","number":77754,"url":"https://github.com/getsentry/sentry/pull/77754","status":"judged","expected":2,"caught":1,"seconds":417,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR77754__20260122/pull/1"},{"case_id":"getsentry-sentry-80168","repository":"sentry","number":80168,"url":"https://github.com/getsentry/sentry/pull/80168","status":"judged","expected":3,"caught":2,"seconds":308,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR80168__20260122/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":2,"caught":1,"seconds":308,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR80528__20260122/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":5,"caught":0,"seconds":288,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR93824__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":1,"caught":0,"seconds":699,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR95633__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":3,"seconds":297,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__coderabbit__PR1__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":4,"caught":0,"seconds":661,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__coderabbit__PR2__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-3","repository":"sentry-greptile","number":3,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3","status":"judged","expected":3,"caught":2,"seconds":382,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__coderabbit__PR3__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-5","repository":"sentry-greptile","number":5,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5","status":"judged","expected":6,"caught":4,"seconds":1050,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__coderabbit__PR5__20260122/pull/1"}]}}}]}