{"family":{"id":"ionwarp-security-review","key":"security-review","agent":"security","name":"Security Review","slug":"security"},"snapshot":{"id":"onerow3-security-2026-09-24-r1","captured_at":"2026-09-25T03:10:07.864194+00:00","status":"methodology_review","source_revision_url":"https://github.com/Winding-Labs/ai-code-review-benchmark"},"release":{"id":"2026-10-12-public-v1","date":"2026-10-12"},"publication":{"protocol":"category-scoped-v4-all-golden-match","status":"methodology_review","label":"Provisional scoped-label matching","ranking_allowed":false,"blockers":["disputed_security_labels","derived_scope_not_independently_adjudicated","generalist_output_rescored"],"methodology_url":"https://ionwarp.com/benchmarks/methodology"},"judge":"IonWarp adapted category-scoped judge, Claude Opus 4.5","scope":"security specialist · fresh local pilot · provisional semantic matching","case_count":13,"issue_count":18,"score_definition":"Reviewer Score = (3 × F1 + latency + $/run) ÷ 5. Each part is this row's place between the worst (0) and best (100) full-cohort row in this family — higher F1, lower latency and lower $/run score higher. Scored once F1, latency and $/run are all measured and at least 3 rows are.","ranking_basis":"Sorted by Reviewer Score (F1 counts 3×, time and cost 1× each).","scenario":{"seats":5,"monthly_reviews":100},"configured_pin":{"model":"deepseek/deepseek-v4.1-flash","reasoning_effort":"medium","retry":{"model":"z-ai/glm-5.3-flash","reasoning_effort":"low"}},"warnings":["Local reviewer execution; no hosted queue or GitHub delivery measurement.","One trial per declared configuration and PR; exploratory comparison.","Pinned source issue matching is not independent finding adjudication.","91 cells are admitted unchanged from the experiments that paid for them (identical case identity, engine and output ceiling, on the runtime of their configuration); they are never re-bought.","Precision is golden-issue matching precision, not independently adjudicated finding correctness.","Competitor outputs are historical; fresh IonWarp runs use the same PR heads and new semantic grading.","Available-evidence checkpoint: unstarted jobs and completed but ungraded outputs remain explicit missing coverage. Scores use only saved grades under the named protocol.","The saved grade files preserve all source rows. This checkpoint projects only its declared case/configuration identities; unselected rows are not pooled into displayed scores."],"notices":["Measured on the reviewer prompt of 2026-09-07; production has changed it since"],"not_measured":[],"license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"cost_unit":"review_run","case_ids":["calcom-cal.com-10600","calcom-cal.com-14740","calcom-cal.com-22532","ai-code-review-evaluation-discourse-graphite-3","ai-code-review-evaluation-discourse-graphite-4","ai-code-review-evaluation-discourse-graphite-10","grafana-grafana-79265","grafana-grafana-103633","keycloak-keycloak-36880","keycloak-keycloak-37038","keycloak-keycloak-37429","keycloak-keycloak-38446","getsentry-sentry-67876"],"benchmark_default":"ionwarp-deepseek-v4-1-flash","benefits":{"row_id":"ionwarp-deepseek-v4-1-flash","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","rivals":4,"wins":[{"metric":"accuracy","percent":16,"text":"+16% accuracy"},{"metric":"speed","percent":24,"text":"24% less wait"},{"metric":"cost","percent":69,"text":"69% cheaper"}],"cost_line":"about 6¢ a review","basis":"DeepSeek V4.1 Flash against the median of 4 review apps on the same 13 PRs. Accuracy is F1, wait is median review time, cost is Pro ($49 a month, 5 seats) at 100 reviews a month against each app's list price for the same team."},"rows":[{"id":"ionwarp-glm-5-3-flash","name":"IonWarp · GLM 5.3 Flash","label":"IonWarp · GLM 5.3 Flash","kind":"ionwarp","model":"z-ai/glm-5.3-flash","model_label":"GLM 5.3 Flash","developer":"z-ai","effort":"low","composite":88.5,"composite_parts":{"f1":85,"latency":87,"cost":100},"f1":0.6667,"f2":0.5952,"precision":0.8333,"recall":0.5556,"tp":10,"fp":2,"fn":8,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.6667,"latency_p50_seconds":98.3,"latency_p90_seconds":1179.7,"display_latency_seconds":98.3,"cost_per_review_usd":0.0118,"cost_basis":"observed_inference","chart_cost":0.0118,"run_dates":["2026-09-10","2026-09-14"],"popovers":{"f1":{"f2":0.5952,"cohort_of":13},"cost":{"estimate":false},"score":{"parts":{"f1":85,"latency":87,"cost":100}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":1,"seconds":223.178,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":1,"seconds":59.105,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":98.303,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":1,"seconds":1179.703,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":3,"seconds":1402.322,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":1,"seconds":46.412,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":168.51,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":1,"seconds":42.897,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":403.817,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":1,"seconds":133.112,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":20.192,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":43.359,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":0,"seconds":53.894,"review_url":null}]}}},{"id":"ionwarp-deepseek-v4-1-flash","name":"IonWarp · DeepSeek V4.1 Flash","label":"IonWarp · DeepSeek V4.1 Flash","kind":"ionwarp","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","developer":"deepseek","effort":"medium","composite":63.4,"composite_parts":{"f1":76,"latency":23,"cost":67},"f1":0.625,"f2":0.5814,"precision":0.7143,"recall":0.5556,"tp":10,"fp":4,"fn":8,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.625,"latency_p50_seconds":356.3,"latency_p90_seconds":799.3,"display_latency_seconds":356.3,"cost_per_review_usd":0.0625,"cost_basis":"observed_inference","chart_cost":0.0625,"run_dates":["2026-09-25","2026-09-25"],"popovers":{"f1":{"f2":0.5814,"cohort_of":13},"cost":{"estimate":false},"score":{"parts":{"f1":76,"latency":23,"cost":67}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":356.307,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":1,"seconds":330.533,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":770.383,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":2,"seconds":244.856,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":2,"seconds":279.693,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":1,"seconds":222.921,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":330.916,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":1,"seconds":427.448,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":865.099,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":1,"seconds":334.074,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":799.319,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":440.218,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":1,"seconds":462.148,"review_url":null}]}}},{"id":"qodo-v2","name":"Qodo v2","label":"Qodo v2","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":62.8,"composite_parts":{"f1":100,"latency":13,"cost":1},"f1":0.7333,"f2":0.6548,"precision":0.9167,"recall":0.6111,"tp":11,"fp":1,"fn":7,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.7333,"latency_p50_seconds":436,"latency_p90_seconds":588,"display_latency_seconds":436,"cost_per_review_usd":1.6667,"cost_basis":"app_list_price_estimate","chart_cost":1.6667,"run_dates":["2026-02-27","2026-02-27"],"popovers":{"f1":{"f2":0.6548,"cohort_of":13},"cost":{"estimate":true},"score":{"parts":{"f1":100,"latency":13,"cost":1}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":436,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR10600__20260227/pull/1"},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":1,"seconds":274,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR14740__20260227/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":321,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22532__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":1,"seconds":382,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR3__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":4,"seconds":462,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR4__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":1,"seconds":449,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR10__20260227/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":0,"seconds":389,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR79265__20260227/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":1,"seconds":410,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR103633__20260227/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":647,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR36880__20260227/pull/1"},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":499,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR37038__20260227/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":588,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR37429__20260227/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":449,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR38446__20260227/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":1,"seconds":406,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR67876__20260227/pull/1"}]}}},{"id":"ionwarp-muse-spark-1-3","name":"IonWarp · Muse Spark 1.3","label":"IonWarp · Muse Spark 1.3","kind":"ionwarp","model":"meta/muse-spark-1.3","model_label":"Muse Spark 1.3","developer":"meta","effort":"medium","composite":42.2,"composite_parts":{"f1":35,"latency":66,"cost":38},"f1":0.4444,"f2":0.3704,"precision":0.6667,"recall":0.3333,"tp":6,"fp":3,"fn":12,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.4444,"latency_p50_seconds":150.1,"latency_p90_seconds":214.7,"display_latency_seconds":150.1,"cost_per_review_usd":0.2582,"cost_basis":"observed_inference","chart_cost":0.2582,"run_dates":["2026-09-08","2026-09-14"],"popovers":{"f1":{"f2":0.3704,"cohort_of":13},"cost":{"estimate":false},"score":{"parts":{"f1":35,"latency":66,"cost":38}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":331.069,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":1,"seconds":150.052,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":125.563,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":1,"seconds":136.547,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":2,"seconds":176.162,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":0,"seconds":148.006,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":0,"seconds":63.338,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":0,"seconds":67.115,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":121.448,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":211.989,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":206.231,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":214.733,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":0,"seconds":186.634,"review_url":null}]}}},{"id":"greptile-v4-1","name":"Greptile v4.1","label":"Greptile v4.1","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":40.2,"composite_parts":{"f1":56,"latency":31,"cost":0},"f1":0.5385,"f2":0.4375,"precision":0.875,"recall":0.3889,"tp":7,"fp":1,"fn":11,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.5385,"latency_p50_seconds":302,"latency_p90_seconds":534,"display_latency_seconds":302,"cost_per_review_usd":1.75,"cost_basis":"app_list_price_estimate","chart_cost":1.75,"run_dates":["2026-04-07","2026-04-07"],"popovers":{"f1":{"f2":0.4375,"cohort_of":13},"cost":{"estimate":true},"score":{"parts":{"f1":56,"latency":31,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":351,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10600__20260406/pull/1"},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":0,"seconds":290,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR14740__20260406/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":302,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22532__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":0,"seconds":255,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR3__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":2,"seconds":254,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR4__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":1,"seconds":344,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR10__20260405/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":267,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR79265__20260405/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":1,"seconds":228,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR103633__20260405/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":418,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36880__20260406/pull/1"},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":534,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37038__20260406/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":725,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37429__20260406/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":306,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR38446__20260406/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":0,"seconds":294,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR67876__20260406/pull/1"}]}}},{"id":"ionwarp-grok-4-6","name":"IonWarp · Grok 4.6","label":"IonWarp · Grok 4.6","kind":"ionwarp","model":"x-ai/grok-4.6","model_label":"Grok 4.6","developer":"x-ai","effort":"medium","composite":39.7,"composite_parts":{"f1":55,"latency":10,"cost":22},"f1":0.5333,"f2":0.4762,"precision":0.6667,"recall":0.4444,"tp":8,"fp":4,"fn":10,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.5333,"latency_p50_seconds":461.5,"latency_p90_seconds":764.7,"display_latency_seconds":461.5,"cost_per_review_usd":0.5751,"cost_basis":"observed_inference","chart_cost":0.5751,"run_dates":["2026-09-08","2026-09-15"],"popovers":{"f1":{"f2":0.4762,"cohort_of":13},"cost":{"estimate":false},"score":{"parts":{"f1":55,"latency":10,"cost":22}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":461.529,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":1,"seconds":481.038,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":115.771,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":1,"seconds":248.582,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":3,"seconds":1079.233,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":0,"seconds":383.089,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":231.952,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":0,"seconds":238.334,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":708.753,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":1,"seconds":287.929,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":741.78,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":594.734,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":1,"seconds":764.715,"review_url":null}]}}},{"id":"ionwarp-gemini-3-8-flash","name":"IonWarp · Gemini 3.8 Flash","label":"IonWarp · Gemini 3.8 Flash","kind":"ionwarp","model":"google/gemini-3.8-flash","model_label":"Gemini 3.8 Flash","developer":"google","effort":"medium","composite":38.5,"composite_parts":{"f1":51,"latency":14,"cost":24},"f1":0.5161,"f2":0.4706,"precision":0.6154,"recall":0.4444,"tp":8,"fp":5,"fn":10,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.5161,"latency_p50_seconds":430.1,"latency_p90_seconds":801.6,"display_latency_seconds":430.1,"cost_per_review_usd":0.5201,"cost_basis":"observed_inference","chart_cost":0.5201,"run_dates":["2026-09-08","2026-09-15"],"popovers":{"f1":{"f2":0.4706,"cohort_of":13},"cost":{"estimate":false},"score":{"parts":{"f1":51,"latency":14,"cost":24}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":430.843,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":1,"seconds":801.586,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":145.325,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":1,"seconds":395.812,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":2,"seconds":532.726,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":0,"seconds":316.011,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":430.136,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":0,"seconds":220.46,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":1076.878,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":356.394,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":68.594,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":468.079,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":1,"seconds":732.716,"review_url":null}]}}},{"id":"coderabbit","name":"CodeRabbit","label":"CodeRabbit","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":34.8,"composite_parts":{"f1":57,"latency":0,"cost":3},"f1":0.5405,"f2":0.5495,"precision":0.5263,"recall":0.5556,"tp":10,"fp":9,"fn":8,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.5405,"latency_p50_seconds":566,"latency_p90_seconds":913,"display_latency_seconds":566,"cost_per_review_usd":1.5,"cost_basis":"app_list_price_estimate","chart_cost":1.5,"run_dates":["2026-01-23","2026-04-05"],"popovers":{"f1":{"f2":0.5495,"cohort_of":13},"cost":{"estimate":true},"score":{"parts":{"f1":57,"latency":0,"cost":3}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":1,"seconds":610,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR10600__20260122/pull/1"},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":1,"seconds":589,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR14740__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":463,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":2,"seconds":641,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR3__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":2,"seconds":527,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR4__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":1,"seconds":566,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":0,"seconds":367,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR79265__20260122/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":1,"seconds":358,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR103633__20260122/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":378,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR36880__20260122/pull/1"},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":817,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR37038__20260122/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":477,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR37429__20260122/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":913,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR38446__20260122/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":0,"seconds":1051,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR67876__20260122/pull/1"}]}}},{"id":"cursor-bugbot","name":"Cursor Bugbot","label":"Cursor Bugbot","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":31.3,"composite_parts":{"f1":48,"latency":6,"cost":7},"f1":0.5,"f2":0.3846,"precision":1,"recall":0.3333,"tp":6,"fp":0,"fn":12,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.5,"latency_p50_seconds":502.5,"latency_p90_seconds":701,"display_latency_seconds":502.5,"cost_per_review_usd":1.25,"cost_basis":"app_list_price_estimate","chart_cost":1.25,"run_dates":["2026-01-28","2026-01-28"],"popovers":{"f1":{"f2":0.3846,"cohort_of":13},"cost":{"estimate":true},"score":{"parts":{"f1":48,"latency":6,"cost":7}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":701,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR10600__20260122/pull/1"},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":1,"seconds":301,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR14740__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":null,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":1,"seconds":517,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR3__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":1,"seconds":310,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR4__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":0,"seconds":314,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":1,"seconds":698,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR79265__20260122/pull/1"},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":0,"seconds":730,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR103633__20260122/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":488,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR36880__20260122/pull/1"},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":2,"seconds":597,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR37038__20260122/pull/1"},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":165,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR37429__20260122/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":663,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR38446__20260122/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":0,"seconds":397,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR67876__20260122/pull/1"}]}}},{"id":"ionwarp-deepseek-v4-pro","name":"IonWarp · DeepSeek V4 Pro","label":"IonWarp · DeepSeek V4 Pro","kind":"ionwarp","model":"deepseek/deepseek-v4-pro-0813","model_label":"DeepSeek V4 Pro","developer":"deepseek","effort":"medium","composite":31.3,"composite_parts":{"f1":14,"latency":63,"cost":52},"f1":0.3478,"f2":0.2597,"precision":0.8,"recall":0.2222,"tp":4,"fp":1,"fn":14,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.3478,"latency_p50_seconds":160.1,"latency_p90_seconds":269.4,"display_latency_seconds":160.1,"cost_per_review_usd":0.1306,"cost_basis":"observed_inference","chart_cost":0.1306,"run_dates":["2026-09-08","2026-09-14"],"popovers":{"f1":{"f2":0.2597,"cohort_of":13},"cost":{"estimate":false},"score":{"parts":{"f1":14,"latency":63,"cost":52}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":0,"seconds":239.185,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":1,"seconds":208.742,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":56.717,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":1,"seconds":172.587,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":0,"seconds":103.904,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":0,"seconds":111.303,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":0,"seconds":160.059,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":1,"seconds":136.828,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":84.972,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":1,"seconds":269.404,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":100.846,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":234.374,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":0,"seconds":326.884,"review_url":null}]}}},{"id":"ionwarp-gpt-5-6-luna-pro","name":"IonWarp · GPT 5.6 Luna Pro","label":"IonWarp · GPT 5.6 Luna Pro","kind":"ionwarp","model":"openai/gpt-5.6-luna-pro","model_label":"GPT 5.6 Luna Pro","developer":"openai","effort":"medium","composite":31.2,"composite_parts":{"f1":0,"latency":100,"cost":56},"f1":0.2857,"f2":0.2,"precision":1,"recall":0.1667,"tp":3,"fp":0,"fn":15,"expected":18,"prs":13,"coverage":{"graded":13,"cohort":13},"display_f1":0.2857,"latency_p50_seconds":76.3,"latency_p90_seconds":148.4,"display_latency_seconds":76.3,"cost_per_review_usd":0.1053,"cost_basis":"observed_inference","chart_cost":0.1053,"run_dates":["2026-09-10","2026-09-14"],"popovers":{"f1":{"f2":0.2,"cohort_of":13},"cost":{"estimate":false},"score":{"parts":{"f1":0,"latency":100,"cost":56}},"prs":{"cases":[{"case_id":"calcom-cal.com-10600","repository":"cal.com","number":10600,"url":"https://github.com/calcom/cal.com/pull/10600","status":"judged","expected":1,"caught":1,"seconds":64.905,"review_url":null},{"case_id":"calcom-cal.com-14740","repository":"cal.com","number":14740,"url":"https://github.com/calcom/cal.com/pull/14740","status":"judged","expected":1,"caught":0,"seconds":47.107,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":24.18,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-3","repository":"discourse-graphite","number":3,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3","status":"judged","expected":2,"caught":1,"seconds":89.083,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":5,"caught":0,"seconds":106.939,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":1,"caught":0,"seconds":77.906,"review_url":null},{"case_id":"grafana-grafana-79265","repository":"grafana","number":79265,"url":"https://github.com/grafana/grafana/pull/79265","status":"judged","expected":1,"caught":0,"seconds":64.89,"review_url":null},{"case_id":"grafana-grafana-103633","repository":"grafana","number":103633,"url":"https://github.com/grafana/grafana/pull/103633","status":"judged","expected":2,"caught":0,"seconds":43.235,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":116.837,"review_url":null},{"case_id":"keycloak-keycloak-37038","repository":"keycloak","number":37038,"url":"https://github.com/keycloak/keycloak/pull/37038","status":"judged","expected":2,"caught":1,"seconds":153.046,"review_url":null},{"case_id":"keycloak-keycloak-37429","repository":"keycloak","number":37429,"url":"https://github.com/keycloak/keycloak/pull/37429","status":"judged","expected":1,"caught":0,"seconds":30.635,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":76.349,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":1,"caught":0,"seconds":148.401,"review_url":null}]}}}]}