{"family":{"id":"ionwarp-log-coverage","key":"log-coverage","agent":"logs","name":"Log & Errors","slug":"logs"},"snapshot":{"id":"onerow3-logs-2026-09-24-r1","captured_at":"2026-09-25T03:12:58.063889+00:00","status":"methodology_review","source_revision_url":"https://github.com/Winding-Labs/ai-code-review-benchmark"},"release":{"id":"2026-10-12-public-v1","date":"2026-10-12"},"publication":{"protocol":"category-scoped-v4-all-golden-match","status":"methodology_review","label":"Provisional scoped-label matching","ranking_allowed":false,"blockers":["derived_scope_not_independently_adjudicated","generalist_output_rescored"],"methodology_url":"https://ionwarp.com/benchmarks/methodology"},"judge":"IonWarp adapted category-scoped judge, Claude Opus 4.5","scope":"logs specialist · fresh local pilot · provisional semantic matching","case_count":17,"issue_count":31,"score_definition":"Reviewer Score = (3 × F1 + latency + $/run) ÷ 5. Each part is this row's place between the worst (0) and best (100) full-cohort row in this family — higher F1, lower latency and lower $/run score higher. Scored once F1, latency and $/run are all measured and at least 3 rows are.","ranking_basis":"Sorted by Reviewer Score (F1 counts 3×, time and cost 1× each).","scenario":{"seats":5,"monthly_reviews":100},"configured_pin":{"model":"deepseek/deepseek-v4.1-flash","reasoning_effort":"medium","retry":{"model":"google/gemini-3.8-flash","reasoning_effort":"medium"}},"warnings":["Local reviewer execution; no hosted queue or GitHub delivery measurement.","One trial per declared configuration and PR; exploratory comparison.","Pinned source issue matching is not independent finding adjudication.","85 cells are admitted unchanged from the experiments that paid for them (identical case identity, engine and output ceiling, on the runtime of their configuration); they are never re-bought.","Precision is golden-issue matching precision, not independently adjudicated finding correctness.","Competitor outputs are historical; fresh IonWarp runs use the same PR heads and new semantic grading.","Available-evidence checkpoint: unstarted jobs and completed but ungraded outputs remain explicit missing coverage. Scores use only saved grades under the named protocol.","The saved grade files preserve all source rows. This checkpoint projects only its declared case/configuration identities; unselected rows are not pooled into displayed scores."],"notices":["Measured on the reviewer prompt of 2026-09-22; production has changed it since"],"not_measured":[],"license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"cost_unit":"review_run","case_ids":["calcom-cal.com-8087","calcom-cal.com-10967","calcom-cal.com-11059","calcom-cal.com-22345","calcom-cal.com-22532","ai-code-review-evaluation-discourse-graphite-4","ai-code-review-evaluation-discourse-graphite-10","grafana-grafana-76186","grafana-grafana-90045","grafana-grafana-97529","keycloak-keycloak-33832","getsentry-sentry-67876","getsentry-sentry-80528","getsentry-sentry-93824","getsentry-sentry-95633","ai-code-review-evaluation-sentry-greptile-1","ai-code-review-evaluation-sentry-greptile-2"],"benchmark_default":"ionwarp-deepseek-v4-1-flash","benefits":{"row_id":"ionwarp-deepseek-v4-1-flash","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","rivals":4,"wins":[{"metric":"speed","percent":38,"text":"38% less wait"},{"metric":"cost","percent":69,"text":"69% cheaper"}],"cost_line":"about 4¢ a review","basis":"DeepSeek V4.1 Flash against the median of 4 review apps on the same 17 PRs. Accuracy is F1, wait is median review time, cost is Pro ($49 a month, 5 seats) at 100 reviews a month against each app's list price for the same team."},"rows":[{"id":"ionwarp-gemini-3-8-flash","name":"IonWarp · Gemini 3.8 Flash","label":"IonWarp · Gemini 3.8 Flash","kind":"ionwarp","model":"google/gemini-3.8-flash","model_label":"Gemini 3.8 Flash","developer":"google","effort":"medium","composite":76.2,"composite_parts":{"f1":86,"latency":72,"cost":50},"f1":0.6774,"f2":0.6774,"precision":0.6774,"recall":0.6774,"tp":21,"fp":10,"fn":10,"expected":31,"prs":17,"coverage":{"graded":17,"cohort":17},"display_f1":0.6774,"latency_p50_seconds":88.5,"latency_p90_seconds":164.1,"display_latency_seconds":88.5,"cost_per_review_usd":0.1003,"cost_basis":"observed_inference","chart_cost":0.1003,"run_dates":["2026-09-14","2026-09-17"],"popovers":{"f1":{"f2":0.6774,"cohort_of":17},"cost":{"estimate":false},"score":{"parts":{"f1":86,"latency":72,"cost":50}},"prs":{"cases":[{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":97.959,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":2,"caught":1,"seconds":80.109,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":5,"caught":2,"seconds":80.461,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":0,"caught":0,"seconds":31.814,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":1,"caught":1,"seconds":88.49,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":3,"caught":2,"seconds":102.648,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":3,"caught":3,"seconds":73.852,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":2,"seconds":135.116,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":3,"seconds":91.595,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":0,"seconds":74.621,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":81.371,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":2,"caught":1,"seconds":112.718,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":0,"caught":0,"seconds":28.384,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":1,"caught":1,"seconds":184.766,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":164.118,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":3,"seconds":84.165,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":2,"caught":1,"seconds":104.775,"review_url":null}]}}},{"id":"qodo-v2","name":"Qodo v2","label":"Qodo v2","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":63.4,"composite_parts":{"f1":100,"latency":16,"cost":1},"f1":0.7273,"f2":0.7547,"precision":0.6857,"recall":0.7742,"tp":24,"fp":11,"fn":7,"expected":31,"prs":17,"coverage":{"graded":17,"cohort":17},"display_f1":0.7273,"latency_p50_seconds":352,"latency_p90_seconds":449,"display_latency_seconds":352,"cost_per_review_usd":1.6667,"cost_basis":"app_list_price_estimate","chart_cost":1.6667,"run_dates":["2026-02-27","2026-02-27"],"popovers":{"f1":{"f2":0.7547,"cohort_of":17},"cost":{"estimate":true},"score":{"parts":{"f1":100,"latency":16,"cost":1}},"prs":{"cases":[{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":222,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR8087__20260227/pull/1"},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":2,"caught":2,"seconds":384,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR10967__20260227/pull/1"},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":5,"caught":4,"seconds":321,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR11059__20260227/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":0,"caught":0,"seconds":352,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22345__20260227/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":1,"caught":1,"seconds":321,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22532__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":3,"caught":3,"seconds":462,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR4__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":3,"caught":2,"seconds":449,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR10__20260227/pull/1"},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":2,"seconds":379,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR76186__20260227/pull/1"},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":2,"seconds":406,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR90045__20260227/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":301,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR97529__20260227/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":349,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR33832__20260227/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":2,"caught":1,"seconds":406,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR67876__20260227/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":0,"caught":0,"seconds":218,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR80528__20260227/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":1,"caught":1,"seconds":337,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR93824__20260227/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":377,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR95633__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":2,"seconds":267,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo-v2__PR1__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":2,"caught":2,"seconds":358,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo-v2__PR2__20260227/pull/1"}]}}},{"id":"ionwarp-deepseek-v4-1-flash","name":"IonWarp · DeepSeek V4.1 Flash","label":"IonWarp · DeepSeek V4.1 Flash","kind":"ionwarp","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","developer":"deepseek","effort":"medium","composite":57,"composite_parts":{"f1":63,"latency":30,"cost":67},"f1":0.5902,"f2":0.5844,"precision":0.6,"recall":0.5806,"tp":18,"fp":12,"fn":13,"expected":31,"prs":17,"coverage":{"graded":17,"cohort":17},"display_f1":0.5902,"latency_p50_seconds":248,"latency_p90_seconds":541.9,"display_latency_seconds":248,"cost_per_review_usd":0.0393,"cost_basis":"observed_inference","chart_cost":0.0393,"run_dates":["2026-09-25","2026-09-25"],"popovers":{"f1":{"f2":0.5844,"cohort_of":17},"cost":{"estimate":false},"score":{"parts":{"f1":63,"latency":30,"cost":67}},"prs":{"cases":[{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":210.253,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":2,"caught":1,"seconds":126.839,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":5,"caught":2,"seconds":168.148,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":0,"caught":0,"seconds":229.393,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":1,"caught":1,"seconds":247.352,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":3,"caught":1,"seconds":159.312,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":3,"caught":2,"seconds":247.974,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":576.724,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":3,"seconds":155.984,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":0,"seconds":541.881,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":408.591,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":2,"caught":1,"seconds":293.405,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":0,"caught":0,"seconds":32.515,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":1,"caught":1,"seconds":263.141,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":415.49,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":2,"seconds":249.113,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":2,"caught":2,"seconds":477.819,"review_url":null}]}}},{"id":"greptile-v4-1","name":"Greptile v4.1","label":"Greptile v4.1","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":56.4,"composite_parts":{"f1":87,"latency":22,"cost":0},"f1":0.678,"f2":0.6579,"precision":0.7143,"recall":0.6452,"tp":20,"fp":8,"fn":11,"expected":31,"prs":17,"coverage":{"graded":17,"cohort":17},"display_f1":0.678,"latency_p50_seconds":302,"latency_p90_seconds":409,"display_latency_seconds":302,"cost_per_review_usd":1.75,"cost_basis":"app_list_price_estimate","chart_cost":1.75,"run_dates":["2026-04-07","2026-04-07"],"popovers":{"f1":{"f2":0.6579,"cohort_of":17},"cost":{"estimate":true},"score":{"parts":{"f1":87,"latency":22,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":312,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR8087__20260406/pull/1"},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":2,"caught":1,"seconds":214,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10967__20260406/pull/1"},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":5,"caught":4,"seconds":409,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR11059__20260406/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":0,"caught":0,"seconds":402,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22345__20260406/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":1,"caught":1,"seconds":302,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22532__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":3,"caught":2,"seconds":254,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR4__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":3,"caught":1,"seconds":344,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR10__20260405/pull/1"},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":176,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR76186__20260405/pull/1"},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":3,"seconds":223,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR90045__20260405/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":345,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR97529__20260405/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":244,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR33832__20260406/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":2,"caught":1,"seconds":294,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR67876__20260406/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":0,"caught":0,"seconds":180,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80528__20260406/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":1,"caught":0,"seconds":176,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR93824__20260406/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":309,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR95633__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":3,"seconds":429,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR1__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":2,"caught":1,"seconds":317,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR2__20260406/pull/1"}]}}},{"id":"cursor-bugbot","name":"Cursor Bugbot","label":"Cursor Bugbot","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":50.8,"composite_parts":{"f1":81,"latency":6,"cost":6},"f1":0.6563,"f2":0.6688,"precision":0.6364,"recall":0.6774,"tp":21,"fp":12,"fn":10,"expected":31,"prs":17,"coverage":{"graded":17,"cohort":17},"display_f1":0.6563,"latency_p50_seconds":454,"latency_p90_seconds":729,"display_latency_seconds":454,"cost_per_review_usd":1.25,"cost_basis":"app_list_price_estimate","chart_cost":1.25,"run_dates":["2026-01-28","2026-01-28"],"popovers":{"f1":{"f2":0.6688,"cohort_of":17},"cost":{"estimate":true},"score":{"parts":{"f1":81,"latency":6,"cost":6}},"prs":{"cases":[{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":431,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR8087__20260122/pull/1"},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":2,"caught":1,"seconds":444,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR10967__20260122/pull/1"},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":5,"caught":4,"seconds":528,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR11059__20260122/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":0,"caught":0,"seconds":757,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR22345__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":1,"caught":0,"seconds":null,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":3,"caught":2,"seconds":310,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR4__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":3,"caught":3,"seconds":314,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":464,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR76186__20260122/pull/1"},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":3,"seconds":329,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR90045__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":0,"seconds":594,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR97529__20260122/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":349,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR33832__20260122/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":2,"caught":1,"seconds":397,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR67876__20260122/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":0,"caught":0,"seconds":326,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR80528__20260122/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":1,"caught":0,"seconds":604,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR93824__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":729,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR95633__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":3,"seconds":562,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR1__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":2,"caught":2,"seconds":579,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR2__20260122/pull/1"}]}}},{"id":"ionwarp-muse-spark-1-3","name":"IonWarp · Muse Spark 1.3","label":"IonWarp · Muse Spark 1.3","kind":"ionwarp","model":"meta/muse-spark-1.3","model_label":"Muse Spark 1.3","developer":"meta","effort":"medium","composite":41.2,"composite_parts":{"f1":21,"latency":90,"cost":53},"f1":0.4364,"f2":0.4054,"precision":0.5,"recall":0.3871,"tp":12,"fp":12,"fn":19,"expected":31,"prs":17,"coverage":{"graded":17,"cohort":17},"display_f1":0.4364,"latency_p50_seconds":55.6,"latency_p90_seconds":97.1,"display_latency_seconds":55.6,"cost_per_review_usd":0.0846,"cost_basis":"observed_inference","chart_cost":0.0846,"run_dates":["2026-09-14","2026-09-17"],"popovers":{"f1":{"f2":0.4054,"cohort_of":17},"cost":{"estimate":false},"score":{"parts":{"f1":21,"latency":90,"cost":53}},"prs":{"cases":[{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":55.688,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":2,"caught":1,"seconds":62.427,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":5,"caught":2,"seconds":67.063,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":0,"caught":0,"seconds":34.449,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":1,"caught":0,"seconds":39.775,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":3,"caught":1,"seconds":66.134,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":3,"caught":0,"seconds":55.59,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":45.365,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":3,"seconds":107.842,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":0,"seconds":34.213,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":59.136,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":2,"caught":1,"seconds":45.211,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":0,"caught":0,"seconds":26.976,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":1,"caught":1,"seconds":46.415,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":65.572,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":1,"seconds":50.921,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":2,"caught":0,"seconds":97.119,"review_url":null}]}}},{"id":"coderabbit","name":"CodeRabbit","label":"CodeRabbit","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":38.4,"composite_parts":{"f1":63,"latency":0,"cost":3},"f1":0.5915,"f2":0.6402,"precision":0.525,"recall":0.6774,"tp":21,"fp":19,"fn":10,"expected":31,"prs":17,"coverage":{"graded":17,"cohort":17},"display_f1":0.5915,"latency_p50_seconds":527,"latency_p90_seconds":730,"display_latency_seconds":527,"cost_per_review_usd":1.5,"cost_basis":"app_list_price_estimate","chart_cost":1.5,"run_dates":["2026-01-23","2026-04-05"],"popovers":{"f1":{"f2":0.6402,"cohort_of":17},"cost":{"estimate":true},"score":{"parts":{"f1":63,"latency":0,"cost":3}},"prs":{"cases":[{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":513,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR8087__20260122/pull/1"},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":2,"caught":2,"seconds":680,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR10967__20260122/pull/1"},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":5,"caught":4,"seconds":677,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR11059__20260122/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":0,"caught":0,"seconds":730,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR22345__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":1,"caught":0,"seconds":463,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":3,"caught":3,"seconds":527,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR4__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":3,"caught":3,"seconds":566,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":277,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR76186__20260122/pull/1"},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":3,"seconds":693,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR90045__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":0,"seconds":445,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR97529__20260122/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":293,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR33832__20260122/pull/1"},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":2,"caught":1,"seconds":1051,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR67876__20260122/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":0,"caught":0,"seconds":308,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR80528__20260122/pull/1"},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":1,"caught":0,"seconds":288,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR93824__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":699,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR95633__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":3,"seconds":297,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__coderabbit__PR1__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":2,"caught":0,"seconds":661,"review_url":"https://github.com/code-review-benchmark/sentry__sentry-greptile__coderabbit__PR2__20260122/pull/1"}]}}},{"id":"ionwarp-glm-5-3-flash","name":"IonWarp · GLM 5.3 Flash","label":"IonWarp · GLM 5.3 Flash","kind":"ionwarp","model":"z-ai/glm-5.3-flash","model_label":"GLM 5.3 Flash","developer":"z-ai","effort":"low","composite":35.9,"composite_parts":{"f1":0,"latency":79,"cost":100},"f1":0.3607,"f2":0.3571,"precision":0.3667,"recall":0.3548,"tp":11,"fp":19,"fn":20,"expected":31,"prs":17,"coverage":{"graded":17,"cohort":17},"display_f1":0.3607,"latency_p50_seconds":74.2,"latency_p90_seconds":242.4,"display_latency_seconds":74.2,"cost_per_review_usd":0.0058,"cost_basis":"observed_inference","chart_cost":0.0058,"run_dates":["2026-09-13","2026-09-17"],"popovers":{"f1":{"f2":0.3571,"cohort_of":17},"cost":{"estimate":false},"score":{"parts":{"f1":0,"latency":79,"cost":100}},"prs":{"cases":[{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":68.961,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":2,"caught":1,"seconds":98.195,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":5,"caught":1,"seconds":89.975,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":0,"caught":0,"seconds":25.748,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":1,"caught":1,"seconds":50.243,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":3,"caught":1,"seconds":74.205,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":3,"caught":0,"seconds":96.566,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":71.94,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":1,"seconds":63.179,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":0,"seconds":82.514,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":330.863,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":2,"caught":0,"seconds":136.059,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":0,"caught":0,"seconds":24.318,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":1,"caught":1,"seconds":52.641,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":242.401,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":2,"seconds":63.669,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":2,"caught":1,"seconds":144.787,"review_url":null}]}}},{"id":"ionwarp-gpt-5-6-luna-pro","name":"IonWarp · GPT 5.6 Luna Pro","label":"IonWarp · GPT 5.6 Luna Pro","kind":"ionwarp","model":"openai/gpt-5.6-luna-pro","model_label":"GPT 5.6 Luna Pro","developer":"openai","effort":"medium","composite":32.7,"composite_parts":{"f1":0,"latency":100,"cost":64},"f1":0.36,"f2":0.3147,"precision":0.4737,"recall":0.2903,"tp":9,"fp":10,"fn":22,"expected":31,"prs":17,"coverage":{"graded":17,"cohort":17},"display_f1":0.36,"latency_p50_seconds":43.7,"latency_p90_seconds":58.6,"display_latency_seconds":43.7,"cost_per_review_usd":0.0463,"cost_basis":"observed_inference","chart_cost":0.0463,"run_dates":["2026-09-14","2026-09-17"],"popovers":{"f1":{"f2":0.3147,"cohort_of":17},"cost":{"estimate":false},"score":{"parts":{"f1":0,"latency":100,"cost":64}},"prs":{"cases":[{"case_id":"calcom-cal.com-8087","repository":"cal.com","number":8087,"url":"https://github.com/calcom/cal.com/pull/8087","status":"judged","expected":1,"caught":1,"seconds":48.02,"review_url":null},{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":2,"caught":0,"seconds":58.643,"review_url":null},{"case_id":"calcom-cal.com-11059","repository":"cal.com","number":11059,"url":"https://github.com/calcom/cal.com/pull/11059","status":"judged","expected":5,"caught":1,"seconds":54.579,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":0,"caught":0,"seconds":24.759,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":1,"caught":0,"seconds":38.601,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-4","repository":"discourse-graphite","number":4,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4","status":"judged","expected":3,"caught":1,"seconds":49.564,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":3,"caught":0,"seconds":39.143,"review_url":null},{"case_id":"grafana-grafana-76186","repository":"grafana","number":76186,"url":"https://github.com/grafana/grafana/pull/76186","status":"judged","expected":3,"caught":1,"seconds":50.501,"review_url":null},{"case_id":"grafana-grafana-90045","repository":"grafana","number":90045,"url":"https://github.com/grafana/grafana/pull/90045","status":"judged","expected":3,"caught":2,"seconds":73.125,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":1,"caught":1,"seconds":40.152,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":0,"caught":0,"seconds":43.726,"review_url":null},{"case_id":"getsentry-sentry-67876","repository":"sentry","number":67876,"url":"https://github.com/getsentry/sentry/pull/67876","status":"judged","expected":2,"caught":0,"seconds":31.8,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":0,"caught":0,"seconds":31.276,"review_url":null},{"case_id":"getsentry-sentry-93824","repository":"sentry","number":93824,"url":"https://github.com/getsentry/sentry/pull/93824","status":"judged","expected":1,"caught":1,"seconds":39.189,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":53.798,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-1","repository":"sentry-greptile","number":1,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1","status":"judged","expected":4,"caught":1,"seconds":36.27,"review_url":null},{"case_id":"ai-code-review-evaluation-sentry-greptile-2","repository":"sentry-greptile","number":2,"url":"https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2","status":"judged","expected":2,"caught":0,"seconds":51.708,"review_url":null}]}}}]}