{"family":{"id":"ionwarp-ai-architecture","key":"ai-architecture","agent":"arch","name":"AI Architecture","slug":"ai-architecture"},"snapshot":{"id":"onerow3-arch-2026-09-24-r1","captured_at":"2026-09-25T03:05:29.526587+00:00","status":"methodology_review","source_revision_url":"https://github.com/Winding-Labs/ai-code-review-benchmark"},"release":{"id":"2026-10-12-public-v1","date":"2026-10-12"},"publication":{"protocol":"category-scoped-v4-all-golden-match","status":"methodology_review","label":"Provisional scoped-label matching","ranking_allowed":false,"blockers":["derived_scope_not_independently_adjudicated","generalist_output_rescored"],"methodology_url":"https://ionwarp.com/benchmarks/methodology"},"judge":"IonWarp adapted category-scoped judge, Claude Opus 4.5","scope":"ai-architecture specialist · fresh local pilot · provisional semantic matching","case_count":11,"issue_count":6,"score_definition":"Reviewer Score = (3 × F1 + latency + $/run) ÷ 5. Each part is this row's place between the worst (0) and best (100) full-cohort row in this family — higher F1, lower latency and lower $/run score higher. Scored once F1, latency and $/run are all measured and at least 3 rows are.","ranking_basis":"Sorted by Reviewer Score (F1 counts 3×, time and cost 1× each).","scenario":{"seats":5,"monthly_reviews":100},"configured_pin":{"model":"deepseek/deepseek-v4.1-flash","reasoning_effort":"medium","retry":{"model":"z-ai/glm-5.3","reasoning_effort":"medium"}},"warnings":["Local reviewer execution; no hosted queue or GitHub delivery measurement.","One trial per declared configuration and PR; exploratory comparison.","Pinned source issue matching is not independent finding adjudication.","88 cells are admitted unchanged from the experiments that paid for them (identical case identity, engine and output ceiling, on the runtime of their configuration); they are never re-bought.","Precision is golden-issue matching precision, not independently adjudicated finding correctness.","Competitor outputs are historical; fresh IonWarp runs use the same PR heads and new semantic grading.","Available-evidence checkpoint: unstarted jobs and completed but ungraded outputs remain explicit missing coverage. Scores use only saved grades under the named protocol.","The saved grade files preserve all source rows. This checkpoint projects only its declared case/configuration identities; unselected rows are not pooled into displayed scores."],"notices":["Measured on the reviewer prompt of 2026-09-07; production has changed it since"],"not_measured":[],"license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"cost_unit":"review_run","case_ids":["calcom-cal.com-10967","calcom-cal.com-22345","calcom-cal.com-22532","ai-code-review-evaluation-discourse-graphite-1","ai-code-review-evaluation-discourse-graphite-10","grafana-grafana-97529","keycloak-keycloak-33832","keycloak-keycloak-36880","keycloak-keycloak-38446","getsentry-sentry-80528","getsentry-sentry-95633"],"benchmark_default":"ionwarp-deepseek-v4-1-flash","benefits":{"row_id":"ionwarp-deepseek-v4-1-flash","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","rivals":4,"wins":[{"metric":"speed","percent":8,"text":"8% less wait"},{"metric":"cost","percent":69,"text":"69% cheaper"}],"cost_line":"about 5¢ a review","basis":"DeepSeek V4.1 Flash against the median of 4 review apps on the same 11 PRs. Accuracy is F1, wait is median review time, cost is Pro ($49 a month, 5 seats) at 100 reviews a month against each app's list price for the same team."},"rows":[{"id":"coderabbit","name":"CodeRabbit","label":"CodeRabbit","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":60.6,"composite_parts":{"f1":100,"latency":0,"cost":3},"f1":0.7273,"f2":0.6897,"precision":0.8,"recall":0.6667,"tp":4,"fp":1,"fn":2,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.7273,"latency_p50_seconds":463,"latency_p90_seconds":730,"display_latency_seconds":463,"cost_per_review_usd":1.5,"cost_basis":"app_list_price_estimate","chart_cost":1.5,"run_dates":["2026-01-23","2026-04-05"],"popovers":{"f1":{"f2":0.6897,"cohort_of":11},"cost":{"estimate":true},"score":{"parts":{"f1":100,"latency":0,"cost":3}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":1,"seconds":680,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR10967__20260122/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":730,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR22345__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":463,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__coderabbit__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":309,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR1__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":566,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__coderabbit__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":445,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__coderabbit__PR97529__20260122/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":293,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR33832__20260122/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":378,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR36880__20260122/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":913,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__coderabbit__PR38446__20260122/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":308,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR80528__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":699,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__coderabbit__PR95633__20260122/pull/1"}]}}},{"id":"ionwarp-glm-5-3","name":"IonWarp · GLM 5.3","label":"IonWarp · GLM 5.3","kind":"ionwarp","model":"z-ai/glm-5.3","model_label":"GLM 5.3","developer":"z-ai","effort":"medium","composite":59.3,"composite_parts":{"f1":55,"latency":75,"cost":56},"f1":0.4706,"f2":0.5714,"precision":0.3636,"recall":0.6667,"tp":4,"fp":7,"fn":2,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.4706,"latency_p50_seconds":89.5,"latency_p90_seconds":126.1,"display_latency_seconds":89.5,"cost_per_review_usd":0.0748,"cost_basis":"observed_inference","chart_cost":0.0748,"run_dates":["2026-09-17","2026-09-22"],"popovers":{"f1":{"f2":0.5714,"cohort_of":11},"cost":{"estimate":false},"score":{"parts":{"f1":55,"latency":75,"cost":56}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":126.132,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":1,"seconds":60.86,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":516.816,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":91.484,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":88.577,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":104.475,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":32.206,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":77.342,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":89.469,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":56.274,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":123.976,"review_url":null}]}}},{"id":"ionwarp-gpt-5-6-luna-pro","name":"IonWarp · GPT 5.6 Luna Pro","label":"IonWarp · GPT 5.6 Luna Pro","kind":"ionwarp","model":"openai/gpt-5.6-luna-pro","model_label":"GPT 5.6 Luna Pro","developer":"openai","effort":"medium","composite":53.8,"composite_parts":{"f1":35,"latency":100,"cost":65},"f1":0.3529,"f2":0.4286,"precision":0.2727,"recall":0.5,"tp":3,"fp":8,"fn":3,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.3529,"latency_p50_seconds":51.4,"latency_p90_seconds":97,"display_latency_seconds":51.4,"cost_per_review_usd":0.0463,"cost_basis":"observed_inference","chart_cost":0.0463,"run_dates":["2026-09-10","2026-09-17"],"popovers":{"f1":{"f2":0.4286,"cohort_of":11},"cost":{"estimate":false},"score":{"parts":{"f1":35,"latency":100,"cost":65}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":62.581,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":51.393,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":49.515,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":125.905,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":50.94,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":48.333,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":75.998,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":55.308,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":49.42,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":40.866,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":96.998,"review_url":null}]}}},{"id":"ionwarp-deepseek-v4-pro","name":"IonWarp · DeepSeek V4 Pro","label":"IonWarp · DeepSeek V4 Pro","kind":"ionwarp","model":"deepseek/deepseek-v4-pro-0813","model_label":"DeepSeek V4 Pro","developer":"deepseek","effort":"medium","composite":53.4,"composite_parts":{"f1":51,"latency":65,"cost":51},"f1":0.4444,"f2":0.5556,"precision":0.3333,"recall":0.6667,"tp":4,"fp":8,"fn":2,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.4444,"latency_p50_seconds":112.3,"latency_p90_seconds":187.7,"display_latency_seconds":112.3,"cost_per_review_usd":0.1016,"cost_basis":"observed_inference","chart_cost":0.1016,"run_dates":["2026-09-08","2026-09-17"],"popovers":{"f1":{"f2":0.5556,"cohort_of":11},"cost":{"estimate":false},"score":{"parts":{"f1":51,"latency":65,"cost":51}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":175.38,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":1,"seconds":180.024,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":149.174,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":325.626,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":106.688,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":76.096,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":79.656,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":187.713,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":112.328,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":31.777,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":79.623,"review_url":null}]}}},{"id":"greptile-v4-1","name":"Greptile v4.1","label":"Greptile v4.1","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":50.5,"composite_parts":{"f1":78,"latency":19,"cost":0},"f1":0.6,"f2":0.5357,"precision":0.75,"recall":0.5,"tp":3,"fp":1,"fn":3,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.6,"latency_p50_seconds":306,"latency_p90_seconds":402,"display_latency_seconds":306,"cost_per_review_usd":1.75,"cost_basis":"app_list_price_estimate","chart_cost":1.75,"run_dates":["2026-04-07","2026-04-07"],"popovers":{"f1":{"f2":0.5357,"cohort_of":11},"cost":{"estimate":true},"score":{"parts":{"f1":78,"latency":19,"cost":0}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":214,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10967__20260406/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":402,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22345__20260406/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":302,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22532__20260406/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":247,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR1__20260405/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":344,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR10__20260405/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":345,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR97529__20260405/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":244,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR33832__20260406/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":418,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36880__20260406/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":306,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR38446__20260406/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":180,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80528__20260406/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":309,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR95633__20260406/pull/1"}]}}},{"id":"ionwarp-deepseek-v4-1-flash","name":"IonWarp · DeepSeek V4.1 Flash","label":"IonWarp · DeepSeek V4.1 Flash","kind":"ionwarp","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","developer":"deepseek","effort":"medium","composite":47.8,"composite_parts":{"f1":55,"latency":10,"cost":63},"f1":0.4706,"f2":0.5714,"precision":0.3636,"recall":0.6667,"tp":4,"fp":7,"fn":2,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.4706,"latency_p50_seconds":374.1,"latency_p90_seconds":638.6,"display_latency_seconds":374.1,"cost_per_review_usd":0.05,"cost_basis":"observed_inference","chart_cost":0.05,"run_dates":["2026-09-24","2026-09-24"],"popovers":{"f1":{"f2":0.5714,"cohort_of":11},"cost":{"estimate":false},"score":{"parts":{"f1":55,"latency":10,"cost":63}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":638.585,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":1,"seconds":189.829,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":496.93,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":255.978,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":242.548,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":412.944,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":504.107,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":691.646,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":363.57,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":275.422,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":374.139,"review_url":null}]}}},{"id":"ionwarp-glm-5-3-flash","name":"IonWarp · GLM 5.3 Flash","label":"IonWarp · GLM 5.3 Flash","kind":"ionwarp","model":"z-ai/glm-5.3-flash","model_label":"GLM 5.3 Flash","developer":"z-ai","effort":"low","composite":38.3,"composite_parts":{"f1":0,"latency":92,"cost":100},"f1":0.1538,"f2":0.1613,"precision":0.1429,"recall":0.1667,"tp":1,"fp":6,"fn":5,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.1538,"latency_p50_seconds":61.7,"latency_p90_seconds":224.1,"display_latency_seconds":61.7,"cost_per_review_usd":0.0064,"cost_basis":"observed_inference","chart_cost":0.0064,"run_dates":["2026-09-10","2026-09-17"],"popovers":{"f1":{"f2":0.1613,"cohort_of":11},"cost":{"estimate":false},"score":{"parts":{"f1":0,"latency":92,"cost":100}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":36.399,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":112.612,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":61.716,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":0,"seconds":224.113,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":27.479,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":665.148,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":0,"seconds":206.021,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":37.345,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":47.304,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":15.46,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":129.079,"review_url":null}]}}},{"id":"cursor-bugbot","name":"Cursor Bugbot","label":"Cursor Bugbot","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":37.4,"composite_parts":{"f1":60,"latency":0,"cost":6},"f1":0.5,"f2":0.5882,"precision":0.4,"recall":0.6667,"tp":4,"fp":6,"fn":2,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.5,"latency_p50_seconds":466,"latency_p90_seconds":729,"display_latency_seconds":466,"cost_per_review_usd":1.25,"cost_basis":"app_list_price_estimate","chart_cost":1.25,"run_dates":["2026-01-28","2026-01-28"],"popovers":{"f1":{"f2":0.5882,"cohort_of":11},"cost":{"estimate":true},"score":{"parts":{"f1":60,"latency":0,"cost":6}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":444,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR10967__20260122/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":1,"seconds":757,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR22345__20260122/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":null,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__bugbot__PR22532__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":360,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR1__20260122/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":314,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__bugbot__PR10__20260122/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":594,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__bugbot__PR97529__20260122/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":349,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR33832__20260122/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":488,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR36880__20260122/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":663,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__bugbot__PR38446__20260122/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":326,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR80528__20260122/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":729,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__bugbot__PR95633__20260122/pull/1"}]}}},{"id":"ionwarp-grok-4-6","name":"IonWarp · Grok 4.6","label":"IonWarp · Grok 4.6","kind":"ionwarp","model":"x-ai/grok-4.6","model_label":"Grok 4.6","developer":"x-ai","effort":"medium","composite":34,"composite_parts":{"f1":34,"latency":36,"cost":32},"f1":0.3478,"f2":0.4878,"precision":0.2353,"recall":0.6667,"tp":4,"fp":13,"fn":2,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.3478,"latency_p50_seconds":209.4,"latency_p90_seconds":252.2,"display_latency_seconds":209.4,"cost_per_review_usd":0.2843,"cost_basis":"observed_inference","chart_cost":0.2843,"run_dates":["2026-09-08","2026-09-17"],"popovers":{"f1":{"f2":0.4878,"cohort_of":11},"cost":{"estimate":false},"score":{"parts":{"f1":34,"latency":36,"cost":32}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":211.155,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":1,"seconds":153.389,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":308.232,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":209.377,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":218.275,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":136.186,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":101.309,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":190.807,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":210.029,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":145.554,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":252.175,"review_url":null}]}}},{"id":"ionwarp-muse-spark-1-3","name":"IonWarp · Muse Spark 1.3","label":"IonWarp · Muse Spark 1.3","kind":"ionwarp","model":"meta/muse-spark-1.3","model_label":"Muse Spark 1.3","developer":"meta","effort":"medium","composite":33.6,"composite_parts":{"f1":23,"latency":60,"cost":39},"f1":0.2857,"f2":0.3846,"precision":0.2,"recall":0.5,"tp":3,"fp":12,"fn":3,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.2857,"latency_p50_seconds":124.1,"latency_p90_seconds":178.2,"display_latency_seconds":124.1,"cost_per_review_usd":0.1932,"cost_basis":"observed_inference","chart_cost":0.1932,"run_dates":["2026-09-08","2026-09-17"],"popovers":{"f1":{"f2":0.3846,"cohort_of":11},"cost":{"estimate":false},"score":{"parts":{"f1":23,"latency":60,"cost":39}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":178.169,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":216.537,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":175.535,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":84.223,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":116.465,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":48.705,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":79.315,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":158.059,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":85.694,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":124.105,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":169.674,"review_url":null}]}}},{"id":"ionwarp-gemini-3-8-flash","name":"IonWarp · Gemini 3.8 Flash","label":"IonWarp · Gemini 3.8 Flash","kind":"ionwarp","model":"google/gemini-3.8-flash","model_label":"Gemini 3.8 Flash","developer":"google","effort":"medium","composite":32.3,"composite_parts":{"f1":31,"latency":32,"cost":35},"f1":0.3333,"f2":0.4167,"precision":0.25,"recall":0.5,"tp":3,"fp":9,"fn":3,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.3333,"latency_p50_seconds":229.7,"latency_p90_seconds":314,"display_latency_seconds":229.7,"cost_per_review_usd":0.2384,"cost_basis":"observed_inference","chart_cost":0.2384,"run_dates":["2026-09-08","2026-09-22"],"popovers":{"f1":{"f2":0.4167,"cohort_of":11},"cost":{"estimate":false},"score":{"parts":{"f1":31,"latency":32,"cost":35}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":229.659,"review_url":null},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":185.136,"review_url":null},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":314.482,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":290.697,"review_url":null},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":162.483,"review_url":null},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":116.468,"review_url":null},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":1,"seconds":210.114,"review_url":null},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":1,"seconds":314.045,"review_url":null},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":298.061,"review_url":null},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":138.991,"review_url":null},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":313.185,"review_url":null}]}}},{"id":"qodo-v2","name":"Qodo v2","label":"Qodo v2","kind":"competitor","model":null,"model_label":null,"developer":null,"effort":null,"composite":12.8,"composite_parts":{"f1":17,"latency":13,"cost":1},"f1":0.25,"f2":0.1923,"precision":0.5,"recall":0.1667,"tp":1,"fp":1,"fn":5,"expected":6,"prs":11,"coverage":{"graded":11,"cohort":11},"display_f1":0.25,"latency_p50_seconds":352,"latency_p90_seconds":449,"display_latency_seconds":352,"cost_per_review_usd":1.6667,"cost_basis":"app_list_price_estimate","chart_cost":1.6667,"run_dates":["2026-02-27","2026-02-27"],"popovers":{"f1":{"f2":0.1923,"cohort_of":11},"cost":{"estimate":true},"score":{"parts":{"f1":17,"latency":13,"cost":1}},"prs":{"cases":[{"case_id":"calcom-cal.com-10967","repository":"cal.com","number":10967,"url":"https://github.com/calcom/cal.com/pull/10967","status":"judged","expected":1,"caught":0,"seconds":384,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR10967__20260227/pull/1"},{"case_id":"calcom-cal.com-22345","repository":"cal.com","number":22345,"url":"https://github.com/calcom/cal.com/pull/22345","status":"judged","expected":1,"caught":0,"seconds":352,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22345__20260227/pull/1"},{"case_id":"calcom-cal.com-22532","repository":"cal.com","number":22532,"url":"https://github.com/calcom/cal.com/pull/22532","status":"judged","expected":0,"caught":0,"seconds":321,"review_url":"https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22532__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-1","repository":"discourse-graphite","number":1,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1","status":"judged","expected":1,"caught":1,"seconds":294,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR1__20260227/pull/1"},{"case_id":"ai-code-review-evaluation-discourse-graphite-10","repository":"discourse-graphite","number":10,"url":"https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10","status":"judged","expected":0,"caught":0,"seconds":449,"review_url":"https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-v2__PR10__20260227/pull/1"},{"case_id":"grafana-grafana-97529","repository":"grafana","number":97529,"url":"https://github.com/grafana/grafana/pull/97529","status":"judged","expected":0,"caught":0,"seconds":301,"review_url":"https://github.com/code-review-benchmark/grafana__grafana__qodo-v2__PR97529__20260227/pull/1"},{"case_id":"keycloak-keycloak-33832","repository":"keycloak","number":33832,"url":"https://github.com/keycloak/keycloak/pull/33832","status":"judged","expected":1,"caught":0,"seconds":349,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR33832__20260227/pull/1"},{"case_id":"keycloak-keycloak-36880","repository":"keycloak","number":36880,"url":"https://github.com/keycloak/keycloak/pull/36880","status":"judged","expected":1,"caught":0,"seconds":647,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR36880__20260227/pull/1"},{"case_id":"keycloak-keycloak-38446","repository":"keycloak","number":38446,"url":"https://github.com/keycloak/keycloak/pull/38446","status":"judged","expected":0,"caught":0,"seconds":449,"review_url":"https://github.com/code-review-benchmark/keycloak__keycloak__qodo-v2__PR38446__20260227/pull/1"},{"case_id":"getsentry-sentry-80528","repository":"sentry","number":80528,"url":"https://github.com/getsentry/sentry/pull/80528","status":"judged","expected":1,"caught":0,"seconds":218,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR80528__20260227/pull/1"},{"case_id":"getsentry-sentry-95633","repository":"sentry","number":95633,"url":"https://github.com/getsentry/sentry/pull/95633","status":"judged","expected":0,"caught":0,"seconds":377,"review_url":"https://github.com/code-review-benchmark/sentry__sentry__qodo-v2__PR95633__20260227/pull/1"}]}}}]}