{"family":{"id":"ionwarp-cicd","key":"cicd-performance","agent":"cicd","name":"CI/CD Performance","slug":"cicd"},"snapshot":{"id":"model-tables-2026-09-20-r1","captured_at":"2026-09-21T15:23:43.547098+00:00","status":"methodology_review","source_revision_url":"https://github.com/Winding-Labs/ai-code-review-benchmark"},"release":{"id":"2026-10-12-public-v1","date":"2026-10-12"},"publication":{"protocol":"category-scoped-v4-all-golden-match","status":"methodology_review","label":"Provisional scoped-label matching","ranking_allowed":false,"blockers":["operator_labels_not_official_corpus","generalist_output_rescored"],"methodology_url":"https://ionwarp.com/benchmarks/methodology"},"judge":"IonWarp adapted category-scoped judge, Claude Opus 4.5","scope":"cicd specialist · fresh local pilot · provisional semantic matching","case_count":18,"issue_count":13,"score_definition":"Reviewer Score = (3 × F1 + latency + $/run) ÷ 5. Each part is this row's place between the worst (0) and best (100) full-cohort row in this family — higher F1, lower latency and lower $/run score higher. Scored once F1, latency and $/run are all measured and at least 3 rows are.","ranking_basis":"Sorted by Reviewer Score (F1 counts 3×, time and cost 1× each).","scenario":{"seats":5,"monthly_reviews":100},"configured_pin":{"model":"deepseek/deepseek-v4.1-flash","reasoning_effort":"medium","retry":{"model":"deepseek/deepseek-v4-pro-0813","reasoning_effort":"medium"}},"warnings":["Local reviewer execution; no hosted queue or GitHub delivery measurement.","One trial per declared configuration and PR; exploratory comparison.","Pinned source issue matching is not independent finding adjudication.","18 cells are admitted unchanged from the experiments that paid for them (identical case identity, engine and output ceiling, on the runtime of their configuration); they are never re-bought.","Contains operator-labelled PRs outside the official Martian corpus. These results are not an official-corpus leaderboard score.","Competitor outputs are historical; fresh IonWarp runs use the same PR heads and new semantic grading.","Precision is golden-issue matching precision, not independently adjudicated finding correctness.","Available-evidence checkpoint: unstarted jobs and completed but ungraded outputs remain explicit missing coverage. Scores use only saved grades under the named protocol.","3 provider calls have unavailable billing records. Their full $2.18962554 reservations remain in budget exposure. Reported spend is settled spend only; affected review costs remain null and cannot support cost comparisons or routing recommendations.","Provider fallbacks on from the first paid call, same per-model maximum price bound on every candidate provider (plan 2026-09-20 §4).","The saved grade files preserve all source rows. This checkpoint projects only its declared case/configuration identities; unselected rows are not pooled into displayed scores."],"notices":["Provisional: one agent searched for these pull requests by this lens's own defect classes and wrote the goldens; a second rater admitted them. Two goldens a later audit could not support (Kiali #10167, ESPHome #9396) are held: still counted here, dropped at the next release. IonWarp models only."],"not_measured":["CodeRabbit","Cursor Bugbot","Greptile v4.1","Qodo v2"],"license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/"},"cost_unit":"review_run","case_ids":["microsoft-aspire.dev-964","django-django-21457","esphome-esphome-9396","mozilla-gcp-ingestion-2901","herdrdev-herdr-4023","juspay-hyperswitch-14081","home-assistant-ios-5002","kiali-kiali-10167","touchlab-kmmbridge-266","marimo-team-marimo-9312","awslabs-mountpoint-s3-csi-driver-461","open-telemetry-opentelemetry-collector-contrib-37106","outline-outline-8534","plonky3-plonky3-1969","software-mansion-react-native-reanimated-8929","astral-sh-ruff-28610","home-assistant-supervisor-6807","wxsms-uiv-903"],"benchmark_default":"ionwarp-deepseek-v4-1-flash","benefits":{"row_id":"ionwarp-deepseek-v4-1-flash","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","rivals":0,"wins":[],"cost_line":"under 1¢ a review","basis":"No review app was measured on all 18 PRs of this lane, so there is no comparison yet."},"rows":[{"id":"ionwarp-deepseek-v4-pro","name":"IonWarp · DeepSeek V4 Pro","label":"IonWarp · DeepSeek V4 Pro","kind":"ionwarp","model":"deepseek/deepseek-v4-pro-0813","model_label":"DeepSeek V4 Pro","developer":"deepseek","effort":"medium","composite":60.5,"composite_parts":{"f1":100,"latency":0,"cost":3},"f1":0.7273,"f2":0.8333,"precision":0.6,"recall":0.9231,"tp":12,"fp":8,"fn":1,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.7273,"latency_p50_seconds":82.4,"latency_p90_seconds":403.9,"display_latency_seconds":82.4,"cost_per_review_usd":0.0323,"cost_basis":"observed_inference","chart_cost":0.0323,"run_dates":["2026-09-21","2026-09-21"],"popovers":{"f1":{"f2":0.8333,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":100,"latency":0,"cost":3}},"prs":{"cases":[{"case_id":"microsoft-aspire.dev-964","repository":"aspire.dev","number":964,"url":"https://github.com/microsoft/aspire.dev/pull/964","status":"judged","expected":0,"caught":0,"seconds":77.23,"review_url":null},{"case_id":"django-django-21457","repository":"django","number":21457,"url":"https://github.com/django/django/pull/21457","status":"judged","expected":1,"caught":1,"seconds":133.054,"review_url":null},{"case_id":"esphome-esphome-9396","repository":"esphome","number":9396,"url":"https://github.com/esphome/esphome/pull/9396","status":"judged","expected":1,"caught":1,"seconds":129.038,"review_url":null},{"case_id":"mozilla-gcp-ingestion-2901","repository":"gcp-ingestion","number":2901,"url":"https://github.com/mozilla/gcp-ingestion/pull/2901","status":"judged","expected":1,"caught":1,"seconds":53.279,"review_url":null},{"case_id":"herdrdev-herdr-4023","repository":"herdr","number":4023,"url":"https://github.com/herdrdev/herdr/pull/4023","status":"judged","expected":1,"caught":1,"seconds":96.942,"review_url":null},{"case_id":"juspay-hyperswitch-14081","repository":"hyperswitch","number":14081,"url":"https://github.com/juspay/hyperswitch/pull/14081","status":"judged","expected":1,"caught":1,"seconds":80.36,"review_url":null},{"case_id":"home-assistant-ios-5002","repository":"iOS","number":5002,"url":"https://github.com/home-assistant/iOS/pull/5002","status":"judged","expected":1,"caught":0,"seconds":403.891,"review_url":null},{"case_id":"kiali-kiali-10167","repository":"kiali","number":10167,"url":"https://github.com/kiali/kiali/pull/10167","status":"judged","expected":1,"caught":1,"seconds":36.714,"review_url":null},{"case_id":"touchlab-kmmbridge-266","repository":"KMMBridge","number":266,"url":"https://github.com/touchlab/KMMBridge/pull/266","status":"judged","expected":1,"caught":1,"seconds":77.4,"review_url":null},{"case_id":"marimo-team-marimo-9312","repository":"marimo","number":9312,"url":"https://github.com/marimo-team/marimo/pull/9312","status":"judged","expected":0,"caught":0,"seconds":39.713,"review_url":null},{"case_id":"awslabs-mountpoint-s3-csi-driver-461","repository":"mountpoint-s3-csi-driver","number":461,"url":"https://github.com/awslabs/mountpoint-s3-csi-driver/pull/461","status":"judged","expected":1,"caught":1,"seconds":80.046,"review_url":null},{"case_id":"open-telemetry-opentelemetry-collector-contrib-37106","repository":"opentelemetry-collector-contrib","number":37106,"url":"https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/37106","status":"judged","expected":1,"caught":1,"seconds":198.806,"review_url":null},{"case_id":"outline-outline-8534","repository":"outline","number":8534,"url":"https://github.com/outline/outline/pull/8534","status":"judged","expected":1,"caught":1,"seconds":84.476,"review_url":null},{"case_id":"plonky3-plonky3-1969","repository":"Plonky3","number":1969,"url":"https://github.com/Plonky3/Plonky3/pull/1969","status":"judged","expected":1,"caught":1,"seconds":146.43,"review_url":null},{"case_id":"software-mansion-react-native-reanimated-8929","repository":"react-native-reanimated","number":8929,"url":"https://github.com/software-mansion/react-native-reanimated/pull/8929","status":"judged","expected":1,"caught":1,"seconds":521.156,"review_url":null},{"case_id":"astral-sh-ruff-28610","repository":"ruff","number":28610,"url":"https://github.com/astral-sh/ruff/pull/28610","status":"judged","expected":0,"caught":0,"seconds":308.571,"review_url":null},{"case_id":"home-assistant-supervisor-6807","repository":"supervisor","number":6807,"url":"https://github.com/home-assistant/supervisor/pull/6807","status":"judged","expected":0,"caught":0,"seconds":50.954,"review_url":null},{"case_id":"wxsms-uiv-903","repository":"uiv","number":903,"url":"https://github.com/wxsms/uiv/pull/903","status":"judged","expected":0,"caught":0,"seconds":29.435,"review_url":null}]}}},{"id":"ionwarp-glm-5-3-flash","name":"IonWarp · GLM 5.3 Flash","label":"IonWarp · GLM 5.3 Flash","kind":"ionwarp","model":"z-ai/glm-5.3-flash","model_label":"GLM 5.3 Flash","developer":"z-ai","effort":"low","composite":53.5,"composite_parts":{"f1":37,"latency":56,"cost":100},"f1":0.55,"f2":0.6962,"precision":0.4074,"recall":0.8462,"tp":11,"fp":16,"fn":2,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.55,"latency_p50_seconds":49.7,"latency_p90_seconds":215.2,"display_latency_seconds":49.7,"cost_per_review_usd":0.0027,"cost_basis":"observed_inference","chart_cost":0.0027,"run_dates":["2026-09-21","2026-09-21"],"popovers":{"f1":{"f2":0.6962,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":37,"latency":56,"cost":100}},"prs":{"cases":[{"case_id":"microsoft-aspire.dev-964","repository":"aspire.dev","number":964,"url":"https://github.com/microsoft/aspire.dev/pull/964","status":"judged","expected":0,"caught":0,"seconds":73.033,"review_url":null},{"case_id":"django-django-21457","repository":"django","number":21457,"url":"https://github.com/django/django/pull/21457","status":"judged","expected":1,"caught":1,"seconds":56.868,"review_url":null},{"case_id":"esphome-esphome-9396","repository":"esphome","number":9396,"url":"https://github.com/esphome/esphome/pull/9396","status":"judged","expected":1,"caught":1,"seconds":158.511,"review_url":null},{"case_id":"mozilla-gcp-ingestion-2901","repository":"gcp-ingestion","number":2901,"url":"https://github.com/mozilla/gcp-ingestion/pull/2901","status":"judged","expected":1,"caught":1,"seconds":32.579,"review_url":null},{"case_id":"herdrdev-herdr-4023","repository":"herdr","number":4023,"url":"https://github.com/herdrdev/herdr/pull/4023","status":"judged","expected":1,"caught":0,"seconds":23.619,"review_url":null},{"case_id":"juspay-hyperswitch-14081","repository":"hyperswitch","number":14081,"url":"https://github.com/juspay/hyperswitch/pull/14081","status":"judged","expected":1,"caught":1,"seconds":42.542,"review_url":null},{"case_id":"home-assistant-ios-5002","repository":"iOS","number":5002,"url":"https://github.com/home-assistant/iOS/pull/5002","status":"judged","expected":1,"caught":0,"seconds":386.845,"review_url":null},{"case_id":"kiali-kiali-10167","repository":"kiali","number":10167,"url":"https://github.com/kiali/kiali/pull/10167","status":"judged","expected":1,"caught":1,"seconds":94.123,"review_url":null},{"case_id":"touchlab-kmmbridge-266","repository":"KMMBridge","number":266,"url":"https://github.com/touchlab/KMMBridge/pull/266","status":"judged","expected":1,"caught":1,"seconds":28.343,"review_url":null},{"case_id":"marimo-team-marimo-9312","repository":"marimo","number":9312,"url":"https://github.com/marimo-team/marimo/pull/9312","status":"judged","expected":0,"caught":0,"seconds":22.924,"review_url":null},{"case_id":"awslabs-mountpoint-s3-csi-driver-461","repository":"mountpoint-s3-csi-driver","number":461,"url":"https://github.com/awslabs/mountpoint-s3-csi-driver/pull/461","status":"judged","expected":1,"caught":1,"seconds":76.271,"review_url":null},{"case_id":"open-telemetry-opentelemetry-collector-contrib-37106","repository":"opentelemetry-collector-contrib","number":37106,"url":"https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/37106","status":"judged","expected":1,"caught":1,"seconds":215.164,"review_url":null},{"case_id":"outline-outline-8534","repository":"outline","number":8534,"url":"https://github.com/outline/outline/pull/8534","status":"judged","expected":1,"caught":1,"seconds":129.985,"review_url":null},{"case_id":"plonky3-plonky3-1969","repository":"Plonky3","number":1969,"url":"https://github.com/Plonky3/Plonky3/pull/1969","status":"judged","expected":1,"caught":1,"seconds":31.865,"review_url":null},{"case_id":"software-mansion-react-native-reanimated-8929","repository":"react-native-reanimated","number":8929,"url":"https://github.com/software-mansion/react-native-reanimated/pull/8929","status":"judged","expected":1,"caught":1,"seconds":90.796,"review_url":null},{"case_id":"astral-sh-ruff-28610","repository":"ruff","number":28610,"url":"https://github.com/astral-sh/ruff/pull/28610","status":"judged","expected":0,"caught":0,"seconds":31.881,"review_url":null},{"case_id":"home-assistant-supervisor-6807","repository":"supervisor","number":6807,"url":"https://github.com/home-assistant/supervisor/pull/6807","status":"judged","expected":0,"caught":0,"seconds":21.222,"review_url":null},{"case_id":"wxsms-uiv-903","repository":"uiv","number":903,"url":"https://github.com/wxsms/uiv/pull/903","status":"judged","expected":0,"caught":0,"seconds":15.384,"review_url":null}]}}},{"id":"ionwarp-deepseek-v4-1-flash","name":"IonWarp · DeepSeek V4.1 Flash","label":"IonWarp · DeepSeek V4.1 Flash","kind":"ionwarp","model":"deepseek/deepseek-v4.1-flash","model_label":"DeepSeek V4.1 Flash","developer":"deepseek","effort":"medium","composite":51.2,"composite_parts":{"f1":53,"latency":46,"cost":51},"f1":0.5946,"f2":0.7237,"precision":0.4583,"recall":0.8462,"tp":11,"fp":13,"fn":2,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.5946,"latency_p50_seconds":54.1,"latency_p90_seconds":661,"display_latency_seconds":54.1,"cost_per_review_usd":0.0095,"cost_basis":"observed_inference","chart_cost":0.0095,"run_dates":["2026-09-21","2026-09-21"],"popovers":{"f1":{"f2":0.7237,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":53,"latency":46,"cost":51}},"prs":{"cases":[{"case_id":"microsoft-aspire.dev-964","repository":"aspire.dev","number":964,"url":"https://github.com/microsoft/aspire.dev/pull/964","status":"judged","expected":0,"caught":0,"seconds":24.345,"review_url":null},{"case_id":"django-django-21457","repository":"django","number":21457,"url":"https://github.com/django/django/pull/21457","status":"judged","expected":1,"caught":1,"seconds":46.898,"review_url":null},{"case_id":"esphome-esphome-9396","repository":"esphome","number":9396,"url":"https://github.com/esphome/esphome/pull/9396","status":"judged","expected":1,"caught":1,"seconds":802.019,"review_url":null},{"case_id":"mozilla-gcp-ingestion-2901","repository":"gcp-ingestion","number":2901,"url":"https://github.com/mozilla/gcp-ingestion/pull/2901","status":"judged","expected":1,"caught":1,"seconds":23.602,"review_url":null},{"case_id":"herdrdev-herdr-4023","repository":"herdr","number":4023,"url":"https://github.com/herdrdev/herdr/pull/4023","status":"judged","expected":1,"caught":1,"seconds":205.366,"review_url":null},{"case_id":"juspay-hyperswitch-14081","repository":"hyperswitch","number":14081,"url":"https://github.com/juspay/hyperswitch/pull/14081","status":"judged","expected":1,"caught":1,"seconds":113.142,"review_url":null},{"case_id":"home-assistant-ios-5002","repository":"iOS","number":5002,"url":"https://github.com/home-assistant/iOS/pull/5002","status":"judged","expected":1,"caught":0,"seconds":490.443,"review_url":null},{"case_id":"kiali-kiali-10167","repository":"kiali","number":10167,"url":"https://github.com/kiali/kiali/pull/10167","status":"judged","expected":1,"caught":1,"seconds":25.219,"review_url":null},{"case_id":"touchlab-kmmbridge-266","repository":"KMMBridge","number":266,"url":"https://github.com/touchlab/KMMBridge/pull/266","status":"judged","expected":1,"caught":1,"seconds":56.348,"review_url":null},{"case_id":"marimo-team-marimo-9312","repository":"marimo","number":9312,"url":"https://github.com/marimo-team/marimo/pull/9312","status":"judged","expected":0,"caught":0,"seconds":141.904,"review_url":null},{"case_id":"awslabs-mountpoint-s3-csi-driver-461","repository":"mountpoint-s3-csi-driver","number":461,"url":"https://github.com/awslabs/mountpoint-s3-csi-driver/pull/461","status":"judged","expected":1,"caught":1,"seconds":57.663,"review_url":null},{"case_id":"open-telemetry-opentelemetry-collector-contrib-37106","repository":"opentelemetry-collector-contrib","number":37106,"url":"https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/37106","status":"judged","expected":1,"caught":1,"seconds":22.866,"review_url":null},{"case_id":"outline-outline-8534","repository":"outline","number":8534,"url":"https://github.com/outline/outline/pull/8534","status":"judged","expected":1,"caught":1,"seconds":23.907,"review_url":null},{"case_id":"plonky3-plonky3-1969","repository":"Plonky3","number":1969,"url":"https://github.com/Plonky3/Plonky3/pull/1969","status":"judged","expected":1,"caught":1,"seconds":72.897,"review_url":null},{"case_id":"software-mansion-react-native-reanimated-8929","repository":"react-native-reanimated","number":8929,"url":"https://github.com/software-mansion/react-native-reanimated/pull/8929","status":"judged","expected":1,"caught":0,"seconds":44.927,"review_url":null},{"case_id":"astral-sh-ruff-28610","repository":"ruff","number":28610,"url":"https://github.com/astral-sh/ruff/pull/28610","status":"judged","expected":0,"caught":0,"seconds":51.946,"review_url":null},{"case_id":"home-assistant-supervisor-6807","repository":"supervisor","number":6807,"url":"https://github.com/home-assistant/supervisor/pull/6807","status":"judged","expected":0,"caught":0,"seconds":661.007,"review_url":null},{"case_id":"wxsms-uiv-903","repository":"uiv","number":903,"url":"https://github.com/wxsms/uiv/pull/903","status":"judged","expected":0,"caught":0,"seconds":15.576,"review_url":null}]}}},{"id":"ionwarp-gpt-5-6-luna-pro","name":"IonWarp · GPT 5.6 Luna Pro","label":"IonWarp · GPT 5.6 Luna Pro","kind":"ionwarp","model":"openai/gpt-5.6-luna-pro","model_label":"GPT 5.6 Luna Pro","developer":"openai","effort":"medium","composite":48.5,"composite_parts":{"f1":36,"latency":100,"cost":35},"f1":0.5455,"f2":0.625,"precision":0.45,"recall":0.6923,"tp":9,"fp":11,"fn":4,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.5455,"latency_p50_seconds":33.2,"latency_p90_seconds":53.1,"display_latency_seconds":33.2,"cost_per_review_usd":0.0141,"cost_basis":"observed_inference","chart_cost":0.0141,"run_dates":["2026-09-21","2026-09-21"],"popovers":{"f1":{"f2":0.625,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":36,"latency":100,"cost":35}},"prs":{"cases":[{"case_id":"microsoft-aspire.dev-964","repository":"aspire.dev","number":964,"url":"https://github.com/microsoft/aspire.dev/pull/964","status":"judged","expected":0,"caught":0,"seconds":38.785,"review_url":null},{"case_id":"django-django-21457","repository":"django","number":21457,"url":"https://github.com/django/django/pull/21457","status":"judged","expected":1,"caught":1,"seconds":41.038,"review_url":null},{"case_id":"esphome-esphome-9396","repository":"esphome","number":9396,"url":"https://github.com/esphome/esphome/pull/9396","status":"judged","expected":1,"caught":1,"seconds":35.31,"review_url":null},{"case_id":"mozilla-gcp-ingestion-2901","repository":"gcp-ingestion","number":2901,"url":"https://github.com/mozilla/gcp-ingestion/pull/2901","status":"judged","expected":1,"caught":1,"seconds":32.643,"review_url":null},{"case_id":"herdrdev-herdr-4023","repository":"herdr","number":4023,"url":"https://github.com/herdrdev/herdr/pull/4023","status":"judged","expected":1,"caught":0,"seconds":31.629,"review_url":null},{"case_id":"juspay-hyperswitch-14081","repository":"hyperswitch","number":14081,"url":"https://github.com/juspay/hyperswitch/pull/14081","status":"judged","expected":1,"caught":0,"seconds":28.92,"review_url":null},{"case_id":"home-assistant-ios-5002","repository":"iOS","number":5002,"url":"https://github.com/home-assistant/iOS/pull/5002","status":"judged","expected":1,"caught":0,"seconds":24.033,"review_url":null},{"case_id":"kiali-kiali-10167","repository":"kiali","number":10167,"url":"https://github.com/kiali/kiali/pull/10167","status":"judged","expected":1,"caught":1,"seconds":24.835,"review_url":null},{"case_id":"touchlab-kmmbridge-266","repository":"KMMBridge","number":266,"url":"https://github.com/touchlab/KMMBridge/pull/266","status":"judged","expected":1,"caught":0,"seconds":53.119,"review_url":null},{"case_id":"marimo-team-marimo-9312","repository":"marimo","number":9312,"url":"https://github.com/marimo-team/marimo/pull/9312","status":"judged","expected":0,"caught":0,"seconds":33.829,"review_url":null},{"case_id":"awslabs-mountpoint-s3-csi-driver-461","repository":"mountpoint-s3-csi-driver","number":461,"url":"https://github.com/awslabs/mountpoint-s3-csi-driver/pull/461","status":"judged","expected":1,"caught":1,"seconds":34.688,"review_url":null},{"case_id":"open-telemetry-opentelemetry-collector-contrib-37106","repository":"opentelemetry-collector-contrib","number":37106,"url":"https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/37106","status":"judged","expected":1,"caught":1,"seconds":42.482,"review_url":null},{"case_id":"outline-outline-8534","repository":"outline","number":8534,"url":"https://github.com/outline/outline/pull/8534","status":"judged","expected":1,"caught":1,"seconds":52.779,"review_url":null},{"case_id":"plonky3-plonky3-1969","repository":"Plonky3","number":1969,"url":"https://github.com/Plonky3/Plonky3/pull/1969","status":"judged","expected":1,"caught":1,"seconds":29.545,"review_url":null},{"case_id":"software-mansion-react-native-reanimated-8929","repository":"react-native-reanimated","number":8929,"url":"https://github.com/software-mansion/react-native-reanimated/pull/8929","status":"judged","expected":1,"caught":1,"seconds":57.693,"review_url":null},{"case_id":"astral-sh-ruff-28610","repository":"ruff","number":28610,"url":"https://github.com/astral-sh/ruff/pull/28610","status":"judged","expected":0,"caught":0,"seconds":23.448,"review_url":null},{"case_id":"home-assistant-supervisor-6807","repository":"supervisor","number":6807,"url":"https://github.com/home-assistant/supervisor/pull/6807","status":"judged","expected":0,"caught":0,"seconds":30.002,"review_url":null},{"case_id":"wxsms-uiv-903","repository":"uiv","number":903,"url":"https://github.com/wxsms/uiv/pull/903","status":"judged","expected":0,"caught":0,"seconds":19.298,"review_url":null}]}}},{"id":"ionwarp-muse-spark-1-3","name":"IonWarp · Muse Spark 1.3","label":"IonWarp · Muse Spark 1.3","kind":"ionwarp","model":"meta/muse-spark-1.3","model_label":"Muse Spark 1.3","developer":"meta","effort":"medium","composite":11.6,"composite_parts":{"f1":0,"latency":58,"cost":0},"f1":0.4444,"f2":0.4545,"precision":0.4286,"recall":0.4615,"tp":6,"fp":8,"fn":7,"expected":13,"prs":18,"coverage":{"graded":18,"cohort":18},"display_f1":0.4444,"latency_p50_seconds":48.8,"latency_p90_seconds":102.4,"display_latency_seconds":48.8,"cost_per_review_usd":0.0344,"cost_basis":"observed_inference","chart_cost":0.0344,"run_dates":["2026-09-21","2026-09-21"],"popovers":{"f1":{"f2":0.4545,"cohort_of":18},"cost":{"estimate":false},"score":{"parts":{"f1":0,"latency":58,"cost":0}},"prs":{"cases":[{"case_id":"microsoft-aspire.dev-964","repository":"aspire.dev","number":964,"url":"https://github.com/microsoft/aspire.dev/pull/964","status":"judged","expected":0,"caught":0,"seconds":69.696,"review_url":null},{"case_id":"django-django-21457","repository":"django","number":21457,"url":"https://github.com/django/django/pull/21457","status":"judged","expected":1,"caught":1,"seconds":75.385,"review_url":null},{"case_id":"esphome-esphome-9396","repository":"esphome","number":9396,"url":"https://github.com/esphome/esphome/pull/9396","status":"judged","expected":1,"caught":1,"seconds":49.663,"review_url":null},{"case_id":"mozilla-gcp-ingestion-2901","repository":"gcp-ingestion","number":2901,"url":"https://github.com/mozilla/gcp-ingestion/pull/2901","status":"judged","expected":1,"caught":0,"seconds":59.762,"review_url":null},{"case_id":"herdrdev-herdr-4023","repository":"herdr","number":4023,"url":"https://github.com/herdrdev/herdr/pull/4023","status":"judged","expected":1,"caught":0,"seconds":37.166,"review_url":null},{"case_id":"juspay-hyperswitch-14081","repository":"hyperswitch","number":14081,"url":"https://github.com/juspay/hyperswitch/pull/14081","status":"judged","expected":1,"caught":0,"seconds":40.252,"review_url":null},{"case_id":"home-assistant-ios-5002","repository":"iOS","number":5002,"url":"https://github.com/home-assistant/iOS/pull/5002","status":"judged","expected":1,"caught":0,"seconds":33.177,"review_url":null},{"case_id":"kiali-kiali-10167","repository":"kiali","number":10167,"url":"https://github.com/kiali/kiali/pull/10167","status":"judged","expected":1,"caught":1,"seconds":62.509,"review_url":null},{"case_id":"touchlab-kmmbridge-266","repository":"KMMBridge","number":266,"url":"https://github.com/touchlab/KMMBridge/pull/266","status":"judged","expected":1,"caught":1,"seconds":67.612,"review_url":null},{"case_id":"marimo-team-marimo-9312","repository":"marimo","number":9312,"url":"https://github.com/marimo-team/marimo/pull/9312","status":"judged","expected":0,"caught":0,"seconds":45.116,"review_url":null},{"case_id":"awslabs-mountpoint-s3-csi-driver-461","repository":"mountpoint-s3-csi-driver","number":461,"url":"https://github.com/awslabs/mountpoint-s3-csi-driver/pull/461","status":"judged","expected":1,"caught":0,"seconds":47.87,"review_url":null},{"case_id":"open-telemetry-opentelemetry-collector-contrib-37106","repository":"opentelemetry-collector-contrib","number":37106,"url":"https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/37106","status":"judged","expected":1,"caught":1,"seconds":75.029,"review_url":null},{"case_id":"outline-outline-8534","repository":"outline","number":8534,"url":"https://github.com/outline/outline/pull/8534","status":"judged","expected":1,"caught":1,"seconds":108.432,"review_url":null},{"case_id":"plonky3-plonky3-1969","repository":"Plonky3","number":1969,"url":"https://github.com/Plonky3/Plonky3/pull/1969","status":"judged","expected":1,"caught":0,"seconds":31.992,"review_url":null},{"case_id":"software-mansion-react-native-reanimated-8929","repository":"react-native-reanimated","number":8929,"url":"https://github.com/software-mansion/react-native-reanimated/pull/8929","status":"judged","expected":1,"caught":0,"seconds":102.446,"review_url":null},{"case_id":"astral-sh-ruff-28610","repository":"ruff","number":28610,"url":"https://github.com/astral-sh/ruff/pull/28610","status":"judged","expected":0,"caught":0,"seconds":34.716,"review_url":null},{"case_id":"home-assistant-supervisor-6807","repository":"supervisor","number":6807,"url":"https://github.com/home-assistant/supervisor/pull/6807","status":"judged","expected":0,"caught":0,"seconds":34.342,"review_url":null},{"case_id":"wxsms-uiv-903","repository":"uiv","number":903,"url":"https://github.com/wxsms/uiv/pull/903","status":"judged","expected":0,"caught":0,"seconds":31.095,"review_url":null}]}}}]}