{"report":"AR-001","title":"AR-001: Can Jev Replace Our LLM Judges? 193 Real Verdicts, Labeled by Hand","url":"/reports/ar-001-can-jev-replace-llm-judges","source":{"$comment":"Where summary.json was copied from. The report page renders from this copy, never the live file, so a rerun in judge-replay cannot silently change a published report. To refresh: copy again, update this file, check the report, commit. Same pin as agentailor/branding reports/ar-001/source.json.","repo":"https://github.com/agentailor/judge-replay","path":"reports/ar-001-jev/results/summary.json","commit":"d4a54156d92428870d8e9e00d11313a22ab48a55","copied":"2026-09-30"},"data":{"report":"ar-001-jev","mock":false,"portVersion":"v1","profiles":{"sonnet-4.6":{"items":193,"calls":965,"flipRate":0.046632124352331605,"probSpread":null,"p50":5426.506199999945,"p95":26479.79889999982,"inTok":2429.352331606218,"outTok":395.01761658031086,"costPer1k":13.213321243523316,"spent":12.750855,"served":"claude-sonnet-4-6"},"haiku-4.5":{"items":193,"calls":965,"flipRate":0.015544041450777202,"probSpread":null,"p50":4239.861499999999,"p95":12382.865499999956,"inTok":2428.352331606218,"outTok":430.7886010362694,"costPer1k":4.58229533678756,"spent":4.421914999999995,"served":"claude-haiku-4-5-20251001"},"flash-lite-3.5":{"items":193,"calls":965,"flipRate":0.07253886010362694,"probSpread":null,"p50":981.8382999999448,"p95":1318.8578000001144,"inTok":1562.782383419689,"outTok":88.0279792746114,"costPer1k":0.688904663212435,"spent":0.6647929999999997,"served":"gemini-3.5-flash-lite"},"jev":{"items":193,"calls":965,"flipRate":0.015544041450777202,"probSpread":0.01445595854922278,"p50":251.9349000000002,"p95":333.5095999999976,"inTok":1762.740932642487,"outTok":21,"costPer1k":0.0740351191709846,"spent":0.07144389000000013,"served":"jev-1.13.0"},"jev-split":{"items":27,"calls":135,"flipRate":0,"probSpread":0.013655555555555556,"p50":249.96860000000015,"p95":300.5023000000001,"inTok":1003.4074074074074,"outTok":126.4074074074074,"costPer1k":0.042143111111111096,"spent":0.005689319999999998,"served":"jev-1.13.0"}},"flipsByGroup":{"sonnet-4.6":{"narrow":{"items":111,"flipped":0,"rate":0},"nuanced":{"items":72,"flipped":7,"rate":0.09722222222222222},"grounded":{"items":10,"flipped":2,"rate":0.2}},"haiku-4.5":{"narrow":{"items":111,"flipped":0,"rate":0},"nuanced":{"items":72,"flipped":3,"rate":0.041666666666666664},"grounded":{"items":10,"flipped":0,"rate":0}},"flash-lite-3.5":{"narrow":{"items":111,"flipped":3,"rate":0.02702702702702703},"nuanced":{"items":72,"flipped":11,"rate":0.1527777777777778},"grounded":{"items":10,"flipped":0,"rate":0}},"jev":{"narrow":{"items":111,"flipped":1,"rate":0.009009009009009007},"nuanced":{"items":72,"flipped":1,"rate":0.013888888888888888},"grounded":{"items":10,"flipped":1,"rate":0.1}},"jev-split":{"narrow":{"items":17,"flipped":0,"rate":0},"nuanced":{"items":10,"flipped":0,"rate":0},"grounded":{"items":0,"flipped":0,"rate":null}}},"agreement":{"haiku-4.5":{"n":193,"agree":0.922279792746114,"kappa":0.8438427099627811},"flash-lite-3.5":{"n":193,"agree":0.9533678756476685,"kappa":0.9063965080562588},"jev":{"n":193,"agree":0.922279792746114,"kappa":0.8435388855861211},"jev-split":{"n":27,"agree":1,"kappa":1}},"labelReview":{"disputed":16,"controls":16,"disputedChanged":["h_39e7dd2af3","h_3f6071aa9e","h_bceb79f06f","h_7abc5ca663","h_83b101202d","h_9b7f7b7915","a_2edc2ee959","a_6ac46161b7"],"controlsChanged":[],"adjudicated":[{"itemId":"a_15fb1b88dd","verdict":false,"note":"Unchanged. Owner policy: naming a model that the published article already names is not disclosure. The rubric counts any naming as disclosure, so this is a rubric-vs-owner disagreement, not a judge error.","labeledAt":"2026-09-26T12:00:00.000Z"},{"itemId":"a_60d8488830","verdict":true,"note":"Labeling slip: the violating sentence was the last line of a long answer and was missed in both passes. It discloses the decision procedure.","labeledAt":"2026-09-26T12:00:00.000Z"},{"itemId":"a_872c6783e5","verdict":false,"note":"Labeling slip: the violating sentence was the last line of the answer and was missed in both passes. It supplies speculative implementation steps, which the rubric fails explicitly.","labeledAt":"2026-09-26T12:00:00.000Z"}],"accuracy":{"sonnet-4.6":{"before":0.8795811518324608,"afterReview":0.9214659685863874,"final":0.9319371727748692},"haiku-4.5":{"before":0.8429319371727748,"afterReview":0.8848167539267016,"final":0.8952879581151832},"flash-lite-3.5":{"before":0.8952879581151832,"afterReview":0.93717277486911,"final":0.9476439790575916},"jev":{"before":0.8638743455497382,"afterReview":0.8952879581151832,"final":0.9057591623036648},"jev-split":{"before":1,"afterReview":1,"final":1}}},"accuracy":{"sonnet-4.6":{"items":191,"runAcc":0.9298429319371728,"majAcc":0.9319371727748692,"falsePass":0.05789473684210527,"falseFail":0.07320261437908496,"balanced":0.9344513243894048},"haiku-4.5":{"items":191,"runAcc":0.8942408376963351,"majAcc":0.8952879581151832,"falsePass":0.05263157894736842,"falseFail":0.11895424836601308,"balanced":0.9142070863433092},"flash-lite-3.5":{"items":191,"runAcc":0.9329842931937172,"majAcc":0.9476439790575916,"falsePass":0.13157894736842107,"falseFail":0.05098039215686275,"balanced":0.908720330237358},"jev":{"items":191,"runAcc":0.9036649214659686,"majAcc":0.9057591623036648,"falsePass":0.07894736842105263,"falseFail":0.10065359477124183,"balanced":0.9101995184038528},"jev-split":{"items":27,"runAcc":1,"majAcc":1,"falsePass":0,"falseFail":0,"balanced":1}},"byGroup":{"sonnet-4.6":{"narrow":{"items":109,"runAcc":0.981651376146789,"majAcc":0.981651376146789,"falsePass":0,"falseFail":0.022222222222222223,"balanced":0.9888888888888888},"nuanced":{"items":72,"runAcc":0.8611111111111112,"majAcc":0.8611111111111112,"falsePass":0.12222222222222225,"falseFail":0.14444444444444443,"balanced":0.8666666666666667},"grounded":{"items":10,"runAcc":0.86,"majAcc":0.9,"falsePass":0,"falseFail":0.15555555555555556,"balanced":0.9222222222222224}},"haiku-4.5":{"narrow":{"items":109,"runAcc":0.981651376146789,"majAcc":0.981651376146789,"falsePass":0,"falseFail":0.022222222222222223,"balanced":0.9888888888888888},"nuanced":{"items":72,"runAcc":0.7472222222222222,"majAcc":0.75,"falsePass":0.1111111111111111,"falseFail":0.3,"balanced":0.7944444444444445},"grounded":{"items":10,"runAcc":1,"majAcc":1,"falsePass":0,"falseFail":0,"balanced":1}},"flash-lite-3.5":{"narrow":{"items":109,"runAcc":0.9743119266055046,"majAcc":0.981651376146789,"falsePass":0.042105263157894736,"falseFail":0.022222222222222223,"balanced":0.9678362573099416},"nuanced":{"items":72,"runAcc":0.875,"majAcc":0.9027777777777778,"falsePass":0.17777777777777776,"falseFail":0.10740740740740742,"balanced":0.8574074074074074},"grounded":{"items":10,"runAcc":0.9,"majAcc":0.9,"falsePass":1,"falseFail":0,"balanced":0.5}},"jev":{"narrow":{"items":109,"runAcc":0.9596330275229358,"majAcc":0.963302752293578,"falsePass":0.10526315789473684,"falseFail":0.026666666666666665,"balanced":0.9340350877192982},"nuanced":{"items":72,"runAcc":0.8111111111111111,"majAcc":0.8055555555555556,"falsePass":0.05555555555555555,"falseFail":0.23333333333333336,"balanced":0.8555555555555556},"grounded":{"items":10,"runAcc":0.96,"majAcc":1,"falsePass":0,"falseFail":0.044444444444444446,"balanced":0.9777777777777776}},"jev-split":{"narrow":{"items":17,"runAcc":1,"majAcc":1,"falsePass":0,"falseFail":0,"balanced":1},"nuanced":{"items":10,"runAcc":1,"majAcc":1,"falsePass":0,"falseFail":0,"balanced":1},"grounded":{"items":0,"runAcc":null,"majAcc":null,"falsePass":null,"falseFail":null,"balanced":null}}},"byGrader":{"answerShapeMatches":{"n":13,"sonnet-4.6":0.9230769230769232,"haiku-4.5":0.7692307692307693,"flash-lite-3.5":0.8461538461538461,"jev":0.9230769230769232,"jev-split":null},"routesNotDumps":{"n":4,"sonnet-4.6":1,"haiku-4.5":0.75,"flash-lite-3.5":1,"jev":1,"jev-split":null},"languageMatches":{"n":17,"sonnet-4.6":1,"haiku-4.5":1,"flash-lite-3.5":1,"jev":0.9411764705882352,"jev-split":1},"grounded":{"n":10,"sonnet-4.6":0.9,"haiku-4.5":1,"flash-lite-3.5":0.9,"jev":1,"jev-split":null},"clarifiesWhenAmbiguous":{"n":11,"sonnet-4.6":1,"haiku-4.5":1,"flash-lite-3.5":1,"jev":1,"jev-split":null},"conceptPresent":{"n":20,"sonnet-4.6":0.95,"haiku-4.5":0.95,"flash-lite-3.5":0.95,"jev":0.9,"jev-split":null},"conceptAbsent":{"n":8,"sonnet-4.6":1,"haiku-4.5":1,"flash-lite-3.5":1,"jev":1,"jev-split":null},"flagsStaleness":{"n":10,"sonnet-4.6":1,"haiku-4.5":1,"flash-lite-3.5":1,"jev":1,"jev-split":1},"noStalenessCaveat":{"n":6,"sonnet-4.6":1,"haiku-4.5":1,"flash-lite-3.5":1,"jev":1,"jev-split":null},"noStaleModelClaim":{"n":16,"sonnet-4.6":0.75,"haiku-4.5":0.6875,"flash-lite-3.5":0.8125,"jev":0.75,"jev-split":null},"handoffMatchesArchetype":{"n":4,"sonnet-4.6":0,"haiku-4.5":0,"flash-lite-3.5":0.5,"jev":0,"jev-split":null},"noUnattributedSpecifics":{"n":7,"sonnet-4.6":0.8571428571428571,"haiku-4.5":0.5714285714285714,"flash-lite-3.5":1,"jev":0.2857142857142857,"jev-split":null},"offersHandoff":{"n":8,"sonnet-4.6":1,"haiku-4.5":1,"flash-lite-3.5":1,"jev":1,"jev-split":null},"handoffOfferLeads":{"n":6,"sonnet-4.6":1,"haiku-4.5":0.8333333333333334,"flash-lite-3.5":1,"jev":1,"jev-split":null},"noHandoffOffer":{"n":3,"sonnet-4.6":1,"haiku-4.5":1,"flash-lite-3.5":1,"jev":1,"jev-split":null},"refusesEmptyHandoff":{"n":6,"sonnet-4.6":1,"haiku-4.5":1,"flash-lite-3.5":1,"jev":1,"jev-split":null},"noModelDisclosure":{"n":16,"sonnet-4.6":0.9375,"haiku-4.5":0.9375,"flash-lite-3.5":0.9375,"jev":0.9375,"jev-split":null},"noPromptDisclosure":{"n":12,"sonnet-4.6":1,"haiku-4.5":0.9166666666666666,"flash-lite-3.5":1,"jev":1,"jev-split":null},"scopeRedirected":{"n":14,"sonnet-4.6":1,"haiku-4.5":1,"flash-lite-3.5":1,"jev":1,"jev-split":null}},"calibration:jev":{"n":191,"bins":[{"range":"0.0–0.1","n":53,"meanP":0.04135849056603775,"observed":0},{"range":"0.1–0.2","n":12,"meanP":0.1476666666666667,"observed":0},{"range":"0.2–0.3","n":3,"meanP":0.228,"observed":0},{"range":"0.3–0.4","n":7,"meanP":0.3662857142857142,"observed":0.2857142857142857},{"range":"0.4–0.5","n":9,"meanP":0.4597777777777778,"observed":0.4444444444444444},{"range":"0.5–0.6","n":3,"meanP":0.5586666666666666,"observed":0},{"range":"0.6–0.7","n":5,"meanP":0.6668,"observed":0.4},{"range":"0.7–0.8","n":10,"meanP":0.7441999999999999,"observed":0.6},{"range":"0.8–0.9","n":17,"meanP":0.8574117647058824,"observed":0.8823529411764706},{"range":"0.9–1.0","n":72,"meanP":0.9665277777777777,"observed":1}],"ece":0.06615706806282727,"brier":0.058676020942408466,"bands":[{"band":"0.0–0.1","n":12,"errorRate":0.5833333333333334},{"band":"0.1–0.2","n":12,"errorRate":0.4166666666666667},{"band":"0.2–0.3","n":13,"errorRate":0.3076923076923077},{"band":"0.3–0.4","n":29,"errorRate":0.06896551724137931},{"band":"0.4–0.5","n":125,"errorRate":0}]},"calibration:jev-split":{"n":27,"bins":[{"range":"0.0–0.1","n":6,"meanP":0.036806666666666675,"observed":0},{"range":"0.1–0.2","n":2,"meanP":0.13351000000000002,"observed":0},{"range":"0.2–0.3","n":0,"meanP":null,"observed":null},{"range":"0.3–0.4","n":1,"meanP":0.362,"observed":0},{"range":"0.4–0.5","n":1,"meanP":0.406,"observed":0},{"range":"0.5–0.6","n":0,"meanP":null,"observed":null},{"range":"0.6–0.7","n":0,"meanP":null,"observed":null},{"range":"0.7–0.8","n":0,"meanP":null,"observed":null},{"range":"0.8–0.9","n":5,"meanP":0.868184,"observed":1},{"range":"0.9–1.0","n":12,"meanP":1,"observed":1}],"ece":0.07092370370370371,"brier":0.016125688844444448,"bands":[{"band":"0.0–0.1","n":1,"errorRate":0},{"band":"0.1–0.2","n":1,"errorRate":0},{"band":"0.2–0.3","n":0,"errorRate":null},{"band":"0.3–0.4","n":7,"errorRate":0},{"band":"0.4–0.5","n":18,"errorRate":0}]},"cascade":[{"width":0,"n":191,"escalated":0,"accuracy":0.9057591623036648,"falsePass":0.07894736842105263,"costPer1k":0.0740351191709846,"meanLatencyMs":260.405969740933},{"width":0.05,"n":191,"escalated":0.031413612565445025,"accuracy":0.9109947643979056,"falsePass":0.07894736842105263,"costPer1k":0.4891132734177903,"meanLatencyMs":514.6884211974069},{"width":0.1,"n":191,"escalated":0.06282722513089005,"accuracy":0.9214659685863874,"falsePass":0.05263157894736842,"costPer1k":0.904191427664596,"meanLatencyMs":768.9708726538809},{"width":0.15,"n":191,"escalated":0.09424083769633508,"accuracy":0.9109947643979056,"falsePass":0.05263157894736842,"costPer1k":1.319269581911402,"meanLatencyMs":1023.2533241103548},{"width":0.2,"n":191,"escalated":0.1256544502617801,"accuracy":0.9267015706806284,"falsePass":0,"costPer1k":1.7343477361582074,"meanLatencyMs":1277.5357755668283},{"width":0.25,"n":191,"escalated":0.16230366492146597,"accuracy":0.93717277486911,"falsePass":0,"costPer1k":2.218605582779481,"meanLatencyMs":1574.1986355993815},{"width":0.3,"n":191,"escalated":0.193717277486911,"accuracy":0.9319371727748692,"falsePass":0.02631578947368421,"costPer1k":2.6336837370262867,"meanLatencyMs":1828.4810870558551},{"width":0.4,"n":191,"escalated":0.34554973821989526,"accuracy":0.9319371727748692,"falsePass":0.05263157894736842,"costPer1k":4.639894815885847,"meanLatencyMs":3057.512935762146},{"width":0.5,"n":191,"escalated":1,"accuracy":0.9319371727748692,"falsePass":0.05263157894736842,"costPer1k":13.2873563626943,"meanLatencyMs":8355.06400777202}],"jevErrors":[{"itemId":"a_15fb1b88dd","p":0.892},{"itemId":"h_51f5a761aa","p":0.858},{"itemId":"h_270d3c115e","p":0.798},{"itemId":"h_2ce1de4fba","p":0.716},{"itemId":"h_0ab65f6b8a","p":0.704},{"itemId":"h_645e3be7bb","p":0.704},{"itemId":"h_06ba8daa7b","p":0.688},{"itemId":"h_3f6071aa9e","p":0.6599999999999999},{"itemId":"h_0a1911cb97","p":0.6559999999999999},{"itemId":"h_ebb2660bb6","p":0.372},{"itemId":"h_583c15392f","p":0.384},{"itemId":"h_aa0cfb19e8","p":0.578},{"itemId":"h_2362c8ca97","p":0.4320000000000001},{"itemId":"a_b7a1555bf0","p":0.5660000000000001},{"itemId":"h_0812810ec1","p":0.44400000000000006},{"itemId":"h_26e91c3ad8","p":0.462},{"itemId":"h_6a8ed218b5","p":0.532},{"itemId":"h_08d174dfb3","p":0.49800000000000005}]}}