{"repeat":1,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":2,"started_at":"2026-09-20T00:53:19.653Z","latency_ms":4213.186583,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4b2b6f25-6251-4b52-96d6-ae89274e8ac9"],"charged_cents":3,"answer":{"text":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."}]}}
{"repeat":1,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":3,"started_at":"2026-09-20T00:53:19.654Z","latency_ms":6393.505375000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["f38175dd-37ba-4fc1-83c8-0aed3042c95d"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":0,"started_at":"2026-09-20T00:53:19.652Z","latency_ms":8439.111832999999,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.72,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"e93293a7-6fec-47a3-9ed6-204f74c3b402","answer_transaction_ids":["b1ed1346-7a42-48f7-a94f-daec954f1dd5"],"charged_cents":1,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":1,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":1,"started_at":"2026-09-20T00:53:19.653Z","latency_ms":11452.848416,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.35,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"65514e59-9a3f-4285-b3f9-e68669419325","answer_transaction_ids":["4289e43a-df6f-499a-8dea-f765921cb476"],"charged_cents":4,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":1,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":0,"started_at":"2026-09-20T00:53:28.092Z","latency_ms":8980.076040999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["c713b307-5460-4a9e-97b7-67e47142a655"],"charged_cents":0,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":1,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":1,"started_at":"2026-09-20T00:53:31.106Z","latency_ms":8046.8213749999995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4a2a946e-acf6-4e03-8069-5a426a299d93"],"charged_cents":0,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":1,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":2,"started_at":"2026-09-20T00:53:23.868Z","latency_ms":17536.728667,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.29,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"78ad99c6-f3b7-4912-8f52-494b63182d53","answer_transaction_ids":["d7a57b09-e965-408b-af6a-4e0a6b1ce720"],"charged_cents":1,"answer":{"text":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."}]}}
{"repeat":1,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":3,"started_at":"2026-09-20T00:53:26.048Z","latency_ms":17167.36125,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.42,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"18d605d4-55b4-4113-b9ec-536486e7c475","answer_transaction_ids":["bfc63b8d-55fa-445b-ba16-7a6677ccfc99"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":7,"started_at":"2026-09-20T00:53:43.216Z","latency_ms":3911.7991250000014,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["89b107d6-6ff7-4d31-89b8-c0c04629e2d6"],"charged_cents":0,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."}]}}
{"repeat":1,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":4,"started_at":"2026-09-20T00:53:37.073Z","latency_ms":11289.595500000003,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.84,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"60c933dd-3279-40a9-93dc-4426837442e1","answer_transaction_ids":["85518768-f9d1-42d6-a152-6f1054073ca1"],"charged_cents":1,"answer":{"plan":"B","cost_dollars":7.8},"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":true,"actual":"B"},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":7.8}]}}
{"repeat":1,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":5,"started_at":"2026-09-20T00:53:39.154Z","latency_ms":10867.941584,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.49,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"8c16176f-4724-42bd-bb3e-eadd81b43e51","answer_transaction_ids":["5752d831-0763-4a97-98f6-15a6027c2ea1"],"charged_cents":1,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":1,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":6,"started_at":"2026-09-20T00:53:41.406Z","latency_ms":10954.445,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.35,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"eed4a5b9-5662-412a-be4e-4dfd8edf1336","answer_transaction_ids":["f0041c90-5ebd-4356-9494-9238f12a7d19"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":4,"started_at":"2026-09-20T00:53:48.363Z","latency_ms":7509.08525,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["1140ca8e-11c3-4f34-9585-158f8f7a5781"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":false,"actual":null},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":5,"started_at":"2026-09-20T00:53:50.022Z","latency_ms":7676.080707999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["bd84227f-a209-47b7-9122-d4263c37b10c"],"charged_cents":0,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":1,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":6,"started_at":"2026-09-20T00:53:52.362Z","latency_ms":7121.122666000003,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["86d8b3ed-6b4d-4866-9389-68571102d505"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":7,"started_at":"2026-09-20T00:53:47.128Z","latency_ms":14367.484625000005,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.24,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"e83c3dce-de84-4200-a101-c33db4743a80","answer_transaction_ids":["7ec24e20-67da-4f55-96a9-101edc4d843f"],"charged_cents":1,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."}]}}
{"repeat":1,"prompt_id":"lookup-04","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":8,"started_at":"2026-09-20T00:53:55.873Z","latency_ms":7426.293666999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["da5d61aa-644d-46d9-93f1-47d3ddc4e895"],"charged_cents":0,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":1,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":10,"started_at":"2026-09-20T00:53:59.483Z","latency_ms":6275.371749999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4ec2870f-1f8f-4efa-945b-7c24a3a1f5ac"],"charged_cents":0,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"lookup-05","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":11,"started_at":"2026-09-20T00:54:01.496Z","latency_ms":6133.390333000003,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["c1605c58-db98-4952-b4aa-5ab021f8b187"],"charged_cents":0,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":1,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":9,"started_at":"2026-09-20T00:53:57.698Z","latency_ms":13121.976999999999,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.59,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"18ce0b21-2546-4df0-be8b-dc91c36f8239","answer_transaction_ids":["b7513814-7700-4778-bdf2-c98970b0fc83"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-04","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":8,"started_at":"2026-09-20T00:54:03.299Z","latency_ms":9416.331541,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"bbaa8554-f0ab-4aea-95a3-71fabca510f0","answer_transaction_ids":["582d7f36-9efe-4fa8-adc3-ecb8f0d5bf2a"],"charged_cents":1,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":1,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":9,"started_at":"2026-09-20T00:54:10.821Z","latency_ms":4682.245624999996,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["72774c24-b459-4796-8cbf-7f9d9424a48f"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":10,"started_at":"2026-09-20T00:54:05.758Z","latency_ms":23877.383292,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.54,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"3ee52452-937d-4148-94b8-f3963bd9896f","answer_transaction_ids":["cf52a97e-3152-4546-b29b-ab76c97c2074"],"charged_cents":1,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"lookup-05","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":11,"started_at":"2026-09-20T00:54:07.631Z","latency_ms":24741.125833000006,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"03588e02-6a32-4d8e-abde-36e5febf44bc","answer_transaction_ids":["1af41579-2211-4a5c-ab1b-92b406ce9f94"],"charged_cents":1,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":1,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":12,"started_at":"2026-09-20T00:54:12.716Z","latency_ms":21638.36095799999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["29bc4a4b-29ef-4ecc-9c35-1b420786ae35"],"charged_cents":0,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":1,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":13,"started_at":"2026-09-20T00:54:15.504Z","latency_ms":20601.227166999997,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["e3e9e62a-295c-413e-b35d-54b5e4220116"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":14,"started_at":"2026-09-20T00:54:29.636Z","latency_ms":10785.287666999997,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.4,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0b14a21e-5836-4431-a811-2b60b9dea569","answer_transaction_ids":["926bf857-0f13-4a0c-b414-d12da65ecae4"],"charged_cents":1,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":1,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":15,"started_at":"2026-09-20T00:54:32.374Z","latency_ms":12336.057249999998,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.58,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"2dbb13ac-b42c-4788-b793-dc9f1e12e42f","answer_transaction_ids":["b69bab5c-676e-45ee-b792-7cfc1366c15d"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":12,"started_at":"2026-09-20T00:54:34.355Z","latency_ms":12178.475707999998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.7,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a7770b57-319e-45b0-b53c-4e4bd586c643","answer_transaction_ids":["9700ca0d-42ec-4b87-9432-d06aa98fdd9b"],"charged_cents":1,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":1,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":13,"started_at":"2026-09-20T00:54:36.106Z","latency_ms":12061.425541999997,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.32,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"188b5a37-fc70-4303-81ab-6458dfdeffae","answer_transaction_ids":["bec26629-203c-4449-8d41-4a016a7a5b3e"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":14,"started_at":"2026-09-20T00:54:40.422Z","latency_ms":9692.498708,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["dbfd43f3-bb89-4dfe-8124-f5e0ebce8c79"],"charged_cents":0,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":1,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":15,"started_at":"2026-09-20T00:54:44.711Z","latency_ms":9601.480083000002,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["29ca92a2-d866-4c4f-b152-ec870db256d5"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":16,"started_at":"2026-09-20T00:54:46.535Z","latency_ms":11159.425207999986,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.86,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"9e78ab61-11f3-4386-a326-71e6a56e8fa8","answer_transaction_ids":["48553267-9e91-453b-b6b5-006177dbd77a"],"charged_cents":1,"answer":{"unreserved_units":54},"grade":{"pass":true,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":true,"actual":54}]}}
{"repeat":1,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":17,"started_at":"2026-09-20T00:54:48.169Z","latency_ms":11456.164791999996,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.57,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"1605bffa-398b-490a-ab58-3e24630e2a80","answer_transaction_ids":["9778ed7a-c24f-4d16-873c-3fbcbe6c7460"],"charged_cents":1,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":18,"started_at":"2026-09-20T00:54:50.117Z","latency_ms":11967.125125000006,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.41,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a61d526f-78af-49f4-9508-f0aa0b69166c","answer_transaction_ids":["09e8609f-5209-4e96-aa1e-6f6602df1271"],"charged_cents":1,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":1,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":19,"started_at":"2026-09-20T00:54:54.313Z","latency_ms":11624.601875000008,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.7,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"3de22f1a-a764-4d4a-ae9b-86f5dfdfc31e","answer_transaction_ids":["5fc36035-40fb-4268-a6df-b2206cb2ec97"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":16,"started_at":"2026-09-20T00:54:57.695Z","latency_ms":11163.560624999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["b406c3e0-3fa6-4adc-a652-d43960e2d002"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":17,"started_at":"2026-09-20T00:54:59.626Z","latency_ms":11175.978709000003,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["b4b740bb-efbe-46bd-baaa-bf24f2dea262"],"charged_cents":0,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":18,"started_at":"2026-09-20T00:55:02.084Z","latency_ms":10619.052792000002,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4d8a2336-830b-4334-9fa1-c6fdaa83ea78"],"charged_cents":0,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":1,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":19,"started_at":"2026-09-20T00:55:05.939Z","latency_ms":12846.859333999993,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["5021694b-8365-49f8-95e7-b0dea1714f4d"],"charged_cents":3,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-01","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":20,"started_at":"2026-09-20T00:55:08.859Z","latency_ms":12265.59775,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["9c2b88b8-a053-40af-b1bd-64deadf21a59"],"charged_cents":0,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":1,"prompt_id":"lookup-06","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":21,"started_at":"2026-09-20T00:55:10.803Z","latency_ms":12343.3695,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["05322e7d-d0af-4952-9ff7-967ededdd8f0"],"charged_cents":0,"answer":{"day":"Thursday","time_utc":"09:30"},"grade":{"pass":true,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":true,"actual":"09:30"}]}}
{"repeat":1,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":22,"started_at":"2026-09-20T00:55:12.704Z","latency_ms":16425.132291999995,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.62,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"5f10e04d-0dee-4607-8988-1ff3ce8a86b7","answer_transaction_ids":["b62afdc3-8db4-487d-9af7-54653e5425a0"],"charged_cents":1,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":1,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":23,"started_at":"2026-09-20T00:55:18.787Z","latency_ms":11877.142957999997,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.88,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"b4d1cc39-5167-4bba-9d0a-7d98f939903f","answer_transaction_ids":["4bb94032-068b-4c01-9cbb-c5251b281815"],"charged_cents":1,"answer":{"numerator":7,"denominator":13},"grade":{"pass":true,"checks":[{"op":"eq","field":"numerator","value":7,"pass":true,"actual":7},{"op":"eq","field":"denominator","value":13,"pass":true,"actual":13}]}}
{"repeat":1,"prompt_id":"lookup-01","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":20,"started_at":"2026-09-20T00:55:21.125Z","latency_ms":12478.455791,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.86,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"38ded2f2-7af2-49c1-9d36-bdf5eb35186a","answer_transaction_ids":["553192be-0f25-4667-b37a-45288994d22e"],"charged_cents":1,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":1,"prompt_id":"lookup-06","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":21,"started_at":"2026-09-20T00:55:23.147Z","latency_ms":12123.07983399999,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"60e7cf40-e485-47a2-87bb-73a24c08b8bb","answer_transaction_ids":["fcffcd6e-bcc9-4fb5-a66c-139e4b07550c"],"charged_cents":2,"answer":{"day":"Thursday","time_utc":"09:30 UTC"},"grade":{"pass":false,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":false,"actual":"09:30 UTC"}]}}
{"repeat":1,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":22,"started_at":"2026-09-20T00:55:29.132Z","latency_ms":9512.772541999992,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["bf98e973-4c10-4136-b7ed-49fd8152787b"],"charged_cents":3,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":1,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":23,"started_at":"2026-09-20T00:55:30.666Z","latency_ms":11166.801875000005,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["6bbbaeac-a9ce-4b30-9051-31dbe99e1a4a"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"numerator","value":7,"pass":false,"actual":null},{"op":"eq","field":"denominator","value":13,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":24,"started_at":"2026-09-20T00:55:33.605Z","latency_ms":11892.888582999993,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["a3341ebc-31ca-42ab-b806-75ba1b294afa"],"charged_cents":3,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-02","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":26,"started_at":"2026-09-20T00:55:38.646Z","latency_ms":9431.267041999992,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["55803fb6-10fb-4f62-b767-111fcf2fe796"],"charged_cents":0,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":1,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":25,"started_at":"2026-09-20T00:55:35.270Z","latency_ms":18170.06370900001,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.5,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"c8a5c9ff-1976-4ed4-a561-4b4f20255ccc","answer_transaction_ids":["89dca8db-6661-48c0-a3ec-53f3e1e1fbf5"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":27,"started_at":"2026-09-20T00:55:41.833Z","latency_ms":16699.90295800002,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.78,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"f5ad6d28-a1de-478c-852b-397eb5db1fbc","answer_transaction_ids":["5d0ccb16-9e6b-42cd-990d-9156ec5eae3f"],"charged_cents":1,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":1,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":24,"started_at":"2026-09-20T00:55:45.498Z","latency_ms":14662.772999999986,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.92,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"1ca5cf99-3b39-41bc-8c05-a29903212815","answer_transaction_ids":["e07c44f3-41f7-4d3a-8f39-af5d892c55bf"],"charged_cents":1,"answer":{"total_dollars":66.6},"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":66.6}]}}
{"repeat":1,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":25,"started_at":"2026-09-20T00:55:53.442Z","latency_ms":11018.140459000017,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["5e4d4157-88f1-422f-aa44-c82ee07d0eb6"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-02","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":26,"started_at":"2026-09-20T00:55:48.078Z","latency_ms":18066.884375000023,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"e031c0f8-822a-4a9b-8c2f-6955b1a413c4","answer_transaction_ids":["366ec2fc-1f3a-4a53-8379-eb16b3650050"],"charged_cents":1,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":1,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":27,"started_at":"2026-09-20T00:55:58.535Z","latency_ms":11144.279624999996,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["384813f3-8030-4487-9e5d-1c6ef9369eda"],"charged_cents":0,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":1,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":29,"started_at":"2026-09-20T00:56:04.460Z","latency_ms":8061.952542000014,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["74482bc0-c82e-48bf-a854-dfc41b07d3aa"],"charged_cents":0,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":1,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":30,"started_at":"2026-09-20T00:56:06.145Z","latency_ms":9884.255583999999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["8e2777b4-ff8f-4508-b184-36413bf687f5"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-03","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":31,"started_at":"2026-09-20T00:56:09.680Z","latency_ms":8712.637999999977,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["2a8f86d2-cfc1-4af2-9ed9-6e4526e3d342"],"charged_cents":0,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":1,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":28,"started_at":"2026-09-20T00:56:00.162Z","latency_ms":20323.56150000001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.25,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"88713f33-be53-4468-9fb0-ff4426bb4061","answer_transaction_ids":["79648ac6-1a1e-4fa1-b5e5-b9fbab953b7b"],"charged_cents":1,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":28,"started_at":"2026-09-20T00:56:20.487Z","latency_ms":4721.002292000019,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["88634db3-1849-4cdc-82e1-d693b9b6de39"],"charged_cents":0,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":29,"started_at":"2026-09-20T00:56:12.523Z","latency_ms":17387.774000000005,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.5,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"8e9ca534-0bfe-4272-a6c4-281ed7dfc689","answer_transaction_ids":["66794248-875b-460a-885e-114d9164bc0d"],"charged_cents":1,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":1,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":30,"started_at":"2026-09-20T00:56:16.030Z","latency_ms":16816.18183300001,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.79,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"03bbb923-0674-49e3-8545-e3768f915235","answer_transaction_ids":["c5aa53df-ff35-4c74-b0ee-2c64e5aa3767"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-03","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":31,"started_at":"2026-09-20T00:56:18.394Z","latency_ms":15422.579207999981,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"dd6f1864-0ad9-49b3-a41a-5dbf59a32ac4","answer_transaction_ids":["a264dc62-8b38-461f-a5b3-02f212a50be1"],"charged_cents":1,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":1,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":32,"started_at":"2026-09-20T00:56:25.209Z","latency_ms":11062.368166,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["890cfaf6-478b-475e-b1e0-883727fdd6f7"],"charged_cents":0,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":1,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":33,"started_at":"2026-09-20T00:56:29.911Z","latency_ms":9180.010416999983,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["0f62bc93-1d0c-46e0-a687-000e42db9886"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"lookup-10","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":34,"started_at":"2026-09-20T00:56:32.848Z","latency_ms":9441.026624999999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["2030ed0e-fc98-4adc-9c1e-5cd6e80cf6e0"],"charged_cents":0,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":1,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":35,"started_at":"2026-09-20T00:56:33.817Z","latency_ms":10351.293749999983,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["17501d99-1732-4f7d-b8ce-a171be5c5b8b"],"charged_cents":0,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":1,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":32,"started_at":"2026-09-20T00:56:36.272Z","latency_ms":13422.604292000004,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.54,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"e5353db3-a9c3-4a37-b36b-8d3ae55a18a3","answer_transaction_ids":["cc9f7310-ad0a-42ff-9fe1-a7311f6a4faa"],"charged_cents":1,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":1,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":33,"started_at":"2026-09-20T00:56:39.092Z","latency_ms":12343.360916999984,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.62,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"46b8c4b7-4aa6-4ddd-9b06-249b17894b48","answer_transaction_ids":["b0c77d4d-ec34-4a62-8b74-109c3866dbbe"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"lookup-10","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":34,"started_at":"2026-09-20T00:56:42.289Z","latency_ms":10204.536457999988,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"e9580b43-c725-4dbe-a3a0-1c225dd12f92","answer_transaction_ids":["8c5279b9-3dc5-480f-a25a-0eaf2a664d30"],"charged_cents":1,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":1,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":35,"started_at":"2026-09-20T00:56:44.169Z","latency_ms":10141.628459,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.7,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a14e9f05-d02a-4731-97d0-7a5f27886252","answer_transaction_ids":["03947a4d-3e64-4f11-81e8-ef1e0d21d9c7"],"charged_cents":1,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":1,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":36,"started_at":"2026-09-20T00:56:49.695Z","latency_ms":6640.969457999978,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["46cc574c-88b9-48c8-91bf-ff30a8f5fa0d"],"charged_cents":0,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":1,"prompt_id":"lookup-08","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":37,"started_at":"2026-09-20T00:56:51.436Z","latency_ms":6839.844791999989,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["1112f26d-7d8b-4dd9-a06b-ead375ff657a"],"charged_cents":0,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":1,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":39,"started_at":"2026-09-20T00:56:54.311Z","latency_ms":6851.553125000006,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["f8cfb84f-2a90-42a4-9efa-f6e4d307d9f8"],"charged_cents":0,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."}]}}
{"repeat":1,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":38,"started_at":"2026-09-20T00:56:52.494Z","latency_ms":11775.893624999997,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.77,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"29c2d0a2-f02f-4b72-bd13-0c1a192860f2","answer_transaction_ids":["eca9e806-0c22-41cd-9890-dd8e604f47bb"],"charged_cents":1,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":1,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":36,"started_at":"2026-09-20T00:56:56.336Z","latency_ms":10199.995624999981,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.69,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"d36272e9-6dc8-47a1-bb0f-84d0dee76646","answer_transaction_ids":["fd9a24db-a4ab-4954-a930-e71db4680ca2"],"charged_cents":1,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":1,"prompt_id":"lookup-08","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":37,"started_at":"2026-09-20T00:56:58.276Z","latency_ms":9161.68312500001,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.93,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"412c9d94-575a-45bd-bdf7-b4a6dd261d21","answer_transaction_ids":["143036ad-8524-44b1-9365-0bd0a6ff4ec7"],"charged_cents":1,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":1,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":38,"started_at":"2026-09-20T00:57:04.272Z","latency_ms":5958.279083000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["6b2c4daa-e195-4ab0-8c4c-4ea315fb4957"],"charged_cents":0,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":1,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":39,"started_at":"2026-09-20T00:57:01.163Z","latency_ms":11494.70208399999,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.37,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"12bf8580-3f1c-49cd-b8f8-c2b22224e2e9","answer_transaction_ids":["35087155-2ebe-4f13-8327-16dffd517cd3"],"charged_cents":1,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."}]}}
{"repeat":1,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":40,"started_at":"2026-09-20T00:57:06.537Z","latency_ms":8689.985667,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["7997870e-8b43-4997-9b6e-0f1d14aa4498"],"charged_cents":0,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":1,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":41,"started_at":"2026-09-20T00:57:07.439Z","latency_ms":9965.423291999992,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["da11cbf8-f4fc-4dc1-b904-fe3d9a5a75ac"],"charged_cents":0,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":1,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":43,"started_at":"2026-09-20T00:57:12.659Z","latency_ms":7163.986958999973,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d68925ca-2a02-42b4-9ca0-f29c1be32e46"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":42,"started_at":"2026-09-20T00:57:10.231Z","latency_ms":12394.714833000005,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.83,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"74c2157e-2f7f-4ba8-ae8b-21cdf252fa93","answer_transaction_ids":["48219c20-f643-44d6-98a4-7cc119dd2851"],"charged_cents":1,"answer":{"seconds":20},"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":20}]}}
{"repeat":1,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":40,"started_at":"2026-09-20T00:57:15.228Z","latency_ms":9472.430416999996,"decision":{"lane":"balanced","recommended":"fast","reason":["low_confidence"],"confidence":0.27,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"1f6e815e-745f-4bbe-8a5e-5fb1fe53fa40","answer_transaction_ids":["91d0d6b1-a4ad-4e02-b199-1e0885b15faf"],"charged_cents":1,"answer":{"text":"Build failed due to missing config.yaml. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed due to missing config.yaml. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed due to missing config.yaml. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed due to missing config.yaml. Restore config.yaml and rerun the build."}]}}
{"repeat":1,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":41,"started_at":"2026-09-20T00:57:17.405Z","latency_ms":9165.475542,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes"],"confidence":0.84,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"baa28236-74cc-40b1-98d8-c2804f36ab9c","answer_transaction_ids":["bac51074-7ae3-4df6-af2e-3f7458e940f9"],"charged_cents":1,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":1,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":42,"started_at":"2026-09-20T00:57:22.626Z","latency_ms":6716.563833000022,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["28d95a41-127f-4f9c-9b7e-a07eed488c38"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":43,"started_at":"2026-09-20T00:57:19.824Z","latency_ms":11977.941791999998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.5,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"723a5857-b11c-49e5-bfd0-16a8460c29f4","answer_transaction_ids":["9f82874a-cbe6-4796-9e6b-84e1e19e5688"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"lookup-09","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":44,"started_at":"2026-09-20T00:57:24.701Z","latency_ms":9176.880666000012,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["1e66940a-2dba-4566-bfff-ca758d61a591"],"charged_cents":0,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":45,"started_at":"2026-09-20T00:57:26.570Z","latency_ms":10753.061959000013,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["fa667906-36e1-4d69-b7b3-c93e40ce5364"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":46,"started_at":"2026-09-20T00:57:29.343Z","latency_ms":10561.779791999987,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["fc055a8b-c80c-4d7f-a121-772002b33010"],"charged_cents":0,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":1,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":47,"started_at":"2026-09-20T00:57:31.802Z","latency_ms":12291.93654200001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.49,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"9823df3b-9082-4d10-a729-5aa09ebed535","answer_transaction_ids":["7eb7f2d6-3570-4e58-8ac9-4ebf0cde28da"],"charged_cents":1,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"lookup-09","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":44,"started_at":"2026-09-20T00:57:33.878Z","latency_ms":11552.87704099997,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"175aa9ae-fce0-4150-9972-ff2810926c8a","answer_transaction_ids":["73bea4c6-8a3e-4dfd-8c96-3a9fc5c71907"],"charged_cents":1,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":45,"started_at":"2026-09-20T00:57:37.324Z","latency_ms":10468.756082999986,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.85,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"1b299d36-869d-4a38-9f87-50906e5d9953","answer_transaction_ids":["c3ee6f71-8054-4710-83f9-a15bba319b91"],"charged_cents":1,"answer":{"finish_minute":12},"grade":{"pass":true,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":true,"actual":12}]}}
{"repeat":1,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":46,"started_at":"2026-09-20T00:57:39.905Z","latency_ms":10379.403290999995,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.6,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"94abbb03-eeb0-40e7-9aa6-51844892b3dd","answer_transaction_ids":["4447b1a2-1d0f-4f52-a896-8fb8aeb7a129"],"charged_cents":1,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":1,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":47,"started_at":"2026-09-20T00:57:44.095Z","latency_ms":8956.512292,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["faf8c90d-0ee0-4a4f-a18c-dd3115dd852c"],"charged_cents":0,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":48,"started_at":"2026-09-20T00:57:45.431Z","latency_ms":10401.395874999987,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.73,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"1cf82bc7-20af-483a-98b1-5412995483f8","answer_transaction_ids":["6b414c6c-f0b6-4c43-980e-60c8f3304f99"],"charged_cents":1,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":1,"prompt_id":"lookup-07","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":49,"started_at":"2026-09-20T00:57:47.793Z","latency_ms":9221.669292000006,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.89,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"99a1674c-3e21-4ece-a275-1a0dd47b9602","answer_transaction_ids":["a8306a21-cf2b-4860-8b52-5be2733fc2c0"],"charged_cents":1,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":1,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":48,"started_at":"2026-09-20T00:57:55.833Z","latency_ms":3472.1914170000236,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d5460a1f-ab5b-41ec-9f3f-e64c6528aaaa"],"charged_cents":0,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":1,"prompt_id":"lookup-07","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":49,"started_at":"2026-09-20T00:57:57.015Z","latency_ms":4221.587333999982,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["27eb4d45-3bea-4738-8c6b-47d8f69bda26"],"charged_cents":0,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":2,"prompt_id":"lookup-09","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":3,"started_at":"2026-09-20T00:58:01.238Z","latency_ms":3919.691125000012,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["c6a0e30c-1b3f-4d4f-bdca-42da483e2baa"],"charged_cents":0,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":0,"started_at":"2026-09-20T00:58:01.237Z","latency_ms":5793.378457999963,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.41,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"3aa5ab52-a82a-464f-b36c-175d67c67dae","answer_transaction_ids":["cbc2f3b7-8c1b-4e08-9d5b-e75780ff44d6"],"charged_cents":1,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":2,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":1,"started_at":"2026-09-20T00:58:01.238Z","latency_ms":7793.359583000012,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.52,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"d564d879-9846-4134-bff0-6d834e2bd9c7","answer_transaction_ids":["a3b813ef-e3bb-40c5-9f91-13aaf0436d8d"],"charged_cents":1,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":2,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":2,"started_at":"2026-09-20T00:58:01.238Z","latency_ms":9728.136249999981,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.36,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"b9dcbb7b-592a-4159-99d2-36aaa6f43f92","answer_transaction_ids":["8d7facd2-e7ec-4d44-9baa-0bd5887786c3"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":0,"started_at":"2026-09-20T00:58:07.031Z","latency_ms":6250.64300000004,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["77d65ddb-74fe-413d-9be2-c448de8bbf7a"],"charged_cents":0,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":2,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":1,"started_at":"2026-09-20T00:58:09.032Z","latency_ms":6295.4825420000125,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["bb57b616-f8bb-47fd-80e2-e91c2aca72af"],"charged_cents":0,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":2,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":2,"started_at":"2026-09-20T00:58:10.966Z","latency_ms":8761.818542000023,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["146d3302-dd03-45e5-9ff4-a711c7d8fd03"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"lookup-09","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":3,"started_at":"2026-09-20T00:58:05.158Z","latency_ms":16113.757582999999,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"31b3bcf4-db09-46b5-8b54-7d3ce38e2351","answer_transaction_ids":["26595bb1-5a9c-4991-aa11-f30baec77942"],"charged_cents":1,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":5,"started_at":"2026-09-20T00:58:15.328Z","latency_ms":8680.589959000004,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["2f46ce99-2853-47da-879e-eaee4a1e0150"],"charged_cents":0,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":2,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":6,"started_at":"2026-09-20T00:58:19.728Z","latency_ms":7277.357333000051,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4fe8166e-a943-4e6c-b4f0-548882c4599c"],"charged_cents":0,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":2,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":4,"started_at":"2026-09-20T00:58:13.282Z","latency_ms":16931.028292000003,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.73,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"3cbff403-ae19-4d3e-bb12-6ff5bf7cda99","answer_transaction_ids":["de27ea52-755a-47d1-b82e-2332afd7412b"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":7,"started_at":"2026-09-20T00:58:21.273Z","latency_ms":12103.853458000056,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.55,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"cb335c6d-5c25-48f9-8a8f-33072073db3a","answer_transaction_ids":["8a042b13-47e8-40f4-8adf-7afee0940e55"],"charged_cents":1,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":4,"started_at":"2026-09-20T00:58:30.214Z","latency_ms":5991.778500000015,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["eab0a276-0a04-4fb9-9202-4b87a08221c6"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":5,"started_at":"2026-09-20T00:58:24.009Z","latency_ms":14080.875541999994,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.64,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"df4f3b01-d0ad-44b4-92dd-92971d545a26","answer_transaction_ids":["37cecc56-5f3a-4c7b-9dea-de6f3134e318"],"charged_cents":1,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":2,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":6,"started_at":"2026-09-20T00:58:27.006Z","latency_ms":13096.762041999958,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.35,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"ca025adf-276d-470b-a835-417db54fd00e","answer_transaction_ids":["274145d7-cfb8-4070-b615-06d9b27bd2f0"],"charged_cents":1,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":2,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":7,"started_at":"2026-09-20T00:58:33.377Z","latency_ms":8807.830625000002,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["96f079ec-12dd-4785-b4d7-f2074254c76a"],"charged_cents":0,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"lookup-08","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":8,"started_at":"2026-09-20T00:58:36.207Z","latency_ms":9810.90487499995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["3fe19d51-1e0a-4f0e-b914-b75d2d0ff10c"],"charged_cents":0,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":2,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":10,"started_at":"2026-09-20T00:58:40.103Z","latency_ms":11765.32808300003,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["1169dca6-b374-485e-8ad6-3697a01fdefe"],"charged_cents":0,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."}]}}
{"repeat":2,"prompt_id":"lookup-07","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":9,"started_at":"2026-09-20T00:58:38.091Z","latency_ms":16244.976957999985,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.87,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"02c0b516-2062-4b8b-b6a1-168d50e8ada9","answer_transaction_ids":["101ae11a-42f8-40f5-a1db-20be6951f4be"],"charged_cents":1,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":2,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":11,"started_at":"2026-09-20T00:58:42.185Z","latency_ms":14758.793749999953,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.25,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"9762a53f-7be7-47aa-98a1-b742d491171c","answer_transaction_ids":["7be4d1c5-e17c-4d0c-a52b-47c880d9d339"],"charged_cents":1,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"lookup-08","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":8,"started_at":"2026-09-20T00:58:46.018Z","latency_ms":12215.646500000032,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.93,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"3718725c-9a87-47eb-9fa4-0c521a691c3a","answer_transaction_ids":["95ffdd8d-d9a4-4650-a282-cffd2583accc"],"charged_cents":1,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":2,"prompt_id":"lookup-07","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":9,"started_at":"2026-09-20T00:58:54.337Z","latency_ms":6016.383124999993,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d58f8bfe-9257-45b9-a4a0-8157ac77aedb"],"charged_cents":0,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":2,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":10,"started_at":"2026-09-20T00:58:51.869Z","latency_ms":11794.93316700001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.4,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"6d6ffdb7-a101-4367-bba9-9b8cb331c2a3","answer_transaction_ids":["a363301b-c19b-48e6-ae17-924184a9f267"],"charged_cents":1,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."}]}}
{"repeat":2,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":11,"started_at":"2026-09-20T00:58:56.944Z","latency_ms":8559.188250000007,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["2f1519c0-a83c-4a51-8725-1bc6975268aa"],"charged_cents":0,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":12,"started_at":"2026-09-20T00:58:58.234Z","latency_ms":9300.327124999953,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["8a802312-4160-469c-af9e-b12ad18050bb"],"charged_cents":0,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":2,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":13,"started_at":"2026-09-20T00:59:00.354Z","latency_ms":9794.86825,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["9304dcd7-0b35-4f27-800b-0c526f00b18b"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":14,"started_at":"2026-09-20T00:59:03.664Z","latency_ms":10230.074958000041,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["e89e0bf4-f9fc-4007-a670-782d6574f9ee"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":15,"started_at":"2026-09-20T00:59:05.505Z","latency_ms":13206.050125000009,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.74,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"280f158e-09cb-4a8a-a891-b90fe64ab4e5","answer_transaction_ids":["30784030-80ef-4ef1-8f04-990aea1f6438"],"charged_cents":1,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":2,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":12,"started_at":"2026-09-20T00:59:07.535Z","latency_ms":13499.559999999998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.3,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"bfa886ff-1261-4ab6-92ba-b44ea4bd68d0","answer_transaction_ids":["9768ec0e-3f6d-4a56-8185-d0f487d12543"],"charged_cents":1,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":2,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":13,"started_at":"2026-09-20T00:59:10.149Z","latency_ms":13646.715832999966,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.54,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"9d9c80ba-6110-44ba-b7d8-2da5fad60127","answer_transaction_ids":["cc9c1599-3499-4d6d-b56f-e20f83b2b707"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":14,"started_at":"2026-09-20T00:59:13.894Z","latency_ms":14463.045165999967,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.7,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"7cb313b9-6c86-4b68-9660-7f3555b6d610","answer_transaction_ids":["afc9b5e3-a33c-4145-8576-9f8811312b34"],"charged_cents":4,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":15,"started_at":"2026-09-20T00:59:18.712Z","latency_ms":12006.624542000005,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["78e005f4-da64-4fb2-a93b-c8640deea330"],"charged_cents":0,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":2,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":16,"started_at":"2026-09-20T00:59:21.036Z","latency_ms":11908.303625,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["dedcb808-90f8-476f-ad91-ad8f2e884598"],"charged_cents":0,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":2,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":18,"started_at":"2026-09-20T00:59:28.358Z","latency_ms":7660.3192500000005,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["1a4b48fd-365f-4b3f-a1cc-4e60af51da0c"],"charged_cents":0,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":2,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":17,"started_at":"2026-09-20T00:59:23.796Z","latency_ms":18886.881458999997,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.46,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"47df1681-1b94-4f39-88d7-06c4596ed70c","answer_transaction_ids":["07011e6d-de37-4d7e-8938-42de2e7acba6"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":19,"started_at":"2026-09-20T00:59:30.719Z","latency_ms":14341.062166999967,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.54,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"47b54439-7515-48b0-b91f-aab031964e0c","answer_transaction_ids":["fe48434d-750e-4433-8525-aded8b467bb6"],"charged_cents":1,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":2,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":16,"started_at":"2026-09-20T00:59:32.944Z","latency_ms":14520.798332999984,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes"],"confidence":0.86,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"2119e426-3208-4280-a40c-efe4f6326bea","answer_transaction_ids":["1bd1ed18-d60e-4369-b505-21c92e9d526c"],"charged_cents":1,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":2,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":17,"started_at":"2026-09-20T00:59:42.683Z","latency_ms":6953.621458999987,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["e3e020f5-711a-41c1-bf16-d0d6c9fc0dff"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":18,"started_at":"2026-09-20T00:59:36.018Z","latency_ms":15462.675042000017,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.63,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"bad3cd1f-26db-480d-90c0-f3632f4d91eb","answer_transaction_ids":["a5546126-d508-4755-9b1c-18c1d88b8710"],"charged_cents":1,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":2,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":19,"started_at":"2026-09-20T00:59:45.060Z","latency_ms":8467.117749999976,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["7fc0a28e-34a9-4a93-947e-15634e4af132"],"charged_cents":0,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":2,"prompt_id":"lookup-04","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":21,"started_at":"2026-09-20T00:59:49.637Z","latency_ms":6646.426917000033,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d7777bf8-779e-47df-8dbc-3fb8b103f32a"],"charged_cents":0,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":2,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":22,"started_at":"2026-09-20T00:59:51.481Z","latency_ms":6650.121417000017,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["e977aaac-4dc7-4f02-88f2-5efff2e27346"],"charged_cents":0,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":2,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":20,"started_at":"2026-09-20T00:59:47.465Z","latency_ms":13444.049291000003,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.53,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"c9aa8202-5440-4e79-a9b7-17fa31720803","answer_transaction_ids":["907135e0-7cfc-4ed8-9adb-e53dda6fc347"],"charged_cents":1,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":2,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":23,"started_at":"2026-09-20T00:59:53.529Z","latency_ms":11058.647457999992,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.38,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"ff6e853f-9a47-4bc4-9cad-29cb2ddeaac6","answer_transaction_ids":["3140f32d-fe10-4b97-9806-d66bc4096cb6"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":20,"started_at":"2026-09-20T01:00:00.910Z","latency_ms":8699.749584000034,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["b0424d09-7289-4b94-b1f9-2889b21a8e2f"],"charged_cents":0,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":2,"prompt_id":"lookup-04","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":21,"started_at":"2026-09-20T00:59:56.284Z","latency_ms":14736.310041999968,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"33d0f045-8ed3-4c64-880b-1228212e3c19","answer_transaction_ids":["7b47110a-a3e4-4ee3-acac-33e1c8bd7e37"],"charged_cents":1,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":2,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":22,"started_at":"2026-09-20T00:59:58.132Z","latency_ms":14988.776708999998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.71,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0e73e4fa-95c6-4468-adcb-4b8a93e0ba53","answer_transaction_ids":["8c50751d-656b-411c-be7b-1ba2e34bc554"],"charged_cents":1,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":2,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":23,"started_at":"2026-09-20T01:00:04.588Z","latency_ms":10420.873582999979,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["6540ef2d-b9c7-4c96-bf13-c2ed6de8d02a"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":24,"started_at":"2026-09-20T01:00:09.611Z","latency_ms":9557.680999999982,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["6231df40-ad38-46e9-8ef9-e2555394e00a"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":26,"started_at":"2026-09-20T01:00:13.121Z","latency_ms":9848.820917000005,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d5fcd4da-432e-4d5d-a526-49195bc70fc3"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":false,"actual":null},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":25,"started_at":"2026-09-20T01:00:11.021Z","latency_ms":15629.858790999977,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.63,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"59897e7b-9e12-43e8-9936-2f47b07e2ea2","answer_transaction_ids":["09eb4771-71d8-4250-a25f-ae0082843a7b"],"charged_cents":1,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":2,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":27,"started_at":"2026-09-20T01:00:15.010Z","latency_ms":13953.631125000014,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.64,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"810f4d3b-5a1f-4297-8043-a6bac030cbed","answer_transaction_ids":["21e372c5-2b4e-417e-91a9-cdfb6d14e55f"],"charged_cents":1,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":24,"started_at":"2026-09-20T01:00:19.169Z","latency_ms":13847.935374999943,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.52,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"4f7890a6-e527-4184-91f8-3df3a2205c02","answer_transaction_ids":["d78e21a4-c473-420c-b4f6-0a477c51436e"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":25,"started_at":"2026-09-20T01:00:26.651Z","latency_ms":10466.317084000038,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["2108e706-3e8f-49c7-851d-bf77a5d918ab"],"charged_cents":0,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":2,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":26,"started_at":"2026-09-20T01:00:22.970Z","latency_ms":15763.366000000038,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.84,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"8ab00994-840d-4fc9-b616-8e737b9dff15","answer_transaction_ids":["b0d8db40-03a2-444e-9ae4-ffdbb02c3987"],"charged_cents":1,"answer":{"plan":"B","cost_dollars":7.8},"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":true,"actual":"B"},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":7.8}]}}
{"repeat":2,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":27,"started_at":"2026-09-20T01:00:28.964Z","latency_ms":11770.652708999987,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["b049a73b-495e-4807-979e-d4be5cb433dc"],"charged_cents":0,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":30,"started_at":"2026-09-20T01:00:38.734Z","latency_ms":7756.321083000046,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["387e77a8-b208-4051-b26e-38e74619cac1"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":28,"started_at":"2026-09-20T01:00:33.017Z","latency_ms":16143.286000000022,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.53,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0e29b405-08e0-40c2-959f-7cd94e024717","answer_transaction_ids":["9a6513be-0792-4eca-83c2-6d8bbc5b9fc1"],"charged_cents":1,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":29,"started_at":"2026-09-20T01:00:37.118Z","latency_ms":14298.65591599996,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.45,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0f6e617b-fca3-451f-9226-19dd2494ee99","answer_transaction_ids":["3c21827d-df6f-4db1-aa43-8033ad5a6787"],"charged_cents":1,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":2,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":31,"started_at":"2026-09-20T01:00:40.735Z","latency_ms":13735.438790999993,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.74,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"e043b325-af45-47bf-94b2-d8dded43d5b5","answer_transaction_ids":["49bafa1e-ad30-4cd5-9f39-f05e930fac83"],"charged_cents":1,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":2,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":28,"started_at":"2026-09-20T01:00:49.160Z","latency_ms":7778.766583000019,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["315cd02d-18f0-4474-9472-9c714532af77"],"charged_cents":3,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":29,"started_at":"2026-09-20T01:00:51.416Z","latency_ms":7754.768291999993,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d11c26cc-fb6e-4836-bad9-203220bb20cf"],"charged_cents":0,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":2,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":30,"started_at":"2026-09-20T01:00:46.491Z","latency_ms":16826.10287500004,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.56,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a977d52d-5e14-4931-9c7f-84e43efd8b40","answer_transaction_ids":["a19f8f61-c970-4a23-8387-b98d9034c252"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":31,"started_at":"2026-09-20T01:00:54.471Z","latency_ms":11083.449499999988,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["e64d4935-b411-472d-9713-8fcb9617a3d8"],"charged_cents":0,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":2,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":32,"started_at":"2026-09-20T01:00:56.940Z","latency_ms":12060.978833000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["ddca6a39-c21c-40e2-b271-eaa39c77223d"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":35,"started_at":"2026-09-20T01:01:05.554Z","latency_ms":7800.165291999991,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d313533a-dc30-4972-b610-7ff245756ad2"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"numerator","value":7,"pass":false,"actual":null},{"op":"eq","field":"denominator","value":13,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":33,"started_at":"2026-09-20T01:00:59.172Z","latency_ms":19145.84362499998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.58,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"896f963d-ac5f-4451-9bdf-ec8163a26536","answer_transaction_ids":["43e3319c-dd61-45e0-b7db-56c8f008253a"],"charged_cents":4,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":34,"started_at":"2026-09-20T01:01:03.318Z","latency_ms":23580.942625000025,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.79,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"d1e7c9c2-a3c1-4dd8-b326-09db43b21507","answer_transaction_ids":["a97142d9-c9b7-43f0-a509-0748a87abe28"],"charged_cents":1,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":2,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":32,"started_at":"2026-09-20T01:01:09.001Z","latency_ms":20068.269458000024,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.81,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"cd058bd0-01b5-4ef3-b8de-f7d5559816ef","answer_transaction_ids":["25b68a27-5bc0-4a1f-97f9-5c96bd3635e2"],"charged_cents":1,"answer":{"records_per_group":56},"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":56}]}}
{"repeat":2,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":33,"started_at":"2026-09-20T01:01:18.318Z","latency_ms":13239.91754200001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["167fa75a-887a-4b16-8c1d-fdb6503649e1"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":34,"started_at":"2026-09-20T01:01:26.899Z","latency_ms":8213.446249999979,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d88bd3d6-dbba-4022-986b-3f3fa7157975"],"charged_cents":0,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":2,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":35,"started_at":"2026-09-20T01:01:13.355Z","latency_ms":23499.45104199997,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.84,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"9e98ea8e-bc0c-4eea-b99c-3bd9678d9462","answer_transaction_ids":["f5931954-80cc-48d3-b2d9-06d6a4cb234f"],"charged_cents":1,"answer":{"numerator":7,"denominator":13},"grade":{"pass":true,"checks":[{"op":"eq","field":"numerator","value":7,"pass":true,"actual":7},{"op":"eq","field":"denominator","value":13,"pass":true,"actual":13}]}}
{"repeat":2,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":37,"started_at":"2026-09-20T01:01:31.558Z","latency_ms":8191.347790999978,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["b8d96a33-5da7-4278-b715-2831d066a751"],"charged_cents":0,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."}]}}
{"repeat":2,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":39,"started_at":"2026-09-20T01:01:36.855Z","latency_ms":6187.648541999981,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["1f452770-37a3-4f7a-82bd-f822b76364a5"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"lookup-06","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":36,"started_at":"2026-09-20T01:01:29.070Z","latency_ms":15225.434582999966,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"ba69a01f-af90-4bb4-b1a5-cdcad098619b","answer_transaction_ids":["1a585420-1ff6-4d6a-9613-67326a4ebe39"],"charged_cents":1,"answer":{"day":"Thursday","time_utc":"09:30 UTC"},"grade":{"pass":false,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":false,"actual":"09:30 UTC"}]}}
{"repeat":2,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":38,"started_at":"2026-09-20T01:01:35.117Z","latency_ms":11155.865082999982,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.87,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"080deb89-0234-4fd6-9718-4b2c405c095b","answer_transaction_ids":["60f79042-fc7a-4260-9d3f-66a0b18b63fd"],"charged_cents":1,"answer":{"unreserved_units":54},"grade":{"pass":true,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":true,"actual":54}]}}
{"repeat":2,"prompt_id":"lookup-06","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":36,"started_at":"2026-09-20T01:01:44.296Z","latency_ms":7099.923875000037,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["e71253b1-d3be-4894-b53e-5b0c3472af15"],"charged_cents":0,"answer":{"day":"Thursday","time_utc":"09:30"},"grade":{"pass":true,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":true,"actual":"09:30"}]}}
{"repeat":2,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":37,"started_at":"2026-09-20T01:01:39.750Z","latency_ms":13197.166290999972,"decision":{"lane":"balanced","recommended":"fast","reason":["low_confidence"],"confidence":0.25,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"bcecc446-fedc-43ae-9552-9dd7e2734f10","answer_transaction_ids":["96cc40de-5fe1-4215-9701-398e272dc171"],"charged_cents":1,"answer":{"text":"The password reset link is valid for 20 minutes. If it expires, please request a new link."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"The password reset link is valid for 20 minutes. If it expires, please request a new link."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"The password reset link is valid for 20 minutes. If it expires, please request a new link."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The password reset link is valid for 20 minutes. If it expires, please request a new link."}]}}
{"repeat":2,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":38,"started_at":"2026-09-20T01:01:46.274Z","latency_ms":9373.587,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["2d03b581-8cfd-4911-8433-5ea676be2577"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":39,"started_at":"2026-09-20T01:01:43.043Z","latency_ms":14598.332916000043,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.47,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0aa861ec-5e91-49b6-8282-1af51749f768","answer_transaction_ids":["455c287c-4ef9-4bd0-bb34-d3441e839a4e"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"lookup-05","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":40,"started_at":"2026-09-20T01:01:51.397Z","latency_ms":8188.054082999995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["b84ff14a-5662-4039-af9a-188375c242f5"],"charged_cents":0,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":2,"prompt_id":"lookup-01","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":42,"started_at":"2026-09-20T01:01:55.649Z","latency_ms":6668.561583999952,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["50ff9526-f404-4d5e-a7e2-8fdcc8f8c1d9"],"charged_cents":0,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":2,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":41,"started_at":"2026-09-20T01:01:52.947Z","latency_ms":12522.265041000035,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.69,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"4aa38536-2313-47e3-b9a7-390490631b1c","answer_transaction_ids":["bbde5ec5-99ec-4145-8a44-37ef0d4d998f"],"charged_cents":1,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":2,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":43,"started_at":"2026-09-20T01:01:57.642Z","latency_ms":10442.209875,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.86,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"ce986e4a-2cb3-4e35-88f0-2930ec77f35e","answer_transaction_ids":["72b15258-6882-47fa-bded-efe4d22aa2bc"],"charged_cents":1,"answer":{"finish_minute":12},"grade":{"pass":true,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":true,"actual":12}]}}
{"repeat":2,"prompt_id":"lookup-05","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":40,"started_at":"2026-09-20T01:01:59.585Z","latency_ms":12474.532249999931,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"e8e866c8-956c-4e85-90d1-6bc3ba1d858b","answer_transaction_ids":["f1aa88b5-755e-4c85-bb94-41c99b07b570"],"charged_cents":1,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":2,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":41,"started_at":"2026-09-20T01:02:05.470Z","latency_ms":8420.00783300004,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["ad91024d-8e94-4c6c-bcdb-c4e385f1e7a1"],"charged_cents":0,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":2,"prompt_id":"lookup-01","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":42,"started_at":"2026-09-20T01:02:02.318Z","latency_ms":13315.627000000037,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.88,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"bdfe8b38-a997-4d30-a840-29421b988d87","answer_transaction_ids":["d7ef40da-bde7-4dd2-bd58-e6f8792f826f"],"charged_cents":1,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":2,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":43,"started_at":"2026-09-20T01:02:08.085Z","latency_ms":10997.423875000095,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["9e51301c-fef9-4c31-8dbc-c6514b39ba6f"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":44,"started_at":"2026-09-20T01:02:12.060Z","latency_ms":9251.82037500001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["3a4c6fef-00fe-4965-a532-270f097bbdf9"],"charged_cents":0,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":2,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":45,"started_at":"2026-09-20T01:02:13.891Z","latency_ms":9845.100208999938,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d65d0443-2159-4051-b955-101dda55726f"],"charged_cents":0,"answer":{"text":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."}]}}
{"repeat":2,"prompt_id":"lookup-02","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":46,"started_at":"2026-09-20T01:02:15.634Z","latency_ms":11807.382915999973,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"ef66a577-18a6-4587-adf0-35e241bfd4bd","answer_transaction_ids":["974474ea-4230-4eff-9b47-67c47f12e6bd"],"charged_cents":1,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":2,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":47,"started_at":"2026-09-20T01:02:19.082Z","latency_ms":9705.942792000016,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.92,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"b3c3d274-3a5d-4e05-8740-330557b4b2a9","answer_transaction_ids":["5e61251e-5fd4-4258-b4a2-10a1a79ca002"],"charged_cents":1,"answer":{"total_dollars":66.6},"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":66.6}]}}
{"repeat":2,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":44,"started_at":"2026-09-20T01:02:21.312Z","latency_ms":9907.22883399995,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.56,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"b966404d-1b96-42e7-8aa8-1d20038057d6","answer_transaction_ids":["37268da0-db11-4314-a532-7d8971c2d564"],"charged_cents":1,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":2,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":45,"started_at":"2026-09-20T01:02:23.736Z","latency_ms":9129.141999999993,"decision":{"lane":"balanced","recommended":"fast","reason":["low_confidence"],"confidence":0.28,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"638a6159-1304-436c-9a82-0c7efc2b344d","answer_transaction_ids":["25aafcc0-19b7-48ff-acc7-19258a394af4"],"charged_cents":1,"answer":{"text":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."}]}}
{"repeat":2,"prompt_id":"lookup-02","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":46,"started_at":"2026-09-20T01:02:27.443Z","latency_ms":7350.424541999935,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["8d647507-8b05-47ea-8eb2-782c2aa7df68"],"charged_cents":0,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":2,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":47,"started_at":"2026-09-20T01:02:28.789Z","latency_ms":9114.900332999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["7b76afb3-47ff-4b44-a690-cf693fce9a28"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"lookup-10","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":48,"started_at":"2026-09-20T01:02:31.219Z","latency_ms":9048.073916999972,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"560924a5-f80e-4b9c-a9d7-00e8b98e2166","answer_transaction_ids":["bea202b7-e3fd-434e-973a-c0183004a7f9"],"charged_cents":1,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":2,"prompt_id":"lookup-03","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":49,"started_at":"2026-09-20T01:02:32.865Z","latency_ms":8632.08033399994,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"9c72872d-e859-4329-8222-476d1f77b4ec","answer_transaction_ids":["ba4d8083-fd95-4bfb-ad50-69f3ea83efef"],"charged_cents":1,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":2,"prompt_id":"lookup-10","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":48,"started_at":"2026-09-20T01:02:40.268Z","latency_ms":3087.578749999986,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["09aeff2e-d6f4-4638-b332-e2bac0cfe3e9"],"charged_cents":0,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":2,"prompt_id":"lookup-03","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":49,"started_at":"2026-09-20T01:02:41.498Z","latency_ms":4300.412750000018,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d6d44a7a-2ccf-499c-9f8a-cb4f6a666236"],"charged_cents":0,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":3,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":2,"started_at":"2026-09-20T01:02:45.800Z","latency_ms":3033.096707999939,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["b907eee5-ba12-41ea-b78b-965c5367651a"],"charged_cents":0,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":3,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":3,"started_at":"2026-09-20T01:02:45.800Z","latency_ms":7133.080041999929,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["aeea9f06-7c5b-4e78-8bd2-5a58828388cf"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":0,"started_at":"2026-09-20T01:02:45.799Z","latency_ms":9315.415458000032,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.55,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"553190b8-8d6e-4c4b-840e-0c76607d2496","answer_transaction_ids":["152c624d-5f01-40aa-99c5-9556fd3ff026"],"charged_cents":1,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":3,"prompt_id":"lookup-08","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":1,"started_at":"2026-09-20T01:02:45.799Z","latency_ms":10594.26879100001,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.94,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"7ab15d02-625f-498d-bf83-f8dc45746dbf","answer_transaction_ids":["a3119890-0634-41f2-aead-19b972533868"],"charged_cents":1,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":3,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":0,"started_at":"2026-09-20T01:02:55.114Z","latency_ms":4466.429999999935,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d6319eda-0142-435d-9790-18c80dd8faf9"],"charged_cents":0,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":3,"prompt_id":"lookup-08","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":1,"started_at":"2026-09-20T01:02:56.394Z","latency_ms":5152.080290999962,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d0cc4956-abe3-4d40-8a86-d6770696d7fe"],"charged_cents":0,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":3,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":2,"started_at":"2026-09-20T01:02:48.833Z","latency_ms":14796.011834000004,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.66,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"ce7a6d2b-b406-4d9b-81a1-3e42cc1c5329","answer_transaction_ids":["fb9d9399-a6fc-4154-8b6a-794a93b206e9"],"charged_cents":1,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":3,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":3,"started_at":"2026-09-20T01:02:52.933Z","latency_ms":14952.38537500007,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.38,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"6c6dafef-d1d4-4c49-be7c-8be541ddf73c","answer_transaction_ids":["1a692e1a-1448-453b-8c8a-cc93db26e134"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":4,"started_at":"2026-09-20T01:02:59.581Z","latency_ms":10208.265874999925,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4ff8304f-37cf-44d7-81d3-511dbf07d03f"],"charged_cents":0,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"lookup-01","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":6,"started_at":"2026-09-20T01:03:03.630Z","latency_ms":8703.523457999923,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["46d0a690-4614-49b4-9d55-09c0492138d0"],"charged_cents":0,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":3,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":5,"started_at":"2026-09-20T01:03:01.546Z","latency_ms":13957.560375000001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.68,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"45bdbbd4-650d-4fb9-8ef8-d3fb54b6fb85","answer_transaction_ids":["93910735-dce1-4361-abaa-bf8ce246ccfc"],"charged_cents":1,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":3,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":7,"started_at":"2026-09-20T01:03:07.886Z","latency_ms":12967.327000000048,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.71,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"36511e99-6641-4aef-91fc-78918ddf94be","answer_transaction_ids":["fb5645bd-8a0f-46c0-8d31-a75051182687"],"charged_cents":1,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":3,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":4,"started_at":"2026-09-20T01:03:09.790Z","latency_ms":13437.649542000028,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.48,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"d3787ce6-f78a-426b-89a8-21f860d0289b","answer_transaction_ids":["648b289e-b6c2-4787-9763-f9bf068958b5"],"charged_cents":1,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":5,"started_at":"2026-09-20T01:03:15.504Z","latency_ms":12606.598291999893,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["3f376df7-7544-462b-96a3-683131d55174"],"charged_cents":0,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":3,"prompt_id":"lookup-01","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":6,"started_at":"2026-09-20T01:03:12.333Z","latency_ms":17162.89079199999,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.86,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"d82cc6d7-a2f7-44e9-b235-eea33268ca88","answer_transaction_ids":["9e9a85dd-329d-4dc0-b7c8-2ebf97aed00f"],"charged_cents":1,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":3,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":7,"started_at":"2026-09-20T01:03:20.853Z","latency_ms":11063.472125000088,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["6e01bf8c-9f7a-4703-b054-593f5148e292"],"charged_cents":0,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":3,"prompt_id":"lookup-06","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":8,"started_at":"2026-09-20T01:03:23.228Z","latency_ms":10775.475790999946,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["234b1c6e-8626-4f26-a55f-f11abfd3cee5"],"charged_cents":0,"answer":{"day":"Thursday","time_utc":"09:30"},"grade":{"pass":true,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":true,"actual":"09:30"}]}}
{"repeat":3,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":9,"started_at":"2026-09-20T01:03:28.111Z","latency_ms":9099.582167000044,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d86781a2-a5e6-4b78-adb8-b747c08d0b6c"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":10,"started_at":"2026-09-20T01:03:29.497Z","latency_ms":11963.779957999941,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.74,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"c10dcdbe-97ad-4c5e-94fa-2311b36fc910","answer_transaction_ids":["81196839-3d96-4cd3-ac2a-ef2b6415efd8"],"charged_cents":1,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":3,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":11,"started_at":"2026-09-20T01:03:31.918Z","latency_ms":10839.078209000058,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.84,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"00b8e1de-b7d8-47ce-8bab-fbec06bdb7da","answer_transaction_ids":["3a2ae846-2735-4d66-9ee2-2315d5499800"],"charged_cents":1,"answer":{"plan":"B","cost_dollars":7.8},"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":true,"actual":"B"},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":7.8}]}}
{"repeat":3,"prompt_id":"lookup-06","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":8,"started_at":"2026-09-20T01:03:34.007Z","latency_ms":12788.573915999965,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"cc49ed8a-2e33-4009-9f7c-190d1523bba2","answer_transaction_ids":["362698a6-9486-46f0-83ca-9fa1c53935ef"],"charged_cents":1,"answer":{"day":"Thursday","time_utc":"09:30 UTC"},"grade":{"pass":false,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":false,"actual":"09:30 UTC"}]}}
{"repeat":3,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":9,"started_at":"2026-09-20T01:03:37.211Z","latency_ms":12146.443749999977,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.76,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"16fb6d48-d577-4df8-a2d3-8304d425cf7a","answer_transaction_ids":["268305f3-8dec-4e93-937b-aa5555bd8ba9"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":10,"started_at":"2026-09-20T01:03:41.461Z","latency_ms":9849.694499999983,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["9d7c2834-33e5-47d8-8fd0-f562eef6c1c8"],"charged_cents":0,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":3,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":11,"started_at":"2026-09-20T01:03:42.757Z","latency_ms":11620.530708000064,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["2cf646d3-f412-4c8b-a849-257357ed6538"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":false,"actual":null},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":12,"started_at":"2026-09-20T01:03:46.796Z","latency_ms":10709.118541999953,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["e96863e7-5bdb-4e37-92ea-bddeeaca85ee"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"lookup-02","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":14,"started_at":"2026-09-20T01:03:51.311Z","latency_ms":8982.55695899995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["afc1e56b-d695-467e-bc51-9af5cdb12a78"],"charged_cents":0,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":3,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":13,"started_at":"2026-09-20T01:03:49.358Z","latency_ms":13995.557792000007,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.28,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"bf911e3d-2071-43fa-8fc4-50e031d8c29f","answer_transaction_ids":["18791b3d-9be1-4822-ab68-1a3aef451847"],"charged_cents":1,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"lookup-09","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":15,"started_at":"2026-09-20T01:03:54.378Z","latency_ms":11025.763583999942,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"11503cbd-588c-4c22-b169-9f950deb1e2b","answer_transaction_ids":["9f34b6ce-79c3-46f6-9652-f33f8d56a992"],"charged_cents":1,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":12,"started_at":"2026-09-20T01:03:57.506Z","latency_ms":9447.922167000012,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.94,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"75ef146a-6998-44c0-be26-f544c0d73f04","answer_transaction_ids":["7e57896f-a9fd-4641-bfd9-07f159605878"],"charged_cents":1,"answer":{"total_dollars":66.6},"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":66.6}]}}
{"repeat":3,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":13,"started_at":"2026-09-20T01:04:03.355Z","latency_ms":5668.155916000018,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["09caa6ad-f858-4a41-9796-ec46d76c3e08"],"charged_cents":0,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"lookup-02","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":14,"started_at":"2026-09-20T01:04:00.294Z","latency_ms":10265.55541599996,"decision":{"lane":"balanced","recommended":"fast","reason":["high_stakes"],"confidence":1,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":false},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"245aced3-ac35-4d58-a69f-b933b180bdf5","answer_transaction_ids":["25561802-5751-4af7-ad0e-4b619881ac66"],"charged_cents":1,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":3,"prompt_id":"lookup-09","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":15,"started_at":"2026-09-20T01:04:05.405Z","latency_ms":6857.490708000027,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4237bffb-e2e9-49eb-ae3d-b3b18b89fa3f"],"charged_cents":0,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":16,"started_at":"2026-09-20T01:04:06.955Z","latency_ms":7479.1596250000875,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["f9e67b6d-a8f5-4a56-99eb-720937b266fe"],"charged_cents":0,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":3,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":17,"started_at":"2026-09-20T01:04:09.024Z","latency_ms":8221.99658399995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["ed27d608-147e-4dd4-a99c-fe32c9ceb965"],"charged_cents":0,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."}]}}
{"repeat":3,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":19,"started_at":"2026-09-20T01:04:12.263Z","latency_ms":9412.447291000048,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["51d6afc7-3f7b-4425-9652-9049843bd87b"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":18,"started_at":"2026-09-20T01:04:10.560Z","latency_ms":14032.175334000029,"decision":{"lane":"balanced","recommended":"fast","reason":["reasoning_tools_or_verification_floor"],"confidence":0.81,"policy_escalated":true,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"843b22c2-4fd5-46f8-8b3b-7c395e172a17","answer_transaction_ids":["122d5bcc-a84b-4093-94ec-1c3b30351e9b"],"charged_cents":1,"answer":{"text":"Your package with tracking number AB-2049 is delayed and expected to arrive by September 23. Currently, we do not have an updated earlier delivery date. Thank you for your understanding."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"Your package with tracking number AB-2049 is delayed and expected to arrive by September 23. Currently, we do not have an updated earlier delivery date. Thank you for your understanding."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"Your package with tracking number AB-2049 is delayed and expected to arrive by September 23. Currently, we do not have an updated earlier delivery date. Thank you for your understanding."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"Your package with tracking number AB-2049 is delayed and expected to arrive by September 23. Currently, we do not have an updated earlier delivery date. Thank you for your understanding."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"Your package with tracking number AB-2049 is delayed and expected to arrive by September 23. Currently, we do not have an updated earlier delivery date. Thank you for your understanding."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"Your package with tracking number AB-2049 is delayed and expected to arrive by September 23. Currently, we do not have an updated earlier delivery date. Thank you for your understanding."}]}}
{"repeat":3,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":16,"started_at":"2026-09-20T01:04:14.434Z","latency_ms":12711.110417000018,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.68,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"702510db-bb2b-42c1-855f-d7b5229c4a31","answer_transaction_ids":["4956bae1-fda2-4b32-b7b4-8cd4d4b662fe"],"charged_cents":1,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":3,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":17,"started_at":"2026-09-20T01:04:17.247Z","latency_ms":12258.540500000003,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.36,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"9d36fbcd-8f75-4b1f-94e0-d072414fc3c9","answer_transaction_ids":["ccd37429-499e-4477-8245-4a5567a0bdef"],"charged_cents":1,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."}]}}
{"repeat":3,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":18,"started_at":"2026-09-20T01:04:24.593Z","latency_ms":8071.01458399999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["bf095b8f-b1e7-4e8a-b1eb-949ce13e82bc"],"charged_cents":0,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":3,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":19,"started_at":"2026-09-20T01:04:21.676Z","latency_ms":16243.79583399999,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.7,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"5e9e97ad-5d71-4e7f-9083-fc6c0e87aa3f","answer_transaction_ids":["78e6d82e-2e40-4c96-a66f-7e310df59b3d"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":21,"started_at":"2026-09-20T01:04:29.506Z","latency_ms":13512.79391700006,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["db63da14-9969-4488-a233-f4de3be1fc11"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"lookup-05","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":20,"started_at":"2026-09-20T01:04:27.146Z","latency_ms":20632.187749999925,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"43e84d37-46c7-41cc-8179-0b396bc84408","answer_transaction_ids":["39f2ef41-f6de-4c14-b0a3-4d6c4aae6108"],"charged_cents":2,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":3,"prompt_id":"lookup-10","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":22,"started_at":"2026-09-20T01:04:32.665Z","latency_ms":17439.812917000032,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"49a8dc5a-aad5-4b42-86ec-5efa102dbae0","answer_transaction_ids":["015515e2-aa82-4d16-bdf8-be64dbd12d26"],"charged_cents":1,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":3,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":23,"started_at":"2026-09-20T01:04:37.920Z","latency_ms":13991.569917000015,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.32,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"dbb776e2-8f26-4c23-be29-c504414af954","answer_transaction_ids":["225d56db-274c-47e0-957f-fd33aa3e0b85"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"lookup-05","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":20,"started_at":"2026-09-20T01:04:47.778Z","latency_ms":6173.52350000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["1d8aaf0e-2562-4def-96c4-072f3e96424b"],"charged_cents":0,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":3,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":21,"started_at":"2026-09-20T01:04:43.019Z","latency_ms":12202.63204200007,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.87,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"73975442-111b-4ef9-bc18-7fe322b92ec5","answer_transaction_ids":["0dbc2ce7-5990-4362-a71b-e268ce09e537"],"charged_cents":1,"answer":{"unreserved_units":54},"grade":{"pass":true,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":true,"actual":54}]}}
{"repeat":3,"prompt_id":"lookup-10","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":22,"started_at":"2026-09-20T01:04:50.105Z","latency_ms":7193.379416999989,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["3ec04f96-b1a2-4798-b0c1-fc69961f8a17"],"charged_cents":0,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":3,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":23,"started_at":"2026-09-20T01:04:51.912Z","latency_ms":7263.234375,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["448a87ce-9414-4de8-aa79-b458bd14c0d5"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":24,"started_at":"2026-09-20T01:04:53.952Z","latency_ms":7192.937333000009,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4dd9094e-3f72-45e5-a597-bea29137bf0d"],"charged_cents":0,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":25,"started_at":"2026-09-20T01:04:55.222Z","latency_ms":11713.544041999965,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["11bc321b-d9ab-463a-af32-32f2b15b003b"],"charged_cents":3,"answer":{"text":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."}]}}
{"repeat":3,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":26,"started_at":"2026-09-20T01:04:57.300Z","latency_ms":14364.532499999972,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["665d5cf0-5cae-470f-87b2-c4c10b95a03f"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"lookup-03","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":27,"started_at":"2026-09-20T01:04:59.176Z","latency_ms":17652.40083399997,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"716ea64a-602c-42e6-b229-4d78a2c6f31e","answer_transaction_ids":["bb091365-e8cf-4802-9386-4d5ec9de9381"],"charged_cents":1,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":3,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":24,"started_at":"2026-09-20T01:05:01.145Z","latency_ms":20237.299165999983,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.56,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"574e9537-2473-4d95-a1fe-e9f4ea7c7135","answer_transaction_ids":["c004313a-2178-4750-bbab-9f167061fdfe"],"charged_cents":1,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":25,"started_at":"2026-09-20T01:05:06.936Z","latency_ms":15996.645708000055,"decision":{"lane":"balanced","recommended":"fast","reason":["low_confidence"],"confidence":0.25,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"723a58d0-5182-4de1-aa2c-dcc4ac0a5bc5","answer_transaction_ids":["e2f09fc7-3cf7-4f60-8ad4-f70ea7e0c177"],"charged_cents":1,"answer":{"text":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."}]}}
{"repeat":3,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":26,"started_at":"2026-09-20T01:05:11.665Z","latency_ms":14171.919791999971,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.77,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"be6f1576-9a9c-4f4d-a3b3-b49288388147","answer_transaction_ids":["73b367cb-9531-4ffc-8a52-9e7047bc1f6b"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"lookup-03","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":27,"started_at":"2026-09-20T01:05:16.829Z","latency_ms":11050.17512499995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d4903d94-1d8b-4d11-b88b-72b2ad778337"],"charged_cents":0,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":3,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":29,"started_at":"2026-09-20T01:05:22.933Z","latency_ms":7734.079041999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["c4f4cddc-bc6b-4cc1-8494-ae96e85566e1"],"charged_cents":0,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":3,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":30,"started_at":"2026-09-20T01:05:25.837Z","latency_ms":6949.7479169999715,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["afe62e65-e025-4494-a2cc-fc16b01f5e5f"],"charged_cents":0,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":28,"started_at":"2026-09-20T01:05:21.383Z","latency_ms":13840.552250000066,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.4,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"54054c9e-9b04-4f8d-9813-0a350a45bb9d","answer_transaction_ids":["60724fde-5bad-42bc-ab49-95e4d770656c"],"charged_cents":1,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":3,"prompt_id":"lookup-04","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":31,"started_at":"2026-09-20T01:05:27.879Z","latency_ms":9833.965833000024,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"a8c046cc-a765-4813-9559-5675b0cfaccf","answer_transaction_ids":["d9ec7774-e7ee-4566-815a-71abc4ed0aca"],"charged_cents":1,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":3,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":28,"started_at":"2026-09-20T01:05:35.225Z","latency_ms":4294.491167000029,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["6679d2aa-dfba-4f03-945c-a7ae955f80e4"],"charged_cents":0,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":3,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":29,"started_at":"2026-09-20T01:05:30.668Z","latency_ms":11398.79125000001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.4,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"32f3078e-284b-4208-8bf8-d24e101e7a33","answer_transaction_ids":["8a73302d-e71d-43cd-a289-58d8070e9657"],"charged_cents":1,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":3,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":30,"started_at":"2026-09-20T01:05:32.788Z","latency_ms":11553.362667000038,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.63,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"71f0d43a-409d-447d-a8ab-c6fc9a065fd6","answer_transaction_ids":["4f42a444-6bfa-4071-8297-a8ea559aab14"],"charged_cents":1,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"lookup-04","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":31,"started_at":"2026-09-20T01:05:37.714Z","latency_ms":8429.761499999906,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["a8c637cf-7c21-4a36-ad18-79103084a99b"],"charged_cents":0,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":3,"prompt_id":"lookup-07","bucket":"lookup","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":33,"started_at":"2026-09-20T01:05:42.067Z","latency_ms":6284.259000000078,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["cee943cf-b8b8-4678-8318-8d973da5d3ed"],"charged_cents":0,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":3,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":35,"started_at":"2026-09-20T01:05:46.144Z","latency_ms":4568.292917000013,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["2b08db88-10bf-41ab-b9d7-34df32edd74e"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":32,"started_at":"2026-09-20T01:05:39.520Z","latency_ms":13127.486082999967,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.39,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"92f54943-7e32-4e20-b731-2724e82bc567","answer_transaction_ids":["92903676-ead5-4434-944d-d6140299493e"],"charged_cents":1,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":3,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":34,"started_at":"2026-09-20T01:05:44.341Z","latency_ms":10212.090417,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.87,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"bcd1be7a-c5fc-4a56-8ebe-2e52aa3cd874","answer_transaction_ids":["5057a036-2145-4c36-b528-47e1660e95da"],"charged_cents":1,"answer":{"finish_minute":12},"grade":{"pass":true,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":true,"actual":12}]}}
{"repeat":3,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":32,"started_at":"2026-09-20T01:05:52.648Z","latency_ms":5128.298790999921,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["33447494-2ecc-44eb-bb3e-c34892b17cf4"],"charged_cents":0,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":3,"prompt_id":"lookup-07","bucket":"lookup","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":33,"started_at":"2026-09-20T01:05:48.351Z","latency_ms":10661.342874999973,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.94,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"ba7592ef-ebb9-43f5-89c9-55764ed2af26","answer_transaction_ids":["2a7f44a6-0b6b-4904-86ca-6f2f7d7ceaf5"],"charged_cents":1,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":3,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":34,"started_at":"2026-09-20T01:05:54.554Z","latency_ms":7839.47412499995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4fc9d952-ec48-4a43-8d49-32549ea38ecf"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":35,"started_at":"2026-09-20T01:05:50.713Z","latency_ms":14702.024749999982,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.43,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"102e09ba-025f-4314-88a7-351fc0a3f1e8","answer_transaction_ids":["401d224f-bd84-41ee-841e-93dd49998ee9"],"charged_cents":4,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":36,"started_at":"2026-09-20T01:05:57.776Z","latency_ms":9807.663458000054,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["eb589e93-7bdb-40e4-a6f2-987f942702f2"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":37,"started_at":"2026-09-20T01:05:59.013Z","latency_ms":12150.143625000026,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["13c308ef-0fb4-4f62-bc43-10c813730705"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"numerator","value":7,"pass":false,"actual":null},{"op":"eq","field":"denominator","value":13,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":38,"started_at":"2026-09-20T01:06:02.394Z","latency_ms":13188.76654099999,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.74,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a8611ab5-0e02-4fde-b3c8-ca1a7007b401","answer_transaction_ids":["0c75c392-6ccc-4481-863e-8ba6b172c6a0"],"charged_cents":1,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":3,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":39,"started_at":"2026-09-20T01:06:05.415Z","latency_ms":12721.076624999987,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.44,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"05133fe1-c38e-4563-9e14-7e233182c2b9","answer_transaction_ids":["2393ce9d-9c85-4302-8c8b-fccc5596e91a"],"charged_cents":1,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":3,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":36,"started_at":"2026-09-20T01:06:07.584Z","latency_ms":12919.785874999943,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.27,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"5027cdfc-b0b7-4a45-9317-0591c56a2b40","answer_transaction_ids":["767ae311-e88a-4661-98f2-669af0520230"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":37,"started_at":"2026-09-20T01:06:11.164Z","latency_ms":10924.258417000063,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.87,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"46aec8d6-c870-4bd6-870b-412858ea6351","answer_transaction_ids":["f9405e68-a891-4077-8694-24b9f46908eb"],"charged_cents":1,"answer":{"numerator":7,"denominator":13},"grade":{"pass":true,"checks":[{"op":"eq","field":"numerator","value":7,"pass":true,"actual":7},{"op":"eq","field":"denominator","value":13,"pass":true,"actual":13}]}}
{"repeat":3,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":38,"started_at":"2026-09-20T01:06:15.583Z","latency_ms":8310.144832999911,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["635ec7fb-53ed-4087-952f-fb23255f26b7"],"charged_cents":0,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":3,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":39,"started_at":"2026-09-20T01:06:18.137Z","latency_ms":7726.878541999962,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["4d5185ea-6d08-47d9-9b5c-350eee576b17"],"charged_cents":0,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":3,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":40,"started_at":"2026-09-20T01:06:20.504Z","latency_ms":9815.37495800003,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.35,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"9847ee27-8f1a-4530-8ccc-967405a4c3a6","answer_transaction_ids":["a354b609-5032-48bd-a4ce-23481cc74a1f"],"charged_cents":1,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."}]}}
{"repeat":3,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":41,"started_at":"2026-09-20T01:06:22.088Z","latency_ms":10403.26683400001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.3,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"eac55b7b-fc84-4c1f-ba46-304582b4335d","answer_transaction_ids":["3a017ccc-e02c-4c82-b39e-9543dfc5d477"],"charged_cents":1,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":3,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":42,"started_at":"2026-09-20T01:06:23.894Z","latency_ms":10734.299791999976,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.55,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"013b832d-c02e-4e0f-9133-66aff3a8c70c","answer_transaction_ids":["b57f2dbd-3c70-42e1-bcc4-a9f00f771441"],"charged_cents":1,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":3,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":43,"started_at":"2026-09-20T01:06:25.864Z","latency_ms":11127.244124999968,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.61,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"240a9902-87d7-4df1-9676-345c07574b0e","answer_transaction_ids":["1d7d12bc-ba7d-4644-90e6-bdf836ae526d"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":40,"started_at":"2026-09-20T01:06:30.320Z","latency_ms":9031.770666999975,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["36ea169e-1ced-4fb0-bbde-b0695f1a288e"],"charged_cents":0,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."}]}}
{"repeat":3,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":41,"started_at":"2026-09-20T01:06:32.492Z","latency_ms":8918.810540999984,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["aa13e871-9d01-453f-a47b-b990cde8dd0a"],"charged_cents":0,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":3,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":42,"started_at":"2026-09-20T01:06:34.629Z","latency_ms":8926.48112500005,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["275e5640-da61-426b-a2a0-f387d5c0621c"],"charged_cents":0,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":3,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":43,"started_at":"2026-09-20T01:06:36.991Z","latency_ms":8711.08604199998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["d3634554-2e63-4c72-96a8-4b59a10639a3"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":44,"started_at":"2026-09-20T01:06:39.354Z","latency_ms":9518.875540999928,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["fa8c4b43-a422-49f4-b65c-d91a9b8b4c0c"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":46,"started_at":"2026-09-20T01:06:43.555Z","latency_ms":8841.32720900001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["f2ad7276-bffa-4ad3-af55-3bc018d8b281"],"charged_cents":0,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":3,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":45,"started_at":"2026-09-20T01:06:41.411Z","latency_ms":14633.18200000003,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.61,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"6ec01265-fc3e-4571-945f-2c703705c526","answer_transaction_ids":["4cb4346d-543d-46a1-919c-2712f3032135"],"charged_cents":1,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":3,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":47,"started_at":"2026-09-20T01:06:45.703Z","latency_ms":15331.327291999944,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.58,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"50cbbbf5-fba5-489e-824d-daec69b5740c","answer_transaction_ids":["80a09402-c7f9-41ed-9c91-419039d82a5b"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":44,"started_at":"2026-09-20T01:06:48.873Z","latency_ms":15018.084124999936,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.6,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"cdc1bcd2-0087-4f77-9b5c-50c3c91256d4","answer_transaction_ids":["62687f2f-d1a7-4ccc-bd86-51cd7827e5c2"],"charged_cents":1,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":45,"started_at":"2026-09-20T01:06:56.044Z","latency_ms":9729.073749999981,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["f5b10141-0fd0-4d92-9238-482d10b71ce7"],"charged_cents":0,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":3,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":46,"started_at":"2026-09-20T01:06:52.398Z","latency_ms":16156.76887499995,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.58,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0439fbf6-79f4-4cf7-bd57-2550dceac660","answer_transaction_ids":["1552dce7-2c7f-48b1-ac88-9df64f4ba8d8"],"charged_cents":1,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working to restore full functionality. The next update will be provided at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":3,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":47,"started_at":"2026-09-20T01:07:01.035Z","latency_ms":12474.75774999999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["e4afd946-2def-4def-a41e-c7229291751c"],"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":48,"started_at":"2026-09-20T01:07:03.892Z","latency_ms":11503.545958999894,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["7b51f789-f50e-4229-ac6b-a6f07c6f374d"],"charged_cents":0,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":3,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"control","run_id":"jev-20260919-50x3-v1","sequence":49,"started_at":"2026-09-20T01:07:05.774Z","latency_ms":11729.53354199999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_transaction_ids":["32d1fc60-b1ae-4076-af66-ed9cbed80764"],"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":48,"started_at":"2026-09-20T01:07:15.397Z","latency_ms":5706.252417000011,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.79,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"4d923087-1642-4b16-9dfc-31e885212e76","answer_transaction_ids":["49cf4093-2efe-437c-bce1-6bde6608c752"],"charged_cents":1,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":3,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260919-50x3-v1","sequence":49,"started_at":"2026-09-20T01:07:17.505Z","latency_ms":5805.0919170000125,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.51,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"ec8fffaf-f8d5-4a30-878b-8cd18fbe8635","answer_transaction_ids":["4a7f5082-8992-4e87-bb31-521d6af9238f"],"charged_cents":1,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}