[ { "repeat": 1, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 4, "started_at": "2026-09-20T00:53:37.073Z", "latency_ms": 11289.595500000003, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.84, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "60c933dd-3279-40a9-93dc-4426837442e1", "answer_transaction_ids": [ "85518768-f9d1-42d6-a152-6f1054073ca1" ], "charged_cents": 1, "answer": { "plan": "B", "cost_dollars": 7.8 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": true, "actual": "B" }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": 7.8 } ] } }, { "repeat": 1, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 4, "started_at": "2026-09-20T00:53:48.363Z", "latency_ms": 7509.08525, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "1140ca8e-11c3-4f34-9585-158f8f7a5781" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": false, "actual": null }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 9, "started_at": "2026-09-20T00:53:57.698Z", "latency_ms": 13121.976999999999, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.59, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "18ce0b21-2546-4df0-be8b-dc91c36f8239", "answer_transaction_ids": [ "b7513814-7700-4778-bdf2-c98970b0fc83" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 9, "started_at": "2026-09-20T00:54:10.821Z", "latency_ms": 4682.245624999996, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "72774c24-b459-4796-8cbf-7f9d9424a48f" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 15, "started_at": "2026-09-20T00:54:32.374Z", "latency_ms": 12336.057249999998, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes", "low_confidence" ], "confidence": 0.58, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "2dbb13ac-b42c-4788-b793-dc9f1e12e42f", "answer_transaction_ids": [ "b69bab5c-676e-45ee-b792-7cfc1366c15d" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 15, "started_at": "2026-09-20T00:54:44.711Z", "latency_ms": 9601.480083000002, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "29ca92a2-d866-4c4f-b152-ec870db256d5" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 19, "started_at": "2026-09-20T00:54:54.313Z", "latency_ms": 11624.601875000008, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.7, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "3de22f1a-a764-4d4a-ae9b-86f5dfdfc31e", "answer_transaction_ids": [ "5fc36035-40fb-4268-a6df-b2206cb2ec97" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-07", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 16, "started_at": "2026-09-20T00:54:57.695Z", "latency_ms": 11163.560624999998, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "b406c3e0-3fa6-4adc-a652-d43960e2d002" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "unreserved_units", "value": 54, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 19, "started_at": "2026-09-20T00:55:05.939Z", "latency_ms": 12846.859333999993, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "5021694b-8365-49f8-95e7-b0dea1714f4d" ], "charged_cents": 3, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "lookup-06", "bucket": "lookup", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 21, "started_at": "2026-09-20T00:55:23.147Z", "latency_ms": 12123.07983399999, "decision": { "lane": "fast", "recommended": "fast", "reason": [], "confidence": 0.99, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "fast", "final_lane": "fast", "fallback": false, "router_transaction_id": "60e7cf40-e485-47a2-87bb-73a24c08b8bb", "answer_transaction_ids": [ "fcffcd6e-bcc9-4fb5-a66c-139e4b07550c" ], "charged_cents": 2, "answer": { "day": "Thursday", "time_utc": "09:30 UTC" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "day", "value": "Thursday", "pass": true, "actual": "Thursday" }, { "op": "eq", "field": "time_utc", "value": "09:30", "pass": false, "actual": "09:30 UTC" } ] } }, { "repeat": 1, "prompt_id": "multi-step-05", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 23, "started_at": "2026-09-20T00:55:30.666Z", "latency_ms": 11166.801875000005, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "6bbbaeac-a9ce-4b30-9051-31dbe99e1a4a" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "numerator", "value": 7, "pass": false, "actual": null }, { "op": "eq", "field": "denominator", "value": 13, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 24, "started_at": "2026-09-20T00:55:33.605Z", "latency_ms": 11892.888582999993, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "a3341ebc-31ca-42ab-b806-75ba1b294afa" ], "charged_cents": 3, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 25, "started_at": "2026-09-20T00:55:35.270Z", "latency_ms": 18170.06370900001, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.5, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "c8a5c9ff-1976-4ed4-a561-4b4f20255ccc", "answer_transaction_ids": [ "89dca8db-6661-48c0-a3ec-53f3e1e1fbf5" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 24, "started_at": "2026-09-20T00:55:45.498Z", "latency_ms": 14662.772999999986, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.92, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "1ca5cf99-3b39-41bc-8c05-a29903212815", "answer_transaction_ids": [ "e07c44f3-41f7-4d3a-8f39-af5d892c55bf" ], "charged_cents": 1, "answer": { "total_dollars": 66.6 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": 66.6 } ] } }, { "repeat": 1, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 25, "started_at": "2026-09-20T00:55:53.442Z", "latency_ms": 11018.140459000017, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "5e4d4157-88f1-422f-aa44-c82ee07d0eb6" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 30, "started_at": "2026-09-20T00:56:06.145Z", "latency_ms": 9884.255583999999, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "8e2777b4-ff8f-4508-b184-36413bf687f5" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 30, "started_at": "2026-09-20T00:56:16.030Z", "latency_ms": 16816.18183300001, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.79, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "03bbb923-0674-49e3-8545-e3768f915235", "answer_transaction_ids": [ "c5aa53df-ff35-4c74-b0ee-2c64e5aa3767" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 41, "started_at": "2026-09-20T00:57:07.439Z", "latency_ms": 9965.423291999992, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "da11cbf8-f4fc-4dc1-b904-fe3d9a5a75ac" ], "charged_cents": 0, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 1, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 42, "started_at": "2026-09-20T00:57:10.231Z", "latency_ms": 12394.714833000005, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.83, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "74c2157e-2f7f-4ba8-ae8b-21cdf252fa93", "answer_transaction_ids": [ "48219c20-f643-44d6-98a4-7cc119dd2851" ], "charged_cents": 1, "answer": { "seconds": 20 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": 20 } ] } }, { "repeat": 1, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 41, "started_at": "2026-09-20T00:57:17.405Z", "latency_ms": 9165.475542, "decision": { "lane": "frontier", "recommended": "fast", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes" ], "confidence": 0.84, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": true, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "baa28236-74cc-40b1-98d8-c2804f36ab9c", "answer_transaction_ids": [ "bac51074-7ae3-4df6-af2e-3f7458e940f9" ], "charged_cents": 1, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 1, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 42, "started_at": "2026-09-20T00:57:22.626Z", "latency_ms": 6716.563833000022, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "28d95a41-127f-4f9c-9b7e-a07eed488c38" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-03", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 45, "started_at": "2026-09-20T00:57:26.570Z", "latency_ms": 10753.061959000013, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "fa667906-36e1-4d69-b7b3-c93e40ce5364" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "finish_minute", "value": 12, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 4, "started_at": "2026-09-20T00:58:13.282Z", "latency_ms": 16931.028292000003, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.73, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "3cbff403-ae19-4d3e-bb12-6ff5bf7cda99", "answer_transaction_ids": [ "de27ea52-755a-47d1-b82e-2332afd7412b" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 4, "started_at": "2026-09-20T00:58:30.214Z", "latency_ms": 5991.778500000015, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "eab0a276-0a04-4fb9-9202-4b87a08221c6" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 13, "started_at": "2026-09-20T00:59:00.354Z", "latency_ms": 9794.86825, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "9304dcd7-0b35-4f27-800b-0c526f00b18b" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 14, "started_at": "2026-09-20T00:59:03.664Z", "latency_ms": 10230.074958000041, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "e89e0bf4-f9fc-4007-a670-782d6574f9ee" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 13, "started_at": "2026-09-20T00:59:10.149Z", "latency_ms": 13646.715832999966, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.54, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "9d9c80ba-6110-44ba-b7d8-2da5fad60127", "answer_transaction_ids": [ "cc9c1599-3499-4d6d-b56f-e20f83b2b707" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 14, "started_at": "2026-09-20T00:59:13.894Z", "latency_ms": 14463.045165999967, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.7, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "7cb313b9-6c86-4b68-9660-7f3555b6d610", "answer_transaction_ids": [ "afc9b5e3-a33c-4145-8576-9f8811312b34" ], "charged_cents": 4, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 16, "started_at": "2026-09-20T00:59:21.036Z", "latency_ms": 11908.303625, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "dedcb808-90f8-476f-ad91-ad8f2e884598" ], "charged_cents": 0, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 2, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 16, "started_at": "2026-09-20T00:59:32.944Z", "latency_ms": 14520.798332999984, "decision": { "lane": "frontier", "recommended": "fast", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes" ], "confidence": 0.86, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": true, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "2119e426-3208-4280-a40c-efe4f6326bea", "answer_transaction_ids": [ "1bd1ed18-d60e-4369-b505-21c92e9d526c" ], "charged_cents": 1, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 2, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 24, "started_at": "2026-09-20T01:00:09.611Z", "latency_ms": 9557.680999999982, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "6231df40-ad38-46e9-8ef9-e2555394e00a" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 26, "started_at": "2026-09-20T01:00:13.121Z", "latency_ms": 9848.820917000005, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "d5fcd4da-432e-4d5d-a526-49195bc70fc3" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": false, "actual": null }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 24, "started_at": "2026-09-20T01:00:19.169Z", "latency_ms": 13847.935374999943, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.52, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "4f7890a6-e527-4184-91f8-3df3a2205c02", "answer_transaction_ids": [ "d78e21a4-c473-420c-b4f6-0a477c51436e" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 26, "started_at": "2026-09-20T01:00:22.970Z", "latency_ms": 15763.366000000038, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.84, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "8ab00994-840d-4fc9-b616-8e737b9dff15", "answer_transaction_ids": [ "b0d8db40-03a2-444e-9ae4-ffdbb02c3987" ], "charged_cents": 1, "answer": { "plan": "B", "cost_dollars": 7.8 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": true, "actual": "B" }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": 7.8 } ] } }, { "repeat": 2, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 30, "started_at": "2026-09-20T01:00:38.734Z", "latency_ms": 7756.321083000046, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "387e77a8-b208-4051-b26e-38e74619cac1" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 30, "started_at": "2026-09-20T01:00:46.491Z", "latency_ms": 16826.10287500004, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes", "low_confidence" ], "confidence": 0.56, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "a977d52d-5e14-4931-9c7f-84e43efd8b40", "answer_transaction_ids": [ "a19f8f61-c970-4a23-8387-b98d9034c252" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 32, "started_at": "2026-09-20T01:00:56.940Z", "latency_ms": 12060.978833000001, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "ddca6a39-c21c-40e2-b271-eaa39c77223d" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-05", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 35, "started_at": "2026-09-20T01:01:05.554Z", "latency_ms": 7800.165291999991, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "d313533a-dc30-4972-b610-7ff245756ad2" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "numerator", "value": 7, "pass": false, "actual": null }, { "op": "eq", "field": "denominator", "value": 13, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 32, "started_at": "2026-09-20T01:01:09.001Z", "latency_ms": 20068.269458000024, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.81, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "cd058bd0-01b5-4ef3-b8de-f7d5559816ef", "answer_transaction_ids": [ "25b68a27-5bc0-4a1f-97f9-5c96bd3635e2" ], "charged_cents": 1, "answer": { "records_per_group": 56 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": 56 } ] } }, { "repeat": 2, "prompt_id": "lookup-06", "bucket": "lookup", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 36, "started_at": "2026-09-20T01:01:29.070Z", "latency_ms": 15225.434582999966, "decision": { "lane": "fast", "recommended": "fast", "reason": [], "confidence": 0.99, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "fast", "final_lane": "fast", "fallback": false, "router_transaction_id": "ba69a01f-af90-4bb4-b1a5-cdcad098619b", "answer_transaction_ids": [ "1a585420-1ff6-4d6a-9613-67326a4ebe39" ], "charged_cents": 1, "answer": { "day": "Thursday", "time_utc": "09:30 UTC" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "day", "value": "Thursday", "pass": true, "actual": "Thursday" }, { "op": "eq", "field": "time_utc", "value": "09:30", "pass": false, "actual": "09:30 UTC" } ] } }, { "repeat": 2, "prompt_id": "multi-step-07", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 38, "started_at": "2026-09-20T01:01:46.274Z", "latency_ms": 9373.587, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "2d03b581-8cfd-4911-8433-5ea676be2577" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "unreserved_units", "value": 54, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-03", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 43, "started_at": "2026-09-20T01:02:08.085Z", "latency_ms": 10997.423875000095, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "9e51301c-fef9-4c31-8dbc-c6514b39ba6f" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "finish_minute", "value": 12, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 47, "started_at": "2026-09-20T01:02:19.082Z", "latency_ms": 9705.942792000016, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.92, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "b3c3d274-3a5d-4e05-8740-330557b4b2a9", "answer_transaction_ids": [ "5e61251e-5fd4-4258-b4a2-10a1a79ca002" ], "charged_cents": 1, "answer": { "total_dollars": 66.6 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": 66.6 } ] } }, { "repeat": 2, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 47, "started_at": "2026-09-20T01:02:28.789Z", "latency_ms": 9114.900332999998, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "7b76afb3-47ff-4b44-a690-cf693fce9a28" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 3, "started_at": "2026-09-20T01:02:45.800Z", "latency_ms": 7133.080041999929, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "aeea9f06-7c5b-4e78-8bd2-5a58828388cf" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 3, "started_at": "2026-09-20T01:02:52.933Z", "latency_ms": 14952.38537500007, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.38, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "6c6dafef-d1d4-4c49-be7c-8be541ddf73c", "answer_transaction_ids": [ "1a692e1a-1448-453b-8c8a-cc93db26e134" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 9, "started_at": "2026-09-20T01:03:28.111Z", "latency_ms": 9099.582167000044, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "d86781a2-a5e6-4b78-adb8-b747c08d0b6c" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 11, "started_at": "2026-09-20T01:03:31.918Z", "latency_ms": 10839.078209000058, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.84, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "00b8e1de-b7d8-47ce-8bab-fbec06bdb7da", "answer_transaction_ids": [ "3a2ae846-2735-4d66-9ee2-2315d5499800" ], "charged_cents": 1, "answer": { "plan": "B", "cost_dollars": 7.8 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": true, "actual": "B" }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": 7.8 } ] } }, { "repeat": 3, "prompt_id": "lookup-06", "bucket": "lookup", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 8, "started_at": "2026-09-20T01:03:34.007Z", "latency_ms": 12788.573915999965, "decision": { "lane": "fast", "recommended": "fast", "reason": [], "confidence": 0.99, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "fast", "final_lane": "fast", "fallback": false, "router_transaction_id": "cc49ed8a-2e33-4009-9f7c-190d1523bba2", "answer_transaction_ids": [ "362698a6-9486-46f0-83ca-9fa1c53935ef" ], "charged_cents": 1, "answer": { "day": "Thursday", "time_utc": "09:30 UTC" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "day", "value": "Thursday", "pass": true, "actual": "Thursday" }, { "op": "eq", "field": "time_utc", "value": "09:30", "pass": false, "actual": "09:30 UTC" } ] } }, { "repeat": 3, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 9, "started_at": "2026-09-20T01:03:37.211Z", "latency_ms": 12146.443749999977, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.76, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "16fb6d48-d577-4df8-a2d3-8304d425cf7a", "answer_transaction_ids": [ "268305f3-8dec-4e93-937b-aa5555bd8ba9" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 11, "started_at": "2026-09-20T01:03:42.757Z", "latency_ms": 11620.530708000064, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "2cf646d3-f412-4c8b-a849-257357ed6538" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": false, "actual": null }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 12, "started_at": "2026-09-20T01:03:46.796Z", "latency_ms": 10709.118541999953, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "e96863e7-5bdb-4e37-92ea-bddeeaca85ee" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 12, "started_at": "2026-09-20T01:03:57.506Z", "latency_ms": 9447.922167000012, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.94, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "75ef146a-6998-44c0-be26-f544c0d73f04", "answer_transaction_ids": [ "7e57896f-a9fd-4641-bfd9-07f159605878" ], "charged_cents": 1, "answer": { "total_dollars": 66.6 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": 66.6 } ] } }, { "repeat": 3, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 19, "started_at": "2026-09-20T01:04:12.263Z", "latency_ms": 9412.447291000048, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "51d6afc7-3f7b-4425-9652-9049843bd87b" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 19, "started_at": "2026-09-20T01:04:21.676Z", "latency_ms": 16243.79583399999, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.7, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "5e9e97ad-5d71-4e7f-9083-fc6c0e87aa3f", "answer_transaction_ids": [ "78e6d82e-2e40-4c96-a66f-7e310df59b3d" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-07", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 21, "started_at": "2026-09-20T01:04:29.506Z", "latency_ms": 13512.79391700006, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "db63da14-9969-4488-a233-f4de3be1fc11" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "unreserved_units", "value": 54, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 26, "started_at": "2026-09-20T01:04:57.300Z", "latency_ms": 14364.532499999972, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "665d5cf0-5cae-470f-87b2-c4c10b95a03f" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 26, "started_at": "2026-09-20T01:05:11.665Z", "latency_ms": 14171.919791999971, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.77, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "be6f1576-9a9c-4f4d-a3b3-b49288388147", "answer_transaction_ids": [ "73b367cb-9531-4ffc-8a52-9e7047bc1f6b" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-03", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 34, "started_at": "2026-09-20T01:05:54.554Z", "latency_ms": 7839.47412499995, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "4fc9d952-ec48-4a43-8d49-32549ea38ecf" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "finish_minute", "value": 12, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-05", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 37, "started_at": "2026-09-20T01:05:59.013Z", "latency_ms": 12150.143625000026, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "13c308ef-0fb4-4f62-bc43-10c813730705" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "numerator", "value": 7, "pass": false, "actual": null }, { "op": "eq", "field": "denominator", "value": 13, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 44, "started_at": "2026-09-20T01:06:39.354Z", "latency_ms": 9518.875540999928, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "fa8c4b43-a422-49f4-b65c-d91a9b8b4c0c" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 47, "started_at": "2026-09-20T01:06:45.703Z", "latency_ms": 15331.327291999944, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes", "low_confidence" ], "confidence": 0.58, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "50cbbbf5-fba5-489e-824d-daec69b5740c", "answer_transaction_ids": [ "80a09402-c7f9-41ed-9c91-419039d82a5b" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 44, "started_at": "2026-09-20T01:06:48.873Z", "latency_ms": 15018.084124999936, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.6, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "cdc1bcd2-0087-4f77-9b5c-50c3c91256d4", "answer_transaction_ids": [ "62687f2f-d1a7-4ccc-bd86-51cd7827e5c2" ], "charged_cents": 1, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 47, "started_at": "2026-09-20T01:07:01.035Z", "latency_ms": 12474.75774999999, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "e4afd946-2def-4def-a41e-c7229291751c" ], "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "control", "run_id": "jev-20260919-50x3-v1", "sequence": 48, "started_at": "2026-09-20T01:07:03.892Z", "latency_ms": 11503.545958999894, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_transaction_ids": [ "7b51f789-f50e-4229-ac6b-a6f07c6f374d" ], "charged_cents": 0, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 3, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "treatment", "run_id": "jev-20260919-50x3-v1", "sequence": 48, "started_at": "2026-09-20T01:07:15.397Z", "latency_ms": 5706.252417000011, "decision": { "lane": "frontier", "recommended": "fast", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes", "low_confidence" ], "confidence": 0.79, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": true, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "4d923087-1642-4b16-9dfc-31e885212e76", "answer_transaction_ids": [ "49cf4093-2efe-437c-bce1-6bde6608c752" ], "charged_cents": 1, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } } ]