{"protocol":"ocb-competitor-pilot-0.1","attempt":"pilot4","generated_at":"2026-10-20T02:38:02.066Z","scope":"Two development task families, one repeat, fresh native threads. No independent holdout or statistical superiority claim.","complete":true,"plan_sha256":"cc71c2468c5c2b6bdb17046a92e75c67ef099ea5f606812ca162eba7ad74d868","image_id":"sha256:9c55dec64c64cafe68e3d4bac49f829ded5f1b82cf595be201d41bd901be01c4","models":{"openai":{"model":"gpt-6.1-sol","input":2,"cacheRead":0.1,"cacheWrite":2.5,"output":10},"anthropic":{"model":"claude-opus-5-5","input":4,"cacheRead":0.2,"cacheWrite":5,"cacheWriteLong":8,"output":20}},"source_sha256":"45f5bc53cc79cc6e520eae157d92528d30a493367998d2a2ea012fb659c1cdf8","groups":[{"condition":"cairntir","planned":6,"completed":6,"successes":6,"cost_usd":1.5308413,"cost_per_success_usd":0.2551402166666667,"tokens":{"input":1522725,"uncached":230,"cacheRead":1297653,"cacheWrite":224842,"cacheWriteLong":0,"output":38737},"total_seconds":877.891},{"condition":"handoff_md","planned":6,"completed":6,"successes":6,"cost_usd":0.69415773,"cost_per_success_usd":0.115692955,"tokens":{"input":541772,"uncached":147,"cacheRead":439948,"cacheWrite":101677,"cacheWriteLong":0,"output":19906},"total_seconds":442.932},{"condition":"onlybots","planned":6,"completed":6,"successes":6,"cost_usd":0.58617585,"cost_per_success_usd":0.097695975,"tokens":{"input":490914,"uncached":1351,"cacheRead":395702,"cacheWrite":93861,"cacheWriteLong":0,"output":15251},"total_seconds":383.057}],"rows":[{"run_id":"pilot2-same-pilot-retry-after-openai-cairntir","lane":"same","task":"pilot-retry-after","provider":"openai","receiver_provider":"openai","condition":"cairntir","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.23373904,"tokens":{"input":346326,"uncached":60,"cacheRead":289715,"cacheWrite":56551,"cacheWriteLong":0,"output":6327},"seconds":199.809,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":true,"cairntir_receipts":[{"episode":"investigate","successful_writes":2,"successful_reads":2,"tool_errors":1,"tool_receipts":6},{"episode":"implement","successful_writes":3,"successful_reads":2,"tool_errors":0,"tool_receipts":7}],"integrity_passed":true},{"run_id":"pilot2-same-pilot-retry-after-openai-handoff_md","lane":"same","task":"pilot-retry-after","provider":"openai","receiver_provider":"openai","condition":"handoff_md","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.10790911,"tokens":{"input":110369,"uncached":30,"cacheRead":84941,"cacheWrite":25398,"cacheWriteLong":0,"output":3586},"seconds":104.734,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot2-same-pilot-retry-after-openai-onlybots","lane":"same","task":"pilot-retry-after","provider":"openai","receiver_provider":"openai","condition":"onlybots","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.09105052,"tokens":{"input":97817,"uncached":244,"cacheRead":73725,"cacheWrite":23848,"cacheWriteLong":0,"output":2357},"seconds":78.839,"onlybots_context_actions_confirmed":true,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot2-same-pilot-retry-after-anthropic-cairntir","lane":"same","task":"pilot-retry-after","provider":"anthropic","receiver_provider":"anthropic","condition":"cairntir","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.32388021,"tokens":{"input":221055,"uncached":26,"cacheRead":193656,"cacheWrite":27373,"cacheWriteLong":0,"output":7409},"seconds":104.335,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":true,"cairntir_receipts":[{"episode":"investigate","successful_writes":3,"successful_reads":2,"tool_errors":1,"tool_receipts":6},{"episode":"implement","successful_writes":2,"successful_reads":2,"tool_errors":0,"tool_receipts":4}],"integrity_passed":true},{"run_id":"pilot2-same-pilot-retry-after-anthropic-handoff_md","lane":"same","task":"pilot-retry-after","provider":"anthropic","receiver_provider":"anthropic","condition":"handoff_md","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.15725402,"tokens":{"input":75810,"uncached":20,"cacheRead":62670,"cacheWrite":13120,"cacheWriteLong":0,"output":3952},"seconds":55.974,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot2-same-pilot-retry-after-anthropic-onlybots","lane":"same","task":"pilot-retry-after","provider":"anthropic","receiver_provider":"anthropic","condition":"onlybots","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.10239243,"tokens":{"input":58474,"uncached":16,"cacheRead":52242,"cacheWrite":6216,"cacheWriteLong":0,"output":3040},"seconds":48.428,"onlybots_context_actions_confirmed":true,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot4-same-pilot-invoice-rounding-openai-cairntir","lane":"same","task":"pilot-invoice-rounding","provider":"openai","receiver_provider":"openai","condition":"cairntir","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.20015622,"tokens":{"input":300853,"uncached":51,"cacheRead":255617,"cacheWrite":45185,"cacheWriteLong":0,"output":6153},"seconds":189.835,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":true,"cairntir_receipts":[{"episode":"investigate","successful_writes":2,"successful_reads":2,"tool_errors":1,"tool_receipts":6},{"episode":"implement","successful_writes":4,"successful_reads":2,"tool_errors":1,"tool_receipts":9}],"integrity_passed":true},{"run_id":"pilot4-same-pilot-invoice-rounding-openai-handoff_md","lane":"same","task":"pilot-invoice-rounding","provider":"openai","receiver_provider":"openai","condition":"handoff_md","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.09712062,"tokens":{"input":121107,"uncached":33,"cacheRead":96571,"cacheWrite":24503,"cacheWriteLong":0,"output":2614},"seconds":92.579,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot4-same-pilot-invoice-rounding-openai-onlybots","lane":"same","task":"pilot-invoice-rounding","provider":"openai","receiver_provider":"openai","condition":"onlybots","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.09125763,"tokens":{"input":120200,"uncached":248,"cacheRead":96141,"cacheWrite":23811,"cacheWriteLong":0,"output":2162},"seconds":86.05,"onlybots_context_actions_confirmed":true,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot2-same-pilot-invoice-rounding-anthropic-cairntir","lane":"same","task":"pilot-invoice-rounding","provider":"anthropic","receiver_provider":"anthropic","condition":"cairntir","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.20868561,"tokens":{"input":177190,"uncached":22,"cacheRead":164538,"cacheWrite":12630,"cacheWriteLong":0,"output":5627},"seconds":83.31,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":true,"cairntir_receipts":[{"episode":"investigate","successful_writes":2,"successful_reads":2,"tool_errors":0,"tool_receipts":4},{"episode":"implement","successful_writes":2,"successful_reads":1,"tool_errors":1,"tool_receipts":4}],"integrity_passed":true},{"run_id":"pilot2-same-pilot-invoice-rounding-anthropic-handoff_md","lane":"same","task":"pilot-invoice-rounding","provider":"anthropic","receiver_provider":"anthropic","condition":"handoff_md","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.11092944,"tokens":{"input":58127,"uncached":16,"cacheRead":51277,"cacheWrite":6834,"cacheWriteLong":0,"output":3322},"seconds":44.363,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot3-same-pilot-invoice-rounding-anthropic-onlybots","lane":"same","task":"pilot-invoice-rounding","provider":"anthropic","receiver_provider":"anthropic","condition":"onlybots","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.08988142,"tokens":{"input":59515,"uncached":16,"cacheRead":53737,"cacheWrite":5762,"cacheWriteLong":0,"output":2513},"seconds":46.574,"onlybots_context_actions_confirmed":true,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot4-cross-pilot-retry-after-openai-onlybots","lane":"cross","task":"pilot-retry-after","provider":"openai","receiver_provider":"anthropic","condition":"onlybots","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.10997572,"tokens":{"input":82047,"uncached":27,"cacheRead":62656,"cacheWrite":19364,"cacheWriteLong":0,"output":2046},"seconds":53.55,"onlybots_context_actions_confirmed":true,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot4-cross-pilot-retry-after-openai-handoff_md","lane":"cross","task":"pilot-retry-after","provider":"openai","receiver_provider":"anthropic","condition":"handoff_md","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.10682643,"tokens":{"input":96062,"uncached":26,"cacheRead":79906,"cacheWrite":16130,"cacheWriteLong":0,"output":2930},"seconds":70.465,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot4-cross-pilot-retry-after-openai-cairntir","lane":"cross","task":"pilot-retry-after","provider":"openai","receiver_provider":"anthropic","condition":"cairntir","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.29073772,"tokens":{"input":237477,"uncached":35,"cacheRead":196449,"cacheWrite":40993,"cacheWriteLong":0,"output":6752},"seconds":159.684,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":true,"cairntir_receipts":[{"episode":"investigate","successful_writes":2,"successful_reads":2,"tool_errors":1,"tool_receipts":6},{"episode":"implement","successful_writes":2,"successful_reads":2,"tool_errors":0,"tool_receipts":4}],"integrity_passed":true},{"run_id":"pilot3-cross-pilot-retry-after-anthropic-onlybots","lane":"cross","task":"pilot-retry-after","provider":"anthropic","receiver_provider":"openai","condition":"onlybots","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.10161813,"tokens":{"input":72861,"uncached":800,"cacheRead":57201,"cacheWrite":14860,"cacheWriteLong":0,"output":3133},"seconds":69.616,"onlybots_context_actions_confirmed":true,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot4-cross-pilot-retry-after-anthropic-handoff_md","lane":"cross","task":"pilot-retry-after","provider":"anthropic","receiver_provider":"openai","condition":"handoff_md","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.11411811,"tokens":{"input":80297,"uncached":22,"cacheRead":64583,"cacheWrite":15692,"cacheWriteLong":0,"output":3502},"seconds":74.817,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":null,"cairntir_receipts":null,"integrity_passed":true},{"run_id":"pilot4-cross-pilot-retry-after-anthropic-cairntir","lane":"cross","task":"pilot-retry-after","provider":"anthropic","receiver_provider":"openai","condition":"cairntir","status":"completed","success":true,"failure":null,"error":null,"cost_usd":0.2736425,"tokens":{"input":239824,"uncached":36,"cacheRead":197678,"cacheWrite":42110,"cacheWriteLong":0,"output":6469},"seconds":140.918,"onlybots_context_actions_confirmed":null,"cairntir_tools_invoked":true,"cairntir_receipts":[{"episode":"investigate","successful_writes":2,"successful_reads":1,"tool_errors":1,"tool_receipts":4},{"episode":"implement","successful_writes":4,"successful_reads":3,"tool_errors":1,"tool_receipts":10}],"integrity_passed":true}],"other_attempts":[{"run_id":"pilot1-same-pilot-retry-after-openai-cairntir","status":"infrastructure_error","success":false,"error":"Cairntir read/write treatment was not used in every expected episode; inspect integration before scoring","cost_usd":0.09060212},{"run_id":"pilot1-same-pilot-retry-after-anthropic-cairntir","status":"completed","success":true,"error":null,"cost_usd":0.31742584},{"run_id":"pilot2-same-pilot-invoice-rounding-openai-cairntir","status":"infrastructure_error","success":false,"error":"Cairntir read/write treatment was not used in every expected episode; inspect integration before scoring","cost_usd":0.22690982},{"run_id":"pilot3-same-pilot-invoice-rounding-openai-cairntir","status":"completed","success":false,"error":"terminated: HPE_INVALID_EOF_STATE: Response does not match the HTTP/1.1 protocol (Invalid EOF state)","cost_usd":0.1189885}],"campaign_spend":{"openai":{"cap_usd":20,"accounted_usd":1.63867871,"retained_uncertain_usd":0.0810575,"unresolved_requests":0},"anthropic":{"cap_usd":20,"accounted_usd":1.92642245,"retained_uncertain_usd":0,"unresolved_requests":0}},"updated":"2026-10-20","hosts":{"codex":"0.159.3","claude_code":"2.1.301"},"competitor":{"name":"Cairntir","version":"1.12.3","configuration":"Official MCP and unmodified host policy; default production embeddings"},"conditions":[{"condition":"onlybots","trials":6,"successes":6,"cost_usd":0.58617585,"total_tokens":506165},{"condition":"handoff_md","trials":6,"successes":6,"cost_usd":0.69415773,"total_tokens":561678},{"condition":"cairntir","trials":6,"successes":6,"cost_usd":1.5308413,"total_tokens":1561462}],"comparisons":[{"baseline":"handoff_md","cost_savings_fraction":0.15555813229941262,"token_savings_fraction":0.09883420749967065},{"baseline":"cairntir","cost_savings_fraction":0.6170890803638496,"token_savings_fraction":0.6758390533999546}],"source":{"file":"benchmark/continuity/results/competitor-pilot-2026-10/summary.json","sha256":"86a8c4ffa570ded0490b824af8aa0f9bdea329bfbe63f8870ab6b87c0d4c9dbb"}}