forked from BasedHardware/omi
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathjit_cost_evidence_driver.py
More file actions
3321 lines (3146 loc) · 155 KB
/
Copy pathjit_cost_evidence_driver.py
File metadata and controls
3321 lines (3146 loc) · 155 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""Build and validate a bounded, matched JIT cost-evidence run.
This driver deliberately has no provider client. ``--plan`` emits
the exact synthetic inputs and source-derived route/prompt hashes that a later
operator run must use. ``--join-receipts`` joins content-free endpoint
observations (including the exact ``X-Omi-Request-ID``) to an exported
``llm_gateway_attempts`` ledger, preserving every retry attempt. The resulting
envelope is consumed by ``--validate-receipts`` together with a harness
sidecar. The AccountingEvent remains the authority for
provider/model/rate-card/usage/cost; the sidecar joins each ``attempt_id`` to
the synthetic case and records the matched prompt/evidence hashes and all
gateway/tool/cache counters. Missing usage, model, route, hash, attempt,
tool-round, cache-unit, or cost fields remain unknown and block the
comparison; they are never converted to zero.
The capture modes read only an explicitly named QA agent-state snapshot and,
for ``--export-attempts``/``--export-jit-receipt``, an explicitly fenced
development Firestore ledger;
they never invoke a provider.
The released desktop proactivity response exposes only lane/model and limited
cache usage; it is not a cost receipt. Legacy and nano therefore require a
durable ``llm_gateway_attempts`` event joined by the exact backend request ID.
The default sample is a three-case synthetic prompt-only proxy. A real
producer-derived qualification uses exactly two already-completed JIT full
turns, one planned and one ambient, and never launches another full turn just
to populate this evidence. It stays within the unchanged 3/8/3/1 daily caps.
The fixture is prompt-only evidence until a receipt file is supplied and
validated.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import sqlite3
import stat
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Mapping, Sequence
DEFAULT_FIXTURE = (
Path(__file__).resolve().parents[1]
/ "testing"
/ "jit_processing"
/ "fixtures"
/ "jit_architecture_quality_cost_v2.json"
)
DEFAULT_CASE_IDS = ("actionable_deadline", "ambiguous_context", "already_visible")
CAPS = {
"notifications_per_day": 3,
"nano_triage_per_day": 8,
"full_turns_per_day": 3,
"full_turns_per_candidate": 1,
}
BUDGET_CAP_MICRO_USD = 5_000_000
JIT_FULL_RESERVATION_MICRO_USD = 50_000
# These are the ceilings enforced by the qualification budget and the
# OpenAI-compatible gateway request guard. The gateway intentionally counts
# serialized UTF-8 bytes as a conservative, tokenizer-independent input-token
# upper bound (see ``_apply_jit_request_budget``), so this driver measures the
# same representation before any network call.
JIT_MAX_INPUT_ENVELOPE_BYTES = 32_768
JIT_MAX_OUTPUT_TOKENS = 2_048
JIT_RUNTIME_GUARD_SOURCE = (
"backend/llm_gateway/gateway/jit_budget.py:24-25; " "backend/llm_gateway/routers/openai_compatible.py:823-842"
)
MAX_TOOL_MANIFEST_BYTES = 256 * 1024
RECEIPT_SCHEMA_VERSION = "omi.jit.cost_evidence.receipts.v1"
QA_OWNER_UID = "vi7SA9ckQCe4ccobWNxlbdcNdC23"
# AgentRuntimeProcess.defaultStateDirectory() scopes state by the QA bundle
# identifier. Keep this distinct from the QA Firestore database ID (jit-qa).
QA_BUNDLE_IDENTIFIER = "com.omi.omi-jit-qa"
QA_STATE_DIR_NAME = QA_BUNDLE_IDENTIFIER
AGENT_DATABASE_FILENAME = "omi-agentd.sqlite3"
QA_STATE_PATH_SUFFIX = Path("Application Support") / "Omi" / "AgentRuntime" / QA_BUNDLE_IDENTIFIER
MAX_AGENT_TOOL_ROUNDS = 500
MAX_FIRESTORE_REQUEST_IDS = 30
MAX_JIT_GATEWAY_ATTEMPTS = 500
PRODUCER_LANES = ("planned", "ambient")
MAX_PRODUCER_RUNS = len(PRODUCER_LANES)
SOURCE_PROJECTION_SCHEMA_VERSION = "omi.jit.proactivity.source_projection.v1"
# New producer runs persist the source-owned projection as a dedicated run
# input field. The metadata spelling is retained only for explicitly opted-in
# reads of old private QA records during this migration.
SOURCE_PROJECTION_RUN_INPUT_KEY = "jitCostEvidenceProjection"
SOURCE_PROJECTION_LEGACY_METADATA_KEY = "jitCostEvidenceProjection"
SOURCE_PROJECTION_METADATA_KEY = SOURCE_PROJECTION_LEGACY_METADATA_KEY
NANO_BILLING_SCHEMA_VERSION = "omi.jit.proactivity.nano_billing.v1"
# Only these AccountingEvent fields cross the evidence boundary. In
# particular, a broad Firestore export may contain user identifiers or other
# metadata; the comparison needs attribution and pricing fields, never
# prompts or account content.
ACCOUNTING_RECEIPT_FIELDS = (
"attempt_id",
"request_id",
"api_surface",
"invocation_id",
"provider",
"configured_model",
"actual_model_version",
"usage_status",
"uncached_input_tokens",
"cached_input_tokens",
"cache_write_tokens",
"cache_write_ttl",
"output_tokens",
"reasoning_tokens",
"cache_status",
"cost_status",
"estimated_cost_micro_usd",
"rate_card_id",
"cost_basis",
"retry_ordinal",
)
SIDECAR_FIELDS = (
"run_id",
"gateway_run_id",
"agent_run_id",
"agent_request_id",
"attempt_ids",
"request_id",
"case_id",
"architecture",
"stage",
"gateway_lane",
"producer_lane",
"evidence_sha256",
"prompt_sha256",
"uncached_prompt_sha256",
"system_prompt_sha256",
"tool_rounds",
"tool_invocations",
"receipt_origin",
)
class EvidenceError(ValueError):
"""A receipt or fixture cannot support a cost comparison."""
@dataclass(frozen=True)
class Route:
architecture: str
stage: str
gateway_lane: str
provider: str
served_model: str
rate_card_id: str
prompt_hash_key: str
system_prompt_hash_key: str | None = None
def _sha256(value: str) -> str:
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def _load_json(path: Path) -> Mapping[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise EvidenceError(f"cannot read JSON fixture {path}: {exc}") from exc
if not isinstance(value, Mapping):
raise EvidenceError(f"JSON root must be an object: {path}")
return value
def _fixture_routes(fixture: Mapping[str, Any]) -> dict[tuple[str, str], Route]:
try:
raw = fixture["billing_receipt_contract"]["runtime_route_contract"]
except (KeyError, TypeError) as exc:
raise EvidenceError("fixture has no runtime route contract") from exc
try:
return {
("legacy", "full"): Route(
architecture="legacy",
stage="full",
gateway_lane=raw["legacy_director"]["gateway_lane"],
provider=raw["legacy_director"]["provider"],
served_model=raw["legacy_director"]["model"],
rate_card_id=raw["legacy_director"]["rate_card_id"],
prompt_hash_key="prompt_sha256",
),
("jit", "nano"): Route(
architecture="jit",
stage="nano",
gateway_lane=raw["jit_nano"]["gateway_lane"],
provider=raw["jit_nano"]["provider"],
served_model=raw["jit_nano"]["model"],
rate_card_id=raw["jit_nano"]["rate_card_id"],
prompt_hash_key="nano_prompt_sha256",
),
("jit", "full"): Route(
architecture="jit",
stage="full",
gateway_lane=raw["jit_full"]["gateway_lane"],
provider=raw["jit_full"]["provider"],
served_model=raw["jit_full"]["model"],
rate_card_id=raw["jit_full"]["rate_card_id"],
prompt_hash_key="full_prompt_sha256",
system_prompt_hash_key="full_system_prompt_sha256",
),
}
except (KeyError, TypeError) as exc:
raise EvidenceError(f"fixture route contract is incomplete: {exc}") from exc
def _case_map(fixture: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]:
cases = fixture.get("cases")
if not isinstance(cases, list):
raise EvidenceError("fixture cases must be a list")
result: dict[str, Mapping[str, Any]] = {}
for case in cases:
if not isinstance(case, Mapping) or not isinstance(case.get("case_id"), str):
raise EvidenceError("fixture contains a malformed case")
result[case["case_id"]] = case
return result
def _evidence_hash(case: Mapping[str, Any]) -> str:
evidence = case.get("shared_evidence")
if not isinstance(evidence, Mapping):
raise EvidenceError(f"case {case.get('case_id')} has no shared evidence")
canonical = json.dumps(evidence, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
return _sha256(canonical)
def _matched_inputs(case: Mapping[str, Any]) -> dict[str, str]:
evidence = case["shared_evidence"]
legacy = case["prompt_inputs"]["legacy_probe_projection"]
jit = case["prompt_inputs"]["jit_projection"]
now = evidence.get("now")
timezone = evidence.get("timezone")
captured_at = legacy.get("captured_at")
context_id = jit.get("context_id")
if not all(isinstance(value, str) and value for value in (now, timezone, captured_at, context_id)):
raise EvidenceError(f"case {case['case_id']} lacks evaluation time, timezone, or context ID")
if now != captured_at:
raise EvidenceError(f"case {case['case_id']} has mismatched evaluation and captured times")
if context_id != legacy.get("bucket_id"):
raise EvidenceError(f"case {case['case_id']} has mismatched legacy/JIT context IDs")
return {
"evaluation_time": now,
"timezone": timezone,
"context_id": context_id,
"evidence_sha256": _evidence_hash(case),
}
def _expected_prompt(case: Mapping[str, Any], route: Route) -> dict[str, str]:
prompts = case["prompts"]
if route.architecture == "legacy":
legacy = prompts["legacy"]
if not isinstance(legacy.get("prompt_sha256"), str) or not isinstance(
legacy.get("uncached_prompt_sha256"), str
):
raise EvidenceError(f"case {case['case_id']} lacks the legacy prompt hashes")
return {
"prompt_sha256": legacy["prompt_sha256"],
"uncached_prompt_sha256": legacy["uncached_prompt_sha256"],
}
jit = prompts["jit"]
expected = {"prompt_sha256": jit[route.prompt_hash_key]}
if route.system_prompt_hash_key:
expected["system_prompt_sha256"] = jit[route.system_prompt_hash_key]
return expected
def _materialized_prompts(case: Mapping[str, Any], architecture: str, stage: str) -> dict[str, str]:
"""Return source-materialized prompt strings without exposing them in output."""
prompts = case.get("prompts")
if not isinstance(prompts, Mapping):
raise EvidenceError(f"case {case.get('case_id')} has no prompt materialization")
if architecture == "legacy":
source = prompts.get("legacy")
fields = {
"prompt": "materialized_prompt",
"uncached_prompt": "materialized_uncached_prompt",
}
elif stage == "nano":
source = prompts.get("jit")
fields = {"prompt": "materialized_nano_prompt"}
elif stage == "full":
source = prompts.get("jit")
fields = {
"prompt": "materialized_full_prompt",
"system_prompt": "materialized_full_system_prompt",
}
else:
raise EvidenceError(f"unsupported materialized route: {architecture}/{stage}")
if not isinstance(source, Mapping):
raise EvidenceError(f"case {case.get('case_id')} has no {architecture}/{stage} prompt materialization")
result: dict[str, str] = {}
for output_key, source_key in fields.items():
value = source.get(source_key)
if not isinstance(value, str):
raise EvidenceError(f"case {case.get('case_id')} lacks {source_key}")
result[output_key] = value
return result
def _validate_materialized_prompts(
case: Mapping[str, Any], routes: Mapping[tuple[str, str], Route], architecture: str, stage: str
) -> dict[str, str]:
"""Check the fixture's claimed hashes against its actual UTF-8 prompt bytes."""
route = routes[(architecture, stage)]
materialized = _materialized_prompts(case, architecture, stage)
expected = _expected_prompt(case, route)
hash_sources = {
"prompt_sha256": "prompt",
"uncached_prompt_sha256": "uncached_prompt",
"system_prompt_sha256": "system_prompt",
}
for hash_key, value in expected.items():
source_key = hash_sources.get(hash_key)
if source_key is None or source_key not in materialized:
raise EvidenceError(f"case {case['case_id']} has no materialized value for {hash_key}")
if _sha256(materialized[source_key]) != value:
raise EvidenceError(f"case {case['case_id']} {source_key} hash does not match materialized UTF-8 bytes")
return materialized
def _validate_fixture(fixture: Mapping[str, Any]) -> None:
if fixture.get("schema_version") != "jit_architecture_quality_cost.v2":
raise EvidenceError("driver requires the v2 matched-input fixture")
if fixture.get("prompt_replay_scope", {}).get("status") != "prompt_only_proxy":
raise EvidenceError("fixture scope must remain explicitly prompt_only_proxy")
if fixture.get("prompt_replay_scope", {}).get("provider_calls_executed") != 0:
raise EvidenceError("fixture claims provider calls; refusing to treat it as a zero-call proxy")
if fixture.get("execution_contract", {}).get("hard_caps") != CAPS:
raise EvidenceError("fixture caps changed; refresh the operator contract before running")
if fixture.get("execution_contract", {}).get("operational_cost_cap_usd") != 5.0:
raise EvidenceError("fixture cost cap changed; refresh the operator contract before running")
_fixture_routes(fixture)
_case_map(fixture)
def build_plan(fixture: Mapping[str, Any], case_ids: Sequence[str]) -> dict[str, Any]:
"""Return a no-call execution plan for matched synthetic inputs."""
_validate_fixture(fixture)
cases = _case_map(fixture)
routes = _fixture_routes(fixture)
if not case_ids:
raise EvidenceError("at least one case is required")
if len(set(case_ids)) != len(case_ids):
raise EvidenceError("case IDs must be unique")
if len(case_ids) > CAPS["notifications_per_day"]:
raise EvidenceError("selected sample exceeds the unchanged notification cap")
planned_cases: list[dict[str, Any]] = []
for case_id in case_ids:
case = cases.get(case_id)
if case is None:
raise EvidenceError(f"unknown fixture case: {case_id}")
if case.get("comparability", {}).get("status") == "blocked_context_gap":
raise EvidenceError(f"case {case_id} has a blocked context projection")
matched = _matched_inputs(case)
legacy_route = routes[("legacy", "full")]
nano_route = routes[("jit", "nano")]
full_route = routes[("jit", "full")]
planned_cases.append(
{
"case_id": case_id,
"category": case.get("category"),
"matched_input": matched,
"legacy": {
"route": legacy_route.__dict__,
"prompt_hashes": _expected_prompt(case, legacy_route),
"operation_count_exact": 1,
"gateway_attempts": "all durable attempt rows; retries are not capped by this operation count",
},
"jit": {
"nano": {
"route": nano_route.__dict__,
"prompt_hashes": _expected_prompt(case, nano_route),
"operation_count_exact": 1,
"gateway_attempts": "all durable attempt rows; retries are not capped by this operation count",
},
"full": {
"route": full_route.__dict__,
"prompt_hashes": _expected_prompt(case, full_route),
"full_turns_max": 1,
"gateway_attempts": "all producer receipt attempt IDs",
},
},
}
)
return {
"schema_version": "omi.jit.cost_evidence.plan.v1",
"status": "matched_input_plan",
"evidence_scope": "prompt_only_proxy; no provider calls",
"fixture_schema_version": fixture["schema_version"],
"same_synthetic_context_per_case": True,
"same_evaluation_time_and_timezone_per_case": True,
"caps": CAPS,
"budget_cap_micro_usd": BUDGET_CAP_MICRO_USD,
"jit_full_reservation_bound_micro_usd": JIT_FULL_RESERVATION_MICRO_USD,
"minimum_runtime_sample": {
"matched_cases": len(planned_cases),
"legacy_operations_exact": len(planned_cases),
"jit_nano_operations_exact": len(planned_cases),
"jit_full_turns_max": len(planned_cases),
"maximum_reserved_jit_full_usd": len(planned_cases) * JIT_FULL_RESERVATION_MICRO_USD / 1_000_000,
"quality_judgment": "root-owned after trusted receipts and adjudication",
},
"cases": planned_cases,
"receipt_contract": {
# Legacy and JIT nano proactivity return envelopes whose durable
# source is the backend AccountingEvent. JIT full uses the
# separate, prompt-free durable receipt below. Do not make the
# harness pretend that case IDs or prompt hashes are provider
# fields.
"legacy_accounting_event_fields": [
"attempt_id",
"invocation_id",
"request_id",
"api_surface",
"provider",
"configured_model",
"actual_model_version",
"outcome",
"usage_status",
"prompt_tokens",
"uncached_input_tokens",
"cached_input_tokens",
"cache_write_tokens",
"output_tokens",
"reasoning_tokens",
"cache_write_ttl",
"cache_status",
"cost_status",
"estimated_cost_micro_usd",
"rate_card_id",
"cost_basis",
],
"legacy_response_observation": {
"endpoint": "POST /v1/desktop/proactivity/completions",
"exposes": [
"operation",
"lane",
"provider_model",
"usage.cached_tokens",
"usage.cache_write_tokens",
"cache_write",
"fallback_class",
],
"does_not_expose": [
"attempt_id",
"request_id",
"invocation_id",
"actual_model_version",
"rate_card_id",
"cost_status",
"estimated_cost_micro_usd",
],
"consequence": (
"The endpoint response alone cannot support a trusted cost comparison. "
"Join the durable llm_gateway_attempts event by the exact backend request_id; "
"if that join is unavailable, leave legacy/nano cost unknown and stop."
),
},
"legacy_receipt_source": (
"durable backend llm_gateway_attempts AccountingEvent, joined by exact request_id; "
"the endpoint response is metadata only"
),
"jit_receipt_source": (
"jit-gateway-receipt-v1 rebuilt from durable llm_gateway_attempts by exact jit_run_id, "
"joined to its content-free harness sidecar"
),
"jit_gateway_attempt_fields": [
"attempt_id",
"provider",
"configured_model",
"actual_model_version",
"rate_card_id",
"cost_basis",
"usage_status",
"cost_status",
"normalized_uncached_input_tokens",
"cached_input_tokens",
"cache_write_tokens",
"output_tokens",
"reasoning_tokens",
"estimated_cost_micro_usd",
],
"jit_gateway_aggregate_fields": [
"attempt_count",
"normalized_uncached_input_tokens",
"cached_input_tokens",
"cache_write_tokens",
"output_tokens",
"estimated_cost_micro_usd",
"cost_status",
],
"harness_sidecar_fields": [
"run_id",
"gateway_run_id",
"agent_run_id",
"agent_request_id",
"attempt_ids",
"case_id",
"architecture",
"stage",
"gateway_lane",
"producer_lane",
"evidence_sha256",
"prompt_sha256",
"tool_rounds",
"tool_invocations",
],
"unknown_policy": "missing, malformed, aggregate-only, or zero-placeholder cost/counters block; never infer zero",
"counting": (
"sum every trusted provider-completion attempt; gateway_attempts is the number of distinct attempt IDs. "
"The SQLite tool ledger is a provider/tool invocation count, not a model round count; only an explicit "
"tool_rounds sidecar value may be reported as rounds. Cache units are cached plus cache-write token "
"units from the receipt."
),
"join": "sidecar.attempt_ids covers provider attempt IDs; run_id is sidecar-owned and stable for all operations in one case; gateway_run_id is the JIT budget execution ID when it differs",
"legacy_receipts_key": "legacy_provider_receipts",
"jit_nano_receipts_key": "jit_nano_provider_receipts",
"actual_jit_nano_receipts_key": "actual_jit_nano_provider_receipts",
"actual_jit_nano_receipt_origin": (
"producer nano_billing.request_id joined to durable llm_gateway_attempts; replay nano is excluded from actual architecture cost"
),
"jit_receipts_key": "jit_gateway_receipts",
},
}
def _json_bytes(value: Any) -> bytes:
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
def _sha256_bytes(value: bytes) -> str:
return hashlib.sha256(value).hexdigest()
def _notification_schema(*, allow_lookup: bool = False) -> dict[str, Any]:
"""Mirror ContextProactivityEngine.schema without importing Swift code."""
properties: dict[str, Any] = {
"decision": {
"type": "string",
"enum": ["suggest", "insight", "task_candidate", "resurface", "silence"],
},
"title": {"type": "string", "description": "The specific thing this is about."},
"message": {"type": "string", "description": "What the user should know or do."},
"reasoning": {"type": "string"},
"bucket_entry_refs": {"type": "array", "items": {"type": "string"}},
"fact_ids": {"type": "array", "items": {"type": "string"}},
"task_refs": {"type": "array", "items": {"type": "string"}},
}
required = ["decision", "title", "message", "reasoning", "bucket_entry_refs", "fact_ids", "task_refs"]
if allow_lookup:
properties["lookup_query"] = {"type": "string"}
required.append("lookup_query")
return {
"type": "object",
"properties": properties,
"required": required,
"additionalProperties": False,
}
def _nano_schema() -> dict[str, Any]:
return {
"type": "object",
"properties": {"approved": {"type": "boolean"}},
"required": ["approved"],
"additionalProperties": False,
}
def _materialized_descriptor(materialized: Mapping[str, str], key: str) -> dict[str, Any]:
value = materialized[key]
encoded = value.encode("utf-8")
return {"sha256": _sha256_bytes(encoded), "utf8_bytes": len(encoded)}
def _proactive_request_payload(
*, operation: str, prompt: str, uncached_prompt: str | None, max_completion_tokens: int, cache_key: str | None
) -> dict[str, Any]:
content: list[dict[str, str]] = [{"type": "text", "text": prompt}]
if uncached_prompt:
content.append({"type": "text", "text": uncached_prompt})
payload: dict[str, Any] = {
"operation": operation,
"messages": [{"role": "user", "content": content}],
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "desktop_proactivity",
"strict": True,
"schema": _nano_schema() if operation == "proactive_extraction" else _notification_schema(),
},
},
"max_completion_tokens": max_completion_tokens,
}
if cache_key:
payload["cache_key"] = cache_key
return payload
def _proactive_gateway_payload(client_payload: Mapping[str, Any], *, gateway_lane: str) -> dict[str, Any]:
"""Apply the current server-side desktop-proactivity payload projection.
The input endpoint body is not what ``_apply_jit_request_budget`` measures:
the router adds the lane, metadata, and (for the legacy cache key) an
explicit breakpoint before forwarding the body to the gateway. Measuring
this projection avoids a false fit caused by omitting those fields.
"""
operation = client_payload.get("operation")
if not isinstance(operation, str):
raise EvidenceError("proactivity payload has no operation")
payload = {key: value for key, value in client_payload.items() if key not in {"operation", "cache_key"}}
payload["model"] = gateway_lane
payload["metadata"] = {
"omi_feature": f"desktop_{operation}",
"prompt_version": f"desktop_{operation}.v1",
"parser_version": "desktop_proactive_json.v1",
}
# The current backend raises the legacy reasoning request to its known
# recovery floor. This is a small body-size detail, but keeping it here
# means the no-call measurement matches the forwarded request.
if operation == "proactive_reasoning":
payload["max_completion_tokens"] = max(int(payload["max_completion_tokens"]), 2_400)
cache_key = client_payload.get("cache_key")
# ``has_cacheable_prefix`` in the backend only emits these cache fields
# when the first stable text is at least 1,024 tokens (the production
# heuristic is four characters per token). Keep the projection exact so
# the dry-run does not claim cache accounting for a prefix the server
# would leave unmarked.
stable_prefix = ""
messages = payload.get("messages")
if isinstance(messages, list):
for message in reversed(messages):
if not isinstance(message, Mapping):
continue
content = message.get("content")
if isinstance(content, list):
first = next((part for part in content if isinstance(part, Mapping)), None)
if isinstance(first, Mapping) and first.get("type") == "text" and isinstance(first.get("text"), str):
stable_prefix = first["text"]
break
if isinstance(content, str):
stable_prefix = content
break
if isinstance(cache_key, str) and cache_key and len(stable_prefix) >= 4_096:
payload["prompt_cache_key"] = cache_key
payload["prompt_cache_options"] = {"mode": "explicit", "ttl": "30m"}
if isinstance(messages, list) and messages:
copied_messages = [dict(message) for message in messages if isinstance(message, Mapping)]
if copied_messages and isinstance(copied_messages[0].get("content"), list):
parts = list(copied_messages[0]["content"])
marker = {"type": "text", "text": "", "prompt_cache_breakpoint": {"mode": "explicit"}}
if not any(isinstance(part, Mapping) and part.get("prompt_cache_breakpoint") for part in parts):
parts.insert(1, marker)
copied_messages[0]["content"] = parts
payload["messages"] = copied_messages
return payload
def _openai_tool_payload(tools: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
"""Convert the source MCP definitions to the OpenAI tool wire shape."""
result: list[dict[str, Any]] = []
for tool in tools:
result.append(
{
"type": "function",
"function": {
"name": tool["name"],
"description": tool["description"],
"parameters": tool["inputSchema"],
},
}
)
return result
def _load_tool_manifest(path: Path) -> tuple[list[Mapping[str, Any]], dict[str, Any]]:
try:
raw = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise EvidenceError(f"cannot read tool manifest {path}: {exc}") from exc
metadata: dict[str, Any] = {}
if isinstance(raw, Mapping):
candidate = raw.get("tools")
for key in ("adapter_id", "manifest_version", "manifest_digest", "context"):
if key in raw:
metadata[key] = raw[key]
else:
candidate = raw
if not isinstance(candidate, list) or not candidate:
raise EvidenceError("tool manifest must be a non-empty JSON list or an object with a tools list")
if len(_json_bytes(candidate)) > MAX_TOOL_MANIFEST_BYTES:
raise EvidenceError("tool manifest exceeds the bounded preflight size")
names: set[str] = set()
normalized: list[Mapping[str, Any]] = []
for tool in candidate:
if not isinstance(tool, Mapping):
raise EvidenceError("tool manifest contains a malformed entry")
name = tool.get("name")
description = tool.get("description")
input_schema = tool.get("inputSchema")
if not isinstance(name, str) or not name or name in names:
raise EvidenceError("tool manifest contains a missing or duplicate name")
if not isinstance(description, str) or not description:
raise EvidenceError(f"tool manifest entry {name} lacks a description")
if not isinstance(input_schema, Mapping):
raise EvidenceError(f"tool manifest entry {name} lacks inputSchema")
names.add(name)
normalized.append({"name": name, "description": description, "inputSchema": input_schema})
metadata.update(
{
"tool_count": len(normalized),
"manifest_utf8_bytes": len(_json_bytes(normalized)),
"manifest_sha256": _sha256_bytes(_json_bytes(normalized)),
}
)
return normalized, metadata
def _body_summary(body: Mapping[str, Any]) -> dict[str, Any]:
encoded = _json_bytes(body)
size = len(encoded)
return {
"request_body_sha256": _sha256_bytes(encoded),
"request_utf8_bytes": size,
"input_envelope_limit_bytes": JIT_MAX_INPUT_ENVELOPE_BYTES,
"fits_input_envelope": size <= JIT_MAX_INPUT_ENVELOPE_BYTES,
}
def preflight_payloads(
fixture: Mapping[str, Any],
case_ids: Sequence[str],
*,
tool_manifest: Sequence[Mapping[str, Any]] | None = None,
tool_manifest_metadata: Mapping[str, Any] | None = None,
kernel_system_prompt: str | None = None,
) -> dict[str, Any]:
"""Measure source-derived request envelopes without making any provider call.
Legacy and nano use the exact Swift HTTP body shape. The full path uses the
actual ``omi-sonnet`` OpenAI-compatible body shape, including the source
MCP tool definitions. The full summary includes the kernel policy when the
operator supplies the built runtime's policy artifact; without it the
smaller result is explicitly labelled minimum-only and remains blocked.
"""
plan = build_plan(fixture, case_ids)
routes = _fixture_routes(fixture)
tool_metadata = dict(tool_manifest_metadata or {})
if tool_manifest is not None:
tools = list(tool_manifest)
if not tools:
raise EvidenceError("tool manifest must be non-empty")
# Re-run the same structural checks for callers that already loaded a
# manifest from a test or an API rather than a file.
names: set[str] = set()
for tool in tools:
if not isinstance(tool, Mapping):
raise EvidenceError("tool manifest contains a malformed entry")
if (
not isinstance(tool.get("name"), str)
or not tool["name"]
or tool["name"] in names
or not isinstance(tool.get("description"), str)
or not isinstance(tool.get("inputSchema"), Mapping)
):
raise EvidenceError("tool manifest contains an invalid entry")
names.add(tool["name"])
openai_tools = _openai_tool_payload(tools)
tool_metadata.setdefault("tool_count", len(tools))
tool_metadata.setdefault("manifest_utf8_bytes", len(_json_bytes(tools)))
tool_metadata.setdefault("manifest_sha256", _sha256_bytes(_json_bytes(tools)))
else:
openai_tools = []
case_results: list[dict[str, Any]] = []
blocking_reasons: list[str] = []
for planned in plan["cases"]:
case_id = planned["case_id"]
case = _case_map(fixture)[case_id]
legacy_materialized = _validate_materialized_prompts(case, routes, "legacy", "full")
nano_materialized = _validate_materialized_prompts(case, routes, "jit", "nano")
full_materialized = _validate_materialized_prompts(case, routes, "jit", "full")
legacy_body = _proactive_request_payload(
operation="proactive_reasoning",
prompt=legacy_materialized["prompt"],
uncached_prompt=legacy_materialized["uncached_prompt"],
max_completion_tokens=800,
cache_key="director:v1",
)
nano_body = _proactive_request_payload(
operation="proactive_extraction",
prompt=nano_materialized["prompt"],
uncached_prompt=None,
max_completion_tokens=120,
cache_key=None,
)
legacy_summary = {
"endpoint": "POST /v1/desktop/proactivity/completions",
"gateway_lane": routes[("legacy", "full")].gateway_lane,
"provider": routes[("legacy", "full")].provider,
"served_model": routes[("legacy", "full")].served_model,
"prompt": _materialized_descriptor(legacy_materialized, "prompt"),
"uncached_prompt": _materialized_descriptor(legacy_materialized, "uncached_prompt"),
**_body_summary(
_proactive_gateway_payload(
legacy_body,
gateway_lane=routes[("legacy", "full")].gateway_lane,
)
),
}
nano_summary = {
"endpoint": "POST /v1/desktop/proactivity/completions",
"gateway_lane": routes[("jit", "nano")].gateway_lane,
"provider": routes[("jit", "nano")].provider,
"served_model": routes[("jit", "nano")].served_model,
"prompt": _materialized_descriptor(nano_materialized, "prompt"),
**_body_summary(
_proactive_gateway_payload(
nano_body,
gateway_lane=routes[("jit", "nano")].gateway_lane,
)
),
}
system_parts = [full_materialized["system_prompt"]]
if kernel_system_prompt is not None:
system_parts.insert(0, kernel_system_prompt)
full_body = {
"model": "omi-sonnet",
"messages": [
{"role": "system", "content": "\n".join(system_parts)},
{"role": "user", "content": full_materialized["prompt"]},
],
"tools": openai_tools,
"max_tokens": JIT_MAX_OUTPUT_TOKENS,
"stream": True,
"stream_options": {"include_usage": True},
}
full_summary: dict[str, Any] = {
"endpoint": "POST /v2/chat/completions",
"requested_model": "omi-sonnet",
"gateway_lane": routes[("jit", "full")].gateway_lane,
"provider": routes[("jit", "full")].provider,
"served_model": routes[("jit", "full")].served_model,
"prompt": _materialized_descriptor(full_materialized, "prompt"),
"jit_system_prompt": _materialized_descriptor(full_materialized, "system_prompt"),
"tool_manifest": tool_metadata or None,
"kernel_system_prompt_supplied": kernel_system_prompt is not None,
**_body_summary(full_body),
}
case_result = {
"case_id": case_id,
"legacy": legacy_summary,
"jit_nano": nano_summary,
"jit_full": full_summary,
}
case_results.append(case_result)
if not legacy_summary["fits_input_envelope"]:
blocking_reasons.append(f"{case_id} legacy request exceeds the 32768-byte JIT envelope")
if not nano_summary["fits_input_envelope"]:
blocking_reasons.append(f"{case_id} nano request exceeds the 32768-byte JIT envelope")
if tool_manifest is None:
blocking_reasons.append("full JIT preflight requires the source-generated MCP tool manifest")
if kernel_system_prompt is None:
blocking_reasons.append("full JIT preflight requires the built kernel system-policy artifact")
if not full_summary["fits_input_envelope"]:
blocking_reasons.append(f"{case_id} full request exceeds the 32768-byte JIT envelope")
if not tool_manifest:
tool_metadata = None
return {
"schema_version": "omi.jit.cost_evidence.preflight.v1",
"status": "blocked" if blocking_reasons else "ready_for_runtime",
"evidence_scope": (
"no-call serialized-envelope preflight; proves source/hash/size bounds only, "
"not provider quality or architecture cost"
),
"fixture_schema_version": fixture["schema_version"],
"runtime_guard_source": JIT_RUNTIME_GUARD_SOURCE,
"input_envelope": {
"encoding": "UTF-8",
"limit_bytes": JIT_MAX_INPUT_ENVELOPE_BYTES,
"output_token_cap": JIT_MAX_OUTPUT_TOKENS,
"modality": "text-only; image/audio calls remain rejected by the runtime guard",
},
"tool_manifest": tool_metadata,
"kernel_system_prompt": {
"supplied": kernel_system_prompt is not None,
"utf8_bytes": len(kernel_system_prompt.encode("utf-8")) if kernel_system_prompt is not None else None,
"sha256": _sha256(kernel_system_prompt) if kernel_system_prompt is not None else None,
},
"blocking_reasons": sorted(set(blocking_reasons)),
"cases": case_results,
"operator_recipe": [
"npm --prefix desktop/macos/agent run build --silent",
"Generate the service/coordinator omi-tools-stdio manifest from the built runtime with jitKnowledgeToolsEnabled=true and jitProactivity=true; assert the exact four read-only JIT tools and write JSON only to a temporary artifact.",
"Generate kernelSystemPolicy(\"service\", \"coordinator\") from the same built runtime; write UTF-8 text only to a temporary artifact.",
"Run jit_cost_evidence_driver.py --plan --plan-file <plan.json> and then --preflight --tool-manifest <manifest.json> --kernel-system-prompt <policy.txt> --plan-file <plan.json>.",
"After parent approval, capture X-Omi-Request-ID for every legacy and nano response, join exact IDs to exported llm_gateway_attempts with --join-receipts, then validate the joined envelope; archive only content-free receipts and sidecars.",
],
}
def _required_int(receipt: Mapping[str, Any], key: str, *, minimum: int = 0) -> int:
value = receipt.get(key)
if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
raise EvidenceError(f"receipt {receipt.get('case_id', '?')} has unknown or invalid {key}")
return value
def _required_string(value: Any, label: str) -> str:
if not isinstance(value, str) or not value.strip():
raise EvidenceError(f"{label} is missing or empty")
return value
def _content_free_accounting_receipt(row: Mapping[str, Any]) -> dict[str, Any]:
"""Keep only the prompt-free fields needed by ``summarize_receipts``."""
return {key: row[key] for key in ACCOUNTING_RECEIPT_FIELDS if key in row}
def _content_free_sidecar(sidecar: Mapping[str, Any]) -> dict[str, Any]:
"""Keep only the opaque IDs and hashes needed to join a receipt."""
return {key: sidecar[key] for key in SIDECAR_FIELDS if key in sidecar}
def _content_free_jit_receipt(receipt: Mapping[str, Any]) -> dict[str, Any]:
"""Copy the JIT receipt envelope without accepting arbitrary payload data."""
allowed = ("schema_version", "run_id", "contract_version", "attempts", "aggregate")
result = {key: receipt[key] for key in allowed if key in receipt}
attempts = receipt.get("attempts")
if isinstance(attempts, list):
attempt_fields = (
"attempt_id",
"provider",
"configured_model",
"actual_model_version",
"rate_card_id",
"cost_basis",
"usage_status",
"cost_status",
"normalized_uncached_input_tokens",
"cached_input_tokens",
"cache_write_tokens",
"output_tokens",
"reasoning_tokens",
"estimated_cost_micro_usd",
)
result["attempts"] = [
{key: item[key] for key in attempt_fields if key in item} for item in attempts if isinstance(item, Mapping)
]
aggregate = receipt.get("aggregate")
if isinstance(aggregate, Mapping):
aggregate_fields = (
"attempt_count",
"normalized_uncached_input_tokens",
"cached_input_tokens",
"cache_write_tokens",
"output_tokens",
"reasoning_tokens",
"estimated_cost_micro_usd",
"cost_status",
)
result["aggregate"] = {key: aggregate[key] for key in aggregate_fields if key in aggregate}
return result
def _canonical_json_hash(value: Any) -> str:
"""Hash canonical JSON without returning the JSON material."""
return hashlib.sha256(_json_bytes(value)).hexdigest()
def _required_identifier(value: Any, label: str) -> str:
if not isinstance(value, str) or not value.strip() or len(value) > 256:
raise EvidenceError(f"{label} is missing or malformed")
return value.strip()
def _validate_nano_billing_observation(
raw: Any,
*,
owner_id: str,
producer_lane: str,
execution_id: str,
) -> dict[str, Any]:
"""Keep the producer's content-free nano observation for a durable join.
The desktop can identify the actual nano request, but it cannot price it.
Accounting rows joined by that exact request ID remain authoritative; this
projection deliberately carries no cost estimate.
"""