forked from BasedHardware/omi
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathscreen_frame.py
More file actions
248 lines (187 loc) · 8.82 KB
/
Copy pathscreen_frame.py
File metadata and controls
248 lines (187 loc) · 8.82 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
"""Wire types and internal models for meeting-note screenshot egress.
Mirror these exactly in Swift (Lane B) and TS (Lane C) — see
`data/reports/meeting-screenshots/DESIGN-sol.md` for rationale, and the
authoritative contract at the time this was written for the exact shapes.
The one property that must not be broken: the client never decides what may
be stored. It uploads candidate bytes; the server canonicalises them, judges
those exact bytes, mints an internal approval, and only a holder of that
approval may write to the screenshot bucket. `ScreenFrameApprovalClaims` is
never serialized into any response model — it does not leave the process.
"""
from __future__ import annotations
import unicodedata
from datetime import datetime
from enum import Enum
from typing import List, Literal, Optional
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator
class ScreenFrameEgressPurpose(str, Enum):
MEETING_NOTE_V1 = "meeting_note_v1"
class ScreenFrameRetentionClass(str, Enum):
WITH_SUBJECT = "with_subject"
# ---------------------------------------------------------------------------
# Request wire types
# ---------------------------------------------------------------------------
class ScreenFrameSubjectIn(BaseModel):
model_config = ConfigDict(extra="forbid")
kind: Literal["conversation"]
id: str = Field(min_length=1, max_length=256)
class ScreenFrameCandidateIn(BaseModel):
model_config = ConfigDict(extra="forbid")
client_frame_id: str = Field(min_length=1, max_length=128)
captured_at: datetime
mime_type: Literal["image/jpeg", "image/png"]
declared_width: int = Field(ge=1, le=10000)
declared_height: int = Field(ge=1, le=10000)
sha256_base64: str = Field(min_length=44, max_length=44) # transport check only
bytes_base64: str
class ScreenFrameAdjudicationRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
schema_version: Literal[1]
attempt_id: UUID
purpose: Literal["meeting_note_v1"]
subject: ScreenFrameSubjectIn
candidates: List[ScreenFrameCandidateIn] = Field(min_length=1, max_length=8)
class ScreenFrameSharingUpdateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
enabled: bool
class ScreenFrameSettings(BaseModel):
"""The account-level setting gating screen-frame egress admission
(contract §6, setting_key="meeting_note_screenshots_enabled"). Shared and
authoritative across every device — desktop, web, a reinstall — because
it protects the user from themselves (accidentally leaving the feature
on), not third parties from the user; the privacy judge is what protects
people appearing in frames. It does not need to be tamper-proof, only
consistent, so it is a plain user-profile field, not a signed claim.
"""
model_config = ConfigDict(extra="forbid")
meeting_note_screenshots_enabled: bool
class ScreenFrameSettingsUpdateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
meeting_note_screenshots_enabled: bool
# ---------------------------------------------------------------------------
# Stored / response wire types
# ---------------------------------------------------------------------------
class NormalizedRect(BaseModel):
model_config = ConfigDict(extra="forbid")
x: float
y: float
width: float
height: float
class ScreenFrameGround(BaseModel):
"""Gradient stops derived from the canonical bytes at approval time.
Both clients render the banner from these; neither samples pixels. A
signed cross-origin URL cannot be read back from a canvas (no guaranteed
CORS headers), and two independent extractions (Swift on macOS, JS on
web) would drift from each other anyway. Extracted once, server-side,
from the canonical JPEG — see utils/screen_frames/palette.py, a port of
desktop/macos/Desktop/Sources/MeetingScreenshots/MeetingBannerPalette.swift.
"""
model_config = ConfigDict(extra="forbid")
stops: List[str] = Field(min_length=2, max_length=2) # "#RRGGBB"
is_neutral: bool
class ConversationScreenFrame(BaseModel):
id: str
captured_at: datetime
role: Literal["banner", "strip"]
rank: int = Field(ge=0, le=6)
caption: str = Field(max_length=160)
labels: List[str] = Field(max_length=8)
source_badge: Optional[Literal["code", "browser", "document", "slides", "product"]] = None
focal_region: Optional[NormalizedRect] = None
width: int
height: int
content_url: str # signed, 60 min
thumbnail_url: str # signed, 60 min
url_expires_at: datetime
ground: ScreenFrameGround
class ConversationScreenFrameSet(BaseModel):
revision: int
banner: Optional[ConversationScreenFrame] = None
strip: List[ConversationScreenFrame] = Field(default_factory=list, max_length=6)
adjudicated_at: Optional[datetime] = None
"""When an adjudication pass last ran, whatever it decided.
A client must use this, not `revision`, to decide whether to offer candidates. `revision`
only moves when something was approved, so an all-rejected pass leaves it at 0 and reads as
"never attempted" — and the client then re-uploads the very frames the judge refused, on
every reopen. Those are the sensitive ones by definition.
"""
class ScreenFrameAdjudicationResponse(BaseModel):
attempt_id: UUID
outcome: Literal["committed", "no_approved_frames"]
frame_set: ConversationScreenFrameSet
# ---------------------------------------------------------------------------
# The judge's strict discriminated output (§4 steps 3-4 of the contract).
#
# There is deliberately NO separate decision+sensitivity pair: the measured
# judge produced contradictory sensitive+publish verdicts twice
# (FINDINGS.md:100-108). A single discriminated outcome cannot contradict
# itself the way two independent fields can.
# ---------------------------------------------------------------------------
_JOINERS = "\u200d\u200c\ufe0f\ufe0e"
class ScreenFrameJudgement(BaseModel):
model_config = ConfigDict(extra="forbid")
outcome: Literal["approved_clean", "rejected"]
reject_reason: Optional[
Literal[
"credentials",
"private_messages",
"email",
"banking",
"medical",
"identifiable_person",
"personal_document",
"unreadable",
"other",
]
] = None
# Deliberately NOT max_length/max_items constrained, unlike the wire model above.
# Vertex treats a responseSchema's maxLength as advisory, so gemini-2.5-flash-lite
# overruns it in normal use (measured 2026-08-25: one caption in seven came back
# over 160 chars) even though the prompt states the limit too. A strict constraint
# here does not shorten the caption — it raises inside .with_structured_output(),
# which judge_frame() turns into judge_call_failed, and the frame is dropped. That
# trades a cosmetic overrun for silently losing a frame the judge approved, with no
# safety benefit: caption and labels are descriptive metadata and carry no part of
# the verdict. So normalise instead, and let the wire model keep the real contract.
caption: str
labels: List[str]
source_badge: Optional[Literal["code", "browser", "document", "slides", "product"]] = None
banner_suitability: float = Field(ge=0, le=1)
@field_validator("caption", mode="before")
@classmethod
def _truncate_caption(cls, value: object) -> object:
if not isinstance(value, str) or len(value) <= 160:
return value
# A plain [:160] can cut inside a grapheme cluster and leave a dangling
# zero-width joiner or combining mark, which every client then renders as
# a broken glyph. Back off to the last codepoint that can end a string.
cut = value[:160]
while cut and (unicodedata.combining(cut[-1]) or cut[-1] in _JOINERS):
cut = cut[:-1]
return cut
@field_validator("labels", mode="before")
@classmethod
def _cap_labels(cls, value: object) -> object:
return value[:8] if isinstance(value, list) else value
# ---------------------------------------------------------------------------
# The internal approval object. This NEVER appears in a response model and is
# never sent to a client — see module docstring.
# ---------------------------------------------------------------------------
class ScreenFrameApprovalClaims(BaseModel):
iss: Literal["omi-screen-frame-adjudicator"]
aud: Literal["omi-screen-frame-writer"]
jti: UUID # one-use
uid: str
purpose: str
subject_kind: Literal["conversation"]
subject_id: str
canonical_sha256: str
model: str
policy_version: str
prompt_version: str
retention: str
decision: Literal["approved_clean"]
labels_digest: str
issued_at: datetime
expires_at: datetime # <= 10 minutes