forked from BasedHardware/omi
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathGeneratedToolCapabilities.swift
More file actions
669 lines (661 loc) · 28.6 KB
/
Copy pathGeneratedToolCapabilities.swift
File metadata and controls
669 lines (661 loc) · 28.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
// Generated by agent/scripts/generate-tool-surfaces.mjs — do not edit.
import Foundation
enum GeneratedToolCapabilities {
enum Surface: Hashable {
case desktopChat
case realtimeHub
case onboarding
case taskChat
}
enum LatencyClass: String {
case fastLocal = "fast local"
case fastNetwork = "fast network"
case asyncBackground = "async background"
}
struct Capability {
let toolName: String
let title: String
let latency: LatencyClass
let surfaces: Set<Surface>
let summary: String
let bullets: [String]
func supports(_ surface: Surface) -> Bool {
surfaces.contains(surface)
}
}
static let capabilities: [Capability] = [
Capability(
toolName: "execute_sql",
title: "Execute SQL",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Run SQL on the local omi.db database for structured local data.",
bullets: [
"Supports SELECT, INSERT, UPDATE, DELETE.",
"Use for personal facts, app usage stats, time queries, task lookups, conversations, memories, aggregations, and anything structured.",
"Supports FTS5 MATCH queries for keyword search; see the schema footer for FTS tables and patterns.",
"SELECT queries auto-limit to 200 rows. UPDATE/DELETE require WHERE. DROP/ALTER/CREATE are blocked.",
"Prefer semantic_search for fuzzy screen-history questions and backend task tools for creating/updating tasks.",
"Use execute_sql for quantitative queries (counts, sums, date ranges, aggregations).",
"Use semantic_search instead for fuzzy or conceptual queries about screen content."
]
),
Capability(
toolName: "semantic_search",
title: "Semantic Search",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Vector similarity search on the user's screen history.",
bullets: [
"Use for fuzzy/conceptual questions about what the user saw, read, or worked on where exact SQL keywords will not work.",
"Examples: \"reading about machine learning\", \"working on design mockups\".",
"Parameters: query (required), days (default 7), app_filter (optional).",
"Prefer semantic_search over execute_sql when the user asks about something they 'saw' or worked on."
]
),
Capability(
toolName: "search_screen_history",
title: "Search Screen History",
latency: .fastLocal,
surfaces: Set([.realtimeHub]),
summary: "Search the user's on-screen history by meaning.",
bullets: [
"Use for what the user saw, read, or worked on. Speak a short summary of the result."
]
),
Capability(
toolName: "get_daily_recap",
title: "Daily Recap",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Pre-formatted activity recap: apps, conversations, tasks, focus, memories, and observations.",
bullets: [
"Use for what the user did today/yesterday/this week; it is faster than composing many SQL queries.",
"Parameters: days_ago (0=today, 1=yesterday, 7=past week; default 1)."
]
),
Capability(
toolName: "fill_cloud_connector_form",
title: "Fill Cloud Connector Form",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Fill the visible ChatGPT or Claude custom MCP connector form using Omi's native macOS Accessibility automation.",
bullets: [
"Call this first for ChatGPT or Claude cloud MCP connector setup when the connector form is visible.",
"Do not install browser extensions before trying this tool.",
"If it reports missing Accessibility permission, missing form, or missing required fields, wait for the missing condition or use guarded screenshots before any keyboard automation."
]
),
Capability(
toolName: "list_agent_sessions",
title: "List Agent Sessions",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "List Omi-managed agent sessions from the local runtime kernel.",
bullets: [
"Use for current or recent kernel-backed Omi agents/subagents across chat, PTT/realtime, task chat, and floating-bar pills.",
"Returns task_agents and floating_agent_pills alongside canonical session summaries.",
"For a prior child agent's final answer, do not infer run completion from session status or restrict discovery to status='open'. List recent sessions, then call get_agent_run with the returned runId and answer from run.finalText without exposing the internal id."
]
),
Capability(
toolName: "get_agent_run",
title: "Get Agent Run",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Inspect one canonical Omi agent run.",
bullets: [
"Use a runId from list_agent_sessions or a correlated Omi result.",
"Returns the run, attempts, adapter bindings, events, and artifact metadata.",
"For a completed child, use run.finalText to answer the user and keep the internal runId out of the user-visible response."
]
),
Capability(
toolName: "build_desktop_awareness_snapshot",
title: "Build Desktop Awareness Snapshot",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Build a local coordinator snapshot from kernel sessions, runs, dispatches, deliveries, candidates, and runtime health.",
bullets: [
"Use before routing new local work or summarizing open agent loops.",
"Returns metadata and local state summaries, not raw transcripts or screenshot bytes."
]
),
Capability(
toolName: "list_desktop_action_queue",
title: "List Desktop Action Queue",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Return the derived Desktop action queue from runs, dispatches, deliveries, candidates, legacy projections, and overrides.",
bullets: [
"Use for approvals, failed runs, artifact review, stale work, and candidate review.",
"The queue is derived and not persisted as authority."
]
),
Capability(
toolName: "get_desktop_open_loops",
title: "Get Desktop Open Loops",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Summarize unresolved local coordinator loops: blocking dispatches, failed/stale runs, undelivered artifacts, and candidate reviews.",
bullets: [
"Use for quick status answers and voice status summaries."
]
),
Capability(
toolName: "build_desktop_context_packet",
title: "Build Desktop Context Packet",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Persist a minimized DesktopContextPacket plus context-access audit rows from explicit selected snippets.",
bullets: [
"Use selected snippets with provenance, not full transcripts or screenshot image bytes.",
"Requires a positive TTL and writes context-access audit rows."
]
),
Capability(
toolName: "route_desktop_intent",
title: "Route Desktop Intent",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Run deterministic local intent routing over action queue and reusable session candidates.",
bullets: [
"Use before creating a new run when existing local context may be relevant."
]
),
Capability(
toolName: "evaluate_desktop_tool_policy",
title: "Evaluate Desktop Tool Policy",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Evaluate local coordinator policy for a tool/capability request without executing the tool.",
bullets: [
"Use to explain why a sensitive local action needs dispatch or approval."
]
),
Capability(
toolName: "create_desktop_dispatch",
title: "Create Desktop Dispatch",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Create a durable local DesktopCoordinatorDispatch for approvals, routing choices, artifact review, candidates, or sensitive context.",
bullets: [
"Use when user attention or approval is required before crossing a boundary."
]
),
Capability(
toolName: "resolve_desktop_dispatch",
title: "Resolve Desktop Dispatch",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Resolve or cancel a pending local DesktopCoordinatorDispatch, optionally creating a scoped allow grant for an explicit approval.",
bullets: [
"Use only for explicit user approval/denial/cancel decisions."
]
),
Capability(
toolName: "cancel_agent_run",
title: "Cancel Agent Run",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Request cancellation for one canonical Omi agent run through the runtime kernel.",
bullets: [
"Use when the user asks to stop a running Omi agent/subagent.",
"Returns whether cancellation was accepted, dispatched, and acknowledged."
]
),
Capability(
toolName: "inspect_agent_artifacts",
title: "Inspect Agent Artifacts",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Inspect canonical artifact metadata for an Omi agent artifact, session, run, or attempt.",
bullets: [
"Returns artifact references and metadata only.",
"Use after get_agent_run when the user asks what files or outputs an agent produced."
]
),
Capability(
toolName: "read_tool_output",
title: "Read Tool Output",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Read a bounded excerpt from a canonical Omi tool-output artifact.",
bullets: [
"Requires a canonical artifact id and keeps provider payloads bounded.",
"Use an artifactId returned by a toolResultEnvelope fullOutputRef or inspect_agent_artifacts.",
"The response is bounded; use search_tool_output for targeted retrieval."
]
),
Capability(
toolName: "search_tool_output",
title: "Search Tool Output",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Search a canonical Omi tool-output artifact without returning the complete artifact.",
bullets: [
"Requires a canonical artifact id and returns bounded matching lines.",
"Use after a truncated toolResultEnvelope to find the relevant local output."
]
),
Capability(
toolName: "update_agent_artifact_lifecycle",
title: "Update Agent Artifact Lifecycle",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Update metadata-only lifecycle state for one canonical Omi agent artifact.",
bullets: [
"Use to mark artifact metadata as retained, dismissed, or opened after a user-visible artifact decision.",
"Pass sessionId, runId, or attemptId when available as a scope guard.",
"This never reads artifact contents and has no OS side effects."
]
),
Capability(
toolName: "send_agent_message",
title: "Send Agent Message",
latency: .asyncBackground,
surfaces: Set([.desktopChat]),
summary: "Send a follow-up message to an existing canonical Omi agent session.",
bullets: [
"Use when continuing a multi-turn conversation with an Omi-managed agent by sessionId.",
"Creates a new run in the existing session."
]
),
Capability(
toolName: "spawn_background_agent",
title: "Spawn Background Agent",
latency: .asyncBackground,
surfaces: Set([]),
summary: "Internal Swift coordinator entrypoint for creating canonical floating-bar runs.",
bullets: [
"Swift coordinator entrypoint only; not advertised to agent-facing surfaces.",
"Swift coordinator entrypoint only."
]
),
Capability(
toolName: "spawn_agent",
title: "Spawn Agent",
latency: .asyncBackground,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Start canonical Omi background work and optionally project it into floating-bar pills.",
bullets: [
"Creates a canonical kernel session/run; visible runs project into floating-bar pills.",
"Calling spawn_agent is the only way to start a visible floating-bar background agent; saying you will start one does not start it.",
"Use visible=false for parent-linked background work that should not appear as a pill.",
"The primary coordinator decides in its model loop whether to call spawn_agent. When the current user explicitly asks OpenClaw or Hermes to do work, call spawn_agent in that same turn with that provider; do not delegate that instruction to another agent, use a text-pattern handoff, or narrate that only another chat surface can do it.",
"Pass provider='openclaw' or provider='hermes' only when the current user explicitly names that provider; otherwise omit provider so Omi starts its regular managed agent.",
"Pass toolPolicy.allowedToolNames to restrict which Omi tools the child agent may call; it can only narrow, never widen, the child's tool set.",
"Inspect progress with list_agent_sessions or get_agent_run."
]
),
Capability(
toolName: "run_agent_and_wait",
title: "Run Agent And Wait",
latency: .asyncBackground,
surfaces: Set([.desktopChat]),
summary: "Run a parent-linked child agent synchronously and return its structured result.",
bullets: [
"Use for synchronous structured child results linked to a known parent run."
]
),
Capability(
toolName: "set_desktop_attention_override",
title: "Set Desktop Attention Override",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Dismiss or hide a kernel-derived attention subject such as a floating-bar run.",
bullets: [
"Use dismissed=true to hide floating-bar pills without deleting canonical run state.",
"Use dismissed=true to hide a floating-bar pill without deleting its canonical run.",
"Use subjectKind=run and subjectId=<runId> for pill dismissal."
]
),
Capability(
toolName: "search_tasks",
title: "Search Tasks",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Vector similarity search on tasks (action_items + staged_tasks).",
bullets: [
"Use for finding tasks by meaning, not exact keywords, e.g. \"find tasks about shopping\".",
"Examples: \"tasks about shopping\", \"anything related to the presentation\".",
"Parameters: query (required), include_completed (default false).",
"More reliable than hand-writing MATCH queries for task search."
]
),
Capability(
toolName: "complete_task",
title: "Complete Task",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Toggle a task's completion status by backendId.",
bullets: [
"Use after finding the task with execute_sql or search_tasks."
]
),
Capability(
toolName: "delete_task",
title: "Delete Task",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Delete a task permanently by backendId.",
bullets: [
"Use after finding the task with execute_sql or search_tasks."
]
),
Capability(
toolName: "load_skill",
title: "Load Skill",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Load the full instructions for a named skill listed in available_skills.",
bullets: [
"Use the exact skill name from available_skills."
]
),
Capability(
toolName: "search_skills",
title: "Search Skills",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Search installed skill names and compact descriptions before loading a specialized workflow.",
bullets: [
"Use only when the user's request may benefit from a specialized workflow.",
"Load a returned skill only when it is relevant to the user's request.",
"Use only when the current user request plausibly needs a specialized workflow.",
"Do not browse skills merely to explore options or because a related term appears in conversation context."
]
),
Capability(
toolName: "save_knowledge_graph",
title: "Save Knowledge Graph",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Save a knowledge graph of entities and relationships extracted from the user's data.",
bullets: [
"Parameters: nodes (array of {id, label, node_type, aliases}), edges (array of {source_id, target_id, label}).",
"node_type must be one of: person, organization, place, thing, concept.",
"Use when exploring the user's files during onboarding to build their knowledge graph.",
"Deduplication is handled automatically; provide all entities you find.",
"Use when exploring the user's files during onboarding or knowledge-graph building.",
"Deduplication is handled automatically; include all meaningful entities and relationships you found."
]
),
Capability(
toolName: "get_conversations",
title: "Get Conversations",
latency: .fastNetwork,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Retrieve conversations by recency or date range.",
bullets: [
"Use for latest/recent conversations and time-based conversation retrieval.",
"For voice, this returns summaries only and should be spoken briefly."
]
),
Capability(
toolName: "search_conversations",
title: "Search Conversations",
latency: .fastNetwork,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Semantic search across the user's past conversations.",
bullets: [
"Use for specific topics, decisions, or events discussed in conversations."
]
),
Capability(
toolName: "get_memories",
title: "Get Memories",
latency: .fastNetwork,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Retrieve stored facts, preferences, habits, people, and background about the user.",
bullets: [
"Use for broad 'what do you know about me' questions or personal facts."
]
),
Capability(
toolName: "search_memories",
title: "Search Memories",
latency: .fastNetwork,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Semantic search across user memories.",
bullets: [
"Use for a specific personal fact that is not already in the visible user context."
]
),
Capability(
toolName: "get_action_items",
title: "Get Action Items",
latency: .fastNetwork,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Retrieve the user's tasks with optional completion and due-date filters.",
bullets: [
"Use for completed tasks, date ranges, or the full task list.",
"For voice, prefer get_tasks for plain overdue/due-today questions."
]
),
Capability(
toolName: "create_action_item",
title: "Create Action Item",
latency: .fastNetwork,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Create a new task, to-do, or reminder.",
bullets: [
"Use when the user explicitly asks to add something to their list.",
"Pass a concise description and due_at only when the user gave a time."
]
),
Capability(
toolName: "update_action_item",
title: "Update Action Item",
latency: .fastNetwork,
surfaces: Set([.desktopChat, .realtimeHub]),
summary: "Update an existing task's status, description, or due date.",
bullets: [
"Find the task first, then update the matching id. Do not guess task ids."
]
),
Capability(
toolName: "capture_screen",
title: "Capture Screen",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Capture a live current-screen image after the user asks about what is visible now.",
bullets: [
"For a direct current-screen question, use this live capture instead of treating screen history as current evidence.",
"Use capture_screen only when raw pixels are necessary; it requires explicit approval before image bytes are shared.",
"The result lists the full-screen image path plus native-resolution detail tiles on large screens; use Read to view them.",
"For a direct current-screen question, capture a live image instead of using get_work_context as current visual evidence.",
"After capture_screen returns, use Read to view the full-screen image.",
"The full screenshot is downscaled before you see it — before quoting small on-screen text (titles, prices, sizes, labels) or choosing between similar-looking items, Read the detail tile covering that item and take the exact text from the tile.",
"Keep every detail you cite (title, price, badge, position) bound to one on-screen item; if text is not legible even in a tile, say so instead of inferring.",
"Do NOT use bash screencapture - always use this tool instead."
]
),
Capability(
toolName: "check_permission_status",
title: "Check Permission Status",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub, .onboarding]),
summary: "Check whether a required macOS permission has been granted.",
bullets: [
"Use before requesting a permission or after request_permission returns pending.",
"Omit type to check all supported permissions."
]
),
Capability(
toolName: "request_permission",
title: "Request Permission",
latency: .fastLocal,
surfaces: Set([.desktopChat, .realtimeHub, .onboarding]),
summary: "Open or guide the user through granting a required macOS permission. Screen sharing is the macOS Screen Recording permission.",
bullets: [
"Call only when the current user message names one permission, clearly affirms your immediately preceding one-permission request, or directly says to request it/that permission.",
"Treat screen share, screen sharing, and screen-share as the screen_recording permission type.",
"Ask the user to choose when their request is generic or names multiple permissions.",
"The user must still complete the native macOS prompt or Settings toggle.",
"Call only when the current user message explicitly requests one named permission, clearly affirms your immediately preceding one-permission request, or directly says to request it/that permission.",
"For generic or multi-permission requests, ask the user which permission they want to grant.",
"Use strict permission types only. Do not invent permission names.",
"After requesting, explain any returned requires_restart or pending status."
]
),
Capability(
toolName: "scan_files",
title: "Scan Files",
latency: .asyncBackground,
surfaces: Set([.onboarding]),
summary: "Scan selected files/folders during onboarding to build local context.",
bullets: [
"Onboarding-only."
]
),
Capability(
toolName: "set_user_preferences",
title: "Set User Preferences",
latency: .fastLocal,
surfaces: Set([.onboarding]),
summary: "Persist onboarding preferences such as name and language.",
bullets: [
"Onboarding-only."
]
),
Capability(
toolName: "ask_followup",
title: "Ask Followup",
latency: .asyncBackground,
surfaces: Set([.onboarding]),
summary: "Ask the user a follow-up onboarding question with optional quick replies.",
bullets: [
"Onboarding-only."
]
),
Capability(
toolName: "complete_onboarding",
title: "Complete Onboarding",
latency: .fastLocal,
surfaces: Set([.onboarding]),
summary: "Complete onboarding after required goals and context are collected.",
bullets: [
"Onboarding-only."
]
),
Capability(
toolName: "get_email_insights",
title: "Get Email Insights",
latency: .fastLocal,
surfaces: Set([.onboarding]),
summary: "Read precomputed email/calendar onboarding insights.",
bullets: [
"Onboarding-only; requires background insights to be loaded."
]
),
Capability(
toolName: "get_tasks",
title: "Get Tasks",
latency: .fastLocal,
surfaces: Set([.realtimeHub]),
summary: "Read the user's overdue and due-today tasks locally.",
bullets: [
"Use for plain voice questions like what are my tasks, what's due today, or what's on my list.",
"Prefer get_action_items for completed tasks, date ranges, or the full list."
]
),
Capability(
toolName: "create_calendar_event",
title: "Create Calendar Event",
latency: .fastNetwork,
surfaces: Set([.realtimeHub]),
summary: "Create a new Google Calendar event.",
bullets: [
"Use when the user asks to add, create, schedule, or put a specific event on their calendar.",
"Pass title, start_time, and end_time as ISO-8601 strings with timezone; include location, description, and attendees when provided.",
"This capability creates one specified event; it does not find availability, reschedule, delete, or coordinate with people."
]
),
Capability(
toolName: "ask_higher_model",
title: "Ask Higher Model",
latency: .fastNetwork,
surfaces: Set([.realtimeHub]),
summary: "Get a second opinion from the larger model when the user pushes back or current facts are needed.",
bullets: [
"Use sparingly; answer simple or creative requests yourself."
]
),
Capability(
toolName: "screenshot",
title: "Screenshot",
latency: .fastLocal,
surfaces: Set([.realtimeHub]),
summary: "Capture the user's current screen.",
bullets: [
"Use when the user asks about what is on screen."
]
),
Capability(
toolName: "report_screen_observation",
title: "Report Screen Observation",
latency: .fastLocal,
surfaces: Set([.realtimeHub]),
summary: "Verify grounding from the current-screen image.",
bullets: [
"Only call after screenshot returns the current image.",
"Submit a concise visual observation, then answer the user's original request naturally."
]
),
Capability(
toolName: "point_click",
title: "Point Click",
latency: .fastLocal,
surfaces: Set([.realtimeHub]),
summary: "Click at on-screen pixel coordinates.",
bullets: [
"Use only when the user clearly asks you to click something."
]
),
Capability(
toolName: "get_local_status",
title: "Get Local Status",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Report whether local Omi Desktop context is available.",
bullets: [
"Local API only."
]
),
Capability(
toolName: "get_screenshot",
title: "Get Screenshot",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Fetch a local Rewind screenshot image by screenshot_id.",
bullets: [
"Local API only."
]
),
Capability(
toolName: "get_work_context",
title: "Get Work Context",
latency: .fastLocal,
surfaces: Set([.desktopChat]),
summary: "Get the user's current screen plus a compressed timeline of recent on-screen activity.",
bullets: [
"Call this first for \"what is on my screen\", \"do you see my screen\", and current-work questions.",
"Returns availability, a screenshot_id for follow-up, OCR preview, and recent timeline without raw image bytes.",
"If raw pixels are needed after this, request get_screenshot/capture_screen approval.",
"Use this for recent work/activity history, not for direct current-screen questions.",
"Its screen_now and timeline fields are historical unless this turn separately attached a live image.",
"For current visual detail, use capture_screen when approval is available rather than answering from this tool."
]
)
]
static func capabilities(for surface: Surface) -> [Capability] {
capabilities.filter { $0.supports(surface) }
}
static var desktopToolNames: [String] {
capabilities(for: .desktopChat).map(\.toolName)
}
static var realtimeToolNames: [String] {
["ask_higher_model","cancel_agent_run","check_permission_status","create_action_item","create_calendar_event","get_action_items","get_agent_run","get_conversations","get_daily_recap","get_memories","get_tasks","inspect_agent_artifacts","list_agent_sessions","point_click","report_screen_observation","request_permission","screenshot","search_conversations","search_memories","search_screen_history","set_desktop_attention_override","spawn_agent","update_action_item","update_agent_artifact_lifecycle"]
}
}