ScreenContextTelemetry.swift

import AppKit
import CoreGraphics
import Foundation

enum ScreenContextFailureCode: String, CaseIterable {
  case permissionDenied = "permission_denied"
  case databaseUnavailable = "database_unavailable"
  case screenNowUnavailable = "screen_now_unavailable"
  case screenshotPending = "screenshot_pending"
  case screenshotFileMissing = "screenshot_file_missing"
  case screenshotChunkCorrupted = "screenshot_chunk_corrupted"
  case screenshotSharingDisabled = "screenshot_sharing_disabled"
  case imageUnavailable = "image_unavailable"
  case policyApprovalRequired = "policy_approval_required"
  case captureFailed = "capture_failed"
  /// The ask came from Omi's own window and no usable frame of another app
  /// existed to stand in for the screen (none found, unreadable, or stale).
  /// Distinct from a capture failure so the explicit-screen funnel can tell
  /// "Omi was the subject" turns apart from broken ones.
  case omiFrontmostNoFrame = "omi_frontmost_no_frame"
  case unknown = "unknown"
}

struct ScreenshotUnavailableClassification {
  let code: ScreenContextFailureCode
  let reason: String
  let hint: String
}

enum ScreenContextInterestDetector {
  private static let exactPhrases = [
    "what is on my screen",
    "what's on my screen",
    "whats on my screen",
    "what do you see on my screen",
    "can you see my screen",
    "do you see my screen",
    "look at my screen",
    "look on my screen",
    "see my screen",
    "view my screen",
    "current screen",
    "this screen",
    "my screen",
    "what am i looking at",
    "what i'm looking at",
    "this error",
    "this page",
    "this window",
    "this app",
    "on the left",
    "on the right",
    "at the top",
    "at the bottom",
  ]

  private static let contextualVerbs = [
    "look",
    "see",
    "view",
    "read",
    "inspect",
    "debug",
    "identify",
  ]

  private static let visualReferences = [
    "page",
    "window",
    "error",
    "dialog",
    "button",
    "option",
    "screen",
  ]

  static func isScreenContextRequest(_ text: String) -> Bool {
    let lower = text.lowercased()
      .replacingOccurrences(of: "’", with: "'")
      .trimmingCharacters(in: .whitespacesAndNewlines)
    guard !lower.isEmpty else { return false }
    if exactPhrases.contains(where: { lower.contains($0) }) {
      return true
    }
    let words = lower.split { !$0.isLetter && !$0.isNumber }.map(String.init)
    guard !words.isEmpty else { return false }
    let wordSet = Set(words)
    return contextualVerbs.contains(where: wordSet.contains) && visualReferences.contains(where: wordSet.contains)
  }

  /// The narrow reading used when the message carries attachments: only a message that names the
  /// screen itself is about the screen. "Look at this page" beside a PDF is about the PDF; the
  /// deictic cues `isScreenContextRequest` accepts ("this", "look", "page") are exactly the words
  /// people use to point at the file they just attached.
  static func namesTheScreen(_ text: String) -> Bool {
    text.lowercased().contains("screen")
  }
}

enum ScreenContextAutoIncludeReason: Equatable {
  case explicitScreenRequest
  case ambientSurfaceContext

  var isExplicitScreenRequest: Bool {
    self == .explicitScreenRequest
  }
}

enum ScreenContextAutoIncludePolicy {
  /// `hasAttachments`: the message carries files or conversation references the user chose to
  /// attach. Those are the message's subject, so no ambient screen context is added alongside
  /// them and a capture is taken only when the text names the screen outright — otherwise the
  /// model is handed a desktop to describe next to the file it was asked about.
  static func reason(
    userText: String,
    systemPromptStyle: ChatSystemPromptStyle,
    turnOwner: ChatTurnOwner,
    onboardingActive: Bool = false,
    hasAttachments: Bool = false
  ) -> ScreenContextAutoIncludeReason? {
    if hasAttachments {
      return ScreenContextInterestDetector.namesTheScreen(userText) ? .explicitScreenRequest : nil
    }
    if ScreenContextInterestDetector.isScreenContextRequest(userText) {
      return .explicitScreenRequest
    }

    switch turnOwner {
    case .floatingDefault, .floatingVoice:
      // The onboarding demo's whole premise is "Omi reads your screen", but its
      // suggested query has no screen-cue words. Treat onboarding floating turns
      // as explicit so a real capture is attempted and capture/permission
      // failures surface in the answer instead of a silently blind reply.
      return onboardingActive ? .explicitScreenRequest : .ambientSurfaceContext
    case .taskChat, .agentPill:
      return .ambientSurfaceContext
    case .mainChat:
      return systemPromptStyle == .floating ? .ambientSurfaceContext : nil
    }
  }

  static func shouldInclude(
    userText: String,
    systemPromptStyle: ChatSystemPromptStyle,
    turnOwner: ChatTurnOwner,
    hasAttachments: Bool = false
  ) -> Bool {
    reason(
      userText: userText, systemPromptStyle: systemPromptStyle, turnOwner: turnOwner,
      hasAttachments: hasAttachments) != nil
  }
}

struct ScreenContextTelemetryContext {
  let surface: String
  let surfaceKind: String?
  let externalRefKind: String?
  let externalRefId: String?
  let runId: String?
  let pillId: String?

  static let desktopChat = ScreenContextTelemetryContext(surface: "desktop_chat")

  init(
    surface: String,
    surfaceKind: String? = nil,
    externalRefKind: String? = nil,
    externalRefId: String? = nil,
    runId: String? = nil,
    pillId: String? = nil
  ) {
    self.surface = surface
    self.surfaceKind = surfaceKind
    self.externalRefKind = externalRefKind
    self.externalRefId = externalRefId
    self.runId = runId
    self.pillId = pillId
  }

  static func from(
    surfaceRef: AgentSurfaceReference?,
    fallbackSurface: String = "desktop_chat",
    runId: String? = nil
  ) -> ScreenContextTelemetryContext {
    guard let surfaceRef else {
      return ScreenContextTelemetryContext(surface: fallbackSurface, runId: runId)
    }
    let surface = surfaceRef.surfaceKind.isEmpty ? fallbackSurface : surfaceRef.surfaceKind
    let pillId = surfaceRef.externalRefKind == "pill" ? surfaceRef.externalRefId : nil
    let resolvedRunId = runId ?? (surfaceRef.externalRefKind == "run" ? surfaceRef.externalRefId : nil)
    return ScreenContextTelemetryContext(
      surface: surface,
      surfaceKind: surfaceRef.surfaceKind,
      externalRefKind: surfaceRef.externalRefKind,
      externalRefId: surfaceRef.externalRefId,
      runId: resolvedRunId,
      pillId: pillId
    )
  }
}

enum ScreenContextToolTelemetry {
  private static let screenContextTools: Set<String> = [
    "get_work_context",
    "capture_screen",
    "get_screenshot",
    "look_at_frame",
    "show_rewind_evidence",
    "search_screen_history",
    "semantic_search",
  ]

  static func isScreenContextTool(_ toolName: String) -> Bool {
    screenContextTools.contains(toolName)
  }

  static func imageBytesBucket(_ byteCount: Int?) -> String? {
    guard let byteCount else { return nil }
    if byteCount <= 0 { return "0" }
    if byteCount <= 50 * 1024 { return "1-50kb" }
    if byteCount <= 250 * 1024 { return "50-250kb" }
    return "250kb+"
  }

  static func trackToolResult(
    toolName: String,
    context: ScreenContextTelemetryContext = .desktopChat,
    ok: Bool,
    failureCode: ScreenContextFailureCode? = nil,
    screenNowAvailable: Bool? = nil,
    timelineCount: Int? = nil,
    latestCaptureAgeSeconds: Int? = nil,
    hasOCRPreview: Bool? = nil,
    imageBytes: Int? = nil,
    permissionTCCGranted: Bool? = nil,
    sckAvailable: Bool? = nil
  ) {
    Task { @MainActor in
      AnalyticsManager.shared.screenContextToolResult(
        toolName: toolName,
        context: context,
        ok: ok,
        failureCode: failureCode?.rawValue,
        screenNowAvailable: screenNowAvailable,
        timelineCount: timelineCount,
        latestCaptureAgeSeconds: latestCaptureAgeSeconds,
        hasOCRPreview: hasOCRPreview,
        imageBytesBucket: imageBytesBucket(imageBytes),
        permissionTCCGranted: permissionTCCGranted,
        sckAvailable: sckAvailable
      )
    }
  }

  static func trackInvariant(
    _ name: String,
    context: ScreenContextTelemetryContext = .desktopChat,
    toolName: String? = nil,
    properties: [String: Any] = [:]
  ) {
    let propertiesBox = RuntimeJSONPayloadBox(properties)
    Task { @MainActor in
      AnalyticsManager.shared.screenContextInvariant(
        name: name,
        context: context,
        toolName: toolName,
        properties: propertiesBox.value
      )
    }
  }

  static func classifyScreenshotUnavailable(
    screenshot: Screenshot,
    activeChunk: String?,
    error: Error
  ) -> ScreenshotUnavailableClassification? {
    if let rewindError = error as? RewindError, case .corruptedVideoChunk = rewindError {
      return ScreenshotUnavailableClassification(
        code: .screenshotChunkCorrupted,
        reason: "The video chunk backing this screenshot is corrupted and cannot be decoded.",
        hint: "Pick a different screenshot_id; this frame's pixels are unrecoverable."
      )
    }
    if screenshot.usesVideoStorage, let chunk = screenshot.videoChunkPath, chunk == activeChunk {
      return ScreenshotUnavailableClassification(
        code: .screenshotPending,
        reason: "The frame is in the active recording segment that has not been flushed to disk yet.",
        hint: "Retry in ~60s, or choose an older screenshot_id whose video chunk is already finalized."
      )
    }
    if !screenshot.usesVideoStorage, (screenshot.imagePath ?? "").isEmpty {
      return ScreenshotUnavailableClassification(
        code: .imageUnavailable,
        reason: "This screenshot row has no stored image.",
        hint: "Pick a different screenshot_id from a recent search_screen_history result."
      )
    }
    if error as? RewindError != nil {
      return ScreenshotUnavailableClassification(
        code: .screenshotFileMissing,
        reason: "The image data for this screenshot is no longer on disk.",
        hint: "Pick a more recent screenshot_id whose pixels are still retained."
      )
    }
    return nil
  }

  static func toolResultFacts(toolName: String, output: String) -> ScreenContextToolFacts? {
    if output.hasPrefix("EXECUTION_PRECONDITION_FAILED:"),
      output.contains("\"code\":\"execution_precondition_failed\""),
      output.contains("\"reason\":\"screenshot_sharing_disabled\"")
    {
      return ScreenContextToolFacts(
        requested: true,
        succeeded: false,
        approvalRequired: false,
        failureCode: .screenshotSharingDisabled
      )
    }
    if output.hasPrefix("POLICY_DENIED:"), output.contains("\"code\":\"approval_required\"") {
      return ScreenContextToolFacts(
        requested: true,
        succeeded: false,
        approvalRequired: true,
        failureCode: .policyApprovalRequired
      )
    }
    if output.hasPrefix("PERMISSION_REQUIRED:"), output.contains("\"code\":\"permission_required\"") {
      if !permissionErrorHasNextTool(output) {
        trackInvariant("screen_tool_permission_error_missing_next_tool", toolName: toolName)
      }
      return ScreenContextToolFacts(
        requested: true,
        succeeded: false,
        approvalRequired: false,
        failureCode: .permissionDenied
      )
    }

    let normalizedOutput =
      output
      .replacingOccurrences(of: "EXECUTION_PRECONDITION_FAILED: ", with: "")
      .replacingOccurrences(of: "POLICY_DENIED: ", with: "")
      .replacingOccurrences(of: "PERMISSION_REQUIRED: ", with: "")
    guard
      let data = normalizedOutput.data(using: .utf8),
      let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
    else {
      return nil
    }

    let ok = (json["ok"] as? Bool) == true
    if toolName == "get_work_context" {
      let screenNow = json["screen_now"] as? [String: Any]
      let screenAvailable = (screenNow?["available"] as? Bool) == true
      let failureRaw = (json["failure_code"] as? String) ?? (screenNow?["failure_code"] as? String)
      if failureRaw == ScreenContextFailureCode.permissionDenied.rawValue, !jsonPermissionErrorHasNextTool(json) {
        trackInvariant("screen_tool_permission_error_missing_next_tool", toolName: toolName)
      }
      return ScreenContextToolFacts(
        requested: true,
        succeeded: ok && screenAvailable,
        approvalRequired: false,
        failureCode: failureRaw.flatMap(ScreenContextFailureCode.init(rawValue:))
      )
    }

    if toolName == "get_screenshot" || toolName == "capture_screen" || toolName == "show_rewind_evidence" {
      let failureRaw = (json["error"] as? String) ?? (json["code"] as? String)
      return ScreenContextToolFacts(
        requested: true,
        succeeded: ok,
        approvalRequired: failureRaw == "approval_required",
        failureCode: failureRaw.flatMap(ScreenContextFailureCode.init(rawValue:))
      )
    }

    return nil
  }

  private static func permissionErrorHasNextTool(_ output: String) -> Bool {
    let normalizedOutput =
      output
      .replacingOccurrences(of: "PERMISSION_REQUIRED: ", with: "")
    guard
      let data = normalizedOutput.data(using: .utf8),
      let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
    else {
      return false
    }
    return jsonPermissionErrorHasNextTool(json)
  }

  private static func jsonPermissionErrorHasNextTool(_ json: [String: Any]) -> Bool {
    guard json["next_tool"] as? String == "request_permission",
      let args = json["next_tool_arguments"] as? [String: Any]
    else {
      return false
    }
    return args["type"] as? String == "screen_recording"
  }
}

struct ScreenContextToolFacts {
  let requested: Bool
  let succeeded: Bool
  let approvalRequired: Bool
  let failureCode: ScreenContextFailureCode?
}

struct ScreenContextChatCycleSnapshot {
  let screenToolRequested: Bool
  let screenToolSucceeded: Bool
  let screenToolApprovalRequired: Bool
  let screenToolFailureCodes: [String]
}

final class ScreenContextChatCycleMetrics: @unchecked Sendable {
  private let lock = NSLock()
  private var screenToolRequested = false
  private var screenToolSucceeded = false
  private var screenToolApprovalRequired = false
  private var failureCodes: Set<String> = []

  func recordToolRequested(_ toolName: String) {
    guard ScreenContextToolTelemetry.isScreenContextTool(toolName) else { return }
    lock.lock()
    screenToolRequested = true
    lock.unlock()
  }

  func recordToolResult(name: String, output: String) {
    guard ScreenContextToolTelemetry.isScreenContextTool(name) else { return }
    guard let facts = ScreenContextToolTelemetry.toolResultFacts(toolName: name, output: output) else {
      lock.lock()
      screenToolRequested = true
      lock.unlock()
      return
    }
    lock.lock()
    screenToolRequested = screenToolRequested || facts.requested
    screenToolSucceeded = screenToolSucceeded || facts.succeeded
    screenToolApprovalRequired = screenToolApprovalRequired || facts.approvalRequired
    if let failureCode = facts.failureCode {
      failureCodes.insert(failureCode.rawValue)
    }
    lock.unlock()
  }

  func snapshot() -> ScreenContextChatCycleSnapshot {
    lock.lock()
    defer { lock.unlock() }
    return ScreenContextChatCycleSnapshot(
      screenToolRequested: screenToolRequested,
      screenToolSucceeded: screenToolSucceeded,
      screenToolApprovalRequired: screenToolApprovalRequired,
      screenToolFailureCodes: failureCodes.sorted()
    )
  }
}

enum ScreenContextWorkContextBuilder {
  static let staleCaptureThresholdSeconds = 60
  static let voiceTurnStaleCaptureThresholdSeconds = 15

  static func isoFormatter(timeZone: TimeZone = .current) -> ISO8601DateFormatter {
    let formatter = ISO8601DateFormatter()
    formatter.timeZone = timeZone
    return formatter
  }

  /// Ambient turns without Screen Recording get this instead of silence, so the
  /// model can explain a blind answer when the question was screen-dependent —
  /// without manufacturing a permission request for generic utterances.
  static func ambientPermissionUnavailablePayload() -> [String: Any] {
    [
      "ok": false,
      "name": "get_work_context",
      "failure_code": ScreenContextFailureCode.permissionDenied.rawValue,
      "permission": [
        "screen_recording": "not_granted"
      ],
      "screen_now": [
        "available": false,
        "failure_code": ScreenContextFailureCode.permissionDenied.rawValue,
      ],
      "timeline": [],
      "guidance":
        "No screen context is available because Screen Recording is not enabled for Omi. ONLY if the user's question depends on seeing their screen: start your reply by telling them to enable Screen Recording for Omi in System Settings > Privacy & Security, then answer what you can. For questions that do not need the screen, answer normally and do not mention permissions.",
    ]
  }

  /// Direct “what is on my screen?” requests are a turn-scoped visual action,
  /// not a request for Rewind history. The caller attaches the same image bytes
  /// to the model request; this envelope makes that provenance explicit.
  static func explicitCurrentScreenPayload(
    screenRecordingGranted: Bool,
    imageAttached: Bool,
    capturedAt: Date = Date(),
    formatter: ISO8601DateFormatter = isoFormatter()
  ) -> [String: Any] {
    guard screenRecordingGranted else {
      return permissionDeniedPayload(windowMinutes: 1)
    }
    guard imageAttached else {
      return [
        "ok": false,
        "name": "get_work_context",
        "failure_code": ScreenContextFailureCode.imageUnavailable.rawValue,
        "screen_now": [
          "available": false,
          "failure_code": ScreenContextFailureCode.imageUnavailable.rawValue,
          "source": "turn_scoped_live_capture",
        ],
        "timeline": [],
        "guidance":
          "A live capture failed even though Screen Recording shows granted. The usual cause is that the permission was granted after Omi launched and only takes effect after a relaunch. START your reply by telling the user to quit and reopen Omi to activate Screen Recording, then answer what you can without the screen. Do not answer from screen history.",
      ]
    }
    return [
      "ok": true,
      "name": "get_work_context",
      "screen_now": [
        "available": true,
        "source": "turn_scoped_live_capture",
        "captured_at": formatter.string(from: capturedAt),
        "image_delivered_to_model": true,
        "latest_capture_age_seconds": 0,
      ],
      "timeline": [],
      "guidance":
        "The attached image is the only current-screen evidence for this turn. Answer the user's question from it; do not substitute stored history or OCR metadata.",
    ]
  }

  /// The explicit current-screen question was asked from Omi's own window, so
  /// a fresh capture would photograph Omi describing itself. The attached
  /// image is instead the screen as the user left it — either the summon-
  /// boundary capture or the most recent store frame of another app. The
  /// envelope names that provenance positively and forbids the substitution
  /// in the direction the failure actually happens: describing Omi.
  static func explicitLastExternalFramePayload(
    source: String,
    appName: String,
    windowTitle: String?,
    frameAgeSeconds: Int,
    capturedAt: Date,
    formatter: ISO8601DateFormatter = isoFormatter()
  ) -> [String: Any] {
    let windowPart =
      (windowTitle.map { $0.trimmingCharacters(in: .whitespaces) }.flatMap { $0.isEmpty ? nil : $0 })
      .map { ", \"\($0)\"" } ?? ""
    let originSentence: String
    switch source {
    case "summon_boundary_capture":
      originSentence =
        "It was captured \(max(0, frameAgeSeconds)) seconds ago, at the moment Omi's window came to the front."
    default:
      originSentence =
        "It was captured \(max(0, frameAgeSeconds)) seconds before Omi came to the front."
    }
    return [
      "ok": true,
      "name": "get_work_context",
      "screen_now": [
        "available": true,
        "source": source,
        "app_name": appName,
        "captured_at": formatter.string(from: capturedAt),
        "image_delivered_to_model": true,
        "latest_capture_age_seconds": max(0, frameAgeSeconds),
      ],
      "timeline": [],
      "guidance":
        "The user asked about their screen from inside Omi's own window, so a capture at send time would show Omi, not what they mean. The attached image shows \(appName)\(windowPart). \(originSentence) Treat it as the screen the user is asking about. Do not describe Omi, its interface, or this conversation. Answer about \(appName) from the attached image only; do not substitute OCR text or stored history.",
    ]
  }

  /// The explicit current-screen question was asked from Omi's own window and
  /// no honest stand-in exists. The self-referential capture is never shipped
  /// "with a warning" — a payload that names the failure and the way out is
  /// the file's standing doctrine for unavailable evidence.
  static func selfFrontmostUnavailablePayload(
    reason: ScreenContextFallbackUnavailable,
    lastExternalAppName: String? = nil,
    lastExternalFrameAgeSeconds: Int? = nil,
    formatter: ISO8601DateFormatter = isoFormatter()
  ) -> [String: Any] {
    var screenNow: [String: Any] = [
      "available": false,
      "failure_code": ScreenContextFailureCode.omiFrontmostNoFrame.rawValue,
      "source": "omi_frontmost_no_external_frame",
    ]
    if let lastExternalAppName {
      screenNow["last_external_app_name"] = lastExternalAppName
    }
    if let lastExternalFrameAgeSeconds {
      screenNow["last_external_frame_age_seconds"] = lastExternalFrameAgeSeconds
    }
    let stalenessNote: String
    switch reason {
    case .noAttachableFrame:
      stalenessNote = ""
    case .frameTooStale(let ageSeconds):
      stalenessNote =
        " The most recent frame of another app is \(ageSeconds) seconds old, past the freshness limit."
    }
    // The recovery matches the failure. A missing frame usually means the
    // user has been in Omi for a while, and the honest way out is handing the
    // model fresh pixels from inside Omi — the picker and ⌘V both exist —
    // while a too-stale frame means they may simply have taken a while to
    // send. The switch-apps round trip stays as the fallback in both.
    let recovery: String
    switch reason {
    case .noAttachableFrame:
      recovery =
        "START your reply by asking the user to switch to the app they want summarized and ask again from there. You can also mention they can attach a recent screen from the paperclip's menu or paste a screenshot with ⌘V."
    case .frameTooStale:
      recovery =
        "START your reply by telling the user the frame on file is too old to answer from, and offer the ways to hand you fresh pixels without leaving Omi: attach a recent screen from the paperclip's menu, or paste a screenshot with ⌘V. If they would rather switch to the app they want summarized and ask again from there, that works too."
    }
    return [
      "ok": false,
      "name": "get_work_context",
      "failure_code": ScreenContextFailureCode.omiFrontmostNoFrame.rawValue,
      "screen_now": screenNow,
      "timeline": [],
      "guidance":
        "This question was asked from Omi's own window, so a live capture would show Omi itself, and no recent frame of another app is available to stand in for the screen.\(stalenessNote) \(recovery) Do not describe Omi's own window or interface, and do not answer from stored history.",
    ]
  }

  /// Pixels + envelope for an explicit current-screen request, resolved
  /// through `ScreenContextFallbackPolicy`. The policy owns the decision; this
  /// gathers its inputs and performs the effects (live capture or frame load).
  /// On the main chat the live-capture branch never fires, because the
  /// composer is Omi's own window — the policy's one self-referential surface.
  ///
  /// Main-actor isolated: the frame loader and the workspace state behind the
  /// defaults live there. The capture itself is dispatched off-main below.
  @MainActor
  static func explicitScreenEvidence(
    turnOwner: ChatTurnOwner,
    now: Date = Date(),
    frontmostBundleIdentifier: String? = NSWorkspace.shared.frontmostApplication?.bundleIdentifier,
    omiBundleIdentifier: String? = Bundle.main.bundleIdentifier,
    loader: RewindFrameLoader = .shared,
    isScreenRecordingGranted: @Sendable () -> Bool = { CGPreflightScreenCaptureAccess() },
    captureNow: @Sendable @escaping () -> Data? = { ScreenCaptureManager.captureScreenData() }
  ) async -> (imageData: Data?, payload: [String: Any]) {
    // Recorded, not gated: frontmost-at-send is near-tautological for typed
    // main-chat sends and adds nothing to the decision, but it keeps the
    // funnel able to distinguish a summon-and-ask from other paths.
    let isOmiFrontmost =
      frontmostBundleIdentifier != nil && frontmostBundleIdentifier == omiBundleIdentifier
    // Without the permission there is no evidence on any surface — no live
    // capture and no guarantee about what old frames contain — so the
    // enable-Screen-Recording payload answers, exactly as it did before the
    // fallback existed.
    guard isScreenRecordingGranted() else {
      return (nil, explicitCurrentScreenPayload(screenRecordingGranted: false, imageAttached: false))
    }
    // The staleness input needs the newest honest candidate. Two exist for a
    // main-chat ask: the newest attachable store row and the summon-boundary
    // capture (the screen as the user left it when Omi took the front) — the
    // newest wins, because both depict the same referent at different ages.
    // Non-main-chat surfaces never fall back, so no work is paid there.
    var fallbackCandidate:
      (
        timestamp: Date, source: String, appName: String, windowTitle: String?,
        provide: @Sendable () async -> Data?
      )?
    if turnOwner == .mainChat {
      if let row = await loader.latestAttachableRow() {
        fallbackCandidate = (
          row.timestamp,
          "last_external_frame",
          row.appName,
          row.windowTitle,
          { await loader.loadData(for: row) }
        )
      }
      if let boundary = loader.currentSummonBoundary(),
        fallbackCandidate?.timestamp ?? .distantPast < boundary.timestamp
      {
        fallbackCandidate = (
          boundary.timestamp,
          "summon_boundary_capture",
          boundary.appName,
          boundary.windowTitle,
          { boundary.data }
        )
      }
    }
    let frameAge = fallbackCandidate.map { max(0, now.timeIntervalSince($0.timestamp)) }

    func stamped(_ payload: [String: Any]) -> [String: Any] {
      var payload = payload
      var screenNow = payload["screen_now"] as? [String: Any] ?? [:]
      screenNow["omi_frontmost_at_send"] = isOmiFrontmost
      payload["screen_now"] = screenNow
      return payload
    }

    switch ScreenContextFallbackPolicy.evidenceSource(
      turnOwner: turnOwner,
      lastExternalFrameAgeSeconds: frameAge
    ) {
    case .turnScopedLiveCapture:
      // An explicit current-screen question gets one capture scoped to this
      // exact turn. Never let a Rewind frame or OCR summary impersonate the
      // image the model receives.
      let data = await Task.detached(priority: .userInitiated) {
        captureNow()
      }.value
      return (
        data,
        stamped(
          explicitCurrentScreenPayload(screenRecordingGranted: true, imageAttached: data != nil)
        )
      )
    case .lastExternalFrame:
      guard let candidate = fallbackCandidate, let frameAge else {
        return (nil, stamped(selfFrontmostUnavailablePayload(reason: .noAttachableFrame)))
      }
      // Candidate first (summon-boundary capture or newest store row). A
      // transient decode failure on the newest store row must not fail the
      // turn when an older attachable row is still loadable: fall through
      // to the loader, which skips unreadable rows under the same freshness
      // bound the policy just applied.
      var data = await candidate.provide()
      var served = candidate
      if data == nil {
        if let frame = await loader.loadLatestAttachableFrame(
          maxAgeSeconds: ScreenContextFallbackPolicy.maxFallbackFrameAgeSeconds,
          now: now
        ) {
          data = frame.data
          served = (
            frame.timestamp, "last_external_frame", frame.appName, frame.windowTitle,
            { nil }
          )
        }
      }
      guard let data else {
        return (
          nil,
          stamped(
            selfFrontmostUnavailablePayload(
              reason: .noAttachableFrame,
              lastExternalAppName: candidate.appName,
              lastExternalFrameAgeSeconds: Int(frameAge.rounded())
            )
          )
        )
      }
      return (
        data,
        stamped(
          explicitLastExternalFramePayload(
            source: served.source,
            appName: served.appName,
            windowTitle: served.windowTitle,
            frameAgeSeconds: Int(max(0, now.timeIntervalSince(served.timestamp)).rounded()),
            capturedAt: served.timestamp
          )
        )
      )
    case .unavailable(let reason):
      return (
        nil,
        stamped(
          selfFrontmostUnavailablePayload(
            reason: reason,
            lastExternalAppName: fallbackCandidate?.appName,
            lastExternalFrameAgeSeconds: frameAge.map { Int($0.rounded()) }
          )
        )
      )
    }
  }

  static func payload(arguments: RuntimeJSONPayloadBox) async -> [String: Any] {
    await payload(arguments: arguments.value)
  }

  static func payloadBox(arguments: RuntimeJSONPayloadBox) async -> RuntimeJSONPayloadBox {
    RuntimeJSONPayloadBox(await payload(arguments: arguments.value))
  }

  static func payload(arguments: [String: Any]) async -> [String: Any] {
    let minutes = max(1, min(120, Int(parseInt64(arguments["minutes"]) ?? 10)))
    let staleThresholdSeconds = max(
      1,
      min(300, Int(parseInt64(arguments["max_age_seconds"]) ?? Int64(staleCaptureThresholdSeconds)))
    )
    let includeScreen = parseBool(arguments["include_screen"]) ?? false
    let now = Date()
    let start = now.addingTimeInterval(-Double(minutes) * 60)
    let formatter = isoFormatter()

    // Cheap index first. A durable handle (URL / file) already names the document the
    // user means, so answering "where was that pricing doc" must not require Screen
    // Recording, a video-chunk decode, or an OCR dump. Tape is the fallback for a
    // question the handles cannot answer, and the caller asks for it by name.
    let index = await workHistoryIndex(start: start, now: now)
    if !includeScreen, index.hasDurableHandle {
      var cheap: [String: Any] = [
        "ok": true,
        "name": "get_work_context",
        "window_minutes": minutes,
        "screen_now": [
          "available": false,
          "reason": "not_requested",
        ],
        "timeline": [],
        "memories_hint": memoriesHint,
        "guidance": handleFirstGuidance,
        // The tape path reported `latest_capture_age_seconds`; this path reported nothing at
        // all, leaving wall-clock visit times with no present to measure against.
        "generated_at": formatter.string(from: now),
      ]
      attach(index, to: &cheap, now: now)
      return cheap
    }

    guard CGPreflightScreenCaptureAccess() else {
      var denied = permissionDeniedPayload(windowMinutes: minutes)
      // Screen Recording gates pixels, not the work index. Handles recorded before the
      // permission lapsed still answer a "where was that" question, so they ride along
      // instead of being thrown away with the screen.
      attach(index, to: &denied, now: now)
      return denied
    }

    guard await RewindDatabase.shared.getDatabaseQueue() != nil else {
      if let fresh = freshScreenCapturePayload(now: now, formatter: formatter) {
        return [
          "ok": true,
          "name": "get_work_context",
          "window_minutes": minutes,
          "screen_now": fresh,
          "timeline": [],
          "latest_capture_age_seconds": 0,
          "memories_hint": "For the user's operating principles/preferences, also call search_memories (omi-memory).",
          "guidance":
            "The local Rewind timeline database is unavailable, but a fresh live screen capture succeeded. Use capture_screen if raw pixels are necessary.",
        ]
      }
      return [
        "ok": false,
        "name": "get_work_context",
        "window_minutes": minutes,
        "failure_code": ScreenContextFailureCode.databaseUnavailable.rawValue,
        "screen_now": ["available": false, "failure_code": ScreenContextFailureCode.databaseUnavailable.rawValue],
        "timeline": [],
        "guidance": "Omi Desktop local screen history is not available yet.",
      ]
    }

    var screenNow: [String: Any] = ["available": false]
    var failureCode: ScreenContextFailureCode?
    var latestCaptureAgeSeconds: Int?
    let activeChunk = await VideoChunkEncoder.shared.currentChunkPath

    if let recent = try? await RewindDatabase.shared.getRecentScreenshots(limit: 25) {
      latestCaptureAgeSeconds = recent.first.map { max(0, Int(now.timeIntervalSince($0.timestamp))) }
      var firstUnavailable: ScreenContextFailureCode?
      for shot in recent {
        guard let sid = shot.id else { continue }
        if shot.usesVideoStorage, let chunk = shot.videoChunkPath, chunk == activeChunk {
          firstUnavailable = firstUnavailable ?? .screenshotPending
          continue
        }
        do {
          let data = try await loadScreenshotDataEnsuringStorage(for: shot)
          screenNow = [
            "available": true,
            "screenshot_id": sid,
            "timestamp": formatter.string(from: shot.timestamp),
            "app_name": shot.appName,
            "window_title": shot.windowTitle ?? NSNull(),
            "ocr_preview": String((shot.ocrText ?? "").prefix(800)),
            "image_bytes": data.count,
            "latest_capture_age_seconds": max(0, Int(now.timeIntervalSince(shot.timestamp))),
            "note":
              "Latest available finalized frame (may be up to ~1 min old, and can predate window_minutes). Call get_screenshot with this screenshot_id only when you need raw pixels.",
          ]
          failureCode = nil
          break
        } catch {
          let classified = ScreenContextToolTelemetry.classifyScreenshotUnavailable(
            screenshot: shot,
            activeChunk: activeChunk,
            error: error
          )
          firstUnavailable = firstUnavailable ?? classified?.code ?? .imageUnavailable
        }
      }
      if (screenNow["available"] as? Bool) != true {
        failureCode = firstUnavailable ?? .screenNowUnavailable
        screenNow["failure_code"] = failureCode?.rawValue
      }
    } else {
      failureCode = .screenNowUnavailable
      screenNow["failure_code"] = failureCode?.rawValue
    }

    if shouldUseFreshCapture(
      screenNow: screenNow,
      latestCaptureAgeSeconds: latestCaptureAgeSeconds,
      staleThresholdSeconds: staleThresholdSeconds
    ) {
      if let fresh = freshScreenCapturePayload(now: now, formatter: formatter) {
        screenNow = fresh
        failureCode = nil
        latestCaptureAgeSeconds = 0
      } else if let latestCaptureAgeSeconds, latestCaptureAgeSeconds > staleThresholdSeconds {
        failureCode = .imageUnavailable
        screenNow = [
          "available": false,
          "failure_code": failureCode?.rawValue ?? ScreenContextFailureCode.imageUnavailable.rawValue,
          "latest_capture_age_seconds": latestCaptureAgeSeconds,
          "note":
            "Latest finalized work-context frame was older than \(staleThresholdSeconds) seconds and live capture was unavailable.",
        ]
        ScreenContextToolTelemetry.trackInvariant(
          "stale_inspection_ignored",
          toolName: "get_work_context",
          properties: ["latest_capture_age_seconds": latestCaptureAgeSeconds]
        )
      }
    }

    var timeline: [[String: Any]] = []
    let calendar = Calendar.current
    func clock(_ date: Date) -> String {
      let c = calendar.dateComponents([.hour, .minute], from: date)
      return String(format: "%02d:%02d", c.hour ?? 0, c.minute ?? 0)
    }
    if let shots = try? await RewindDatabase.shared.getScreenshotsSampled(from: start, to: now, targetCount: 80) {
      var runs: [(app: String, window: String, start: String, end: String, frames: Int)] = []
      for shot in shots {
        let window = normalizeWindow(shot.windowTitle ?? "")
        let cl = clock(shot.timestamp)
        if var last = runs.last, last.app == shot.appName, last.window == window {
          last.end = cl
          last.frames += 1
          runs[runs.count - 1] = last
        } else {
          runs.append((shot.appName, window, cl, cl, 1))
        }
      }
      for run in runs.reversed().prefix(20) {
        timeline.append([
          "start": run.start,
          "end": run.end,
          "app": run.app,
          "window": run.window,
          "frames": run.frames,
        ])
      }
    }

    var payload: [String: Any] = [
      "ok": true,
      "name": "get_work_context",
      "window_minutes": minutes,
      "screen_now": screenNow,
      "timeline": timeline,
      "memories_hint": memoriesHint,
      "guidance": handleFirstGuidance,
    ]
    attach(index, to: &payload, now: now)
    if let failureCode {
      payload["failure_code"] = failureCode.rawValue
    }
    if let latestCaptureAgeSeconds {
      payload["latest_capture_age_seconds"] = latestCaptureAgeSeconds
    }
    payload["freshness_threshold_seconds"] = staleThresholdSeconds
    payload["generated_at"] = formatter.string(from: now)
    return payload
  }

  // MARK: - Work-history index (cheap path)

  static let memoriesHint =
    "For the user's operating principles/preferences, also call search_memories (omi-memory)."

  static let handleFirstGuidance =
    "Identify the document, URL, or file the user means from visits[].handles and briefs[].handles, then open or read that source. Timeline runs and screenshot_id are fallback evidence for a question the handles cannot answer; ask for them with include_screen=true. This is recent historical activity, not proof of the current visible screen."

  struct WorkHistorySnapshot: Sendable {
    var visits: [WorkHistoryVisitRecord] = []
    var briefs: [WorkstreamBrief] = []

    /// Whether the index can actually name a source.
    ///
    /// Skipping the tape is only an improvement when a handle is an *address* — a URL or a
    /// file path the model can open. `app_window` is the fallback kind: it carries the same
    /// app and window title the timeline already showed, so returning it alone and
    /// suppressing the timeline would trade evidence for nothing. Measured on this
    /// machine's Beta profile, 483 of 483 handles across 523 visits were `app_window`
    /// (Accessibility was never granted, so no URL or file could be read), which is exactly
    /// the state this guard keeps on today's behavior.
    var hasDurableHandle: Bool {
      visits.contains { $0.handles.contains(where: \.isDurable) }
        || briefs.contains { $0.handles.contains(where: \.isDurable) }
    }
  }

  /// Read the durable work index. This is the only read on the default path, and it is
  /// deliberately independent of Screen Recording and of the Rewind frame store.
  static func workHistoryIndex(start: Date, now: Date) async -> WorkHistorySnapshot {
    guard let queue = await RewindDatabase.shared.getDatabaseQueue() else { return WorkHistorySnapshot() }
    let excluded = RewindSettings.shared.excludedApps
    let visits =
      (try? await queue.read { db in
        try WorkHistoryIndex.fetchRecentVisits(
          in: db, from: start, to: now, limit: 20, excludedApps: excluded)
      }) ?? []
    let briefs =
      (try? await queue.read { db in
        try WorkHistoryIndex.fetchBriefs(in: db, limit: 5, excludedApps: excluded)
      }) ?? []
    return WorkHistorySnapshot(visits: visits, briefs: briefs)
  }

  static func attach(_ index: WorkHistorySnapshot, to payload: inout [String: Any], now: Date) {
    guard !index.visits.isEmpty || !index.briefs.isEmpty else { return }
    let calendar = Calendar.current
    func clock(_ date: Date) -> String {
      let c = calendar.dateComponents([.hour, .minute], from: date)
      return String(format: "%02d:%02d", c.hour ?? 0, c.minute ?? 0)
    }
    payload["visits"] = index.visits.map { $0.jsonObject(clock: clock, now: now) }
    payload["briefs"] = index.briefs.map { $0.jsonObject() }
  }

  static func parseBool(_ value: Any?) -> Bool? {
    if let value = value as? Bool { return value }
    if let value = value as? NSNumber { return value.boolValue }
    if let value = value as? String {
      switch value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() {
      case "true", "1", "yes": return true
      case "false", "0", "no": return false
      default: return nil
      }
    }
    return nil
  }

  static func shouldUseFreshCapture(
    screenNow: [String: Any],
    latestCaptureAgeSeconds: Int?,
    staleThresholdSeconds: Int = staleCaptureThresholdSeconds
  ) -> Bool {
    guard (screenNow["available"] as? Bool) == true else { return true }
    guard let latestCaptureAgeSeconds else { return true }
    return latestCaptureAgeSeconds > staleThresholdSeconds
  }

  static func freshScreenCapturePayload(
    now: Date = Date(),
    formatter: ISO8601DateFormatter = isoFormatter()
  ) -> [String: Any]? {
    guard ScreenCaptureManager.captureScreenData() != nil else { return nil }
    return [
      "available": true,
      "source": "live_capture_stale_rewind",
      "timestamp": formatter.string(from: now),
      "latest_capture_age_seconds": 0,
      "fresh_capture_available": true,
      "raw_image_tool": "capture_screen",
      "note":
        "Fresh live screenshot capture succeeded because the latest finalized work-context frame was missing or stale. Raw pixels are not included here; call capture_screen if the current screen contents matter.",
    ]
  }

  static func permissionDeniedPayload(windowMinutes minutes: Int) -> [String: Any] {
    [
      "ok": false,
      "name": "get_work_context",
      "window_minutes": max(1, min(120, minutes)),
      "failure_code": ScreenContextFailureCode.permissionDenied.rawValue,
      "permission": [
        "screen_recording": "not_granted"
      ],
      "screen_now": [
        "available": false,
        "failure_code": ScreenContextFailureCode.permissionDenied.rawValue,
      ],
      "timeline": [],
      "guidance":
        "Omi does not have Screen Recording permission for current screen access. Tell the user plainly, then call request_permission with type=screen_recording if current screen access is needed.",
      "next_tool": "request_permission",
      "next_tool_arguments": [
        "type": "screen_recording"
      ],
    ]
  }

  static func ambientPayload(from payload: [String: Any]) -> [String: Any] {
    var minimized: [String: Any] = [
      "ok": payload["ok"] as? Bool ?? false,
      "name": "get_work_context",
      "ambient": true,
    ]
    if let failureCode = payload["failure_code"] as? String {
      minimized["failure_code"] = failureCode
    }
    if let screenNow = payload["screen_now"] as? [String: Any] {
      var compactScreen: [String: Any] = [:]
      for key in [
        "available", "app_name", "window_title", "captured_at", "age_seconds", "latest_capture_age_seconds", "source",
      ] {
        if let value = screenNow[key] {
          compactScreen[key] = value
        }
      }
      minimized["screen_now"] = compactScreen
    }
    if let timeline = payload["timeline"] as? [[String: Any]] {
      minimized["timeline_count"] = timeline.count
    }
    return minimized
  }

  static func telemetryValues(from payload: [String: Any]) -> (
    ok: Bool, failureCode: ScreenContextFailureCode?, screenNowAvailable: Bool?, timelineCount: Int?,
    latestCaptureAgeSeconds: Int?, hasOCRPreview: Bool?, imageBytes: Int?
  ) {
    let screenNow = payload["screen_now"] as? [String: Any]
    let screenNowAvailable = screenNow?["available"] as? Bool
    let failureRaw = (payload["failure_code"] as? String) ?? (screenNow?["failure_code"] as? String)
    let timelineCount = (payload["timeline"] as? [[String: Any]])?.count
    let latestAge =
      (screenNow?["latest_capture_age_seconds"] as? Int) ?? (payload["latest_capture_age_seconds"] as? Int)
    let ocr = screenNow?["ocr_preview"] as? String
    return (
      ok: (payload["ok"] as? Bool) == true,
      failureCode: failureRaw.flatMap(ScreenContextFailureCode.init(rawValue:)),
      screenNowAvailable: screenNowAvailable,
      timelineCount: timelineCount,
      latestCaptureAgeSeconds: latestAge,
      hasOCRPreview: ocr.map { !$0.isEmpty },
      imageBytes: screenNow?["image_bytes"] as? Int
    )
  }

  /// One "read a frame's bytes, rebuilding storage if the read finds it gone"
  /// path for every frame consumer — the tape loop here and
  /// `RewindFrameLoader`'s chat surfaces. Duplicating this would let the two
  /// recovery behaviors drift in the worst place.
  static func loadScreenshotDataEnsuringStorage(for screenshot: Screenshot) async throws -> Data {
    do {
      return try await RewindStorage.shared.loadScreenshotData(for: screenshot)
    } catch {
      try await RewindStorage.shared.initialize()
      return try await RewindStorage.shared.loadScreenshotData(for: screenshot)
    }
  }

  private static func parseInt64(_ value: Any?) -> Int64? {
    if let value = value as? Int64 { return value }
    if let value = value as? Int { return Int64(value) }
    if let value = value as? Double { return Int64(value) }
    if let value = value as? String { return Int64(value.trimmingCharacters(in: .whitespacesAndNewlines)) }
    return nil
  }

  private static func normalizeWindow(_ raw: String) -> String {
    var window = String(
      raw.unicodeScalars.filter { scalar in
        let category = scalar.properties.generalCategory
        return category != .format && category != .control
      })
    if let range = window.range(of: #"\s*\(\d{2,}\)\s*$"#, options: .regularExpression) {
      window.removeSubrange(range)
    }
    if let range = window.range(of: #"^\(\d+\)\s*"#, options: .regularExpression) {
      window.removeSubrange(range)
    }
    return window.trimmingCharacters(in: .whitespacesAndNewlines)
  }
}