forked from BasedHardware/omi
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathrewind.ts
More file actions
159 lines (153 loc) · 7.25 KB
/
Copy pathrewind.ts
File metadata and controls
159 lines (153 loc) · 7.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
import { ipcMain, BrowserWindow } from 'electron'
import { readFile } from 'fs/promises'
import { resolve, sep } from 'path'
import { getPrimarySourceId } from '../rewind/sourceId'
import {
listRewindFrames,
listRewindFramesSampled,
searchRewindFrames,
rewindDayBounds,
rewindFrameCount,
getRewindFrameOcrLines,
searchRewindEmbeddings,
rewindFramesByIds
} from './db'
import { groupFrames } from '../rewind/rewindGrouping'
import { configureRewindEmbedSession, embedRewindQuery } from '../rewind/embeddingService'
import { mergeRewindSearchResults, type VectorHit } from '../rewind/vectorSearchMerge'
import {
getRewindSettings,
updateRewindSettings,
ingestRewindFrame
} from '../rewind/captureService'
import { getCaptureDirective } from '../rewind/captureDirective'
import { pruneRewindOnce } from '../rewind/retentionRunner'
import { rebuildRewindIndexFromDisk } from '../rewind/rebuildIndex'
import { rewindRoot } from '../rewind/paths'
import type { RewindSettings } from '../../shared/types'
/** How many semantic neighbours to pull before the similarity floor + the
* already-in-FTS filter thin them out. */
const VECTOR_TOP_K = 50
/**
* Semantic hits for a query, or [] when semantic search is unavailable (signed
* out, embedding backend down, nothing indexed yet). Never throws: on macOS the
* whole vector leg is a `try?`, and keyword results must render regardless.
*/
async function vectorHits(query: string): Promise<VectorHit[]> {
try {
const vec = await embedRewindQuery(query)
if (!vec) return []
const scored = await searchRewindEmbeddings(vec, VECTOR_TOP_K)
const frames = rewindFramesByIds(scored.map((s) => s.frameId))
const byId = new Map(frames.map((f) => [f.id, f]))
return scored
.map((s) => {
const frame = byId.get(s.frameId)
return frame ? { frame, similarity: s.similarity } : null
})
.filter((h): h is VectorHit => h !== null)
} catch (e) {
console.warn(`[rewind-embed] vector search failed, keyword-only: ${(e as Error).message}`)
return []
}
}
// Monotonic id for the newest search. The vector leg is slow and its result is
// delivered out-of-band, so a stale one must never overwrite a newer query's
// results (type "invoice", then "receipt": invoice's vectors land last).
let searchSeq = 0
export function registerRewindHandlers(): void {
ipcMain.handle('rewind:frames', async (_e, from: number, to: number) =>
listRewindFrames(from, to)
)
// A day's frames, evenly down-sampled to ~500 (macOS parity + row-limit backstop).
// The day-scoped timeline loads through this; 'rewind:frames' stays the unsampled
// primitive for the small incremental live-append.
ipcMain.handle('rewind:framesSampled', async (_e, from: number, to: number) =>
listRewindFramesSampled(from, to)
)
ipcMain.handle('rewind:dayBounds', async () => rewindDayBounds())
ipcMain.handle('rewind:frameCount', async () => rewindFrameCount())
// Hybrid search, in TWO PHASES.
//
// Phase 1 (this handler, synchronous): keyword results (FTS5/BM25), returned
// immediately. Phase 2 (below, out-of-band): the same list with semantic hits
// merged in, pushed on 'rewind:search-results' when — and if — they arrive.
//
// Keyword search must NEVER wait on the network, and it used to: the handler
// awaited the query embedding, which is up to 3 attempts x a 30s timeout plus
// backoff — about 91 seconds on a captive-portal/flaky network. The FTS rows
// were sitting in hand from the first millisecond the whole time, and the user
// stared at an empty result list. Vector search is ADDITIVE recall; its failure
// is supposed to degrade silently to keyword-only (macOS wraps the whole leg in
// `try?`), which is only true if keyword results don't depend on it.
ipcMain.handle('rewind:search', async (e, query: string) => {
const q = query.trim()
if (!q) return []
const seq = ++searchSeq
const fts = searchRewindFrames(q)
// Fire-and-forget: nothing about the reply below depends on this resolving,
// and it must never reject into the handler.
void (async () => {
const hits = await vectorHits(q)
if (hits.length === 0) return // keyword-only; the phase-1 reply already stands
if (seq !== searchSeq) return // a newer query has since been issued
if (e.sender.isDestroyed()) return
// Which frames were keyword hits, so groupFrames can flag the purely-semantic
// groups (those that exist only because vector recall added them).
const keywordIds = new Set(fts.map((f) => f.id).filter((id): id is number => id != null))
e.sender.send('rewind:search-results', {
query: q,
groups: groupFrames(mergeRewindSearchResults(fts, hits), q, { keywordIds })
})
})()
return groupFrames(fts, q)
})
// Relay of the renderer's Firebase session — the embedding indexer and the
// query embedder are inert without it (the token only exists in the renderer).
ipcMain.handle(
'rewind:setEmbedSession',
async (_e, s: { desktopApiBase: string; token: string } | null) =>
configureRewindEmbedSession(s)
)
// --- Track 4 --- Per-line OCR bounding boxes for the search highlight overlay.
ipcMain.handle('rewind:frameOcrLines', async (_e, frameId: number) =>
getRewindFrameOcrLines(frameId)
)
ipcMain.handle('rewind:frameImage', async (_e, imagePath: string) => {
const root = resolve(rewindRoot())
const full = resolve(imagePath)
if (full !== root && !full.startsWith(root + sep)) {
throw new Error('invalid frame path')
}
const buf = await readFile(full)
return `data:image/jpeg;base64,${buf.toString('base64')}`
})
ipcMain.handle('rewind:getSettings', async () => getRewindSettings())
ipcMain.handle('rewind:setSettings', async (_e, next: RewindSettings) => {
updateRewindSettings(next)
const current = getRewindSettings()
// Notify the renderer capture host so it can start/stop the stream and
// re-pace immediately, without waiting for a re-mount or a poll.
for (const w of BrowserWindow.getAllWindows()) {
w.webContents.send('rewind:settings', current)
}
return current
})
// Current runtime capture directive (pause + effective cadence). The capture
// host fetches this on mount, then reacts to pushes on 'rewind:capture-directive'.
ipcMain.handle('rewind:getCaptureDirective', async () => getCaptureDirective())
ipcMain.handle('rewind:pruneNow', async () => pruneRewindOnce())
// Recovery affordance: re-create rewind_frames rows for the JPEGs still on disk
// after a whole-DB reset/recovery wiped them (surfaced from DbRecoveryNotice).
// Only ever INSERTs missing rows — never deletes, idempotent. Returns the count.
ipcMain.handle('rewind:rebuildIndex', async () => rebuildRewindIndexFromDisk())
// Cached primary-screen id. The underlying desktopCapturer.getSources() can
// take several seconds on some machines, so it's prewarmed at startup; this
// is an instant cache hit in the normal case.
ipcMain.handle('rewind:primarySourceId', async () => getPrimarySourceId())
// Receive a sampled JPEG frame from the renderer capture host and store it
// (after foreground-window metadata + idle/lock/dup gating).
ipcMain.handle('rewind:saveFrame', async (_e, data: Uint8Array) =>
ingestRewindFrame(Buffer.from(data))
)
}