forked from BasedHardware/omi
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathResourceMonitor.swift
More file actions
796 lines (680 loc) · 29.3 KB
/
Copy pathResourceMonitor.swift
File metadata and controls
796 lines (680 loc) · 29.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
import AppKit
import Foundation
import Sentry
private func boundedDiagnosticTag(_ value: Any?) -> String {
switch value {
case let value as String:
return value
case let value as Bool:
return value ? "true" : "false"
default:
return "unknown"
}
}
enum MemoryPressureLevel: String, Equatable, Sendable {
case nominal
case warning
case critical
case extreme
}
struct MemoryPressureDecision: Equatable, Sendable {
let previousLevel: MemoryPressureLevel
let level: MemoryPressureLevel
let shouldReportCritical: Bool
let shouldRemediate: Bool
var reportPhase: String {
previousLevel == .critical || previousLevel == .extreme ? "sustained" : "entered"
}
}
/// Owns one memory-pressure episode independently from the sampling timer.
///
/// The old monitor used only a five-minute timestamp cooldown. A process that
/// remained above 800 MB consequently opened the same Sentry issue every five
/// minutes forever, while a small dip around the threshold could start another
/// apparent episode. Hysteresis gives recovery one authoritative owner, and a
/// slower sustained reminder preserves visibility for pressure that never
/// clears without flooding triage.
struct MemoryPressureEpisodeTracker: Sendable {
private(set) var level: MemoryPressureLevel = .nominal
private var lastCriticalReportAt: Date?
private var lastRemediationAt: Date?
let warningThresholdMB: UInt64
let criticalThresholdMB: UInt64
let extremeThresholdMB: UInt64
let warningRecoveryMB: UInt64
let criticalRecoveryMB: UInt64
let sustainedReportInterval: TimeInterval
let remediationInterval: TimeInterval
init(
warningThresholdMB: UInt64 = 500,
criticalThresholdMB: UInt64 = 800,
extremeThresholdMB: UInt64 = 3000,
warningRecoveryMB: UInt64 = 450,
criticalRecoveryMB: UInt64 = 720,
sustainedReportInterval: TimeInterval = 15 * 60,
remediationInterval: TimeInterval = 5 * 60
) {
precondition(warningRecoveryMB < warningThresholdMB)
precondition(criticalRecoveryMB < criticalThresholdMB)
precondition(warningThresholdMB < criticalThresholdMB)
precondition(criticalThresholdMB < extremeThresholdMB)
self.warningThresholdMB = warningThresholdMB
self.criticalThresholdMB = criticalThresholdMB
self.extremeThresholdMB = extremeThresholdMB
self.warningRecoveryMB = warningRecoveryMB
self.criticalRecoveryMB = criticalRecoveryMB
self.sustainedReportInterval = sustainedReportInterval
self.remediationInterval = remediationInterval
}
mutating func evaluate(memoryFootprintMB: UInt64, at now: Date) -> MemoryPressureDecision {
let previous = level
let next = nextLevel(for: memoryFootprintMB)
level = next
guard next == .critical else {
if next == .nominal || next == .warning {
lastCriticalReportAt = nil
lastRemediationAt = nil
}
return MemoryPressureDecision(
previousLevel: previous,
level: next,
shouldReportCritical: false,
shouldRemediate: false)
}
let enteredCritical = previous != .critical && previous != .extreme
let shouldReport =
enteredCritical
|| lastCriticalReportAt.map { now.timeIntervalSince($0) >= sustainedReportInterval } ?? true
let shouldRemediate =
enteredCritical
|| lastRemediationAt.map { now.timeIntervalSince($0) >= remediationInterval } ?? true
if shouldReport {
lastCriticalReportAt = now
}
if shouldRemediate {
lastRemediationAt = now
}
return MemoryPressureDecision(
previousLevel: previous,
level: next,
shouldReportCritical: shouldReport,
shouldRemediate: shouldRemediate)
}
private func nextLevel(for memoryFootprintMB: UInt64) -> MemoryPressureLevel {
if memoryFootprintMB >= extremeThresholdMB {
return .extreme
}
if memoryFootprintMB >= criticalThresholdMB {
return .critical
}
if level == .critical || level == .extreme, memoryFootprintMB >= criticalRecoveryMB {
return .critical
}
if memoryFootprintMB >= warningThresholdMB {
return .warning
}
if level == .warning, memoryFootprintMB >= warningRecoveryMB {
return .warning
}
return .nominal
}
}
/// Monitors system resources (memory, CPU, disk) and reports to Sentry
@MainActor
class ResourceMonitor {
static let shared = ResourceMonitor()
/// Check if this is a non-production build (avoids Sentry calls in test apps)
private let isDevBuild: Bool = AppBuild.isNonProduction
// MARK: - Configuration
/// How often to sample resources (seconds)
private let sampleInterval: TimeInterval = 30
/// Memory threshold (MB) - warn when exceeded
private var memoryWarningThreshold: UInt64 { memoryPressureTracker.warningThresholdMB }
/// Memory threshold (MB) - critical alert
private var memoryCriticalThreshold: UInt64 { memoryPressureTracker.criticalThresholdMB }
/// Memory growth rate threshold (MB/min) - detect leaks
private let memoryGrowthRateThreshold: Double = 50
/// Extreme memory threshold - auto-restart to prevent system from becoming unusable.
/// Keep this well below the point where free RAM is exhausted — at 4GB the system
/// has ~120MB free and the new instance fails to launch, leaving the user stuck.
/// At 3GB there is still ~10-13GB free on typical 16GB machines.
private var memoryAutoRestartThreshold: UInt64 { memoryPressureTracker.extremeThresholdMB }
// MARK: - State
private var monitorTimer: Timer?
private var isMonitoring = false
private var memorySamples: [(timestamp: Date, memoryMB: UInt64)] = []
private let maxSamples = 20 // Keep last 20 samples for trend analysis
private var lastWarningTime: Date?
private var peakMemoryObserved: UInt64 = 0 // Track peak memory manually
private var autoRestartTriggered = false // Only auto-restart once per session
private var memoryPressureTracker = MemoryPressureEpisodeTracker()
// Minimum time between warnings (prevent spam)
private let warningCooldown: TimeInterval = 300 // 5 minutes
private init() {}
// MARK: - Public API
/// Start monitoring resources
func start() {
guard !isMonitoring else { return }
isMonitoring = true
log("ResourceMonitor: Starting resource monitoring (interval: \(Int(sampleInterval))s)")
// Take initial sample
Task {
await sampleResources()
}
// Start periodic sampling
monitorTimer = Timer.scheduledTimer(withTimeInterval: sampleInterval, repeats: true) {
[weak self] _ in
Task { @MainActor in
await self?.sampleResources()
}
}
}
/// Stop monitoring resources
func stop() {
guard isMonitoring else { return }
isMonitoring = false
monitorTimer?.invalidate()
monitorTimer = nil
memorySamples.removeAll()
memoryPressureTracker = MemoryPressureEpisodeTracker()
lastWarningTime = nil
log("ResourceMonitor: Stopped resource monitoring")
}
/// Get current resource snapshot
func getCurrentResources() -> ResourceSnapshot {
return ResourceSnapshot(
memoryUsageMB: getMemoryUsageMB(),
memoryFootprintMB: getMemoryFootprintMB(),
peakMemoryMB: getPeakMemoryMB(),
memoryPercent: getMemoryPercentage(),
totalSystemRAM_MB: getTotalSystemRAM(),
systemMemoryPressure: getSystemMemoryPressure(),
cpuUsage: getCPUUsage(),
diskUsedGB: getDiskUsedGB(),
diskFreeGB: getDiskFreeGB(),
threadCount: getThreadCount(),
timestamp: Date()
)
}
/// Manually report current resources to Sentry (call before known heavy operations)
func reportResourcesNow(context: String) {
let snapshot = getCurrentResources()
// Add as breadcrumb (skip in dev builds)
if !isDevBuild {
let breadcrumb = Breadcrumb(level: .info, category: "resources")
breadcrumb.message =
"[\(context)] Memory: \(snapshot.memoryUsageMB)MB, Footprint: \(snapshot.memoryFootprintMB)MB, CPU: \(String(format: "%.1f", snapshot.cpuUsage))%"
breadcrumb.data = snapshot.asDictionary()
SentrySDK.addBreadcrumb(breadcrumb)
}
log("ResourceMonitor: [\(context)] \(snapshot.summary)")
}
// MARK: - Private Methods
private func sampleResources() async {
let snapshot = getCurrentResources()
// Store memory sample for trend analysis
memorySamples.append((timestamp: snapshot.timestamp, memoryMB: snapshot.memoryFootprintMB))
if memorySamples.count > maxSamples {
memorySamples.removeFirst()
}
// Update Sentry context with current resources
updateSentryContext(snapshot)
await updateRuntimeActivityContext()
// Check for issues
await checkMemoryThresholds(snapshot)
checkMemoryGrowthRate()
// Log periodically (every 5th sample = ~2.5 min)
if memorySamples.count % 5 == 0 {
log("ResourceMonitor: \(snapshot.summary)")
}
// Log per-component memory diagnostics every 10th sample (~5 min)
if memorySamples.count % 10 == 0 {
await logComponentDiagnostics(snapshot: snapshot)
}
}
/// Collect and log per-component memory diagnostics to help identify leak sources
@discardableResult
private func logComponentDiagnostics(snapshot: ResourceSnapshot) async -> [String: Any] {
var components: [String: Any] = [:]
// LiveNotesMonitor buffers (MainActor — direct access)
let liveNotes = LiveNotesMonitor.shared
components["liveNotes_wordBuffer"] = liveNotes.wordBufferCount
components["liveNotes_notesContext"] = liveNotes.existingNotesContextCount
components["liveNotes_notesCount"] = liveNotes.notes.count
// VideoChunkEncoder buffer/lifecycle (actor — await)
let bufferStatus = await VideoChunkEncoder.shared.getBufferStatus()
components["videoEncoder_frameCount"] = bufferStatus.frameCount
components["videoEncoder_maxBufferFrames"] = bufferStatus.maxBufferFrames
components["videoEncoder_isEncoderRunning"] = bufferStatus.isEncoderRunning
components["videoEncoder_consecutiveWriteFailures"] = bufferStatus.consecutiveWriteFailures
components["videoEncoder_restartCount"] = bufferStatus.encoderRestartCount
components["videoEncoder_emergencyResetCount"] = bufferStatus.emergencyResetCount
components["videoEncoder_writerNotReadyCount"] = bufferStatus.writerNotReadyCount
components["videoEncoder_lifecyclePhase"] = bufferStatus.lifecyclePhase
components["videoEncoder_queueBucket"] = bufferStatus.queueBucket
components["videoEncoder_isInitialized"] = bufferStatus.isInitialized
components["videoEncoder_hasStalenessTimer"] = bufferStatus.hasStalenessTimer
components["videoEncoder_finalizationWaiterBucket"] = bufferStatus.finalizationWaiterBucket
if let age = bufferStatus.oldestFrameAge {
components["videoEncoder_oldestFrameAgeSec"] = Int(age)
}
if let age = bufferStatus.currentChunkAge {
components["videoEncoder_currentChunkAgeSec"] = Int(age)
}
// FocusAssistant pending tasks (actor — await, optional since it may not be initialized)
if let focusAssistant = ProactiveAssistantsPlugin.shared.currentFocusAssistant {
components["focus_pendingTasks"] = await focusAssistant.pendingTasksCount
components["focus_historyCount"] = await focusAssistant.analysisHistoryCount
}
// Rewind backpressure stats (MainActor — direct access)
let plugin = ProactiveAssistantsPlugin.shared
components["rewind_droppedFrames"] = plugin.droppedFrameCount
components["rewind_isProcessing"] = plugin.isProcessingRewindFrame
components["rewind_isMonitoring"] = plugin.isMonitoring
components["rewind_captureIntervalSec"] = RewindSettings.shared.captureInterval
components["rewind_effectiveCaptureIntervalSec"] = RewindSettings.shared.effectiveCaptureInterval(
isOnBattery: PowerMonitor.cachedBatteryState())
components["rewind_retentionDays"] = RewindSettings.shared.retentionDays
components["system_memoryPressurePercent"] = snapshot.systemMemoryPressure
// Thread count is already in snapshot
components["threadCount"] = snapshot.threadCount
let componentSummary = components.map { "\($0.key)=\($0.value)" }.sorted().joined(
separator: ", ")
log("ResourceMonitor: COMPONENTS: \(componentSummary)")
// Add to Sentry context for crash diagnostics
if !isDevBuild {
SentrySDK.configureScope { scope in
scope.setContext(value: components, key: "memory_components")
}
// Add breadcrumb when memory is elevated
if snapshot.memoryFootprintMB >= memoryWarningThreshold {
let breadcrumb = Breadcrumb(level: .warning, category: "memory_diagnostics")
breadcrumb.message = "Component diagnostics at \(snapshot.memoryFootprintMB)MB"
breadcrumb.data = components
SentrySDK.addBreadcrumb(breadcrumb)
}
}
return components
}
private func updateSentryContext(_ snapshot: ResourceSnapshot) {
// Set resource context that will be attached to all future events (skip in dev builds)
guard !isDevBuild else { return }
SentrySDK.configureScope { scope in
scope.setContext(value: snapshot.asDictionary(), key: "resources")
}
}
/// Keep bounded owner/phase context fresh for SDK-generated events such as
/// app hangs. Full component counters are logged less often, but a five-minute
/// old writer phase is not useful when classifying a three-second hang.
private func updateRuntimeActivityContext() async {
guard !isDevBuild else { return }
let encoder = await VideoChunkEncoder.shared.getBufferStatus()
let plugin = ProactiveAssistantsPlugin.shared
let activity: [String: Any] = [
"video_encoder_phase": encoder.lifecyclePhase,
"video_encoder_queue": encoder.queueBucket,
"video_encoder_finalization_waiters": encoder.finalizationWaiterBucket,
"video_encoder_initialized": encoder.isInitialized,
"video_encoder_staleness_timer": encoder.hasStalenessTimer,
"rewind_monitoring": plugin.isMonitoring,
"rewind_processing_frame": plugin.isProcessingRewindFrame,
]
SentrySDK.configureScope { scope in
scope.setTag(value: encoder.lifecyclePhase, key: "video_encoder_phase")
scope.setTag(value: encoder.queueBucket, key: "video_encoder_queue")
scope.setTag(
value: encoder.finalizationWaiterBucket,
key: "video_encoder_finalization_waiters")
scope.setTag(
value: plugin.isMonitoring ? "true" : "false",
key: "rewind_monitoring")
scope.setContext(value: activity, key: "runtime_activity")
}
}
private func checkMemoryThresholds(_ snapshot: ResourceSnapshot) async {
let now = Date()
let decision = memoryPressureTracker.evaluate(
memoryFootprintMB: snapshot.memoryFootprintMB,
at: now)
// Extreme threshold - auto-restart to prevent the system from becoming unresponsive.
// Without this, memory can climb to 7GB+, causing SQLite I/O failures and making
// the app impossible to reopen without a full computer restart.
if snapshot.memoryFootprintMB >= memoryAutoRestartThreshold && !autoRestartTriggered
&& !isDevBuild
{
autoRestartTriggered = true
log(
"ResourceMonitor: EXTREME memory \(snapshot.memoryFootprintMB)MB — auto-restarting to prevent system degradation"
)
SentrySDK.capture(message: "App Auto-Restarting Due to Extreme Memory") { scope in
scope.setLevel(.fatal)
scope.setTag(value: "auto_restart", key: "resource_alert")
scope.setContext(value: snapshot.asDictionary(), key: "resources")
}
// Give Sentry 3 seconds to flush, then relaunch and terminate.
// Only terminate if the relaunch succeeds — otherwise the user would be
// left with no running app and would need a full computer restart.
DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) {
let task = Process()
task.launchPath = "/usr/bin/open"
task.arguments = ["-n", Bundle.main.bundleURL.path]
do {
try task.run()
NSApp.terminate(nil)
} catch {
logError(
"ResourceMonitor: Failed to relaunch app during auto-restart, aborting terminate to avoid leaving user stuck",
error: error)
self.autoRestartTriggered = false // Allow retry on next threshold check
}
}
return
}
// Critical threshold
if decision.level == .critical {
if decision.shouldReportCritical || decision.shouldRemediate {
// Collect before capture. Previously this ran in an unstructured Task
// racing the Sentry event, so the event that needed component ownership
// most often carried the previous sample's context (or none).
let components = await logComponentDiagnostics(snapshot: snapshot)
if decision.shouldRemediate {
triggerMemoryRemediation()
}
if decision.shouldReportCritical {
log(
"ResourceMonitor: CRITICAL - Memory usage \(snapshot.memoryFootprintMB)MB exceeds \(memoryCriticalThreshold)MB threshold"
)
// Send Sentry event (skip in dev builds)
if !isDevBuild {
let threshold = self.memoryCriticalThreshold
SentrySDK.capture(message: "Critical Memory Usage") { scope in
scope.setLevel(.error)
scope.setFingerprint(["resource-monitor", "memory-critical"])
scope.setTag(value: "memory_critical", key: "resource_alert")
scope.setTag(value: decision.reportPhase, key: "memory_episode_phase")
scope.setTag(
value: boundedDiagnosticTag(components["videoEncoder_lifecyclePhase"]),
key: "video_encoder_phase")
scope.setTag(
value: boundedDiagnosticTag(components["videoEncoder_queueBucket"]),
key: "video_encoder_queue")
scope.setTag(
value: boundedDiagnosticTag(components["rewind_isMonitoring"]),
key: "rewind_monitoring")
scope.setContext(value: snapshot.asDictionary(), key: "resources")
scope.setContext(value: components, key: "memory_components")
scope.setContext(
value: [
"threshold_mb": threshold,
"current_mb": snapshot.memoryFootprintMB,
"peak_mb": snapshot.peakMemoryMB,
], key: "memory_details")
}
}
}
}
}
// Warning threshold
else if snapshot.memoryFootprintMB >= memoryWarningThreshold {
if lastWarningTime == nil || now.timeIntervalSince(lastWarningTime!) > warningCooldown {
lastWarningTime = now
log(
"ResourceMonitor: WARNING - Memory usage \(snapshot.memoryFootprintMB)MB exceeds \(memoryWarningThreshold)MB threshold"
)
// Add warning breadcrumb (skip in dev builds)
if !isDevBuild {
let breadcrumb = Breadcrumb(level: .warning, category: "resources")
breadcrumb.message = "High memory usage: \(snapshot.memoryFootprintMB)MB"
breadcrumb.data = snapshot.asDictionary()
SentrySDK.addBreadcrumb(breadcrumb)
}
}
}
}
private func checkMemoryGrowthRate() {
guard memorySamples.count >= 5 else { return }
// Calculate growth rate over last 5 samples
let recentSamples = Array(memorySamples.suffix(5))
guard let first = recentSamples.first, let last = recentSamples.last else { return }
let timeDiffMinutes = last.timestamp.timeIntervalSince(first.timestamp) / 60.0
guard timeDiffMinutes > 0 else { return }
let memoryGrowthMB = Double(Int64(last.memoryMB) - Int64(first.memoryMB))
let growthRateMBPerMin = memoryGrowthMB / timeDiffMinutes
// Detect potential memory leak
if growthRateMBPerMin > memoryGrowthRateThreshold {
log(
"ResourceMonitor: WARNING - Memory growing at \(String(format: "%.1f", growthRateMBPerMin))MB/min (potential leak)"
)
// Add breadcrumb (skip in dev builds)
if !isDevBuild {
let breadcrumb = Breadcrumb(level: .warning, category: "resources")
breadcrumb.message =
"Potential memory leak detected: \(String(format: "%.1f", growthRateMBPerMin))MB/min growth rate"
breadcrumb.data = [
"growth_rate_mb_per_min": growthRateMBPerMin,
"samples_analyzed": recentSamples.count,
"time_span_minutes": timeDiffMinutes,
"start_memory_mb": first.memoryMB,
"end_memory_mb": last.memoryMB,
]
SentrySDK.addBreadcrumb(breadcrumb)
}
}
}
// MARK: - Memory Remediation
/// Attempt to free memory by flushing heavy components.
/// Called at most once per warningCooldown (5 min) when critical threshold is exceeded.
/// Closure called during memory remediation to trim transcript state.
/// Set by AppState on init to avoid tight coupling.
var onMemoryPressureTrimTranscript: (() -> Void)?
private func triggerMemoryRemediation() {
log(
"ResourceMonitor: Triggering memory remediation — flushing video encoder, clearing assistant pending work, trimming transcript, pausing AgentSync"
)
let memoryBefore = getMemoryFootprintMB()
// Clear queued frames in assistant coordinator
AssistantCoordinator.shared.clearAllPendingWork()
// Trim in-memory transcript segments (already persisted in SQLite)
onMemoryPressureTrimTranscript?()
Task {
// Flush VideoChunkEncoder and await completion; ResourceMonitor records
// reset counters in component diagnostics so hang/memory Sentry events can
// be correlated.
do {
_ = try await RewindStorage.shared.flushCurrentVideoChunk()
} catch {
logError("ResourceMonitor: Failed to flush video chunk during memory remediation", error: error)
}
// Clear focus assistant pending tasks specifically
if let focusAssistant = ProactiveAssistantsPlugin.shared.currentFocusAssistant {
await focusAssistant.clearPendingWork()
}
// Pause AgentSync to reduce memory pressure and resume after 60s
await AgentSyncService.shared.pause()
Task {
try? await Task.sleep(nanoseconds: 60_000_000_000) // 60s
await AgentSyncService.shared.resume()
log("ResourceMonitor: AgentSync resumed after 60s cooldown")
}
let memoryAfter = await MainActor.run { self.getMemoryFootprintMB() }
log("ResourceMonitor: Memory remediation completed — \(memoryBefore)MB -> \(memoryAfter)MB")
}
if !isDevBuild {
let breadcrumb = Breadcrumb(level: .warning, category: "memory_remediation")
breadcrumb.message = "Memory remediation triggered at critical threshold"
breadcrumb.data = [
"memory_footprint_mb": memoryBefore,
"threshold_mb": memoryCriticalThreshold,
]
SentrySDK.addBreadcrumb(breadcrumb)
}
}
// MARK: - Resource Getters (macOS specific)
/// Get current memory usage in MB (resident set size)
private func getMemoryUsageMB() -> UInt64 {
var info = mach_task_basic_info()
var count = mach_msg_type_number_t(MemoryLayout<mach_task_basic_info>.size) / 4
let result = withUnsafeMutablePointer(to: &info) {
$0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) {
task_info(mach_task_self_, task_flavor_t(MACH_TASK_BASIC_INFO), $0, &count)
}
}
if result == KERN_SUCCESS {
return info.resident_size / (1024 * 1024)
}
return 0
}
/// Get physical memory footprint in MB (more accurate for macOS)
private func getMemoryFootprintMB() -> UInt64 {
var info = task_vm_info_data_t()
var count = mach_msg_type_number_t(MemoryLayout<task_vm_info>.size) / 4
let result = withUnsafeMutablePointer(to: &info) {
$0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) {
task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count)
}
}
if result == KERN_SUCCESS {
return UInt64(info.phys_footprint) / (1024 * 1024)
}
return getMemoryUsageMB() // Fallback
}
/// Get peak memory usage in MB (tracked manually since phys_footprint_peak unavailable)
private func getPeakMemoryMB() -> UInt64 {
let current = getMemoryFootprintMB()
if current > peakMemoryObserved {
peakMemoryObserved = current
}
return peakMemoryObserved
}
/// Get CPU usage percentage (0-100+, can exceed 100% on multi-core)
private func getCPUUsage() -> Double {
var threadList: thread_act_array_t?
var threadCount: mach_msg_type_number_t = 0
guard task_threads(mach_task_self_, &threadList, &threadCount) == KERN_SUCCESS,
let threads = threadList
else {
return 0
}
defer {
vm_deallocate(
mach_task_self_, vm_address_t(bitPattern: threads),
vm_size_t(threadCount) * vm_size_t(MemoryLayout<thread_t>.size))
}
var totalCPU: Double = 0
for i in 0..<Int(threadCount) {
var info = thread_basic_info()
var count = mach_msg_type_number_t(
MemoryLayout<thread_basic_info>.size / MemoryLayout<natural_t>.size)
let result = withUnsafeMutablePointer(to: &info) {
$0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) {
thread_info(threads[i], thread_flavor_t(THREAD_BASIC_INFO), $0, &count)
}
}
if result == KERN_SUCCESS && (info.flags & TH_FLAGS_IDLE) == 0 {
totalCPU += Double(info.cpu_usage) / Double(TH_USAGE_SCALE) * 100.0
}
}
return totalCPU
}
/// Get disk space used in GB
private func getDiskUsedGB() -> Double {
let homeDir = FileManager.default.homeDirectoryForCurrentUser
do {
let values = try homeDir.resourceValues(forKeys: [
.volumeTotalCapacityKey, .volumeAvailableCapacityKey,
])
let total = values.volumeTotalCapacity ?? 0
let available = values.volumeAvailableCapacity ?? 0
return Double(total - available) / (1024 * 1024 * 1024)
} catch {
return 0
}
}
/// Get disk space free in GB
private func getDiskFreeGB() -> Double {
let homeDir = FileManager.default.homeDirectoryForCurrentUser
do {
let values = try homeDir.resourceValues(forKeys: [.volumeAvailableCapacityKey])
return Double(values.volumeAvailableCapacity ?? 0) / (1024 * 1024 * 1024)
} catch {
return 0
}
}
/// Get current thread count
private func getThreadCount() -> Int {
var threadList: thread_act_array_t?
var threadCount: mach_msg_type_number_t = 0
guard task_threads(mach_task_self_, &threadList, &threadCount) == KERN_SUCCESS,
let threads = threadList
else {
return 0
}
vm_deallocate(
mach_task_self_, vm_address_t(bitPattern: threads),
vm_size_t(threadCount) * vm_size_t(MemoryLayout<thread_t>.size))
return Int(threadCount)
}
/// Get total system RAM in MB
private func getTotalSystemRAM() -> UInt64 {
return UInt64(ProcessInfo.processInfo.physicalMemory) / (1024 * 1024)
}
/// Get app's memory usage as percentage of total system RAM
private func getMemoryPercentage() -> Double {
let totalRAM = getTotalSystemRAM()
guard totalRAM > 0 else { return 0 }
let footprint = getMemoryFootprintMB()
return (Double(footprint) / Double(totalRAM)) * 100.0
}
/// Get system-wide memory pressure (percentage of total RAM in use by all apps)
private func getSystemMemoryPressure() -> Double {
var stats = vm_statistics64()
var count = mach_msg_type_number_t(
MemoryLayout<vm_statistics64>.size / MemoryLayout<integer_t>.size)
let result = withUnsafeMutablePointer(to: &stats) {
$0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) {
host_statistics64(mach_host_self(), HOST_VM_INFO64, $0, &count)
}
}
guard result == KERN_SUCCESS else { return 0 }
let pageSize = UInt64(getpagesize())
let totalRAM = ProcessInfo.processInfo.physicalMemory
// Active + Wired + Compressed = memory in use
let activeBytes = UInt64(stats.active_count) * pageSize
let wiredBytes = UInt64(stats.wire_count) * pageSize
let compressedBytes = UInt64(stats.compressor_page_count) * pageSize
let usedBytes = activeBytes + wiredBytes + compressedBytes
return (Double(usedBytes) / Double(totalRAM)) * 100.0
}
}
// MARK: - Resource Snapshot
struct ResourceSnapshot {
let memoryUsageMB: UInt64 // Resident set size
let memoryFootprintMB: UInt64 // Physical footprint (more accurate)
let peakMemoryMB: UInt64 // Peak memory since launch
let memoryPercent: Double // App memory as % of total RAM
let totalSystemRAM_MB: UInt64 // Total system RAM
let systemMemoryPressure: Double // System-wide RAM usage %
let cpuUsage: Double // CPU percentage
let diskUsedGB: Double // Disk used
let diskFreeGB: Double // Disk free
let threadCount: Int // Number of threads
let timestamp: Date
var summary: String {
"Memory: \(memoryFootprintMB)MB/\(totalSystemRAM_MB / 1024)GB (\(String(format: "%.2f", memoryPercent))%), System RAM: \(String(format: "%.1f", systemMemoryPressure))% used, CPU: \(String(format: "%.1f", cpuUsage))%, Threads: \(threadCount)"
}
func asDictionary() -> [String: Any] {
return [
"memory_usage_mb": memoryUsageMB,
"memory_footprint_mb": memoryFootprintMB,
"peak_memory_mb": peakMemoryMB,
"memory_percent": memoryPercent,
"total_system_ram_mb": totalSystemRAM_MB,
"system_memory_pressure_percent": systemMemoryPressure,
"cpu_usage_percent": cpuUsage,
"disk_used_gb": diskUsedGB,
"disk_free_gb": diskFreeGB,
"thread_count": threadCount,
"timestamp": ISO8601DateFormatter().string(from: timestamp),
]
}
}