forked from BasedHardware/omi
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathvoice_recorder_provider.dart
More file actions
552 lines (471 loc) · 18.5 KB
/
Copy pathvoice_recorder_provider.dart
File metadata and controls
552 lines (471 loc) · 18.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
import 'dart:async';
import 'dart:io';
import 'dart:math' as math;
import 'dart:typed_data';
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:path_provider/path_provider.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:omi/backend/http/api/messages.dart';
import 'package:omi/backend/preferences.dart';
import 'package:omi/app_globals.dart';
import 'package:omi/services/services.dart';
import 'package:omi/utils/alerts/app_snackbar.dart';
import 'package:omi/utils/audio/wav_bytes.dart';
import 'package:omi/utils/logger.dart';
import 'package:omi/utils/l10n_extensions.dart';
enum VoiceRecorderState { idle, recording, transcribing, transcribeSuccess, transcribeFailed, pendingRecovery }
typedef VoiceMessageTranscriber = Future<String> Function(List<File> audioFiles);
class VoiceRecorderProvider extends ChangeNotifier {
static const _wavPathKey = 'voice_recorder_pending_wav_path';
/// Maximum WAV chunk size in bytes (10 MB of PCM data per chunk).
/// Each chunk gets its own 44-byte WAV header so the backend can decode it independently.
static const maxChunkPcmBytes = 10 * 1024 * 1024;
VoiceRecorderState _state = VoiceRecorderState.idle;
String _transcript = '';
bool _isProcessing = false;
// Disk-based recording: PCM chunks stream to a temp file instead of RAM
IOSink? _pcmSink;
File? _pcmFile;
int _pcmBytesWritten = 0;
// Persisted WAV file for retry (kept until transcription succeeds or user closes)
File? _wavFile;
// Audio visualization — more bars give the wave a denser, more "speech-like"
// look that matches modern voice-mode designs.
final List<double> _audioLevels = List.generate(50, (_) => 0.05);
Timer? _waveformTimer;
// Callbacks for UI integration
Function(String transcript, bool autoSend)? _onTranscriptReady;
VoidCallback? _onClose;
// Set by the caller (e.g. tapping the send button mid-recording) before
// processRecording() — instructs the chat page to send immediately after
// filling the text field instead of waiting for a manual send tap.
bool _autoSendRequested = false;
void requestAutoSendOnNextTranscript() {
_autoSendRequested = true;
}
final VoiceMessageTranscriber _transcribeVoiceMessage;
// Injected only by tests — ServiceManager is a singleton initialised from
// native plumbing, so the mic is resolved lazily at call time otherwise.
final IMicRecorderService? _micOverride;
VoiceRecorderProvider({VoiceMessageTranscriber? transcriber, IMicRecorderService? mic})
: _transcribeVoiceMessage = transcriber ?? transcribeVoiceMessage,
_micOverride = mic;
IMicRecorderService get _mic => _micOverride ?? ServiceManager.instance().mic;
VoiceRecorderState get state => _state;
String get transcript => _transcript;
bool get isProcessing => _isProcessing;
List<double> get audioLevels => List.unmodifiable(_audioLevels);
bool get isRecording => _state == VoiceRecorderState.recording;
bool get isActive => _state != VoiceRecorderState.idle;
bool get hasPendingRecording => _state == VoiceRecorderState.pendingRecovery;
/// Check for a WAV file persisted from a previous session.
/// Call this on app startup to recover interrupted recordings.
Future<void> checkPendingRecording() async {
final path = SharedPreferencesUtil().getString(_wavPathKey);
if (path.isEmpty) return;
final file = File(path);
if (file.existsSync()) {
_wavFile = file;
_state = VoiceRecorderState.pendingRecovery;
notifyListeners();
} else {
// File was cleaned up externally — clear the stale preference
await SharedPreferencesUtil().remove(_wavPathKey);
}
}
void setCallbacks({Function(String transcript, bool autoSend)? onTranscriptReady, VoidCallback? onClose}) {
_onTranscriptReady = onTranscriptReady;
_onClose = onClose;
}
void clearCallbacks() {
_onTranscriptReady = null;
_onClose = null;
}
static const _audioSessionChannel = MethodChannel('com.omi.ios/audioSession');
Future<void> startRecording() async {
if (_state == VoiceRecorderState.recording) return;
_state = VoiceRecorderState.recording;
_transcript = '';
_pcmBytesWritten = 0;
// Clean up any previous WAV file
await _cleanupWavFile();
// Create a persisted PCM file for streaming audio to disk.
final recordingsDir = await _recordingsDirectory();
_pcmFile = File('${recordingsDir.path}/voice_recording_${DateTime.now().millisecondsSinceEpoch}.pcm');
// Create the file before opening the sink. IOSink.openWrite() may defer
// file creation until the first event-loop turn, which can leave the
// recording session with no visible PCM file immediately after a
// successful start under load.
await _pcmFile!.create();
_pcmSink = _pcmFile!.openWrite();
// Reset audio levels
for (int i = 0; i < _audioLevels.length; i++) {
_audioLevels[i] = 0.1;
}
notifyListeners();
await Permission.microphone.request();
// Configure audio session for Bluetooth before starting recorder.
if (Platform.isIOS) {
try {
await _audioSessionChannel.invokeMethod('configureForBluetooth');
} catch (e) {
Logger.debug('VoiceRecorderProvider: Failed to configure audio session for Bluetooth: $e');
}
}
// Repaint at ~60Hz so the wave flows smoothly. Levels are shifted in
// onByteReceived (audio callback rate, much faster than the UI), but the
// canvas only re-renders on notifyListeners — so a slower timer made the
// wave appear frozen / laggy.
_waveformTimer = Timer.periodic(const Duration(milliseconds: 16), (_) {
if (_state == VoiceRecorderState.recording) {
notifyListeners();
}
});
try {
await _mic.start(
onByteReceived: (bytes) {
if (_state == VoiceRecorderState.recording) {
// Write to disk instead of accumulating in RAM
_pcmSink?.add(bytes);
_pcmBytesWritten += bytes.length;
// Update audio visualization based on actual audio levels
if (bytes.isNotEmpty) {
double rms = 0;
for (int i = 0; i < bytes.length - 1; i += 2) {
int sample = bytes[i] | (bytes[i + 1] << 8);
if (sample > 32767) {
sample = sample - 65536;
}
rms += sample * sample;
}
int sampleCount = bytes.length ~/ 2;
if (sampleCount > 0) {
rms = math.sqrt(rms / sampleCount) / 32768.0;
} else {
rms = 0;
}
// Wider dynamic range so quiet sections stay near zero and loud
// peaks reach the full bar height. The 0.5 exponent boosts mid
// levels so normal speech has visible amplitude.
final level = math.pow(rms, 0.5).toDouble().clamp(0.02, 1.0);
for (int i = 0; i < _audioLevels.length - 1; i++) {
_audioLevels[i] = _audioLevels[i + 1];
}
_audioLevels[_audioLevels.length - 1] = level;
}
}
},
onRecording: () {
Logger.debug('VoiceRecorderProvider: Recording started');
_state = VoiceRecorderState.recording;
// Reset audio levels
for (int i = 0; i < _audioLevels.length; i++) {
_audioLevels[i] = 0.1;
}
notifyListeners();
},
onStop: () {
Logger.debug('VoiceRecorderProvider: Recording stopped');
},
onInitializing: () {
Logger.debug('VoiceRecorderProvider: Initializing');
},
);
} catch (e) {
// Mic contention (a conversation already holds the recorder) or a native
// start failure. Never rethrows: this runs from a fire-and-forget tap
// handler, so an escaping error is an unhandled async crash.
Logger.debug('VoiceRecorderProvider: failed to start recording: $e');
await _unwindFailedStart();
}
}
/// Undo everything [startRecording] armed before handing the mic over — the
/// waveform timer, the open PCM sink and its file — and return to idle so the
/// chat bar drops back to the text field instead of wedging in `recording`.
Future<void> _unwindFailedStart() async {
_waveformTimer?.cancel();
_waveformTimer = null;
try {
await _pcmSink?.close();
} catch (e) {
Logger.debug('Error closing PCM sink after failed start: $e');
}
_pcmSink = null;
await _cleanupPcmFile();
_pcmBytesWritten = 0;
_autoSendRequested = false;
_state = VoiceRecorderState.idle;
for (int i = 0; i < _audioLevels.length; i++) {
_audioLevels[i] = 0.1;
}
notifyListeners();
}
void stopRecording() {
_waveformTimer?.cancel();
_mic.stop();
}
Future<void> processRecording() async {
if (_isProcessing) return;
_state = VoiceRecorderState.transcribing;
_isProcessing = true;
notifyListeners();
stopRecording();
try {
// Flush and close the PCM sink
await _pcmSink?.flush();
await _pcmSink?.close();
_pcmSink = null;
// Check minimum audio length (0.5 seconds at 16kHz PCM16 = 16000 bytes)
const int minAudioBytes = 16000;
final pcmLength = _pcmFile != null && _pcmFile!.existsSync() ? await _pcmFile!.length() : _pcmBytesWritten;
if (pcmLength < minAudioBytes) {
Logger.debug('Audio too short ($pcmLength bytes), closing without error');
close();
return;
}
// Convert PCM file to WAV file (reads from disk, writes to disk — no full-file RAM copy)
// Keep PCM file until WAV is confirmed on disk — if conversion fails, PCM is the only copy
_wavFile = await _convertPcmFileToWavFile(_pcmFile!, 16000, 1);
// Persist WAV path so user can retry if the app is closed
await SharedPreferencesUtil().saveString(_wavPathKey, _wavFile!.path);
// WAV conversion succeeded — safe to delete PCM file now
await _cleanupPcmFile();
// Split into chunks if the WAV is large, then transcribe
final chunks = await splitWavFileIfNeeded(_wavFile!, 16000, 1);
try {
final transcript = await _transcribeVoiceMessage(chunks);
if (transcript.trim().isNotEmpty) {
_transcript = transcript;
_state = VoiceRecorderState.transcribeSuccess;
_isProcessing = false;
notifyListeners();
final autoSend = _autoSendRequested;
_autoSendRequested = false;
_onTranscriptReady?.call(transcript, autoSend);
close();
} else {
Logger.debug('Empty transcript received; preserving recording for retry');
_markTranscriptionFailed();
}
} finally {
_cleanupChunkFiles(chunks);
}
} catch (e) {
Logger.debug('Error processing recording: $e');
// Only clean up PCM if WAV exists (conversion succeeded).
// If WAV conversion failed, keep PCM as the only surviving copy.
if (_wavFile != null && _wavFile!.existsSync()) {
await _cleanupPcmFile();
}
_state = VoiceRecorderState.transcribeFailed;
_isProcessing = false;
notifyListeners();
_showTranscriptionFailedSnackbar();
}
}
Future<void> retry() async {
if (_wavFile != null && _wavFile!.existsSync()) {
// Retry transcription with existing WAV file on disk (no re-encoding needed)
await _retryTranscription();
} else if (_pcmFile != null && _pcmFile!.existsSync()) {
// WAV conversion failed but PCM survived — retry from PCM
await processRecording();
} else {
await startRecording();
}
}
Future<void> _retryTranscription() async {
if (_isProcessing) return;
_state = VoiceRecorderState.transcribing;
_isProcessing = true;
notifyListeners();
try {
final chunks = await splitWavFileIfNeeded(_wavFile!, 16000, 1);
try {
final transcript = await _transcribeVoiceMessage(chunks);
if (transcript.trim().isNotEmpty) {
_transcript = transcript;
_state = VoiceRecorderState.transcribeSuccess;
_isProcessing = false;
notifyListeners();
final autoSend = _autoSendRequested;
_autoSendRequested = false;
_onTranscriptReady?.call(transcript, autoSend);
close();
} else {
Logger.debug('Empty transcript received on retry; preserving recording for retry');
_markTranscriptionFailed();
}
} finally {
_cleanupChunkFiles(chunks);
}
} catch (e) {
Logger.debug('Error retrying transcription: $e');
_state = VoiceRecorderState.transcribeFailed;
_isProcessing = false;
notifyListeners();
_showTranscriptionFailedSnackbar();
}
}
static Future<Directory> _recordingsDirectory() async {
final supportDir = await getApplicationSupportDirectory();
final directory = Directory('${supportDir.path}/voice_recordings');
if (!directory.existsSync()) {
await directory.create(recursive: true);
}
return directory;
}
/// Convert a PCM file on disk to a WAV file on disk.
/// Reads and writes in chunks to avoid loading the entire file into memory.
static Future<File> _convertPcmFileToWavFile(File pcmFile, int sampleRate, int channels) async {
final pcmLength = await pcmFile.length();
final wavHeader = WavBytesUtil.getWavHeader(pcmLength, sampleRate, channelCount: channels);
final recordingsDir = await _recordingsDirectory();
final wavPath = '${recordingsDir.path}/voice_recording_${DateTime.now().millisecondsSinceEpoch}.wav';
final wavFile = File(wavPath);
final sink = wavFile.openWrite();
// Write 44-byte WAV header
sink.add(wavHeader);
// Stream PCM data in chunks — never loads entire file into RAM
final reader = pcmFile.openRead();
await for (final chunk in reader) {
sink.add(chunk);
}
await sink.flush();
await sink.close();
Logger.debug('WAV file created: $wavPath (${pcmLength + 44} bytes from $pcmLength PCM bytes)');
return wavFile;
}
Future<void> _cleanupPcmFile() async {
try {
if (_pcmFile != null && _pcmFile!.existsSync()) {
await _pcmFile!.delete();
}
} catch (e) {
Logger.debug('Error cleaning up PCM file: $e');
}
_pcmFile = null;
}
Future<void> _cleanupWavFile() async {
try {
if (_wavFile != null && _wavFile!.existsSync()) {
await _wavFile!.delete();
}
} catch (e) {
Logger.debug('Error cleaning up WAV file: $e');
}
_wavFile = null;
await SharedPreferencesUtil().remove(_wavPathKey);
}
/// Split a WAV file into multiple chunk files if it exceeds [maxChunkPcmBytes].
/// Each chunk is a standalone WAV file with its own header.
/// Returns the original file in a single-element list if no splitting is needed.
@visibleForTesting
static Future<List<File>> splitWavFileIfNeeded(File wavFile, int sampleRate, int channels) async {
final fileLength = await wavFile.length();
final pcmLength = fileLength - 44; // subtract WAV header
if (pcmLength <= maxChunkPcmBytes) {
return [wavFile];
}
final chunks = <File>[];
final tempDir = await getTemporaryDirectory();
final timestamp = DateTime.now().millisecondsSinceEpoch;
final reader = wavFile.openRead(44); // skip the original WAV header
int chunkIndex = 0;
int pcmBytesRemaining = pcmLength;
final buffer = BytesBuilder(copy: false);
await for (final data in reader) {
buffer.add(data);
while (buffer.length >= maxChunkPcmBytes && pcmBytesRemaining > 0) {
final chunkBytes = buffer.takeBytes();
final chunkPcmSize = chunkBytes.length < maxChunkPcmBytes ? chunkBytes.length : maxChunkPcmBytes;
final chunkFile = File('${tempDir.path}/voice_chunk_${timestamp}_$chunkIndex.wav');
final sink = chunkFile.openWrite();
sink.add(WavBytesUtil.getWavHeader(chunkPcmSize, sampleRate, channelCount: channels));
sink.add(chunkBytes.sublist(0, chunkPcmSize));
await sink.flush();
await sink.close();
chunks.add(chunkFile);
pcmBytesRemaining -= chunkPcmSize;
chunkIndex++;
// Put leftover bytes back
if (chunkBytes.length > chunkPcmSize) {
buffer.add(chunkBytes.sublist(chunkPcmSize));
}
}
}
// Write remaining bytes as the final chunk
if (buffer.length > 0) {
final remaining = buffer.takeBytes();
final chunkFile = File('${tempDir.path}/voice_chunk_${timestamp}_$chunkIndex.wav');
final sink = chunkFile.openWrite();
sink.add(WavBytesUtil.getWavHeader(remaining.length, sampleRate, channelCount: channels));
sink.add(remaining);
await sink.flush();
await sink.close();
chunks.add(chunkFile);
}
Logger.debug('Split WAV into ${chunks.length} chunks from $pcmLength PCM bytes');
return chunks;
}
/// Clean up chunk files after transcription (but not the original WAV).
void _cleanupChunkFiles(List<File> chunks) {
for (final chunk in chunks) {
// Don't delete the original WAV file — only delete split chunks
if (chunk.path != _wavFile?.path) {
try {
if (chunk.existsSync()) chunk.deleteSync();
} catch (e) {
Logger.debug('Error cleaning up chunk file: $e');
}
}
}
}
void _markTranscriptionFailed() {
_state = VoiceRecorderState.transcribeFailed;
_isProcessing = false;
notifyListeners();
_showTranscriptionFailedSnackbar();
}
void _showTranscriptionFailedSnackbar() {
final context = globalNavigatorKey.currentContext;
if (context == null) return;
AppSnackbar.showSnackbarError(context.l10n.voiceFailedToTranscribe);
}
void close() {
if (_state == VoiceRecorderState.idle) {
return;
}
if (_state == VoiceRecorderState.recording) {
stopRecording();
}
_waveformTimer?.cancel();
_state = VoiceRecorderState.idle;
_transcript = '';
_isProcessing = false;
_pcmBytesWritten = 0;
_autoSendRequested = false;
// Close PCM sink if still open
_pcmSink?.close();
_pcmSink = null;
// Clean up temp files
_cleanupPcmFile();
_cleanupWavFile();
// Reset audio levels
for (int i = 0; i < _audioLevels.length; i++) {
_audioLevels[i] = 0.1;
}
notifyListeners();
_onClose?.call();
}
@override
void dispose() {
_waveformTimer?.cancel();
_pcmSink?.close();
if (_state == VoiceRecorderState.recording) {
_mic.stop();
}
super.dispose();
}
}