forked from BasedHardware/omi
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_failure_modes.py
More file actions
224 lines (187 loc) · 8.57 KB
/
Copy pathtest_failure_modes.py
File metadata and controls
224 lines (187 loc) · 8.57 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
"""
Scenario 3: Failure Modes
Tests that selected invalid inputs and edge cases behave deterministically.
Full LLM, Redis-unavailable, and STT failure simulations are explicit v2 work.
"""
import pytest
from fakes.firestore import read_conversation, seed_conversation
class TestLLMFailureDegradation:
"""Verify graceful degradation when LLM services are unavailable."""
@pytest.mark.skip(reason="Requires per-test HTTP server reconfiguration — TODO for v2")
def test_llm_500_graceful_degradation(self, client, auth_headers):
"""
When LLM returns 500, conversations should still be saved.
The processing step may fail but the original conversation data
must persist in Firestore.
"""
# This test requires configuring llm_httpserver to return 500
# after a conversation is created, then verifying it's still readable.
# For v1, we verify the pattern exists in code via static analysis.
pass
def test_conversation_persists_after_failed_processing(self, client, auth_headers):
"""
Even if post-processing fails, the raw conversation should be
retrievable. We seed data directly and verify read works.
"""
conv_data = {
"id": "fail-persist-001",
"created_at": "2025-01-15T15:00:00Z",
"started_at": "2025-01-15T15:00:00Z",
"finished_at": "2025-01-15T15:03:00Z",
"source": "omi",
"language": "en",
"structured": {
"title": "",
"overview": "",
"emoji": "🧠",
"category": "other",
"action_items": [],
"events": [],
},
"transcript_segments": [
{
"id": "seg-fail-1",
"text": "This conversation should persist even if processing fails.",
"speaker": "SPEAKER_00",
"is_user": True,
"start": 0.0,
"end": 4.0,
}
],
"discarded": False,
"status": "in_progress",
"is_locked": False,
"data_protection_level": "standard",
}
seed_conversation("123", conv_data)
# Verify readable via API
resp = client.get(f"/v1/conversations/{conv_data['id']}", headers=auth_headers)
assert resp.status_code == 200, f"Conversation should be readable: {resp.text}"
body = resp.json()
assert body["id"] == conv_data["id"]
# Should still have transcript even if not processed
assert len(body.get("transcript_segments", [])) >= 1
class TestRedisFakePaths:
"""Verify CRUD routes work with fakeredis-backed Redis paths."""
def test_crud_works_with_fake_redis(self, client, auth_headers):
"""Basic CRUD operations succeed with fakeredis-backed Redis operations."""
# Create action item (triggers rate limiting + Redis ops)
resp = client.post(
"/v1/action-items",
json={"description": "Redis fail-open test"},
headers=auth_headers,
)
assert resp.status_code == 200, f"Should succeed with fakeredis: {resp.text}"
# Create memory (also uses Redis for caching)
resp = client.post(
"/v3/memories",
json={"content": "Redis fail-open memory", "category": "system"},
headers=auth_headers,
)
assert resp.status_code == 200, f"Memory create should work: {resp.text}"
def test_conversation_list_works_with_fake_redis_cache(self, client, auth_headers):
"""Listing conversations works with fakeredis cache layer."""
resp = client.get("/v1/conversations", headers=auth_headers)
# Should return 200 with empty list or existing conversations
assert resp.status_code == 200, f"List should work: {resp.text}"
def test_voice_duration_lua_uses_fake_redis(self, client, fake_redis):
"""Lua scripts imported outside database.redis_db must stay on FakeRedis."""
from utils import voice_duration_limiter
allowed, used_ms, _remaining_ms = voice_duration_limiter.try_consume_budget('e2e-voice-script', 1000)
assert allowed is True
assert used_ms == 1000
assert fake_redis.zcard('voice_duration:e2e-voice-script') == 1
class TestSTTFailureHandling:
"""Test STT failure/timeout scenarios."""
@pytest.mark.skip(
reason="STT WebSocket fake not yet implemented — " "requires async WS handler simulating Deepgram protocol"
)
def test_stt_timeout_handled_gracefully(self, client, auth_headers):
"""
When Deepgram STT times out, the backend should handle the error
without crashing the connection.
"""
pass
@pytest.mark.skip(reason="STT WebSocket fake not yet implemented")
def test_stt_error_returns_useful_message(self, client, auth_headers):
"""STT errors return a meaningful error to the client."""
pass
class TestInvalidInputHandling:
"""Verify the API rejects malformed input gracefully."""
def test_missing_auth_returns_401(self, client):
"""Requests without auth header get 401."""
resp = client.get("/v1/conversations")
assert resp.status_code == 401
def test_invalid_conversation_id_returns_404(self, client, auth_headers):
"""Non-existent conversation ID returns 404."""
resp = client.get("/v1/conversations/nonexistent-id-12345", headers=auth_headers)
assert resp.status_code == 404
def test_invalid_memory_id_returns_404(self, client, auth_headers):
"""Non-existent memory ID returns 404."""
resp = client.delete("/v3/memories/nonexistent-mem-id", headers=auth_headers)
assert resp.status_code == 404
def test_empty_action_item_description_rejected(self, client, auth_headers):
"""Empty descriptions are rejected by TaskCreate min_length=1 validation."""
resp = client.post(
"/v1/action-items",
json={"description": ""},
headers=auth_headers,
)
assert resp.status_code == 422, resp.text
class TestEdgeCases:
"""Edge cases and boundary conditions."""
def test_unicode_content_roundtrip(self, client, auth_headers):
"""Unicode text survives create→read round-trip."""
unicode_text = "Hello 世界 🌍 Привет 日本語"
resp = client.post(
"/v3/memories",
json={"content": unicode_text, "category": "manual"},
headers=auth_headers,
)
assert resp.status_code == 200, resp.text
mem_id = resp.json()["id"]
resp = client.get("/v3/memories", headers=auth_headers)
assert resp.status_code == 200, resp.text
memories = resp.json()
found = [m for m in memories if m["id"] == mem_id]
assert found, f"Memory {mem_id} not found"
assert found[0]["content"] == unicode_text
def test_long_action_item_description(self, client, auth_headers):
"""Very long descriptions are handled correctly."""
long_desc = "A" * 1000
resp = client.post(
"/v1/action-items",
json={"description": long_desc},
headers=auth_headers,
)
assert resp.status_code == 200, f"Long AI should work: {resp.text}"
ai_id = resp.json()["id"]
read_resp = client.get(f"/v1/action-items/{ai_id}", headers=auth_headers)
assert read_resp.status_code == 200, read_resp.text
assert read_resp.json()["description"] == long_desc
def test_special_characters_in_title(self, client, auth_headers):
"""Special characters in conversation title are preserved."""
conv = {
"id": "special-chars-001",
"created_at": "2025-01-15T16:00:00Z",
"started_at": "2025-01-15T16:00:00Z",
"finished_at": "2025-01-15T16:01:00Z",
"source": "omi",
"structured": {
"title": "Meeting with O'Brien & Associates <script>",
"overview": "",
"emoji": "🧠",
"category": "other",
"action_items": [],
"events": [],
},
"transcript_segments": [],
"discarded": False,
"status": "completed",
"is_locked": False,
}
seed_conversation("123", conv)
resp = client.get(f"/v1/conversations/{conv['id']}", headers=auth_headers)
assert resp.status_code == 200, resp.text
body = resp.json()
assert "O'Brien" in body["structured"]["title"]