forked from MakazhanAlpamys/Soup
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_cost.py
More file actions
136 lines (118 loc) · 4.2 KB
/
Copy pathtest_cost.py
File metadata and controls
136 lines (118 loc) · 4.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
"""Tests for soup cost command."""
import json
from typer.testing import CliRunner
from soup_cli.cli import app
runner = CliRunner()
def test_cost_with_config_file(tmp_path):
"""Test basic cost table output."""
config_file = tmp_path / "soup.yaml"
config_file.write_text(
"base: meta-llama/Llama-3.1-8B-Instruct\n"
"task: sft\n"
"data:\n"
" train: ./data/train.jsonl\n"
" max_length: 2048\n"
"training:\n"
" epochs: 3\n"
" batch_size: 4\n"
" quantization: 4bit\n"
" lora:\n"
" r: 64\n"
"output: ./output\n"
)
result = runner.invoke(app, ["cost", "--config", str(config_file)])
assert result.exit_code == 0
assert "Training Cost Estimate" in result.output
assert "Provider" in result.output
assert "RunPod" in result.output
def test_cost_json_output(tmp_path):
"""Test JSON output for automation."""
config_file = tmp_path / "soup.yaml"
config_file.write_text(
"base: meta-llama/Llama-3.1-8B-Instruct\n"
"task: sft\n"
"data:\n"
" train: ./data/train.jsonl\n"
" max_length: 2048\n"
"training:\n"
" batch_size: 4\n"
"output: ./output\n"
)
result = runner.invoke(app, ["cost", "--config", str(config_file), "--json"])
assert result.exit_code == 0
data = json.loads(result.output)
assert isinstance(data, list)
assert len(data) > 0
assert "total_cost" in data[0]
assert "provider" in data[0]
def test_cost_with_gpu_filter(tmp_path):
"""Test filtering by specific GPU."""
config_file = tmp_path / "soup.yaml"
config_file.write_text(
"base: meta-llama/Llama-3.1-8B-Instruct\n"
"task: sft\n"
"data:\n"
" train: ./data/train.jsonl\n"
" max_length: 2048\n"
"training:\n"
" batch_size: 4\n"
"output: ./output\n"
)
result = runner.invoke(app, ["cost", "--config", str(config_file), "--gpu", "H100"])
assert result.exit_code == 0
assert "H100" in result.output
# Filtering to H100 must exclude other GPUs from the pricing table
assert "RTX 4090" not in result.output
def test_cost_with_unknown_gpu(tmp_path):
"""Test filtering by unknown GPU."""
config_file = tmp_path / "soup.yaml"
config_file.write_text(
"base: meta-llama/Llama-3.1-8B-Instruct\n"
"task: sft\n"
"data:\n"
" train: ./data/train.jsonl\n"
" max_length: 2048\n"
"training:\n"
" batch_size: 4\n"
"output: ./output\n"
)
result = runner.invoke(app, ["cost", "--config", str(config_file), "--gpu", "nonexistent"])
assert result.exit_code == 1
assert "No matching GPUs found" in result.output
def test_cost_missing_config():
"""Test cost fails gracefully when config doesn't exist."""
result = runner.invoke(app, ["cost", "--config", "nonexistent.yaml"])
assert result.exit_code != 0
def test_cost_warns_when_dataset_unreadable(tmp_path):
"""When dataset cannot be read, user should see a fallback warning."""
config_file = tmp_path / "soup.yaml"
# train path points nowhere, so the loader cannot read it
config_file.write_text(
"base: meta-llama/Llama-3.1-8B-Instruct\n"
"task: sft\n"
"data:\n"
" train: ./data/missing_train.jsonl\n"
" max_length: 2048\n"
"training:\n"
" batch_size: 4\n"
"output: ./output\n"
)
result = runner.invoke(app, ["cost", "--config", str(config_file)])
assert result.exit_code == 0
assert "Could not read training dataset" in result.output
def test_cost_shows_variance_disclaimer(tmp_path):
"""Table output must include a variance disclaimer for table consumers."""
config_file = tmp_path / "soup.yaml"
config_file.write_text(
"base: meta-llama/Llama-3.1-8B-Instruct\n"
"task: sft\n"
"data:\n"
" train: ./data/train.jsonl\n"
" max_length: 2048\n"
"training:\n"
" batch_size: 4\n"
"output: ./output\n"
)
result = runner.invoke(app, ["cost", "--config", str(config_file)])
assert result.exit_code == 0
assert "estimates are approximate" in result.output