forked from MakazhanAlpamys/Soup
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdisplay.py
More file actions
139 lines (118 loc) · 4.87 KB
/
Copy pathdisplay.py
File metadata and controls
139 lines (118 loc) · 4.87 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
"""Rich live training dashboard in the terminal."""
from typing import Any, Mapping, Optional
from rich.console import Console
from rich.live import Live
from rich.panel import Panel
from soup_cli.config.schema import SoupConfig
console = Console()
def format_gate_row(state: Optional[Mapping[str, Any]]) -> str:
"""Render the eval-gate status row for the live training panel (#36).
Returns an empty string when ``state`` is None or empty (so the row is
hidden when eval-gate is disabled).
Format example::
Gate: helpfulness 7.8 [green]✓[/] | math 0.82 [red]✗[/] (-0.06 from base) | STOP
Pure formatter — no I/O, no side effects — so it is trivially testable
via ``Console(file=StringIO())`` without spinning up a Live display.
"""
if not state:
return ""
tasks = state.get("tasks") or []
if not tasks:
return ""
parts: list[str] = []
for task in tasks:
name = str(task.get("name", "?"))
score = task.get("score")
# Explicit ``is True`` so a missing field renders the ``?`` mark
# rather than the false-y red ✗ (e.g. tasks still pending).
passed = task.get("passed") is True
delta = task.get("delta")
score_str = f"{score:.2f}" if isinstance(score, (int, float)) else "—"
mark = "[green]✓[/]" if passed else "[red]✗[/]"
chunk = f"{name} {score_str} {mark}"
if delta is not None and isinstance(delta, (int, float)):
sign = "+" if delta >= 0 else ""
chunk += f" ({sign}{delta:.2f})"
parts.append(chunk)
body = " | ".join(parts)
action = state.get("action")
suffix = ""
if action == "stop":
suffix = " | [bold red]STOP[/]"
elif action == "warn":
suffix = " | [yellow]WARN[/]"
return f"[bold]Gate:[/] {body}{suffix}"
class TrainingDisplay:
"""Live-updating terminal dashboard for training progress."""
def __init__(self, config: SoupConfig, device_name: str = ""):
self.config = config
self.device_name = device_name
self.current_step = 0
self.total_steps = 0
self.current_epoch = 0
self.loss = 0.0
self.lr = 0.0
self.grad_norm = 0.0
self.gpu_mem = ""
self.speed = 0.0
#: None until an evaluation actually runs. Its own series --
#: never folded into `self.loss`, which is the training curve.
self.val_loss = None
self._live: Optional[Live] = None
def start(self, total_steps: int):
"""Start the live display."""
self.total_steps = total_steps
self._live = Live(self._render(), console=console, refresh_per_second=2)
self._live.start()
def update(self, step: int, epoch: float, loss: float, lr: float, **kwargs):
"""Update display with new metrics."""
self.current_step = step
self.current_epoch = epoch
self.loss = loss
self.lr = lr
self.grad_norm = kwargs.get("grad_norm", 0.0)
self.speed = kwargs.get("speed", 0.0)
self.gpu_mem = kwargs.get("gpu_mem", "")
# Sticky: an evaluation happens every N steps, so the last
# measured value stays on screen between evaluations rather
# than blinking out on every training step.
if kwargs.get("val_loss") is not None:
self.val_loss = kwargs["val_loss"]
if self._live:
self._live.update(self._render())
def stop(self):
"""Stop the live display. Safe to call multiple times."""
if self._live:
self._live.stop()
self._live = None
def _render(self) -> Panel:
"""Render the dashboard panel."""
if self.total_steps > 0:
progress_pct = self.current_step / self.total_steps * 100
else:
progress_pct = 0
bar_width = 30
filled = int(bar_width * progress_pct / 100)
bar = "#" * filled + "-" * (bar_width - filled)
epochs = self.config.training.epochs
epoch_str = f"Epoch {self.current_epoch:.1f}/{epochs}"
lines = []
lines.append(f"{epoch_str} [{bar}] {progress_pct:.0f}%")
lines.append(f"Step: {self.current_step}/{self.total_steps}")
lines.append(f"Loss: {self.loss:.4f} LR: {self.lr:.2e}")
if self.val_loss is not None:
lines.append(f"Val loss: {self.val_loss:.4f}")
if self.speed > 0:
lines.append(f"Speed: {self.speed:.2f} it/s")
if self.gpu_mem:
lines.append(f"GPU peak: {self.gpu_mem}")
if self.grad_norm > 0:
lines.append(f"Grad: {self.grad_norm:.4f}")
content = "\n".join(lines)
name = self.config.experiment_name or self.config.base
return Panel(
content,
title=f"[bold green]Soup Training: {name}[/]",
subtitle=f"[dim]{self.device_name}[/]",
border_style="green",
)