b73b5251ae
Full-stack burn-in orchestration dashboard (Stages 1–6d complete): FastAPI backend, SQLite/WAL, SSE live dashboard, mock TrueNAS server, SMTP/webhook notifications, batch burn-in, settings UI, audit log, stats page, cancel SMART/burn-in, drag-to-reorder stages. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
453 lines
18 KiB
Python
453 lines
18 KiB
Python
"""
|
|
Daily status email — sent at smtp_report_hour (local time) every day.
|
|
|
|
Disabled when SMTP_HOST is not set.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import smtplib
|
|
import ssl
|
|
from datetime import datetime, timedelta, timezone
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.text import MIMEText
|
|
|
|
import aiosqlite
|
|
|
|
from app.config import settings
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HTML email template
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _chip(state: str) -> str:
|
|
colours = {
|
|
"PASSED": ("#1a4731", "#3fb950", "#3fb950"),
|
|
"passed": ("#1a4731", "#3fb950", "#3fb950"),
|
|
"FAILED": ("#4b1113", "#f85149", "#f85149"),
|
|
"failed": ("#4b1113", "#f85149", "#f85149"),
|
|
"running": ("#0d2d6b", "#58a6ff", "#58a6ff"),
|
|
"queued": ("#4b3800", "#d29922", "#d29922"),
|
|
"cancelled": ("#222", "#8b949e", "#8b949e"),
|
|
"unknown": ("#222", "#8b949e", "#8b949e"),
|
|
"idle": ("#222", "#8b949e", "#8b949e"),
|
|
"UNKNOWN": ("#222", "#8b949e", "#8b949e"),
|
|
}
|
|
bg, fg, bd = colours.get(state, ("#222", "#8b949e", "#8b949e"))
|
|
label = state.upper()
|
|
return (
|
|
f'<span style="background:{bg};color:{fg};border:1px solid {bd};'
|
|
f'border-radius:4px;padding:2px 8px;font-size:11px;font-weight:600;'
|
|
f'letter-spacing:.04em;white-space:nowrap">{label}</span>'
|
|
)
|
|
|
|
|
|
def _temp_colour(c) -> str:
|
|
if c is None:
|
|
return "#8b949e"
|
|
if c < 40:
|
|
return "#3fb950"
|
|
if c < 50:
|
|
return "#d29922"
|
|
return "#f85149"
|
|
|
|
|
|
def _fmt_bytes(b) -> str:
|
|
if b is None:
|
|
return "—"
|
|
tb = b / 1_000_000_000_000
|
|
if tb >= 1:
|
|
return f"{tb:.0f} TB"
|
|
return f"{b / 1_000_000_000:.0f} GB"
|
|
|
|
|
|
def _fmt_dt(iso: str | None) -> str:
|
|
if not iso:
|
|
return "—"
|
|
try:
|
|
dt = datetime.fromisoformat(iso)
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
return dt.astimezone().strftime("%Y-%m-%d %H:%M")
|
|
except Exception:
|
|
return iso or "—"
|
|
|
|
|
|
def _drive_rows_html(drives: list[dict]) -> str:
|
|
if not drives:
|
|
return '<tr><td colspan="8" style="text-align:center;color:#8b949e;padding:24px">No drives found</td></tr>'
|
|
|
|
rows = []
|
|
for d in drives:
|
|
health = d.get("smart_health") or "UNKNOWN"
|
|
temp = d.get("temperature_c")
|
|
bi = d.get("burnin") or {}
|
|
bi_state = bi.get("state", "—") if bi else "—"
|
|
|
|
short = d.get("smart_short") or {}
|
|
long_ = d.get("smart_long") or {}
|
|
short_state = short.get("state", "idle")
|
|
long_state = long_.get("state", "idle")
|
|
|
|
row_bg = "#1c0a0a" if health == "FAILED" else "#0d1117"
|
|
|
|
rows.append(f"""
|
|
<tr style="background:{row_bg};border-bottom:1px solid #30363d">
|
|
<td style="padding:9px 12px;font-weight:600;color:#c9d1d9">{d.get('devname','—')}</td>
|
|
<td style="padding:9px 12px;color:#8b949e;font-size:12px">{d.get('model','—')}</td>
|
|
<td style="padding:9px 12px;font-family:monospace;font-size:12px;color:#8b949e">{d.get('serial','—')}</td>
|
|
<td style="padding:9px 12px;text-align:right;color:#8b949e">{_fmt_bytes(d.get('size_bytes'))}</td>
|
|
<td style="padding:9px 12px;text-align:right;color:{_temp_colour(temp)};font-weight:500">{f'{temp}°C' if temp is not None else '—'}</td>
|
|
<td style="padding:9px 12px">{_chip(health)}</td>
|
|
<td style="padding:9px 12px">{_chip(short_state)}</td>
|
|
<td style="padding:9px 12px">{_chip(long_state)}</td>
|
|
<td style="padding:9px 12px">{_chip(bi_state) if bi else '—'}</td>
|
|
</tr>""")
|
|
return "\n".join(rows)
|
|
|
|
|
|
def _build_html(drives: list[dict], generated_at: str) -> str:
|
|
total = len(drives)
|
|
failed_drives = [d for d in drives if d.get("smart_health") == "FAILED"]
|
|
running_burnin = [d for d in drives if (d.get("burnin") or {}).get("state") == "running"]
|
|
passed_burnin = [d for d in drives if (d.get("burnin") or {}).get("state") == "passed"]
|
|
|
|
# Alert banner
|
|
alert_html = ""
|
|
if failed_drives:
|
|
names = ", ".join(d["devname"] for d in failed_drives)
|
|
alert_html = f"""
|
|
<div style="background:#4b1113;border:1px solid #f85149;border-radius:6px;padding:14px 18px;margin-bottom:20px;color:#f85149;font-weight:500">
|
|
⚠ SMART health FAILED on {len(failed_drives)} drive(s): {names}
|
|
</div>"""
|
|
|
|
drive_rows = _drive_rows_html(drives)
|
|
|
|
return f"""<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>TrueNAS Burn-In — Daily Report</title>
|
|
</head>
|
|
<body style="margin:0;padding:0;background:#0d1117;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',system-ui,sans-serif;font-size:14px;color:#c9d1d9">
|
|
<table width="100%" cellpadding="0" cellspacing="0" style="background:#0d1117;min-height:100vh">
|
|
<tr><td align="center" style="padding:32px 16px">
|
|
|
|
<table width="700" cellpadding="0" cellspacing="0" style="max-width:700px;width:100%">
|
|
|
|
<!-- Header -->
|
|
<tr>
|
|
<td style="background:#161b22;border:1px solid #30363d;border-radius:10px 10px 0 0;padding:20px 24px;border-bottom:none">
|
|
<table width="100%" cellpadding="0" cellspacing="0">
|
|
<tr>
|
|
<td><span style="font-size:18px;font-weight:700;color:#f0f6fc">TrueNAS Burn-In</span>
|
|
<span style="color:#8b949e;font-size:13px;margin-left:10px">Daily Status Report</span></td>
|
|
<td align="right" style="color:#8b949e;font-size:12px">{generated_at}</td>
|
|
</tr>
|
|
</table>
|
|
</td>
|
|
</tr>
|
|
|
|
<!-- Body -->
|
|
<tr>
|
|
<td style="background:#0d1117;border:1px solid #30363d;border-top:none;border-bottom:none;padding:24px">
|
|
|
|
{alert_html}
|
|
|
|
<!-- Summary chips -->
|
|
<table cellpadding="0" cellspacing="0" style="margin-bottom:24px">
|
|
<tr>
|
|
<td style="padding-right:10px">
|
|
<div style="background:#161b22;border:1px solid #30363d;border-radius:8px;padding:12px 18px;text-align:center;min-width:80px">
|
|
<div style="font-size:24px;font-weight:700;color:#f0f6fc">{total}</div>
|
|
<div style="font-size:11px;color:#8b949e;text-transform:uppercase;letter-spacing:.06em;margin-top:2px">Drives</div>
|
|
</div>
|
|
</td>
|
|
<td style="padding-right:10px">
|
|
<div style="background:#161b22;border:1px solid #30363d;border-radius:8px;padding:12px 18px;text-align:center;min-width:80px">
|
|
<div style="font-size:24px;font-weight:700;color:#f85149">{len(failed_drives)}</div>
|
|
<div style="font-size:11px;color:#8b949e;text-transform:uppercase;letter-spacing:.06em;margin-top:2px">Failed</div>
|
|
</div>
|
|
</td>
|
|
<td style="padding-right:10px">
|
|
<div style="background:#161b22;border:1px solid #30363d;border-radius:8px;padding:12px 18px;text-align:center;min-width:80px">
|
|
<div style="font-size:24px;font-weight:700;color:#58a6ff">{len(running_burnin)}</div>
|
|
<div style="font-size:11px;color:#8b949e;text-transform:uppercase;letter-spacing:.06em;margin-top:2px">Running</div>
|
|
</div>
|
|
</td>
|
|
<td>
|
|
<div style="background:#161b22;border:1px solid #30363d;border-radius:8px;padding:12px 18px;text-align:center;min-width:80px">
|
|
<div style="font-size:24px;font-weight:700;color:#3fb950">{len(passed_burnin)}</div>
|
|
<div style="font-size:11px;color:#8b949e;text-transform:uppercase;letter-spacing:.06em;margin-top:2px">Passed</div>
|
|
</div>
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
|
|
<!-- Drive table -->
|
|
<table width="100%" cellpadding="0" cellspacing="0" style="border:1px solid #30363d;border-radius:8px;overflow:hidden">
|
|
<thead>
|
|
<tr style="background:#161b22">
|
|
<th style="padding:9px 12px;font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:#8b949e;text-align:left;border-bottom:1px solid #30363d">Drive</th>
|
|
<th style="padding:9px 12px;font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:#8b949e;text-align:left;border-bottom:1px solid #30363d">Model</th>
|
|
<th style="padding:9px 12px;font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:#8b949e;text-align:left;border-bottom:1px solid #30363d">Serial</th>
|
|
<th style="padding:9px 12px;font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:#8b949e;text-align:right;border-bottom:1px solid #30363d">Size</th>
|
|
<th style="padding:9px 12px;font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:#8b949e;text-align:right;border-bottom:1px solid #30363d">Temp</th>
|
|
<th style="padding:9px 12px;font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:#8b949e;text-align:left;border-bottom:1px solid #30363d">Health</th>
|
|
<th style="padding:9px 12px;font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:#8b949e;text-align:left;border-bottom:1px solid #30363d">Short</th>
|
|
<th style="padding:9px 12px;font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:#8b949e;text-align:left;border-bottom:1px solid #30363d">Long</th>
|
|
<th style="padding:9px 12px;font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:#8b949e;text-align:left;border-bottom:1px solid #30363d">Burn-In</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{drive_rows}
|
|
</tbody>
|
|
</table>
|
|
|
|
</td>
|
|
</tr>
|
|
|
|
<!-- Footer -->
|
|
<tr>
|
|
<td style="background:#161b22;border:1px solid #30363d;border-top:none;border-radius:0 0 10px 10px;padding:14px 24px;text-align:center">
|
|
<span style="font-size:12px;color:#8b949e">Generated by TrueNAS Burn-In Dashboard · {generated_at}</span>
|
|
</td>
|
|
</tr>
|
|
|
|
</table>
|
|
</td></tr>
|
|
</table>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Send
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Standard ports for each SSL mode — used when smtp_port is not overridden
|
|
_MODE_PORTS: dict[str, int] = {"starttls": 587, "ssl": 465, "plain": 25}
|
|
|
|
|
|
def _smtp_port() -> int:
|
|
"""Derive port from ssl_mode; fall back to settings.smtp_port if explicitly set."""
|
|
mode = (settings.smtp_ssl_mode or "starttls").lower()
|
|
return _MODE_PORTS.get(mode, 587)
|
|
|
|
|
|
def _send_email(subject: str, html: str) -> None:
|
|
recipients = [r.strip() for r in settings.smtp_to.split(",") if r.strip()]
|
|
if not recipients:
|
|
log.warning("SMTP_TO is empty — skipping send")
|
|
return
|
|
|
|
msg = MIMEMultipart("alternative")
|
|
msg["Subject"] = subject
|
|
msg["From"] = settings.smtp_from or settings.smtp_user
|
|
msg["To"] = ", ".join(recipients)
|
|
msg.attach(MIMEText(html, "html", "utf-8"))
|
|
|
|
ctx = ssl.create_default_context()
|
|
mode = (settings.smtp_ssl_mode or "starttls").lower()
|
|
timeout = int(settings.smtp_timeout or 60)
|
|
port = _smtp_port()
|
|
|
|
if mode == "ssl":
|
|
server = smtplib.SMTP_SSL(settings.smtp_host, port, context=ctx, timeout=timeout)
|
|
server.ehlo()
|
|
server.login(settings.smtp_user, settings.smtp_password)
|
|
server.sendmail(msg["From"], recipients, msg.as_string())
|
|
server.quit()
|
|
else:
|
|
with smtplib.SMTP(settings.smtp_host, port, timeout=timeout) as server:
|
|
server.ehlo()
|
|
if mode == "starttls":
|
|
server.starttls(context=ctx)
|
|
server.ehlo()
|
|
server.login(settings.smtp_user, settings.smtp_password)
|
|
server.sendmail(msg["From"], recipients, msg.as_string())
|
|
|
|
log.info("Email sent to %s", recipients)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Data fetch
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def _fetch_report_data() -> list[dict]:
|
|
"""Pull drives + latest burnin state from DB."""
|
|
from app.routes import _fetch_drives_for_template # local import avoids circular
|
|
async with aiosqlite.connect(settings.db_path) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
await db.execute("PRAGMA journal_mode=WAL")
|
|
return await _fetch_drives_for_template(db)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scheduler
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _build_alert_html(
|
|
job_id: int,
|
|
devname: str,
|
|
serial: str | None,
|
|
model: str | None,
|
|
state: str,
|
|
error_text: str | None,
|
|
generated_at: str,
|
|
) -> str:
|
|
is_fail = state == "failed"
|
|
color = "#f85149" if is_fail else "#3fb950"
|
|
bg = "#4b1113" if is_fail else "#1a4731"
|
|
icon = "✕" if is_fail else "✓"
|
|
|
|
error_section = ""
|
|
if error_text:
|
|
error_section = f"""
|
|
<div style="background:#4b1113;border:1px solid #f85149;border-radius:6px;
|
|
padding:12px 16px;margin-top:16px;color:#f85149;font-size:13px">
|
|
<strong>Error:</strong> {error_text}
|
|
</div>"""
|
|
|
|
return f"""<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head><meta charset="UTF-8"><title>Burn-In {state.title()} Alert</title></head>
|
|
<body style="margin:0;padding:0;background:#0d1117;font-family:-apple-system,sans-serif;
|
|
font-size:14px;color:#c9d1d9">
|
|
<table width="100%" cellpadding="0" cellspacing="0">
|
|
<tr><td align="center" style="padding:32px 16px">
|
|
<table width="480" cellpadding="0" cellspacing="0" style="max-width:480px;width:100%">
|
|
<tr>
|
|
<td style="background:{bg};border:2px solid {color};border-radius:10px;padding:24px">
|
|
<div style="font-size:26px;font-weight:700;color:{color};margin-bottom:16px">
|
|
{icon} Burn-In {state.upper()}
|
|
</div>
|
|
<table cellpadding="0" cellspacing="0" style="width:100%">
|
|
<tr>
|
|
<td style="color:#8b949e;font-size:12px;padding:5px 0">Device</td>
|
|
<td style="font-weight:600;text-align:right;font-size:15px">{devname}</td>
|
|
</tr>
|
|
<tr>
|
|
<td style="color:#8b949e;font-size:12px;padding:5px 0">Model</td>
|
|
<td style="text-align:right">{model or '—'}</td>
|
|
</tr>
|
|
<tr>
|
|
<td style="color:#8b949e;font-size:12px;padding:5px 0">Serial</td>
|
|
<td style="font-family:monospace;text-align:right">{serial or '—'}</td>
|
|
</tr>
|
|
<tr>
|
|
<td style="color:#8b949e;font-size:12px;padding:5px 0">Job #</td>
|
|
<td style="font-family:monospace;text-align:right">{job_id}</td>
|
|
</tr>
|
|
</table>
|
|
{error_section}
|
|
<div style="margin-top:16px;font-size:11px;color:#8b949e">{generated_at}</div>
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
</td></tr>
|
|
</table>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
async def send_job_alert(
|
|
job_id: int,
|
|
devname: str,
|
|
serial: str | None,
|
|
model: str | None,
|
|
state: str,
|
|
error_text: str | None,
|
|
) -> None:
|
|
"""Send an immediate per-job alert email (pass or fail)."""
|
|
icon = "✕" if state == "failed" else "✓"
|
|
subject = f"{icon} Burn-In {state.upper()}: {devname} ({serial or 'no serial'})"
|
|
now_str = datetime.now().strftime("%Y-%m-%d %H:%M")
|
|
html = _build_alert_html(job_id, devname, serial, model, state, error_text, now_str)
|
|
await asyncio.to_thread(_send_email, subject, html)
|
|
|
|
|
|
async def test_smtp_connection() -> dict:
|
|
"""
|
|
Try to establish an SMTP connection using current settings.
|
|
Returns {"ok": True/False, "error": str|None}.
|
|
Does NOT send any email.
|
|
"""
|
|
if not settings.smtp_host:
|
|
return {"ok": False, "error": "SMTP_HOST is not configured"}
|
|
|
|
def _test() -> dict:
|
|
try:
|
|
ctx = ssl.create_default_context()
|
|
mode = (settings.smtp_ssl_mode or "starttls").lower()
|
|
timeout = int(settings.smtp_timeout or 60)
|
|
port = _smtp_port()
|
|
|
|
if mode == "ssl":
|
|
server = smtplib.SMTP_SSL(settings.smtp_host, port,
|
|
context=ctx, timeout=timeout)
|
|
server.ehlo()
|
|
else:
|
|
server = smtplib.SMTP(settings.smtp_host, port, timeout=timeout)
|
|
server.ehlo()
|
|
if mode == "starttls":
|
|
server.starttls(context=ctx)
|
|
server.ehlo()
|
|
|
|
if settings.smtp_user:
|
|
server.login(settings.smtp_user, settings.smtp_password)
|
|
server.quit()
|
|
return {"ok": True, "error": None}
|
|
except Exception as exc:
|
|
return {"ok": False, "error": str(exc)}
|
|
|
|
return await asyncio.to_thread(_test)
|
|
|
|
|
|
async def send_report_now() -> None:
|
|
"""Send a report immediately (used by on-demand API endpoint)."""
|
|
drives = await _fetch_report_data()
|
|
now_str = datetime.now().strftime("%Y-%m-%d %H:%M")
|
|
html = _build_html(drives, now_str)
|
|
subject = f"Burn-In Report — {datetime.now().strftime('%Y-%m-%d')} ({len(drives)} drives)"
|
|
await asyncio.to_thread(_send_email, subject, html)
|
|
|
|
|
|
async def run() -> None:
|
|
"""Background loop: send daily report at smtp_report_hour local time."""
|
|
if not settings.smtp_host:
|
|
log.info("SMTP not configured — daily email disabled")
|
|
return
|
|
|
|
log.info(
|
|
"Mailer started — daily report at %02d:00 local time",
|
|
settings.smtp_report_hour,
|
|
)
|
|
|
|
while True:
|
|
now = datetime.now()
|
|
target = now.replace(
|
|
hour=settings.smtp_report_hour,
|
|
minute=0, second=0, microsecond=0,
|
|
)
|
|
if target <= now:
|
|
target += timedelta(days=1)
|
|
|
|
wait = (target - now).total_seconds()
|
|
log.info("Next report in %.0f seconds (%s)", wait, target.strftime("%Y-%m-%d %H:%M"))
|
|
await asyncio.sleep(wait)
|
|
|
|
if settings.smtp_daily_report_enabled:
|
|
try:
|
|
await send_report_now()
|
|
except Exception as exc:
|
|
log.error("Failed to send daily report: %s", exc)
|
|
else:
|
|
log.info("Daily report skipped — smtp_daily_report_enabled is False")
|
|
|
|
# Sleep briefly past the hour to avoid drift from re-triggering immediately
|
|
await asyncio.sleep(60)
|