"""
Daily status email — sent at smtp_report_hour (local time) every day.
Disabled when SMTP_HOST is not set.
"""
import asyncio
import html
import logging
import smtplib
import ssl
from datetime import datetime, timedelta, timezone
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import aiosqlite
from app.config import settings
log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# HTML email template
# ---------------------------------------------------------------------------
def _chip(state: str) -> str:
colours = {
"PASSED": ("#1a4731", "#3fb950", "#3fb950"),
"passed": ("#1a4731", "#3fb950", "#3fb950"),
"FAILED": ("#4b1113", "#f85149", "#f85149"),
"failed": ("#4b1113", "#f85149", "#f85149"),
"running": ("#0d2d6b", "#58a6ff", "#58a6ff"),
"queued": ("#4b3800", "#d29922", "#d29922"),
"cancelled": ("#222", "#8b949e", "#8b949e"),
"unknown": ("#222", "#8b949e", "#8b949e"),
"idle": ("#222", "#8b949e", "#8b949e"),
"UNKNOWN": ("#222", "#8b949e", "#8b949e"),
}
bg, fg, bd = colours.get(state, ("#222", "#8b949e", "#8b949e"))
label = state.upper()
return (
f'{label}'
)
def _temp_colour(c) -> str:
if c is None:
return "#8b949e"
if c < 40:
return "#3fb950"
if c < 50:
return "#d29922"
return "#f85149"
def _fmt_bytes(b) -> str:
if b is None:
return "—"
tb = b / 1_000_000_000_000
if tb >= 1:
return f"{tb:.0f} TB"
return f"{b / 1_000_000_000:.0f} GB"
def _fmt_dt(iso: str | None) -> str:
if not iso:
return "—"
try:
dt = datetime.fromisoformat(iso)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt.astimezone().strftime("%Y-%m-%d %H:%M")
except Exception:
return iso or "—"
def _drive_rows_html(drives: list[dict]) -> str:
if not drives:
return '| No drives found |
'
rows = []
for d in drives:
health = d.get("smart_health") or "UNKNOWN"
temp = d.get("temperature_c")
bi = d.get("burnin") or {}
bi_state = bi.get("state", "—") if bi else "—"
short = d.get("smart_short") or {}
long_ = d.get("smart_long") or {}
short_state = short.get("state", "idle")
long_state = long_.get("state", "idle")
row_bg = "#1c0a0a" if health == "FAILED" else "#0d1117"
rows.append(f"""
| {d.get('devname','—')} |
{d.get('model','—')} |
{d.get('serial','—')} |
{_fmt_bytes(d.get('size_bytes'))} |
{f'{temp}°C' if temp is not None else '—'} |
{_chip(health)} |
{_chip(short_state)} |
{_chip(long_state)} |
{_chip(bi_state) if bi else '—'} |
""")
return "\n".join(rows)
def _build_unlock_banner_html(events: list[dict]) -> str:
"""Banner listing every pool-drive unlock granted in the last 24h.
Every interpolated DB field is run through html.escape — operator and
reason are free-text from the unlock modal and otherwise inject into
the email body verbatim.
"""
if not events:
return ""
rows = []
for e in events:
evt = e.get("event_type") or ""
is_boot = evt == "boot_pool_drive_unlocked"
is_exported = evt == "exported_pool_drive_unlocked"
is_mounted = evt == "mounted_drive_unlocked"
kind = (
"BOOT POOL" if is_boot
else "EXPORTED ZFS" if is_exported
else "MOUNTED FILESYSTEM" if is_mounted
else "pool"
)
when = html.escape((e.get("created_at") or "")[:19])
operator = html.escape(e.get("operator") or "?")
devname = html.escape(e.get("devname") or "?")
# `message` already includes pool name, devname, and the operator's
# reason — surface it verbatim so the audit trail is faithful.
message = html.escape(e.get("message") or "")
rows.append(
f"{when} · "
f"{operator} unlocked a {kind} drive "
f"({devname}): "
f"{message}"
)
return f"""
⚠ {len(events)} pool-drive unlock(s) in the last 24h
"""
def _build_html(drives: list[dict], generated_at: str,
unlock_events: list[dict] | None = None) -> str:
total = len(drives)
failed_drives = [d for d in drives if d.get("smart_health") == "FAILED"]
running_burnin = [d for d in drives if (d.get("burnin") or {}).get("state") == "running"]
passed_burnin = [d for d in drives if (d.get("burnin") or {}).get("state") == "passed"]
# Alert banners (unlock events first — the audit-grade signal)
alert_html = _build_unlock_banner_html(unlock_events or [])
if failed_drives:
names = ", ".join(d["devname"] for d in failed_drives)
alert_html += f"""
⚠ SMART health FAILED on {len(failed_drives)} drive(s): {names}
"""
drive_rows = _drive_rows_html(drives)
return f"""
TrueNAS Burn-In — Daily Report
| TrueNAS Burn-In
Daily Status Report |
{generated_at} |
|
{alert_html}
|
|
{len(failed_drives)}
Failed
|
{len(running_burnin)}
Running
|
{len(passed_burnin)}
Passed
|
| Drive |
Model |
Serial |
Size |
Temp |
Health |
Short |
Long |
Burn-In |
{drive_rows}
|
|
Generated by TrueNAS Burn-In Dashboard · {generated_at}
|
|
"""
# ---------------------------------------------------------------------------
# Send
# ---------------------------------------------------------------------------
# Standard ports for each SSL mode — used when smtp_port is not overridden
_MODE_PORTS: dict[str, int] = {"starttls": 587, "ssl": 465, "plain": 25}
def _smtp_port() -> int:
"""Derive port from ssl_mode; fall back to settings.smtp_port if explicitly set."""
mode = (settings.smtp_ssl_mode or "starttls").lower()
return _MODE_PORTS.get(mode, 587)
def _send_email(subject: str, html: str) -> None:
recipients = [r.strip() for r in settings.smtp_to.split(",") if r.strip()]
if not recipients:
log.warning("SMTP_TO is empty — skipping send")
return
msg = MIMEMultipart("alternative")
msg["Subject"] = subject
msg["From"] = settings.smtp_from or settings.smtp_user
msg["To"] = ", ".join(recipients)
msg.attach(MIMEText(html, "html", "utf-8"))
ctx = ssl.create_default_context()
mode = (settings.smtp_ssl_mode or "starttls").lower()
timeout = int(settings.smtp_timeout or 60)
port = _smtp_port()
if mode == "ssl":
server = smtplib.SMTP_SSL(settings.smtp_host, port, context=ctx, timeout=timeout)
server.ehlo()
server.login(settings.smtp_user, settings.smtp_password)
server.sendmail(msg["From"], recipients, msg.as_string())
server.quit()
else:
with smtplib.SMTP(settings.smtp_host, port, timeout=timeout) as server:
server.ehlo()
if mode == "starttls":
server.starttls(context=ctx)
server.ehlo()
server.login(settings.smtp_user, settings.smtp_password)
server.sendmail(msg["From"], recipients, msg.as_string())
log.info("Email sent to %s", recipients)
# ---------------------------------------------------------------------------
# Data fetch
# ---------------------------------------------------------------------------
async def _fetch_report_data() -> list[dict]:
"""Pull drives + latest burnin state from DB."""
from app.routes import _fetch_drives_for_template # local import avoids circular
async with aiosqlite.connect(settings.db_path) as db:
db.row_factory = aiosqlite.Row
await db.execute("PRAGMA journal_mode=WAL")
return await _fetch_drives_for_template(db)
async def _fetch_unlock_events_24h() -> list[dict]:
"""Return pool-drive unlock audit events from the last 24 hours.
These are operator overrides of the pool-membership lock — every entry
represents a deliberate decision to risk a pool, so the daily report
surfaces them as an audit-grade banner.
"""
async with aiosqlite.connect(settings.db_path) as db:
db.row_factory = aiosqlite.Row
await db.execute("PRAGMA journal_mode=WAL")
# julianday() handles the 'YYYY-MM-DDTHH:MM:SS.fff+00:00' format
# we write from Python; comparing the raw string against
# datetime('now','-1 day') (which formats as 'YYYY-MM-DD HH:MM:SS')
# produces subtle off-by-up-to-a-day errors because of the
# 'T' vs ' ' separator and the '+00:00' suffix.
cur = await db.execute("""
SELECT ae.event_type, ae.operator, ae.message, ae.created_at,
d.devname, d.pool_name, d.pool_role
FROM audit_events ae
LEFT JOIN drives d ON d.id = ae.drive_id
WHERE ae.event_type IN (
'pool_drive_unlocked',
'boot_pool_drive_unlocked',
'exported_pool_drive_unlocked',
'mounted_drive_unlocked')
AND julianday(ae.created_at) >= julianday('now', '-1 day')
ORDER BY ae.created_at DESC
""")
return [dict(r) for r in await cur.fetchall()]
# ---------------------------------------------------------------------------
# Scheduler
# ---------------------------------------------------------------------------
def _build_alert_html(
job_id: int,
devname: str,
serial: str | None,
model: str | None,
state: str,
error_text: str | None,
generated_at: str,
) -> str:
is_fail = state == "failed"
color = "#f85149" if is_fail else "#3fb950"
bg = "#4b1113" if is_fail else "#1a4731"
icon = "✕" if is_fail else "✓"
error_section = ""
if error_text:
error_section = f"""
Error: {error_text}
"""
return f"""
Burn-In {state.title()} Alert
|
{icon} Burn-In {state.upper()}
| Device |
{devname} |
| Model |
{model or '—'} |
| Serial |
{serial or '—'} |
| Job # |
{job_id} |
{error_section}
{generated_at}
|
|
"""
async def send_job_alert(
job_id: int,
devname: str,
serial: str | None,
model: str | None,
state: str,
error_text: str | None,
) -> None:
"""Send an immediate per-job alert email (pass or fail)."""
icon = "✕" if state == "failed" else "✓"
subject = f"{icon} Burn-In {state.upper()}: {devname} ({serial or 'no serial'})"
now_str = datetime.now().strftime("%Y-%m-%d %H:%M")
html = _build_alert_html(job_id, devname, serial, model, state, error_text, now_str)
await asyncio.to_thread(_send_email, subject, html)
async def test_smtp_connection() -> dict:
"""
Try to establish an SMTP connection using current settings.
Returns {"ok": True/False, "error": str|None}.
Does NOT send any email.
"""
if not settings.smtp_host:
return {"ok": False, "error": "SMTP_HOST is not configured"}
def _test() -> dict:
try:
ctx = ssl.create_default_context()
mode = (settings.smtp_ssl_mode or "starttls").lower()
timeout = int(settings.smtp_timeout or 60)
port = _smtp_port()
if mode == "ssl":
server = smtplib.SMTP_SSL(settings.smtp_host, port,
context=ctx, timeout=timeout)
server.ehlo()
else:
server = smtplib.SMTP(settings.smtp_host, port, timeout=timeout)
server.ehlo()
if mode == "starttls":
server.starttls(context=ctx)
server.ehlo()
if settings.smtp_user:
server.login(settings.smtp_user, settings.smtp_password)
server.quit()
return {"ok": True, "error": None}
except Exception as exc:
return {"ok": False, "error": str(exc)}
return await asyncio.to_thread(_test)
async def send_report_now() -> None:
"""Send a report immediately (used by on-demand API endpoint)."""
drives = await _fetch_report_data()
unlock_events = await _fetch_unlock_events_24h()
now_str = datetime.now().strftime("%Y-%m-%d %H:%M")
html = _build_html(drives, now_str, unlock_events)
suffix = ""
if unlock_events:
suffix = f" — {len(unlock_events)} pool unlock(s)"
subject = (
f"Burn-In Report — {datetime.now().strftime('%Y-%m-%d')} "
f"({len(drives)} drives){suffix}"
)
await asyncio.to_thread(_send_email, subject, html)
async def run() -> None:
"""Background loop: send daily report at smtp_report_hour local time."""
if not settings.smtp_host:
log.info("SMTP not configured — daily email disabled")
return
log.info(
"Mailer started — daily report at %02d:00 local time",
settings.smtp_report_hour,
)
while True:
now = datetime.now()
target = now.replace(
hour=settings.smtp_report_hour,
minute=0, second=0, microsecond=0,
)
if target <= now:
target += timedelta(days=1)
wait = (target - now).total_seconds()
log.info("Next report in %.0f seconds (%s)", wait, target.strftime("%Y-%m-%d %H:%M"))
await asyncio.sleep(wait)
if settings.smtp_daily_report_enabled:
try:
await send_report_now()
except Exception as exc:
log.error("Failed to send daily report: %s", exc)
else:
log.info("Daily report skipped — smtp_daily_report_enabled is False")
# Sleep briefly past the hour to avoid drift from re-triggering immediately
await asyncio.sleep(60)