11218753ce
Closes #5 of the post-Codex hardening list: * Settings UI now shows a `[set]` (green) or `[unset]` (gray) badge next to every password/key field. Tells the operator at a glance which secrets are configured without ever rendering the value. * SSH key gets a granular source label: `set (environment variable)`, `set (mounted secret)`, or `set (stored in settings DB — prefer a mounted secret in production)`. Same hint copy in the field's help text now actively recommends `/run/secrets/ssh_key` over the textarea. * New `GET /api/v1/settings/redacted` admin-only endpoint dumps every editable setting with secrets replaced by `***`, plus the per-secret status map. Useful for ops triage ("what's actually loaded?") without the secrets ever leaving the container or hitting a transcript. * `POST /api/v1/settings` writes a `settings_secret_changed` audit event whenever a non-empty secret is rotated. Records field names, operator, source IP — never the value. Lets the audit page answer "who rotated the SMTP password last week?". Internal: `_SECRET_FIELDS` constant in routes.py is now the single source of truth for which fields get the redaction / audit treatment. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1498 lines
54 KiB
Python
1498 lines
54 KiB
Python
import asyncio
|
|
import csv
|
|
import io
|
|
import json
|
|
from datetime import datetime, timezone
|
|
|
|
import aiosqlite
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request, WebSocket
|
|
from fastapi.responses import HTMLResponse, RedirectResponse, StreamingResponse
|
|
from sse_starlette.sse import EventSourceResponse
|
|
|
|
from app import auth, burnin, mailer, poller, settings_store
|
|
from app.config import settings
|
|
from app.database import get_db
|
|
from app.models import (
|
|
BurninJobResponse, BurninStageResponse,
|
|
CancelBurninRequest, DriveResponse,
|
|
SmartTestState, StartBurninRequest, UnlockPoolDriveRequest,
|
|
UpdateDriveRequest,
|
|
)
|
|
from app.renderer import templates
|
|
|
|
router = APIRouter()
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Internal helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _eta_seconds(eta_at: str | None) -> int | None:
|
|
if not eta_at:
|
|
return None
|
|
try:
|
|
eta_ts = datetime.fromisoformat(eta_at)
|
|
if eta_ts.tzinfo is None:
|
|
eta_ts = eta_ts.replace(tzinfo=timezone.utc)
|
|
remaining = (eta_ts - datetime.now(timezone.utc)).total_seconds()
|
|
return max(0, int(remaining))
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _is_stale(last_polled_at: str) -> bool:
|
|
try:
|
|
last = datetime.fromisoformat(last_polled_at)
|
|
if last.tzinfo is None:
|
|
last = last.replace(tzinfo=timezone.utc)
|
|
return (datetime.now(timezone.utc) - last).total_seconds() > settings.stale_threshold_seconds
|
|
except Exception:
|
|
return True
|
|
|
|
|
|
def _compute_eta_seconds(started_at: str | None, percent: int) -> int | None:
|
|
"""Linear ETA extrapolation from started_at and percent complete."""
|
|
if not started_at or percent <= 0:
|
|
return None
|
|
try:
|
|
start = datetime.fromisoformat(started_at)
|
|
if start.tzinfo is None:
|
|
start = start.replace(tzinfo=timezone.utc)
|
|
elapsed = (datetime.now(timezone.utc) - start).total_seconds()
|
|
total_est = elapsed / (percent / 100)
|
|
remaining = max(0, int(total_est - elapsed))
|
|
return remaining
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _build_smart(row: aiosqlite.Row, prefix: str) -> SmartTestState:
|
|
eta_at = row[f"{prefix}_eta_at"]
|
|
return SmartTestState(
|
|
state=row[f"{prefix}_state"] or "idle",
|
|
percent=row[f"{prefix}_percent"],
|
|
eta_seconds=_eta_seconds(eta_at),
|
|
eta_timestamp=eta_at,
|
|
started_at=row[f"{prefix}_started_at"],
|
|
finished_at=row[f"{prefix}_finished_at"],
|
|
error_text=row[f"{prefix}_error"],
|
|
)
|
|
|
|
|
|
def _row_to_drive(row: aiosqlite.Row) -> DriveResponse:
|
|
return DriveResponse(
|
|
id=row["id"],
|
|
devname=row["devname"],
|
|
serial=row["serial"],
|
|
model=row["model"],
|
|
size_bytes=row["size_bytes"],
|
|
temperature_c=row["temperature_c"],
|
|
smart_health=row["smart_health"] or "UNKNOWN",
|
|
last_polled_at=row["last_polled_at"],
|
|
is_stale=_is_stale(row["last_polled_at"]),
|
|
smart_short=_build_smart(row, "short"),
|
|
smart_long=_build_smart(row, "long"),
|
|
notes=row["notes"],
|
|
location=row["location"],
|
|
pool_name=row["pool_name"],
|
|
pool_role=row["pool_role"],
|
|
pool_unlocked_until=burnin.unlock_expiry(
|
|
row["id"], row["pool_name"], row["pool_role"],
|
|
),
|
|
)
|
|
|
|
|
|
def _compute_status(drive: dict) -> str:
|
|
short = (drive.get("smart_short") or {}).get("state", "idle")
|
|
long_ = (drive.get("smart_long") or {}).get("state", "idle")
|
|
health = drive.get("smart_health", "UNKNOWN")
|
|
if "running" in (short, long_):
|
|
return "running"
|
|
if short == "failed" or long_ == "failed" or health == "FAILED":
|
|
return "failed"
|
|
if "passed" in (short, long_):
|
|
return "passed"
|
|
return "idle"
|
|
|
|
|
|
_DRIVES_QUERY = """
|
|
SELECT
|
|
d.id, d.devname, d.serial, d.model, d.size_bytes,
|
|
d.temperature_c, d.smart_health, d.last_polled_at,
|
|
d.notes, d.location, d.pool_name, d.pool_role,
|
|
s.state AS short_state,
|
|
s.percent AS short_percent,
|
|
s.started_at AS short_started_at,
|
|
s.eta_at AS short_eta_at,
|
|
s.finished_at AS short_finished_at,
|
|
s.error_text AS short_error,
|
|
l.state AS long_state,
|
|
l.percent AS long_percent,
|
|
l.started_at AS long_started_at,
|
|
l.eta_at AS long_eta_at,
|
|
l.finished_at AS long_finished_at,
|
|
l.error_text AS long_error
|
|
FROM drives d
|
|
LEFT JOIN smart_tests s ON s.drive_id = d.id AND s.test_type = 'short'
|
|
LEFT JOIN smart_tests l ON l.drive_id = d.id AND l.test_type = 'long'
|
|
WHERE d.last_seen_at >= datetime('now', '-7 days')
|
|
{where}
|
|
ORDER BY d.devname
|
|
"""
|
|
|
|
|
|
async def _fetch_burnin_by_drive(db: aiosqlite.Connection) -> dict[int, dict]:
|
|
"""Return latest burn-in job (any state) keyed by drive_id.
|
|
|
|
Jobs created before the drive's last_reset_at are excluded so the
|
|
dashboard burn-in column clears after a reset while history is preserved.
|
|
"""
|
|
cur = await db.execute("""
|
|
SELECT bj.*
|
|
FROM burnin_jobs bj
|
|
JOIN drives d ON d.id = bj.drive_id
|
|
WHERE bj.id IN (SELECT MAX(id) FROM burnin_jobs GROUP BY drive_id)
|
|
AND (d.last_reset_at IS NULL OR bj.created_at > d.last_reset_at)
|
|
""")
|
|
rows = await cur.fetchall()
|
|
return {r["drive_id"]: dict(r) for r in rows}
|
|
|
|
|
|
async def _fetch_drives_for_template(db: aiosqlite.Connection) -> list[dict]:
|
|
cur = await db.execute(_DRIVES_QUERY.format(where=""))
|
|
rows = await cur.fetchall()
|
|
burnin_by_drive = await _fetch_burnin_by_drive(db)
|
|
|
|
# For burn-ins that include SMART stages, fetch those stages so we can
|
|
# mirror their progress/result in the Short/Long SMART columns.
|
|
# This covers both running stages (showing live progress) and completed
|
|
# stages (showing passed/failed after the burn-in moves to the next stage).
|
|
bi_smart_stages: dict[int, dict[str, dict]] = {} # job_id -> {stage_name: row}
|
|
bi_ids_with_smart = [
|
|
bi["id"] for bi in burnin_by_drive.values()
|
|
if bi["state"] in ("running", "queued")
|
|
]
|
|
if bi_ids_with_smart:
|
|
placeholders = ",".join("?" * len(bi_ids_with_smart))
|
|
# placeholders is purely structural ("?,?,?"); IDs themselves are
|
|
# bound via the parameter tuple. SQL built via concatenation so
|
|
# bandit's B608 (which fires on any f-string SQL) doesn't flag it.
|
|
sql = (
|
|
"SELECT bs.burnin_job_id, bs.stage_name, bs.state, bs.percent, "
|
|
" bs.started_at, bs.finished_at, bs.error_text "
|
|
"FROM burnin_stages bs "
|
|
"WHERE bs.burnin_job_id IN (" + placeholders + ") "
|
|
" AND bs.stage_name IN ('short_smart', 'long_smart') "
|
|
" AND bs.state IN ('running', 'passed', 'failed')"
|
|
)
|
|
cur = await db.execute(sql, bi_ids_with_smart)
|
|
for r in await cur.fetchall():
|
|
bi_smart_stages.setdefault(r["burnin_job_id"], {})[r["stage_name"]] = dict(r)
|
|
|
|
drives = []
|
|
for row in rows:
|
|
d = _row_to_drive(row).model_dump()
|
|
d["status"] = _compute_status(d)
|
|
bi = burnin_by_drive.get(d["id"])
|
|
d["burnin"] = bi
|
|
|
|
# Overlay burn-in SMART stage progress/results onto the SMART columns
|
|
if bi and bi["id"] in bi_smart_stages:
|
|
for stage_name, stage in bi_smart_stages[bi["id"]].items():
|
|
target = "smart_short" if stage_name == "short_smart" else "smart_long"
|
|
# Only overlay if the standalone SMART column is idle/empty
|
|
existing = d.get(target) or {}
|
|
if existing.get("state") not in (None, "idle"):
|
|
continue
|
|
pct = stage["percent"] or 0
|
|
d[target] = {
|
|
"state": stage["state"],
|
|
"percent": pct if stage["state"] == "running" else (100 if stage["state"] == "passed" else 0),
|
|
"eta_seconds": _compute_eta_seconds(stage["started_at"], pct) if stage["state"] == "running" else None,
|
|
"eta_timestamp": None,
|
|
"started_at": stage["started_at"],
|
|
"finished_at": stage["finished_at"],
|
|
"error_text": stage["error_text"],
|
|
}
|
|
|
|
drives.append(d)
|
|
return drives
|
|
|
|
|
|
def _stale_context(poller_state: dict) -> dict:
|
|
last = poller_state.get("last_poll_at")
|
|
if not last:
|
|
return {"stale": False, "stale_seconds": 0}
|
|
try:
|
|
dt = datetime.fromisoformat(last)
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
elapsed = int((datetime.now(timezone.utc) - dt).total_seconds())
|
|
stale = elapsed > settings.stale_threshold_seconds
|
|
return {"stale": stale, "stale_seconds": elapsed}
|
|
except Exception:
|
|
return {"stale": False, "stale_seconds": 0}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Auth — login / logout / first-user setup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/login", response_class=HTMLResponse)
|
|
async def login_page(request: Request, next: str = "/", error: str | None = None):
|
|
needs_setup = (await auth.user_count()) == 0
|
|
return templates.TemplateResponse(request, "login.html", {
|
|
"request": request,
|
|
"needs_setup": needs_setup,
|
|
"error": error,
|
|
"next": next if next.startswith("/") else "/",
|
|
})
|
|
|
|
|
|
def _client_ip(request: Request) -> str:
|
|
fwd = (request.headers.get("X-Forwarded-For") or "").split(",")[0].strip()
|
|
return fwd or (request.client.host if request.client else "unknown")
|
|
|
|
|
|
@router.post("/login")
|
|
async def login_submit(request: Request):
|
|
form = await request.form()
|
|
username = (form.get("username") or "").strip()
|
|
password = form.get("password") or ""
|
|
next_url = form.get("next") or "/"
|
|
if not next_url.startswith("/"):
|
|
next_url = "/"
|
|
ip = _client_ip(request)
|
|
|
|
# Rate-limit gate — checked BEFORE bcrypt so an attacker can't burn CPU.
|
|
locked_until = auth.login_locked_until(username, ip)
|
|
if locked_until is not None:
|
|
remaining = int(locked_until - __import__("time").time())
|
|
return templates.TemplateResponse(request, "login.html", {
|
|
"request": request,
|
|
"needs_setup": False,
|
|
"error": f"Too many failed attempts. Try again in {remaining // 60} min.",
|
|
"next": next_url,
|
|
}, status_code=429)
|
|
|
|
found = await auth.get_user_by_username(username)
|
|
if not found or not auth.verify_password(password, found[1]):
|
|
# Constant-ish-time: still call verify on a junk hash if user missing
|
|
# so the timing of "user not found" matches "wrong password."
|
|
if not found:
|
|
auth.verify_password(password, "$2b$12$" + "x" * 53)
|
|
tripped = auth.record_login_failure(username, ip)
|
|
await auth.audit_auth_event(
|
|
"user_login_locked_out" if tripped else "user_login_failed",
|
|
username,
|
|
f"Failed login from {ip}" + (
|
|
f" — IP/user locked out for {auth.LOGIN_LOCKOUT_SECONDS // 60} min"
|
|
if tripped else ""
|
|
),
|
|
)
|
|
return templates.TemplateResponse(request, "login.html", {
|
|
"request": request,
|
|
"needs_setup": False,
|
|
"error": "Invalid username or password.",
|
|
"next": next_url,
|
|
}, status_code=401)
|
|
|
|
user = found[0]
|
|
auth.clear_login_failures(username, ip)
|
|
request.session["user_id"] = user.id
|
|
request.session["username"] = user.username
|
|
await auth.touch_last_login(user.id)
|
|
await auth.audit_auth_event(
|
|
"user_login", user.username, f"Signed in from {ip}"
|
|
)
|
|
return RedirectResponse(url=next_url, status_code=303)
|
|
|
|
|
|
@router.post("/api/v1/auth/setup")
|
|
async def auth_first_user_setup(request: Request):
|
|
"""Create the first admin from the login page when the users table is
|
|
empty. Public endpoint — but only does anything when zero users exist."""
|
|
if (await auth.user_count()) > 0:
|
|
raise HTTPException(status_code=409, detail="Users already exist.")
|
|
form = await request.form()
|
|
username = (form.get("username") or "").strip()
|
|
password = form.get("password") or ""
|
|
full_name = (form.get("full_name") or "").strip() or None
|
|
try:
|
|
user = await auth.create_user(username, password, full_name, is_admin=True)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc))
|
|
request.session["user_id"] = user.id
|
|
request.session["username"] = user.username
|
|
await auth.touch_last_login(user.id)
|
|
return RedirectResponse(url="/", status_code=303)
|
|
|
|
|
|
@router.get("/logout")
|
|
@router.post("/logout")
|
|
async def logout(request: Request):
|
|
user = request.state.current_user if hasattr(request.state, "current_user") else None
|
|
if user:
|
|
await auth.audit_auth_event(
|
|
"user_logout", user.username, f"Signed out from {_client_ip(request)}"
|
|
)
|
|
request.session.clear()
|
|
return RedirectResponse(url="/login", status_code=303)
|
|
|
|
|
|
@router.post("/api/v1/auth/change-password")
|
|
async def change_password(request: Request):
|
|
user = request.state.current_user if hasattr(request.state, "current_user") else None
|
|
if not user:
|
|
raise HTTPException(status_code=401, detail="Authentication required")
|
|
form = await request.form()
|
|
current = form.get("current_password") or ""
|
|
new_pw = form.get("new_password") or ""
|
|
confirm = form.get("confirm_password") or ""
|
|
if new_pw != confirm:
|
|
raise HTTPException(status_code=400, detail="New passwords do not match.")
|
|
try:
|
|
await auth.change_password(user.id, current, new_pw)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc))
|
|
await auth.audit_auth_event(
|
|
"user_password_changed", user.username,
|
|
f"Password changed from {_client_ip(request)}",
|
|
)
|
|
return {"ok": True}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dashboard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/", response_class=HTMLResponse)
|
|
async def dashboard(request: Request, db: aiosqlite.Connection = Depends(get_db)):
|
|
drives = await _fetch_drives_for_template(db)
|
|
ps = poller.get_state()
|
|
return templates.TemplateResponse(request, "dashboard.html", {
|
|
"request": request,
|
|
"drives": drives,
|
|
"poller": ps,
|
|
**_stale_context(ps),
|
|
})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SSE — live drive table updates
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/sse/drives")
|
|
async def sse_drives(request: Request):
|
|
q = poller.subscribe()
|
|
|
|
async def generate():
|
|
try:
|
|
while True:
|
|
# Wait for next poll notification or keepalive timeout
|
|
try:
|
|
payload = await asyncio.wait_for(q.get(), timeout=25.0)
|
|
except asyncio.TimeoutError:
|
|
if await request.is_disconnected():
|
|
break
|
|
yield {"event": "keepalive", "data": ""}
|
|
continue
|
|
|
|
if await request.is_disconnected():
|
|
break
|
|
|
|
# Extract alert from payload (may be None for regular polls)
|
|
alert = None
|
|
if isinstance(payload, dict):
|
|
alert = payload.get("alert")
|
|
|
|
# Render fresh table HTML
|
|
async with aiosqlite.connect(settings.db_path) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
await db.execute("PRAGMA journal_mode=WAL")
|
|
drives = await _fetch_drives_for_template(db)
|
|
|
|
html = templates.env.get_template(
|
|
"components/drives_table.html"
|
|
).render(drives=drives)
|
|
|
|
yield {"event": "drives-update", "data": html}
|
|
|
|
# Push system sensor state so JS can update temp chips live
|
|
ps = poller.get_state()
|
|
yield {
|
|
"event": "system-sensors",
|
|
"data": json.dumps({
|
|
"system_temps": ps.get("system_temps", {}),
|
|
"thermal_pressure": ps.get("thermal_pressure", "ok"),
|
|
"temp_warn_c": settings.temp_warn_c,
|
|
"temp_crit_c": settings.temp_crit_c,
|
|
}),
|
|
}
|
|
|
|
# Push browser notification event if this was a job completion
|
|
if alert:
|
|
yield {"event": "job-alert", "data": json.dumps(alert)}
|
|
|
|
finally:
|
|
poller.unsubscribe(q)
|
|
|
|
return EventSourceResponse(generate())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# JSON API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/health")
|
|
async def health(db: aiosqlite.Connection = Depends(get_db)):
|
|
"""Real readiness check, not just process-is-running.
|
|
|
|
Verifies (a) DB writable, (b) poller has succeeded recently relative
|
|
to the configured stale_threshold_seconds, (c) SSH reachable when
|
|
configured. Returns 503 when any check fails so a proxy/orchestrator
|
|
health probe can take the container out of rotation.
|
|
"""
|
|
from datetime import datetime, timezone
|
|
from fastapi.responses import JSONResponse
|
|
from app import ssh_client as _ssh
|
|
|
|
checks: dict[str, dict] = {}
|
|
|
|
# DB probe — confirm the journal is healthy (PRAGMA reads journal_mode
|
|
# and would fail loudly if WAL is wedged or the file is unreadable).
|
|
try:
|
|
cur = await db.execute("PRAGMA journal_mode")
|
|
await cur.fetchone()
|
|
checks["db"] = {"ok": True}
|
|
except Exception as exc:
|
|
checks["db"] = {"ok": False, "error": str(exc)}
|
|
|
|
ps = poller.get_state()
|
|
last = ps.get("last_poll_at")
|
|
poll_age = None
|
|
if last:
|
|
try:
|
|
t = datetime.fromisoformat(last)
|
|
if t.tzinfo is None:
|
|
t = t.replace(tzinfo=timezone.utc)
|
|
poll_age = (datetime.now(timezone.utc) - t).total_seconds()
|
|
except Exception:
|
|
poll_age = None
|
|
poll_ok = ps.get("healthy") and (
|
|
poll_age is None or poll_age <= settings.stale_threshold_seconds * 3
|
|
)
|
|
checks["poller"] = {
|
|
"ok": bool(poll_ok),
|
|
"last_error": ps.get("last_error"),
|
|
"last_poll_at": last,
|
|
"age_seconds": int(poll_age) if poll_age is not None else None,
|
|
}
|
|
|
|
# SSH probe — only when configured. Cheap (single sensors -j).
|
|
if _ssh.is_configured():
|
|
try:
|
|
r = await _ssh.test_connection()
|
|
checks["ssh"] = {"ok": bool(r.get("ok")),
|
|
"error": r.get("error")}
|
|
except Exception as exc:
|
|
checks["ssh"] = {"ok": False, "error": str(exc)}
|
|
else:
|
|
checks["ssh"] = {"ok": True, "skipped": True}
|
|
|
|
cur = await db.execute("SELECT COUNT(*) FROM drives")
|
|
row = await cur.fetchone()
|
|
drives_tracked = row[0] if row else 0
|
|
|
|
status_ok = all(c["ok"] for c in checks.values())
|
|
body = {
|
|
"status": "ok" if status_ok else "degraded",
|
|
"checks": checks,
|
|
"drives_tracked": drives_tracked,
|
|
"poll_interval_s": settings.poll_interval_seconds,
|
|
"version": settings.app_version,
|
|
}
|
|
return JSONResponse(body, status_code=200 if status_ok else 503)
|
|
|
|
|
|
@router.get("/api/v1/drives", response_model=list[DriveResponse])
|
|
async def list_drives(db: aiosqlite.Connection = Depends(get_db)):
|
|
cur = await db.execute(_DRIVES_QUERY.format(where=""))
|
|
rows = await cur.fetchall()
|
|
return [_row_to_drive(r) for r in rows]
|
|
|
|
|
|
@router.get("/api/v1/drives/{drive_id}/drawer")
|
|
async def drive_drawer(drive_id: int, db: aiosqlite.Connection = Depends(get_db)):
|
|
"""Data for the log drawer — latest burn-in job + stages, SMART tests, audit events."""
|
|
cur = await db.execute(_DRIVES_QUERY.format(where="AND d.id = ?"), (drive_id,))
|
|
row = await cur.fetchone()
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Drive not found")
|
|
drive = _row_to_drive(row)
|
|
|
|
# Latest burn-in job + its stages (include log_text and bad_blocks)
|
|
cur = await db.execute(
|
|
"SELECT * FROM burnin_jobs WHERE drive_id=? ORDER BY id DESC LIMIT 1",
|
|
(drive_id,),
|
|
)
|
|
job_row = await cur.fetchone()
|
|
burnin = None
|
|
if job_row:
|
|
job = dict(job_row)
|
|
cur = await db.execute(
|
|
"SELECT id, stage_name, state, percent, started_at, finished_at, "
|
|
"duration_seconds, error_text, log_text, bad_blocks "
|
|
"FROM burnin_stages WHERE burnin_job_id=? ORDER BY id",
|
|
(job_row["id"],),
|
|
)
|
|
job["stages"] = [dict(r) for r in await cur.fetchall()]
|
|
burnin = job
|
|
|
|
# SMART raw output from smart_tests table
|
|
cur = await db.execute(
|
|
"SELECT test_type, state, percent, started_at, finished_at, error_text, raw_output "
|
|
"FROM smart_tests WHERE drive_id=?",
|
|
(drive_id,),
|
|
)
|
|
smart_rows = {r["test_type"]: dict(r) for r in await cur.fetchall()}
|
|
|
|
# Cached SMART attributes (JSON blob on drives table)
|
|
import json as _json
|
|
smart_attrs = None
|
|
cur = await db.execute("SELECT smart_attrs FROM drives WHERE id=?", (drive_id,))
|
|
attrs_row = await cur.fetchone()
|
|
if attrs_row and attrs_row["smart_attrs"]:
|
|
try:
|
|
smart_attrs = _json.loads(attrs_row["smart_attrs"])
|
|
except Exception:
|
|
pass
|
|
|
|
# Last 50 audit events for this drive (newest first)
|
|
cur = await db.execute("""
|
|
SELECT id, event_type, operator, message, created_at
|
|
FROM audit_events
|
|
WHERE drive_id = ?
|
|
ORDER BY id DESC
|
|
LIMIT 50
|
|
""", (drive_id,))
|
|
events = [dict(r) for r in await cur.fetchall()]
|
|
|
|
def _smart_card(test_type: str) -> dict:
|
|
smart_obj = drive.smart_short if test_type == "short" else drive.smart_long
|
|
base = smart_obj.model_dump() if smart_obj else {}
|
|
row = smart_rows.get(test_type, {})
|
|
base["raw_output"] = row.get("raw_output")
|
|
return base
|
|
|
|
return {
|
|
"drive": {
|
|
"id": drive.id,
|
|
"devname": drive.devname,
|
|
"serial": drive.serial,
|
|
"model": drive.model,
|
|
"size_bytes": drive.size_bytes,
|
|
},
|
|
"burnin": burnin,
|
|
"smart": {
|
|
"short": _smart_card("short"),
|
|
"long": _smart_card("long"),
|
|
"attrs": smart_attrs,
|
|
},
|
|
"events": events,
|
|
}
|
|
|
|
|
|
@router.get("/api/v1/drives/{drive_id}", response_model=DriveResponse)
|
|
async def get_drive(drive_id: int, db: aiosqlite.Connection = Depends(get_db)):
|
|
cur = await db.execute(
|
|
_DRIVES_QUERY.format(where="AND d.id = ?"), (drive_id,)
|
|
)
|
|
row = await cur.fetchone()
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Drive not found")
|
|
return _row_to_drive(row)
|
|
|
|
|
|
@router.post("/api/v1/drives/{drive_id}/smart/start")
|
|
async def smart_start(
|
|
drive_id: int,
|
|
body: dict,
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
"""Start a standalone SHORT or LONG SMART test on a single drive.
|
|
|
|
Uses SSH (smartctl) when configured — required for TrueNAS SCALE 25.10+
|
|
where the REST smart/test endpoint no longer exists.
|
|
Falls back to TrueNAS REST API for older versions.
|
|
"""
|
|
from app import burnin as _burnin, ssh_client
|
|
|
|
test_type = (body.get("type") or "").upper()
|
|
if test_type not in ("SHORT", "LONG"):
|
|
raise HTTPException(status_code=422, detail="type must be SHORT or LONG")
|
|
|
|
cur = await db.execute("SELECT devname FROM drives WHERE id=?", (drive_id,))
|
|
row = await cur.fetchone()
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Drive not found")
|
|
devname = row[0]
|
|
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
ttype_lower = test_type.lower()
|
|
|
|
if ssh_client.is_configured():
|
|
# SSH path — works on TrueNAS SCALE 25.10+ and CORE
|
|
try:
|
|
output = await ssh_client.start_smart_test(devname, test_type)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=502, detail=f"SSH error: {exc}")
|
|
|
|
# Mark as running in DB (truenas_job_id=NULL signals SSH-managed test)
|
|
# Store smartctl start output as proof the test was initiated
|
|
await db.execute(
|
|
"""INSERT INTO smart_tests (drive_id, test_type, state, percent, started_at, raw_output)
|
|
VALUES (?,?,?,?,?,?)
|
|
ON CONFLICT(drive_id, test_type) DO UPDATE SET
|
|
state='running', percent=0, truenas_job_id=NULL,
|
|
started_at=excluded.started_at, finished_at=NULL, error_text=NULL,
|
|
raw_output=excluded.raw_output""",
|
|
(drive_id, ttype_lower, "running", 0, now, output),
|
|
)
|
|
await db.commit()
|
|
from app import poller as _poller
|
|
_poller._notify_subscribers()
|
|
return {"devname": devname, "type": test_type, "message": output[:200]}
|
|
|
|
else:
|
|
# REST path — older TrueNAS CORE / SCALE versions
|
|
client = _burnin._client
|
|
if client is None:
|
|
raise HTTPException(status_code=503, detail="TrueNAS client not ready")
|
|
try:
|
|
tn_job_id = await client.start_smart_test([devname], test_type)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=502, detail=f"TrueNAS error: {exc}")
|
|
return {"job_id": tn_job_id, "devname": devname, "type": test_type}
|
|
|
|
|
|
@router.post("/api/v1/drives/{drive_id}/smart/cancel")
|
|
async def smart_cancel(
|
|
drive_id: int,
|
|
body: dict,
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
"""Cancel a running standalone SMART test on a drive."""
|
|
from app import burnin as _burnin
|
|
|
|
test_type = (body.get("type") or "").lower()
|
|
if test_type not in ("short", "long"):
|
|
raise HTTPException(status_code=422, detail="type must be 'short' or 'long'")
|
|
|
|
cur = await db.execute("SELECT devname FROM drives WHERE id=?", (drive_id,))
|
|
row = await cur.fetchone()
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Drive not found")
|
|
devname = row[0]
|
|
|
|
client = _burnin._client
|
|
if client is None:
|
|
raise HTTPException(status_code=503, detail="TrueNAS client not ready")
|
|
|
|
from app import ssh_client
|
|
|
|
if ssh_client.is_configured():
|
|
# SSH path — abort via smartctl -X
|
|
try:
|
|
await ssh_client.abort_smart_test(devname)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=502, detail=f"SSH abort error: {exc}")
|
|
else:
|
|
# REST path — find TrueNAS job and abort it
|
|
try:
|
|
jobs = await client.get_smart_jobs()
|
|
tn_job_id = None
|
|
for j in jobs:
|
|
if j.get("state") != "RUNNING":
|
|
continue
|
|
args = j.get("arguments", [])
|
|
if not args or not isinstance(args[0], dict):
|
|
continue
|
|
if devname in args[0].get("disks", []):
|
|
tn_job_id = j["id"]
|
|
break
|
|
|
|
if tn_job_id is None:
|
|
raise HTTPException(status_code=404, detail="No running SMART test found for this drive")
|
|
|
|
await client.abort_job(tn_job_id)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=502, detail=f"TrueNAS error: {exc}")
|
|
|
|
# Update local DB state
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
await db.execute(
|
|
"UPDATE smart_tests SET state='aborted', finished_at=? WHERE drive_id=? AND test_type=? AND state='running'",
|
|
(now, drive_id, test_type),
|
|
)
|
|
await db.commit()
|
|
|
|
return {"cancelled": True, "devname": devname, "type": test_type}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Burn-in API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _row_to_burnin(row: aiosqlite.Row, stages: list[aiosqlite.Row]) -> BurninJobResponse:
|
|
return BurninJobResponse(
|
|
id=row["id"],
|
|
drive_id=row["drive_id"],
|
|
profile=row["profile"],
|
|
state=row["state"],
|
|
percent=row["percent"] or 0,
|
|
stage_name=row["stage_name"],
|
|
operator=row["operator"],
|
|
created_at=row["created_at"],
|
|
started_at=row["started_at"],
|
|
finished_at=row["finished_at"],
|
|
error_text=row["error_text"],
|
|
stages=[
|
|
BurninStageResponse(
|
|
id=s["id"],
|
|
stage_name=s["stage_name"],
|
|
state=s["state"],
|
|
percent=s["percent"] or 0,
|
|
started_at=s["started_at"],
|
|
finished_at=s["finished_at"],
|
|
error_text=s["error_text"],
|
|
)
|
|
for s in stages
|
|
],
|
|
)
|
|
|
|
|
|
@router.post("/api/v1/burnin/start")
|
|
async def burnin_start(req: StartBurninRequest):
|
|
results = []
|
|
errors = []
|
|
for drive_id in req.drive_ids:
|
|
try:
|
|
job_id = await burnin.start_job(
|
|
drive_id, req.profile, req.operator, stage_order=req.stage_order
|
|
)
|
|
results.append({"drive_id": drive_id, "job_id": job_id})
|
|
except burnin.PoolMemberError as exc:
|
|
errors.append({
|
|
"drive_id": drive_id,
|
|
"error": str(exc),
|
|
"pool_name": exc.pool_name,
|
|
"pool_role": exc.pool_role,
|
|
"pool_locked": True,
|
|
})
|
|
except ValueError as exc:
|
|
errors.append({"drive_id": drive_id, "error": str(exc)})
|
|
if errors and not results:
|
|
# Surface the first error's structured fields so the UI can render
|
|
# an unlock affordance instead of a generic toast.
|
|
raise HTTPException(status_code=409, detail=errors[0])
|
|
return {"queued": results, "errors": errors}
|
|
|
|
|
|
@router.post("/api/v1/drives/{drive_id}/unlock")
|
|
async def unlock_pool_drive(drive_id: int, req: UnlockPoolDriveRequest):
|
|
try:
|
|
expiry = await burnin.grant_pool_unlock(
|
|
drive_id, req.confirm_token, req.operator, req.reason,
|
|
)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc))
|
|
return {"unlocked": True, "expires_at": expiry,
|
|
"ttl_seconds": burnin.UNLOCK_TTL_SECONDS}
|
|
|
|
|
|
@router.post("/api/v1/burnin/{job_id}/cancel")
|
|
async def burnin_cancel(job_id: int, req: CancelBurninRequest):
|
|
ok = await burnin.cancel_job(job_id, req.operator)
|
|
if not ok:
|
|
raise HTTPException(status_code=409, detail="Job not found or not cancellable")
|
|
return {"cancelled": True}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# History pages
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_PAGE_SIZE = 50
|
|
|
|
_ALL_STATES = ("queued", "running", "passed", "failed", "cancelled", "unknown")
|
|
|
|
_HISTORY_QUERY = """
|
|
SELECT
|
|
bj.id, bj.drive_id, bj.profile, bj.state, bj.operator,
|
|
bj.created_at, bj.started_at, bj.finished_at, bj.error_text,
|
|
d.devname, d.serial, d.model, d.size_bytes,
|
|
CAST(
|
|
(julianday(bj.finished_at) - julianday(bj.started_at)) * 86400
|
|
AS INTEGER
|
|
) AS duration_seconds
|
|
FROM burnin_jobs bj
|
|
JOIN drives d ON d.id = bj.drive_id
|
|
{where}
|
|
ORDER BY bj.id DESC
|
|
"""
|
|
|
|
|
|
def _state_where(state: str) -> tuple[str, list]:
|
|
if state == "all":
|
|
return "", []
|
|
return "WHERE bj.state = ?", [state]
|
|
|
|
|
|
@router.get("/history", response_class=HTMLResponse)
|
|
async def history_list(
|
|
request: Request,
|
|
state: str = Query(default="all"),
|
|
page: int = Query(default=1, ge=1),
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
if state not in ("all",) + _ALL_STATES:
|
|
state = "all"
|
|
|
|
where_clause, params = _state_where(state)
|
|
|
|
# Total count
|
|
count_sql = f"SELECT COUNT(*) FROM burnin_jobs bj JOIN drives d ON d.id = bj.drive_id {where_clause}"
|
|
cur = await db.execute(count_sql, params)
|
|
total_count = (await cur.fetchone())[0]
|
|
total_pages = max(1, (total_count + _PAGE_SIZE - 1) // _PAGE_SIZE)
|
|
page = min(page, total_pages)
|
|
offset = (page - 1) * _PAGE_SIZE
|
|
|
|
# Per-state counts for badges
|
|
cur = await db.execute(
|
|
"SELECT state, COUNT(*) FROM burnin_jobs GROUP BY state"
|
|
)
|
|
counts = {"all": total_count if state == "all" else 0}
|
|
for r in await cur.fetchall():
|
|
counts[r[0]] = r[1]
|
|
if state != "all":
|
|
cur2 = await db.execute("SELECT COUNT(*) FROM burnin_jobs")
|
|
counts["all"] = (await cur2.fetchone())[0]
|
|
|
|
# Job rows
|
|
sql = _HISTORY_QUERY.format(where=where_clause) + " LIMIT ? OFFSET ?"
|
|
cur = await db.execute(sql, params + [_PAGE_SIZE, offset])
|
|
rows = await cur.fetchall()
|
|
jobs = [dict(r) for r in rows]
|
|
|
|
ps = poller.get_state()
|
|
return templates.TemplateResponse(request, "history.html", {
|
|
"request": request,
|
|
"jobs": jobs,
|
|
"active_state": state,
|
|
"counts": counts,
|
|
"page": page,
|
|
"total_pages": total_pages,
|
|
"total_count": total_count,
|
|
"poller": ps,
|
|
**_stale_context(ps),
|
|
})
|
|
|
|
|
|
@router.get("/history/{job_id}", response_class=HTMLResponse)
|
|
async def history_detail(
|
|
request: Request,
|
|
job_id: int,
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
# Job + drive info
|
|
cur = await db.execute("""
|
|
SELECT
|
|
bj.*, d.devname, d.serial, d.model, d.size_bytes,
|
|
CAST(
|
|
(julianday(bj.finished_at) - julianday(bj.started_at)) * 86400
|
|
AS INTEGER
|
|
) AS duration_seconds
|
|
FROM burnin_jobs bj
|
|
JOIN drives d ON d.id = bj.drive_id
|
|
WHERE bj.id = ?
|
|
""", (job_id,))
|
|
row = await cur.fetchone()
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Burn-in job not found")
|
|
job = dict(row)
|
|
|
|
# Stages (with duration)
|
|
cur = await db.execute("""
|
|
SELECT *,
|
|
CAST(
|
|
(julianday(finished_at) - julianday(started_at)) * 86400
|
|
AS INTEGER
|
|
) AS duration_seconds
|
|
FROM burnin_stages
|
|
WHERE burnin_job_id = ?
|
|
ORDER BY id
|
|
""", (job_id,))
|
|
job["stages"] = [dict(r) for r in await cur.fetchall()]
|
|
|
|
ps = poller.get_state()
|
|
return templates.TemplateResponse(request, "job_detail.html", {
|
|
"request": request,
|
|
"job": job,
|
|
"poller": ps,
|
|
**_stale_context(ps),
|
|
})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CSV export
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/api/v1/burnin/export.csv")
|
|
async def burnin_export_csv(db: aiosqlite.Connection = Depends(get_db)):
|
|
cur = await db.execute("""
|
|
SELECT
|
|
bj.id AS job_id,
|
|
bj.drive_id,
|
|
d.devname,
|
|
d.serial,
|
|
d.model,
|
|
bj.profile,
|
|
bj.state,
|
|
bj.operator,
|
|
bj.created_at,
|
|
bj.started_at,
|
|
bj.finished_at,
|
|
CAST(
|
|
(julianday(bj.finished_at) - julianday(bj.started_at)) * 86400
|
|
AS INTEGER
|
|
) AS duration_seconds,
|
|
bj.error_text
|
|
FROM burnin_jobs bj
|
|
JOIN drives d ON d.id = bj.drive_id
|
|
ORDER BY bj.id DESC
|
|
""")
|
|
rows = await cur.fetchall()
|
|
|
|
buf = io.StringIO()
|
|
writer = csv.writer(buf)
|
|
writer.writerow([
|
|
"job_id", "drive_id", "devname", "serial", "model",
|
|
"profile", "state", "operator",
|
|
"created_at", "started_at", "finished_at", "duration_seconds",
|
|
"error_text",
|
|
])
|
|
for r in rows:
|
|
writer.writerow(list(r))
|
|
|
|
buf.seek(0)
|
|
return StreamingResponse(
|
|
iter([buf.getvalue()]),
|
|
media_type="text/csv",
|
|
headers={"Content-Disposition": "attachment; filename=burnin_history.csv"},
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# On-demand email report
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.post("/api/v1/report/send")
|
|
async def send_report_now():
|
|
"""Trigger the daily status email immediately (for testing SMTP config)."""
|
|
if not settings.smtp_host:
|
|
raise HTTPException(status_code=503, detail="SMTP not configured (SMTP_HOST is empty)")
|
|
try:
|
|
await mailer.send_report_now()
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=502, detail=f"Mail send failed: {exc}")
|
|
return {"sent": True, "to": settings.smtp_to}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Drive notes / location update
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.patch("/api/v1/drives/{drive_id}")
|
|
async def update_drive(
|
|
drive_id: int,
|
|
req: UpdateDriveRequest,
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
cur = await db.execute("SELECT id FROM drives WHERE id=?", (drive_id,))
|
|
if not await cur.fetchone():
|
|
raise HTTPException(status_code=404, detail="Drive not found")
|
|
|
|
await db.execute(
|
|
"UPDATE drives SET notes=?, location=? WHERE id=?",
|
|
(req.notes, req.location, drive_id),
|
|
)
|
|
await db.commit()
|
|
return {"updated": True}
|
|
|
|
|
|
@router.post("/api/v1/drives/{drive_id}/reset")
|
|
async def reset_drive(
|
|
drive_id: int,
|
|
body: dict,
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
"""
|
|
Clear SMART test results for a drive so it shows as fresh.
|
|
Only allowed when no burn-in job is active (queued or running).
|
|
Preserves all job history — just resets the display state.
|
|
"""
|
|
cur = await db.execute("SELECT id FROM drives WHERE id=?", (drive_id,))
|
|
if not await cur.fetchone():
|
|
raise HTTPException(status_code=404, detail="Drive not found")
|
|
|
|
# Reject if any active burn-in
|
|
cur = await db.execute(
|
|
"SELECT COUNT(*) FROM burnin_jobs WHERE drive_id=? AND state IN ('queued','running')",
|
|
(drive_id,),
|
|
)
|
|
if (await cur.fetchone())[0] > 0:
|
|
raise HTTPException(status_code=409, detail="Cannot reset while a burn-in is active")
|
|
|
|
operator = body.get("operator", "operator")
|
|
|
|
# Reset SMART test state to idle
|
|
await db.execute(
|
|
"""UPDATE smart_tests SET state='idle', percent=0, started_at=NULL,
|
|
eta_at=NULL, finished_at=NULL, error_text=NULL, raw_output=NULL
|
|
WHERE drive_id=?""",
|
|
(drive_id,),
|
|
)
|
|
# Clear SMART attrs cache + stamp reset time (hides prior burn-in from dashboard)
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
await db.execute(
|
|
"UPDATE drives SET smart_attrs=NULL, last_reset_at=? WHERE id=?",
|
|
(now, drive_id),
|
|
)
|
|
|
|
# Audit event
|
|
await db.execute(
|
|
"""INSERT INTO audit_events (event_type, drive_id, operator, message)
|
|
VALUES (?,?,?,?)""",
|
|
("drive_reset", drive_id, operator, "Drive reset — SMART state cleared"),
|
|
)
|
|
await db.commit()
|
|
|
|
poller._notify_subscribers()
|
|
return {"reset": True}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Audit log page
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_AUDIT_QUERY = """
|
|
SELECT
|
|
ae.id, ae.event_type, ae.operator, ae.message, ae.created_at,
|
|
d.devname, d.serial
|
|
FROM audit_events ae
|
|
LEFT JOIN drives d ON d.id = ae.drive_id
|
|
ORDER BY ae.id DESC
|
|
LIMIT 200
|
|
"""
|
|
|
|
_AUDIT_EVENT_COLORS = {
|
|
"burnin_queued": "yellow",
|
|
"burnin_started": "blue",
|
|
"burnin_passed": "passed",
|
|
"burnin_failed": "failed",
|
|
"burnin_cancelled": "cancelled",
|
|
"burnin_stuck": "failed",
|
|
"burnin_unknown": "unknown",
|
|
}
|
|
|
|
|
|
@router.get("/audit", response_class=HTMLResponse)
|
|
async def audit_log(
|
|
request: Request,
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
cur = await db.execute(_AUDIT_QUERY)
|
|
rows = [dict(r) for r in await cur.fetchall()]
|
|
ps = poller.get_state()
|
|
return templates.TemplateResponse(request, "audit.html", {
|
|
"request": request,
|
|
"events": rows,
|
|
"event_colors": _AUDIT_EVENT_COLORS,
|
|
"poller": ps,
|
|
**_stale_context(ps),
|
|
})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stats / analytics page
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/stats", response_class=HTMLResponse)
|
|
async def stats_page(
|
|
request: Request,
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
# Overall counts
|
|
cur = await db.execute("""
|
|
SELECT
|
|
COUNT(*) as total,
|
|
SUM(CASE WHEN state='passed' THEN 1 ELSE 0 END) as passed,
|
|
SUM(CASE WHEN state='failed' THEN 1 ELSE 0 END) as failed,
|
|
SUM(CASE WHEN state='running' THEN 1 ELSE 0 END) as running,
|
|
SUM(CASE WHEN state='cancelled' THEN 1 ELSE 0 END) as cancelled
|
|
FROM burnin_jobs
|
|
""")
|
|
overall = dict(await cur.fetchone())
|
|
|
|
# Failure rate by drive model (only completed jobs)
|
|
cur = await db.execute("""
|
|
SELECT
|
|
COALESCE(d.model, 'Unknown') AS model,
|
|
COUNT(*) AS total,
|
|
SUM(CASE WHEN bj.state='passed' THEN 1 ELSE 0 END) AS passed,
|
|
SUM(CASE WHEN bj.state='failed' THEN 1 ELSE 0 END) AS failed,
|
|
ROUND(100.0 * SUM(CASE WHEN bj.state='passed' THEN 1 ELSE 0 END) / COUNT(*), 1) AS pass_rate
|
|
FROM burnin_jobs bj
|
|
JOIN drives d ON d.id = bj.drive_id
|
|
WHERE bj.state IN ('passed', 'failed')
|
|
GROUP BY COALESCE(d.model, 'Unknown')
|
|
ORDER BY total DESC
|
|
LIMIT 20
|
|
""")
|
|
by_model = [dict(r) for r in await cur.fetchall()]
|
|
|
|
# Activity last 14 days
|
|
cur = await db.execute("""
|
|
SELECT
|
|
date(created_at) AS day,
|
|
COUNT(*) AS total,
|
|
SUM(CASE WHEN state='passed' THEN 1 ELSE 0 END) AS passed,
|
|
SUM(CASE WHEN state='failed' THEN 1 ELSE 0 END) AS failed
|
|
FROM burnin_jobs
|
|
WHERE created_at >= date('now', '-14 days')
|
|
GROUP BY date(created_at)
|
|
ORDER BY day DESC
|
|
""")
|
|
by_day = [dict(r) for r in await cur.fetchall()]
|
|
|
|
# Average test duration by drive size (rounded to nearest TB)
|
|
cur = await db.execute("""
|
|
SELECT
|
|
CAST(ROUND(CAST(d.size_bytes AS REAL) / 1e12) AS INTEGER) AS size_tb,
|
|
COUNT(*) AS total,
|
|
ROUND(AVG(
|
|
(julianday(bj.finished_at) - julianday(bj.started_at)) * 86400 / 3600.0
|
|
), 1) AS avg_hours
|
|
FROM burnin_jobs bj
|
|
JOIN drives d ON d.id = bj.drive_id
|
|
WHERE bj.state IN ('passed', 'failed')
|
|
AND bj.started_at IS NOT NULL
|
|
AND bj.finished_at IS NOT NULL
|
|
GROUP BY size_tb
|
|
ORDER BY size_tb
|
|
""")
|
|
by_size = [dict(r) for r in await cur.fetchall()]
|
|
|
|
# Failure breakdown by stage (which stage caused the failure)
|
|
cur = await db.execute("""
|
|
SELECT
|
|
COALESCE(bj.stage_name, 'unknown') AS failed_stage,
|
|
COUNT(*) AS count
|
|
FROM burnin_jobs bj
|
|
WHERE bj.state = 'failed'
|
|
GROUP BY failed_stage
|
|
ORDER BY count DESC
|
|
""")
|
|
by_failure_stage = [dict(r) for r in await cur.fetchall()]
|
|
|
|
# Drives tracked
|
|
cur = await db.execute("SELECT COUNT(*) FROM drives")
|
|
drives_total = (await cur.fetchone())[0]
|
|
|
|
ps = poller.get_state()
|
|
return templates.TemplateResponse(request, "stats.html", {
|
|
"request": request,
|
|
"overall": overall,
|
|
"by_model": by_model,
|
|
"by_day": by_day,
|
|
"by_size": by_size,
|
|
"by_failure_stage": by_failure_stage,
|
|
"drives_total": drives_total,
|
|
"poller": ps,
|
|
**_stale_context(ps),
|
|
})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Settings page
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/settings", response_class=HTMLResponse)
|
|
async def settings_page(
|
|
request: Request,
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
# Editable values — real values for form fields (secrets excluded)
|
|
editable = {
|
|
# SMTP
|
|
"smtp_host": settings.smtp_host,
|
|
"smtp_port": settings.smtp_port,
|
|
"smtp_ssl_mode": settings.smtp_ssl_mode or "starttls",
|
|
"smtp_timeout": settings.smtp_timeout,
|
|
"smtp_user": settings.smtp_user,
|
|
"smtp_from": settings.smtp_from,
|
|
"smtp_to": settings.smtp_to,
|
|
"smtp_report_hour": settings.smtp_report_hour,
|
|
"smtp_daily_report_enabled": settings.smtp_daily_report_enabled,
|
|
"smtp_alert_on_fail": settings.smtp_alert_on_fail,
|
|
"smtp_alert_on_pass": settings.smtp_alert_on_pass,
|
|
# Webhook
|
|
"webhook_url": settings.webhook_url,
|
|
# Burn-in behaviour
|
|
"stuck_job_hours": settings.stuck_job_hours,
|
|
"max_parallel_burnins": settings.max_parallel_burnins,
|
|
"temp_warn_c": settings.temp_warn_c,
|
|
"temp_crit_c": settings.temp_crit_c,
|
|
"bad_block_threshold": settings.bad_block_threshold,
|
|
"surface_validate_block_size": settings.surface_validate_block_size,
|
|
"surface_validate_block_buffer": settings.surface_validate_block_buffer,
|
|
"surface_validate_passes": settings.surface_validate_passes,
|
|
# SSH credentials (take effect immediately — each SSH call reads live settings)
|
|
"ssh_host": settings.ssh_host,
|
|
"ssh_port": settings.ssh_port,
|
|
"ssh_user": settings.ssh_user,
|
|
# Note: ssh_password and ssh_key intentionally omitted from display (sensitive)
|
|
# System settings (restart required to fully apply)
|
|
"truenas_base_url": settings.truenas_base_url,
|
|
"truenas_verify_tls": settings.truenas_verify_tls,
|
|
"poll_interval_seconds": settings.poll_interval_seconds,
|
|
"stale_threshold_seconds": settings.stale_threshold_seconds,
|
|
"allowed_ips": settings.allowed_ips,
|
|
"log_level": settings.log_level,
|
|
# Note: truenas_api_key intentionally omitted from display (sensitive)
|
|
}
|
|
|
|
from app import ssh_client as _ssh
|
|
ps = poller.get_state()
|
|
return templates.TemplateResponse(request, "settings.html", {
|
|
"request": request,
|
|
"editable": editable,
|
|
"secret_status": _secret_status(),
|
|
"smtp_enabled": bool(settings.smtp_host),
|
|
"ssh_configured": _ssh.is_configured(),
|
|
"app_version": settings.app_version,
|
|
"poller": ps,
|
|
**_stale_context(ps),
|
|
})
|
|
|
|
|
|
# Field names that hold secrets and must never be rendered to the UI or
|
|
# included in the redacted-settings dump verbatim.
|
|
_SECRET_FIELDS = ("smtp_password", "ssh_password", "ssh_key", "truenas_api_key")
|
|
|
|
|
|
def _secret_status() -> dict[str, str]:
|
|
"""Per-secret display string for the settings page so the operator can
|
|
see whether each secret is configured (and how) without ever rendering
|
|
the value. Distinguishes env-var, mounted-file, and DB-stored sources
|
|
for ssh_key — the others can only come from the live settings object."""
|
|
import os as _os
|
|
from app.ssh_client import _MOUNTED_KEY_PATH
|
|
|
|
def _has(field: str) -> bool:
|
|
v = getattr(settings, field, "")
|
|
return bool(v)
|
|
|
|
# ssh_key gets the most granular treatment because we actively prefer
|
|
# the mounted file path in production but the textarea is still wired.
|
|
if _os.environ.get("SSH_KEY"):
|
|
ssh_key_status = "set (environment variable)"
|
|
elif _has("ssh_key"):
|
|
ssh_key_status = "set (stored in settings DB — prefer a mounted secret in production)"
|
|
elif _os.path.exists(
|
|
_os.environ.get("SSH_KEY_FILE", _MOUNTED_KEY_PATH)
|
|
):
|
|
ssh_key_status = "set (mounted secret)"
|
|
else:
|
|
ssh_key_status = "unset"
|
|
|
|
return {
|
|
"smtp_password": "set" if _has("smtp_password") else "unset",
|
|
"ssh_password": "set" if _has("ssh_password") else "unset",
|
|
"ssh_key": ssh_key_status,
|
|
"truenas_api_key": "set" if _has("truenas_api_key") else "unset",
|
|
}
|
|
|
|
|
|
@router.get("/api/v1/settings/redacted")
|
|
async def get_settings_redacted(request: Request):
|
|
"""Admin-only diagnostic dump of every editable setting with secrets
|
|
replaced by '***'. Useful for ops triage ("what's actually loaded
|
|
right now?") without leaking the real values into the transcript."""
|
|
user = request.state.current_user
|
|
if not user:
|
|
raise HTTPException(status_code=401, detail="Authentication required")
|
|
if not user.is_admin:
|
|
raise HTTPException(status_code=403, detail="Admin only")
|
|
out: dict[str, object] = {}
|
|
for field in settings_store._EDITABLE.keys():
|
|
val = getattr(settings, field, None)
|
|
if field in _SECRET_FIELDS:
|
|
out[field] = "***" if val else None
|
|
else:
|
|
out[field] = val
|
|
out["_secret_status"] = _secret_status()
|
|
return out
|
|
|
|
|
|
@router.post("/api/v1/settings")
|
|
async def save_settings(request: Request, body: dict):
|
|
"""Save editable runtime settings. Secrets are only updated if non-empty."""
|
|
user = request.state.current_user
|
|
# Don't overwrite secrets if client sent empty string. Track which
|
|
# ones DID get a real change so we can audit the rotation.
|
|
rotated: list[str] = []
|
|
for secret_field in _SECRET_FIELDS:
|
|
if secret_field in body:
|
|
if body[secret_field] == "":
|
|
del body[secret_field]
|
|
else:
|
|
rotated.append(secret_field)
|
|
|
|
try:
|
|
saved = settings_store.save(body)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=422, detail=str(exc))
|
|
|
|
# Audit secret rotations — never log the value, only the field name +
|
|
# operator + source IP. Lets `audit` page answer "who rotated the
|
|
# SMTP password last week?"
|
|
if rotated and user:
|
|
await auth.audit_auth_event(
|
|
"settings_secret_changed",
|
|
user.username,
|
|
f"Rotated secrets from {_client_ip(request)}: {', '.join(sorted(rotated))}",
|
|
)
|
|
|
|
return {"saved": True, "keys": saved, "rotated_secrets": rotated}
|
|
|
|
|
|
@router.post("/api/v1/settings/test-smtp")
|
|
async def test_smtp():
|
|
"""Test the current SMTP configuration without sending an email."""
|
|
result = await mailer.test_smtp_connection()
|
|
if not result["ok"]:
|
|
raise HTTPException(status_code=502, detail=result["error"])
|
|
return {"ok": True}
|
|
|
|
|
|
@router.post("/api/v1/settings/test-ssh")
|
|
async def test_ssh():
|
|
"""Test the current SSH configuration."""
|
|
from app import ssh_client
|
|
result = await ssh_client.test_connection()
|
|
if not result["ok"]:
|
|
raise HTTPException(status_code=502, detail=result.get("error", "Connection failed"))
|
|
return {"ok": True}
|
|
|
|
|
|
@router.websocket("/ws/terminal")
|
|
async def terminal_ws(websocket: WebSocket):
|
|
"""WebSocket endpoint bridging the browser xterm.js terminal to an SSH PTY."""
|
|
from app import terminal as _term
|
|
await _term.handle(websocket)
|
|
|
|
|
|
@router.get("/api/v1/updates/check")
|
|
async def check_updates():
|
|
"""Check for a newer release on Forgejo."""
|
|
import httpx
|
|
current = settings.app_version
|
|
try:
|
|
async with httpx.AsyncClient(timeout=8.0) as client:
|
|
r = await client.get(
|
|
"https://git.hellocomputer.xyz/api/v1/repos/brandon/truenas-burnin/releases/latest",
|
|
headers={"Accept": "application/json"},
|
|
)
|
|
if r.status_code == 200:
|
|
data = r.json()
|
|
latest = data.get("tag_name", "").lstrip("v")
|
|
up_to_date = not latest or latest == current
|
|
return {
|
|
"current": current,
|
|
"latest": latest or None,
|
|
"update_available": not up_to_date,
|
|
"message": None,
|
|
}
|
|
elif r.status_code == 404:
|
|
return {"current": current, "latest": None, "update_available": False,
|
|
"message": "No releases published yet"}
|
|
else:
|
|
return {"current": current, "latest": None, "update_available": False,
|
|
"message": f"Forgejo API returned {r.status_code}"}
|
|
except Exception as exc:
|
|
return {"current": current, "latest": None, "update_available": False,
|
|
"message": f"Could not reach update server: {exc}"}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Print view (must be BEFORE /{job_id} int route)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/history/{job_id}/print", response_class=HTMLResponse)
|
|
async def history_print(
|
|
request: Request,
|
|
job_id: int,
|
|
db: aiosqlite.Connection = Depends(get_db),
|
|
):
|
|
cur = await db.execute("""
|
|
SELECT
|
|
bj.*, d.devname, d.serial, d.model, d.size_bytes,
|
|
CAST(
|
|
(julianday(bj.finished_at) - julianday(bj.started_at)) * 86400
|
|
AS INTEGER
|
|
) AS duration_seconds
|
|
FROM burnin_jobs bj
|
|
JOIN drives d ON d.id = bj.drive_id
|
|
WHERE bj.id = ?
|
|
""", (job_id,))
|
|
row = await cur.fetchone()
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
job = dict(row)
|
|
|
|
cur = await db.execute("""
|
|
SELECT *,
|
|
CAST(
|
|
(julianday(finished_at) - julianday(started_at)) * 86400
|
|
AS INTEGER
|
|
) AS duration_seconds
|
|
FROM burnin_stages WHERE burnin_job_id=? ORDER BY id
|
|
""", (job_id,))
|
|
job["stages"] = [dict(r) for r in await cur.fetchall()]
|
|
|
|
return templates.TemplateResponse(request, "job_print.html", {
|
|
"request": request,
|
|
"job": job,
|
|
})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Burn-in job detail API (must be after export.csv to avoid int coercion)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/api/v1/burnin/{job_id}", response_model=BurninJobResponse)
|
|
async def burnin_get(job_id: int, db: aiosqlite.Connection = Depends(get_db)):
|
|
db.row_factory = aiosqlite.Row
|
|
cur = await db.execute("SELECT * FROM burnin_jobs WHERE id=?", (job_id,))
|
|
row = await cur.fetchone()
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Burn-in job not found")
|
|
cur = await db.execute(
|
|
"SELECT * FROM burnin_stages WHERE burnin_job_id=? ORDER BY id", (job_id,)
|
|
)
|
|
stages = await cur.fetchall()
|
|
return _row_to_burnin(row, stages)
|