3e0000528f
Go live against real TrueNAS SCALE 25.10: - Remove mock-truenas dependency; mount SSH key as Docker secret - Filter expired disk records from /api/v2.0/disk (expiretime field) - Route all SMART operations through SSH (SCALE 25.10 removed REST smart/test endpoint) - Poll drive temperatures via POST /api/v2.0/disk/temperatures (SCALE-specific) - Store raw smartctl output in smart_tests.raw_output for proof of test execution - Fix percent-remaining=0 false jump to 100% on test start - Fix terminal WebSocket: add mounted key file fallback (/run/secrets/ssh_key) - Fix WebSocket support: uvicorn → uvicorn[standard] (installs websockets) HBA/system sensor temps on dashboard: - SSH to TrueNAS and run sensors -j each poll cycle - Parse coretemp (CPU package) and pch_* (PCH/chipset — storage I/O proxy) - Render as compact chips in stats bar, color-coded green/yellow/red - Live updates via new SSE system-sensors event every 12s Adaptive concurrency signal: - Thermal pressure indicator in stats bar: hidden when OK, WARM/HOT when running burn-in drives hit temp_warn_c / temp_crit_c thresholds - Thermal gate in burn-in queue: jobs wait up to 3 min before acquiring semaphore slot if running drives are already at warning temp; times out and proceeds Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
403 lines
14 KiB
Python
403 lines
14 KiB
Python
"""
|
|
Polling loop — fetches TrueNAS state every POLL_INTERVAL_SECONDS and
|
|
normalizes it into SQLite.
|
|
|
|
Design notes:
|
|
- Opens its own DB connection per cycle (WAL allows concurrent readers).
|
|
- Skips a cycle if TrueNAS is unreachable; marks poller unhealthy.
|
|
- Never overwrites a 'running' state with stale history.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timezone, timedelta
|
|
from typing import Any
|
|
|
|
import aiosqlite
|
|
|
|
from app.config import settings
|
|
from app.truenas import TrueNASClient
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
# Shared state read by the /health endpoint and dashboard template
|
|
_state: dict[str, Any] = {
|
|
"last_poll_at": None,
|
|
"last_error": None,
|
|
"healthy": False,
|
|
"drives_seen": 0,
|
|
"consecutive_failures": 0,
|
|
"system_temps": {}, # {"cpu_c": int|None, "pch_c": int|None}
|
|
"thermal_pressure": "ok", # "ok" | "warn" | "crit" — based on running burn-in drive temps
|
|
}
|
|
|
|
# SSE subscriber queues — notified after each successful poll
|
|
_subscribers: list[asyncio.Queue] = []
|
|
|
|
|
|
def get_state() -> dict:
|
|
return _state.copy()
|
|
|
|
|
|
def subscribe() -> asyncio.Queue:
|
|
q: asyncio.Queue = asyncio.Queue(maxsize=1)
|
|
_subscribers.append(q)
|
|
return q
|
|
|
|
|
|
def unsubscribe(q: asyncio.Queue) -> None:
|
|
try:
|
|
_subscribers.remove(q)
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
def _notify_subscribers(alert: dict | None = None) -> None:
|
|
payload = {"alert": alert}
|
|
for q in list(_subscribers):
|
|
try:
|
|
q.put_nowait(payload)
|
|
except asyncio.QueueFull:
|
|
pass # Client is behind; skip this update
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def _eta_from_progress(percent: int, started_iso: str | None) -> str | None:
|
|
"""Linear ETA extrapolation from elapsed time and percent complete."""
|
|
if not started_iso or percent <= 0:
|
|
return None
|
|
try:
|
|
start = datetime.fromisoformat(started_iso)
|
|
if start.tzinfo is None:
|
|
start = start.replace(tzinfo=timezone.utc)
|
|
elapsed = (datetime.now(timezone.utc) - start).total_seconds()
|
|
total_est = elapsed / (percent / 100)
|
|
remaining = max(0.0, total_est - elapsed)
|
|
return (datetime.now(timezone.utc) + timedelta(seconds=remaining)).isoformat()
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _map_history_state(status: str) -> str:
|
|
return "passed" if "without error" in status.lower() else "failed"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DB helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def _upsert_drive(db: aiosqlite.Connection, disk: dict, now: str) -> int:
|
|
await db.execute(
|
|
"""
|
|
INSERT INTO drives
|
|
(truenas_disk_id, devname, serial, model, size_bytes,
|
|
temperature_c, smart_health, last_seen_at, last_polled_at)
|
|
VALUES (?,?,?,?,?,?,?,?,?)
|
|
ON CONFLICT(truenas_disk_id) DO UPDATE SET
|
|
temperature_c = excluded.temperature_c,
|
|
smart_health = excluded.smart_health,
|
|
last_seen_at = excluded.last_seen_at,
|
|
last_polled_at = excluded.last_polled_at
|
|
""",
|
|
(
|
|
disk["identifier"],
|
|
disk["devname"],
|
|
disk.get("serial"),
|
|
disk.get("model"),
|
|
disk.get("size"),
|
|
disk.get("temperature"),
|
|
disk.get("smart_health", "UNKNOWN"),
|
|
now,
|
|
now,
|
|
),
|
|
)
|
|
cur = await db.execute(
|
|
"SELECT id FROM drives WHERE truenas_disk_id = ?", (disk["identifier"],)
|
|
)
|
|
row = await cur.fetchone()
|
|
return row["id"]
|
|
|
|
|
|
async def _upsert_test(db: aiosqlite.Connection, drive_id: int, ttype: str, data: dict) -> None:
|
|
await db.execute(
|
|
"""
|
|
INSERT INTO smart_tests
|
|
(drive_id, test_type, state, percent, truenas_job_id,
|
|
started_at, eta_at, finished_at, error_text)
|
|
VALUES (?,?,?,?,?,?,?,?,?)
|
|
ON CONFLICT(drive_id, test_type) DO UPDATE SET
|
|
state = excluded.state,
|
|
percent = excluded.percent,
|
|
truenas_job_id = excluded.truenas_job_id,
|
|
started_at = COALESCE(excluded.started_at, smart_tests.started_at),
|
|
eta_at = excluded.eta_at,
|
|
finished_at = excluded.finished_at,
|
|
error_text = excluded.error_text
|
|
""",
|
|
(
|
|
drive_id,
|
|
ttype,
|
|
data["state"],
|
|
data.get("percent", 0),
|
|
data.get("truenas_job_id"),
|
|
data.get("started_at"),
|
|
data.get("eta_at"),
|
|
data.get("finished_at"),
|
|
data.get("error_text"),
|
|
),
|
|
)
|
|
|
|
|
|
async def _apply_running_job(
|
|
db: aiosqlite.Connection, drive_id: int, ttype: str, job: dict
|
|
) -> None:
|
|
pct = job["progress"]["percent"]
|
|
await _upsert_test(db, drive_id, ttype, {
|
|
"state": "running",
|
|
"percent": pct,
|
|
"truenas_job_id": job["id"],
|
|
"started_at": job.get("time_started"),
|
|
"eta_at": _eta_from_progress(pct, job.get("time_started")),
|
|
"finished_at": None,
|
|
"error_text": None,
|
|
})
|
|
|
|
|
|
async def _sync_history(
|
|
db: aiosqlite.Connection,
|
|
client: TrueNASClient,
|
|
drive_id: int,
|
|
devname: str,
|
|
ttype: str,
|
|
) -> None:
|
|
"""Pull most recent completed test from history.
|
|
|
|
This is only called when the drive+type is NOT in the active running-jobs
|
|
dict, so it's safe to overwrite any previous 'running' state — the job
|
|
has finished (or was never started).
|
|
"""
|
|
try:
|
|
results = await client.get_smart_results(devname)
|
|
except Exception:
|
|
return # History fetch failure is non-fatal
|
|
|
|
if not results:
|
|
return
|
|
|
|
for test in results[0].get("tests", []):
|
|
t_name = test.get("type", "").lower()
|
|
is_short = "short" in t_name
|
|
if (ttype == "short") != is_short:
|
|
continue # Wrong test type
|
|
|
|
state = _map_history_state(test.get("status", ""))
|
|
await _upsert_test(db, drive_id, ttype, {
|
|
"state": state,
|
|
"percent": 100 if state == "passed" else 0,
|
|
"truenas_job_id": None,
|
|
"started_at": None,
|
|
"eta_at": None,
|
|
"finished_at": None,
|
|
"error_text": test.get("status_verbose") if state == "failed" else None,
|
|
})
|
|
break # Most recent only
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Poll cycle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def _poll_smart_via_ssh(db: aiosqlite.Connection, now: str) -> None:
|
|
"""
|
|
Poll progress for SMART tests started via SSH (truenas_job_id IS NULL).
|
|
Used on TrueNAS SCALE 25.10+ where the REST smart/test API no longer exists.
|
|
"""
|
|
from app import ssh_client
|
|
if not ssh_client.is_configured():
|
|
return
|
|
|
|
cur = await db.execute(
|
|
"""SELECT st.id, st.test_type, st.drive_id, d.devname, st.started_at
|
|
FROM smart_tests st
|
|
JOIN drives d ON d.id = st.drive_id
|
|
WHERE st.state = 'running' AND st.truenas_job_id IS NULL"""
|
|
)
|
|
rows = await cur.fetchall()
|
|
if not rows:
|
|
return
|
|
|
|
for row in rows:
|
|
test_id, ttype, drive_id, devname, started_at = row[0], row[1], row[2], row[3], row[4]
|
|
try:
|
|
progress = await ssh_client.poll_smart_progress(devname)
|
|
except Exception as exc:
|
|
log.warning("SSH SMART poll failed for %s: %s", devname, exc)
|
|
continue
|
|
|
|
state = progress["state"]
|
|
pct_remaining = progress.get("percent_remaining") # None = not yet in output
|
|
raw_output = progress.get("output", "")
|
|
|
|
if state == "running":
|
|
# pct_remaining=None means smartctl output doesn't have the % line yet
|
|
# (test just started) — keep percent at 0 rather than jumping to 100
|
|
if pct_remaining is None:
|
|
pct = 0
|
|
else:
|
|
pct = max(0, 100 - pct_remaining)
|
|
eta = _eta_from_progress(pct, started_at)
|
|
await db.execute(
|
|
"UPDATE smart_tests SET percent=?, eta_at=?, raw_output=? WHERE id=?",
|
|
(pct, eta, raw_output, test_id),
|
|
)
|
|
elif state == "passed":
|
|
await db.execute(
|
|
"UPDATE smart_tests SET state='passed', percent=100, finished_at=?, raw_output=? WHERE id=?",
|
|
(now, raw_output, test_id),
|
|
)
|
|
log.info("SSH SMART %s passed on %s", ttype, devname)
|
|
elif state == "failed":
|
|
await db.execute(
|
|
"UPDATE smart_tests SET state='failed', percent=0, finished_at=?, "
|
|
"error_text=?, raw_output=? WHERE id=?",
|
|
(now, f"SMART {ttype.upper()} test failed", raw_output, test_id),
|
|
)
|
|
log.warning("SSH SMART %s FAILED on %s", ttype, devname)
|
|
# state == "unknown" → keep polling, no update
|
|
|
|
await db.commit()
|
|
|
|
|
|
async def poll_cycle(client: TrueNASClient) -> int:
|
|
"""Run one full poll. Returns number of drives seen."""
|
|
now = _now()
|
|
|
|
disks = await client.get_disks()
|
|
running_jobs = await client.get_smart_jobs(state="RUNNING")
|
|
|
|
# Fetch temperatures via SCALE-specific endpoint.
|
|
# CORE doesn't have this endpoint — silently skip on any error.
|
|
try:
|
|
temps = await client.get_disk_temperatures()
|
|
except Exception:
|
|
temps = {}
|
|
|
|
# Inject temperature into each disk dict (SCALE 25.10 has no temp in /disk)
|
|
for disk in disks:
|
|
devname = disk.get("devname", "")
|
|
t = temps.get(devname)
|
|
if t is not None:
|
|
disk["temperature"] = int(round(t))
|
|
|
|
# Index running jobs by (devname, test_type)
|
|
active: dict[tuple[str, str], dict] = {}
|
|
for job in running_jobs:
|
|
try:
|
|
args = job["arguments"][0]
|
|
devname = args["disks"][0]
|
|
ttype = args["type"].lower()
|
|
active[(devname, ttype)] = job
|
|
except (KeyError, IndexError, TypeError):
|
|
pass
|
|
|
|
async with aiosqlite.connect(settings.db_path) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
await db.execute("PRAGMA journal_mode=WAL")
|
|
await db.execute("PRAGMA foreign_keys=ON")
|
|
|
|
for disk in disks:
|
|
devname = disk["devname"]
|
|
drive_id = await _upsert_drive(db, disk, now)
|
|
|
|
for ttype in ("short", "long"):
|
|
if (devname, ttype) in active:
|
|
await _apply_running_job(db, drive_id, ttype, active[(devname, ttype)])
|
|
else:
|
|
await _sync_history(db, client, drive_id, devname, ttype)
|
|
|
|
await db.commit()
|
|
|
|
# SSH SMART polling — for tests started via smartctl (no TrueNAS REST job)
|
|
await _poll_smart_via_ssh(db, now)
|
|
|
|
return len(disks)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Background loop
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def run(client: TrueNASClient) -> None:
|
|
log.info("Poller started", extra={"poll_interval": settings.poll_interval_seconds})
|
|
cycle = 0
|
|
while True:
|
|
try:
|
|
count = await poll_cycle(client)
|
|
cycle += 1
|
|
_state["last_poll_at"] = _now()
|
|
_state["last_error"] = None
|
|
_state["healthy"] = True
|
|
_state["drives_seen"] = count
|
|
_state["consecutive_failures"] = 0
|
|
log.debug("Poll OK", extra={"drives": count})
|
|
|
|
# System sensor temps via SSH (non-fatal)
|
|
from app import ssh_client as _ssh
|
|
if _ssh.is_configured():
|
|
try:
|
|
_state["system_temps"] = await _ssh.get_system_sensors()
|
|
except Exception:
|
|
pass
|
|
|
|
# Thermal pressure: max temp of drives currently under burn-in
|
|
try:
|
|
async with aiosqlite.connect(settings.db_path) as _tdb:
|
|
_tdb.row_factory = aiosqlite.Row
|
|
await _tdb.execute("PRAGMA journal_mode=WAL")
|
|
_cur = await _tdb.execute("""
|
|
SELECT MAX(d.temperature_c)
|
|
FROM drives d
|
|
JOIN burnin_jobs bj ON bj.drive_id = d.id
|
|
WHERE bj.state = 'running' AND d.temperature_c IS NOT NULL
|
|
""")
|
|
_row = await _cur.fetchone()
|
|
_max_t = _row[0] if _row and _row[0] is not None else None
|
|
if _max_t is None:
|
|
_state["thermal_pressure"] = "ok"
|
|
elif _max_t >= settings.temp_crit_c:
|
|
_state["thermal_pressure"] = "crit"
|
|
elif _max_t >= settings.temp_warn_c:
|
|
_state["thermal_pressure"] = "warn"
|
|
else:
|
|
_state["thermal_pressure"] = "ok"
|
|
except Exception:
|
|
_state["thermal_pressure"] = "ok"
|
|
|
|
_notify_subscribers()
|
|
|
|
# Check for stuck jobs every 5 cycles (~1 min at default 12s interval)
|
|
if cycle % 5 == 0:
|
|
try:
|
|
from app import burnin as _burnin
|
|
await _burnin.check_stuck_jobs()
|
|
except Exception as exc:
|
|
log.error("Stuck-job check failed: %s", exc)
|
|
|
|
except Exception as exc:
|
|
failures = _state["consecutive_failures"] + 1
|
|
_state["consecutive_failures"] = failures
|
|
_state["last_error"] = str(exc)
|
|
_state["healthy"] = False
|
|
if failures >= 5:
|
|
log.critical(
|
|
"Poller has failed %d consecutive times: %s",
|
|
failures, exc,
|
|
extra={"consecutive_failures": failures},
|
|
)
|
|
else:
|
|
log.error("Poll failed: %s", exc, extra={"consecutive_failures": failures})
|
|
|
|
await asyncio.sleep(settings.poll_interval_seconds)
|