fix(dashboard): не обнулять узлы кластера при REST-рефетче и сбоях WS
Some checks failed
Backend Tests / pytest (Python 3.12) (push) Successful in 56s
Deploy to Production / Trigger production update (push) Failing after 9s

Данные по узлам (nodes) приходят только из WebSocket dashboard_metrics,
а REST /api/resources/all их не содержит. Каждый 15-секундный рефетч REST
затирал ноды -> таблица узлов и счётчик обнулялись, WS возвращал через 2с.

- frontend: при REST-рефетче переносим последние известные nodes из кэша,
  чтобы node-таблица не мигала
- backend: кэш последнего удачного опроса по каждому серверу
  (_server_last_good) — транзиентный таймаут Proxmox больше не отдаёт
  пустые vms/containers/nodes

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
markmorado
2026-07-05 00:56:08 +05:00
parent d4c5c6110a
commit b9e5ef9426
2 changed files with 43 additions and 4 deletions

View File

@@ -44,12 +44,35 @@ _DISK_CACHE_TTL = 30.0
# I/O rate tracking for instances-list channel: key = "server_id:vmid" # I/O rate tracking for instances-list channel: key = "server_id:vmid"
_list_io_prev: Dict[str, Tuple[float, Dict[str, float]]] = {} _list_io_prev: Dict[str, Tuple[float, Dict[str, float]]] = {}
# Last-good per-server dashboard payload: server_id → {vms, containers, nodes}.
# Used so a transient Proxmox timeout on one poll cycle does not blank the
# dashboard (readings would otherwise flash to zero and back every 2 s).
_server_last_good: Dict[int, dict] = {}
def get_cached_metrics(channel: str) -> Optional[Any]: def get_cached_metrics(channel: str) -> Optional[Any]:
"""Return the last cached payload for a channel, or None.""" """Return the last cached payload for a channel, or None."""
return _metrics_cache.get(channel) return _metrics_cache.get(channel)
def _merge_server_cache(server_id: int, base: dict) -> dict:
"""
Fill empty vms/containers/nodes from the last successful poll for this
server, and remember any non-empty values for future degraded cycles.
Keeps the dashboard stable when a single poll returns partial data.
"""
cached = _server_last_good.get(server_id, {})
for key in ("vms", "containers", "nodes"):
if base.get(key):
cached[key] = base[key]
elif cached.get(key):
base[key] = cached[key]
base["vms_count"] = len(base.get("vms") or [])
base["containers_count"] = len(base.get("containers") or [])
_server_last_good[server_id] = cached
return base
# ────────────────────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────────────────────
# Synchronous helpers (run inside a thread via asyncio.to_thread) # Synchronous helpers (run inside a thread via asyncio.to_thread)
# ────────────────────────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────────────────────────
@@ -112,14 +135,15 @@ def _fetch_server_resources_with_nodes(server) -> dict:
base["vms_count"] = len(vms) base["vms_count"] = len(vms)
base["containers_count"] = len(containers) base["containers_count"] = len(containers)
except Exception as exc: except Exception as exc:
# VM/container fetch failed — fall through to node fetch and let the
# last-good cache backfill vms/containers at the end.
base["error"] = str(exc) base["error"] = str(exc)
return base
# ── Node status + RRD mini-slices ───────────────────────────────────── # ── Node status + RRD mini-slices ─────────────────────────────────────
try: try:
client = _get_proxmox_client(server) client = _get_proxmox_client(server)
if not client.is_connected(): if not client.is_connected():
return base return _merge_server_cache(server.id, base)
nodes_raw = client.get_nodes() nodes_raw = client.get_nodes()
now = time.monotonic() now = time.monotonic()
nodes_out = [] nodes_out = []
@@ -170,7 +194,7 @@ def _fetch_server_resources_with_nodes(server) -> dict:
base["nodes"] = nodes_out base["nodes"] = nodes_out
except Exception as exc: except Exception as exc:
logger.debug(f"[metrics_broadcaster] node stats error for server {server.id}: {exc}") logger.debug(f"[metrics_broadcaster] node stats error for server {server.id}: {exc}")
return base return _merge_server_cache(server.id, base)
def _fetch_node_status(db_factory, server_id: int, node: str): def _fetch_node_status(db_factory, server_id: int, node: str):

View File

@@ -370,7 +370,22 @@ export default function DashboardPage() {
const { data: resources } = useQuery({ const { data: resources } = useQuery({
queryKey: ['resources'], queryKey: ['resources'],
queryFn: () => apiClient.get<Rec>('/proxmox/api/resources/all'), queryFn: async () => {
const fresh = await apiClient.get<Rec>('/proxmox/api/resources/all')
// The REST endpoint returns VMs/containers but NOT per-node stats — those
// arrive only via the dashboard_metrics WebSocket. Graft the last known
// nodes so this 15 s refetch doesn't blank the node table each cycle.
const prev = qc.getQueryData<Rec>(['resources'])
const prevServers = (prev?.servers as Rec[]) ?? []
if (prevServers.length) {
const nodesById = new Map(prevServers.map(s => [s.id, s.nodes]))
const servers = ((fresh.servers as Rec[]) ?? []).map(s =>
(s.nodes as Rec[])?.length ? s : { ...s, nodes: nodesById.get(s.id) ?? [] }
)
return { ...fresh, servers }
}
return fresh
},
refetchInterval: 15000, refetchInterval: 15000,
}) })