markmorado/pvemanager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

feat(servers): distinguish rejected credentials from unreachable nodes

Browse Source 
An offline server used to report a single "Failed to connect" state, but
"the node does not answer" and "the node answered 401" are fixed in
completely different ways — the first is power or network, the second is a
stale token, which is what happens when a node joins a cluster and its
user.cfg and priv/token.cfg are replaced by the cluster ones.

- add ProxmoxClient.check_connection() returning (ok, error_kind, error);
  token auth surfaces 401 as ResourceException rather than
  AuthenticationError, since proxmoxer validates the token on first use
- store the reason in proxmox_servers.last_error_kind (migration 40) and
  expose it through the server response
- surface an auth_error state in the UI with a hint explaining the usual
  cause

Storage listings now also report per-node availability: the cluster config
declares a storage for the whole cluster but says nothing about whether it
actually came up on a given node, so a missing volume group or pool stayed
invisible. Availability comes from a single /cluster/resources call.
This commit is contained in:
markmorado
2026-07-24 04:42:59 +05:00
parent db7411b1bf
commit 075b8cdaaa
15 changed files with 218 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/app/api/proxmox/backup.py
View File

@@ -39,6 +39,14 @@ def get_storages(
try:
client = _get_proxmox_client(server)
storages = client.get_cluster_storages()
# Конфиг кластера общий для всех нод и ничего не говорит о том, поднялось ли
# хранилище на конкретной ноде. Доступность берём из /cluster/resources —
# один запрос на весь кластер.
node_status = client.get_storage_node_status()
for s in storages:
statuses = node_status.get(s.get("storage"), [])
s["nodes_status"] = statuses
s["inactive_nodes"] = [n["node"] for n in statuses if not n["active"]]
return JSONResponse(content={"storages": storages})
except Exception as e:
logger.error(f"Error listing storages for server {server_id}: {e}")

10
backend/app/api/proxmox/servers.py
View File

@@ -142,10 +142,11 @@ def create_proxmox_server(
# в 30 секунд (период monitoring_worker.run_server_availability_check).
try:
client = _get_proxmox_client(server)
if client.is_connected():
ok, error_kind, check_error = client.check_connection()
if ok:
server.update_status(True)
else:
server.update_status(False, "Failed to connect")
server.update_status(False, check_error or "Failed to connect", error_kind)
db.commit()
db.refresh(server)
except Exception as e:
@@ -297,10 +298,11 @@ def update_proxmox_server(
if creds_changed:
try:
client = _get_proxmox_client(server)
if client.is_connected():
ok, error_kind, check_error = client.check_connection()
if ok:
server.update_status(True)
else:
server.update_status(False, "Failed to connect")
server.update_status(False, check_error or "Failed to connect", error_kind)
db.commit()
db.refresh(server)
if server.is_online:

7
backend/app/models/proxmox.py
View File

@@ -103,6 +103,9 @@ class ProxmoxServer(Base):
is_online = Column(Boolean, nullable=True, default=None)
last_error = Column(Text, nullable=True)
# Почему сервер недоступен: 'auth' — API ответил 401 (протух токен/пароль),
# 'connection' — не отвечает вовсе. Лечится по-разному, поэтому различаем.
last_error_kind = Column(String(20), nullable=True)
__table_args__ = (
Index('idx_proxmox_status', 'is_online', 'last_check'),
@@ -125,13 +128,15 @@ class ProxmoxServer(Base):
'last_check': self.last_check
}
def update_status(self, is_online: bool, error: str = None):
def update_status(self, is_online: bool, error: str = None, error_kind: str = None):
self.is_online = is_online
self.last_check = utcnow()
if error:
self.last_error = error
self.last_error_kind = error_kind or 'connection'
elif is_online:
self.last_error = None
self.last_error_kind = None
assigned_users = relationship("User", secondary="user_servers", back_populates="assigned_servers", lazy="select")

41
backend/app/proxmox/client.py
View File

@@ -140,6 +140,10 @@ class ProxmoxClient(VmMixin, LxcMixin, ClusterMixin, StorageMixin, NetworkMixin,
self.connection_key = f"{host}:{user}:{token_name or 'password'}"
self.proxmox = None
self.last_used = time.time()
# Причина неудачного подключения: 'auth' | 'connection' | None.
# Нужна, чтобы отличить протухший токен от недоступной ноды.
self.connect_error_kind = None
self.connect_error = None
# Check cache first
with connection_cache_lock:
@@ -198,6 +202,13 @@ class ProxmoxClient(VmMixin, LxcMixin, ClusterMixin, StorageMixin, NetworkMixin,
logger.debug(f"Cached connection for {self.connection_key}")
except Exception as e:
logger.error(f"Failed to test Proxmox connection for {host}: {e}")
# При токен-авторизации 401 прилетает не AuthenticationError,
# а ResourceException со status_code — proxmoxer не делает
# отдельный логин, токен проверяется на первом же запросе.
self.connect_error_kind = (
'auth' if getattr(e, 'status_code', None) == 401 else 'connection'
)
self.connect_error = str(e)
self.proxmox = None
# Удаляем неудачное подключение из кеша
with connection_cache_lock:
@@ -206,6 +217,8 @@ class ProxmoxClient(VmMixin, LxcMixin, ClusterMixin, StorageMixin, NetworkMixin,
except AuthenticationError as e:
logger.error(f"Ошибка аутентификации Proxmox {host}: {e}")
self.connect_error_kind = 'auth'
self.connect_error = str(e)
self.proxmox = None
# Удаляем из кеша при ошибке аутентификации
with connection_cache_lock:
@@ -213,6 +226,10 @@ class ProxmoxClient(VmMixin, LxcMixin, ClusterMixin, StorageMixin, NetworkMixin,
del connection_cache[self.connection_key]
except Exception as e:
logger.error(f"Ошибка подключения к Proxmox {host}: {e}")
self.connect_error_kind = (
'auth' if getattr(e, 'status_code', None) == 401 else 'connection'
)
self.connect_error = str(e)
self.proxmox = None
# Удаляем из кеша при ошибке подключения
if self.connection_key in connection_cache:
@@ -229,6 +246,30 @@ class ProxmoxClient(VmMixin, LxcMixin, ClusterMixin, StorageMixin, NetworkMixin,
except Exception:
return False
def check_connection(self) -> tuple:
"""
Проверить API, отличая отказ авторизации от недоступности ноды.
«Нода не отвечает» и «нода ответила 401» чинятся по-разному: первое —
сеть/питание, второе — протухший токен (например, после ввода ноды в
кластер, когда user.cfg и priv/token.cfg замещаются кластерными).
Returns:
(ok, error_kind, error) — error_kind: None | 'auth' | 'connection'
"""
if not self.proxmox:
return (False, self.connect_error_kind or 'connection',
self.connect_error or 'Failed to connect')
try:
self.proxmox.version.get()
self.last_used = time.time()
return (True, None, None)
except AuthenticationError as e:
return (False, 'auth', str(e))
except Exception as e:
kind = 'auth' if getattr(e, 'status_code', None) == 401 else 'connection'
return (False, kind, str(e))
@asynccontextmanager
async def ensure_connection(self):
"""Ensure connection is active before use"""

32
backend/app/proxmox/mixins/storage.py
View File

@@ -152,6 +152,38 @@ class StorageMixin:
logger.error(f"Ошибка получения хранилищ ({content}) ноды {node}: {e}")
return result
def get_storage_node_status(self) -> Dict[str, List[Dict[str, Any]]]:
"""
Доступность каждого хранилища по нодам — одним запросом /cluster/resources.
Конфиг кластера (/storage) не знает, поднялось ли хранилище на конкретной
ноде: VG/пул может быть объявлен, но физически отсутствовать — типичный
случай после ввода новой ноды в кластер, где локальные хранилища ноды
заменяются кластерным конфигом.
Хранилища, отключённые в конфиге (disable), в /cluster/resources не
попадают, поэтому в карте их не будет.
Returns:
{storage_id: [{'node': 'prod', 'active': True}, ...]}
"""
if not self.proxmox:
return {}
status: Dict[str, List[Dict[str, Any]]] = {}
try:
for res in self.get_cluster_resources(type_='storage'):
storage_id = res.get('storage')
node = res.get('node')
if not storage_id or not node:
continue
status.setdefault(storage_id, []).append({
'node': node,
'active': res.get('status') == 'available',
})
except Exception as e:
logger.error(f"Ошибка получения статусов хранилищ с {self.host}: {e}")
return status
def get_node_storages(self, node: str, content: str = None) -> List[Dict]:
"""Get storages available on a specific node, optionally filtered by content type"""
if not self.proxmox:

2
backend/app/schemas.py
View File

@@ -224,6 +224,8 @@ class ProxmoxServerResponse(BaseModel):
last_check: Optional[datetime] = None
is_online: Optional[bool] = None
last_error: Optional[str] = None
# 'auth' — сервер отвечает, но отверг учётные данные; 'connection' — не отвечает
last_error_kind: Optional[str] = None
workspaces: List[ServerWorkspaceBrief] = []
# Exclude sensitive fields

54
backend/app/workers/monitoring_worker.py
View File

@@ -90,6 +90,17 @@ def run_async(coro):
return asyncio.run(coro)
class ProbeFailure(Exception):
"""Неудачная проверка доступности сервера с указанием причины.
error_kind: 'auth' — API ответил 401, 'connection' — не отвечает.
"""
def __init__(self, message: str, error_kind: str = None):
super().__init__(message)
self.error_kind = error_kind or 'connection'
class MonitoringWorker:
"""Background worker for monitoring and generating notifications"""
@@ -112,20 +123,22 @@ class MonitoringWorker:
raise ValueError(f"Server {server.name} has no valid authentication configured")
return ProxmoxClient.from_server(server)
def _notify_server_offline(self, db, server: ProxmoxServer, error: str, users: List[User]):
def _notify_server_offline(self, db, server: ProxmoxServer, error: str, users: List[User],
error_kind: str = None):
"""Send notification that server went offline"""
server_id = server.id
# Check cooldown (don't spam alerts)
last_alert = self.last_server_alerts.get(server_id)
if last_alert:
# 10 minute cooldown between alerts for same server
if (datetime.now().timestamp() - last_alert) < 600:
return
# Update server status in DB
server.is_online = False
server.last_error = error
server.last_error_kind = error_kind or 'connection'
server.last_check = utcnow()
db.commit()
@@ -247,39 +260,42 @@ class MonitoringWorker:
for server in servers:
try:
client = self._create_proxmox_client(server)
# Quick connectivity check - just test if API responds
if client.is_connected():
ok, error_kind, check_error = client.check_connection()
if ok:
# Server is online
was_offline = self.last_server_states.get(server.id) == False
# Update server status in DB
if not server.is_online or was_offline:
server.is_online = True
server.last_error = None
server.last_error_kind = None
server.last_check = utcnow()
db.commit()
# Notify immediately if server recovered
if was_offline:
self._notify_server_recovered(db, server, users)
self.last_server_states[server.id] = True
else:
raise ConnectionError("API not responding")
raise ProbeFailure(check_error or "API not responding", error_kind)
except ValueError as e:
# Configuration error - log but don't mark as offline
logger.warning(f"[SERVER CHECK] Server {server.name} configuration error: {e}")
except Exception as e:
# Server is offline
error_msg = str(e)
was_online = self.last_server_states.get(server.id, True)
# Update server status in DB
server.is_online = False
server.last_error = error_msg[:500]
server.last_error_kind = getattr(e, 'error_kind', None) or 'connection'
server.last_check = utcnow()
db.commit()
@@ -298,7 +314,8 @@ class MonitoringWorker:
self.last_server_alerts[server.id] = now
if was_online:
logger.warning(f"[SERVER CHECK] Server {server.name} went OFFLINE: {error_msg}")
reason = 'AUTH FAILED' if server.last_error_kind == 'auth' else 'OFFLINE'
logger.warning(f"[SERVER CHECK] Server {server.name} went {reason}: {error_msg}")
else:
logger.warning(f"[SERVER CHECK] Server {server.name} still OFFLINE (repeat notification)")
@@ -405,8 +422,12 @@ class MonitoringWorker:
client = self._create_proxmox_client(server)
# Explicitly test connection - this will return False if offline
if not client.is_connected():
raise ConnectionError(f"Cannot connect to Proxmox server at {server.hostname}")
ok, error_kind, check_error = client.check_connection()
if not ok:
raise ProbeFailure(
check_error or f"Cannot connect to Proxmox server at {server.hostname}",
error_kind,
)
# Server is online - notify if it was offline before
self._notify_server_online(db, server, users)
@@ -457,7 +478,8 @@ class MonitoringWorker:
# Connection error - server is offline
error_msg = str(e)
logger.error(f"[MONITORING] Server {server.name} connection failed: {error_msg}")
self._notify_server_offline(db, server, error_msg, users)
self._notify_server_offline(db, server, error_msg, users,
error_kind=getattr(e, 'error_kind', None))
except Exception as e:
logger.error(f"[MONITORING] Critical error in VM status monitoring: {e}", exc_info=True)

16
backend/migrations/migrations.py
View File

@@ -1562,6 +1562,14 @@ def migrate_appstore_default_credentials(conn):
logger.info("✓ App Store catalog_apps.default_credentials column added")
def migrate_server_error_kind(conn):
"""Add last_error_kind column to proxmox_servers ('auth' | 'connection')."""
if not table_exists(conn, 'proxmox_servers'):
return
if add_column_if_not_exists(conn, 'proxmox_servers', 'last_error_kind', 'VARCHAR(20)'):
logger.info("✓ proxmox_servers.last_error_kind column added")
def migrate_script_repo_metadata_format(conn):
"""Add metadata_format column to script_git_repos (header-comment | community-scripts-ct)."""
if not table_exists(conn, 'script_git_repos'):
@@ -2015,6 +2023,14 @@ def run_all_migrations(engine, db_session=None):
logger.warning(f"App Store default_credentials migration: {e}")
conn.rollback()
# Migration 40: proxmox_servers.last_error_kind ('auth' | 'connection')
try:
migrate_server_error_kind(conn)
conn.commit()
except Exception as e:
logger.warning(f"Server last_error_kind migration: {e}")
conn.rollback()
logger.info("=" * 50)
logger.info("✓ All migrations completed successfully")
logger.info("=" * 50)

2
frontend/src/components/shared/status-dot.tsx
View File

@@ -8,6 +8,8 @@ const statusColors: Record<string, string> = {
unknown: 'bg-muted-foreground/30',
online: 'bg-success',
offline: 'bg-danger',
// Сервер отвечает, но отверг учётные данные — чинится не так, как «не отвечает».
auth_error: 'bg-warning',
};
interface StatusDotProps {

32
frontend/src/features/backups/StoragesTab.tsx
View File

@@ -1,6 +1,6 @@
import { useState, useEffect } from 'react';
import { useTranslation } from 'react-i18next';
import { Plus, Pencil, Trash2, Database, CalendarClock, Power, PowerOff } from 'lucide-react';
import { Plus, Pencil, Trash2, Database, CalendarClock, Power, PowerOff, AlertTriangle } from 'lucide-react';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import { Button } from '@/components/ui/button';
import { Checkbox } from '@/components/ui/checkbox';
@@ -50,6 +50,8 @@ interface StorageRow {
used?: number;
total?: number;
avail?: number;
/** Ноды, где хранилище объявлено, но недоступно (нет VG/пула, отвалился mount). */
inactive_nodes?: string[];
}
interface StorageForm {
@@ -563,15 +565,34 @@ export default function StoragesTab() {
<TableHead className="text-right">{t('common.actions')}</TableHead>
</TableRow></TableHeader>
<TableBody>
{storages.map(s => (
<TableRow key={s.storage}>
{storages.map(s => {
const inactiveNodes = s.inactive_nodes ?? [];
return (
<TableRow key={s.storage} className={inactiveNodes.length > 0 ? 'bg-destructive/5' : undefined}>
<TableCell className="font-mono font-medium">
{s.storage}
{s.disable === 1 && <Badge variant="secondary" className="ml-2 text-2xs">{t('storages.disabled')}</Badge>}
{inactiveNodes.length > 0 && (
<Badge
variant="destructive"
className="ml-2 text-2xs"
title={t('storages.inactive_hint', { nodes: inactiveNodes.join(', ') })}
>
<AlertTriangle className="h-3 w-3" />
{t('storages.inactive')}
</Badge>
)}
</TableCell>
<TableCell><Badge variant="outline">{s.type}</Badge></TableCell>
<TableCell className="text-xs">{s.content || '—'}</TableCell>
<TableCell className="text-xs text-muted-foreground">{s.nodes || t('storages.all_nodes')}</TableCell>
<TableCell className="text-xs text-muted-foreground">
{s.nodes || t('storages.all_nodes')}
{inactiveNodes.length > 0 && (
<span className="block text-2xs text-destructive">
{t('storages.inactive_on', { nodes: inactiveNodes.join(', ') })}
</span>
)}
</TableCell>
<TableCell className="text-xs text-muted-foreground">
{s.total ? `${formatBytes(s.used || 0)} / ${formatBytes(s.total)}` : '—'}
</TableCell>
@@ -591,7 +612,8 @@ export default function StoragesTab() {
</div>
</TableCell>
</TableRow>
))}
);
})}
</TableBody>
</Table>
)}

16
frontend/src/features/nodes/NodeDetailPage.tsx
View File

@@ -116,8 +116,20 @@ export default function NodeDetailPage() {
: t('nodes.standalone')}
</Badge>
)}
<Badge variant={server?.is_online ? 'default' : 'destructive'}>
{server?.is_online ? t('common.online') : t('common.offline')}
<Badge
variant={
server?.is_online ? 'default'
: server?.last_error_kind === 'auth' ? 'outline'
: 'destructive'
}
className={server?.last_error_kind === 'auth' && !server?.is_online
? 'border-warning/40 text-warning' : undefined}
title={server?.last_error_kind === 'auth' && !server?.is_online
? t('nodes.auth_error_hint') : server?.last_error || undefined}
>
{server?.is_online ? t('common.online')
: server?.last_error_kind === 'auth' ? t('nodes.auth_error')
: t('common.offline')}
</Badge>
</div>
</div>

17
frontend/src/features/nodes/NodesPage.tsx
View File

@@ -13,6 +13,7 @@ import { useServers, useCreateServer, useUpdateServer, useDeleteServer, useTestS
import { useVirtualMachines } from '@/hooks/use-instances';
import type { ProxmoxServer, ProxmoxServerCreate } from '@/types';
import { formatUptime } from '@/lib/format';
import { cn } from '@/lib/utils';
import { toast } from 'sonner';
import { useConfirm } from '@/components/shared/ConfirmDialog';
import { StatusDot } from '@/components/shared/status-dot';
@@ -358,6 +359,8 @@ export default function NodesPage() {
{servers.map(srv => {
const srvVMs = vms.filter(v => v.server_id === srv.id && v.type === 'qemu');
const srvCTs = vms.filter(v => v.server_id === srv.id && v.type === 'lxc');
// Сервер ответил 401 — он жив, дело в учётных данных, а не в связи.
const authError = !srv.is_online && srv.last_error_kind === 'auth';
return (
<Card key={srv.id} className="relative group">
<CardContent className="p-5 space-y-4">
@@ -446,9 +449,17 @@ export default function NodesPage() {
<span className="font-mono tabular-nums">{srvCTs.length} LXC</span>
</div>
</div>
<Badge variant={srv.is_online ? 'default' : 'destructive'} className="gap-1.5">
<StatusDot status={srv.is_online ? 'online' : 'offline'} pulse className="h-1.5 w-1.5" />
{srv.is_online ? t('common.online') : t('common.offline')}
<Badge
variant={srv.is_online ? 'default' : authError ? 'outline' : 'destructive'}
className={cn('gap-1.5', authError && 'border-warning/40 text-warning')}
title={authError ? t('nodes.auth_error_hint') : srv.last_error || undefined}
>
<StatusDot
status={srv.is_online ? 'online' : authError ? 'auth_error' : 'offline'}
pulse
className="h-1.5 w-1.5"
/>
{srv.is_online ? t('common.online') : authError ? t('nodes.auth_error') : t('common.offline')}
</Badge>
</div>

5
frontend/src/locales/en.json
View File

@@ -589,6 +589,8 @@
},
"nodes": {
"title": "Servers",
"auth_error": "Auth error",
"auth_error_hint": "The server is reachable but rejected the credentials (HTTP 401). Check the API token or password — this usually means the token was revoked or replaced, e.g. after the node joined a cluster.",
"vms": "VMs",
"containers": "Containers",
"cpu": "CPU",
@@ -1056,6 +1058,9 @@
"enabled_ok": "Storage enabled",
"disabled": "Disabled",
"enabled": "Enabled",
"inactive": "Inactive",
"inactive_on": "unavailable on: {{nodes}}",
"inactive_hint": "Declared in the cluster config but not available on: {{nodes}}. The volume group, pool or mount is missing on those nodes.",
"id_required": "Storage ID is required",
"created": "Storage created",
"saved": "Storage updated",

5
frontend/src/locales/ru.json
View File

@@ -589,6 +589,8 @@
},
"nodes": {
"title": "Серверы",
"auth_error": "Ошибка авторизации",
"auth_error_hint": "Сервер доступен, но отверг учётные данные (HTTP 401). Проверьте API-токен или пароль — обычно это значит, что токен отозван или заменён, например после ввода ноды в кластер.",
"vms": "ВМ",
"containers": "Контейнеры",
"cpu": "CPU",
@@ -1056,6 +1058,9 @@
"enabled_ok": "Хранилище включено",
"disabled": "Отключено",
"enabled": "Включено",
"inactive": "Недоступно",
"inactive_on": "недоступно на: {{nodes}}",
"inactive_hint": "Объявлено в конфиге кластера, но недоступно на: {{nodes}}. На этих нодах нет нужной volume group, пула или монтирования.",
"id_required": "Укажите ID хранилища",
"created": "Хранилище создано",
"saved": "Хранилище обновлено",

2
frontend/src/types/index.ts
View File

@@ -60,6 +60,8 @@ export interface ProxmoxServer {
last_check?: string;
is_online?: boolean;
last_error?: string;
/** Почему сервер недоступен: 'auth' — отверг учётные данные, 'connection' — не отвечает. */
last_error_kind?: 'auth' | 'connection' | null;
workspaces?: ServerWorkspaceBrief[];
}