fix: wait for Proxmox resize task before VM start (v1.2.0)

- configure_vm: capture resize UPID and call wait_for_task(120s)
  before returning; prevents lock conflict when VM is started right
  after disk resize
- resize_vm_disk / resize_container_disk (x2): same fix applied
- remove unnecessary restart_vm() after manual QEMU disk resize —
  resize is a storage-layer operation and does not require a reboot;
  the auto-restart was causing a second lock conflict

Fixes: can't lock file '/var/lock/qemu-server/lock-xxx.conf' - got timeout
Fixes: command '/usr/bin/qemu-img resize' failed: got timeout
This commit is contained in:
markmorado
2026-03-16 20:23:06 +01:00
parent bd1777f67b
commit 4ef79112be
15 changed files with 342 additions and 156 deletions

View File

@@ -4,6 +4,18 @@ All notable changes to PVEmanager will be documented in this file.
---
## [v1.2.0] - 2026-03-16
### 🐛 Bug Fix — Disk Resize Lock Conflict
- **Root cause fixed** — `resize.put()` in the Proxmox API is asynchronous and returns a UPID task; the old code did not wait for this task to finish before starting the VM, causing Proxmox to report `can't lock file '/var/lock/qemu-server/lock-xxx.conf' — got timeout` and `command '/usr/bin/qemu-img resize' failed: got timeout`
- **`configure_vm` (VM creation from template)** — now captures the resize UPID and calls `wait_for_task(120 s)` before returning; the VM can only be started after the disk has been fully resized
- **`resize_vm_disk`** — same fix applied; the resize task completes before the function returns
- **`resize_container_disk` (both LXC methods)** — same fix; the restart of the container is now issued only after the resize task is confirmed complete
- **Removed unnecessary post-resize VM restart** — QEMU disk resize operates at the storage layer and does not require a VM reboot; the automatic `restart_vm()` call after manual disk resize has been removed to prevent a second lock conflict
---
## [v1.1.8] - 2026-03-16
### 👤 User → Server Assignment

View File

@@ -4,7 +4,7 @@
**Modern web panel for managing Proxmox servers, virtual machines and LXC containers**
[![Version](https://img.shields.io/badge/version-1.1.8-blue?style=flat-square)](CHANGELOG.md)
[![Version](https://img.shields.io/badge/version-1.2.0-blue?style=flat-square)](CHANGELOG.md)
[![License](https://img.shields.io/badge/license-MIT-green?style=flat-square)](LICENSE)
[![Docker](https://img.shields.io/badge/docker-ready-2496ED?style=flat-square&logo=docker&logoColor=white)](compose.yml)
[![Python](https://img.shields.io/badge/python-3.12-3776AB?style=flat-square&logo=python&logoColor=white)](backend/)

View File

@@ -1 +1 @@
1.1.8
1.2.0

View File

@@ -1,6 +1,6 @@
# 📖 PVEmanager - Documentation
> Complete guide for installation, configuration and usage of PVEmanager v1.1.8
> Complete guide for installation, configuration and usage of PVEmanager v1.2.0
---
@@ -475,6 +475,8 @@ RAM: 512MB - 128GB
Disk: Increase size (decrease not possible)
```
> **Note (v1.2.0):** Disk resize is fully asynchronous. The panel waits for the Proxmox resize task to complete before starting the VM, preventing `lock file timeout` errors that occurred when the VM was started immediately after a resize request.
### High Availability (HA)
Requires a Proxmox cluster with HA manager configured.

View File

@@ -512,7 +512,8 @@ def delete_proxmox_server(
):
"""Удалить Proxmox сервер и связанные OS Templates"""
from ...proxmox_client import clear_server_cache
from ...models import OSTemplate, OSTemplateGroup, VMInstance
from ...models import OSTemplate, OSTemplateGroup, VMInstance, Notification
from ...workers import monitoring_worker
server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
if not server:
@@ -558,11 +559,26 @@ def delete_proxmox_server(
logger.info(f"User {current_user.username} deleted Proxmox server: {server_name} ({server_ip})")
# Удаляем уведомления о сервере (offline/online) чтобы они не показывались после удаления
notifications_deleted = db.query(Notification).filter(
Notification.source_id == str(server_id),
Notification.source.in_(["monitoring", "server_monitor"])
).delete(synchronize_session=False)
if notifications_deleted > 0:
logger.info(f"Deleted {notifications_deleted} notifications for server {server_name}")
# Очищаем кеш подключений для этого сервера
clear_server_cache(server_ip)
db.delete(server)
db.commit()
# Очищаем in-memory состояние мониторинга для удалённого сервера
try:
monitoring_worker.cleanup_server_state(server_id)
except Exception as e:
logger.warning(f"Failed to cleanup monitoring state for server {server_id}: {e}")
return None

View File

@@ -1014,15 +1014,10 @@ async def resize_vm_disk(
db.commit()
logger.info(f"Updated disk size in database: VM {vmid} -> {size_gb}GB")
# Перезапускаем VM для применения изменений
restart_success = client.restart_vm(node, vmid)
if restart_success:
logger.info(f"VM {vmid} restarted to apply disk resize")
logger.info(f"User {current_user.username} resized disk {disk} of VM {vmid} to {size}")
return JSONResponse(content={
"status": "success",
"message": f"Размер диска {disk} изменен на {size}. VM перезапускается для применения изменений."
"message": f"Размер диска {disk} изменен на {size}. Для применения изменений внутри ОС выполните resize файловой системы."
})
else:
raise HTTPException(status_code=500, detail="Не удалось изменить размер диска")

View File

@@ -208,6 +208,7 @@
"full_name": "Full Name",
"email": "Email",
"change_password": "Change Password",
"change_credentials": "Change Credentials",
"current_password": "Current Password",
"new_password": "New Password",
"confirm_password": "Confirm Password",
@@ -1147,5 +1148,43 @@
"workspace_name_required": "Enter workspace name",
"failed_to_load_workspace": "Failed to load workspace data",
"failed_to_save": "Failed to save",
"failed_to_delete": "Failed to delete"
"failed_to_delete": "Failed to delete",
"action_executed_successfully": "Action executed successfully",
"are_you_sure_delete": "Are you sure you want to delete",
"are_you_sure_to": "Are you sure you want to",
"assign_user": "Assign User",
"auto": "Auto",
"cancelled": "Cancelled",
"cannot_detect_proxmox_node": "Cannot detect Proxmox node",
"command_sent_successfully": "Command sent successfully",
"config_saved_successfully": "Configuration saved successfully",
"container": "container",
"dashboard": "Dashboard",
"delete_network": "Delete Network",
"delete_resource": "Delete Resource",
"deleted_successfully": "deleted successfully",
"enter_repository_url": "Enter repository URL",
"error_loading_data": "Error loading data",
"error_saving": "Error saving",
"failed_to_execute_command": "Failed to execute command",
"failed_to_load_server_data": "Failed to load server data",
"failed_to_update_server": "Failed to update server",
"id_mismatch_cancelled": "ID mismatch. Deletion cancelled.",
"input_required": "Input Required",
"ip_allocated_successfully": "IP allocated successfully",
"ip_released_successfully": "IP released successfully",
"lxc_template": "LXC Template",
"network_created_successfully": "Network created successfully",
"network_deleted_successfully": "Network deleted successfully",
"no_owner": "No owner",
"no_servers": "No servers",
"notification": "Notification",
"reinstall_error": "Reinstall error",
"reinstall_started": "Reinstall started. VM will be available in a few minutes.",
"release_ip": "Release IP",
"select_all": "Select all",
"send": "Send",
"server_updated_successfully": "Server updated successfully",
"update_check_disabled": "Update check disabled",
"virtual_machine": "virtual machine"
}

View File

@@ -208,6 +208,7 @@
"full_name": "Полное имя",
"email": "Email",
"change_password": "Изменить пароль",
"change_credentials": "Изменить учётные данные",
"current_password": "Текущий пароль",
"new_password": "Новый пароль",
"confirm_password": "Подтвердите пароль",
@@ -1147,5 +1148,43 @@
"workspace_name_required": "Введите название workspace",
"failed_to_load_workspace": "Не удалось загрузить данные workspace",
"failed_to_save": "Не удалось сохранить",
"failed_to_delete": "Не удалось удалить"
"failed_to_delete": "Не удалось удалить",
"action_executed_successfully": "Действие успешно выполнено",
"are_you_sure_delete": "Вы уверены, что хотите удалить",
"are_you_sure_to": "Вы уверены, что хотите",
"assign_user": "Назначить пользователя",
"auto": "Авто",
"cancelled": "Отменено",
"cannot_detect_proxmox_node": "Не удалось определить ноду Proxmox",
"command_sent_successfully": "Команда успешно отправлена",
"config_saved_successfully": "Конфигурация сохранена успешно",
"container": "контейнер",
"dashboard": "Главная",
"delete_network": "Удалить сеть",
"delete_resource": "Удаление ресурса",
"deleted_successfully": "успешно удалён",
"enter_repository_url": "Введите URL репозитория",
"error_loading_data": "Ошибка загрузки данных",
"error_saving": "Ошибка сохранения",
"failed_to_execute_command": "Не удалось выполнить команду",
"failed_to_load_server_data": "Ошибка загрузки данных сервера",
"failed_to_update_server": "Ошибка обновления сервера",
"id_mismatch_cancelled": "ID не совпадает. Удаление отменено.",
"input_required": "Обязательное поле",
"ip_allocated_successfully": "IP успешно выделен",
"ip_released_successfully": "IP успешно освобождён",
"lxc_template": "LXC шаблон",
"network_created_successfully": "Сеть успешно создана",
"network_deleted_successfully": "Сеть успешно удалена",
"no_owner": "Не назначен",
"no_servers": "Нет серверов",
"notification": "Уведомление",
"reinstall_error": "Ошибка переустановки",
"reinstall_started": "Переустановка запущена. VM будет доступна через несколько минут.",
"release_ip": "Освободить IP",
"select_all": "Выбрать все",
"send": "Отправить",
"server_updated_successfully": "Сервер успешно обновлён",
"update_check_disabled": "Проверка обновлений отключена",
"virtual_machine": "виртуальная машина"
}

View File

@@ -1387,10 +1387,13 @@ class ProxmoxClient:
if disk_key:
# Only resize if requested size is larger than current
if disk_size > current_size_gb:
self.proxmox.nodes(node).qemu(vmid).resize.put(
resize_upid = self.proxmox.nodes(node).qemu(vmid).resize.put(
disk=disk_key,
size=f'{disk_size}G'
)
# Wait for resize task to complete before proceeding
if isinstance(resize_upid, str):
self.wait_for_task(node, resize_upid, timeout=120)
logger.info(f"Диск {disk_key} VM {vmid} изменен с {current_size_gb}G до {disk_size}G")
else:
logger.info(f"Диск {disk_key} VM {vmid} уже имеет размер {current_size_gb}G >= запрошенного {disk_size}G")
@@ -1613,7 +1616,10 @@ class ProxmoxClient:
return False
try:
self.proxmox.nodes(node).qemu(vmid).resize.put(disk=disk, size=size)
resize_upid = self.proxmox.nodes(node).qemu(vmid).resize.put(disk=disk, size=size)
# Wait for resize task to complete to avoid lock conflicts
if isinstance(resize_upid, str):
self.wait_for_task(node, resize_upid, timeout=120)
logger.info(f"Размер диска {disk} VM {vmid} изменен на {size}")
return True
except Exception as e:
@@ -1697,7 +1703,10 @@ class ProxmoxClient:
return False
try:
self.proxmox.nodes(node).lxc(vmid).resize.put(disk=disk, size=size)
resize_upid = self.proxmox.nodes(node).lxc(vmid).resize.put(disk=disk, size=size)
# Wait for resize task to complete to avoid lock conflicts
if isinstance(resize_upid, str):
self.wait_for_task(node, resize_upid, timeout=120)
logger.info(f"Размер диска {disk} контейнера {vmid} изменен на {size}")
return True
except Exception as e:
@@ -2375,7 +2384,10 @@ class ProxmoxClient:
return False
try:
self.proxmox.nodes(node).lxc(vmid).resize.put(disk=disk, size=size)
resize_upid = self.proxmox.nodes(node).lxc(vmid).resize.put(disk=disk, size=size)
# Wait for resize task to complete to avoid lock conflicts
if isinstance(resize_upid, str):
self.wait_for_task(node, resize_upid, timeout=120)
logger.info(f"Диск {disk} LXC {vmid} изменен на {size}")
return True
except Exception as e:

View File

@@ -3237,6 +3237,14 @@ body {
font-size: 0.8125rem;
font-weight: 600;
color: var(--text-primary);
flex: 1;
min-width: 0;
}
.node-name-text {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.node-dot-online {

View File

@@ -705,7 +705,12 @@ body.update-in-progress .layout {
{% endif %}
</div>
<!-- System Status -->
{% block sidebar_status %}{% endblock %}
<div class="sidebar-status-section" id="sidebarStatusSection">
<div class="status-chip status-ok" id="sidebarStatusChip">
<span class="status-chip-dot"></span>
<span id="sidebarStatusText">...</span>
</div>
</div>
<!-- Workspace Selector -->
<div class="sidebar-workspace-section" id="sidebarWorkspaceSection">
<div class="sidebar-workspace-label">Workspace</div>
@@ -1457,6 +1462,39 @@ body.update-in-progress .layout {
}
});
async function loadSidebarStatus() {
const chip = document.getElementById('sidebarStatusChip');
const text = document.getElementById('sidebarStatusText');
if (!chip || !text) return;
try {
const resp = await fetchWithAuth('/api/dashboard/alerts', { hideOnForbidden: true });
if (!resp || !resp.ok) return;
const data = await resp.json();
const total = data.total || 0;
const critical = (data.alerts || []).filter(a => a.level === 'critical').length;
chip.classList.remove('status-ok', 'status-warning', 'status-error');
if (critical > 0) {
chip.classList.add('status-error');
chip.style.cursor = 'pointer';
chip.onclick = () => { if (typeof openAlertsModal === 'function') openAlertsModal(); };
text.textContent = `${critical} ${t('critical') || 'critical'}`;
} else if (total > 0) {
chip.classList.add('status-warning');
chip.style.cursor = 'pointer';
chip.onclick = () => { if (typeof openAlertsModal === 'function') openAlertsModal(); };
text.textContent = `${total} ${t('warning') || 'warning'}`;
} else {
chip.classList.add('status-ok');
chip.style.cursor = '';
chip.onclick = null;
text.textContent = t('all_systems_operational') || 'All systems operational';
}
} catch (e) {
const text2 = document.getElementById('sidebarStatusText');
if (text2) text2.textContent = t('all_systems_operational') || 'All systems operational';
}
}
async function loadSidebarWorkspace() {
try {
// Determine if current user is admin
@@ -1958,6 +1996,10 @@ body.update-in-progress .layout {
// Load workspace selector for sidebar
loadSidebarWorkspace();
// Load sidebar status chip
loadSidebarStatus();
setInterval(loadSidebarStatus, 60000);
}
// Add click handlers for sidebar menu items to update page title

View File

@@ -11,23 +11,6 @@
</div>
{% endblock %}
{% block sidebar_status %}
<div class="sidebar-status-section">
<div class="status-chip {% if stats.total_alerts == 0 %}status-ok{% elif stats.critical_alerts > 0 %}status-error{% else %}status-warning{% endif %}" id="dashStatusChip"{% if stats.total_alerts > 0 %} onclick="openAlertsModal()" style="cursor:pointer;" title="{{ t('view_alerts') if t('view_alerts') else 'Посмотреть предупреждения' }}"{% endif %}>
<span class="status-chip-dot"></span>
<span id="dashStatusText">
{%- if stats.total_alerts == 0 -%}
{{ t('all_systems_operational') }}
{%- elif stats.critical_alerts > 0 -%}
{{ stats.critical_alerts }} {{ t('critical') | lower }} {{ t('alerts_label') | lower }}
{%- else -%}
{{ stats.total_alerts }} {{ t('warning') | lower }}{{ 's' if stats.total_alerts != 1 else '' }}
{%- endif -%}
</span>
</div>
</div>
{% endblock %}
{% block content %}
<!-- ======== 5 STAT CARDS ======== -->
<div class="stats-grid-5">
@@ -337,15 +320,15 @@ async function _refreshNodeStats(onlineServers) {
Promise.all(uniqueNodes.map(async ({ server, nodeName, maxVms }) => {
try {
const r = await fetchWithAuth(`/proxmox/api/${server.id}/node/status?node=${nodeName}`, { hideOnForbidden: true });
if (!r || !r.ok) return { nodeName, cpuPct: 0, ramPct: 0, diskPct: 0, vmCount: maxVms };
if (!r || !r.ok) return { nodeName, serverName: server.name, cpuPct: 0, ramPct: 0, diskPct: 0, vmCount: maxVms };
const st = await r.json();
return {
nodeName, vmCount: maxVms,
nodeName, serverName: server.name, vmCount: maxVms,
cpuPct: Math.round((st.cpu || 0) * 100),
ramPct: st.memory && st.memory.total > 0 ? Math.round((st.memory.used / st.memory.total) * 100) : 0,
diskPct: st.rootfs && st.rootfs.total > 0 ? Math.round((st.rootfs.used / st.rootfs.total) * 100) : 0,
};
} catch (e_) { return { nodeName, cpuPct: 0, ramPct: 0, diskPct: 0, vmCount: maxVms }; }
} catch (e_) { return { nodeName, serverName: server.name, cpuPct: 0, ramPct: 0, diskPct: 0, vmCount: maxVms }; }
})),
Promise.all(uniqueNodes.map(async ({ server, nodeName }) => {
try {
@@ -420,7 +403,7 @@ async function _refreshNodeStats(onlineServers) {
<div class="node-item-header">
<div class="node-item-name">
<span class="node-dot-online"></span>
${escapeHtml(node.nodeName)}
<span class="node-name-text">${escapeHtml(node.serverName || node.nodeName)}</span>
</div>
<span class="node-vm-count">${node.vmCount > 0 ? node.vmCount + ' ' + t('vms') : ''}</span>
</div>
@@ -530,7 +513,7 @@ function renderDashboardData(data) {
const allNodes = [];
for (const server of (data.servers || [])) {
if (Array.isArray(server.nodes)) {
for (const node of server.nodes) allNodes.push(node);
for (const node of server.nodes) allNodes.push({ ...node, serverName: server.name });
}
}
if (allNodes.length > 0) {
@@ -583,7 +566,7 @@ function renderDashboardData(data) {
<div class="node-item-header">
<div class="node-item-name">
<span class="node-dot-online"></span>
${escapeHtml(node.name)}
<span class="node-name-text">${escapeHtml(node.serverName || node.name)}</span>
</div>
</div>
<div class="node-bar-row">

View File

@@ -1292,6 +1292,9 @@ function createResourceCard(resource, serverId, type) {
}
// Модальное окно мониторинга
// Track open monitor channel for cleanup
window._monitorWsChannel = null;
function showMonitorModal(serverId, node, vmid, type, name) {
const modal = document.getElementById('monitorModal');
document.getElementById('monitorTitle').innerHTML = `
@@ -1299,75 +1302,85 @@ function showMonitorModal(serverId, node, vmid, type, name) {
<span id="updateIndicator" style="display: inline-block; margin-left: 0.5rem; width: 8px; height: 8px; background: var(--accent); border-radius: 50%; opacity: 0; transition: opacity 0.3s ease;"></span>
`;
document.getElementById('monitorContent').innerHTML = '<div style="text-align: center; padding: 2rem; color: var(--text-muted);"><i class="fa-solid fa-spinner fa-spin"></i> Загрузка данных...</div>';
modal.style.display = 'flex';
// Загрузить данные мониторинга
// Clean up any previous WS subscription
if (window._monitorWsChannel && typeof unsubscribeWsChannel === 'function') {
unsubscribeWsChannel(window._monitorWsChannel);
}
window._monitorWsChannel = `instance_metrics:${serverId}_${vmid}_${type}_${node}`;
// Initial load via HTTP (RRD) + WS status (if cached) — fallback to HTTP
loadMonitorData(serverId, node, vmid, type);
// Обновлять каждые 5 секунд
if (window.monitorInterval) clearInterval(window.monitorInterval);
window.monitorInterval = setInterval(() => {
if (modal.style.display === 'flex') {
loadMonitorData(serverId, node, vmid, type);
}
}, 5000);
// Subscribe to live WS updates
if (typeof subscribeWsChannel === 'function') {
subscribeWsChannel(window._monitorWsChannel);
}
if (typeof registerWsHandler === 'function') {
registerWsHandler('metrics_update', function _monitorWsHandler(msg) {
if (msg.channel !== window._monitorWsChannel) return;
if (!msg.data) return;
const monitorContent = document.getElementById('monitorContent');
if (!monitorContent) return;
const updateIndicator = document.getElementById('updateIndicator');
if (updateIndicator) { updateIndicator.style.opacity = '1'; setTimeout(() => { updateIndicator.style.opacity = '0'; }, 300); }
if (monitorContent.dataset.initialized) {
updateMonitorData(msg.data, null);
} else {
// First time: need RRD too — fetch once then mark initialized
fetchWithAuth(`/proxmox/api/${serverId}/${type}/${vmid}/rrddata?node=${node}&timeframe=hour`)
.then(r => r.ok ? r.json() : null)
.catch(() => null)
.then(rrdData => {
renderMonitorData(msg.data, rrdData || [], type);
monitorContent.dataset.initialized = 'true';
});
}
});
}
}
function closeMonitorModal() {
const modal = document.getElementById('monitorModal');
const monitorContent = document.getElementById('monitorContent');
modal.style.display = 'none';
// Очистить флаг инициализации для следующего открытия
if (monitorContent) {
delete monitorContent.dataset.initialized;
}
if (window.monitorInterval) {
clearInterval(window.monitorInterval);
window.monitorInterval = null;
if (monitorContent) delete monitorContent.dataset.initialized;
// Unsubscribe WS
if (window._monitorWsChannel && typeof unsubscribeWsChannel === 'function') {
unsubscribeWsChannel(window._monitorWsChannel);
window._monitorWsChannel = null;
}
}
async function loadMonitorData(serverId, node, vmid, type) {
const monitorContent = document.getElementById('monitorContent');
const isFirstLoad = !monitorContent.dataset.initialized;
try {
// Показать индикатор обновления
const updateIndicator = document.getElementById('updateIndicator');
if (updateIndicator) {
updateIndicator.style.opacity = '1';
}
if (updateIndicator) updateIndicator.style.opacity = '1';
const [statusRes, rrdRes] = await Promise.all([
fetchWithAuth(`/proxmox/api/${serverId}/${type}/${vmid}/status?node=${node}`),
fetchWithAuth(`/proxmox/api/${serverId}/${type}/${vmid}/rrddata?node=${node}&timeframe=hour`)
]);
if (!statusRes.ok || !rrdRes.ok) {
throw new Error(`HTTP error: ${statusRes.status} / ${rrdRes.status}`);
}
if (!statusRes.ok || !rrdRes.ok) throw new Error(`HTTP error: ${statusRes.status} / ${rrdRes.status}`);
const status = await statusRes.json();
const rrdData = await rrdRes.json();
// Первая загрузка - рендерим всё, последующие - только обновляем данные
if (isFirstLoad) {
renderMonitorData(status, rrdData, type);
monitorContent.dataset.initialized = 'true';
} else {
updateMonitorData(status, rrdData);
}
// Скрыть индикатор
if (updateIndicator) {
setTimeout(() => {
updateIndicator.style.opacity = '0';
}, 300);
}
if (updateIndicator) setTimeout(() => { updateIndicator.style.opacity = '0'; }, 300);
} catch (error) {
console.error('Error loading monitor data:', error);
document.getElementById('monitorContent').innerHTML = `

View File

@@ -1906,29 +1906,77 @@ function revertVMStatus(serverId, vmid) {
delete statusEl.dataset.prevStatus;
}
// Poll VM status until it changes
// Poll VM status until it changes — via WebSocket instance_metrics
async function pollVMStatus(serverId, vmid, type, node, action, attempts = 0, phase = 'initial') {
const maxAttempts = 30;
const pollInterval = 1500;
if (attempts >= maxAttempts) {
loadAllVMs();
// Prefer WS subscription over HTTP polling
if (typeof subscribeWsChannel === 'function' && typeof registerWsHandler === 'function') {
const channel = `instance_metrics:${serverId}_${vmid}_${type}_${node}`;
let _resolved = false;
let _timeoutId = null;
const _cleanup = () => {
clearTimeout(_timeoutId);
if (typeof unsubscribeWsChannel === 'function') unsubscribeWsChannel(channel);
};
// Timeout: fall back to HTTP after 45 s
_timeoutId = setTimeout(() => {
if (_resolved) return;
_resolved = true;
_cleanup();
_pollVMStatusHTTP(serverId, vmid, type, node, action, 0, phase);
}, 45000);
subscribeWsChannel(channel);
registerWsHandler('metrics_update', function _wsHandler(msg) {
if (msg.channel !== channel || _resolved) return;
const newStatus = msg.data && msg.data.status;
if (!newStatus) return;
if (action === 'restart') {
if (phase === 'initial' && newStatus === 'stopped') {
phase = 'restarting';
updateVMStatusUI(serverId, vmid, 'pending', 'restart');
return;
} else if (phase === 'restarting' && newStatus === 'running') {
_resolved = true;
_cleanup();
updateVMStatusUI(serverId, vmid, newStatus);
updateVMActionButton(serverId, vmid, type, node, newStatus);
}
} else {
const expected = action === 'start' ? 'running' : action === 'stop' ? 'stopped' : null;
if (expected && newStatus === expected) {
_resolved = true;
_cleanup();
updateVMStatusUI(serverId, vmid, newStatus);
updateVMActionButton(serverId, vmid, type, node, newStatus);
}
}
});
return;
}
// Fallback: HTTP polling (no WS available)
_pollVMStatusHTTP(serverId, vmid, type, node, action, attempts, phase);
}
async function _pollVMStatusHTTP(serverId, vmid, type, node, action, attempts = 0, phase = 'initial') {
const maxAttempts = 30;
const pollInterval = 1500;
if (attempts >= maxAttempts) { loadAllVMs(); return; }
try {
const apiType = type === 'lxc' ? 'container' : 'vm';
const response = await fetchWithAuth(`/proxmox/api/${serverId}/${apiType}/${vmid}/status?node=${node}`);
if (response && response.ok) {
const data = await response.json();
const newStatus = data.status || 'stopped';
// For restart, we need to wait for stopped first, then running
if (action === 'restart') {
if (phase === 'initial' && newStatus === 'stopped') {
updateVMStatusUI(serverId, vmid, 'pending', 'restart');
setTimeout(() => pollVMStatus(serverId, vmid, type, node, action, 0, 'restarting'), pollInterval);
setTimeout(() => _pollVMStatusHTTP(serverId, vmid, type, node, action, 0, 'restarting'), pollInterval);
return;
} else if (phase === 'restarting' && newStatus === 'running') {
updateVMStatusUI(serverId, vmid, newStatus);
@@ -1936,24 +1984,18 @@ async function pollVMStatus(serverId, vmid, type, node, action, attempts = 0, ph
return;
}
} else {
const expectedStatus = action === 'start' ? 'running' :
action === 'stop' ? 'stopped' : null;
if (expectedStatus && newStatus === expectedStatus) {
const expected = action === 'start' ? 'running' : action === 'stop' ? 'stopped' : null;
if (expected && newStatus === expected) {
updateVMStatusUI(serverId, vmid, newStatus);
updateVMActionButton(serverId, vmid, type, node, newStatus);
return;
}
}
// Continue polling
setTimeout(() => pollVMStatus(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
setTimeout(() => _pollVMStatusHTTP(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
} else {
setTimeout(() => pollVMStatus(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
setTimeout(() => _pollVMStatusHTTP(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
}
} catch (error) {
setTimeout(() => pollVMStatus(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
}
} catch { setTimeout(() => _pollVMStatusHTTP(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval); }
}
// Update action button with new status for context menu
@@ -3200,64 +3242,34 @@ window.addEventListener('DOMContentLoaded', () => {
}, 1000);
}
// Start real-time status monitoring
startStatusMonitoring();
});
// Real-time status monitoring
let statusMonitorInterval = null;
function startStatusMonitoring() {
// Poll for status changes every 10 seconds
statusMonitorInterval = setInterval(async () => {
if (allVMs.length === 0) return;
// Group VMs by server for efficient API calls
const serverGroups = {};
allVMs.forEach(vm => {
if (!serverGroups[vm.server_id]) {
serverGroups[vm.server_id] = [];
}
serverGroups[vm.server_id].push(vm);
});
// Check status for each VM
for (const [serverId, vms] of Object.entries(serverGroups)) {
for (const vm of vms) {
try {
const apiType = vm.type === 'lxc' ? 'container' : 'vm';
const response = await fetchWithAuth(`/proxmox/api/${serverId}/${apiType}/${vm.vmid}/status?node=${vm.node}`);
if (response && response.ok) {
const data = await response.json();
const newStatus = data.status || 'stopped';
// Only update if status changed
if (vm.status !== newStatus) {
console.log(`[Monitor] Status changed for ${vm.name}: ${vm.status} -> ${newStatus}`);
vm.status = newStatus;
updateVMStatusUI(parseInt(serverId), vm.vmid, newStatus);
updateVMActionButton(parseInt(serverId), vm.vmid, vm.type, vm.node, newStatus);
}
// Subscribe to WS dashboard_metrics for live status updates (replaces HTTP polling)
if (typeof subscribeWsChannel === 'function') {
subscribeWsChannel('dashboard_metrics');
}
if (typeof registerWsHandler === 'function') {
registerWsHandler('metrics_update', (msg) => {
if (msg.channel !== 'dashboard_metrics' || !msg.data) return;
if (allVMs.length === 0) return;
// Update only VMs whose status changed (no DOM rebuild)
for (const server of (msg.data.servers || [])) {
const sid = server.id;
for (const item of [...(server.vms || []), ...(server.containers || [])]) {
const vm = allVMs.find(v => v.server_id === sid && v.vmid === item.vmid);
if (!vm) continue;
const newStatus = item.status || 'stopped';
if (vm.status !== newStatus) {
vm.status = newStatus;
updateVMStatusUI(sid, item.vmid, newStatus);
updateVMActionButton(sid, item.vmid, vm.type, vm.node, newStatus);
}
} catch (error) {
// Silently ignore errors during background monitoring
}
}
}
}, 10000); // Every 10 seconds
}
function stopStatusMonitoring() {
if (statusMonitorInterval) {
clearInterval(statusMonitorInterval);
statusMonitorInterval = null;
updateStats();
});
}
}
// Clean up on page unload
window.addEventListener('beforeunload', () => {
stopStatusMonitoring();
window.addEventListener('beforeunload', () => {
if (typeof unsubscribeWsChannel === 'function') unsubscribeWsChannel('dashboard_metrics');
});
});
</script>

View File

@@ -972,6 +972,19 @@ class MonitoringWorker:
finally:
db.close()
def cleanup_server_state(self, server_id: int):
"""Remove all in-memory state for a deleted server to prevent stale data"""
self.last_server_states.pop(server_id, None)
self.last_server_alerts.pop(server_id, None)
# Remove all VM state entries belonging to this server
stale_keys = [k for k in self.last_vm_states if k.startswith(f"{server_id}:")]
for key in stale_keys:
del self.last_vm_states[key]
stale_alert_keys = [k for k in self.last_resource_alerts if k.startswith(f"{server_id}:")]
for key in stale_alert_keys:
del self.last_resource_alerts[key]
logger.info(f"[MONITORING] Cleared in-memory state for deleted server {server_id}")
# Global worker instance
monitoring_worker = MonitoringWorker()