fix: wait for Proxmox resize task before VM start (v1.2.0)
- configure_vm: capture resize UPID and call wait_for_task(120s) before returning; prevents lock conflict when VM is started right after disk resize - resize_vm_disk / resize_container_disk (x2): same fix applied - remove unnecessary restart_vm() after manual QEMU disk resize — resize is a storage-layer operation and does not require a reboot; the auto-restart was causing a second lock conflict Fixes: can't lock file '/var/lock/qemu-server/lock-xxx.conf' - got timeout Fixes: command '/usr/bin/qemu-img resize' failed: got timeout
This commit is contained in:
12
CHANGELOG.md
12
CHANGELOG.md
@@ -4,6 +4,18 @@ All notable changes to PVEmanager will be documented in this file.
|
||||
|
||||
---
|
||||
|
||||
## [v1.2.0] - 2026-03-16
|
||||
|
||||
### 🐛 Bug Fix — Disk Resize Lock Conflict
|
||||
|
||||
- **Root cause fixed** — `resize.put()` in the Proxmox API is asynchronous and returns a UPID task; the old code did not wait for this task to finish before starting the VM, causing Proxmox to report `can't lock file '/var/lock/qemu-server/lock-xxx.conf' — got timeout` and `command '/usr/bin/qemu-img resize' failed: got timeout`
|
||||
- **`configure_vm` (VM creation from template)** — now captures the resize UPID and calls `wait_for_task(120 s)` before returning; the VM can only be started after the disk has been fully resized
|
||||
- **`resize_vm_disk`** — same fix applied; the resize task completes before the function returns
|
||||
- **`resize_container_disk` (both LXC methods)** — same fix; the restart of the container is now issued only after the resize task is confirmed complete
|
||||
- **Removed unnecessary post-resize VM restart** — QEMU disk resize operates at the storage layer and does not require a VM reboot; the automatic `restart_vm()` call after manual disk resize has been removed to prevent a second lock conflict
|
||||
|
||||
---
|
||||
|
||||
## [v1.1.8] - 2026-03-16
|
||||
|
||||
### 👤 User → Server Assignment
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
**Modern web panel for managing Proxmox servers, virtual machines and LXC containers**
|
||||
|
||||
[](CHANGELOG.md)
|
||||
[](CHANGELOG.md)
|
||||
[](LICENSE)
|
||||
[](compose.yml)
|
||||
[](backend/)
|
||||
|
||||
4
WIKI.md
4
WIKI.md
@@ -1,6 +1,6 @@
|
||||
# 📖 PVEmanager - Documentation
|
||||
|
||||
> Complete guide for installation, configuration and usage of PVEmanager v1.1.8
|
||||
> Complete guide for installation, configuration and usage of PVEmanager v1.2.0
|
||||
|
||||
---
|
||||
|
||||
@@ -475,6 +475,8 @@ RAM: 512MB - 128GB
|
||||
Disk: Increase size (decrease not possible)
|
||||
```
|
||||
|
||||
> **Note (v1.2.0):** Disk resize is fully asynchronous. The panel waits for the Proxmox resize task to complete before starting the VM, preventing `lock file timeout` errors that occurred when the VM was started immediately after a resize request.
|
||||
|
||||
### High Availability (HA)
|
||||
|
||||
Requires a Proxmox cluster with HA manager configured.
|
||||
|
||||
@@ -512,7 +512,8 @@ def delete_proxmox_server(
|
||||
):
|
||||
"""Удалить Proxmox сервер и связанные OS Templates"""
|
||||
from ...proxmox_client import clear_server_cache
|
||||
from ...models import OSTemplate, OSTemplateGroup, VMInstance
|
||||
from ...models import OSTemplate, OSTemplateGroup, VMInstance, Notification
|
||||
from ...workers import monitoring_worker
|
||||
|
||||
server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
|
||||
if not server:
|
||||
@@ -558,11 +559,26 @@ def delete_proxmox_server(
|
||||
|
||||
logger.info(f"User {current_user.username} deleted Proxmox server: {server_name} ({server_ip})")
|
||||
|
||||
# Удаляем уведомления о сервере (offline/online) чтобы они не показывались после удаления
|
||||
notifications_deleted = db.query(Notification).filter(
|
||||
Notification.source_id == str(server_id),
|
||||
Notification.source.in_(["monitoring", "server_monitor"])
|
||||
).delete(synchronize_session=False)
|
||||
if notifications_deleted > 0:
|
||||
logger.info(f"Deleted {notifications_deleted} notifications for server {server_name}")
|
||||
|
||||
# Очищаем кеш подключений для этого сервера
|
||||
clear_server_cache(server_ip)
|
||||
|
||||
db.delete(server)
|
||||
db.commit()
|
||||
|
||||
# Очищаем in-memory состояние мониторинга для удалённого сервера
|
||||
try:
|
||||
monitoring_worker.cleanup_server_state(server_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to cleanup monitoring state for server {server_id}: {e}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -1014,15 +1014,10 @@ async def resize_vm_disk(
|
||||
db.commit()
|
||||
logger.info(f"Updated disk size in database: VM {vmid} -> {size_gb}GB")
|
||||
|
||||
# Перезапускаем VM для применения изменений
|
||||
restart_success = client.restart_vm(node, vmid)
|
||||
if restart_success:
|
||||
logger.info(f"VM {vmid} restarted to apply disk resize")
|
||||
|
||||
logger.info(f"User {current_user.username} resized disk {disk} of VM {vmid} to {size}")
|
||||
return JSONResponse(content={
|
||||
"status": "success",
|
||||
"message": f"Размер диска {disk} изменен на {size}. VM перезапускается для применения изменений."
|
||||
"message": f"Размер диска {disk} изменен на {size}. Для применения изменений внутри ОС выполните resize файловой системы."
|
||||
})
|
||||
else:
|
||||
raise HTTPException(status_code=500, detail="Не удалось изменить размер диска")
|
||||
|
||||
@@ -208,6 +208,7 @@
|
||||
"full_name": "Full Name",
|
||||
"email": "Email",
|
||||
"change_password": "Change Password",
|
||||
"change_credentials": "Change Credentials",
|
||||
"current_password": "Current Password",
|
||||
"new_password": "New Password",
|
||||
"confirm_password": "Confirm Password",
|
||||
@@ -1147,5 +1148,43 @@
|
||||
"workspace_name_required": "Enter workspace name",
|
||||
"failed_to_load_workspace": "Failed to load workspace data",
|
||||
"failed_to_save": "Failed to save",
|
||||
"failed_to_delete": "Failed to delete"
|
||||
"failed_to_delete": "Failed to delete",
|
||||
"action_executed_successfully": "Action executed successfully",
|
||||
"are_you_sure_delete": "Are you sure you want to delete",
|
||||
"are_you_sure_to": "Are you sure you want to",
|
||||
"assign_user": "Assign User",
|
||||
"auto": "Auto",
|
||||
"cancelled": "Cancelled",
|
||||
"cannot_detect_proxmox_node": "Cannot detect Proxmox node",
|
||||
"command_sent_successfully": "Command sent successfully",
|
||||
"config_saved_successfully": "Configuration saved successfully",
|
||||
"container": "container",
|
||||
"dashboard": "Dashboard",
|
||||
"delete_network": "Delete Network",
|
||||
"delete_resource": "Delete Resource",
|
||||
"deleted_successfully": "deleted successfully",
|
||||
"enter_repository_url": "Enter repository URL",
|
||||
"error_loading_data": "Error loading data",
|
||||
"error_saving": "Error saving",
|
||||
"failed_to_execute_command": "Failed to execute command",
|
||||
"failed_to_load_server_data": "Failed to load server data",
|
||||
"failed_to_update_server": "Failed to update server",
|
||||
"id_mismatch_cancelled": "ID mismatch. Deletion cancelled.",
|
||||
"input_required": "Input Required",
|
||||
"ip_allocated_successfully": "IP allocated successfully",
|
||||
"ip_released_successfully": "IP released successfully",
|
||||
"lxc_template": "LXC Template",
|
||||
"network_created_successfully": "Network created successfully",
|
||||
"network_deleted_successfully": "Network deleted successfully",
|
||||
"no_owner": "No owner",
|
||||
"no_servers": "No servers",
|
||||
"notification": "Notification",
|
||||
"reinstall_error": "Reinstall error",
|
||||
"reinstall_started": "Reinstall started. VM will be available in a few minutes.",
|
||||
"release_ip": "Release IP",
|
||||
"select_all": "Select all",
|
||||
"send": "Send",
|
||||
"server_updated_successfully": "Server updated successfully",
|
||||
"update_check_disabled": "Update check disabled",
|
||||
"virtual_machine": "virtual machine"
|
||||
}
|
||||
@@ -208,6 +208,7 @@
|
||||
"full_name": "Полное имя",
|
||||
"email": "Email",
|
||||
"change_password": "Изменить пароль",
|
||||
"change_credentials": "Изменить учётные данные",
|
||||
"current_password": "Текущий пароль",
|
||||
"new_password": "Новый пароль",
|
||||
"confirm_password": "Подтвердите пароль",
|
||||
@@ -1147,5 +1148,43 @@
|
||||
"workspace_name_required": "Введите название workspace",
|
||||
"failed_to_load_workspace": "Не удалось загрузить данные workspace",
|
||||
"failed_to_save": "Не удалось сохранить",
|
||||
"failed_to_delete": "Не удалось удалить"
|
||||
"failed_to_delete": "Не удалось удалить",
|
||||
"action_executed_successfully": "Действие успешно выполнено",
|
||||
"are_you_sure_delete": "Вы уверены, что хотите удалить",
|
||||
"are_you_sure_to": "Вы уверены, что хотите",
|
||||
"assign_user": "Назначить пользователя",
|
||||
"auto": "Авто",
|
||||
"cancelled": "Отменено",
|
||||
"cannot_detect_proxmox_node": "Не удалось определить ноду Proxmox",
|
||||
"command_sent_successfully": "Команда успешно отправлена",
|
||||
"config_saved_successfully": "Конфигурация сохранена успешно",
|
||||
"container": "контейнер",
|
||||
"dashboard": "Главная",
|
||||
"delete_network": "Удалить сеть",
|
||||
"delete_resource": "Удаление ресурса",
|
||||
"deleted_successfully": "успешно удалён",
|
||||
"enter_repository_url": "Введите URL репозитория",
|
||||
"error_loading_data": "Ошибка загрузки данных",
|
||||
"error_saving": "Ошибка сохранения",
|
||||
"failed_to_execute_command": "Не удалось выполнить команду",
|
||||
"failed_to_load_server_data": "Ошибка загрузки данных сервера",
|
||||
"failed_to_update_server": "Ошибка обновления сервера",
|
||||
"id_mismatch_cancelled": "ID не совпадает. Удаление отменено.",
|
||||
"input_required": "Обязательное поле",
|
||||
"ip_allocated_successfully": "IP успешно выделен",
|
||||
"ip_released_successfully": "IP успешно освобождён",
|
||||
"lxc_template": "LXC шаблон",
|
||||
"network_created_successfully": "Сеть успешно создана",
|
||||
"network_deleted_successfully": "Сеть успешно удалена",
|
||||
"no_owner": "Не назначен",
|
||||
"no_servers": "Нет серверов",
|
||||
"notification": "Уведомление",
|
||||
"reinstall_error": "Ошибка переустановки",
|
||||
"reinstall_started": "Переустановка запущена. VM будет доступна через несколько минут.",
|
||||
"release_ip": "Освободить IP",
|
||||
"select_all": "Выбрать все",
|
||||
"send": "Отправить",
|
||||
"server_updated_successfully": "Сервер успешно обновлён",
|
||||
"update_check_disabled": "Проверка обновлений отключена",
|
||||
"virtual_machine": "виртуальная машина"
|
||||
}
|
||||
@@ -1387,10 +1387,13 @@ class ProxmoxClient:
|
||||
if disk_key:
|
||||
# Only resize if requested size is larger than current
|
||||
if disk_size > current_size_gb:
|
||||
self.proxmox.nodes(node).qemu(vmid).resize.put(
|
||||
resize_upid = self.proxmox.nodes(node).qemu(vmid).resize.put(
|
||||
disk=disk_key,
|
||||
size=f'{disk_size}G'
|
||||
)
|
||||
# Wait for resize task to complete before proceeding
|
||||
if isinstance(resize_upid, str):
|
||||
self.wait_for_task(node, resize_upid, timeout=120)
|
||||
logger.info(f"Диск {disk_key} VM {vmid} изменен с {current_size_gb}G до {disk_size}G")
|
||||
else:
|
||||
logger.info(f"Диск {disk_key} VM {vmid} уже имеет размер {current_size_gb}G >= запрошенного {disk_size}G")
|
||||
@@ -1613,7 +1616,10 @@ class ProxmoxClient:
|
||||
return False
|
||||
|
||||
try:
|
||||
self.proxmox.nodes(node).qemu(vmid).resize.put(disk=disk, size=size)
|
||||
resize_upid = self.proxmox.nodes(node).qemu(vmid).resize.put(disk=disk, size=size)
|
||||
# Wait for resize task to complete to avoid lock conflicts
|
||||
if isinstance(resize_upid, str):
|
||||
self.wait_for_task(node, resize_upid, timeout=120)
|
||||
logger.info(f"Размер диска {disk} VM {vmid} изменен на {size}")
|
||||
return True
|
||||
except Exception as e:
|
||||
@@ -1697,7 +1703,10 @@ class ProxmoxClient:
|
||||
return False
|
||||
|
||||
try:
|
||||
self.proxmox.nodes(node).lxc(vmid).resize.put(disk=disk, size=size)
|
||||
resize_upid = self.proxmox.nodes(node).lxc(vmid).resize.put(disk=disk, size=size)
|
||||
# Wait for resize task to complete to avoid lock conflicts
|
||||
if isinstance(resize_upid, str):
|
||||
self.wait_for_task(node, resize_upid, timeout=120)
|
||||
logger.info(f"Размер диска {disk} контейнера {vmid} изменен на {size}")
|
||||
return True
|
||||
except Exception as e:
|
||||
@@ -2375,7 +2384,10 @@ class ProxmoxClient:
|
||||
return False
|
||||
|
||||
try:
|
||||
self.proxmox.nodes(node).lxc(vmid).resize.put(disk=disk, size=size)
|
||||
resize_upid = self.proxmox.nodes(node).lxc(vmid).resize.put(disk=disk, size=size)
|
||||
# Wait for resize task to complete to avoid lock conflicts
|
||||
if isinstance(resize_upid, str):
|
||||
self.wait_for_task(node, resize_upid, timeout=120)
|
||||
logger.info(f"Диск {disk} LXC {vmid} изменен на {size}")
|
||||
return True
|
||||
except Exception as e:
|
||||
|
||||
@@ -3237,6 +3237,14 @@ body {
|
||||
font-size: 0.8125rem;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.node-name-text {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.node-dot-online {
|
||||
|
||||
@@ -705,7 +705,12 @@ body.update-in-progress .layout {
|
||||
{% endif %}
|
||||
</div>
|
||||
<!-- System Status -->
|
||||
{% block sidebar_status %}{% endblock %}
|
||||
<div class="sidebar-status-section" id="sidebarStatusSection">
|
||||
<div class="status-chip status-ok" id="sidebarStatusChip">
|
||||
<span class="status-chip-dot"></span>
|
||||
<span id="sidebarStatusText">...</span>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Workspace Selector -->
|
||||
<div class="sidebar-workspace-section" id="sidebarWorkspaceSection">
|
||||
<div class="sidebar-workspace-label">Workspace</div>
|
||||
@@ -1457,6 +1462,39 @@ body.update-in-progress .layout {
|
||||
}
|
||||
});
|
||||
|
||||
async function loadSidebarStatus() {
|
||||
const chip = document.getElementById('sidebarStatusChip');
|
||||
const text = document.getElementById('sidebarStatusText');
|
||||
if (!chip || !text) return;
|
||||
try {
|
||||
const resp = await fetchWithAuth('/api/dashboard/alerts', { hideOnForbidden: true });
|
||||
if (!resp || !resp.ok) return;
|
||||
const data = await resp.json();
|
||||
const total = data.total || 0;
|
||||
const critical = (data.alerts || []).filter(a => a.level === 'critical').length;
|
||||
chip.classList.remove('status-ok', 'status-warning', 'status-error');
|
||||
if (critical > 0) {
|
||||
chip.classList.add('status-error');
|
||||
chip.style.cursor = 'pointer';
|
||||
chip.onclick = () => { if (typeof openAlertsModal === 'function') openAlertsModal(); };
|
||||
text.textContent = `${critical} ${t('critical') || 'critical'}`;
|
||||
} else if (total > 0) {
|
||||
chip.classList.add('status-warning');
|
||||
chip.style.cursor = 'pointer';
|
||||
chip.onclick = () => { if (typeof openAlertsModal === 'function') openAlertsModal(); };
|
||||
text.textContent = `${total} ${t('warning') || 'warning'}`;
|
||||
} else {
|
||||
chip.classList.add('status-ok');
|
||||
chip.style.cursor = '';
|
||||
chip.onclick = null;
|
||||
text.textContent = t('all_systems_operational') || 'All systems operational';
|
||||
}
|
||||
} catch (e) {
|
||||
const text2 = document.getElementById('sidebarStatusText');
|
||||
if (text2) text2.textContent = t('all_systems_operational') || 'All systems operational';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSidebarWorkspace() {
|
||||
try {
|
||||
// Determine if current user is admin
|
||||
@@ -1958,6 +1996,10 @@ body.update-in-progress .layout {
|
||||
|
||||
// Load workspace selector for sidebar
|
||||
loadSidebarWorkspace();
|
||||
|
||||
// Load sidebar status chip
|
||||
loadSidebarStatus();
|
||||
setInterval(loadSidebarStatus, 60000);
|
||||
}
|
||||
|
||||
// Add click handlers for sidebar menu items to update page title
|
||||
|
||||
@@ -11,23 +11,6 @@
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block sidebar_status %}
|
||||
<div class="sidebar-status-section">
|
||||
<div class="status-chip {% if stats.total_alerts == 0 %}status-ok{% elif stats.critical_alerts > 0 %}status-error{% else %}status-warning{% endif %}" id="dashStatusChip"{% if stats.total_alerts > 0 %} onclick="openAlertsModal()" style="cursor:pointer;" title="{{ t('view_alerts') if t('view_alerts') else 'Посмотреть предупреждения' }}"{% endif %}>
|
||||
<span class="status-chip-dot"></span>
|
||||
<span id="dashStatusText">
|
||||
{%- if stats.total_alerts == 0 -%}
|
||||
{{ t('all_systems_operational') }}
|
||||
{%- elif stats.critical_alerts > 0 -%}
|
||||
{{ stats.critical_alerts }} {{ t('critical') | lower }} {{ t('alerts_label') | lower }}
|
||||
{%- else -%}
|
||||
{{ stats.total_alerts }} {{ t('warning') | lower }}{{ 's' if stats.total_alerts != 1 else '' }}
|
||||
{%- endif -%}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<!-- ======== 5 STAT CARDS ======== -->
|
||||
<div class="stats-grid-5">
|
||||
@@ -337,15 +320,15 @@ async function _refreshNodeStats(onlineServers) {
|
||||
Promise.all(uniqueNodes.map(async ({ server, nodeName, maxVms }) => {
|
||||
try {
|
||||
const r = await fetchWithAuth(`/proxmox/api/${server.id}/node/status?node=${nodeName}`, { hideOnForbidden: true });
|
||||
if (!r || !r.ok) return { nodeName, cpuPct: 0, ramPct: 0, diskPct: 0, vmCount: maxVms };
|
||||
if (!r || !r.ok) return { nodeName, serverName: server.name, cpuPct: 0, ramPct: 0, diskPct: 0, vmCount: maxVms };
|
||||
const st = await r.json();
|
||||
return {
|
||||
nodeName, vmCount: maxVms,
|
||||
nodeName, serverName: server.name, vmCount: maxVms,
|
||||
cpuPct: Math.round((st.cpu || 0) * 100),
|
||||
ramPct: st.memory && st.memory.total > 0 ? Math.round((st.memory.used / st.memory.total) * 100) : 0,
|
||||
diskPct: st.rootfs && st.rootfs.total > 0 ? Math.round((st.rootfs.used / st.rootfs.total) * 100) : 0,
|
||||
};
|
||||
} catch (e_) { return { nodeName, cpuPct: 0, ramPct: 0, diskPct: 0, vmCount: maxVms }; }
|
||||
} catch (e_) { return { nodeName, serverName: server.name, cpuPct: 0, ramPct: 0, diskPct: 0, vmCount: maxVms }; }
|
||||
})),
|
||||
Promise.all(uniqueNodes.map(async ({ server, nodeName }) => {
|
||||
try {
|
||||
@@ -420,7 +403,7 @@ async function _refreshNodeStats(onlineServers) {
|
||||
<div class="node-item-header">
|
||||
<div class="node-item-name">
|
||||
<span class="node-dot-online"></span>
|
||||
${escapeHtml(node.nodeName)}
|
||||
<span class="node-name-text">${escapeHtml(node.serverName || node.nodeName)}</span>
|
||||
</div>
|
||||
<span class="node-vm-count">${node.vmCount > 0 ? node.vmCount + ' ' + t('vms') : ''}</span>
|
||||
</div>
|
||||
@@ -530,7 +513,7 @@ function renderDashboardData(data) {
|
||||
const allNodes = [];
|
||||
for (const server of (data.servers || [])) {
|
||||
if (Array.isArray(server.nodes)) {
|
||||
for (const node of server.nodes) allNodes.push(node);
|
||||
for (const node of server.nodes) allNodes.push({ ...node, serverName: server.name });
|
||||
}
|
||||
}
|
||||
if (allNodes.length > 0) {
|
||||
@@ -583,7 +566,7 @@ function renderDashboardData(data) {
|
||||
<div class="node-item-header">
|
||||
<div class="node-item-name">
|
||||
<span class="node-dot-online"></span>
|
||||
${escapeHtml(node.name)}
|
||||
<span class="node-name-text">${escapeHtml(node.serverName || node.name)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="node-bar-row">
|
||||
|
||||
@@ -1292,6 +1292,9 @@ function createResourceCard(resource, serverId, type) {
|
||||
}
|
||||
|
||||
// Модальное окно мониторинга
|
||||
// Track open monitor channel for cleanup
|
||||
window._monitorWsChannel = null;
|
||||
|
||||
function showMonitorModal(serverId, node, vmid, type, name) {
|
||||
const modal = document.getElementById('monitorModal');
|
||||
document.getElementById('monitorTitle').innerHTML = `
|
||||
@@ -1299,75 +1302,85 @@ function showMonitorModal(serverId, node, vmid, type, name) {
|
||||
<span id="updateIndicator" style="display: inline-block; margin-left: 0.5rem; width: 8px; height: 8px; background: var(--accent); border-radius: 50%; opacity: 0; transition: opacity 0.3s ease;"></span>
|
||||
`;
|
||||
document.getElementById('monitorContent').innerHTML = '<div style="text-align: center; padding: 2rem; color: var(--text-muted);"><i class="fa-solid fa-spinner fa-spin"></i> Загрузка данных...</div>';
|
||||
|
||||
modal.style.display = 'flex';
|
||||
|
||||
// Загрузить данные мониторинга
|
||||
|
||||
// Clean up any previous WS subscription
|
||||
if (window._monitorWsChannel && typeof unsubscribeWsChannel === 'function') {
|
||||
unsubscribeWsChannel(window._monitorWsChannel);
|
||||
}
|
||||
window._monitorWsChannel = `instance_metrics:${serverId}_${vmid}_${type}_${node}`;
|
||||
|
||||
// Initial load via HTTP (RRD) + WS status (if cached) — fallback to HTTP
|
||||
loadMonitorData(serverId, node, vmid, type);
|
||||
|
||||
// Обновлять каждые 5 секунд
|
||||
if (window.monitorInterval) clearInterval(window.monitorInterval);
|
||||
window.monitorInterval = setInterval(() => {
|
||||
if (modal.style.display === 'flex') {
|
||||
loadMonitorData(serverId, node, vmid, type);
|
||||
}
|
||||
}, 5000);
|
||||
|
||||
// Subscribe to live WS updates
|
||||
if (typeof subscribeWsChannel === 'function') {
|
||||
subscribeWsChannel(window._monitorWsChannel);
|
||||
}
|
||||
if (typeof registerWsHandler === 'function') {
|
||||
registerWsHandler('metrics_update', function _monitorWsHandler(msg) {
|
||||
if (msg.channel !== window._monitorWsChannel) return;
|
||||
if (!msg.data) return;
|
||||
const monitorContent = document.getElementById('monitorContent');
|
||||
if (!monitorContent) return;
|
||||
|
||||
const updateIndicator = document.getElementById('updateIndicator');
|
||||
if (updateIndicator) { updateIndicator.style.opacity = '1'; setTimeout(() => { updateIndicator.style.opacity = '0'; }, 300); }
|
||||
|
||||
if (monitorContent.dataset.initialized) {
|
||||
updateMonitorData(msg.data, null);
|
||||
} else {
|
||||
// First time: need RRD too — fetch once then mark initialized
|
||||
fetchWithAuth(`/proxmox/api/${serverId}/${type}/${vmid}/rrddata?node=${node}&timeframe=hour`)
|
||||
.then(r => r.ok ? r.json() : null)
|
||||
.catch(() => null)
|
||||
.then(rrdData => {
|
||||
renderMonitorData(msg.data, rrdData || [], type);
|
||||
monitorContent.dataset.initialized = 'true';
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function closeMonitorModal() {
|
||||
const modal = document.getElementById('monitorModal');
|
||||
const monitorContent = document.getElementById('monitorContent');
|
||||
|
||||
modal.style.display = 'none';
|
||||
|
||||
// Очистить флаг инициализации для следующего открытия
|
||||
if (monitorContent) {
|
||||
delete monitorContent.dataset.initialized;
|
||||
}
|
||||
|
||||
if (window.monitorInterval) {
|
||||
clearInterval(window.monitorInterval);
|
||||
window.monitorInterval = null;
|
||||
if (monitorContent) delete monitorContent.dataset.initialized;
|
||||
// Unsubscribe WS
|
||||
if (window._monitorWsChannel && typeof unsubscribeWsChannel === 'function') {
|
||||
unsubscribeWsChannel(window._monitorWsChannel);
|
||||
window._monitorWsChannel = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadMonitorData(serverId, node, vmid, type) {
|
||||
const monitorContent = document.getElementById('monitorContent');
|
||||
const isFirstLoad = !monitorContent.dataset.initialized;
|
||||
|
||||
|
||||
try {
|
||||
// Показать индикатор обновления
|
||||
const updateIndicator = document.getElementById('updateIndicator');
|
||||
if (updateIndicator) {
|
||||
updateIndicator.style.opacity = '1';
|
||||
}
|
||||
|
||||
if (updateIndicator) updateIndicator.style.opacity = '1';
|
||||
|
||||
const [statusRes, rrdRes] = await Promise.all([
|
||||
fetchWithAuth(`/proxmox/api/${serverId}/${type}/${vmid}/status?node=${node}`),
|
||||
fetchWithAuth(`/proxmox/api/${serverId}/${type}/${vmid}/rrddata?node=${node}&timeframe=hour`)
|
||||
]);
|
||||
|
||||
if (!statusRes.ok || !rrdRes.ok) {
|
||||
throw new Error(`HTTP error: ${statusRes.status} / ${rrdRes.status}`);
|
||||
}
|
||||
|
||||
|
||||
if (!statusRes.ok || !rrdRes.ok) throw new Error(`HTTP error: ${statusRes.status} / ${rrdRes.status}`);
|
||||
|
||||
const status = await statusRes.json();
|
||||
const rrdData = await rrdRes.json();
|
||||
|
||||
// Первая загрузка - рендерим всё, последующие - только обновляем данные
|
||||
|
||||
if (isFirstLoad) {
|
||||
renderMonitorData(status, rrdData, type);
|
||||
monitorContent.dataset.initialized = 'true';
|
||||
} else {
|
||||
updateMonitorData(status, rrdData);
|
||||
}
|
||||
|
||||
// Скрыть индикатор
|
||||
if (updateIndicator) {
|
||||
setTimeout(() => {
|
||||
updateIndicator.style.opacity = '0';
|
||||
}, 300);
|
||||
}
|
||||
|
||||
if (updateIndicator) setTimeout(() => { updateIndicator.style.opacity = '0'; }, 300);
|
||||
} catch (error) {
|
||||
console.error('Error loading monitor data:', error);
|
||||
document.getElementById('monitorContent').innerHTML = `
|
||||
|
||||
@@ -1906,29 +1906,77 @@ function revertVMStatus(serverId, vmid) {
|
||||
delete statusEl.dataset.prevStatus;
|
||||
}
|
||||
|
||||
// Poll VM status until it changes
|
||||
// Poll VM status until it changes — via WebSocket instance_metrics
|
||||
async function pollVMStatus(serverId, vmid, type, node, action, attempts = 0, phase = 'initial') {
|
||||
const maxAttempts = 30;
|
||||
const pollInterval = 1500;
|
||||
|
||||
if (attempts >= maxAttempts) {
|
||||
loadAllVMs();
|
||||
|
||||
// Prefer WS subscription over HTTP polling
|
||||
if (typeof subscribeWsChannel === 'function' && typeof registerWsHandler === 'function') {
|
||||
const channel = `instance_metrics:${serverId}_${vmid}_${type}_${node}`;
|
||||
let _resolved = false;
|
||||
let _timeoutId = null;
|
||||
|
||||
const _cleanup = () => {
|
||||
clearTimeout(_timeoutId);
|
||||
if (typeof unsubscribeWsChannel === 'function') unsubscribeWsChannel(channel);
|
||||
};
|
||||
|
||||
// Timeout: fall back to HTTP after 45 s
|
||||
_timeoutId = setTimeout(() => {
|
||||
if (_resolved) return;
|
||||
_resolved = true;
|
||||
_cleanup();
|
||||
_pollVMStatusHTTP(serverId, vmid, type, node, action, 0, phase);
|
||||
}, 45000);
|
||||
|
||||
subscribeWsChannel(channel);
|
||||
registerWsHandler('metrics_update', function _wsHandler(msg) {
|
||||
if (msg.channel !== channel || _resolved) return;
|
||||
const newStatus = msg.data && msg.data.status;
|
||||
if (!newStatus) return;
|
||||
|
||||
if (action === 'restart') {
|
||||
if (phase === 'initial' && newStatus === 'stopped') {
|
||||
phase = 'restarting';
|
||||
updateVMStatusUI(serverId, vmid, 'pending', 'restart');
|
||||
return;
|
||||
} else if (phase === 'restarting' && newStatus === 'running') {
|
||||
_resolved = true;
|
||||
_cleanup();
|
||||
updateVMStatusUI(serverId, vmid, newStatus);
|
||||
updateVMActionButton(serverId, vmid, type, node, newStatus);
|
||||
}
|
||||
} else {
|
||||
const expected = action === 'start' ? 'running' : action === 'stop' ? 'stopped' : null;
|
||||
if (expected && newStatus === expected) {
|
||||
_resolved = true;
|
||||
_cleanup();
|
||||
updateVMStatusUI(serverId, vmid, newStatus);
|
||||
updateVMActionButton(serverId, vmid, type, node, newStatus);
|
||||
}
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
// Fallback: HTTP polling (no WS available)
|
||||
_pollVMStatusHTTP(serverId, vmid, type, node, action, attempts, phase);
|
||||
}
|
||||
|
||||
async function _pollVMStatusHTTP(serverId, vmid, type, node, action, attempts = 0, phase = 'initial') {
|
||||
const maxAttempts = 30;
|
||||
const pollInterval = 1500;
|
||||
if (attempts >= maxAttempts) { loadAllVMs(); return; }
|
||||
try {
|
||||
const apiType = type === 'lxc' ? 'container' : 'vm';
|
||||
const response = await fetchWithAuth(`/proxmox/api/${serverId}/${apiType}/${vmid}/status?node=${node}`);
|
||||
|
||||
if (response && response.ok) {
|
||||
const data = await response.json();
|
||||
const newStatus = data.status || 'stopped';
|
||||
|
||||
// For restart, we need to wait for stopped first, then running
|
||||
if (action === 'restart') {
|
||||
if (phase === 'initial' && newStatus === 'stopped') {
|
||||
updateVMStatusUI(serverId, vmid, 'pending', 'restart');
|
||||
setTimeout(() => pollVMStatus(serverId, vmid, type, node, action, 0, 'restarting'), pollInterval);
|
||||
setTimeout(() => _pollVMStatusHTTP(serverId, vmid, type, node, action, 0, 'restarting'), pollInterval);
|
||||
return;
|
||||
} else if (phase === 'restarting' && newStatus === 'running') {
|
||||
updateVMStatusUI(serverId, vmid, newStatus);
|
||||
@@ -1936,24 +1984,18 @@ async function pollVMStatus(serverId, vmid, type, node, action, attempts = 0, ph
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
const expectedStatus = action === 'start' ? 'running' :
|
||||
action === 'stop' ? 'stopped' : null;
|
||||
|
||||
if (expectedStatus && newStatus === expectedStatus) {
|
||||
const expected = action === 'start' ? 'running' : action === 'stop' ? 'stopped' : null;
|
||||
if (expected && newStatus === expected) {
|
||||
updateVMStatusUI(serverId, vmid, newStatus);
|
||||
updateVMActionButton(serverId, vmid, type, node, newStatus);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Continue polling
|
||||
setTimeout(() => pollVMStatus(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
|
||||
setTimeout(() => _pollVMStatusHTTP(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
|
||||
} else {
|
||||
setTimeout(() => pollVMStatus(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
|
||||
setTimeout(() => _pollVMStatusHTTP(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
|
||||
}
|
||||
} catch (error) {
|
||||
setTimeout(() => pollVMStatus(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval);
|
||||
}
|
||||
} catch { setTimeout(() => _pollVMStatusHTTP(serverId, vmid, type, node, action, attempts + 1, phase), pollInterval); }
|
||||
}
|
||||
|
||||
// Update action button with new status for context menu
|
||||
@@ -3200,64 +3242,34 @@ window.addEventListener('DOMContentLoaded', () => {
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
// Start real-time status monitoring
|
||||
startStatusMonitoring();
|
||||
});
|
||||
|
||||
// Real-time status monitoring
|
||||
let statusMonitorInterval = null;
|
||||
|
||||
function startStatusMonitoring() {
|
||||
// Poll for status changes every 10 seconds
|
||||
statusMonitorInterval = setInterval(async () => {
|
||||
if (allVMs.length === 0) return;
|
||||
|
||||
// Group VMs by server for efficient API calls
|
||||
const serverGroups = {};
|
||||
allVMs.forEach(vm => {
|
||||
if (!serverGroups[vm.server_id]) {
|
||||
serverGroups[vm.server_id] = [];
|
||||
}
|
||||
serverGroups[vm.server_id].push(vm);
|
||||
});
|
||||
|
||||
// Check status for each VM
|
||||
for (const [serverId, vms] of Object.entries(serverGroups)) {
|
||||
for (const vm of vms) {
|
||||
try {
|
||||
const apiType = vm.type === 'lxc' ? 'container' : 'vm';
|
||||
const response = await fetchWithAuth(`/proxmox/api/${serverId}/${apiType}/${vm.vmid}/status?node=${vm.node}`);
|
||||
|
||||
if (response && response.ok) {
|
||||
const data = await response.json();
|
||||
const newStatus = data.status || 'stopped';
|
||||
|
||||
// Only update if status changed
|
||||
if (vm.status !== newStatus) {
|
||||
console.log(`[Monitor] Status changed for ${vm.name}: ${vm.status} -> ${newStatus}`);
|
||||
vm.status = newStatus;
|
||||
updateVMStatusUI(parseInt(serverId), vm.vmid, newStatus);
|
||||
updateVMActionButton(parseInt(serverId), vm.vmid, vm.type, vm.node, newStatus);
|
||||
}
|
||||
// Subscribe to WS dashboard_metrics for live status updates (replaces HTTP polling)
|
||||
if (typeof subscribeWsChannel === 'function') {
|
||||
subscribeWsChannel('dashboard_metrics');
|
||||
}
|
||||
if (typeof registerWsHandler === 'function') {
|
||||
registerWsHandler('metrics_update', (msg) => {
|
||||
if (msg.channel !== 'dashboard_metrics' || !msg.data) return;
|
||||
if (allVMs.length === 0) return;
|
||||
// Update only VMs whose status changed (no DOM rebuild)
|
||||
for (const server of (msg.data.servers || [])) {
|
||||
const sid = server.id;
|
||||
for (const item of [...(server.vms || []), ...(server.containers || [])]) {
|
||||
const vm = allVMs.find(v => v.server_id === sid && v.vmid === item.vmid);
|
||||
if (!vm) continue;
|
||||
const newStatus = item.status || 'stopped';
|
||||
if (vm.status !== newStatus) {
|
||||
vm.status = newStatus;
|
||||
updateVMStatusUI(sid, item.vmid, newStatus);
|
||||
updateVMActionButton(sid, item.vmid, vm.type, vm.node, newStatus);
|
||||
}
|
||||
} catch (error) {
|
||||
// Silently ignore errors during background monitoring
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 10000); // Every 10 seconds
|
||||
}
|
||||
|
||||
function stopStatusMonitoring() {
|
||||
if (statusMonitorInterval) {
|
||||
clearInterval(statusMonitorInterval);
|
||||
statusMonitorInterval = null;
|
||||
updateStats();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up on page unload
|
||||
window.addEventListener('beforeunload', () => {
|
||||
stopStatusMonitoring();
|
||||
window.addEventListener('beforeunload', () => {
|
||||
if (typeof unsubscribeWsChannel === 'function') unsubscribeWsChannel('dashboard_metrics');
|
||||
});
|
||||
});
|
||||
</script>
|
||||
|
||||
|
||||
@@ -972,6 +972,19 @@ class MonitoringWorker:
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def cleanup_server_state(self, server_id: int):
|
||||
"""Remove all in-memory state for a deleted server to prevent stale data"""
|
||||
self.last_server_states.pop(server_id, None)
|
||||
self.last_server_alerts.pop(server_id, None)
|
||||
# Remove all VM state entries belonging to this server
|
||||
stale_keys = [k for k in self.last_vm_states if k.startswith(f"{server_id}:")]
|
||||
for key in stale_keys:
|
||||
del self.last_vm_states[key]
|
||||
stale_alert_keys = [k for k in self.last_resource_alerts if k.startswith(f"{server_id}:")]
|
||||
for key in stale_alert_keys:
|
||||
del self.last_resource_alerts[key]
|
||||
logger.info(f"[MONITORING] Cleared in-memory state for deleted server {server_id}")
|
||||
|
||||
|
||||
# Global worker instance
|
||||
monitoring_worker = MonitoringWorker()
|
||||
|
||||
Reference in New Issue
Block a user