Dilshod Rasulov
6796903712
A guest was implicitly limited to a single allocation: lookups were
"(server, vmid) -> one row", release_ip_by_vmid freed one address, and
the link tool skipped guests that already had one. An instance can now
hold a primary address plus any number of aliases, managed from the
Network tab of the instance and visible on the IPAM pages.
Schema and service:
- ipam_allocations gains is_primary, assignment_kind, target_interface
and apply_* columns; migration 43 marks existing rows as primary
- find_allocations_by_resource/set_primary; release_ip_by_vmid now frees
every address of a deleted guest instead of just the first one
- next-available search may scan the whole subnet when a network has no
auto-assign pool, skipping node and cached guest addresses so it never
hands out an address that is already live
Applying to the guest (services/guest_ip_service.py):
- reuses script_engine, so LXC goes through pct exec and VMs through the
guest agent; the shell script synchronises the whole alias list of an
interface, which makes add, remove and retry the same code path
- persists via ifupdown, netplan, NetworkManager or sysconfig and never
restarts guest networking; an unknown stack degrades to runtime_only
and an unreachable guest keeps the allocation as pending
API and UI:
- /ipam/api/guests/{server_id}/{vmid}/addresses: list, add, apply,
primary, delete; allocations list accepts guest filters
- instance list exposes ips[] alongside the primary ip
- addresses card on the Network tab, "+N" badge in the IP column,
primary badge in the allocations table
Fixes found along the way:
- link-allocations registers every address of a guest, carries the MAC
over, refuses to take an address away from another live guest and no
longer matches allocations by resource name (which leaked an address
between same-named guests on different servers)
- IPAM history no longer fails to serialise datetime values, which used
to abort re-linking with a 500
1340 lines
62 KiB
Python
1340 lines
62 KiB
Python
"""
|
||
Background monitoring worker for generating notifications
|
||
Uses APScheduler for periodic tasks
|
||
"""
|
||
|
||
import asyncio
|
||
import re
|
||
import time
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Dict, Any, List, Optional
|
||
from loguru import logger
|
||
|
||
|
||
from ..config import utcnow
|
||
|
||
|
||
# qmrestore / vma extract emit lines like "progress 47% (read ...)".
|
||
# Some restores instead print bare "47% ..." markers. Capture either form.
|
||
_PROGRESS_RE = re.compile(r"(?:progress\s+)?(\d{1,3})\s*%")
|
||
|
||
|
||
def _bytes_to_mb(value: Optional[int]) -> Optional[int]:
|
||
"""Перевести байты Proxmox (maxmem) в мегабайты для хранения в БД."""
|
||
if not value:
|
||
return None
|
||
return round(value / (1024 * 1024))
|
||
|
||
|
||
def _bytes_to_gb(value: Optional[int]) -> Optional[int]:
|
||
"""Перевести байты Proxmox (maxdisk) в гигабайты для хранения в БД."""
|
||
if not value:
|
||
return None
|
||
return round(value / (1024 * 1024 * 1024))
|
||
|
||
|
||
def parse_task_progress(log_text: Optional[str]) -> Optional[int]:
|
||
"""Extract the latest restore/backup progress percentage from a task log.
|
||
|
||
Returns the last percentage seen (0-100), or None when the log has none.
|
||
"""
|
||
if not log_text:
|
||
return None
|
||
matches = _PROGRESS_RE.findall(log_text)
|
||
if not matches:
|
||
return None
|
||
# The log is appended in order, so the last match is the most recent progress.
|
||
pct = int(matches[-1])
|
||
return max(0, min(100, pct))
|
||
|
||
|
||
try:
|
||
from backend.app.db import SessionLocal
|
||
from backend.app.models import User, ProxmoxServer, PanelSettings, VMInstance, IPAMAllocation, Notification, ProxmoxTask
|
||
from backend.app.services.notification_service import NotificationService
|
||
from backend.app.logging_service import LoggingService
|
||
from backend.app.proxmox import ProxmoxClient
|
||
from backend.app.schemas import NotificationCreate
|
||
from backend.app.i18n import t
|
||
from backend.app.websocket_manager import broadcast_task_update
|
||
except ImportError:
|
||
from app.db import SessionLocal
|
||
from app.models import User, ProxmoxServer, PanelSettings, VMInstance, IPAMAllocation, Notification, ProxmoxTask
|
||
from app.services.notification_service import NotificationService
|
||
from app.logging_service import LoggingService
|
||
from app.proxmox import ProxmoxClient
|
||
from app.schemas import NotificationCreate
|
||
from app.i18n import t
|
||
from app.websocket_manager import broadcast_task_update
|
||
|
||
|
||
def get_panel_language(db) -> str:
|
||
"""Get panel language from settings"""
|
||
setting = db.query(PanelSettings).filter(PanelSettings.key == "language").first()
|
||
return setting.value if setting else "ru"
|
||
|
||
|
||
def run_async(coro):
|
||
"""Run async coroutine from sync context"""
|
||
try:
|
||
loop = asyncio.get_event_loop()
|
||
if loop.is_running():
|
||
# If loop is running, create task
|
||
import concurrent.futures
|
||
with concurrent.futures.ThreadPoolExecutor() as pool:
|
||
future = pool.submit(asyncio.run, coro)
|
||
return future.result(timeout=30)
|
||
else:
|
||
return loop.run_until_complete(coro)
|
||
except RuntimeError:
|
||
# No event loop, create new one
|
||
return asyncio.run(coro)
|
||
|
||
|
||
class ProbeFailure(Exception):
|
||
"""Неудачная проверка доступности сервера с указанием причины.
|
||
|
||
error_kind: 'auth' — API ответил 401, 'connection' — не отвечает.
|
||
"""
|
||
|
||
def __init__(self, message: str, error_kind: str = None):
|
||
super().__init__(message)
|
||
self.error_kind = error_kind or 'connection'
|
||
|
||
|
||
class MonitoringWorker:
|
||
"""Background worker for monitoring and generating notifications"""
|
||
|
||
# Сколько подряд неудачных опросов статуса UPID терпим, прежде чем пометить
|
||
# задачу failed. Транзитный сетевой сбой не должен «убивать» живую задачу.
|
||
UPID_FAIL_THRESHOLD = 3
|
||
|
||
# Как часто перепроверять IP гостя у Proxmox. Синк идёт каждые 10 секунд,
|
||
# а адрес меняется редко — без троттлинга это сотни лишних запросов в минуту.
|
||
IP_DETECT_TTL = 300
|
||
|
||
def __init__(self):
|
||
self.last_vm_states: Dict[str, str] = {} # vm_id -> status
|
||
self.last_resource_alerts: Dict[str, float] = {} # resource_id -> last_alert_time
|
||
self.last_server_states: Dict[int, bool] = {} # server_id -> is_online
|
||
self.last_server_alerts: Dict[int, float] = {} # server_id -> last_alert_time
|
||
self.upid_fail_counts: Dict[int, int] = {} # ProxmoxTask.id -> consecutive fetch failures
|
||
self.last_ip_detect: Dict[tuple, float] = {} # (server_id, vmid) -> monotonic time
|
||
|
||
def _create_proxmox_client(self, server: ProxmoxServer) -> ProxmoxClient:
|
||
"""Create ProxmoxClient from server model, raising if auth is missing."""
|
||
has_password = server.use_password and server.password
|
||
has_token = server.api_token_name and server.api_token_value
|
||
if not (has_password or has_token):
|
||
raise ValueError(f"Server {server.name} has no valid authentication configured")
|
||
return ProxmoxClient.from_server(server)
|
||
|
||
def _notify_server_offline(self, db, server: ProxmoxServer, error: str, users: List[User],
|
||
error_kind: str = None):
|
||
"""Send notification that server went offline"""
|
||
server_id = server.id
|
||
|
||
# Check cooldown (don't spam alerts)
|
||
last_alert = self.last_server_alerts.get(server_id)
|
||
if last_alert:
|
||
# 10 minute cooldown between alerts for same server
|
||
if (datetime.now().timestamp() - last_alert) < 600:
|
||
return
|
||
|
||
# Update server status in DB
|
||
server.is_online = False
|
||
server.last_error = error
|
||
server.last_error_kind = error_kind or 'connection'
|
||
server.last_check = utcnow()
|
||
db.commit()
|
||
|
||
# Get panel language
|
||
lang = get_panel_language(db)
|
||
|
||
title = t("notify_server_offline_title", lang, server_name=server.name)
|
||
message = t("notify_server_offline_message", lang,
|
||
server_name=server.name,
|
||
hostname=server.hostname,
|
||
error=error[:200])
|
||
data = {
|
||
"server_id": server.id,
|
||
"server_name": server.name,
|
||
"hostname": server.hostname,
|
||
"error": error
|
||
}
|
||
|
||
# Send notification to all users (through all channels: in-app, email, telegram)
|
||
for user in users:
|
||
try:
|
||
# Use async create_and_send for multi-channel delivery
|
||
run_async(
|
||
NotificationService.create_and_send(
|
||
db=db,
|
||
user_id=user.id,
|
||
notification_type="system",
|
||
level="critical",
|
||
title=title,
|
||
message=message,
|
||
data=data,
|
||
source="monitoring",
|
||
source_id=str(server.id)
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"Failed to send notification for user {user.id}: {e}")
|
||
|
||
self.last_server_alerts[server_id] = datetime.now().timestamp()
|
||
self.last_server_states[server_id] = False
|
||
logger.warning(f"Server {server.name} is OFFLINE: {error}")
|
||
|
||
def _notify_server_online(self, db, server: ProxmoxServer, users: List[User]):
|
||
"""Send notification that server came back online"""
|
||
server_id = server.id
|
||
was_offline = self.last_server_states.get(server_id) == False
|
||
|
||
# Update server status in DB
|
||
server.is_online = True
|
||
server.last_error = None
|
||
server.last_check = utcnow()
|
||
db.commit()
|
||
|
||
# Only notify if server was previously offline
|
||
if was_offline:
|
||
# Get panel language
|
||
lang = get_panel_language(db)
|
||
|
||
title = t("notify_server_online_title", lang, server_name=server.name)
|
||
message = t("notify_server_online_message", lang,
|
||
server_name=server.name,
|
||
hostname=server.hostname)
|
||
data = {
|
||
"server_id": server.id,
|
||
"server_name": server.name,
|
||
"hostname": server.hostname
|
||
}
|
||
|
||
for user in users:
|
||
try:
|
||
run_async(
|
||
NotificationService.create_and_send(
|
||
db=db,
|
||
user_id=user.id,
|
||
notification_type="system",
|
||
level="success",
|
||
title=title,
|
||
message=message,
|
||
data=data,
|
||
source="monitoring",
|
||
source_id=str(server.id),
|
||
force_send=True # Always send server recovery notifications
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"Failed to send notification for user {user.id}: {e}")
|
||
|
||
logger.info(f"Server {server.name} is back ONLINE")
|
||
|
||
self.last_server_states[server_id] = True
|
||
|
||
def run_server_availability_check(self):
|
||
"""
|
||
Dedicated server availability check - runs every 30 seconds.
|
||
Sends notifications about offline servers with 10-minute repeat interval.
|
||
Immediately notifies when server comes back online.
|
||
"""
|
||
db = SessionLocal()
|
||
try:
|
||
# Get all active users with admin role for server notifications
|
||
users = db.query(User).filter(
|
||
User.is_active == True,
|
||
User.is_admin == True
|
||
).all()
|
||
|
||
if not users:
|
||
logger.debug("[SERVER CHECK] No admin users, skipping server availability check")
|
||
return
|
||
|
||
# Get all Proxmox servers
|
||
servers = db.query(ProxmoxServer).all()
|
||
|
||
if not servers:
|
||
logger.debug("[SERVER CHECK] No Proxmox servers configured")
|
||
return
|
||
|
||
logger.debug(f"[SERVER CHECK] Checking availability of {len(servers)} servers")
|
||
|
||
for server in servers:
|
||
try:
|
||
client = self._create_proxmox_client(server)
|
||
|
||
# Quick connectivity check - just test if API responds
|
||
ok, error_kind, check_error = client.check_connection()
|
||
if ok:
|
||
# Server is online
|
||
was_offline = self.last_server_states.get(server.id) == False
|
||
|
||
# Update server status in DB
|
||
if not server.is_online or was_offline:
|
||
server.is_online = True
|
||
server.last_error = None
|
||
server.last_error_kind = None
|
||
server.last_check = utcnow()
|
||
db.commit()
|
||
|
||
# Notify immediately if server recovered
|
||
if was_offline:
|
||
self._notify_server_recovered(db, server, users)
|
||
|
||
self.last_server_states[server.id] = True
|
||
else:
|
||
raise ProbeFailure(check_error or "API not responding", error_kind)
|
||
|
||
except ValueError as e:
|
||
# Configuration error - log but don't mark as offline
|
||
logger.warning(f"[SERVER CHECK] Server {server.name} configuration error: {e}")
|
||
|
||
except Exception as e:
|
||
# Server is offline
|
||
error_msg = str(e)
|
||
was_online = self.last_server_states.get(server.id, True)
|
||
|
||
# Update server status in DB
|
||
server.is_online = False
|
||
server.last_error = error_msg[:500]
|
||
server.last_error_kind = getattr(e, 'error_kind', None) or 'connection'
|
||
server.last_check = utcnow()
|
||
db.commit()
|
||
|
||
# Check if we should send notification
|
||
last_alert = self.last_server_alerts.get(server.id, 0)
|
||
now = datetime.now().timestamp()
|
||
time_since_last_alert = now - last_alert
|
||
|
||
# Send notification if:
|
||
# 1. Server just went offline (was_online)
|
||
# 2. OR 10 minutes passed since last notification
|
||
should_notify = was_online or (time_since_last_alert >= 600)
|
||
|
||
if should_notify:
|
||
self._send_server_offline_notification(db, server, error_msg, users)
|
||
self.last_server_alerts[server.id] = now
|
||
|
||
if was_online:
|
||
reason = 'AUTH FAILED' if server.last_error_kind == 'auth' else 'OFFLINE'
|
||
logger.warning(f"[SERVER CHECK] Server {server.name} went {reason}: {error_msg}")
|
||
else:
|
||
logger.warning(f"[SERVER CHECK] Server {server.name} still OFFLINE (repeat notification)")
|
||
|
||
self.last_server_states[server.id] = False
|
||
|
||
except Exception as e:
|
||
logger.error(f"[SERVER CHECK] Critical error: {e}", exc_info=True)
|
||
finally:
|
||
db.close()
|
||
|
||
def _send_server_offline_notification(self, db, server: ProxmoxServer, error: str, users: List[User]):
|
||
"""Send server offline notification to all admin users"""
|
||
lang = get_panel_language(db)
|
||
|
||
title = t("notify_server_offline_title", lang, server_name=server.name)
|
||
message = t("notify_server_offline_message", lang,
|
||
server_name=server.name,
|
||
hostname=server.hostname or server.ip_address,
|
||
error=error[:200])
|
||
data = {
|
||
"server_id": server.id,
|
||
"server_name": server.name,
|
||
"hostname": server.hostname or server.ip_address,
|
||
"error": error
|
||
}
|
||
|
||
for user in users:
|
||
try:
|
||
run_async(
|
||
NotificationService.create_and_send(
|
||
db=db,
|
||
user_id=user.id,
|
||
notification_type="system",
|
||
level="critical",
|
||
title=title,
|
||
message=message,
|
||
data=data,
|
||
source="server_monitor",
|
||
source_id=str(server.id)
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"Failed to send offline notification to user {user.id}: {e}")
|
||
|
||
def _notify_server_recovered(self, db, server: ProxmoxServer, users: List[User]):
|
||
"""Send immediate notification that server came back online"""
|
||
lang = get_panel_language(db)
|
||
|
||
title = t("notify_server_online_title", lang, server_name=server.name)
|
||
message = t("notify_server_online_message", lang,
|
||
server_name=server.name,
|
||
hostname=server.hostname or server.ip_address)
|
||
data = {
|
||
"server_id": server.id,
|
||
"server_name": server.name,
|
||
"hostname": server.hostname or server.ip_address
|
||
}
|
||
|
||
for user in users:
|
||
try:
|
||
run_async(
|
||
NotificationService.create_and_send(
|
||
db=db,
|
||
user_id=user.id,
|
||
notification_type="system",
|
||
level="success",
|
||
title=title,
|
||
message=message,
|
||
data=data,
|
||
source="server_monitor",
|
||
source_id=str(server.id),
|
||
force_send=True # Always send recovery notifications immediately
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"Failed to send recovery notification to user {user.id}: {e}")
|
||
|
||
logger.info(f"[SERVER CHECK] Server {server.name} is back ONLINE - notification sent")
|
||
# Clear alert cooldown so next offline will notify immediately
|
||
self.last_server_alerts.pop(server.id, None)
|
||
|
||
def run_vm_status_monitoring(self):
|
||
"""Monitor VM status changes and server connectivity"""
|
||
db = SessionLocal()
|
||
try:
|
||
# Get all active users
|
||
users = db.query(User).filter(User.is_active == True).all()
|
||
|
||
if not users:
|
||
logger.debug("[MONITORING] No active users, skipping VM monitoring")
|
||
return
|
||
|
||
# Get all Proxmox servers
|
||
servers = db.query(ProxmoxServer).all()
|
||
|
||
if not servers:
|
||
logger.debug("[MONITORING] No Proxmox servers configured, skipping monitoring")
|
||
return
|
||
|
||
logger.debug(f"[MONITORING] Starting VM status check for {len(servers)} servers")
|
||
|
||
for server in servers:
|
||
try:
|
||
client = self._create_proxmox_client(server)
|
||
|
||
# Explicitly test connection - this will return False if offline
|
||
ok, error_kind, check_error = client.check_connection()
|
||
if not ok:
|
||
raise ProbeFailure(
|
||
check_error or f"Cannot connect to Proxmox server at {server.hostname}",
|
||
error_kind,
|
||
)
|
||
|
||
# Server is online - notify if it was offline before
|
||
self._notify_server_online(db, server, users)
|
||
|
||
# Get all VMs
|
||
vms = client.get_vms()
|
||
logger.debug(f"[MONITORING] Server {server.name}: {len(vms)} VMs found")
|
||
|
||
for vm in vms:
|
||
vm_key = f"{server.id}:{vm['vmid']}"
|
||
current_status = vm['status']
|
||
previous_status = self.last_vm_states.get(vm_key)
|
||
|
||
# Detect status change
|
||
if previous_status and previous_status != current_status:
|
||
# Log the status change
|
||
logger.info(f"[VM STATUS CHANGE] Server: {server.name} | VM: {vm['name']} (ID: {vm['vmid']}) | Status: {previous_status} -> {current_status}")
|
||
|
||
# Determine notification level
|
||
level = "info"
|
||
if current_status == "stopped" and previous_status == "running":
|
||
level = "warning"
|
||
logger.warning(f"[VM STOPPED] Server: {server.name} | VM: {vm['name']} (ID: {vm['vmid']}) went from running to stopped")
|
||
elif current_status == "running" and previous_status == "stopped":
|
||
level = "success"
|
||
logger.info(f"[VM STARTED] Server: {server.name} | VM: {vm['name']} (ID: {vm['vmid']}) is now running")
|
||
|
||
# Notify all users (in production, filter by permissions)
|
||
for user in users:
|
||
NotificationService.notify_vm_status(
|
||
db,
|
||
user_id=user.id,
|
||
vm_id=vm['vmid'],
|
||
vm_name=vm['name'],
|
||
status=current_status,
|
||
level=level,
|
||
server_name=server.name,
|
||
old_status=previous_status
|
||
)
|
||
|
||
self.last_vm_states[vm_key] = current_status
|
||
|
||
except ValueError as e:
|
||
# Configuration error - don't treat as offline
|
||
logger.warning(f"[MONITORING] Server {server.name} configuration error: {e}")
|
||
|
||
except Exception as e:
|
||
# Connection error - server is offline
|
||
error_msg = str(e)
|
||
logger.error(f"[MONITORING] Server {server.name} connection failed: {error_msg}")
|
||
self._notify_server_offline(db, server, error_msg, users,
|
||
error_kind=getattr(e, 'error_kind', None))
|
||
|
||
except Exception as e:
|
||
logger.error(f"[MONITORING] Critical error in VM status monitoring: {e}", exc_info=True)
|
||
finally:
|
||
db.close()
|
||
|
||
def run_resource_monitoring(self):
|
||
"""Monitor resource usage (CPU, RAM, Disk)"""
|
||
db = SessionLocal()
|
||
try:
|
||
users = db.query(User).filter(User.is_active == True).all()
|
||
servers = db.query(ProxmoxServer).all()
|
||
|
||
for server in servers:
|
||
try:
|
||
client = self._create_proxmox_client(server)
|
||
|
||
# Get all VMs with stats
|
||
vms = client.get_vms()
|
||
|
||
for vm in vms:
|
||
if vm['status'] != 'running':
|
||
continue
|
||
|
||
vm_key = f"{server.id}:{vm['vmid']}"
|
||
|
||
try:
|
||
# Get VM statistics
|
||
stats = client.get_vm_stats(vm['node'], vm['vmid'])
|
||
|
||
# Check CPU usage
|
||
if 'cpu' in stats and stats['cpu'] is not None:
|
||
cpu_percent = stats['cpu'] * 100
|
||
if cpu_percent > 80:
|
||
alert_key = f"{vm_key}:cpu"
|
||
if self._should_send_alert(alert_key):
|
||
for user in users:
|
||
NotificationService.notify_resource_alert(
|
||
db,
|
||
user_id=user.id,
|
||
resource_type="CPU",
|
||
resource_name=vm['name'],
|
||
usage_percent=cpu_percent,
|
||
threshold=80
|
||
)
|
||
self.last_resource_alerts[alert_key] = datetime.now().timestamp()
|
||
|
||
# Check RAM usage
|
||
if 'mem' in stats and 'maxmem' in stats:
|
||
if stats['maxmem'] > 0:
|
||
ram_percent = (stats['mem'] / stats['maxmem']) * 100
|
||
if ram_percent > 85:
|
||
alert_key = f"{vm_key}:ram"
|
||
if self._should_send_alert(alert_key):
|
||
for user in users:
|
||
NotificationService.notify_resource_alert(
|
||
db,
|
||
user_id=user.id,
|
||
resource_type="RAM",
|
||
resource_name=vm['name'],
|
||
usage_percent=ram_percent,
|
||
threshold=85
|
||
)
|
||
self.last_resource_alerts[alert_key] = datetime.now().timestamp()
|
||
|
||
# Check Disk usage
|
||
if 'disk' in stats and 'maxdisk' in stats:
|
||
if stats['maxdisk'] > 0:
|
||
disk_percent = (stats['disk'] / stats['maxdisk']) * 100
|
||
if disk_percent > 90:
|
||
alert_key = f"{vm_key}:disk"
|
||
if self._should_send_alert(alert_key):
|
||
for user in users:
|
||
NotificationService.notify_resource_alert(
|
||
db,
|
||
user_id=user.id,
|
||
resource_type="Disk",
|
||
resource_name=vm['name'],
|
||
usage_percent=disk_percent,
|
||
threshold=90
|
||
)
|
||
self.last_resource_alerts[alert_key] = datetime.now().timestamp()
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error getting stats for VM {vm['vmid']}: {e}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error monitoring resources for server {server.name}: {e}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error in resource monitoring: {e}")
|
||
finally:
|
||
db.close()
|
||
|
||
def run_cleanup_expired(self):
|
||
"""Clean up expired notifications and old audit logs"""
|
||
db = SessionLocal()
|
||
try:
|
||
count = NotificationService.cleanup_expired(db)
|
||
if count > 0:
|
||
logger.info(f"Cleaned up {count} expired notifications")
|
||
except Exception as e:
|
||
logger.error(f"Error cleaning up notifications: {e}")
|
||
finally:
|
||
db.close()
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
LoggingService.cleanup_old_logs(db, days=30)
|
||
except Exception as e:
|
||
logger.error(f"Error cleaning up old audit logs: {e}")
|
||
finally:
|
||
db.close()
|
||
|
||
def sync_vm_cache(self):
|
||
"""
|
||
Sync VM/container data from all Proxmox servers to local database cache.
|
||
This runs periodically to keep the cache fresh.
|
||
|
||
Deduplication logic:
|
||
- Cluster servers (detected by shared nodes): dedup by vmid within cluster
|
||
- Standalone servers: each server has its own vmid namespace
|
||
- Same vmid on different standalone servers = different VMs
|
||
"""
|
||
db = SessionLocal()
|
||
try:
|
||
servers = db.query(ProxmoxServer).filter(ProxmoxServer.is_online == True).all()
|
||
|
||
if not servers:
|
||
logger.debug("[VM SYNC] No online servers, skipping sync")
|
||
return
|
||
|
||
logger.debug(f"[VM SYNC] Starting VM cache sync for {len(servers)} servers")
|
||
|
||
# Load IPAM allocations for IP lookup
|
||
ipam_allocations = db.query(IPAMAllocation).filter(
|
||
IPAMAllocation.status.in_(['allocated', 'reserved'])
|
||
).all()
|
||
|
||
# Build IPAM lookups.
|
||
# Только по идентификатору гостя: сопоставление по имени давало чужой
|
||
# адрес одноимённым гостям на разных серверах (два разных "wireguard").
|
||
# У гостя может быть несколько адресов — в кэш инстанса пишем
|
||
# основной (is_primary); без флага берём самую раннюю запись.
|
||
ipam_by_server_vmid = {} # (server_id, vmid) -> allocation
|
||
|
||
for alloc in ipam_allocations:
|
||
if alloc.proxmox_server_id and alloc.proxmox_vmid:
|
||
key = (alloc.proxmox_server_id, alloc.proxmox_vmid)
|
||
current = ipam_by_server_vmid.get(key)
|
||
if current is None or (alloc.is_primary and not current.is_primary):
|
||
ipam_by_server_vmid[key] = alloc
|
||
|
||
# Detect clusters: servers that see multiple nodes are part of a cluster
|
||
# Group servers by their node set
|
||
server_nodes = {} # server_id -> set of node names
|
||
cluster_map = {} # server_id -> cluster_id (first server_id in cluster)
|
||
|
||
for server in servers:
|
||
try:
|
||
client = self._create_proxmox_client(server)
|
||
if client.is_connected():
|
||
nodes = client.get_nodes()
|
||
node_names = frozenset([n.get('node', '') for n in nodes if n.get('node')])
|
||
server_nodes[server.id] = node_names
|
||
except Exception as e:
|
||
logger.debug(f"[VM SYNC] Could not get nodes for {server.name}: {e}")
|
||
server_nodes[server.id] = frozenset()
|
||
|
||
# Find clusters: servers with same node set (more than 1 node) are in same cluster
|
||
node_set_to_servers = {} # frozenset of nodes -> list of server_ids
|
||
for server_id, nodes in server_nodes.items():
|
||
if len(nodes) > 1: # Only consider multi-node as cluster
|
||
if nodes not in node_set_to_servers:
|
||
node_set_to_servers[nodes] = []
|
||
node_set_to_servers[nodes].append(server_id)
|
||
|
||
# Assign cluster_id (first server in cluster)
|
||
for nodes, server_ids in node_set_to_servers.items():
|
||
cluster_id = min(server_ids) # Use smallest server_id as cluster identifier
|
||
for sid in server_ids:
|
||
cluster_map[sid] = cluster_id
|
||
logger.debug(f"[VM SYNC] Cluster detected: servers {server_ids} share nodes {list(nodes)})")
|
||
|
||
# Track seen VMs per cluster/server for dedup
|
||
seen_in_cluster = {} # cluster_id -> set of vmid
|
||
seen_standalone = {} # server_id -> set of vmid
|
||
|
||
all_vms_data = [] # List of (server_id, vm_data, vm_type)
|
||
server_clients = {} # server_id (or cluster_id) -> ProxmoxClient
|
||
sync_time = utcnow()
|
||
|
||
# Collect all VMs from all servers with proper dedup
|
||
for server in servers:
|
||
try:
|
||
client = self._create_proxmox_client(server)
|
||
|
||
if not client.is_connected():
|
||
logger.warning(f"[VM SYNC] Server {server.name} not connected, skipping")
|
||
continue
|
||
|
||
# Get all VMs and containers efficiently in one request
|
||
resources = client.get_cluster_resources(type_='vm')
|
||
vms = [res for res in resources if res.get('type') == 'qemu']
|
||
containers = [res for res in resources if res.get('type') == 'lxc']
|
||
|
||
cluster_id = cluster_map.get(server.id)
|
||
|
||
# Determine which dedup set to use
|
||
if cluster_id:
|
||
if cluster_id not in seen_in_cluster:
|
||
seen_in_cluster[cluster_id] = set()
|
||
seen_set = seen_in_cluster[cluster_id]
|
||
use_server_id = cluster_id # Use cluster_id for all VMs in cluster
|
||
# Map cluster_id -> client for IP detection later
|
||
if cluster_id not in server_clients:
|
||
server_clients[cluster_id] = client
|
||
else:
|
||
if server.id not in seen_standalone:
|
||
seen_standalone[server.id] = set()
|
||
seen_set = seen_standalone[server.id]
|
||
use_server_id = server.id
|
||
server_clients[server.id] = client
|
||
|
||
server_vm_count = 0
|
||
server_ct_count = 0
|
||
|
||
for vm in vms:
|
||
if vm.get('template'):
|
||
continue
|
||
vmid = vm.get('vmid')
|
||
if vmid in seen_set:
|
||
continue
|
||
seen_set.add(vmid)
|
||
all_vms_data.append((use_server_id, vm, 'qemu'))
|
||
server_vm_count += 1
|
||
|
||
for ct in containers:
|
||
if ct.get('template'):
|
||
continue
|
||
vmid = ct.get('vmid')
|
||
if vmid in seen_set:
|
||
continue
|
||
seen_set.add(vmid)
|
||
all_vms_data.append((use_server_id, ct, 'lxc'))
|
||
server_ct_count += 1
|
||
|
||
logger.debug(f"[VM SYNC] Server {server.name}: collected {server_vm_count} VMs, {server_ct_count} containers")
|
||
|
||
except Exception as e:
|
||
logger.error(f"[VM SYNC] Error fetching from server {server.name}: {e}")
|
||
|
||
logger.debug(f"[VM SYNC] Collected {len(all_vms_data)} unique VMs/containers")
|
||
|
||
# Track all seen (server_id, vmid) pairs for cleanup
|
||
all_seen = set()
|
||
# Track status changes for WebSocket broadcast
|
||
status_changes = [] # list of (server_id, vmid, node, old_status, new_status)
|
||
# Track newly-created VMs for WebSocket broadcast
|
||
created_vms = [] # list of (server_id, vmid, node, name, vm_type)
|
||
|
||
# Upsert all VMs to database
|
||
for server_id, vm_data, vm_type in all_vms_data:
|
||
try:
|
||
vmid = vm_data.get('vmid')
|
||
vm_name = vm_data.get('name', f"{'VM' if vm_type == 'qemu' else 'CT'}-{vmid}")
|
||
node_name = vm_data.get('node', '')
|
||
|
||
all_seen.add((server_id, vmid))
|
||
|
||
# Get IP from IPAM
|
||
ipam_alloc = ipam_by_server_vmid.get((server_id, vmid))
|
||
ip_address = ipam_alloc.ip_address if ipam_alloc else None
|
||
|
||
# If no IPAM IP, try to detect from Proxmox (LXC config or QEMU guest agent).
|
||
# Троттлим: адрес гостя меняется куда реже, чем идёт синк.
|
||
detect_key = (server_id, vmid)
|
||
detect_due = (time.monotonic() - self.last_ip_detect.get(detect_key, 0.0)) >= self.IP_DETECT_TTL
|
||
if not ip_address and detect_due:
|
||
self.last_ip_detect[detect_key] = time.monotonic()
|
||
try:
|
||
px_client = server_clients.get(server_id)
|
||
if px_client and node_name:
|
||
if vm_type == 'lxc':
|
||
# Статика из netN + живые адреса запущенного
|
||
# контейнера (DHCP виден только там)
|
||
for iface in px_client.get_container_interfaces(
|
||
node_name, vmid,
|
||
include_live=vm_data.get('status') == 'running',
|
||
):
|
||
for ip_info in iface.get('ips') or []:
|
||
if ip_info.get('type') != 'ipv4':
|
||
continue
|
||
addr = ip_info.get('address', '')
|
||
if addr:
|
||
ip_address = addr
|
||
break
|
||
if ip_address:
|
||
break
|
||
elif vm_type == 'qemu' and vm_data.get('status') == 'running':
|
||
result = px_client.proxmox.nodes(node_name).qemu(vmid).agent('network-get-interfaces').get()
|
||
if result and 'result' in result:
|
||
for iface in result['result']:
|
||
if iface.get('name') == 'lo':
|
||
continue
|
||
for ip_info in (iface.get('ip-addresses') or []):
|
||
if ip_info.get('ip-address-type') == 'ipv4':
|
||
addr = ip_info.get('ip-address', '')
|
||
if addr and not addr.startswith('127.'):
|
||
ip_address = addr
|
||
break
|
||
if ip_address:
|
||
break
|
||
except Exception:
|
||
pass
|
||
|
||
# Get OS type
|
||
if vm_type == 'qemu':
|
||
os_type = vm_data.get('ostype', 'QEMU/KVM')
|
||
else:
|
||
os_type = vm_data.get('ostype', 'Linux')
|
||
if os_type:
|
||
os_type = os_type.capitalize()
|
||
|
||
new_status = vm_data.get('status', 'unknown')
|
||
|
||
# Find existing by (server_id, vmid)
|
||
existing = db.query(VMInstance).filter(
|
||
VMInstance.server_id == server_id,
|
||
VMInstance.vmid == vmid
|
||
).first()
|
||
|
||
if existing:
|
||
# Detect status change for WS broadcast
|
||
if existing.status != new_status:
|
||
status_changes.append((server_id, vmid, node_name, existing.status, new_status))
|
||
# If this VM was previously soft-deleted but reappeared,
|
||
# treat it as a new VM for broadcast purposes
|
||
if existing.deleted_at is not None:
|
||
created_vms.append((server_id, vmid, node_name, vm_name, vm_type))
|
||
# Update existing entry
|
||
existing.name = vm_name
|
||
existing.node = node_name
|
||
existing.status = new_status
|
||
existing.cores = vm_data.get('cpus', vm_data.get('maxcpu'))
|
||
# maxmem/maxdisk приходят из Proxmox в БАЙТАХ, а колонки
|
||
# memory/disk_size по соглашению хранятся в MB/GB
|
||
# (так пишут create/resize). Конвертируем, иначе синк
|
||
# затирает корректные значения сырыми байтами.
|
||
existing.memory = _bytes_to_mb(vm_data.get('maxmem'))
|
||
existing.disk_size = _bytes_to_gb(vm_data.get('maxdisk'))
|
||
existing.os_type = os_type
|
||
existing.vm_type = vm_type
|
||
existing.is_template = bool(vm_data.get('template'))
|
||
# Only overwrite ip_address if we have a value;
|
||
# IPAM and detected IPs take priority; otherwise preserve existing
|
||
if ip_address:
|
||
existing.ip_address = ip_address
|
||
existing.last_sync_at = sync_time
|
||
existing.deleted_at = None
|
||
else:
|
||
# Create new entry
|
||
new_vm = VMInstance(
|
||
server_id=server_id,
|
||
vmid=vmid,
|
||
node=node_name,
|
||
vm_type=vm_type,
|
||
name=vm_name,
|
||
status=new_status,
|
||
cores=vm_data.get('cpus', vm_data.get('maxcpu')),
|
||
# см. комментарий выше: Proxmox отдаёт байты, храним MB/GB
|
||
memory=_bytes_to_mb(vm_data.get('maxmem')),
|
||
disk_size=_bytes_to_gb(vm_data.get('maxdisk')),
|
||
os_type=os_type,
|
||
is_template=bool(vm_data.get('template')),
|
||
ip_address=ip_address,
|
||
last_sync_at=sync_time
|
||
)
|
||
db.add(new_vm)
|
||
created_vms.append((server_id, vmid, node_name, vm_name, vm_type))
|
||
|
||
except Exception as e:
|
||
logger.error(f"[VM SYNC] Error processing VM {vmid}: {e}")
|
||
|
||
# Commit all changes
|
||
try:
|
||
db.commit()
|
||
except Exception as e:
|
||
logger.error(f"[VM SYNC] Error committing changes: {e}")
|
||
db.rollback()
|
||
return
|
||
|
||
# Не копим троттлинг-метки по гостям, которых больше нет
|
||
self.last_ip_detect = {
|
||
key: seen_at for key, seen_at in self.last_ip_detect.items() if key in all_seen
|
||
}
|
||
|
||
# Mark VMs that no longer exist as deleted
|
||
# Get all active VMs from synced servers
|
||
synced_server_ids = set(server.id for server in servers)
|
||
# Also include cluster IDs for proper matching
|
||
for server in servers:
|
||
if server.id in cluster_map:
|
||
synced_server_ids.add(cluster_map[server.id])
|
||
|
||
active_vms = db.query(VMInstance).filter(
|
||
VMInstance.deleted_at.is_(None),
|
||
VMInstance.server_id.in_(synced_server_ids)
|
||
).all()
|
||
|
||
deleted_count = 0
|
||
released_ips = 0
|
||
deleted_broadcasts = [] # list of (server_id, vmid, node, name, vm_type)
|
||
for vm in active_vms:
|
||
if (vm.server_id, vm.vmid) not in all_seen:
|
||
vm.deleted_at = sync_time
|
||
deleted_count += 1
|
||
deleted_broadcasts.append((vm.server_id, vm.vmid, vm.node, vm.name, vm.vm_type))
|
||
logger.debug(f"[VM SYNC] Marked VM {vm.name} (server={vm.server_id}, vmid={vm.vmid}) as deleted")
|
||
|
||
# Release IPAM allocation for deleted VM
|
||
try:
|
||
from app.ipam_service import IPAMService
|
||
ipam = IPAMService(db)
|
||
released, released_ip = ipam.release_ip_by_vmid(
|
||
proxmox_server_id=vm.server_id,
|
||
proxmox_vmid=vm.vmid,
|
||
released_by="system",
|
||
reason=f"VM {vm.name} ({vm.vmid}) no longer exists on Proxmox"
|
||
)
|
||
if released:
|
||
released_ips += 1
|
||
logger.debug(f"[VM SYNC] Released IPAM IP {released_ip} for deleted VM {vm.name}")
|
||
except Exception as e:
|
||
logger.warning(f"[VM SYNC] Failed to release IPAM for VM {vm.vmid}: {e}")
|
||
|
||
db.commit()
|
||
logger.debug(f"[VM SYNC] Cache sync completed. Processed {len(all_vms_data)} VMs/containers, marked {deleted_count} as deleted, released {released_ips} IPs")
|
||
|
||
# Broadcast status changes to all connected WebSocket clients
|
||
if status_changes:
|
||
try:
|
||
from app.websocket_manager import ws_manager, run_async_safe
|
||
for srv_id, vmid, node_name, old_status, new_status in status_changes:
|
||
logger.info(f"[VM SYNC] Status change: server={srv_id} vmid={vmid} {old_status} -> {new_status}")
|
||
run_async_safe(ws_manager.broadcast({
|
||
"type": "vm_status_update",
|
||
"server_id": srv_id,
|
||
"vmid": vmid,
|
||
"node": node_name,
|
||
"status": new_status,
|
||
}))
|
||
except Exception as _we:
|
||
logger.warning(f"[VM SYNC] Failed to broadcast status changes: {_we}")
|
||
|
||
# Broadcast deletions so connected clients can update their lists in real time
|
||
if deleted_broadcasts:
|
||
try:
|
||
from app.websocket_manager import ws_manager, run_async_safe
|
||
for srv_id, vmid, node_name, vm_name, vm_type in deleted_broadcasts:
|
||
logger.info(f"[VM SYNC] VM deleted: server={srv_id} vmid={vmid} name={vm_name}")
|
||
run_async_safe(ws_manager.broadcast({
|
||
"type": "vm_deleted",
|
||
"server_id": srv_id,
|
||
"vmid": vmid,
|
||
"node": node_name,
|
||
"name": vm_name,
|
||
"vm_type": vm_type,
|
||
}))
|
||
except Exception as _we:
|
||
logger.warning(f"[VM SYNC] Failed to broadcast deletions: {_we}")
|
||
|
||
# Broadcast newly-created VMs so clients can refetch and show them
|
||
if created_vms:
|
||
try:
|
||
from app.websocket_manager import ws_manager, run_async_safe
|
||
for srv_id, vmid, node_name, vm_name, vm_type in created_vms:
|
||
logger.info(f"[VM SYNC] VM created: server={srv_id} vmid={vmid} name={vm_name}")
|
||
run_async_safe(ws_manager.broadcast({
|
||
"type": "vm_created",
|
||
"server_id": srv_id,
|
||
"vmid": vmid,
|
||
"node": node_name,
|
||
"name": vm_name,
|
||
"vm_type": vm_type,
|
||
}))
|
||
except Exception as _we:
|
||
logger.warning(f"[VM SYNC] Failed to broadcast creations: {_we}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"[VM SYNC] Critical error: {e}", exc_info=True)
|
||
db.rollback()
|
||
finally:
|
||
db.close()
|
||
|
||
def run_upid_task_sync(self) -> None:
|
||
"""
|
||
Poll Proxmox API for status of all 'running' ProxmoxTask records.
|
||
Updates DB status + log, then broadcasts WS update.
|
||
Runs every 5 seconds via APScheduler.
|
||
"""
|
||
db = SessionLocal()
|
||
try:
|
||
running_tasks = (
|
||
db.query(ProxmoxTask)
|
||
.filter(ProxmoxTask.status == "running")
|
||
.all()
|
||
)
|
||
if not running_tasks:
|
||
return
|
||
|
||
logger.debug(f"[UPID SYNC] Checking {len(running_tasks)} running UPID task(s)")
|
||
|
||
for task in running_tasks:
|
||
if not task.server_id or not task.node:
|
||
continue
|
||
try:
|
||
server = db.query(ProxmoxServer).filter(ProxmoxServer.id == task.server_id).first()
|
||
if not server or not server.is_online:
|
||
continue
|
||
|
||
client = self._create_proxmox_client(server)
|
||
if not client.is_connected():
|
||
continue
|
||
|
||
# --- status -----------------------------------------------
|
||
# Транзитный сбой (сеть, перезагрузка ноды) не должен
|
||
# помечать живую задачу как failed: считаем ПОДРЯД идущие
|
||
# неудачи и финализируем только после UPID_FAIL_THRESHOLD.
|
||
try:
|
||
proxmox_status = client.get_task_status(task.node, task.upid, raise_on_error=True)
|
||
fetch_error = None
|
||
except Exception as fe:
|
||
proxmox_status = None
|
||
fetch_error = str(fe)
|
||
|
||
if not proxmox_status:
|
||
fails = self.upid_fail_counts.get(task.id, 0) + 1
|
||
self.upid_fail_counts[task.id] = fails
|
||
if fails < self.UPID_FAIL_THRESHOLD:
|
||
logger.debug(
|
||
f"[UPID SYNC] Task #{task.id}: status fetch failed "
|
||
f"({fails}/{self.UPID_FAIL_THRESHOLD}), will retry: {fetch_error}"
|
||
)
|
||
continue
|
||
logger.warning(
|
||
f"[UPID SYNC] Cannot fetch status for task #{task.id} "
|
||
f"(UPID={task.upid[:30]}...) after {fails} attempts — marking as failed: {fetch_error}"
|
||
)
|
||
task.status = "failed"
|
||
task.exit_status = (fetch_error or "Invalid UPID")[:200]
|
||
task.completed_at = utcnow()
|
||
db.commit()
|
||
self.upid_fail_counts.pop(task.id, None)
|
||
try:
|
||
broadcast_task_update(task.user_id, "task_update", task.to_dict())
|
||
except Exception:
|
||
pass
|
||
continue
|
||
|
||
# Успешный опрос — сбрасываем счётчик неудач
|
||
self.upid_fail_counts.pop(task.id, None)
|
||
|
||
pve_status = proxmox_status.get("status", "running")
|
||
|
||
# --- log --------------------------------------------------
|
||
try:
|
||
log_entries = client.get_task_log(task.node, task.upid, 0, 500) or []
|
||
if isinstance(log_entries, list):
|
||
task.log_text = "\n".join(
|
||
entry.get("t", "") if isinstance(entry, dict) else str(entry)
|
||
for entry in log_entries
|
||
)
|
||
# Extract a live progress % from the log (qmrestore etc.)
|
||
parsed = parse_task_progress(task.log_text)
|
||
if parsed is not None:
|
||
task.progress = parsed
|
||
except Exception:
|
||
pass
|
||
|
||
# --- finalise if stopped ----------------------------------
|
||
if pve_status == "stopped":
|
||
exit_status = proxmox_status.get("exitstatus", "") if proxmox_status else ""
|
||
task.exit_status = exit_status
|
||
task.status = "completed" if exit_status == "OK" else "failed"
|
||
task.completed_at = utcnow()
|
||
# A successfully finished task is 100% done even if the last
|
||
# log line didn't print a percentage.
|
||
if exit_status == "OK" and task.progress is not None:
|
||
task.progress = 100
|
||
logger.info(f"[UPID SYNC] Task #{task.id} finished: {task.status} ({exit_status})")
|
||
|
||
db.commit()
|
||
|
||
# --- WS broadcast -----------------------------------------
|
||
try:
|
||
broadcast_task_update(task.user_id, "task_update", task.to_dict())
|
||
except Exception as _wb:
|
||
logger.debug(f"[UPID SYNC] WS broadcast skipped: {_wb}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"[UPID SYNC] Error syncing task #{task.id} ({task.upid[:30]}): {e}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"[UPID SYNC] Critical error: {e}", exc_info=True)
|
||
finally:
|
||
db.close()
|
||
|
||
def _should_send_alert(self, alert_key: str, cooldown_minutes: int = 30) -> bool:
|
||
"""
|
||
Check if alert should be sent (rate limiting)
|
||
|
||
Args:
|
||
alert_key: Unique alert identifier
|
||
cooldown_minutes: Minimum time between same alerts
|
||
|
||
Returns:
|
||
True if alert should be sent
|
||
"""
|
||
last_alert = self.last_resource_alerts.get(alert_key)
|
||
if not last_alert:
|
||
return True
|
||
|
||
cooldown_seconds = cooldown_minutes * 60
|
||
return (datetime.now().timestamp() - last_alert) > cooldown_seconds
|
||
|
||
def check_for_panel_updates(self):
|
||
"""
|
||
Check for panel updates and notify admin users if a new version is available.
|
||
Runs periodically (every 6 hours) via scheduler.
|
||
"""
|
||
try:
|
||
from app.services.update_service import check_for_updates
|
||
except ImportError:
|
||
from backend.app.services.update_service import check_for_updates
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
logger.debug("[UPDATE CHECK] Checking for panel updates...")
|
||
|
||
# Run async check_for_updates
|
||
result = run_async(check_for_updates())
|
||
|
||
if result.get("error") or result.get("disabled"):
|
||
logger.debug(f"[UPDATE CHECK] Skipped: {result.get('error', 'disabled')}")
|
||
return
|
||
|
||
if not result.get("update_available"):
|
||
logger.debug(f"[UPDATE CHECK] No updates available. Current: {result.get('current_version')}")
|
||
return
|
||
|
||
current_version = result.get("current_version", "unknown")
|
||
new_version = result.get("latest_version", "unknown")
|
||
changelog = result.get("changelog")
|
||
|
||
logger.info(f"[UPDATE CHECK] New version available: {new_version} (current: {current_version})")
|
||
|
||
# Get all admin users to notify
|
||
users = db.query(User).filter(
|
||
User.is_active == True,
|
||
User.is_admin == True
|
||
).all()
|
||
|
||
if not users:
|
||
logger.debug("[UPDATE CHECK] No admin users to notify")
|
||
return
|
||
|
||
# Check if we already notified about this version
|
||
for user in users:
|
||
# Check if notification already exists for this user and version
|
||
existing = db.query(Notification).filter(
|
||
Notification.user_id == user.id,
|
||
Notification.source_id == f"update_{new_version}",
|
||
Notification.type == "update"
|
||
).first()
|
||
|
||
if existing:
|
||
logger.debug(f"[UPDATE CHECK] User {user.id} already notified about version {new_version}")
|
||
continue
|
||
|
||
# Create notification for this user
|
||
try:
|
||
NotificationService.notify_update_available(
|
||
db=db,
|
||
user_id=user.id,
|
||
current_version=current_version,
|
||
new_version=new_version,
|
||
changelog=changelog
|
||
)
|
||
logger.debug(f"[UPDATE CHECK] Notified user {user.username} about update to {new_version}")
|
||
except Exception as e:
|
||
logger.error(f"[UPDATE CHECK] Failed to notify user {user.id}: {e}")
|
||
|
||
db.commit()
|
||
logger.debug(f"[UPDATE CHECK] Completed. Notified {len(users)} admin users about version {new_version}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"[UPDATE CHECK] Error checking for updates: {e}", exc_info=True)
|
||
db.rollback()
|
||
finally:
|
||
db.close()
|
||
|
||
def cleanup_server_state(self, server_id: int):
|
||
"""Remove all in-memory state for a deleted server to prevent stale data"""
|
||
self.last_server_states.pop(server_id, None)
|
||
self.last_server_alerts.pop(server_id, None)
|
||
# Remove all VM state entries belonging to this server
|
||
stale_keys = [k for k in self.last_vm_states if k.startswith(f"{server_id}:")]
|
||
for key in stale_keys:
|
||
del self.last_vm_states[key]
|
||
stale_alert_keys = [k for k in self.last_resource_alerts if k.startswith(f"{server_id}:")]
|
||
for key in stale_alert_keys:
|
||
del self.last_resource_alerts[key]
|
||
logger.info(f"[MONITORING] Cleared in-memory state for deleted server {server_id}")
|
||
|
||
|
||
# Global worker instance
|
||
monitoring_worker = MonitoringWorker()
|
||
|
||
|
||
def start_monitoring_worker():
|
||
"""Start background monitoring tasks using APScheduler"""
|
||
from apscheduler.schedulers.background import BackgroundScheduler
|
||
from apscheduler.triggers.interval import IntervalTrigger
|
||
|
||
scheduler = BackgroundScheduler()
|
||
|
||
# Server availability check - every 30 seconds
|
||
# Sends notifications on offline, repeats every 10 minutes, notifies immediately on recovery
|
||
scheduler.add_job(
|
||
monitoring_worker.run_server_availability_check,
|
||
trigger=IntervalTrigger(seconds=30),
|
||
id='server_availability_check',
|
||
name='Check server availability',
|
||
replace_existing=True
|
||
)
|
||
|
||
# VM status monitoring - every 30 seconds
|
||
scheduler.add_job(
|
||
monitoring_worker.run_vm_status_monitoring,
|
||
trigger=IntervalTrigger(seconds=30),
|
||
id='vm_status_monitoring',
|
||
name='Monitor VM status changes',
|
||
replace_existing=True
|
||
)
|
||
|
||
# Resource monitoring - every 60 seconds
|
||
scheduler.add_job(
|
||
monitoring_worker.run_resource_monitoring,
|
||
trigger=IntervalTrigger(seconds=60),
|
||
id='resource_monitoring',
|
||
name='Monitor resource usage',
|
||
replace_existing=True
|
||
)
|
||
|
||
# Update check - every 6 hours
|
||
scheduler.add_job(
|
||
monitoring_worker.check_for_panel_updates,
|
||
trigger=IntervalTrigger(hours=6),
|
||
id='update_check',
|
||
name='Check for panel updates',
|
||
replace_existing=True
|
||
)
|
||
|
||
# VM cache sync - every 10 seconds
|
||
scheduler.add_job(
|
||
monitoring_worker.sync_vm_cache,
|
||
trigger=IntervalTrigger(seconds=10),
|
||
id='vm_cache_sync',
|
||
name='Sync VM cache from Proxmox',
|
||
replace_existing=True
|
||
)
|
||
|
||
# Task queue processing - every 5 seconds
|
||
try:
|
||
from app.services.task_queue_service import process_task_queue, recover_interrupted_tasks
|
||
# Восстановление задач, прерванных рестартом: без этого зависшая
|
||
# 'running'-задача навсегда блокирует обработку очереди.
|
||
try:
|
||
recovered = recover_interrupted_tasks()
|
||
if recovered:
|
||
logger.info(f"Recovered {recovered} interrupted bulk task(s) after restart")
|
||
except Exception as e:
|
||
logger.warning(f"Interrupted task recovery failed: {e}")
|
||
scheduler.add_job(
|
||
process_task_queue,
|
||
trigger=IntervalTrigger(seconds=5),
|
||
id='task_queue_processing',
|
||
name='Process bulk operation queue',
|
||
replace_existing=True
|
||
)
|
||
logger.info("Task queue processor registered")
|
||
except ImportError as e:
|
||
logger.warning(f"Task queue service not available: {e}")
|
||
|
||
# Proxmox UPID task sync - every 5 seconds
|
||
scheduler.add_job(
|
||
monitoring_worker.run_upid_task_sync,
|
||
trigger=IntervalTrigger(seconds=5),
|
||
id='upid_task_sync',
|
||
name='Sync Proxmox UPID task statuses',
|
||
replace_existing=True
|
||
)
|
||
|
||
# Cleanup expired notifications - every 6 hours
|
||
scheduler.add_job(
|
||
monitoring_worker.run_cleanup_expired,
|
||
trigger=IntervalTrigger(hours=6),
|
||
id='cleanup_notifications',
|
||
name='Clean up expired notifications',
|
||
replace_existing=True
|
||
)
|
||
|
||
# Cleanup expired Proxmox connection cache - every hour
|
||
try:
|
||
from app.proxmox import cleanup_expired_connections
|
||
scheduler.add_job(
|
||
cleanup_expired_connections,
|
||
trigger=IntervalTrigger(hours=1),
|
||
id='proxmox_cache_cleanup',
|
||
name='Cleanup expired Proxmox connections',
|
||
replace_existing=True
|
||
)
|
||
logger.info("Proxmox cache cleanup task registered")
|
||
except ImportError as e:
|
||
logger.warning(f"Proxmox cache cleanup not available: {e}")
|
||
|
||
# Run initial VM cache sync on startup
|
||
try:
|
||
logger.info("Running initial VM cache sync...")
|
||
monitoring_worker.sync_vm_cache()
|
||
except Exception as e:
|
||
logger.warning(f"Initial VM cache sync failed: {e}")
|
||
|
||
# Run initial update check on startup (delayed by 60 seconds to let the app fully start)
|
||
def delayed_update_check():
|
||
import time
|
||
time.sleep(60)
|
||
try:
|
||
logger.info("Running initial update check...")
|
||
monitoring_worker.check_for_panel_updates()
|
||
except Exception as e:
|
||
logger.warning(f"Initial update check failed: {e}")
|
||
|
||
import threading
|
||
threading.Thread(target=delayed_update_check, daemon=True).start()
|
||
|
||
scheduler.start()
|
||
logger.info("Background monitoring worker started")
|
||
|
||
return scheduler
|