Refactor backend: Split models and proxmox_client into modules, add alembic

backend/app/api/proxmox/_helpers.py

@@ -12,7 +12,7 @@ import websockets
 from ...db import get_db
 from ...models import ProxmoxServer, VMInstance, User, IPAMAllocation, IPAMNetwork, VMSnapshotArchive
 from ...schemas import ProxmoxServerCreate, ProxmoxServerUpdate, ProxmoxServerResponse
-from ...proxmox_client import ProxmoxClient, get_proxmox_resources
+from ...proxmox import ProxmoxClient, get_proxmox_resources
 from ...auth import get_current_user, PermissionChecker, require_permission, check_permission
 from ...logging_service import LoggingService
 from ...ipam_service import IPAMService
@@ -72,15 +72,19 @@ def require_vm_access(db: Session, current_user: User, server_id: int, vmid: int
 
 def _get_proxmox_client(server: ProxmoxServer) -> ProxmoxClient:
     """Build a ProxmoxClient from a ProxmoxServer model (password or token auth)."""
-    if server.use_password:
+    host = server.ip_address or getattr(server, 'hostname', '')
+    if hasattr(server, 'port') and server.port and server.port != 8006:
+        host = f"{host}:{server.port}"
+        
+    if getattr(server, 'use_password', False) and getattr(server, 'password', ''):
         return ProxmoxClient(
-            host=server.ip_address,
+            host=host,
             user=server.api_user,
             password=server.password,
             verify_ssl=server.verify_ssl
         )
     return ProxmoxClient(
-        host=server.ip_address,
+        host=host,
         user=server.api_user,
         token_name=server.api_token_name,
         token_value=server.api_token_value,
@@ -110,7 +114,7 @@ def get_next_vmid(db: Session, server_id: int) -> int:
     # Fallback: get from DB if Proxmox not available
     used_vmids = set(
         row[0] for row in db.query(VMInstance.vmid)
-        .filter(VMInstance.server_id == server_id, VMInstance.deleted_at == None)
+        .filter(VMInstance.server_id == server_id, VMInstance.deleted_at.is_(None))
         .all()
     )
 
@@ -291,7 +295,7 @@ def save_vm_instance(
     existing = db.query(VMInstance).filter(
         VMInstance.server_id == server_id,
         VMInstance.vmid == vmid,
-        VMInstance.deleted_at == None
+        VMInstance.deleted_at.is_(None)
     ).first()
 
     if existing:
@@ -322,7 +326,7 @@ def save_vm_instance(
     deleted_existing = db.query(VMInstance).filter(
         VMInstance.server_id == server_id,
         VMInstance.vmid == vmid,
-        VMInstance.deleted_at != None
+        VMInstance.deleted_at.isnot(None)
     ).first()
 
     if deleted_existing:
@@ -386,7 +390,7 @@ def get_vm_instance(db: Session, server_id: int, vmid: int) -> VMInstance:
     return db.query(VMInstance).filter(
         VMInstance.server_id == server_id,
         VMInstance.vmid == vmid,
-        VMInstance.deleted_at == None
+        VMInstance.deleted_at.is_(None)
     ).first()
 
 

backend/app/api/proxmox/async_ops.py

@@ -14,7 +14,7 @@ from loguru import logger
 
 from ...db import SessionLocal
 from ...models import ProxmoxServer, VMInstance, OSTemplate, DeployTask, User
-from ...proxmox_client import ProxmoxClient
+from ...proxmox import ProxmoxClient
 from ...logging_service import LoggingService
 from ._helpers import _get_proxmox_client, get_next_vmid
 

backend/app/api/proxmox/backup.py

@@ -18,6 +18,7 @@ from ...models import BackupJob, ProxmoxServer, User
 from ...auth import PermissionChecker
 from ...logging_service import LoggingService
 from ._helpers import _get_proxmox_client
+from ...proxmox import _run_in_executor
 
 router = APIRouter()
 
@@ -63,7 +64,7 @@ async def create_storage(
 
     try:
         client = _get_proxmox_client(server)
-        result = client.create_storage(storage_id, storage_type, **data)
+        result = await _run_in_executor(client.create_storage, storage_id, storage_type, **data)
         if result.get("success"):
             LoggingService.log_proxmox_action(
                 db=db, action="storage_create", resource_type="storage",
@@ -94,7 +95,7 @@ async def update_storage(
     data = await request.json()
     try:
         client = _get_proxmox_client(server)
-        result = client.update_storage(storage_id, **data)
+        result = await _run_in_executor(client.update_storage, storage_id, **data)
         if result.get("success"):
             return JSONResponse(content={"success": True})
         raise HTTPException(status_code=400, detail=result.get("error", "Failed"))
@@ -216,7 +217,7 @@ async def delete_backup(
 
     try:
         client = _get_proxmox_client(server)
-        result = client.delete_backup(node, storage, volid)
+        result = await _run_in_executor(client.delete_backup, node, storage, volid)
         if result.get("success"):
             LoggingService.log_proxmox_action(
                 db=db, action="backup_delete", resource_type="backup",
@@ -262,7 +263,8 @@ async def create_backup(
 
     try:
         client = _get_proxmox_client(server)
-        result = client.create_backup(
+        result = await _run_in_executor(
+            client.create_backup,
             node=node, vmid=int(vmid), storage=storage,
             mode=mode, compress=compress, remove=remove,
             keep_last=keep_last, notes=notes,
@@ -339,13 +341,15 @@ async def restore_backup(
     try:
         client = _get_proxmox_client(server)
         if vm_type == "lxc":
-            result = client.restore_lxc(
+            result = await _run_in_executor(
+                client.restore_lxc,
                 node=node, vmid=int(vmid), archive=archive,
                 storage=storage, new_vmid=int(new_vmid) if new_vmid else None,
                 start=start,
             )
         else:
-            result = client.restore_vm(
+            result = await _run_in_executor(
+                client.restore_vm,
                 node=node, vmid=int(vmid), archive=archive,
                 storage=storage, new_vmid=int(new_vmid) if new_vmid else None,
                 start=start, unique=unique,

backend/app/api/proxmox/cluster.py

@@ -15,8 +15,9 @@ from typing import Optional
 from ...db import get_db
 from ...models import ProxmoxServer, User
 from ...auth import PermissionChecker
-from ...proxmox_client import ProxmoxClient
+from ...proxmox import ProxmoxClient
 from ._helpers import _get_proxmox_client
+from ...proxmox import _run_in_executor
 
 router = APIRouter()
 
@@ -136,13 +137,13 @@ async def create_cluster(
     client = _get_password_client(server, rootpw)
 
     # Verify server is not already in a cluster on Proxmox side
-    if client.is_cluster():
+    if await _run_in_executor(client.is_cluster):
         raise HTTPException(
             status_code=status.HTTP_409_CONFLICT,
             detail="Нода уже является частью кластера Proxmox. Используйте join для добавления в существующий кластер."
         )
 
-    result = client.create_cluster(cluster_name=cluster_name, link0=link0_ip)
+    result = await _run_in_executor(client.create_cluster, cluster_name=cluster_name, link0=link0_ip)
 
     if not result.get("success"):
         raise HTTPException(
@@ -316,12 +317,12 @@ async def prepare_join(
     server = _get_server_or_404(db, node_server_id)
     client = _get_proxmox_client(server)
 
-    nodes = client.get_nodes()
+    nodes = await _run_in_executor(client.get_nodes)
     node_name = nodes[0].get("node") if nodes else server.hostname
 
     # Collect all guests
-    vms = client.get_vms(node=node_name)
-    containers = client.get_containers(node=node_name)
+    vms = await _run_in_executor(client.get_vms, node=node_name)
+    containers = await _run_in_executor(client.get_containers, node=node_name)
     all_guests = [
         {"vmid": v.get("vmid"), "type": "qemu"} for v in vms
     ] + [
@@ -344,7 +345,8 @@ async def prepare_join(
     upid_map: dict = {}  # vmid -> upid
     for guest in all_guests:
         vmid = guest["vmid"]
-        result = client.vzdump_guest(
+        result = await _run_in_executor(
+            client.vzdump_guest,
             node=node_name,
             vmid=vmid,
             storage=backup_storage,
@@ -367,7 +369,7 @@ async def prepare_join(
         finished_vmids = []
         for vmid, upid in list(upid_map.items()):
             try:
-                task_status = client.proxmox.nodes(node_name).tasks(upid).status.get()
+                task_status = await _run_in_executor(client.proxmox.nodes(node_name).tasks(upid).status.get)
                 if task_status.get("status") == "stopped":
                     finished_vmids.append(vmid)
                     if task_status.get("exitstatus") == "OK":
@@ -400,17 +402,17 @@ async def prepare_join(
         try:
             if guest["type"] == "qemu":
                 # Stop first if running
-                qemu_info = client.proxmox.nodes(node_name).qemu(vmid).status.current.get()
+                qemu_info = await _run_in_executor(client.proxmox.nodes(node_name).qemu(vmid).status.current.get)
                 if qemu_info.get("status") == "running":
-                    client.proxmox.nodes(node_name).qemu(vmid).status.stop.post()
+                    await _run_in_executor(client.proxmox.nodes(node_name).qemu(vmid).status.stop.post)
                     await asyncio.sleep(3)
-                client.proxmox.nodes(node_name).qemu(vmid).delete()
+                await _run_in_executor(client.proxmox.nodes(node_name).qemu(vmid).delete)
             else:
-                ct_info = client.proxmox.nodes(node_name).lxc(vmid).status.current.get()
+                ct_info = await _run_in_executor(client.proxmox.nodes(node_name).lxc(vmid).status.current.get)
                 if ct_info.get("status") == "running":
-                    client.proxmox.nodes(node_name).lxc(vmid).status.stop.post()
+                    await _run_in_executor(client.proxmox.nodes(node_name).lxc(vmid).status.stop.post)
                     await asyncio.sleep(3)
-                client.proxmox.nodes(node_name).lxc(vmid).delete()
+                await _run_in_executor(client.proxmox.nodes(node_name).lxc(vmid).delete)
 
             deleted.append(vmid)
             logger.info(f"Deleted guest vmid {vmid} on {server.name} during prepare-join")
@@ -490,11 +492,11 @@ async def join_cluster(
 
     # --- Pre-check: ноды не должно содержать гестов ---
     node_client = _get_proxmox_client(node_server)
-    nodes = node_client.get_nodes()
+    nodes = await _run_in_executor(node_client.get_nodes)
     node_name = nodes[0].get("node") if nodes else node_server.hostname
 
-    vms = node_client.get_vms(node=node_name)
-    containers = node_client.get_containers(node=node_name)
+    vms = await _run_in_executor(node_client.get_vms, node=node_name)
+    containers = await _run_in_executor(node_client.get_containers, node=node_name)
 
     if vms or containers:
         raise HTTPException(
@@ -513,7 +515,7 @@ async def join_cluster(
 
     # --- Получить join-info с кластерной ноды ---
     cluster_client = _get_proxmox_client(cluster_server)
-    join_info_result = cluster_client.get_cluster_join_info()
+    join_info_result = await _run_in_executor(cluster_client.get_cluster_join_info)
 
     if not join_info_result.get("success"):
         raise HTTPException(
@@ -534,7 +536,8 @@ async def join_cluster(
 
     # --- Выполнить join на присоединяемой ноде (используем пароль из запроса) ---
     joining_client = _get_password_client(node_server, rootpw)
-    result = joining_client.join_cluster(
+    result = await _run_in_executor(
+        joining_client.join_cluster,
         cluster_host=cluster_server.ip_address,
         rootpw=rootpw,
         fingerprint=fingerprint,

backend/app/api/proxmox/lxc.py

@@ -20,7 +20,7 @@ from ...db import get_db, SessionLocal
 from ...ipam_service import IPAMService
 from ...logging_service import LoggingService
 from ...models import DeployTask, IPAMNetwork, ProxmoxServer, User, VMInstance
-from ...proxmox_client import ProxmoxClient
+from ...proxmox import ProxmoxClient
 from ...websocket_manager import broadcast_task_update
 from ._helpers import _get_proxmox_client, get_next_vmid, save_vm_instance
 

backend/app/api/proxmox/networks.py

@@ -7,6 +7,7 @@ from ...db import get_db
 from ...models import ProxmoxServer, User, IPAMNetwork
 from ...auth import PermissionChecker
 from ._helpers import _get_proxmox_client
+from ...proxmox import _run_in_executor
 
 router = APIRouter()
 
@@ -101,7 +102,7 @@ async def create_node_network(
 
     try:
         client = _get_proxmox_client(server)
-        result = client.create_node_network(node, iface_type, **data)
+        result = await _run_in_executor(client.create_node_network, node, iface_type, **data)
         if result.get("success"):
             logger.info(
                 f"User {current_user.username} created {iface_type} interface "
@@ -133,7 +134,7 @@ async def update_node_network(
     data = await request.json()
     try:
         client = _get_proxmox_client(server)
-        result = client.update_node_network(node, iface, **data)
+        result = await _run_in_executor(client.update_node_network, node, iface, **data)
         if result.get("success"):
             logger.info(
                 f"User {current_user.username} updated interface {iface} "

backend/app/api/proxmox/sdn.py

@@ -9,6 +9,7 @@ from ...models import ProxmoxServer, User, IPAMNetwork
 from ...auth import PermissionChecker
 from ...logging_service import LoggingService
 from ._helpers import _get_proxmox_client
+from ...proxmox import _run_in_executor
 
 router = APIRouter()
 
@@ -92,7 +93,7 @@ async def create_sdn_zone(
         
         # Pass additional options from request
         kwargs = {k: v for k, v in data.items() if k not in ['zone', 'type']}
-        result = client.create_sdn_zone(zone_name, zone_type, **kwargs)
+        result = await _run_in_executor(client.create_sdn_zone, zone_name, zone_type, **kwargs)
         
         if result.get('success'):
             logger.info(f"User {current_user.username} created SDN zone: {zone_name}")
@@ -181,7 +182,7 @@ async def create_sdn_vnet(
     try:
         client = _get_proxmox_client(server)
         
-        result = client.create_sdn_vnet(vnet_name, zone, tag=tag, alias=alias, vlanaware=vlanaware)
+        result = await _run_in_executor(client.create_sdn_vnet, vnet_name, zone, tag=tag, alias=alias, vlanaware=vlanaware)
         
         if result.get('success'):
             logger.info(f"User {current_user.username} created SDN vnet: {vnet_name} in zone {zone}")
@@ -271,7 +272,7 @@ async def create_sdn_subnet(
 
     try:
         client = _get_proxmox_client(server)
-        result = client.create_sdn_subnet(vnet, subnet, gateway=gateway, snat=snat, dnszoneprefix=dnszoneprefix)
+        result = await _run_in_executor(client.create_sdn_subnet, vnet, subnet, gateway=gateway, snat=snat, dnszoneprefix=dnszoneprefix)
 
         if result.get('success'):
             logger.info(f"User {current_user.username} created subnet {subnet} in vnet {vnet}")
@@ -356,7 +357,7 @@ async def update_sdn_zone(
     data = await request.json()
     try:
         client = _get_proxmox_client(server)
-        result = client.update_sdn_zone(zone, **data)
+        result = await _run_in_executor(client.update_sdn_zone, zone, **data)
         if result.get('success'):
             logger.info(f"User {current_user.username} updated SDN zone: {zone}")
             return JSONResponse(content=result)
@@ -385,7 +386,7 @@ async def update_sdn_vnet(
     data = await request.json()
     try:
         client = _get_proxmox_client(server)
-        result = client.update_sdn_vnet(vnet, **data)
+        result = await _run_in_executor(client.update_sdn_vnet, vnet, **data)
         if result.get('success'):
             logger.info(f"User {current_user.username} updated SDN vnet: {vnet}")
             return JSONResponse(content=result)

backend/app/api/proxmox/servers.py

@@ -11,8 +11,8 @@ import websockets
 
 from ...db import get_db
 from ...models import ProxmoxServer, VMInstance, User, IPAMAllocation, IPAMNetwork, VMSnapshotArchive
-from ...schemas import ProxmoxServerCreate, ProxmoxServerUpdate, ProxmoxServerResponse
-from ...proxmox_client import ProxmoxClient, get_proxmox_resources
+from app.schemas import ProxmoxServerCreate, ProxmoxServerUpdate, ProxmoxServerResponse
+from ...proxmox import ProxmoxClient, get_proxmox_resources, _run_in_executor
 from ...auth import get_current_user, PermissionChecker, require_permission, check_permission
 from ...logging_service import LoggingService
 from ...ipam_service import IPAMService
@@ -193,7 +193,8 @@ async def auto_setup_proxmox_server(
     # 1. Получить auth ticket
     logger.info(f"Attempting to connect to Proxmox at {base_url}")
     try:
-        auth_response = http_requests.post(
+        auth_response = await _run_in_executor(
+            http_requests.post,
             f"{base_url}/api2/json/access/ticket",
             data={"username": api_user, "password": password},
             verify=verify_ssl,
@@ -239,7 +240,8 @@ async def auto_setup_proxmox_server(
     
     try:
         # Создаем токен через API
-        token_response = http_requests.post(
+        token_response = await _run_in_executor(
+            http_requests.post,
             f"{base_url}/api2/json/access/users/{api_user}/token/{token_name}",
             headers=headers,
             data={"privsep": "0"},  # Полные права как у пользователя
@@ -269,7 +271,8 @@ async def auto_setup_proxmox_server(
     
     try:
         # Получаем список нод
-        nodes_response = http_requests.get(
+        nodes_response = await _run_in_executor(
+            http_requests.get,
             f"{base_url}/api2/json/nodes",
             headers=headers,
             cookies={"PVEAuthCookie": ticket},
@@ -283,7 +286,8 @@ async def auto_setup_proxmox_server(
         
         # Получаем IP адреса нод из cluster/status
         if len(cluster_nodes) > 1:
-            cluster_status_response = http_requests.get(
+            cluster_status_response = await _run_in_executor(
+                http_requests.get,
                 f"{base_url}/api2/json/cluster/status",
                 headers=headers,
                 cookies={"PVEAuthCookie": ticket},
@@ -493,7 +497,7 @@ def update_proxmox_server(
     current_user: User = Depends(PermissionChecker("proxmox.servers.edit"))
 ):
     """Обновить Proxmox сервер"""
-    from ...proxmox_client import clear_server_cache
+    from ...proxmox import clear_server_cache
     
     server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
     if not server:
@@ -567,7 +571,7 @@ def delete_proxmox_server(
     current_user: User = Depends(PermissionChecker("proxmox.servers.delete"))
 ):
     """Удалить Proxmox сервер и связанные OS Templates"""
-    from ...proxmox_client import clear_server_cache
+    from ...proxmox import clear_server_cache
     from ...models import OSTemplate, OSTemplateGroup, VMInstance, Notification
     from ...workers import monitoring_worker
     

backend/app/api/proxmox/snapshots.py

@@ -12,7 +12,7 @@ import websockets
 from ...db import get_db
 from ...models import ProxmoxServer, VMInstance, User, IPAMAllocation, IPAMNetwork, VMSnapshotArchive
 from ...schemas import ProxmoxServerCreate, ProxmoxServerUpdate, ProxmoxServerResponse
-from ...proxmox_client import ProxmoxClient, get_proxmox_resources
+from ...proxmox import ProxmoxClient, get_proxmox_resources, _run_in_executor
 from ...auth import get_current_user, PermissionChecker, require_permission, check_permission
 from ...logging_service import LoggingService
 from ...ipam_service import IPAMService
@@ -81,7 +81,7 @@ async def create_vm_snapshot(
     try:
         client = _get_proxmox_client(server)
         
-        result = client.create_vm_snapshot(node, vmid, snapname, description, vmstate)
+        result = await _run_in_executor(client.create_vm_snapshot, node, vmid, snapname, description, vmstate)
         
         if result.get('success'):
             LoggingService.log_proxmox_action(
@@ -257,7 +257,7 @@ async def create_container_snapshot(
     try:
         client = _get_proxmox_client(server)
         
-        result = client.create_container_snapshot(node, vmid, snapname, description)
+        result = await _run_in_executor(client.create_container_snapshot, node, vmid, snapname, description)
         
         if result.get('success'):
             LoggingService.log_proxmox_action(

backend/app/api/proxmox/tasks.py

@@ -12,7 +12,7 @@ import websockets
 from ...db import get_db
 from ...models import ProxmoxServer, VMInstance, User, IPAMAllocation, IPAMNetwork, VMSnapshotArchive, ProxmoxTask, DeployTask
 from ...schemas import ProxmoxServerCreate, ProxmoxServerUpdate, ProxmoxServerResponse
-from ...proxmox_client import ProxmoxClient, get_proxmox_resources
+from ...proxmox import ProxmoxClient, get_proxmox_resources
 from ...auth import get_current_user, PermissionChecker, require_permission, check_permission
 from ...logging_service import LoggingService
 from ...ipam_service import IPAMService

backend/app/api/proxmox/vms.py

@@ -12,7 +12,7 @@ import websockets
 from ...db import get_db
 from ...models import ProxmoxServer, VMInstance, User, IPAMAllocation, IPAMNetwork, VMSnapshotArchive
 from ...schemas import ProxmoxServerCreate, ProxmoxServerUpdate, ProxmoxServerResponse
-from ...proxmox_client import ProxmoxClient, get_proxmox_resources
+from ...proxmox import ProxmoxClient, get_proxmox_resources
 from ...auth import get_current_user, PermissionChecker, require_permission, check_permission
 from ...logging_service import LoggingService
 from ...ipam_service import IPAMService
@@ -22,6 +22,13 @@ from ._helpers import (check_vm_access, require_vm_access, _get_proxmox_client,
 
 router = APIRouter()
 
+import time as time_lib
+import threading
+
+_live_metrics_cache = {}
+_live_metrics_lock = threading.Lock()
+METRICS_TTL = 10  # seconds
+
 from ...models import TaskQueue
 from ...services.task_queue_service import TaskQueueService, process_task_queue
 from ...api.proxmox.tasks import ProxmoxTaskService
@@ -154,21 +161,39 @@ def get_all_virtual_machines(
     cached_vms = query.order_by(VMInstance.name).all()
 
     # ── Live metrics from Proxmox cluster/resources (CPU, RAM, disk, uptime) ──
-    # One HTTP call per server returns metrics for all VMs/LXC in that cluster.
+    # One HTTP call per cluster returns metrics for all VMs/LXC.
     # Map: (server_id_in_db, vmid) -> live data dict
     live_metrics: dict[tuple[int, int], dict] = {}
-    for server in servers:
-        try:
-            client = _get_proxmox_client(server)
-            if not client.is_connected():
+    
+    with _live_metrics_lock:
+        now = time_lib.time()
+        for server in servers:
+            cache_key = server.id
+            if cache_key in _live_metrics_cache and (now - _live_metrics_cache[cache_key]['time']) < METRICS_TTL:
+                # Use cached metrics
+                for vmid, res in _live_metrics_cache[cache_key]['data'].items():
+                    live_metrics[(server.id, vmid)] = res
                 continue
-            for res in client.get_cluster_resources(type_='vm'):
-                vmid = res.get('vmid')
-                if vmid is None:
+                
+            try:
+                client = _get_proxmox_client(server)
+                if not client.is_connected():
                     continue
-                live_metrics[(server.id, vmid)] = res
-        except Exception as e:
-            logger.debug(f"Could not fetch live metrics from {server.name}: {e}")
+                
+                server_metrics = {}
+                for res in client.get_cluster_resources(type_='vm'):
+                    vmid = res.get('vmid')
+                    if vmid is None:
+                        continue
+                    live_metrics[(server.id, vmid)] = res
+                    server_metrics[vmid] = res
+                    
+                _live_metrics_cache[cache_key] = {
+                    'time': now,
+                    'data': server_metrics
+                }
+            except Exception as e:
+                logger.debug(f"Could not fetch live metrics from {server.name}: {e}")
 
     result = []
     
@@ -738,9 +763,8 @@ async def delete_vm(
                 client.stop_vm(node, vmid, force=True)
             except Exception as _se:
                 logger.warning(f"stop_vm({vmid}) failed: {_se}")
-            import time as _t
             for _ in range(30):
-                _t.sleep(0.5)
+                await asyncio.sleep(0.5)
                 st = client.get_vm_status(node, vmid)
                 if not st or not isinstance(st, dict) or st.get('status') != 'running':
                     break
@@ -1147,9 +1171,8 @@ async def delete_container(
             except Exception as _se:
                 logger.warning(f"stop_container({vmid}) failed: {_se}")
             # Wait up to 15s for it to actually stop
-            import time as _t
             for _ in range(30):
-                _t.sleep(0.5)
+                await asyncio.sleep(0.5)
                 st = client.get_container_status(node, vmid)
                 if not st or not isinstance(st, dict) or st.get('status') != 'running':
                     break