pvemanager/backend/app/api/proxmox/servers.py

from fastapi import APIRouter, Depends, Request, HTTPException, Query, Form, status, WebSocket, WebSocketDisconnect
from fastapi.responses import JSONResponse
from sqlalchemy.orm import Session
from sqlalchemy import func
from loguru import logger
from typing import List
import ssl
import asyncio
import httpx
import websockets

from ...db import get_db
from ...models import ProxmoxServer, VMInstance, User, IPAMAllocation, IPAMNetwork, VMSnapshotArchive
from app.schemas import ProxmoxServerCreate, ProxmoxServerUpdate, ProxmoxServerResponse
from ...proxmox import ProxmoxClient, get_proxmox_resources, _run_in_executor
from ...auth import get_current_user, PermissionChecker, require_permission, check_permission
from ...logging_service import LoggingService
from ...ipam_service import IPAMService
from ._helpers import (check_vm_access, require_vm_access, _get_proxmox_client,
                        get_next_vmid, archive_and_delete_snapshots,
                        save_vm_instance, get_vm_instance, soft_delete_vm_instance)

router = APIRouter()


# ==================== Proxmox Server CRUD ====================

@router.get("/api/servers", response_model=List[ProxmoxServerResponse])
def list_proxmox_servers(
    request: Request,
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("server:view"))
):
    """Получить список всех Proxmox серверов (фильтруется по активному workspace)"""
    from ...api.workspaces import get_workspace_server_ids
    from ...models import WorkspaceUser, WorkspaceServer

    is_privileged = current_user.is_admin or (
        current_user.role and current_user.role.name in ('admin', 'moderator')
    )

    if is_privileged:
        # Privileged: use workspace filter only
        server_ids = get_workspace_server_ids(request, db, current_user)
        query = db.query(ProxmoxServer)
        if server_ids is not None:
            query = query.filter(ProxmoxServer.id.in_(server_ids))
        return query.all()

    # Non-privileged user: must be directly assigned AND server must be in one of their workspaces
    assigned_ids = {s.id for s in current_user.assigned_servers}

    # Collect server IDs accessible through user's workspaces
    user_ws_ids = {
        r.workspace_id for r in
        db.query(WorkspaceUser.workspace_id).filter(WorkspaceUser.user_id == current_user.id).all()
    }
    if user_ws_ids:
        ws_server_ids = {
            r.server_id for r in
            db.query(WorkspaceServer.server_id).filter(
                WorkspaceServer.workspace_id.in_(user_ws_ids)
            ).all()
        }
    else:
        ws_server_ids = set()

    # Intersect: directly assigned AND within user's workspaces
    effective_ids = assigned_ids & ws_server_ids

    # Also apply active-workspace header filter on top
    ws_filter = get_workspace_server_ids(request, db, current_user)
    if ws_filter is not None:
        effective_ids = effective_ids & set(ws_filter)

    if not effective_ids:
        return []

    return db.query(ProxmoxServer).filter(ProxmoxServer.id.in_(effective_ids)).all()


@router.post("/api/servers", response_model=ProxmoxServerResponse, status_code=status.HTTP_201_CREATED)
def create_proxmox_server(
    server_data: ProxmoxServerCreate,
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("server:create"))
):
    """Добавить новый Proxmox сервер"""
    # Разрешаем несколько серверов с одинаковым IP (например, разные порты или кластеры)
    # Проверяем только дубликаты имён
    existing = db.query(ProxmoxServer).filter(ProxmoxServer.name == server_data.name).first()
    if existing:
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail=f"Server with name '{server_data.name}' already exists"
        )
    
    server = ProxmoxServer(**server_data.model_dump())
    db.add(server)
    db.commit()
    db.refresh(server)

    # Сразу проверяем подключение, чтобы статус is_online отобразился без задержки
    # в 30 секунд (период monitoring_worker.run_server_availability_check).
    try:
        client = _get_proxmox_client(server)
        if client.is_connected():
            server.update_status(True)
        else:
            server.update_status(False, "Failed to connect")
        db.commit()
        db.refresh(server)
    except Exception as e:
        logger.warning(f"Initial connectivity check failed for {server.name}: {e}")
        try:
            server.update_status(False, str(e))
            db.commit()
            db.refresh(server)
        except Exception:
            db.rollback()

    # Если сервер онлайн — мгновенно подтягиваем VM/LXC в локальный кэш,
    # чтобы NodeDetailPage и InstancesPage сразу увидели инстансы (без ожидания
    # 10-секундного цикла vm_cache_sync).
    if server.is_online:
        try:
            from ...workers.monitoring_worker import monitoring_worker
            monitoring_worker.sync_vm_cache()
        except Exception as e:
            logger.warning(f"Initial VM cache sync failed for server {server.name}: {e}")

    # Уведомляем все подключённые клиенты, что сервер добавлен — фронтенд
    # сразу инвалидирует список серверов и список VM.
    try:
        from ...websocket_manager import broadcast_event
        broadcast_event(
            "server_added",
            server_id=server.id,
            name=server.name,
            is_online=bool(server.is_online),
        )
    except Exception as e:
        logger.debug(f"Failed to broadcast server_added: {e}")

    logger.info(f"User {current_user.username} added Proxmox server: {server.name}")
    return server




@router.get("/api/servers/{server_id}", response_model=ProxmoxServerResponse)
def get_proxmox_server(
    server_id: int,
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("server:view"))
):
    """Получить информацию о конкретном Proxmox сервере"""
    server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
    if not server:
        raise HTTPException(status_code=404, detail="Proxmox server not found")
    return server


@router.get("/api/servers/{server_id}/cluster-info")
def get_server_cluster_info(
    server_id: int,
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("server:view"))
):
    """
    Получить информацию о кластерном режиме сервера.
    Возвращает is_cluster=true если сервер в кластере (HA доступна).
    """
    server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
    if not server:
        raise HTTPException(status_code=404, detail="Proxmox server not found")
    
    try:
        client = _get_proxmox_client(server)
        
        if not client.is_connected():
            return JSONResponse(content={
                "server_id": server_id,
                "server_name": server.name,
                "is_cluster": False,
                "error": "Cannot connect to server"
            })
        
        is_cluster = client.is_cluster()
        nodes = client.get_nodes()
        node_names = [n.get('node') for n in nodes if n.get('node')]
        # Fallback: если get_nodes() вернул пустой список (например, нет прав),
        # используем hostname сервера как имя ноды
        if not node_names:
            node_names = [server.hostname]

        return JSONResponse(content={
            "server_id": server_id,
            "server_name": server.name,
            "is_cluster": is_cluster,
            "node_count": len(node_names),
            "nodes": node_names
        })
    except Exception as e:
        logger.error(f"Error checking cluster info for server {server_id}: {e}")
        return JSONResponse(content={
            "server_id": server_id,
            "server_name": server.name,
            "is_cluster": False,
            "error": str(e)
        })


@router.put("/api/servers/{server_id}", response_model=ProxmoxServerResponse)
def update_proxmox_server(
    server_id: int,
    server_data: ProxmoxServerUpdate,
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("server:update"))
):
    """Обновить Proxmox сервер"""
    from ...proxmox import clear_server_cache
    
    server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
    if not server:
        raise HTTPException(status_code=404, detail="Proxmox server not found")
    
    # Если изменились учётные данные - очищаем кеш
    update_data = server_data.model_dump(exclude_unset=True)
    if any(key in update_data for key in ['password', 'api_token_name', 'api_token_value', 'api_user']):
        clear_server_cache(server.ip_address)
        logger.info(f"Cleared cache for server {server.ip_address} due to credential update")
    
    # Update only provided fields
    for field, value in update_data.items():
        setattr(server, field, value)
    
    db.commit()
    db.refresh(server)
    
    logger.info(f"User {current_user.username} updated Proxmox server: {server.name}")

    # Если изменились параметры подключения — сразу проверяем доступность,
    # чтобы статус обновился без 30-секундной задержки monitoring_worker.
    creds_changed = any(
        key in update_data
        for key in ('password', 'api_token_name', 'api_token_value', 'api_user',
                    'ip_address', 'hostname', 'port', 'verify_ssl', 'use_password')
    )
    if creds_changed:
        try:
            client = _get_proxmox_client(server)
            if client.is_connected():
                server.update_status(True)
            else:
                server.update_status(False, "Failed to connect")
            db.commit()
            db.refresh(server)
            if server.is_online:
                try:
                    from ...workers.monitoring_worker import monitoring_worker
                    monitoring_worker.sync_vm_cache()
                except Exception as _e:
                    logger.debug(f"VM sync after update failed: {_e}")
        except Exception as e:
            logger.warning(f"Connectivity check after update failed for {server.name}: {e}")
            try:
                server.update_status(False, str(e))
                db.commit()
                db.refresh(server)
            except Exception:
                db.rollback()

    # Broadcast — фронтенд инвалидирует кэши списков серверов и инстансов
    try:
        from ...websocket_manager import broadcast_event
        broadcast_event(
            "server_updated",
            server_id=server.id,
            name=server.name,
            is_online=bool(server.is_online),
        )
    except Exception as e:
        logger.debug(f"Failed to broadcast server_updated: {e}")

    return server


@router.delete("/api/servers/{server_id}", status_code=status.HTTP_204_NO_CONTENT)
def delete_proxmox_server(
    server_id: int,
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("server:delete"))
):
    """Удалить Proxmox сервер и связанные OS Templates"""
    from ...proxmox import clear_server_cache
    from ...models import OSTemplate, OSTemplateGroup, VMInstance, Notification
    from ...workers import monitoring_worker
    
    server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
    if not server:
        raise HTTPException(status_code=404, detail="Proxmox server not found")
    
    server_name = server.name
    server_ip = server.ip_address
    
    # Release ALL IPAM allocations for this server
    try:
        ipam = IPAMService(db)
        released_count = ipam.release_all_by_server(
            proxmox_server_id=server_id,
            released_by=current_user.username
        )
        if released_count > 0:
            logger.info(f"Released {released_count} IPAM allocations for server {server_name}")
    except Exception as e:
        logger.warning(f"Failed to release IPAM allocations for server: {e}")
    
    # Удаляем связанные OS Templates
    templates_deleted = db.query(OSTemplate).filter(OSTemplate.server_id == server_id).delete()
    if templates_deleted > 0:
        logger.info(f"Deleted {templates_deleted} OS templates for server {server_name}")
    
    # Удаляем кэш VM/контейнеров для этого сервера
    vms_deleted = db.query(VMInstance).filter(VMInstance.server_id == server_id).delete()
    if vms_deleted > 0:
        logger.info(f"Deleted {vms_deleted} cached VM instances for server {server_name}")
    
    # Удаляем пустые группы шаблонов (без шаблонов)
    empty_groups = db.query(OSTemplateGroup).filter(
        ~OSTemplateGroup.id.in_(
            db.query(OSTemplate.group_id).filter(OSTemplate.group_id != None).distinct()
        )
    ).all()
    for group in empty_groups:
        # Проверяем что группа действительно пуста
        template_count = db.query(OSTemplate).filter(OSTemplate.group_id == group.id).count()
        if template_count == 0:
            logger.info(f"Deleting empty template group: {group.name}")
            db.delete(group)
    
    logger.info(f"User {current_user.username} deleted Proxmox server: {server_name} ({server_ip})")
    
    # Удаляем уведомления о сервере (offline/online) чтобы они не показывались после удаления
    notifications_deleted = db.query(Notification).filter(
        Notification.source_id == str(server_id),
        Notification.source.in_(["monitoring", "server_monitor"])
    ).delete(synchronize_session=False)
    if notifications_deleted > 0:
        logger.info(f"Deleted {notifications_deleted} notifications for server {server_name}")
    
    # Очищаем кеш подключений для этого сервера
    clear_server_cache(server_ip)
    
    db.delete(server)
    db.commit()
    
    # Очищаем in-memory состояние мониторинга для удалённого сервера
    try:
        monitoring_worker.cleanup_server_state(server_id)
    except Exception as e:
        logger.warning(f"Failed to cleanup monitoring state for server {server_id}: {e}")

    # Broadcast — фронтенд мгновенно убирает удалённый сервер и его инстансы
    try:
        from ...websocket_manager import broadcast_event
        broadcast_event(
            "server_deleted",
            server_id=server_id,
            name=server_name,
        )
    except Exception as e:
        logger.debug(f"Failed to broadcast server_deleted: {e}")

    return None


@router.post("/api/servers/test", status_code=status.HTTP_200_OK)
def test_proxmox_credentials(
    payload: ProxmoxServerCreate,
    current_user: User = Depends(PermissionChecker("server:view"))
):
    """Проверить произвольные учётные данные Proxmox без сохранения в БД."""
    try:
        if payload.use_password:
            client = ProxmoxClient(
                host=payload.ip_address,
                user=payload.api_user,
                password=payload.password,
                verify_ssl=payload.verify_ssl,
            )
        else:
            client = ProxmoxClient(
                host=payload.ip_address,
                user=payload.api_user,
                token_name=payload.api_token_name,
                token_value=payload.api_token_value,
                verify_ssl=payload.verify_ssl,
            )

        if client.is_connected():
            return {"success": True, "status": "success", "message": "Connection successful"}
        return {"success": False, "status": "error", "message": "Failed to connect to Proxmox server"}
    except Exception as e:
        logger.error(f"Error testing Proxmox credentials for {payload.ip_address}: {e}")
        return {"success": False, "status": "error", "message": str(e)}


@router.post("/api/servers/{server_id}/test", status_code=status.HTTP_200_OK)
def test_proxmox_connection(
    server_id: int,
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("server:view"))
):
    """Проверить подключение к Proxmox серверу"""
    server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
    if not server:
        raise HTTPException(status_code=404, detail="Proxmox server not found")

    try:
        # Determine auth method
        client = _get_proxmox_client(server)

        if client.is_connected():
            server.update_status(True)
            db.commit()
            return {"success": True, "status": "success", "message": "Connection successful"}
        else:
            server.update_status(False, "Failed to connect")
            db.commit()
            return {"success": False, "status": "error", "message": "Failed to connect to Proxmox server"}

    except Exception as e:
        server.update_status(False, str(e))
        db.commit()
        logger.error(f"Error testing Proxmox connection to {server.name}: {e}")
        return {"success": False, "status": "error", "message": str(e)}


# ==================== Proxmox Resources ====================

@router.get("/api/{server_id}/resources")
def get_server_resources(
    server_id: int, 
    db: Session = Depends(get_db), 
    current_user: User = Depends(PermissionChecker("vm:view"))
):
    """API для получения всех ресурсов (VM + LXC) с Proxmox сервера"""
    server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
    if not server:
        raise HTTPException(status_code=404, detail="Proxmox server not found")
    
    # Если hostname указан (нода в кластере), фильтруем по ней
    # Проверяем что hostname не пустой и не равен IP адресу (автономный сервер)
    node_filter = None
    if server.hostname and server.hostname != server.ip_address:
        node_filter = server.hostname
    
    try:
        if server.use_password:
            resources = get_proxmox_resources(
                host=server.ip_address,
                user=server.api_user,
                password=server.password,
                verify_ssl=server.verify_ssl,
                node=node_filter
            )
        else:
            resources = get_proxmox_resources(
                host=server.ip_address,
                user=server.api_user,
                token_name=server.api_token_name,
                token_value=server.api_token_value,
                verify_ssl=server.verify_ssl,
                node=node_filter
            )
        
        # Фильтруем шаблоны - они не должны считаться как VM
        vms = [vm for vm in resources.get('vms', []) if not vm.get('template', 0)]
        containers = [ct for ct in resources.get('containers', []) if not ct.get('template', 0)]
        
        # Получаем uptime ноды
        node_uptime = None
        try:
            client = _get_proxmox_client(server)
            nodes = client.get_nodes()
            if nodes:
                target_node = node_filter or (nodes[0].get('node') if nodes else None)
                for n in nodes:
                    if target_node and n.get('node') == target_node:
                        node_uptime = n.get('uptime')
                        break
                if node_uptime is None and nodes:
                    node_uptime = nodes[0].get('uptime')
        except Exception:
            pass
        
        server.update_status(True)
        db.commit()
        
        return JSONResponse(content={
            "server_id": server_id,
            "server_name": server.name,
            "vms": vms,
            "containers": containers,
            "node_uptime": node_uptime
        })
    except Exception as e:
        server.update_status(False, str(e))
        db.commit()
        raise HTTPException(status_code=500, detail=f"Error getting resources: {str(e)}")


@router.get("/api/resources/all")
def get_all_resources(
    db: Session = Depends(get_db), 
    current_user: User = Depends(PermissionChecker("vm:view"))
):
    """API для получения всех ресурсов со всех Proxmox серверов"""
    proxmox_servers = db.query(ProxmoxServer).all()
    
    all_resources = {
        "servers": [],
        "total_vms": 0,
        "total_containers": 0
    }
    
    for server in proxmox_servers:
        try:
            logger.info(f"Getting resources from server {server.name}, use_password={server.use_password}")
            # Создаём клиента для каждого сервера независимо
            if server.use_password:
                resources = get_proxmox_resources(
                    host=server.ip_address,
                    user=server.api_user,
                    password=server.password,
                    verify_ssl=server.verify_ssl
                )
            else:
                logger.info(f"Using API token for {server.name}: user={server.api_user}")
                resources = get_proxmox_resources(
                    host=server.ip_address,
                    user=server.api_user,
                    token_name=server.api_token_name,
                    token_value=server.api_token_value,
                    verify_ssl=server.verify_ssl
                )
            
            vms = resources.get('vms', [])
            containers = resources.get('containers', [])
            
            logger.info(f"Got {len(vms)} VMs and {len(containers)} containers from {server.name}")
            
            server.update_status(True)
            db.commit()
            
            all_resources["servers"].append({
                "id": server.id,
                "name": server.name,
                "ip": server.ip_address,
                "vms": vms,
                "containers": containers,
                "vms_count": len(vms),
                "containers_count": len(containers)
            })
            
            all_resources["total_vms"] += len(vms)
            all_resources["total_containers"] += len(containers)
            
        except Exception as e:
            logger.error(f"Error getting resources from server {server.name} ({server.ip_address}): {e}")
            server.update_status(False, str(e))
            db.commit()
            all_resources["servers"].append({
                "id": server.id,
                "name": server.name,
                "ip": server.ip_address,
                "error": str(e),
                "vms": [],
                "containers": [],
                "vms_count": 0,
                "containers_count": 0
            })
            # Продолжаем обработку следующего сервера
            continue
    
    return JSONResponse(content=all_resources)


# ==================== Node (Host) Status ====================

@router.get("/api/{server_id}/storages")
def get_storages(
    server_id: int,
    node: str = Query(..., description="Node name"),
    content_type: str = Query(None, description="Filter by content type (images, rootdir, vztmpl, etc)"),
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("server:view"))
):
    """Получить список хранилищ на ноде Proxmox"""
    server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
    if not server:
        raise HTTPException(status_code=404, detail="Proxmox server not found")
    
    try:
        client = _get_proxmox_client(server)
        
        if not client.is_connected():
            raise HTTPException(status_code=503, detail="Failed to connect to Proxmox server")
        
        storages = client.get_node_storages(node)
        
        # Фильтрация по типу контента если указано
        if content_type and storages:
            filtered = []
            for storage in storages:
                content = storage.get('content', '')
                if content_type in content.split(','):
                    filtered.append(storage)
            storages = filtered
        
        # Сортировка: сначала активные, потом по имени
        storages.sort(key=lambda x: (not x.get('active', 1), x.get('storage', '')))
        
        return JSONResponse(content={"storages": storages})
    except Exception as e:
        logger.error(f"Error getting storages: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/api/{server_id}/node/status")
def get_node_status(
    server_id: int,
    node: str = Query(..., description="Node name"),
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("vm:view"))
):
    """Получить текущий статус ноды Proxmox (хоста)"""
    server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
    if not server:
        raise HTTPException(status_code=404, detail="Proxmox server not found")
    
    try:
        client = _get_proxmox_client(server)
        
        if not client.is_connected():
            raise HTTPException(status_code=503, detail="Failed to connect to Proxmox server")
        
        status = client.get_node_status(node)
        if not status:
            raise HTTPException(status_code=404, detail="Node status not found")
        
        return JSONResponse(content=status)
    except Exception as e:
        logger.error(f"Error getting node {node} status: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/api/{server_id}/node/rrddata")
def get_node_rrddata(
    server_id: int,
    node: str = Query(..., description="Node name"),
    timeframe: str = Query("hour", regex="^(hour|day|week|month|year)$"),
    db: Session = Depends(get_db),
    current_user: User = Depends(PermissionChecker("vm:view"))
):
    """Получить исторические данные ноды для графиков"""
    server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
    if not server:
        raise HTTPException(status_code=404, detail="Proxmox server not found")
    
    try:
        client = _get_proxmox_client(server)
        
        if not client.is_connected():
            raise HTTPException(status_code=503, detail="Failed to connect to Proxmox server")
        
        rrddata = client.get_node_rrddata(node, timeframe)
        return JSONResponse(content=rrddata)
    except Exception as e:
        logger.error(f"Error getting node {node} RRD data: {e}")
        raise HTTPException(status_code=500, detail=str(e))