Add complete project structure and all files

backend/app/services/logo_service.py

@@ -0,0 +1,225 @@
+"""
+Logo Protection Service with RSA Digital Signature
+===================================================
+
+Protects the panel logo from unauthorized modifications using RSA digital signatures.
+
+How it works:
+1. Logo creator signs the logo with their PRIVATE key (kept secret, never shared)
+2. PUBLIC key is embedded in this code
+3. On each request, signature is verified with public key
+4. Without the private key, it's cryptographically impossible to create a valid signature
+
+For logo updates: Contact the project maintainer with the new logo.
+"""
+
+import base64
+import os
+from pathlib import Path
+from loguru import logger
+
+try:
+    from cryptography.hazmat.primitives import hashes
+    from cryptography.hazmat.primitives.asymmetric import padding
+    from cryptography.hazmat.primitives import serialization
+    CRYPTO_AVAILABLE = True
+except ImportError:
+    CRYPTO_AVAILABLE = False
+    logger.warning("cryptography library not available, logo protection disabled")
+
+# Get the static directory path
+STATIC_DIR = Path(__file__).parent.parent / "static"
+LOGO_PATH = STATIC_DIR / "img" / "logo.png"
+LOGO_SIG_PATH = STATIC_DIR / "img" / "logo.sig"
+
+# ============================================================
+# PUBLIC KEY - Safe to be in open source code
+# This key can only VERIFY signatures, not create them
+# ============================================================
+PUBLIC_KEY_PEM = """-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvM0VCWkP0fg2PMAPYJdn
+JF0YO9EkJtJmnhV2v+VIIqqXdiBnNDVFHxAxxDmeMHkXff9VbZ5PRRDc/AMRSjwY
+f1tN/xCCfQG8GVNlIKDcr11GJCAm7vOjy64bSEs/VpOK/A0l4js0FX6o/nEvqDqF
+hSdrxhrnX4iWbm6UaXL5ZcWrHYXXyD0zsc2g5xEXAtf7sy8U8bHtD6NjaLNNIVWz
+LzZqNTaAzOKXKOgKdwsYT4JXvWNLgrjMHo5TUcYDAnrPDSE2E6Ms2TZDS0BicB7Q
+g6FZ5M8H9NrttpDxvS0Gwen0AfRbI/0Cocn9nG88eiX/hbsodPxmJIlloAUOhtRa
+dwIDAQAB
+-----END PUBLIC KEY-----"""
+
+# Fallback logo (simple base64 encoded placeholder - 64x64 transparent PNG)
+FALLBACK_LOGO_B64 = """
+iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAD
+GUlEQVR4nO2bS27bMBCGP7RZdNNFb9BFT+Au0hv0Br1BeoMe4QSNF+4q3fQGXfQG3fQEi+xatQ8g
+kCIlPkTJUuL5gICRPMP5OZwhKQOGYRiGYRiGYRiGYRhGIpI0dQ/nOy64QC4F/gI/gO/A17LkoyAJ
+cAX4AFwCF8AvYAf8Bn4C38o/+xaQBLgGPgNXZbkA/gDb8s+2wC/gS1nykZAEvAXeA2/Ksg1sgB/l
+n+2ALfBpmXIXBUnAO+AD8Los62AB/C7/bAN8L0s+CpKAG+A98LosK+Av8Kf8sw3wo/yzb4IkwDXw
+CXhdljXwB/hb/tkW+FWWfBQkAa6BT8DrsmyAX8Dv8s82wM+y5CMhCXANfAJelWUN/AZ+lX+2Bn6U
+JR8JSYBr4CPwqixr4Bfwu/yzNfCjLPlISAJcAx+BV2VZAb+A3+WfrYEfZclHQhLgGvgAvCrLEvgJ
+/Cr/bAn8KEs+EpIA18B74GVZFsAP4Ff5ZwvgR1nykZAEuAbeAy/LMgd+AD/LP5sDX8uSj4QkwDXw
+DnhZljnwA/hZ/tkc+FKWfCQkAa6Bd8DLskyBb8DP8s+mwJey5CMhCXANvAVelmUCfAN+ln82Ab6U
+JR8JSYBr4A3wsiw3wFfgR/lnN8CXsuQjIQlwDbwBXpTlGvgK/Cj/7Br4UpZ8JCQBroHXwIuyXAFf
+gB/ln10Bn8uSj4QkwBXwGnhRlkvgM/Cj/LNL4HNZ8pGQBLgCXgMvyjIGPgM/yj8bA5/Lko+EJMAV
+8Bp4UZYx8An4Uf7ZGPhclnwkJAEugdfAi7JcAJ+AH+WfXQCfy5KPhCTAJfAaeFGWMfAR+FH+2Rj4
+VJZ8JCQBLoHXwIuyjIEPwI/yz8bAx7LkIyEJcAm8Bp6XZQS8B76Xfz4C3pclHwlJgAvgNfC8LEPg
+HfC9/PMh8K4s+UhIAlwAr4HnZRkCb4Hv5Z8PgbdlyUdCEuACeA08L8sQeAN8L/98CLwpSz4SkgAX
+wCvgeVkGwGvge/nnA+B1WfKRkAS4AF4Bz8syAF4B38s/HwCvypKPhGEYhmEYhmEYhmEYhmH8D/wD
+F0bN8E0rJrsAAAAASUVORK5CYII=
+""".strip().replace('\n', '')
+
+
+class LogoService:
+    """Service for logo protection using RSA digital signatures"""
+    
+    _public_key = None
+    
+    @classmethod
+    def _get_public_key(cls):
+        """Load and cache public key"""
+        if cls._public_key is None and CRYPTO_AVAILABLE:
+            try:
+                cls._public_key = serialization.load_pem_public_key(
+                    PUBLIC_KEY_PEM.encode()
+                )
+            except Exception as e:
+                logger.error(f"Failed to load public key: {e}")
+        return cls._public_key
+    
+    @staticmethod
+    def verify_signature(data: bytes, signature: bytes) -> bool:
+        """Verify RSA signature of data"""
+        if not CRYPTO_AVAILABLE:
+            return True  # Skip verification if crypto not available
+        
+        public_key = LogoService._get_public_key()
+        if not public_key:
+            return False
+        
+        try:
+            public_key.verify(
+                signature,
+                data,
+                padding.PKCS1v15(),
+                hashes.SHA256()
+            )
+            return True
+        except Exception as e:
+            logger.warning(f"Signature verification failed: {e}")
+            return False
+    
+    @staticmethod
+    def get_stored_signature() -> bytes | None:
+        """Get stored signature from file"""
+        try:
+            if LOGO_SIG_PATH.exists():
+                # Signature is stored as base64
+                sig_b64 = LOGO_SIG_PATH.read_text().strip()
+                return base64.b64decode(sig_b64)
+        except Exception as e:
+            logger.error(f"Error reading logo signature: {e}")
+        return None
+    
+    @staticmethod
+    def verify_logo() -> bool:
+        """Verify logo integrity using RSA signature"""
+        try:
+            if not LOGO_PATH.exists():
+                logger.warning("Logo file not found")
+                return False
+            
+            signature = LogoService.get_stored_signature()
+            if not signature:
+                logger.warning("No signature found for logo")
+                return False
+            
+            logo_data = LOGO_PATH.read_bytes()
+            
+            if not LogoService.verify_signature(logo_data, signature):
+                logger.warning("Logo signature verification FAILED - unauthorized modification detected!")
+                return False
+            
+            logger.debug("Logo signature verified successfully")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Error verifying logo: {e}")
+            return False
+    
+    @staticmethod
+    def get_logo_bytes() -> bytes:
+        """Get logo bytes - verified original or fallback"""
+        if LogoService.verify_logo():
+            return LOGO_PATH.read_bytes()
+        else:
+            logger.info("Using fallback logo (signature verification failed)")
+            return base64.b64decode(FALLBACK_LOGO_B64)
+    
+    @staticmethod
+    def get_logo_base64() -> str:
+        """Get logo as base64 string"""
+        return base64.b64encode(LogoService.get_logo_bytes()).decode()
+    
+    @staticmethod
+    def is_logo_valid() -> bool:
+        """Check if current logo is valid (has valid signature)"""
+        return LogoService.verify_logo()
+
+
+# ============================================================
+# SIGNING FUNCTIONS - Only work with private key
+# These are used by the sign_logo.py script
+# ============================================================
+
+def sign_logo_with_private_key(logo_path: str, private_key_path: str, output_sig_path: str = None) -> tuple[bool, str]:
+    """
+    Sign a logo file with private key.
+    This function is for the logo creator only.
+    
+    Args:
+        logo_path: Path to the logo PNG file
+        private_key_path: Path to the private key PEM file
+        output_sig_path: Path for output signature file (default: logo_path + '.sig')
+    
+    Returns:
+        tuple (success, message)
+    """
+    if not CRYPTO_AVAILABLE:
+        return False, "cryptography library not installed"
+    
+    try:
+        # Load logo
+        logo_path = Path(logo_path)
+        if not logo_path.exists():
+            return False, f"Logo file not found: {logo_path}"
+        
+        logo_data = logo_path.read_bytes()
+        
+        # Verify it's a PNG
+        if not logo_data[:8] == b'\x89PNG\r\n\x1a\n':
+            return False, "File is not a valid PNG"
+        
+        # Load private key
+        private_key_path = Path(private_key_path)
+        if not private_key_path.exists():
+            return False, f"Private key not found: {private_key_path}"
+        
+        private_key = serialization.load_pem_private_key(
+            private_key_path.read_bytes(),
+            password=None
+        )
+        
+        # Create signature
+        signature = private_key.sign(
+            logo_data,
+            padding.PKCS1v15(),
+            hashes.SHA256()
+        )
+        
+        # Save signature as base64
+        sig_path = Path(output_sig_path) if output_sig_path else logo_path.with_suffix('.sig')
+        sig_b64 = base64.b64encode(signature).decode()
+        sig_path.write_text(sig_b64)
+        
+        return True, f"Signature saved to: {sig_path}"
+        
+    except Exception as e:
+        return False, str(e)

backend/app/services/notification_channels.py

@@ -0,0 +1,470 @@
+"""
+Notification Channels - Email (SMTP) and Telegram Bot
+Settings are loaded from database (panel_settings table)
+"""
+
+import asyncio
+import smtplib
+import ssl
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from email.mime.image import MIMEImage
+from typing import Optional, Dict, Any, Tuple
+from datetime import datetime
+from loguru import logger
+
+try:
+    import httpx
+except ImportError:
+    httpx = None
+
+
+def get_local_time() -> str:
+    """Get current local time formatted string"""
+    return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+
+def get_notification_setting(key: str, default: str = None) -> Optional[str]:
+    """Get notification setting from database"""
+    from ..db import SessionLocal
+    from ..models import PanelSettings
+    
+    db = SessionLocal()
+    try:
+        setting = db.query(PanelSettings).filter(PanelSettings.key == key).first()
+        return setting.value if setting else default
+    finally:
+        db.close()
+
+
+def get_panel_name() -> str:
+    """Get panel name from database"""
+    return get_notification_setting('panel_name', 'PVEmanager') or 'PVEmanager'
+
+
+def get_logo_base64() -> Optional[str]:
+    """Get logo as base64 for email embedding"""
+    import base64
+    import os
+    
+    logo_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'static', 'img', 'logo.png')
+    
+    if os.path.exists(logo_path):
+        try:
+            with open(logo_path, 'rb') as f:
+                logo_data = f.read()
+            return base64.b64encode(logo_data).decode('utf-8')
+        except Exception as e:
+            logger.error(f"Failed to read logo: {e}")
+    return None
+
+
+def get_logo_bytes() -> Optional[bytes]:
+    """Get logo as bytes for email attachment"""
+    import os
+    
+    logo_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'static', 'img', 'logo.png')
+    
+    if os.path.exists(logo_path):
+        try:
+            with open(logo_path, 'rb') as f:
+                return f.read()
+        except Exception as e:
+            logger.error(f"Failed to read logo: {e}")
+    return None
+
+
+class EmailChannel:
+    """SMTP Email notification channel - settings from database"""
+    
+    def __init__(self):
+        self._load_settings()
+    
+    def _load_settings(self):
+        """Load SMTP settings from database"""
+        self.smtp_host = get_notification_setting('smtp_host')
+        self.smtp_port = int(get_notification_setting('smtp_port', '587') or '587')
+        self.smtp_user = get_notification_setting('smtp_user')
+        self.smtp_password = get_notification_setting('smtp_password')
+        self.smtp_from = get_notification_setting('smtp_from') or self.smtp_user
+        self.smtp_tls = get_notification_setting('smtp_tls', 'true').lower() == 'true'
+        self.enabled = bool(self.smtp_host and self.smtp_user and self.smtp_password)
+    
+    def reload_settings(self):
+        """Reload settings from database"""
+        self._load_settings()
+    
+    def is_configured(self) -> bool:
+        """Check if SMTP is configured"""
+        self._load_settings()  # Always reload to get latest settings
+        return self.enabled
+    
+    async def send(
+        self,
+        to_email: str,
+        subject: str,
+        body: str,
+        html_body: Optional[str] = None,
+        embedded_images: Optional[Dict[str, bytes]] = None
+    ) -> bool:
+        """Send email notification
+        
+        Args:
+            to_email: Recipient email
+            subject: Email subject
+            body: Plain text body
+            html_body: HTML body (optional)
+            embedded_images: Dict of {cid: image_bytes} for embedding (optional)
+        """
+        if not self.is_configured():
+            logger.warning("SMTP not configured, skipping email notification")
+            return False
+        
+        try:
+            # Create message - use 'related' for embedded images
+            if embedded_images:
+                msg = MIMEMultipart("related")
+                msg_alt = MIMEMultipart("alternative")
+                msg.attach(msg_alt)
+            else:
+                msg = MIMEMultipart("alternative")
+                msg_alt = msg
+            
+            msg["Subject"] = subject
+            msg["From"] = self.smtp_from
+            msg["To"] = to_email
+            
+            # Add plain text
+            msg_alt.attach(MIMEText(body, "plain", "utf-8"))
+            
+            # Add HTML if provided
+            if html_body:
+                msg_alt.attach(MIMEText(html_body, "html", "utf-8"))
+            
+            # Attach embedded images
+            if embedded_images:
+                for cid, image_data in embedded_images.items():
+                    img = MIMEImage(image_data)
+                    img.add_header('Content-ID', f'<{cid}>')
+                    img.add_header('Content-Disposition', 'inline', filename=f'{cid}.png')
+                    msg.attach(img)
+            
+            # Send in thread pool to not block async
+            # Use asyncio.to_thread for better compatibility with FastAPI
+            try:
+                await asyncio.wait_for(
+                    asyncio.to_thread(self._send_sync, to_email, msg),
+                    timeout=15.0  # 15 second timeout for entire operation
+                )
+            except asyncio.TimeoutError:
+                logger.error(f"Email sending to {to_email} timed out after 15 seconds")
+                return False
+            
+            logger.info(f"Email notification sent to {to_email}")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to send email to {to_email}: {e}")
+            return False
+    
+    def _send_sync(self, to_email: str, msg: MIMEMultipart):
+        """Synchronous email sending with timeout"""
+        context = ssl.create_default_context()
+        timeout = 10  # 10 seconds timeout
+        
+        # Port 465 always uses SSL (SMTPS)
+        # Port 587 uses STARTTLS (SMTP with upgrade)
+        # Port 25 uses plain SMTP (or STARTTLS if smtp_tls=True)
+        use_ssl = self.smtp_port == 465
+        
+        logger.info(f"Sending email via {self.smtp_host}:{self.smtp_port} (SSL={use_ssl}, TLS={self.smtp_tls})")
+        
+        if use_ssl:
+            # SSL connection (port 465 - SMTPS)
+            with smtplib.SMTP_SSL(self.smtp_host, self.smtp_port, context=context, timeout=timeout) as server:
+                server.login(self.smtp_user, self.smtp_password)
+                server.sendmail(self.smtp_from, to_email, msg.as_string())
+        else:
+            # STARTTLS connection (port 587, 25, or custom)
+            with smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=timeout) as server:
+                if self.smtp_tls:
+                    server.starttls(context=context)
+                server.login(self.smtp_user, self.smtp_password)
+                server.sendmail(self.smtp_from, to_email, msg.as_string())
+    
+    def format_notification(self, notification: Dict[str, Any]) -> tuple:
+        """Format notification for email - returns (subject, body, html_body, embedded_images)"""
+        level_emoji = {
+            "critical": "🔴",
+            "warning": "🟠",
+            "info": "🔵",
+            "success": "🟢"
+        }
+        
+        level = notification.get("level", "info")
+        emoji = level_emoji.get(level, "🔵")
+        title = notification.get("title", "Notification")
+        message = notification.get("message", "")
+        source = notification.get("source", "system")
+        created_at = notification.get("created_at", "now")
+        
+        # Получаем имя панели
+        panel_name = get_panel_name()
+        
+        subject = f"{emoji} [{panel_name}] {title}"
+        
+        body = f"""
+{title}
+
+{message}
+
+Level: {level.upper()}
+Source: {source}
+Time: {created_at}
+
+---
+{panel_name}
+        """.strip()
+        
+        # Получаем логотип для встраивания
+        logo_bytes = get_logo_bytes()
+        embedded_images = {}
+        
+        if logo_bytes:
+            embedded_images['logo'] = logo_bytes
+            logo_html = f'<img src="cid:logo" alt="{panel_name}" style="max-height: 128px;">'
+        else:
+            logo_html = f'<span style="font-size: 64px;">🖥️</span>'
+        
+        html_body = f"""
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <style>
+        body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; background: #1a1a2e; color: #eee; padding: 20px; margin: 0; }}
+        .container {{ max-width: 600px; margin: 0 auto; background: #16213e; border-radius: 12px; overflow: hidden; }}
+        .header {{ background: linear-gradient(135deg, #667eea 0%, #4f46e5 100%); padding: 24px; text-align: center; }}
+        .logo {{ display: flex; align-items: center; justify-content: center; }}
+        .content {{ padding: 24px; }}
+        .level {{ display: inline-block; padding: 4px 12px; border-radius: 20px; font-size: 12px; font-weight: bold; margin-bottom: 16px; }}
+        .level.critical {{ background: #f44336; color: white; }}
+        .level.warning {{ background: #ff9800; color: white; }}
+        .level.info {{ background: #2196f3; color: white; }}
+        .level.success {{ background: #4caf50; color: white; }}
+        .title {{ font-size: 20px; font-weight: bold; margin-bottom: 12px; color: #fff; }}
+        .message {{ color: #b0b0b0; line-height: 1.6; margin-bottom: 20px; }}
+        .meta {{ color: #666; font-size: 12px; }}
+        .footer {{ padding: 16px 24px; background: #0f0f23; color: #666; font-size: 12px; text-align: center; }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="header">
+            <div class="logo">{logo_html}</div>
+        </div>
+        <div class="content">
+            <span class="level {level}">{level.upper()}</span>
+            <div class="title">{emoji} {title}</div>
+            <div class="message">{message}</div>
+            <div class="meta">
+                Source: {source} | Time: {created_at}
+            </div>
+        </div>
+        <div class="footer">
+            {panel_name} Notifications
+        </div>
+    </div>
+</body>
+</html>
+        """
+        
+        return subject, body, html_body, embedded_images
+
+
+class TelegramChannel:
+    """Telegram Bot notification channel - settings from database"""
+    
+    def __init__(self):
+        self._load_settings()
+    
+    def _load_settings(self):
+        """Load Telegram settings from database"""
+        self.bot_token = get_notification_setting('telegram_bot_token')
+        self.enabled = bool(self.bot_token)
+        self.api_url = f"https://api.telegram.org/bot{self.bot_token}" if self.bot_token else ""
+    
+    def reload_settings(self):
+        """Reload settings from database"""
+        self._load_settings()
+    
+    def is_configured(self) -> bool:
+        """Check if Telegram bot is configured"""
+        self._load_settings()  # Always reload to get latest settings
+        return self.enabled
+    
+    async def send(self, chat_id: str, message: str, parse_mode: str = "HTML") -> bool:
+        """Send Telegram message"""
+        if not self.is_configured():
+            logger.warning("Telegram bot not configured, skipping notification")
+            return False
+        
+        if httpx is None:
+            logger.error("httpx library not installed")
+            return False
+        
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    f"{self.api_url}/sendMessage",
+                    json={
+                        "chat_id": chat_id,
+                        "text": message,
+                        "parse_mode": parse_mode,
+                        "disable_web_page_preview": True
+                    },
+                    timeout=10.0
+                )
+                
+                if response.status_code == 200:
+                    logger.info(f"Telegram notification sent to {chat_id}")
+                    return True
+                else:
+                    logger.error(f"Telegram API error: {response.text}")
+                    return False
+                    
+        except Exception as e:
+            logger.error(f"Failed to send Telegram message to {chat_id}: {e}")
+            return False
+    
+    async def send_photo(self, chat_id: str, photo_bytes: bytes, caption: str, parse_mode: str = "HTML") -> bool:
+        """Send Telegram message with photo"""
+        if not self.is_configured():
+            logger.warning("Telegram bot not configured, skipping notification")
+            return False
+        
+        if httpx is None:
+            logger.error("httpx library not installed")
+            return False
+        
+        try:
+            async with httpx.AsyncClient() as client:
+                files = {'photo': ('logo.png', photo_bytes, 'image/png')}
+                data = {
+                    'chat_id': chat_id,
+                    'caption': caption,
+                    'parse_mode': parse_mode
+                }
+                response = await client.post(
+                    f"{self.api_url}/sendPhoto",
+                    data=data,
+                    files=files,
+                    timeout=15.0
+                )
+                
+                if response.status_code == 200:
+                    logger.info(f"Telegram photo notification sent to {chat_id}")
+                    return True
+                else:
+                    logger.error(f"Telegram API error: {response.text}")
+                    # Fallback to text message
+                    return await self.send(chat_id, caption, parse_mode)
+                    
+        except Exception as e:
+            logger.error(f"Failed to send Telegram photo to {chat_id}: {e}")
+            # Fallback to text message
+            return await self.send(chat_id, caption, parse_mode)
+    
+    async def get_bot_info(self) -> Optional[Dict[str, Any]]:
+        """Get bot information"""
+        if not self.is_configured() or httpx is None:
+            return None
+        
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(f"{self.api_url}/getMe", timeout=10.0)
+                if response.status_code == 200:
+                    return response.json().get("result")
+        except Exception as e:
+            logger.error(f"Failed to get bot info: {e}")
+        return None
+    
+    async def verify_chat_id(self, chat_id: str) -> bool:
+        """Verify chat_id by sending a test message"""
+        local_time = get_local_time()
+        panel_name = get_panel_name()
+        return await self.send(
+            chat_id,
+            f"✅ <b>{panel_name}</b>\n\n"
+            f"Telegram notifications configured successfully!\n\n"
+            f"🕐 Server time: {local_time}",
+            "HTML"
+        )
+    
+    def format_notification(self, notification: Dict[str, Any]) -> Tuple[str, Optional[bytes]]:
+        """Format notification for Telegram - returns (message, logo_bytes)"""
+        level_emoji = {
+            "critical": "🔴",
+            "warning": "🟠", 
+            "info": "🔵",
+            "success": "🟢"
+        }
+        
+        level = notification.get("level", "info")
+        emoji = level_emoji.get(level, "🔵")
+        title = notification.get("title", "Notification")
+        message = notification.get("message", "")
+        source = notification.get("source", "system")
+        created_at = notification.get("created_at", "now")
+        
+        # Получаем имя панели
+        panel_name = get_panel_name()
+        
+        # Получаем логотип
+        logo_bytes = get_logo_bytes()
+        
+        text = f"""
+{emoji} <b>[{panel_name}] {title}</b>
+
+{message}
+
+<i>Level:</i> {level.upper()}
+<i>Source:</i> {source}
+<i>Time:</i> {created_at}
+        """.strip()
+        
+        return text, logo_bytes
+
+
+# Singleton instances - initialized lazily
+_email_channel: Optional[EmailChannel] = None
+_telegram_channel: Optional[TelegramChannel] = None
+
+
+def get_email_channel() -> EmailChannel:
+    """Get or create EmailChannel singleton"""
+    global _email_channel
+    if _email_channel is None:
+        _email_channel = EmailChannel()
+    else:
+        _email_channel.reload_settings()
+    return _email_channel
+
+
+def get_telegram_channel() -> TelegramChannel:
+    """Get or create TelegramChannel singleton"""
+    global _telegram_channel
+    if _telegram_channel is None:
+        _telegram_channel = TelegramChannel()
+    else:
+        _telegram_channel.reload_settings()
+    return _telegram_channel
+
+
+def reload_notification_channels():
+    """Force reload all notification channel settings"""
+    global _email_channel, _telegram_channel
+    if _email_channel:
+        _email_channel.reload_settings()
+    if _telegram_channel:
+        _telegram_channel.reload_settings()

backend/app/services/notification_service.py

@@ -0,0 +1,687 @@
+"""
+Notification Service - Business logic for notifications
+"""
+
+from typing import List, Optional, Dict, Any
+from datetime import datetime, timedelta, timezone
+from sqlalchemy.orm import Session
+from sqlalchemy import and_, or_, desc
+from loguru import logger
+
+
+def utcnow() -> datetime:
+    """Get current UTC time as timezone-aware datetime"""
+    return datetime.now(timezone.utc)
+
+
+from ..models import Notification, NotificationPreference, User, PanelSettings
+from ..schemas import (
+    NotificationCreate,
+    NotificationUpdate,
+    NotificationPreferenceUpdate
+)
+from ..i18n import t
+
+
+def get_panel_language(db: Session) -> str:
+    """Get panel language from settings"""
+    setting = db.query(PanelSettings).filter(PanelSettings.key == "language").first()
+    return setting.value if setting else "ru"
+
+
+class NotificationService:
+    """Service for managing notifications"""
+
+    @staticmethod
+    def create_notification(
+        db: Session,
+        notification_data: NotificationCreate
+    ) -> Notification:
+        """
+        Create a new notification
+        
+        Args:
+            db: Database session
+            notification_data: Notification data
+            
+        Returns:
+            Created notification
+        """
+        user_id = notification_data.user_id
+        level = notification_data.level
+        title = notification_data.title
+        
+        # Check user preferences
+        prefs = NotificationService.get_user_preferences(db, user_id)
+        
+        if not prefs or not prefs.enabled:
+            # User has disabled notifications
+            logger.info(f"[NOTIFICATION SKIPPED] User {user_id}: notifications disabled | Level: {level} | Title: {title}")
+            return None
+        
+        # Check if notification level is allowed
+        if notification_data.level not in prefs.notification_levels:
+            logger.info(f"[NOTIFICATION SKIPPED] User {user_id}: level '{level}' not in allowed levels {prefs.notification_levels} | Title: {title}")
+            return None
+        
+        # Check if notification type is allowed
+        if notification_data.type not in prefs.notification_types:
+            logger.info(f"[NOTIFICATION SKIPPED] User {user_id}: type '{notification_data.type}' not in allowed types | Title: {title}")
+            return None
+        
+        # Check quiet hours
+        if NotificationService._is_quiet_hours(prefs):
+            # Only critical notifications during quiet hours
+            if notification_data.level != "critical":
+                logger.info(f"[NOTIFICATION SKIPPED] User {user_id}: quiet hours active, only critical allowed | Level: {level} | Title: {title}")
+                return None
+        
+        # Create notification
+        notification = Notification(
+            user_id=notification_data.user_id,
+            type=notification_data.type,
+            level=notification_data.level,
+            title=notification_data.title,
+            message=notification_data.message,
+            data=notification_data.data,
+            link=notification_data.link,
+            source=notification_data.source,
+            source_id=notification_data.source_id,
+            expires_at=notification_data.expires_at or (utcnow() + timedelta(days=7))
+        )
+        
+        db.add(notification)
+        db.commit()
+        db.refresh(notification)
+        
+        logger.info(f"[NOTIFICATION CREATED] ID: {notification.id} | User: {user_id} | Level: {level} | Type: {notification_data.type} | Title: {title}")
+        
+        return notification
+
+    @staticmethod
+    def get_user_notifications(
+        db: Session,
+        user_id: int,
+        unread_only: bool = False,
+        level: Optional[str] = None,
+        limit: int = 50,
+        offset: int = 0
+    ) -> tuple[List[Notification], int]:
+        """
+        Get user notifications with filters
+        
+        Args:
+            db: Database session
+            user_id: User ID
+            unread_only: Only unread notifications
+            level: Filter by level (critical, warning, info, success)
+            limit: Maximum number of notifications
+            offset: Offset for pagination
+            
+        Returns:
+            Tuple of (notifications, total_count)
+        """
+        query = db.query(Notification).filter(Notification.user_id == user_id)
+        
+        if unread_only:
+            query = query.filter(Notification.read == False)
+        
+        if level:
+            query = query.filter(Notification.level == level)
+        
+        # Filter out expired notifications
+        query = query.filter(
+            or_(
+                Notification.expires_at.is_(None),
+                Notification.expires_at > utcnow()
+            )
+        )
+        
+        total = query.count()
+        notifications = query.order_by(desc(Notification.created_at)).limit(limit).offset(offset).all()
+        
+        return notifications, total
+
+    @staticmethod
+    def get_unread_count(db: Session, user_id: int) -> int:
+        """Get count of unread notifications"""
+        return db.query(Notification).filter(
+            and_(
+                Notification.user_id == user_id,
+                Notification.read == False,
+                or_(
+                    Notification.expires_at.is_(None),
+                    Notification.expires_at > utcnow()
+                )
+            )
+        ).count()
+
+    @staticmethod
+    def mark_as_read(
+        db: Session,
+        notification_id: int,
+        user_id: int
+    ) -> Optional[Notification]:
+        """Mark notification as read"""
+        notification = db.query(Notification).filter(
+            and_(
+                Notification.id == notification_id,
+                Notification.user_id == user_id
+            )
+        ).first()
+        
+        if notification and not notification.read:
+            notification.read = True
+            notification.read_at = utcnow()
+            db.commit()
+            db.refresh(notification)
+        
+        return notification
+
+    @staticmethod
+    def mark_all_as_read(db: Session, user_id: int) -> int:
+        """Mark all notifications as read for user"""
+        count = db.query(Notification).filter(
+            and_(
+                Notification.user_id == user_id,
+                Notification.read == False
+            )
+        ).update({
+            "read": True,
+            "read_at": utcnow()
+        })
+        db.commit()
+        return count
+
+    @staticmethod
+    def delete_notification(
+        db: Session,
+        notification_id: int,
+        user_id: int
+    ) -> bool:
+        """Delete a notification"""
+        notification = db.query(Notification).filter(
+            and_(
+                Notification.id == notification_id,
+                Notification.user_id == user_id
+            )
+        ).first()
+        
+        if notification:
+            db.delete(notification)
+            db.commit()
+            return True
+        return False
+
+    @staticmethod
+    def delete_all_read(db: Session, user_id: int) -> int:
+        """Delete all read notifications for user"""
+        count = db.query(Notification).filter(
+            and_(
+                Notification.user_id == user_id,
+                Notification.read == True
+            )
+        ).delete()
+        db.commit()
+        return count
+
+    @staticmethod
+    def cleanup_expired(db: Session) -> int:
+        """Delete all expired notifications"""
+        count = db.query(Notification).filter(
+            and_(
+                Notification.expires_at.isnot(None),
+                Notification.expires_at < utcnow()
+            )
+        ).delete()
+        db.commit()
+        return count
+
+    # Preferences management
+    
+    @staticmethod
+    def get_user_preferences(
+        db: Session,
+        user_id: int
+    ) -> Optional[NotificationPreference]:
+        """Get user notification preferences"""
+        prefs = db.query(NotificationPreference).filter(
+            NotificationPreference.user_id == user_id
+        ).first()
+        
+        # Create default preferences if not exist
+        if not prefs:
+            prefs = NotificationPreference(
+                user_id=user_id,
+                enabled=True,
+                email_enabled=False,
+                email_critical_only=True,
+                telegram_enabled=False,
+                notification_levels=["critical", "warning", "info", "success"],
+                notification_types=["vm_status", "resource_alert", "system", "ipam", "template"]
+            )
+            db.add(prefs)
+            db.commit()
+            db.refresh(prefs)
+        
+        return prefs
+
+    @staticmethod
+    def update_user_preferences(
+        db: Session,
+        user_id: int,
+        prefs_data: NotificationPreferenceUpdate
+    ) -> NotificationPreference:
+        """Update user notification preferences"""
+        prefs = NotificationService.get_user_preferences(db, user_id)
+        
+        # Update fields
+        update_data = prefs_data.model_dump(exclude_unset=True)
+        for field, value in update_data.items():
+            setattr(prefs, field, value)
+        
+        prefs.updated_at = utcnow()
+        db.commit()
+        db.refresh(prefs)
+        
+        return prefs
+
+    # Helper methods
+    
+    @staticmethod
+    def _is_quiet_hours(prefs: NotificationPreference) -> bool:
+        """Check if current time is in quiet hours"""
+        if not prefs.quiet_hours_start or not prefs.quiet_hours_end:
+            return False
+        
+        now = utcnow().time()
+        start = datetime.strptime(prefs.quiet_hours_start, "%H:%M").time()
+        end = datetime.strptime(prefs.quiet_hours_end, "%H:%M").time()
+        
+        if start <= end:
+            return start <= now <= end
+        else:
+            # Crosses midnight
+            return now >= start or now <= end
+
+    # Convenience methods for creating specific notification types
+    
+    @staticmethod
+    def notify_vm_status(
+        db: Session,
+        user_id: int,
+        vm_id: int,
+        vm_name: str,
+        status: str,
+        level: str = "info",
+        server_name: str = "",
+        old_status: str = ""
+    ) -> Optional[Notification]:
+        """Create VM status notification"""
+        lang = get_panel_language(db)
+        
+        # Determine title and message based on status
+        if status == "running" and old_status == "stopped":
+            title = t("notify_vm_started_title", lang, vm_name=vm_name)
+            message = t("notify_vm_started_message", lang, vm_name=vm_name, server_name=server_name)
+        elif status == "stopped" and old_status == "running":
+            title = t("notify_vm_stopped_title", lang, vm_name=vm_name)
+            message = t("notify_vm_stopped_message", lang, vm_name=vm_name, server_name=server_name)
+        else:
+            title = t("notify_vm_status_change_title", lang, vm_name=vm_name)
+            message = t("notify_vm_status_change_message", lang, vm_name=vm_name, old_status=old_status, new_status=status)
+        
+        return NotificationService.create_notification(
+            db,
+            NotificationCreate(
+                user_id=user_id,
+                type="vm_status",
+                level=level,
+                title=title,
+                message=message,
+                data={"vm_id": vm_id, "vm_name": vm_name, "status": status, "old_status": old_status},
+                link=f"/vms/instance/{vm_id}",
+                source="proxmox",
+                source_id=str(vm_id)
+            )
+        )
+
+    @staticmethod
+    def notify_resource_alert(
+        db: Session,
+        user_id: int,
+        resource_type: str,
+        resource_name: str,
+        usage_percent: float,
+        threshold: float
+    ) -> Optional[Notification]:
+        """Create resource usage alert"""
+        lang = get_panel_language(db)
+        level = "critical" if usage_percent >= 90 else "warning"
+        
+        title = t("notify_resource_alert_title", lang, resource_type=resource_type)
+        message = t("notify_resource_alert_message", lang,
+                   resource_name=resource_name,
+                   usage=f"{usage_percent:.1f}",
+                   threshold=f"{threshold:.0f}")
+        
+        return NotificationService.create_notification(
+            db,
+            NotificationCreate(
+                user_id=user_id,
+                type="resource_alert",
+                level=level,
+                title=title,
+                message=message,
+                data={
+                    "resource_type": resource_type,
+                    "resource_name": resource_name,
+                    "usage": usage_percent,
+                    "threshold": threshold
+                },
+                source="system"
+            )
+        )
+
+    @staticmethod
+    def notify_system_event(
+        db: Session,
+        user_id: int,
+        title: str,
+        message: str,
+        level: str = "info",
+        data: Optional[dict] = None
+    ) -> Optional[Notification]:
+        """Create generic system notification"""
+        return NotificationService.create_notification(
+            db,
+            NotificationCreate(
+                user_id=user_id,
+                type="system",
+                level=level,
+                title=title,
+                message=message,
+                data=data,
+                source="system"
+            )
+        )
+
+    # ============================================
+    # Async methods for multi-channel notifications
+    # ============================================
+    
+    @staticmethod
+    async def create_and_send(
+        db: Session,
+        user_id: int,
+        notification_type: str,
+        level: str,
+        title: str,
+        message: str,
+        data: Optional[Dict[str, Any]] = None,
+        link: Optional[str] = None,
+        source: Optional[str] = None,
+        source_id: Optional[str] = None,
+        force_send: bool = False
+    ) -> Optional[Notification]:
+        """
+        Create notification and send through enabled channels (Email, Telegram)
+        
+        Args:
+            db: Database session
+            user_id: User ID
+            notification_type: Type of notification
+            level: Level (critical, warning, info, success)
+            title: Notification title
+            message: Notification message
+            data: Additional data
+            link: Optional link
+            source: Source system
+            source_id: Source entity ID
+            force_send: If True, send to external channels even if "Critical Only" is enabled
+            
+        Returns:
+            Created notification
+        """
+        # Create in-app notification
+        notification = NotificationService.create_notification(
+            db,
+            NotificationCreate(
+                user_id=user_id,
+                type=notification_type,
+                level=level,
+                title=title,
+                message=message,
+                data=data,
+                link=link,
+                source=source,
+                source_id=source_id
+            )
+        )
+        
+        if not notification:
+            logger.warning(f"[NOTIFICATION] In-app notification was not created for user {user_id} | Level: {level} | Title: {title}")
+            return None
+        
+        # Get user preferences for external channels
+        prefs = NotificationService.get_user_preferences(db, user_id)
+        if not prefs or not prefs.enabled:
+            logger.info(f"[EXTERNAL CHANNELS SKIPPED] User {user_id}: preferences disabled | Notification ID: {notification.id}")
+            return notification
+        
+        # Get user for email
+        user = db.query(User).filter(User.id == user_id).first()
+        
+        # Prepare notification dict for formatting
+        notification_dict = {
+            "title": title,
+            "message": message,
+            "level": level,
+            "source": source or "system",
+            "created_at": notification.created_at.strftime("%Y-%m-%d %H:%M:%S") if notification.created_at else "now"
+        }
+        
+        # Log channel delivery attempt details
+        logger.info(f"[EXTERNAL CHANNELS] Processing notification {notification.id} | Level: {level} | Email enabled: {prefs.email_enabled} | Telegram enabled: {prefs.telegram_enabled} | Critical only: {prefs.email_critical_only} | Force send: {force_send}")
+        
+        # Send Email
+        if prefs.email_enabled and user and user.email:
+            # Check critical only setting (bypass if force_send is True)
+            if prefs.email_critical_only and level != "critical" and not force_send:
+                logger.info(f"[EMAIL SKIPPED] User {user_id}: critical_only=True but level='{level}' | Notification ID: {notification.id}")
+            else:
+                try:
+                    from .notification_channels import get_email_channel
+                    email_ch = get_email_channel()
+                    logger.info(f"[EMAIL] Attempting to send | To: {user.email} | SMTP configured: {email_ch.is_configured()} | Host: {email_ch.smtp_host}")
+                    if email_ch.is_configured():
+                        subject, body, html_body, embedded_images = email_ch.format_notification(notification_dict)
+                        result = await email_ch.send(user.email, subject, body, html_body, embedded_images)
+                        if result:
+                            logger.info(f"[EMAIL SUCCESS] Sent to {user.email} | Notification ID: {notification.id}")
+                        else:
+                            logger.error(f"[EMAIL FAILED] Could not send to {user.email} | Notification ID: {notification.id}")
+                    else:
+                        logger.warning(f"[EMAIL SKIPPED] SMTP not configured | Host: {email_ch.smtp_host} | User: {email_ch.smtp_user}")
+                except Exception as e:
+                    logger.error(f"[EMAIL ERROR] Failed to send to {user.email}: {e} | Notification ID: {notification.id}")
+        else:
+            if not prefs.email_enabled:
+                logger.debug(f"[EMAIL SKIPPED] User {user_id}: email_enabled=False")
+            elif not user or not user.email:
+                logger.warning(f"[EMAIL SKIPPED] User {user_id}: no email address configured")
+        
+        # Send Telegram
+        if prefs.telegram_enabled and prefs.telegram_chat_id:
+            # Check critical only for Telegram too if set (bypass if force_send is True)
+            if prefs.email_critical_only and level != "critical" and not force_send:
+                logger.info(f"[TELEGRAM SKIPPED] User {user_id}: critical_only=True but level='{level}' | Notification ID: {notification.id}")
+            else:
+                try:
+                    from .notification_channels import get_telegram_channel
+                    tg_ch = get_telegram_channel()
+                    logger.info(f"[TELEGRAM] Attempting to send | Chat ID: {prefs.telegram_chat_id} | Bot configured: {tg_ch.is_configured()}")
+                    if tg_ch.is_configured():
+                        tg_message, logo_bytes = tg_ch.format_notification(notification_dict)
+                        if logo_bytes:
+                            result = await tg_ch.send_photo(prefs.telegram_chat_id, logo_bytes, tg_message)
+                        else:
+                            result = await tg_ch.send(prefs.telegram_chat_id, tg_message)
+                        if result:
+                            logger.info(f"[TELEGRAM SUCCESS] Sent to {prefs.telegram_chat_id} | Notification ID: {notification.id}")
+                        else:
+                            logger.error(f"[TELEGRAM FAILED] Could not send to {prefs.telegram_chat_id} | Notification ID: {notification.id}")
+                    else:
+                        logger.warning(f"[TELEGRAM SKIPPED] Bot not configured")
+                except Exception as e:
+                    logger.error(f"[TELEGRAM ERROR] Failed to send to {prefs.telegram_chat_id}: {e} | Notification ID: {notification.id}")
+        else:
+            if not prefs.telegram_enabled:
+                logger.debug(f"[TELEGRAM SKIPPED] User {user_id}: telegram_enabled=False")
+            elif not prefs.telegram_chat_id:
+                logger.warning(f"[TELEGRAM SKIPPED] User {user_id}: no telegram_chat_id configured")
+        
+        return notification
+
+    @staticmethod
+    async def send_test_notification(
+        db: Session,
+        user_id: int,
+        channel: str = "all"
+    ) -> Dict[str, Any]:
+        """
+        Send test notification through specified channels
+        
+        Args:
+            db: Database session
+            user_id: User ID
+            channel: Channel to test (all, email, telegram, inapp)
+            
+        Returns:
+            Dict with results for each channel
+        """
+        results = {"inapp": False, "email": False, "telegram": False, "errors": []}
+        
+        user = db.query(User).filter(User.id == user_id).first()
+        prefs = NotificationService.get_user_preferences(db, user_id)
+        
+        # Get panel language
+        lang = get_panel_language(db)
+        
+        # Use local time instead of UTC
+        local_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        
+        test_notification = {
+            "title": t("notify_test_title", lang),
+            "message": t("notify_test_message", lang),
+            "level": "info",
+            "source": "system",
+            "created_at": local_time
+        }
+        
+        # In-app notification
+        if channel in ["all", "inapp"]:
+            try:
+                notification = NotificationService.create_notification(
+                    db,
+                    NotificationCreate(
+                        user_id=user_id,
+                        type="system",
+                        level="info",
+                        title=test_notification["title"],
+                        message=test_notification["message"],
+                        source="system"
+                    )
+                )
+                results["inapp"] = notification is not None
+            except Exception as e:
+                results["errors"].append(f"In-app: {str(e)}")
+        
+        # Email
+        if channel in ["all", "email"]:
+            try:
+                from .notification_channels import get_email_channel
+                email_ch = get_email_channel()
+                if email_ch.is_configured():
+                    if user and user.email:
+                        subject, body, html_body, embedded_images = email_ch.format_notification(test_notification)
+                        results["email"] = await email_ch.send(user.email, subject, body, html_body, embedded_images)
+                    else:
+                        results["errors"].append("Email: User has no email address")
+                else:
+                    results["errors"].append("Email: SMTP not configured")
+            except Exception as e:
+                results["errors"].append(f"Email: {str(e)}")
+        
+        # Telegram
+        if channel in ["all", "telegram"]:
+            try:
+                from .notification_channels import get_telegram_channel
+                tg_ch = get_telegram_channel()
+                if tg_ch.is_configured():
+                    if prefs and prefs.telegram_chat_id:
+                        tg_message, logo_bytes = tg_ch.format_notification(test_notification)
+                        if logo_bytes:
+                            results["telegram"] = await tg_ch.send_photo(prefs.telegram_chat_id, logo_bytes, tg_message)
+                        else:
+                            results["telegram"] = await tg_ch.send(prefs.telegram_chat_id, tg_message)
+                    else:
+                        results["errors"].append("Telegram: Chat ID not configured")
+                else:
+                    results["errors"].append("Telegram: Bot token not configured")
+            except Exception as e:
+                results["errors"].append(f"Telegram: {str(e)}")
+        
+        return results
+
+    @staticmethod
+    async def get_channels_status(db: Session, user_id: int) -> Dict[str, Any]:
+        """
+        Get status of notification channels
+        
+        Args:
+            db: Database session
+            user_id: User ID
+            
+        Returns:
+            Dict with channel status information
+        """
+        from .notification_channels import get_email_channel, get_telegram_channel
+        
+        user = db.query(User).filter(User.id == user_id).first()
+        prefs = NotificationService.get_user_preferences(db, user_id)
+        
+        email_ch = get_email_channel()
+        tg_ch = get_telegram_channel()
+        
+        # Get Telegram bot info if configured
+        telegram_bot_info = None
+        if tg_ch.is_configured():
+            try:
+                telegram_bot_info = await tg_ch.get_bot_info()
+            except Exception:
+                pass
+        
+        return {
+            "email": {
+                "configured": email_ch.is_configured(),
+                "enabled": prefs.email_enabled if prefs else False,
+                "user_email": user.email if user else None,
+                "smtp_host": email_ch.smtp_host if email_ch.is_configured() else None,
+                "critical_only": prefs.email_critical_only if prefs else True
+            },
+            "telegram": {
+                "configured": tg_ch.is_configured(),
+                "enabled": prefs.telegram_enabled if prefs else False,
+                "chat_id": prefs.telegram_chat_id if prefs else None,
+                "bot_info": telegram_bot_info
+            },
+            "inapp": {
+                "configured": True,
+                "enabled": prefs.enabled if prefs else True
+            },
+            "quiet_hours": {
+                "start": prefs.quiet_hours_start if prefs else None,
+                "end": prefs.quiet_hours_end if prefs else None
+            }
+        }

backend/app/services/security_service.py

@@ -0,0 +1,599 @@
+"""
+Security Service - Authentication, sessions, and protection
+"""
+
+import secrets
+import hashlib
+from datetime import datetime, timedelta, timezone
+from typing import Optional, Dict, Any, Tuple
+from sqlalchemy.orm import Session
+from sqlalchemy import and_, or_, func
+from loguru import logger
+
+from ..models import (
+    User, Role, ActiveSession, LoginAttempt, BlockedIP, SecuritySetting
+)
+
+
+def utcnow() -> datetime:
+    """Get current UTC time as timezone-aware datetime"""
+    return datetime.now(timezone.utc)
+
+
+class SecurityService:
+    """Service for security operations"""
+    
+    # Cache for security settings
+    _settings_cache: Dict[str, str] = {}
+    _cache_time: datetime = None
+    _cache_ttl = 60  # seconds
+    
+    @classmethod
+    def get_setting(cls, db: Session, key: str, default: str = None) -> str:
+        """Get security setting with caching"""
+        # Check cache
+        now = utcnow()
+        if cls._cache_time and (now - cls._cache_time).seconds < cls._cache_ttl:
+            if key in cls._settings_cache:
+                return cls._settings_cache[key]
+        
+        # Query from DB
+        setting = db.query(SecuritySetting).filter(SecuritySetting.key == key).first()
+        value = setting.value if setting else default
+        
+        # Update cache
+        cls._settings_cache[key] = value
+        cls._cache_time = now
+        
+        return value
+    
+    @classmethod
+    def get_setting_int(cls, db: Session, key: str, default: int = 0) -> int:
+        """Get security setting as integer"""
+        value = cls.get_setting(db, key, str(default))
+        try:
+            return int(value)
+        except (ValueError, TypeError):
+            return default
+    
+    @classmethod
+    def get_setting_bool(cls, db: Session, key: str, default: bool = False) -> bool:
+        """Get security setting as boolean"""
+        value = cls.get_setting(db, key, str(default).lower())
+        
+        # Handle None/empty values gracefully
+        if value is None:
+            return default
+        
+        # Accept real booleans from DB/cache
+        if isinstance(value, bool):
+            return value
+        
+        return str(value).lower() in ('true', '1', 'yes', 'on')
+    
+    @classmethod
+    def update_setting(cls, db: Session, key: str, value: str) -> bool:
+        """Update security setting"""
+        setting = db.query(SecuritySetting).filter(SecuritySetting.key == key).first()
+        if setting:
+            setting.value = value
+            setting.updated_at = utcnow()
+        else:
+            setting = SecuritySetting(key=key, value=value)
+            db.add(setting)
+        
+        db.commit()
+        
+        # Clear cache
+        cls._settings_cache.pop(key, None)
+        return True
+    
+    # ==================== IP Blocking ====================
+    
+    @staticmethod
+    def is_ip_blocked(db: Session, ip_address: str) -> Tuple[bool, Optional[str]]:
+        """Check if IP is blocked, returns (is_blocked, reason)"""
+        blocked = db.query(BlockedIP).filter(BlockedIP.ip_address == ip_address).first()
+        
+        if not blocked:
+            return False, None
+        
+        if blocked.is_permanent:
+            return True, blocked.reason
+        
+        # Compare timezone-aware datetimes
+        now = utcnow()
+        if blocked.expires_at and blocked.expires_at > now:
+            return True, blocked.reason
+        
+        # Block expired, remove it
+        db.delete(blocked)
+        db.commit()
+        return False, None
+    
+    @staticmethod
+    def block_ip(
+        db: Session, 
+        ip_address: str, 
+        reason: str, 
+        blocked_by: str = "system",
+        duration_minutes: int = 60,
+        permanent: bool = False
+    ) -> BlockedIP:
+        """Block an IP address"""
+        blocked = db.query(BlockedIP).filter(BlockedIP.ip_address == ip_address).first()
+        
+        if blocked:
+            blocked.reason = reason
+            blocked.blocked_by = blocked_by
+            blocked.blocked_at = utcnow()
+            blocked.attempts_count += 1
+            blocked.is_permanent = permanent
+            if not permanent:
+                blocked.expires_at = utcnow() + timedelta(minutes=duration_minutes)
+        else:
+            blocked = BlockedIP(
+                ip_address=ip_address,
+                reason=reason,
+                blocked_by=blocked_by,
+                expires_at=None if permanent else utcnow() + timedelta(minutes=duration_minutes),
+                is_permanent=permanent,
+                attempts_count=1
+            )
+            db.add(blocked)
+        
+        db.commit()
+        logger.warning(f"IP {ip_address} blocked: {reason}")
+        return blocked
+    
+    @staticmethod
+    def unblock_ip(db: Session, ip_address: str) -> bool:
+        """Unblock an IP address"""
+        blocked = db.query(BlockedIP).filter(BlockedIP.ip_address == ip_address).first()
+        if blocked:
+            db.delete(blocked)
+            db.commit()
+            logger.info(f"IP {ip_address} unblocked")
+            return True
+        return False
+    
+    # ==================== Login Attempts ====================
+    
+    @staticmethod
+    def record_login_attempt(
+        db: Session,
+        ip_address: str,
+        username: str = None,
+        success: bool = False,
+        failure_reason: str = None,
+        user_agent: str = None
+    ) -> LoginAttempt:
+        """Record a login attempt"""
+        attempt = LoginAttempt(
+            username=username,
+            ip_address=ip_address,
+            user_agent=user_agent[:500] if user_agent else None,
+            success=success,
+            failure_reason=failure_reason
+        )
+        db.add(attempt)
+        db.commit()
+        return attempt
+    
+    @classmethod
+    def check_brute_force(cls, db: Session, ip_address: str, username: str = None) -> Tuple[bool, str]:
+        """
+        Check for brute force attacks
+        Returns (should_block, reason)
+        """
+        threshold = cls.get_setting_int(db, "ip_block_threshold", 10)
+        window_minutes = 30
+        
+        # Count recent failed attempts from this IP
+        since = utcnow() - timedelta(minutes=window_minutes)
+        
+        failed_count = db.query(LoginAttempt).filter(
+            and_(
+                LoginAttempt.ip_address == ip_address,
+                LoginAttempt.success == False,
+                LoginAttempt.attempted_at > since
+            )
+        ).count()
+        
+        if failed_count >= threshold:
+            return True, f"Too many failed login attempts ({failed_count})"
+        
+        return False, None
+    
+    # ==================== Session Management ====================
+    
+    @staticmethod
+    def generate_session_token() -> str:
+        """Generate a secure session token"""
+        return secrets.token_hex(32)
+    
+    @classmethod
+    def create_session(
+        cls,
+        db: Session,
+        user: User,
+        ip_address: str = None,
+        user_agent: str = None
+    ) -> Tuple[ActiveSession, bool]:
+        """
+        Create a new session for user.
+        Returns (session, was_other_session_terminated)
+        """
+        session_timeout = cls.get_setting_int(db, "session_timeout_minutes", 60)
+        
+        terminated_other = False
+        
+        # Create new session
+        token = cls.generate_session_token()
+        expires_at = utcnow() + timedelta(minutes=session_timeout)
+        
+        # Parse device info from user agent
+        device_info = cls._parse_device_info(user_agent) if user_agent else None
+        
+        session = ActiveSession(
+            user_id=user.id,
+            session_token=token,
+            ip_address=ip_address,
+            user_agent=user_agent[:500] if user_agent else None,
+            device_info=device_info,
+            expires_at=expires_at,
+            is_active=True
+        )
+        
+        db.add(session)
+        db.commit()
+        db.refresh(session)
+        
+        logger.info(f"Session created for user {user.username} from {ip_address}")
+        
+        return session, terminated_other
+    
+    @staticmethod
+    def _parse_device_info(user_agent: str) -> str:
+        """Parse device info from user agent"""
+        if not user_agent:
+            return "Unknown"
+        
+        ua = user_agent.lower()
+        
+        # Browser detection
+        browser = "Unknown"
+        if "firefox" in ua:
+            browser = "Firefox"
+        elif "edg" in ua:
+            browser = "Edge"
+        elif "chrome" in ua:
+            browser = "Chrome"
+        elif "safari" in ua:
+            browser = "Safari"
+        elif "opera" in ua:
+            browser = "Opera"
+        
+        # OS detection
+        os = "Unknown"
+        if "windows" in ua:
+            os = "Windows"
+        elif "mac os" in ua or "macos" in ua:
+            os = "macOS"
+        elif "linux" in ua:
+            os = "Linux"
+        elif "android" in ua:
+            os = "Android"
+        elif "iphone" in ua or "ipad" in ua:
+            os = "iOS"
+        
+        return f"{browser} on {os}"
+    
+    @staticmethod
+    def validate_session(db: Session, token: str) -> Optional[ActiveSession]:
+        """Validate session token and return session if valid"""
+        session = db.query(ActiveSession).filter(
+            and_(
+                ActiveSession.session_token == token,
+                ActiveSession.is_active == True
+            )
+        ).first()
+        
+        if not session:
+            return None
+        
+        # Check expiration
+        # Compare timezone-aware datetimes
+        now = utcnow()
+        if session.expires_at < now:
+            session.is_active = False
+            db.commit()
+            return None
+        
+        # Update last activity and extend session expiration (sliding expiration)
+        session.last_activity = utcnow()
+        # Get session timeout setting
+        session_timeout = SecurityService.get_setting_int(db, "session_timeout_minutes", 60)
+        session.expires_at = utcnow() + timedelta(minutes=session_timeout)
+        db.commit()
+        
+        return session
+    
+    @staticmethod
+    def terminate_session(db: Session, token: str) -> bool:
+        """Terminate a session"""
+        session = db.query(ActiveSession).filter(
+            ActiveSession.session_token == token
+        ).first()
+        
+        if session:
+            session.is_active = False
+            db.commit()
+            return True
+        return False
+    
+    @staticmethod
+    def terminate_all_user_sessions(db: Session, user_id: int) -> int:
+        """Terminate all sessions for a user"""
+        count = db.query(ActiveSession).filter(
+            and_(
+                ActiveSession.user_id == user_id,
+                ActiveSession.is_active == True
+            )
+        ).update({"is_active": False})
+        db.commit()
+        return count
+    
+    @staticmethod
+    def get_user_sessions(db: Session, user_id: int) -> list:
+        """Get all active sessions for a user"""
+        return db.query(ActiveSession).filter(
+            and_(
+                ActiveSession.user_id == user_id,
+                ActiveSession.is_active == True
+            )
+        ).all()
+    
+    @staticmethod
+    def cleanup_expired_sessions(db: Session) -> int:
+        """Clean up expired sessions"""
+        count = db.query(ActiveSession).filter(
+            or_(
+                ActiveSession.expires_at < utcnow(),
+                ActiveSession.is_active == False
+            )
+        ).delete()
+        db.commit()
+        logger.info(f"Cleaned up {count} expired sessions")
+        return count
+    
+    # ==================== Account Locking ====================
+    
+    @classmethod
+    def increment_failed_attempts(cls, db: Session, user: User) -> bool:
+        """
+        Increment failed login attempts for user.
+        Returns True if account was locked.
+        """
+        max_attempts = cls.get_setting_int(db, "max_login_attempts", 5)
+        lockout_minutes = cls.get_setting_int(db, "lockout_duration_minutes", 30)
+        
+        user.failed_login_attempts += 1
+        
+        if user.failed_login_attempts >= max_attempts:
+            user.locked_until = utcnow() + timedelta(minutes=lockout_minutes)
+            db.commit()
+            logger.warning(f"User {user.username} locked out after {user.failed_login_attempts} failed attempts")
+            return True
+        
+        db.commit()
+        return False
+    
+    @staticmethod
+    def reset_failed_attempts(db: Session, user: User):
+        """Reset failed login attempts after successful login"""
+        user.failed_login_attempts = 0
+        user.locked_until = None
+        db.commit()
+    
+    @staticmethod
+    def is_account_locked(user: User) -> Tuple[bool, Optional[datetime]]:
+        """Check if account is locked, returns (is_locked, locked_until)"""
+        if not user.locked_until:
+            return False, None
+        
+        # Compare timezone-aware datetimes
+        if user.locked_until > utcnow():
+            return True, user.locked_until
+        
+        return False, None
+    
+    # ==================== Role & Permission Management ====================
+    
+    @staticmethod
+    def get_role(db: Session, role_name: str) -> Optional[Role]:
+        """Get role by name"""
+        return db.query(Role).filter(Role.name == role_name).first()
+    
+    @staticmethod
+    def get_all_roles(db: Session, include_inactive: bool = False) -> list:
+        """Get all roles"""
+        query = db.query(Role)
+        if not include_inactive:
+            query = query.filter(Role.is_active == True)
+        return query.order_by(Role.name).all()
+    
+    @staticmethod
+    def create_role(
+        db: Session,
+        name: str,
+        display_name: str,
+        description: str = None,
+        permissions: dict = None
+    ) -> Role:
+        """Create a new role"""
+        role = Role(
+            name=name.lower(),
+            display_name=display_name,
+            description=description,
+            permissions=permissions or {},
+            is_system=False
+        )
+        db.add(role)
+        db.commit()
+        db.refresh(role)
+        return role
+    
+    @staticmethod
+    def update_role(
+        db: Session,
+        role_id: int,
+        display_name: str = None,
+        description: str = None,
+        permissions: dict = None
+    ) -> Optional[Role]:
+        """Update a role"""
+        role = db.query(Role).filter(Role.id == role_id).first()
+        if not role:
+            return None
+        
+        if display_name:
+            role.display_name = display_name
+        if description is not None:
+            role.description = description
+        if permissions is not None:
+            role.permissions = permissions
+        
+        role.updated_at = utcnow()
+        db.commit()
+        db.refresh(role)
+        return role
+    
+    @staticmethod
+    def delete_role(db: Session, role_id: int) -> Tuple[bool, str]:
+        """Delete a role if it's not a system role"""
+        role = db.query(Role).filter(Role.id == role_id).first()
+        if not role:
+            return False, "Role not found"
+        
+        if role.is_system:
+            return False, "Cannot delete system role"
+        
+        # Check if any users have this role
+        user_count = db.query(User).filter(User.role_id == role_id).count()
+        if user_count > 0:
+            return False, f"Cannot delete role with {user_count} assigned users"
+        
+        db.delete(role)
+        db.commit()
+        return True, "Role deleted"
+    
+    @staticmethod
+    def get_all_permissions() -> Dict[str, Dict[str, str]]:
+        """
+        Get all available permissions grouped by category.
+        Returns permissions in legacy format for backwards compatibility with UI.
+        """
+        try:
+            from ..rbac import get_permission_categories_legacy
+            return get_permission_categories_legacy()
+        except ImportError:
+            # Fallback to static list if RBAC module not available
+            return {
+                "Dashboard": {
+                    "dashboard.view": "View dashboard"
+                },
+                "Proxmox Servers": {
+                    "proxmox.view": "View servers",
+                    "proxmox.manage": "Manage server settings",
+                    "proxmox.servers.add": "Add new servers",
+                    "proxmox.servers.edit": "Edit servers",
+                    "proxmox.servers.delete": "Delete servers"
+                },
+                "Virtual Machines": {
+                    "vms.view": "View VMs",
+                    "vms.create": "Create VMs",
+                    "vms.start": "Start VMs",
+                    "vms.stop": "Stop VMs",
+                    "vms.restart": "Restart VMs",
+                    "vms.delete": "Delete VMs",
+                    "vms.console": "Access VM console",
+                    "vms.migrate": "Migrate VMs"
+                },
+                "Templates": {
+                    "templates.view": "View templates",
+                    "templates.manage": "Manage templates"
+                },
+                "IPAM": {
+                    "ipam.view": "View IPAM",
+                    "ipam.manage": "Manage IPAM"
+                },
+                "Logs": {
+                    "logs.view": "View logs",
+                    "logs.export": "Export logs",
+                    "logs.delete": "Delete logs"
+                },
+                "Settings": {
+                    "settings.view": "View settings",
+                    "settings.panel": "Panel settings",
+                    "settings.security": "Security settings"
+                },
+                "Users": {
+                    "users.view": "View users",
+                    "users.create": "Create users",
+                    "users.edit": "Edit users",
+                    "users.delete": "Delete users"
+                },
+                "Roles": {
+                    "roles.view": "View roles",
+                    "roles.manage": "Manage roles"
+                },
+                "Notifications": {
+                    "notifications.manage": "Manage notifications"
+                }
+            }
+    
+    @staticmethod
+    def get_all_permissions_v2() -> Dict[str, Dict[str, str]]:
+        """
+        Get all available permissions in new format grouped by category.
+        Uses new resource:action format.
+        """
+        try:
+            from ..rbac import get_permission_categories
+            return get_permission_categories()
+        except ImportError:
+            return SecurityService.get_all_permissions()
+    
+    # ==================== Password Validation ====================
+    
+    @classmethod
+    def validate_password(cls, db: Session, password: str) -> Tuple[bool, list]:
+        """
+        Validate password against security requirements.
+        Returns (is_valid, list_of_errors)
+        """
+        errors = []
+        
+        min_length = cls.get_setting_int(db, "password_min_length", 8)
+        require_upper = cls.get_setting_bool(db, "password_require_uppercase", True)
+        require_lower = cls.get_setting_bool(db, "password_require_lowercase", True)
+        require_numbers = cls.get_setting_bool(db, "password_require_numbers", True)
+        require_special = cls.get_setting_bool(db, "password_require_special", False)
+        
+        if len(password) < min_length:
+            errors.append(f"Password must be at least {min_length} characters")
+        
+        if require_upper and not any(c.isupper() for c in password):
+            errors.append("Password must contain at least one uppercase letter")
+        
+        if require_lower and not any(c.islower() for c in password):
+            errors.append("Password must contain at least one lowercase letter")
+        
+        if require_numbers and not any(c.isdigit() for c in password):
+            errors.append("Password must contain at least one number")
+        
+        if require_special and not any(c in "!@#$%^&*()_+-=[]{}|;:,.<>?" for c in password):
+            errors.append("Password must contain at least one special character")
+        
+        return len(errors) == 0, errors

backend/app/services/task_queue_service.py

@@ -0,0 +1,338 @@
+"""
+Task Queue Service for bulk VM/Container operations
+Processes tasks from the queue in background
+"""
+
+from datetime import datetime, timezone
+from typing import List, Dict, Any, Optional
+from loguru import logger
+from sqlalchemy.orm import Session
+
+
+def utcnow() -> datetime:
+    """Get current UTC time as timezone-aware datetime"""
+    return datetime.now(timezone.utc)
+
+
+try:
+    from backend.app.db import SessionLocal
+    from backend.app.models import TaskQueue, ProxmoxServer, User
+    from backend.app.proxmox_client import ProxmoxClient
+except ImportError:
+    from app.db import SessionLocal
+    from app.models import TaskQueue, ProxmoxServer, User
+    from app.proxmox_client import ProxmoxClient
+
+
+class TaskQueueService:
+    """Service for managing and processing bulk operation tasks"""
+    
+    # Valid task types
+    TASK_TYPES = ['bulk_start', 'bulk_stop', 'bulk_restart', 'bulk_delete', 'bulk_shutdown']
+    
+    @staticmethod
+    def create_task(
+        db: Session,
+        task_type: str,
+        user_id: int,
+        items: List[Dict[str, Any]]
+    ) -> TaskQueue:
+        """
+        Create a new bulk operation task
+        
+        Args:
+            db: Database session
+            task_type: Type of operation (bulk_start, bulk_stop, etc.)
+            user_id: ID of user who initiated
+            items: List of VMs/containers to process
+                   Each item: {server_id, vmid, vm_type, name, node}
+        """
+        if task_type not in TaskQueueService.TASK_TYPES:
+            raise ValueError(f"Invalid task type: {task_type}")
+        
+        if not items:
+            raise ValueError("No items provided for bulk operation")
+        
+        task = TaskQueue(
+            task_type=task_type,
+            status='pending',
+            user_id=user_id,
+            total_items=len(items),
+            completed_items=0,
+            failed_items=0,
+            task_data=items,
+            results=[]
+        )
+        db.add(task)
+        db.commit()
+        db.refresh(task)
+        
+        logger.info(f"[TASK QUEUE] Created task #{task.id}: {task_type} with {len(items)} items")
+        return task
+    
+    @staticmethod
+    def get_pending_tasks(db: Session) -> List[TaskQueue]:
+        """Get all pending tasks ordered by creation time"""
+        return db.query(TaskQueue).filter(
+            TaskQueue.status == 'pending'
+        ).order_by(TaskQueue.created_at.asc()).all()
+    
+    @staticmethod
+    def get_running_tasks(db: Session) -> List[TaskQueue]:
+        """Get all currently running tasks"""
+        return db.query(TaskQueue).filter(
+            TaskQueue.status == 'running'
+        ).all()
+    
+    @staticmethod
+    def get_user_tasks(db: Session, user_id: int, limit: int = 20) -> List[TaskQueue]:
+        """Get recent tasks for a user"""
+        return db.query(TaskQueue).filter(
+            TaskQueue.user_id == user_id
+        ).order_by(TaskQueue.created_at.desc()).limit(limit).all()
+    
+    @staticmethod
+    def get_active_tasks(db: Session) -> List[TaskQueue]:
+        """Get all active (pending or running) tasks"""
+        return db.query(TaskQueue).filter(
+            TaskQueue.status.in_(['pending', 'running'])
+        ).order_by(TaskQueue.created_at.asc()).all()
+    
+    @staticmethod
+    def cancel_task(db: Session, task_id: int, user_id: int) -> bool:
+        """Cancel a pending task"""
+        task = db.query(TaskQueue).filter(
+            TaskQueue.id == task_id,
+            TaskQueue.user_id == user_id,
+            TaskQueue.status == 'pending'
+        ).first()
+        
+        if not task:
+            return False
+        
+        task.status = 'cancelled'
+        task.completed_at = utcnow()
+        db.commit()
+        
+        logger.info(f"[TASK QUEUE] Task #{task_id} cancelled by user {user_id}")
+        return True
+
+
+class TaskQueueProcessor:
+    """Processor that executes tasks from the queue"""
+    
+    def __init__(self):
+        self._proxmox_clients: Dict[int, ProxmoxClient] = {}
+    
+    def _get_proxmox_client(self, db: Session, server_id: int) -> Optional[ProxmoxClient]:
+        """Get or create Proxmox client for server"""
+        if server_id in self._proxmox_clients:
+            client = self._proxmox_clients[server_id]
+            if client.is_connected():
+                return client
+        
+        server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
+        if not server:
+            return None
+        
+        try:
+            host = server.ip_address or server.hostname
+            if server.port and server.port != 8006:
+                host = f"{host}:{server.port}"
+            
+            if server.use_password and server.password:
+                client = ProxmoxClient(
+                    host=host,
+                    user=server.api_user,
+                    password=server.password,
+                    verify_ssl=server.verify_ssl
+                )
+            elif server.api_token_name and server.api_token_value:
+                client = ProxmoxClient(
+                    host=host,
+                    user=server.api_user,
+                    token_name=server.api_token_name,
+                    token_value=server.api_token_value,
+                    verify_ssl=server.verify_ssl
+                )
+            else:
+                return None
+            
+            self._proxmox_clients[server_id] = client
+            return client
+        except Exception as e:
+            logger.error(f"[TASK QUEUE] Failed to connect to server {server_id}: {e}")
+            return None
+    
+    def _execute_vm_action(
+        self, 
+        client: ProxmoxClient, 
+        action: str, 
+        vmid: int, 
+        vm_type: str, 
+        node: str
+    ) -> tuple[bool, str]:
+        """Execute action on VM/container"""
+        try:
+            if vm_type == 'lxc':
+                if action == 'start':
+                    client.start_container(node, vmid)
+                elif action == 'stop':
+                    client.stop_container(node, vmid)
+                elif action == 'restart':
+                    client.restart_container(node, vmid)
+                elif action == 'shutdown':
+                    # Shutdown = graceful stop
+                    client.stop_container(node, vmid)
+                elif action == 'delete':
+                    client.delete_container(node, vmid)
+                else:
+                    return False, f"Unknown action: {action}"
+            else:  # qemu
+                if action == 'start':
+                    client.start_vm(node, vmid)
+                elif action == 'stop':
+                    client.stop_vm(node, vmid)
+                elif action == 'restart':
+                    client.restart_vm(node, vmid)
+                elif action == 'shutdown':
+                    # Shutdown = graceful stop
+                    client.stop_vm(node, vmid)
+                elif action == 'delete':
+                    client.delete_vm(node, vmid)
+                else:
+                    return False, f"Unknown action: {action}"
+            
+            return True, "OK"
+        except Exception as e:
+            return False, str(e)
+    
+    def process_task(self, task: TaskQueue) -> None:
+        """Process a single task"""
+        db = SessionLocal()
+        try:
+            # Mark as running
+            task = db.query(TaskQueue).filter(TaskQueue.id == task.id).first()
+            if not task or task.status != 'pending':
+                return
+            
+            task.status = 'running'
+            task.started_at = utcnow()
+            db.commit()
+            
+            logger.info(f"[TASK QUEUE] Starting task #{task.id}: {task.task_type}")
+            
+            # Determine action from task type
+            action_map = {
+                'bulk_start': 'start',
+                'bulk_stop': 'stop',
+                'bulk_restart': 'restart',
+                'bulk_shutdown': 'shutdown',
+                'bulk_delete': 'delete',
+            }
+            action = action_map.get(task.task_type)
+            
+            if not action:
+                task.status = 'failed'
+                task.error_message = f"Unknown task type: {task.task_type}"
+                task.completed_at = utcnow()
+                db.commit()
+                return
+            
+            results = []
+            items = task.task_data or []
+            
+            for item in items:
+                server_id = item.get('server_id')
+                vmid = item.get('vmid')
+                vm_type = item.get('vm_type', 'qemu')
+                node = item.get('node', '')
+                name = item.get('name', f'{vm_type.upper()}-{vmid}')
+                
+                client = self._get_proxmox_client(db, server_id)
+                if not client:
+                    results.append({
+                        'server_id': server_id,
+                        'vmid': vmid,
+                        'name': name,
+                        'success': False,
+                        'message': 'Failed to connect to Proxmox server'
+                    })
+                    task.failed_items += 1
+                else:
+                    success, message = self._execute_vm_action(client, action, vmid, vm_type, node)
+                    results.append({
+                        'server_id': server_id,
+                        'vmid': vmid,
+                        'name': name,
+                        'success': success,
+                        'message': message
+                    })
+                    if success:
+                        task.completed_items += 1
+                    else:
+                        task.failed_items += 1
+                
+                # Update progress
+                task.results = results
+                db.commit()
+            
+            # Mark as completed
+            task.status = 'completed'
+            task.completed_at = utcnow()
+            db.commit()
+            
+            logger.info(f"[TASK QUEUE] Completed task #{task.id}: {task.completed_items} success, {task.failed_items} failed")
+            
+        except Exception as e:
+            logger.error(f"[TASK QUEUE] Task #{task.id} failed with error: {e}")
+            try:
+                task.status = 'failed'
+                task.error_message = str(e)
+                task.completed_at = utcnow()
+                db.commit()
+            except:
+                pass
+        finally:
+            db.close()
+    
+    def process_pending_tasks(self) -> int:
+        """Process all pending tasks. Returns number of tasks processed."""
+        db = SessionLocal()
+        try:
+            # Get pending tasks
+            pending_tasks = TaskQueueService.get_pending_tasks(db)
+            
+            if not pending_tasks:
+                return 0
+            
+            # Check if there's already a running task
+            running_tasks = TaskQueueService.get_running_tasks(db)
+            if running_tasks:
+                logger.debug(f"[TASK QUEUE] {len(running_tasks)} task(s) already running, skipping")
+                return 0
+            
+            # Process first pending task
+            task = pending_tasks[0]
+            db.close()  # Close before processing (will create new session)
+            
+            self.process_task(task)
+            return 1
+            
+        except Exception as e:
+            logger.error(f"[TASK QUEUE] Error processing pending tasks: {e}")
+            return 0
+        finally:
+            try:
+                db.close()
+            except:
+                pass
+
+
+# Global processor instance
+task_processor = TaskQueueProcessor()
+
+
+def process_task_queue():
+    """Function to be called by scheduler to process queue"""
+    return task_processor.process_pending_tasks()

backend/app/services/update_service.py

@@ -0,0 +1,307 @@
+"""
+Update Service - Check for updates and perform system updates
+"""
+import os
+import subprocess
+import asyncio
+import logging
+from typing import Optional, Dict, Any
+from datetime import datetime
+import httpx
+
+logger = logging.getLogger(__name__)
+
+# Путь к корню проекта (монтируется через volume или хост)
+# В Docker это /project, на хосте - определяется через переменную окружения
+PROJECT_ROOT = os.environ.get('PROJECT_ROOT', '/project')
+VERSION_FILE = os.path.join(PROJECT_ROOT, "VERSION")
+
+# Для fallback - путь внутри контейнера где может быть VERSION
+CONTAINER_VERSION_FILE = "/app/VERSION"
+
+# Статус обновления (in-memory)
+update_status = {
+    "is_updating": False,
+    "started_at": None,
+    "stage": None,
+    "progress": 0,
+    "error": None,
+    "completed": False
+}
+
+
+def get_current_version() -> str:
+    """Получить текущую версию из файла VERSION"""
+    # Сначала пробуем файл из проекта (если монтирован)
+    if os.path.exists(VERSION_FILE):
+        try:
+            with open(VERSION_FILE, "r") as f:
+                return f.read().strip()
+        except Exception as e:
+            logger.error(f"Error reading VERSION file from project: {e}")
+    
+    # Fallback - читаем из контейнера
+    if os.path.exists(CONTAINER_VERSION_FILE):
+        try:
+            with open(CONTAINER_VERSION_FILE, "r") as f:
+                return f.read().strip()
+        except Exception as e:
+            logger.error(f"Error reading VERSION file from container: {e}")
+    
+    return "unknown"
+
+
+def is_git_available() -> bool:
+    """Проверить, доступен ли git"""
+    try:
+        result = subprocess.run(
+            ["git", "--version"],
+            capture_output=True,
+            text=True,
+            timeout=5
+        )
+        return result.returncode == 0
+    except Exception:
+        return False
+
+
+def is_project_mounted() -> bool:
+    """Проверить, смонтирован ли проект с .git"""
+    git_dir = os.path.join(PROJECT_ROOT, ".git")
+    return os.path.exists(git_dir)
+
+
+def ensure_safe_directory():
+    """Добавить PROJECT_ROOT в git safe.directory"""
+    try:
+        subprocess.run(
+            ["git", "config", "--global", "--add", "safe.directory", PROJECT_ROOT],
+            capture_output=True,
+            timeout=5
+        )
+    except Exception:
+        pass
+
+
+async def check_for_updates() -> Dict[str, Any]:
+    """
+    Проверить наличие обновлений
+    Сравнивает локальную версию с удалённой
+    """
+    current_version = get_current_version()
+    
+    result = {
+        "current_version": current_version,
+        "latest_version": None,
+        "update_available": False,
+        "changelog": None,
+        "error": None,
+        "git_available": is_git_available(),
+        "project_mounted": is_project_mounted()
+    }
+    
+    # Проверяем доступность git и проекта
+    if not result["git_available"]:
+        result["error"] = "Git is not installed in container. Mount project directory with docker-compose volume."
+        return result
+    
+    if not result["project_mounted"]:
+        result["error"] = "Project directory not mounted. Add volume mount in docker-compose.yml: ./:/project"
+        return result
+    
+    # Добавляем safe.directory для git
+    ensure_safe_directory()
+    
+    try:
+        # Fetch последних изменений без merge
+        fetch_result = subprocess.run(
+            ["git", "fetch", "origin", "main"],
+            cwd=PROJECT_ROOT,
+            capture_output=True,
+            text=True,
+            timeout=30
+        )
+        
+        if fetch_result.returncode != 0:
+            result["error"] = f"Failed to fetch updates: {fetch_result.stderr}"
+            return result
+        
+        # Получить версию из удалённого репозитория
+        remote_version_result = subprocess.run(
+            ["git", "show", "origin/main:VERSION"],
+            cwd=PROJECT_ROOT,
+            capture_output=True,
+            text=True,
+            timeout=10
+        )
+        
+        if remote_version_result.returncode == 0:
+            result["latest_version"] = remote_version_result.stdout.strip()
+        else:
+            result["error"] = "Failed to get remote version"
+            return result
+        
+        # Сравнить версии
+        if result["latest_version"] != current_version:
+            result["update_available"] = True
+            
+            # Получить changelog из удалённого репозитория
+            changelog_result = subprocess.run(
+                ["git", "show", "origin/main:CHANGELOG.md"],
+                cwd=PROJECT_ROOT,
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+            
+            if changelog_result.returncode == 0:
+                # Извлекаем только последнюю версию из changelog
+                changelog_lines = changelog_result.stdout.split('\n')
+                latest_changelog = []
+                in_latest_version = False
+                version_count = 0
+                
+                for line in changelog_lines:
+                    if line.startswith('## [v'):
+                        version_count += 1
+                        if version_count == 1:
+                            in_latest_version = True
+                        elif version_count == 2:
+                            break
+                    
+                    if in_latest_version:
+                        latest_changelog.append(line)
+                
+                result["changelog"] = '\n'.join(latest_changelog)
+        
+        # Проверить количество коммитов позади
+        behind_result = subprocess.run(
+            ["git", "rev-list", "--count", "HEAD..origin/main"],
+            cwd=PROJECT_ROOT,
+            capture_output=True,
+            text=True,
+            timeout=10
+        )
+        
+        if behind_result.returncode == 0:
+            result["commits_behind"] = int(behind_result.stdout.strip())
+        
+    except subprocess.TimeoutExpired:
+        result["error"] = "Timeout while checking for updates"
+    except Exception as e:
+        result["error"] = str(e)
+        logger.error(f"Error checking for updates: {e}")
+    
+    return result
+
+
+def get_update_status() -> Dict[str, Any]:
+    """Получить текущий статус обновления"""
+    return update_status.copy()
+
+
+async def perform_update() -> Dict[str, Any]:
+    """
+    Выполнить обновление системы
+    1. Git pull
+    2. Docker compose build
+    3. Docker compose up
+    """
+    global update_status
+    
+    if update_status["is_updating"]:
+        return {"success": False, "error": "Update already in progress"}
+    
+    # Проверяем доступность
+    if not is_git_available():
+        return {"success": False, "error": "Git not available"}
+    
+    if not is_project_mounted():
+        return {"success": False, "error": "Project not mounted"}
+    
+    # Добавляем safe.directory для git
+    ensure_safe_directory()
+    
+    # Сбросить статус
+    update_status = {
+        "is_updating": True,
+        "started_at": datetime.now().isoformat(),
+        "stage": "initializing",
+        "progress": 0,
+        "error": None,
+        "completed": False
+    }
+    
+    try:
+        # Stage 1: Git pull (10%)
+        update_status["stage"] = "pulling"
+        update_status["progress"] = 10
+        
+        pull_result = subprocess.run(
+            ["git", "pull", "origin", "main"],
+            cwd=PROJECT_ROOT,
+            capture_output=True,
+            text=True,
+            timeout=120
+        )
+        
+        if pull_result.returncode != 0:
+            raise Exception(f"Git pull failed: {pull_result.stderr}")
+        
+        # Stage 2: Docker build (50%)
+        update_status["stage"] = "building"
+        update_status["progress"] = 30
+        
+        # Запускаем rebuild в фоне - это перезапустит сам контейнер
+        # Используем nohup чтобы процесс продолжился после рестарта
+        build_script = f"""
+cd {PROJECT_ROOT}
+docker compose up -d --build app 2>&1 | tee /tmp/update.log
+"""
+        
+        # Записываем скрипт обновления
+        script_path = "/tmp/update_panel.sh"
+        with open(script_path, "w") as f:
+            f.write(build_script)
+        os.chmod(script_path, 0o755)
+        
+        # Запускаем скрипт через nohup
+        subprocess.Popen(
+            ["nohup", "sh", script_path],
+            cwd=PROJECT_ROOT,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            start_new_session=True
+        )
+        
+        update_status["stage"] = "restarting"
+        update_status["progress"] = 80
+        
+        return {
+            "success": True,
+            "message": "Update started. The panel will restart shortly."
+        }
+        
+    except subprocess.TimeoutExpired:
+        update_status["error"] = "Update timeout"
+        update_status["is_updating"] = False
+        return {"success": False, "error": "Update timeout"}
+    except Exception as e:
+        update_status["error"] = str(e)
+        update_status["is_updating"] = False
+        logger.error(f"Update failed: {e}")
+        return {"success": False, "error": str(e)}
+
+
+def reset_update_status():
+    """Сбросить статус обновления"""
+    global update_status
+    update_status = {
+        "is_updating": False,
+        "started_at": None,
+        "stage": None,
+        "progress": 0,
+        "error": None,
+        "completed": False
+    }
+