pvemanager/backend/migrations/migrations.py

"""
PVEmanager Database Migrations
==============================
Unified migration file for all database schema changes.

This file contains all migrations in order:
1. Notifications system (tables: notifications, notification_preferences)
2. RBAC and Security (tables: roles, active_sessions, login_attempts, blocked_ips, security_settings)
3. Cross-node Template Support (columns in os_templates)
4. VM Cache (columns in vm_instances)
5. Task Queue
6. OS Template Icons
7. VM Template Name
8. Enhanced Audit Logs
9. RBAC v2 - Migrate to new permission format

Usage:
    from migrations.migrations import run_all_migrations
    run_all_migrations(engine, db_session)
"""

import json
from sqlalchemy import text
from loguru import logger


# ==================== Default Data ====================

DEFAULT_ROLES = [
    {
        "name": "admin",
        "display_name": "Administrator",
        "description": "Full access to all features",
        "is_system": True,
        "permissions": {
            "dashboard.view": True,
            "proxmox.view": True,
            "proxmox.manage": True,
            "proxmox.servers.add": True,
            "proxmox.servers.edit": True,
            "proxmox.servers.delete": True,
            "vms.view": True,
            "vms.create": True,
            "vms.start": True,
            "vms.stop": True,
            "vms.restart": True,
            "vms.delete": True,
            "vms.console": True,
            "vms.migrate": True,
            "templates.view": True,
            "templates.manage": True,
            "ipam.view": True,
            "ipam.manage": True,
            "logs.view": True,
            "logs.export": True,
            "logs.delete": True,
            "settings.view": True,
            "settings.panel": True,
            "settings.security": True,
            "users.view": True,
            "users.create": True,
            "users.edit": True,
            "users.delete": True,
            "roles.view": True,
            "roles.manage": True,
            "notifications.manage": True,
            "backup:view": True,
            "backup:create": True,
            "backup:restore": True,
            "backup:delete": True,
            "backup:manage": True,
        }
    },
    {
        "name": "moderator",
        "display_name": "Moderator",
        "description": "Can manage VMs and view logs",
        "is_system": True,
        "permissions": {
            "dashboard.view": True,
            "proxmox.view": True,
            "proxmox.manage": False,
            "proxmox.servers.add": False,
            "proxmox.servers.edit": False,
            "proxmox.servers.delete": False,
            "vms.view": True,
            "vms.create": True,
            "vms.start": True,
            "vms.stop": True,
            "vms.restart": True,
            "vms.delete": False,
            "vms.console": True,
            "vms.migrate": False,
            "templates.view": True,
            "templates.manage": False,
            "ipam.view": True,
            "ipam.manage": False,
            "logs.view": True,
            "logs.export": True,
            "logs.delete": False,
            "settings.view": True,
            "settings.panel": False,
            "settings.security": False,
            "users.view": True,
            "users.create": False,
            "users.edit": False,
            "users.delete": False,
            "roles.view": False,
            "roles.manage": False,
            "notifications.manage": True,
            "backup:view": True,
            "backup:create": True,
            "backup:restore": True,
            "backup:delete": False,
            "backup:manage": False,
        }
    },
    {
        "name": "user",
        "display_name": "User",
        "description": "VPS-style user: can only access their own instances",
        "is_system": True,
        "permissions": {
            "dashboard.view": False,
            "proxmox.view": True,
            "proxmox.manage": False,
            "proxmox.servers.add": False,
            "proxmox.servers.edit": False,
            "proxmox.servers.delete": False,
            "vms.view": False,
            "vms:view:own": True,
            "vms.create": False,
            "vms.start": True,
            "vms:start:own": True,
            "vms.stop": True,
            "vms:stop:own": True,
            "vms.restart": True,
            "vms:restart:own": True,
            "vms.delete": False,
            "vms.console": True,
            "vms:console:own": True,
            "vms:snapshots:own": True,
            "vms.migrate": False,
            "templates.view": True,
            "templates.manage": False,
            "ipam.view": False,
            "ipam.manage": False,
            "logs.view": False,
            "logs.export": False,
            "logs.delete": False,
            "settings.view": True,
            "settings.panel": False,
            "settings.security": False,
            "users.view": False,
            "users.create": False,
            "users.edit": False,
            "users.delete": False,
            "roles.view": False,
            "roles.manage": False,
            "notifications.manage": True,
        }
    },
    {
        "name": "demo",
        "display_name": "Demo User",
        "description": "Read-only access for demonstration",
        "is_system": True,
        "permissions": {
            "dashboard.view": True,
            "proxmox.view": True,
            "proxmox.manage": False,
            "proxmox.servers.add": False,
            "proxmox.servers.edit": False,
            "proxmox.servers.delete": False,
            "vms.view": True,
            "vms.create": False,
            "vms.start": False,
            "vms.stop": False,
            "vms.restart": False,
            "vms.delete": False,
            "vms.console": False,
            "vms.migrate": False,
            "templates.view": True,
            "templates.manage": False,
            "ipam.view": True,
            "ipam.manage": False,
            "logs.view": False,
            "logs.export": False,
            "logs.delete": False,
            "settings.view": False,
            "settings.panel": False,
            "settings.security": False,
            "users.view": False,
            "users.create": False,
            "users.edit": False,
            "users.delete": False,
            "roles.view": False,
            "roles.manage": False,
            "notifications.manage": False,
        }
    },
]

SECURITY_DEFAULTS = [
    ("max_login_attempts", "5", "Maximum failed login attempts before lockout"),
    ("lockout_duration_minutes", "30", "Account lockout duration in minutes"),
    ("session_timeout_minutes", "60", "Session timeout in minutes"),
    ("single_session_enabled", "false", "Allow only one active session per user"),
    ("ip_block_threshold", "10", "Failed attempts before IP is blocked"),
    ("ip_block_duration_minutes", "60", "IP block duration in minutes"),
    ("password_min_length", "8", "Minimum password length"),
    ("password_require_uppercase", "true", "Require uppercase letters in password"),
    ("password_require_lowercase", "true", "Require lowercase letters in password"),
    ("password_require_numbers", "true", "Require numbers in password"),
    ("password_require_special", "false", "Require special characters in password"),
    ("password_expiry_days", "0", "Password expiry in days (0 = never)"),
    ("api_rate_limit_per_minute", "60", "API rate limit per minute per user"),
]


# ==================== Helper Functions ====================

def table_exists(conn, table_name: str) -> bool:
    """Check if a table exists"""
    result = conn.execute(text(f"""
        SELECT EXISTS (
            SELECT FROM information_schema.tables 
            WHERE table_name = '{table_name}'
        )
    """))
    return result.scalar()


def column_exists(conn, table_name: str, column_name: str) -> bool:
    """Check if a column exists in a table"""
    result = conn.execute(text(f"""
        SELECT column_name 
        FROM information_schema.columns 
        WHERE table_name = '{table_name}' AND column_name = '{column_name}'
    """))
    return result.fetchone() is not None


def add_column_if_not_exists(conn, table_name: str, column_name: str, column_type: str):
    """Add a column if it doesn't exist"""
    if not column_exists(conn, table_name, column_name):
        conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}"))
        logger.debug(f"Added column {table_name}.{column_name}")
        return True
    return False


# ==================== Migration 1: Notifications ====================

def migrate_notifications(conn):
    """Create notifications system tables"""
    
    if table_exists(conn, 'notifications'):
        logger.info("✓ Notifications tables already exist")
        return
    
    logger.info("Creating notifications tables...")
    
    conn.execute(text("""
        CREATE TABLE IF NOT EXISTS notifications (
            id SERIAL PRIMARY KEY,
            user_id INTEGER REFERENCES users(id) ON DELETE CASCADE,
            type VARCHAR(50) NOT NULL,
            level VARCHAR(20) NOT NULL CHECK (level IN ('critical', 'warning', 'info', 'success')),
            title VARCHAR(255) NOT NULL,
            message TEXT,
            data JSONB,
            link VARCHAR(500),
            source VARCHAR(50),
            source_id VARCHAR(100),
            read BOOLEAN DEFAULT FALSE,
            read_at TIMESTAMP,
            created_at TIMESTAMP DEFAULT NOW(),
            expires_at TIMESTAMP
        )
    """))
    
    conn.execute(text("""
        CREATE INDEX IF NOT EXISTS idx_notifications_user ON notifications(user_id, read, created_at DESC)
    """))
    conn.execute(text("""
        CREATE INDEX IF NOT EXISTS idx_notifications_level ON notifications(level, created_at DESC)
    """))
    conn.execute(text("""
        CREATE INDEX IF NOT EXISTS idx_notifications_created ON notifications(created_at DESC)
    """))
    
    conn.execute(text("""
        CREATE TABLE IF NOT EXISTS notification_preferences (
            id SERIAL PRIMARY KEY,
            user_id INTEGER REFERENCES users(id) UNIQUE ON DELETE CASCADE,
            enabled BOOLEAN DEFAULT TRUE,
            email_enabled BOOLEAN DEFAULT FALSE,
            email_critical_only BOOLEAN DEFAULT TRUE,
            telegram_enabled BOOLEAN DEFAULT FALSE,
            telegram_chat_id VARCHAR(100),
            webhook_url VARCHAR(500),
            notification_levels JSONB DEFAULT '["critical", "warning", "info", "success"]',
            notification_types JSONB DEFAULT '["vm_status", "resource_alert", "system", "ipam", "template"]',
            quiet_hours_start TIME,
            quiet_hours_end TIME,
            created_at TIMESTAMP DEFAULT NOW(),
            updated_at TIMESTAMP DEFAULT NOW()
        )
    """))
    
    conn.execute(text("""
        CREATE INDEX IF NOT EXISTS idx_notification_preferences_user ON notification_preferences(user_id)
    """))
    
    logger.info("✓ Notifications tables created")


# ==================== Migration 2: RBAC & Security ====================

def migrate_rbac_security(conn):
    """Create RBAC and security tables"""
    
    # Roles table
    if not table_exists(conn, 'roles'):
        logger.info("Creating roles table...")
        conn.execute(text("""
            CREATE TABLE IF NOT EXISTS roles (
                id SERIAL PRIMARY KEY,
                name VARCHAR(50) UNIQUE NOT NULL,
                display_name VARCHAR(100) NOT NULL,
                description TEXT,
                permissions JSONB NOT NULL DEFAULT '{}',
                is_system BOOLEAN DEFAULT FALSE,
                is_active BOOLEAN DEFAULT TRUE,
                created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
                updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
            )
        """))
        logger.info("✓ Roles table created")
    
    # Active sessions table
    if not table_exists(conn, 'active_sessions'):
        logger.info("Creating active_sessions table...")
        conn.execute(text("""
            CREATE TABLE IF NOT EXISTS active_sessions (
                id SERIAL PRIMARY KEY,
                user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
                session_token VARCHAR(64) UNIQUE NOT NULL,
                ip_address VARCHAR(45),
                user_agent VARCHAR(500),
                device_info VARCHAR(200),
                created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
                expires_at TIMESTAMP WITH TIME ZONE NOT NULL,
                last_activity TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
                is_active BOOLEAN DEFAULT TRUE
            )
        """))
        logger.info("✓ Active sessions table created")
    
    # Login attempts table
    if not table_exists(conn, 'login_attempts'):
        logger.info("Creating login_attempts table...")
        conn.execute(text("""
            CREATE TABLE IF NOT EXISTS login_attempts (
                id SERIAL PRIMARY KEY,
                username VARCHAR(100),
                ip_address VARCHAR(45) NOT NULL,
                user_agent VARCHAR(500),
                success BOOLEAN DEFAULT FALSE,
                failure_reason VARCHAR(200),
                attempted_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
            )
        """))
        logger.info("✓ Login attempts table created")
    
    # Blocked IPs table
    if not table_exists(conn, 'blocked_ips'):
        logger.info("Creating blocked_ips table...")
        conn.execute(text("""
            CREATE TABLE IF NOT EXISTS blocked_ips (
                id SERIAL PRIMARY KEY,
                ip_address VARCHAR(45) UNIQUE NOT NULL,
                reason VARCHAR(500),
                blocked_by VARCHAR(100),
                blocked_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
                expires_at TIMESTAMP WITH TIME ZONE,
                is_permanent BOOLEAN DEFAULT FALSE,
                attempts_count INTEGER DEFAULT 0
            )
        """))
        logger.info("✓ Blocked IPs table created")
    
    # Security settings table
    if not table_exists(conn, 'security_settings'):
        logger.info("Creating security_settings table...")
        conn.execute(text("""
            CREATE TABLE IF NOT EXISTS security_settings (
                id SERIAL PRIMARY KEY,
                key VARCHAR(100) UNIQUE NOT NULL,
                value VARCHAR(500) NOT NULL,
                description TEXT,
                updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
            )
        """))
        logger.info("✓ Security settings table created")
    
    # Add columns to users table
    user_columns = [
        ("role_id", "INTEGER REFERENCES roles(id)"),
        ("failed_login_attempts", "INTEGER DEFAULT 0"),
        ("locked_until", "TIMESTAMP WITH TIME ZONE"),
        ("last_password_change", "TIMESTAMP WITH TIME ZONE"),
        ("require_password_change", "BOOLEAN DEFAULT FALSE"),
        ("two_factor_enabled", "BOOLEAN DEFAULT FALSE"),
        ("two_factor_secret", "VARCHAR(100)"),
    ]
    
    for col_name, col_type in user_columns:
        try:
            add_column_if_not_exists(conn, 'users', col_name, col_type)
        except Exception as e:
            logger.debug(f"Column users.{col_name}: {e}")
    
    # Create indexes
    indexes = [
        ("idx_sessions_user", "active_sessions(user_id)"),
        ("idx_sessions_token", "active_sessions(session_token)"),
        ("idx_sessions_expires", "active_sessions(expires_at)"),
        ("idx_login_attempts_ip", "login_attempts(ip_address)"),
        ("idx_login_attempts_time", "login_attempts(attempted_at)"),
        ("idx_blocked_ips_ip", "blocked_ips(ip_address)"),
        ("idx_blocked_ips_expires", "blocked_ips(expires_at)"),
        ("idx_users_role", "users(role_id)"),
    ]
    
    for idx_name, idx_def in indexes:
        try:
            conn.execute(text(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {idx_def}"))
        except Exception as e:
            logger.debug(f"Index {idx_name}: {e}")
    
    # Insert default roles
    for role in DEFAULT_ROLES:
        conn.execute(text("""
            INSERT INTO roles (name, display_name, description, permissions, is_system, is_active)
            VALUES (:name, :display_name, :description, :permissions, :is_system, TRUE)
            ON CONFLICT (name) DO NOTHING
        """), {
            "name": role["name"],
            "display_name": role["display_name"],
            "description": role["description"],
            "permissions": json.dumps(role["permissions"]),
            "is_system": role["is_system"]
        })
    
    # Insert default security settings
    for key, value, description in SECURITY_DEFAULTS:
        conn.execute(text("""
            INSERT INTO security_settings (key, value, description)
            VALUES (:key, :value, :description)
            ON CONFLICT (key) DO NOTHING
        """), {"key": key, "value": value, "description": description})
    
    # Set admin role for existing admin users
    conn.execute(text("""
        UPDATE users 
        SET role_id = (SELECT id FROM roles WHERE name = 'admin')
        WHERE is_admin = TRUE AND role_id IS NULL
    """))
    
    # Set user role for non-admin users
    conn.execute(text("""
        UPDATE users 
        SET role_id = (SELECT id FROM roles WHERE name = 'user')
        WHERE is_admin = FALSE AND role_id IS NULL
    """))
    
    logger.info("✓ RBAC and security migration completed")


# ==================== Migration 3: Cross-node Templates ====================

def migrate_cross_node_templates(conn):
    """Add cross-node template support"""
    
    if not table_exists(conn, 'os_templates'):
        logger.info("Table os_templates does not exist, skipping cross-node migration")
        return
    
    # Add source_node column
    if add_column_if_not_exists(conn, 'os_templates', 'source_node', 'VARCHAR(100)'):
        # Copy existing node values to source_node
        conn.execute(text("""
            UPDATE os_templates 
            SET source_node = node 
            WHERE source_node IS NULL
        """))
        logger.info("✓ Added source_node column to os_templates")
    
    # Add replicated_nodes column
    if add_column_if_not_exists(conn, 'os_templates', 'replicated_nodes', "JSONB DEFAULT '{}'::jsonb"):
        logger.info("✓ Added replicated_nodes column to os_templates")

    # Add vm_type column (qemu/lxc) — existing rows default to qemu
    if add_column_if_not_exists(conn, 'os_templates', 'vm_type', "VARCHAR(10) NOT NULL DEFAULT 'qemu'"):
        logger.info("✓ Added vm_type column to os_templates")

    # Add volid column for LXC vztmpl file-based templates
    if add_column_if_not_exists(conn, 'os_templates', 'volid', "VARCHAR(500)"):
        logger.info("✓ Added volid column to os_templates")

    # vmid must be nullable for vztmpl-based templates
    try:
        conn.execute(text("ALTER TABLE os_templates ALTER COLUMN vmid DROP NOT NULL"))
    except Exception:
        pass
    
    # Make node column nullable
    try:
        conn.execute(text("""
            ALTER TABLE os_templates 
            ALTER COLUMN node DROP NOT NULL
        """))
    except Exception:
        pass  # Column might already be nullable
    
    logger.info("✓ Cross-node template migration completed")


# ==================== Migration 4: VM Cache ====================

def migrate_vm_cache(conn):
    """Add VM cache fields to vm_instances table"""
    
    if not table_exists(conn, 'vm_instances'):
        logger.info("Table vm_instances does not exist, skipping VM cache migration")
        return
    
    # Add new columns
    cache_columns = [
        ("status", "VARCHAR(20) DEFAULT 'unknown'"),
        ("is_template", "BOOLEAN DEFAULT FALSE"),
        ("os_type", "VARCHAR(50)"),
        ("last_sync_at", "TIMESTAMP WITH TIME ZONE"),
    ]
    
    for col_name, col_type in cache_columns:
        add_column_if_not_exists(conn, 'vm_instances', col_name, col_type)
    
    # Alter memory and disk_size to BigInteger
    try:
        conn.execute(text("""
            ALTER TABLE vm_instances 
            ALTER COLUMN memory TYPE BIGINT
        """))
    except Exception:
        pass
    
    try:
        conn.execute(text("""
            ALTER TABLE vm_instances 
            ALTER COLUMN disk_size TYPE BIGINT
        """))
    except Exception:
        pass
    
    logger.info("✓ VM cache migration completed")


# ==================== Migration 5: Task Queue ====================

def migrate_task_queue(conn):
    """Create task_queue table for bulk operations"""
    
    if table_exists(conn, 'task_queue'):
        logger.info("Table task_queue already exists, skipping")
        return
    
    conn.execute(text("""
        CREATE TABLE task_queue (
            id SERIAL PRIMARY KEY,
            task_type VARCHAR(50) NOT NULL,
            status VARCHAR(20) NOT NULL DEFAULT 'pending',
            user_id INTEGER NOT NULL,
            total_items INTEGER NOT NULL DEFAULT 0,
            completed_items INTEGER NOT NULL DEFAULT 0,
            failed_items INTEGER NOT NULL DEFAULT 0,
            task_data JSONB NOT NULL,
            results JSONB,
            error_message TEXT,
            created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
            started_at TIMESTAMP WITH TIME ZONE,
            completed_at TIMESTAMP WITH TIME ZONE
        )
    """))
    
    # Create indexes
    conn.execute(text("""
        CREATE INDEX idx_task_queue_task_type ON task_queue (task_type)
    """))
    conn.execute(text("""
        CREATE INDEX idx_task_queue_status ON task_queue (status)
    """))
    conn.execute(text("""
        CREATE INDEX idx_task_queue_user_id ON task_queue (user_id)
    """))
    conn.execute(text("""
        CREATE INDEX idx_task_status_created ON task_queue (status, created_at)
    """))
    conn.execute(text("""
        CREATE INDEX idx_task_user_status ON task_queue (user_id, status)
    """))
    
    logger.info("✓ Task queue table created")


# ==================== Migration 6: Update OS Template Group Icons ====================

# Маппинг названий групп на новые CSS-адаптивные иконки
OS_ICON_MAPPING = {
    'Ubuntu': '<i class="fa-brands fa-ubuntu os-icon os-icon-ubuntu"></i>',
    'Debian': '<i class="fa-brands fa-debian os-icon os-icon-debian"></i>',
    'CentOS': '<i class="fa-brands fa-centos os-icon os-icon-centos"></i>',
    'Rocky Linux': '<span class="os-icon-rocky-svg"></span>',
    'AlmaLinux': '<span class="os-icon-alma-svg"></span>',
    'Fedora': '<i class="fa-brands fa-fedora os-icon os-icon-fedora"></i>',
    'RHEL': '<i class="fa-brands fa-redhat os-icon os-icon-rhel"></i>',
    'Oracle Linux': '<i class="fa-brands fa-linux os-icon os-icon-oracle"></i>',
    'openSUSE': '<i class="fa-brands fa-suse os-icon os-icon-suse"></i>',
    'Arch Linux': '<i class="fa-brands fa-linux os-icon os-icon-arch"></i>',
    'Gentoo': '<i class="fa-brands fa-linux os-icon os-icon-gentoo"></i>',
    'Alpine': '<i class="fa-brands fa-linux os-icon os-icon-alpine"></i>',
    'Windows': '<i class="fa-brands fa-windows os-icon os-icon-windows"></i>',
    'FreeBSD': '<i class="fa-brands fa-freebsd os-icon os-icon-freebsd"></i>',
    'OpenBSD': '<i class="fa-brands fa-linux os-icon os-icon-openbsd"></i>',
    'Other': '<i class="fa-solid fa-compact-disc os-icon os-icon-other"></i>',
}


def migrate_os_template_icons(conn):
    """Update OS template group icons to theme-adaptive CSS classes"""
    logger.info("Migration 6: Updating OS template group icons...")
    
    # Check if os_template_groups table exists
    result = conn.execute(text("""
        SELECT EXISTS (
            SELECT FROM information_schema.tables 
            WHERE table_name = 'os_template_groups'
        )
    """))
    if not result.scalar():
        logger.info("os_template_groups table does not exist, skipping icon migration")
        return
    
    # First, ensure the icon column is large enough for HTML
    try:
        conn.execute(text("""
            ALTER TABLE os_template_groups ALTER COLUMN icon TYPE VARCHAR(200)
        """))
        logger.info("Extended icon column to VARCHAR(200)")
    except Exception as e:
        logger.debug(f"Icon column already extended or error: {e}")
    
    # Update icons for each known group name
    for group_name, icon_html in OS_ICON_MAPPING.items():
        try:
            conn.execute(text("""
                UPDATE os_template_groups 
                SET icon = :icon 
                WHERE name = :name
            """), {"icon": icon_html, "name": group_name})
            logger.debug(f"Updated icon for group: {group_name}")
        except Exception as e:
            logger.warning(f"Failed to update icon for {group_name}: {e}")
    
    logger.info("✓ OS template group icons updated")


# ==================== Migration 7: VM Template Name ====================

def migrate_vm_template_name(conn):
    """Add template_name column to vm_instances for caching template name"""
    logger.info("Migration 7: Adding template_name to vm_instances...")
    
    if not table_exists(conn, 'vm_instances'):
        logger.info("Table vm_instances does not exist, skipping")
        return
    
    add_column_if_not_exists(conn, 'vm_instances', 'template_name', 'VARCHAR(100)')
    
    logger.info("✓ VM template_name migration completed")


# ==================== Migration 8: Enhanced Audit Logs ====================

def migrate_enhanced_audit_logs(conn):
    """Add enhanced logging columns to audit_logs table"""
    logger.info("Migration 8: Enhancing audit_logs table...")
    
    if not table_exists(conn, 'audit_logs'):
        logger.info("Table audit_logs does not exist, skipping")
        return
    
    # New context columns
    add_column_if_not_exists(conn, 'audit_logs', 'request_id', 'VARCHAR(36)')
    add_column_if_not_exists(conn, 'audit_logs', 'session_id', 'VARCHAR(64)')
    
    # Enhanced user tracking
    add_column_if_not_exists(conn, 'audit_logs', 'geo_location', 'VARCHAR(100)')
    
    # Proxmox server context
    add_column_if_not_exists(conn, 'audit_logs', 'server_id', 'INTEGER')
    add_column_if_not_exists(conn, 'audit_logs', 'server_name', 'VARCHAR(100)')
    add_column_if_not_exists(conn, 'audit_logs', 'node_name', 'VARCHAR(100)')
    
    # Request/Response details
    add_column_if_not_exists(conn, 'audit_logs', 'request_body', 'JSONB')
    add_column_if_not_exists(conn, 'audit_logs', 'response_body', 'JSONB')
    add_column_if_not_exists(conn, 'audit_logs', 'query_params', 'VARCHAR(1000)')
    
    # Error details
    add_column_if_not_exists(conn, 'audit_logs', 'error_traceback', 'TEXT')
    
    # Create new indexes
    try:
        conn.execute(text(
            "CREATE INDEX IF NOT EXISTS idx_audit_request_id ON audit_logs(request_id)"
        ))
    except:
        pass
    
    try:
        conn.execute(text(
            "CREATE INDEX IF NOT EXISTS idx_audit_ip ON audit_logs(ip_address)"
        ))
    except:
        pass
    
    try:
        conn.execute(text(
            "CREATE INDEX IF NOT EXISTS idx_audit_server ON audit_logs(server_id)"
        ))
    except:
        pass
    
    logger.info("✓ Enhanced audit_logs migration completed")


# ==================== Migration 9: RBAC v2 - New Permission Format ====================

def migrate_rbac_v2(conn):
    """
    Migrate roles to new RBAC v2 permission format.
    Converts old format (resource.action) to new format (resource:action)
    """
    logger.info("Migration 9: Migrating to RBAC v2 permission format...")
    
    if not table_exists(conn, 'roles'):
        logger.info("Table roles does not exist, skipping RBAC v2 migration")
        return
    
    # Import migration utilities
    try:
        from app.rbac.migration import (
            migrate_all_roles_to_new_format,
            ensure_default_roles_new_format
        )
    except ImportError:
        try:
            from backend.app.rbac.migration import (
                migrate_all_roles_to_new_format,
                ensure_default_roles_new_format
            )
        except ImportError:
            logger.warning("Could not import RBAC migration utilities, skipping v2 migration")
            return
    
    # Migrate existing roles
    migrated_count = migrate_all_roles_to_new_format(conn)
    logger.info(f"Migrated {migrated_count} roles to new permission format")
    
    # Ensure default roles have new format
    ensure_default_roles_new_format(conn)
    
    logger.info("✓ RBAC v2 migration completed")


# ==================== Migration 10: VM Instance Owner ====================

def migrate_vm_instance_owner(conn):
    """
    Add owner_id column to vm_instances for VPS-style user isolation.
    Users with 'user' role will only see their own instances.
    """
    logger.info("Migration 10: Adding VM instance owner support...")
    
    if not table_exists(conn, 'vm_instances'):
        logger.info("Table vm_instances does not exist, skipping owner migration")
        return
    
    # Add owner_id column
    if not column_exists(conn, 'vm_instances', 'owner_id'):
        logger.info("Adding owner_id column to vm_instances...")
        try:
            conn.execute(text(
                """ALTER TABLE vm_instances 
                   ADD COLUMN owner_id INTEGER REFERENCES users(id) ON DELETE SET NULL"""
            ))
            logger.info("✓ Added owner_id column")
        except Exception as e:
            logger.warning(f"Could not add owner_id column: {e}")
    else:
        logger.info("Column owner_id already exists")
    
    # Create index for owner_id
    try:
        conn.execute(text(
            "CREATE INDEX IF NOT EXISTS idx_vm_instance_owner ON vm_instances(owner_id)"
        ))
        logger.info("✓ Created index on owner_id")
    except Exception as e:
        logger.warning(f"Could not create owner_id index: {e}")
    
    # Add VPS-style permissions to user role (idempotent: only add missing keys)
    try:
        result = conn.execute(text("SELECT permissions FROM roles WHERE name = 'user'"))
        row = result.fetchone()
        if row:
            import json
            permissions = row[0] if isinstance(row[0], dict) else json.loads(row[0] or '{}')

            vps_permissions = {
                "vms:view:own": True,      # View only own VMs
                "vms:start:own": True,     # Start own VMs
                "vms:stop:own": True,      # Stop own VMs
                "vms:restart:own": True,   # Restart own VMs
                "vms:console:own": True,   # Console to own VMs
                "vms:snapshots:own": True, # Manage snapshots of own VMs
            }

            # Only add keys that are not already present — never overwrite existing values
            missing = {k: v for k, v in vps_permissions.items() if k not in permissions}
            if missing:
                permissions.update(missing)
                conn.execute(
                    text("UPDATE roles SET permissions = :perms, updated_at = NOW() WHERE name = 'user'"),
                    {"perms": json.dumps(permissions)}
                )
                logger.info(f"✓ Added missing VPS permissions to user role: {list(missing.keys())}")
            else:
                logger.info("✓ User role already has VPS instance ownership permissions, skipping")
    except Exception as e:
        logger.warning(f"Could not update user role permissions: {e}")
    
    logger.info("✓ VM instance owner migration completed")


# ==================== Migration 11: Snapshot Archives ====================

def migrate_snapshot_archives(conn):
    """
    Create vm_snapshot_archives table to store snapshot configurations
    before VM/container deletion for audit and recovery purposes.
    """
    logger.info("Migration 11: Creating snapshot archives table...")
    
    if table_exists(conn, 'vm_snapshot_archives'):
        logger.info("Table vm_snapshot_archives already exists, skipping")
        return
    
    conn.execute(text("""
        CREATE TABLE vm_snapshot_archives (
            id SERIAL PRIMARY KEY,
            server_id INTEGER NOT NULL,
            server_name VARCHAR(100),
            vmid INTEGER NOT NULL,
            vm_name VARCHAR(100),
            vm_type VARCHAR(20) NOT NULL,
            node VARCHAR(100) NOT NULL,
            snapname VARCHAR(100) NOT NULL,
            description TEXT,
            snaptime BIGINT,
            parent VARCHAR(100),
            vmstate BOOLEAN NOT NULL DEFAULT FALSE,
            snapshot_config JSONB,
            deleted_by VARCHAR(100),
            deletion_reason TEXT,
            archived_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL
        )
    """))
    
    # Create indexes
    conn.execute(text("""
        CREATE INDEX idx_snapshot_archive_server_vmid ON vm_snapshot_archives(server_id, vmid)
    """))
    conn.execute(text("""
        CREATE INDEX idx_snapshot_archive_archived ON vm_snapshot_archives(archived_at)
    """))
    conn.execute(text("""
        CREATE INDEX idx_snapshot_archive_vmid ON vm_snapshot_archives(vmid)
    """))
    
    logger.info("✓ Snapshot archives table created")


# ==================== Migration 12: User SSH Public Key ====================

def migrate_user_ssh_key(conn):
    """Add SSH public key column to users table"""
    
    if column_exists(conn, 'users', 'ssh_public_key'):
        logger.info("✓ User SSH public key column already exists")
        return
    
    logger.info("Adding SSH public key column to users table...")
    
    add_column_if_not_exists(conn, 'users', 'ssh_public_key', 'TEXT')
    
    logger.info("✓ User SSH public key column added")


# ==================== Migration 13: Git Repository URL Setting ====================

def migrate_git_repository_setting(conn):
    """Add git repository URL setting to panel_settings"""
    
    logger.info("Adding git repository URL setting...")
    
    # Check if setting already exists
    result = conn.execute(text(
        "SELECT COUNT(*) FROM panel_settings WHERE key = 'git_repository_url'"
    )).scalar()
    
    if result > 0:
        logger.info("✓ Git repository URL setting already exists")
        return
    
    # Insert default repository setting
    conn.execute(text("""
        INSERT INTO panel_settings (key, value, description, created_at, updated_at)
        VALUES (
            'git_repository_url',
            'https://git.tzim.uz/markmorado/pvemanager',
            'Git repository URL for panel updates',
            NOW(),
            NOW()
        )
    """))
    
    logger.info("✓ Git repository URL setting added")


# ==================== Main Migration Function ====================

def migrate_encrypt_passwords(conn):
    """Migration 14: Encrypt plaintext passwords stored in ProxmoxServer and VMInstance.

    Idempotent: already-encrypted values (valid Fernet tokens) are skipped.
    Skipped entirely when FERNET_KEY env var is not set.
    """
    import os
    try:
        from cryptography.fernet import Fernet, InvalidToken
    except ImportError:
        logger.warning("Migration 14: cryptography package not available, skipping")
        return

    fernet_key = os.environ.get("FERNET_KEY")
    if not fernet_key:
        logger.info("Migration 14: FERNET_KEY not set, skipping password encryption")
        return

    f = Fernet(fernet_key.encode())

    def _is_encrypted(value: str) -> bool:
        try:
            f.decrypt(value.encode())
            return True
        except (InvalidToken, Exception):
            return False

    # --- ProxmoxServer.password ---
    rows = conn.execute(
        text("SELECT id, password FROM proxmox_servers WHERE password IS NOT NULL AND password != ''")
    ).fetchall()
    count = 0
    for row in rows:
        if not _is_encrypted(row.password):
            encrypted = f.encrypt(row.password.encode()).decode()
            conn.execute(
                text("UPDATE proxmox_servers SET password = :p WHERE id = :id"),
                {"p": encrypted, "id": row.id}
            )
            count += 1
    if count:
        logger.info(f"Migration 14: Encrypted {count} Proxmox server password(s)")

    # --- VMInstance.cloud_init_password ---
    rows = conn.execute(
        text("SELECT id, cloud_init_password FROM vm_instances WHERE cloud_init_password IS NOT NULL AND cloud_init_password != ''")
    ).fetchall()
    count = 0
    for row in rows:
        if not _is_encrypted(row.cloud_init_password):
            encrypted = f.encrypt(row.cloud_init_password.encode()).decode()
            conn.execute(
                text("UPDATE vm_instances SET cloud_init_password = :p WHERE id = :id"),
                {"p": encrypted, "id": row.id}
            )
            count += 1
    if count:
        logger.info(f"Migration 14: Encrypted {count} VM cloud-init password(s)")


def migrate_backup_jobs(conn):
    """Migration 15: Create backup_jobs table for scheduled backups"""
    logger.info("Migration 15: Creating backup_jobs table...")

    if table_exists(conn, 'backup_jobs'):
        logger.info("Table backup_jobs already exists, skipping")
        return

    conn.execute(text("""
        CREATE TABLE backup_jobs (
            id SERIAL PRIMARY KEY,
            server_id INTEGER NOT NULL REFERENCES proxmox_servers(id) ON DELETE CASCADE,
            node VARCHAR(100) NOT NULL,
            vmids JSONB NOT NULL DEFAULT '[]',
            storage VARCHAR(100) NOT NULL,
            mode VARCHAR(20) NOT NULL DEFAULT 'snapshot',
            compress VARCHAR(20) NOT NULL DEFAULT 'zstd',
            notes VARCHAR(500),
            keep_last INTEGER NOT NULL DEFAULT 3,
            keep_daily INTEGER NOT NULL DEFAULT 7,
            keep_weekly INTEGER NOT NULL DEFAULT 4,
            keep_monthly INTEGER NOT NULL DEFAULT 6,
            cron_expression VARCHAR(100) NOT NULL,
            enabled BOOLEAN NOT NULL DEFAULT TRUE,
            owner_id INTEGER REFERENCES users(id) ON DELETE SET NULL,
            created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
            updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
            last_run_at TIMESTAMP WITH TIME ZONE,
            last_status VARCHAR(20),
            last_error TEXT,
            last_upid VARCHAR(200)
        )
    """))

    conn.execute(text("CREATE INDEX idx_backup_jobs_server ON backup_jobs(server_id)"))
    conn.execute(text("CREATE INDEX idx_backup_jobs_enabled ON backup_jobs(enabled)"))
    conn.execute(text("CREATE INDEX idx_backup_jobs_owner ON backup_jobs(owner_id)"))

    logger.info("✓ backup_jobs table created")


# ==================== Migration 16: IPAM Network Node ====================

def migrate_ipam_network_node(conn):
    """Add proxmox_node column to ipam_networks table"""
    if not table_exists(conn, 'ipam_networks'):
        logger.info("Table ipam_networks does not exist, skipping")
        return

    if add_column_if_not_exists(conn, 'ipam_networks', 'proxmox_node', 'VARCHAR(100)'):
        logger.info("✓ Added proxmox_node column to ipam_networks")
    else:
        logger.info("✓ ipam_networks.proxmox_node already exists")


def migrate_cluster_support(conn):
    """Migration 17: Add cluster_name column to proxmox_servers table"""
    logger.info("Migration 17: Adding cluster support to proxmox_servers...")

    if add_column_if_not_exists(conn, 'proxmox_servers', 'cluster_name', 'VARCHAR(100)'):
        conn.execute(text("""
            CREATE INDEX IF NOT EXISTS idx_proxmox_cluster_name
            ON proxmox_servers(cluster_name)
        """))
        logger.info("✓ Added cluster_name column and index to proxmox_servers")
    else:
        logger.info("✓ proxmox_servers.cluster_name already exists")

    # Ensure admin role has cluster:manage (new format) and remove old proxmox.cluster.manage key
    try:
        result = conn.execute(text("SELECT permissions FROM roles WHERE name = 'admin'")).fetchone()
        if result:
            import json as _json
            perms = _json.loads(result.permissions) if isinstance(result.permissions, str) else result.permissions
            changed = False
            # Remove old dot-format key that triggers full permission reset in Migration 9
            if 'proxmox.cluster.manage' in perms:
                del perms['proxmox.cluster.manage']
                changed = True
            # Add new-format key if missing
            if not perms.get('cluster:manage'):
                perms['cluster:manage'] = True
                changed = True
            if changed:
                conn.execute(
                    text("UPDATE roles SET permissions = :p WHERE name = 'admin'"),
                    {"p": _json.dumps(perms)}
                )
                logger.info("✓ Migrated admin role: proxmox.cluster.manage → cluster:manage")
            else:
                logger.info("✓ Admin role cluster:manage already up to date")
    except Exception as e:
        logger.warning(f"Could not update admin role permissions: {e}")


# ==================== Migration 18: Proxmox Tasks (UPID) ====================

def migrate_proxmox_tasks(conn):
    """Create proxmox_tasks table for tracking long-running Proxmox UPID tasks."""
    logger.info("Migration 18: Creating proxmox_tasks table...")

    if table_exists(conn, 'proxmox_tasks'):
        logger.info("Table proxmox_tasks already exists, skipping")
        return

    conn.execute(text("""
        CREATE TABLE proxmox_tasks (
            id SERIAL PRIMARY KEY,
            upid VARCHAR(200) NOT NULL,
            server_id INTEGER REFERENCES proxmox_servers(id) ON DELETE SET NULL,
            user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
            node VARCHAR(100),
            vmid INTEGER,
            vm_type VARCHAR(10),
            action VARCHAR(100) NOT NULL,
            description VARCHAR(500),
            status VARCHAR(20) NOT NULL DEFAULT 'running',
            exit_status VARCHAR(100),
            log_text TEXT,
            created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
            completed_at TIMESTAMP WITH TIME ZONE
        )
    """))

    conn.execute(text("CREATE INDEX idx_proxmox_tasks_upid ON proxmox_tasks(upid)"))
    conn.execute(text("CREATE INDEX idx_proxmox_tasks_user_id ON proxmox_tasks(user_id)"))
    conn.execute(text("CREATE INDEX idx_proxmox_tasks_status ON proxmox_tasks(status)"))
    conn.execute(text("CREATE INDEX idx_proxmox_tasks_user_status ON proxmox_tasks(user_id, status)"))
    conn.execute(text("CREATE INDEX idx_proxmox_tasks_created ON proxmox_tasks(created_at DESC)"))

    logger.info("✓ proxmox_tasks table created")


def migrate_workspaces(conn):
    """Migration 19: Create workspaces, workspace_servers, workspace_users tables."""
    logger.info("Migration 19: Creating workspaces tables...")

    if table_exists(conn, 'workspaces'):
        logger.info("Table workspaces already exists, skipping")
        return

    # Create workspaces table
    conn.execute(text("""
        CREATE TABLE workspaces (
            id SERIAL PRIMARY KEY,
            name VARCHAR(100) NOT NULL UNIQUE,
            description TEXT,
            color VARCHAR(20) DEFAULT '#667eea',
            is_default BOOLEAN NOT NULL DEFAULT FALSE,
            created_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
            created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
            updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL
        )
    """))
    conn.execute(text("CREATE INDEX idx_workspaces_name ON workspaces(name)"))

    # Create workspace_servers junction table
    conn.execute(text("""
        CREATE TABLE workspace_servers (
            id SERIAL PRIMARY KEY,
            workspace_id INTEGER NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
            server_id INTEGER NOT NULL REFERENCES proxmox_servers(id) ON DELETE CASCADE,
            UNIQUE(workspace_id, server_id)
        )
    """))
    conn.execute(text("CREATE INDEX idx_ws_servers_workspace ON workspace_servers(workspace_id)"))
    conn.execute(text("CREATE INDEX idx_ws_servers_server ON workspace_servers(server_id)"))

    # Create workspace_users junction table
    conn.execute(text("""
        CREATE TABLE workspace_users (
            id SERIAL PRIMARY KEY,
            workspace_id INTEGER NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
            user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
            UNIQUE(workspace_id, user_id)
        )
    """))
    conn.execute(text("CREATE INDEX idx_ws_users_workspace ON workspace_users(workspace_id)"))
    conn.execute(text("CREATE INDEX idx_ws_users_user ON workspace_users(user_id)"))

    # Insert default workspace containing all existing servers
    conn.execute(text("""
        INSERT INTO workspaces (name, description, color, is_default)
        VALUES ('Default', 'Default workspace containing all servers', '#667eea', TRUE)
    """))

    # Add all existing servers to the default workspace
    conn.execute(text("""
        INSERT INTO workspace_servers (workspace_id, server_id)
        SELECT w.id, s.id
        FROM workspaces w, proxmox_servers s
        WHERE w.is_default = TRUE
    """))

    logger.info("✓ workspaces tables created, Default workspace populated")


def migrate_user_servers(conn):
    """Migration 20: Create user_servers association table for server-to-user assignments."""
    logger.info("Migration 20: Creating user_servers table...")

    if table_exists(conn, 'user_servers'):
        logger.info("Table user_servers already exists, skipping")
        return

    conn.execute(text("""
        CREATE TABLE user_servers (
            user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
            server_id INTEGER NOT NULL REFERENCES proxmox_servers(id) ON DELETE CASCADE,
            PRIMARY KEY (user_id, server_id)
        )
    """))
    conn.execute(text("CREATE INDEX idx_user_servers_user ON user_servers(user_id)"))
    conn.execute(text("CREATE INDEX idx_user_servers_server ON user_servers(server_id)"))

    logger.info("✓ user_servers table created")


def migrate_user_role_remove_dashboard(conn):
    """Migration 21: Ensure dashboard:view is False for user role (new format only)."""
    logger.info("Migration 21: Updating user role permissions (dashboard:view = false)...")

    if not table_exists(conn, 'roles'):
        logger.info("Table roles does not exist, skipping")
        return

    result = conn.execute(text("SELECT id, permissions FROM roles WHERE name = 'user'"))
    row = result.fetchone()
    if not row:
        logger.info("Role 'user' not found, skipping")
        return

    import json as _json
    perms = row[1] if isinstance(row[1], dict) else _json.loads(row[1] or '{}')

    changed = False

    # Remove old dot-format key if present (it triggers Migration 9 to "migrate" the role every start)
    if 'dashboard.view' in perms:
        del perms['dashboard.view']
        changed = True

    # Ensure new-format key is False
    if perms.get('dashboard:view') is not False:
        perms['dashboard:view'] = False
        changed = True

    if changed:
        conn.execute(
            text("UPDATE roles SET permissions = :p WHERE name = 'user'"),
            {"p": _json.dumps(perms)}
        )
        logger.info("✓ user role: dashboard:view set to False (old dot-format key removed)")
    else:
        logger.info("✓ user role: dashboard:view already False, skipping")


def migrate_deploy_tasks(conn):
    """Migration 22: Create deploy_tasks table for async VM deployment tracking"""
    conn.execute(text("""
        CREATE TABLE IF NOT EXISTS deploy_tasks (
            id SERIAL PRIMARY KEY,
            status VARCHAR(20) NOT NULL DEFAULT 'pending',
            step VARCHAR(200),
            progress INTEGER NOT NULL DEFAULT 0,
            name VARCHAR(100) NOT NULL,
            template_id INTEGER NOT NULL,
            server_id INTEGER,
            user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
            vmid INTEGER,
            node VARCHAR(100),
            error_message TEXT,
            created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
            started_at TIMESTAMPTZ,
            completed_at TIMESTAMPTZ
        )
    """))
    conn.execute(text("""
        CREATE INDEX IF NOT EXISTS idx_deploy_task_user_status
        ON deploy_tasks (user_id, status)
    """))
    conn.execute(text("""
        CREATE INDEX IF NOT EXISTS idx_deploy_tasks_created
        ON deploy_tasks (created_at)
    """))
    logger.info("✓ deploy_tasks table created")


def migrate_deploy_tasks_kind(conn):
    """Migration 23: Add 'kind' column to deploy_tasks (deploy/reinstall/clone/change_password)"""
    # Check if column exists
    result = conn.execute(text("""
        SELECT column_name FROM information_schema.columns
        WHERE table_name = 'deploy_tasks' AND column_name = 'kind'
    """))
    if result.fetchone():
        logger.info("✓ deploy_tasks.kind already exists")
        return
    conn.execute(text("""
        ALTER TABLE deploy_tasks
        ADD COLUMN kind VARCHAR(30) NOT NULL DEFAULT 'deploy'
    """))
    conn.execute(text("""
        CREATE INDEX IF NOT EXISTS idx_deploy_tasks_kind
        ON deploy_tasks (kind)
    """))
    # template_id may be NULL for non-deploy tasks (clone/reinstall don't always need it)
    conn.execute(text("""
        ALTER TABLE deploy_tasks ALTER COLUMN template_id DROP NOT NULL
    """))
    logger.info("✓ deploy_tasks.kind column added")


def migrate_user_ssh_keys_table(conn):
    """Migration 24: Create user_ssh_keys table for SSH key library per user."""
    if table_exists(conn, 'user_ssh_keys'):
        logger.info("✓ user_ssh_keys table already exists")
        return

    logger.info("Creating user_ssh_keys table...")
    conn.execute(text("""
        CREATE TABLE user_ssh_keys (
            id SERIAL PRIMARY KEY,
            user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
            name VARCHAR(100) NOT NULL,
            public_key TEXT NOT NULL,
            private_key VARCHAR(8000),
            fingerprint VARCHAR(100),
            comment VARCHAR(255),
            created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
        )
    """))
    conn.execute(text("""
        CREATE INDEX idx_user_ssh_keys_user ON user_ssh_keys(user_id)
    """))
    conn.execute(text("""
        CREATE INDEX idx_user_ssh_keys_fp ON user_ssh_keys(fingerprint)
    """))
    logger.info("✓ user_ssh_keys table created")


def migrate_proxmox_task_progress(conn):
    """Migration 25: Add 'progress' column to proxmox_tasks (0-100, parsed from log)."""
    if not table_exists(conn, 'proxmox_tasks'):
        logger.info("Table proxmox_tasks does not exist, skipping progress migration")
        return

    if add_column_if_not_exists(conn, 'proxmox_tasks', 'progress', 'INTEGER'):
        logger.info("✓ Added progress column to proxmox_tasks")
    else:
        logger.info("✓ proxmox_tasks.progress already exists")


def migrate_user_language(conn):
    """Migration 26: Add 'language' column to users table (per-user UI language)."""
    if add_column_if_not_exists(conn, 'users', 'language', "VARCHAR(10) DEFAULT 'ru'"):
        logger.info("✓ Added language column to users")
    else:
        logger.info("✓ users.language already exists")


def run_all_migrations(engine, db_session=None):
    """
    Run all migrations in order.
    
    Args:
        engine: SQLAlchemy engine
        db_session: Optional database session (for compatibility)
    """
    logger.info("=" * 50)
    logger.info("Running PVEmanager database migrations...")
    logger.info("=" * 50)
    
    try:
        with engine.connect() as conn:
            # Migration 1: Notifications
            try:
                migrate_notifications(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Notifications migration: {e}")
                conn.rollback()
            
            # Migration 2: RBAC & Security
            try:
                migrate_rbac_security(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"RBAC migration: {e}")
                conn.rollback()
            
            # Migration 3: Cross-node Templates
            try:
                migrate_cross_node_templates(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Cross-node templates migration: {e}")
                conn.rollback()
            
            # Migration 4: VM Cache
            try:
                migrate_vm_cache(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"VM cache migration: {e}")
                conn.rollback()
            
            # Migration 5: Task Queue
            try:
                migrate_task_queue(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Task queue migration: {e}")
                conn.rollback()
            
            # Migration 6: OS Template Icons (theme-adaptive)
            try:
                migrate_os_template_icons(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"OS template icons migration: {e}")
                conn.rollback()
            
            # Migration 7: VM Template Name
            try:
                migrate_vm_template_name(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"VM template name migration: {e}")
                conn.rollback()
            
            # Migration 8: Enhanced Audit Logs
            try:
                migrate_enhanced_audit_logs(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Enhanced audit logs migration: {e}")
                conn.rollback()
            
            # Migration 9: RBAC v2 - New Permission Format
            try:
                migrate_rbac_v2(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"RBAC v2 migration: {e}")
                conn.rollback()
            
            # Migration 10: VM Instance Owner (VPS-style user isolation)
            try:
                migrate_vm_instance_owner(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"VM instance owner migration: {e}")
                conn.rollback()
            
            # Migration 11: Snapshot Archives
            try:
                migrate_snapshot_archives(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Snapshot archives migration: {e}")
                conn.rollback()
            
            # Migration 12: User SSH Public Key
            try:
                migrate_user_ssh_key(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"User SSH key migration: {e}")
                conn.rollback()
            
            # Migration 13: Git Repository URL Setting
            try:
                migrate_git_repository_setting(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Git repository setting migration: {e}")
                conn.rollback()

            # Migration 14: Encrypt plaintext passwords (idempotent, skipped if FERNET_KEY not set)
            try:
                migrate_encrypt_passwords(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Password encryption migration: {e}")
                conn.rollback()

            # Migration 15: Backup Jobs
            try:
                migrate_backup_jobs(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Backup jobs migration: {e}")
                conn.rollback()

            # Migration 16: IPAM Network Node
            try:
                migrate_ipam_network_node(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"IPAM network node migration: {e}")
                conn.rollback()

            # Migration 17: Cluster Support (cluster_name column)
            try:
                migrate_cluster_support(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Cluster support migration: {e}")
                conn.rollback()

            # Migration 18: Proxmox Tasks (UPID tracking)
            try:
                migrate_proxmox_tasks(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Proxmox tasks migration: {e}")
                conn.rollback()

            # Migration 19: Workspaces
            try:
                migrate_workspaces(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Workspaces migration: {e}")
                conn.rollback()

            # Migration 20: User-Server assignments
            try:
                migrate_user_servers(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"User-Server assignments migration: {e}")
                conn.rollback()

            # Migration 21: Update user role - disable dashboard.view
            try:
                migrate_user_role_remove_dashboard(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"User role dashboard migration: {e}")
                conn.rollback()

            # Migration 22: Deploy Tasks table
            try:
                migrate_deploy_tasks(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Deploy tasks migration: {e}")
                conn.rollback()

            # Migration 23: deploy_tasks.kind column
            try:
                migrate_deploy_tasks_kind(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Deploy tasks kind migration: {e}")
                conn.rollback()

            # Migration 24: user_ssh_keys table
            try:
                migrate_user_ssh_keys_table(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"User SSH keys table migration: {e}")
                conn.rollback()

            # Migration 25: proxmox_tasks.progress column
            try:
                migrate_proxmox_task_progress(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"Proxmox task progress migration: {e}")
                conn.rollback()

            # Migration 26: users.language column
            try:
                migrate_user_language(conn)
                conn.commit()
            except Exception as e:
                logger.warning(f"User language migration: {e}")
                conn.rollback()

        logger.info("=" * 50)
        logger.info("✓ All migrations completed successfully")
        logger.info("=" * 50)
        return True
        
    except Exception as e:
        logger.error(f"Migration failed: {e}")
        return False


# ==================== CLI Entry Point ====================

if __name__ == "__main__":
    print("Running PVEmanager migrations...")
    
    try:
        from app.db import engine
    except ImportError:
        from backend.app.db import engine
    
    success = run_all_migrations(engine)
    
    if success:
        print("✓ All migrations completed successfully")
    else:
        print("✗ Migration failed")
        exit(1)