fix(appstore): DNS, per-server golden template, bind-mount perms, delete race
App Store install/update pipeline reliability fixes: - explicit nameserver (APPSTORE_NAMESERVER) on App Store LXCs and the golden template build CT, instead of inheriting a node resolver that may be unreachable from the container network (e.g. Tailscale-only DNS) - golden template storage/resolution moved from one global setting to per-server_id (migration 37: app_operations.server_id) — building it on one Proxmox server no longer overwrites the template configured for another - golden template automated build fixed: vzdump export step no longer fails on "unable to parse directory volume name 'vztmpl/probe'" - bind-mount host directories are pre-created with permissive ownership before `docker compose up`, fixing Permission denied for non-root containers (e.g. MariaDB) writing to Docker-auto-created root-owned dirs - delete/teardown now waits for Proxmox to actually confirm the destroy task finished (was silently proceeding after a short timeout, letting a new install reuse the VMID while the node was still wiping the old disk) - pct create retries a few times on transient "got lock" config errors - failed pipeline steps now report the tail of command output (4000 chars) instead of the head, so the real error isn't buried under pull/progress noise - install error box in the UI wraps long text and scrolls instead of stretching the dialog Also carries forward in-progress script catalog / instances / RBAC changes that were already uncommitted in the working tree before this session. Bump to v1.9.1, update WIKI.md/docs/CHANGELOG accordingly. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,7 +3,7 @@ from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func
|
||||
from loguru import logger
|
||||
from typing import List
|
||||
from typing import List, Optional
|
||||
import ssl
|
||||
import asyncio
|
||||
import httpx
|
||||
@@ -93,6 +93,83 @@ def _get_proxmox_client(server: ProxmoxServer) -> ProxmoxClient:
|
||||
return ProxmoxClient.from_server(server)
|
||||
|
||||
|
||||
def _extract_fingerprint(join_data: dict, target_ip: str = None) -> Optional[str]:
|
||||
"""
|
||||
Достать SHA-256 fingerprint из ответа GET /cluster/config/join.
|
||||
|
||||
В Proxmox API нет поля `fingerprint` на верхнем уровне — оно лежит в
|
||||
nodelist[].pve_fp у каждой ноды. Выбираем запись целевой кластерной ноды
|
||||
по pve_addr/ring0_addr == target_ip, с фолбэком на preferred_node и на
|
||||
первую ноду с непустым pve_fp.
|
||||
"""
|
||||
if not isinstance(join_data, dict):
|
||||
return None
|
||||
|
||||
# Совместимость: если провайдер всё же вернул fingerprint сверху — берём его.
|
||||
top = join_data.get("fingerprint")
|
||||
if top:
|
||||
return top
|
||||
|
||||
nodelist = join_data.get("nodelist") or []
|
||||
|
||||
def _fp(entry: dict) -> Optional[str]:
|
||||
return entry.get("pve_fp") if isinstance(entry, dict) else None
|
||||
|
||||
# 1) По IP целевой ноды
|
||||
if target_ip:
|
||||
for entry in nodelist:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if target_ip in (entry.get("pve_addr"), entry.get("ring0_addr")) and _fp(entry):
|
||||
return entry["pve_fp"]
|
||||
|
||||
# 2) По preferred_node
|
||||
preferred = join_data.get("preferred_node")
|
||||
if preferred:
|
||||
for entry in nodelist:
|
||||
if isinstance(entry, dict) and entry.get("name") == preferred and _fp(entry):
|
||||
return entry["pve_fp"]
|
||||
|
||||
# 3) Первая нода с непустым pve_fp
|
||||
for entry in nodelist:
|
||||
if _fp(entry):
|
||||
return entry["pve_fp"]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _build_remote_endpoint(target_server: ProxmoxServer, target_node: str) -> str:
|
||||
"""
|
||||
Собрать `target-endpoint` строку для PVE `remote_migrate` API.
|
||||
|
||||
Формат: apitoken=PVEAPIToken={user}!{token}={secret},host={ip}:{port},fingerprint={fp}
|
||||
Требует API-токен у целевого сервера (remote_migrate не принимает пароль).
|
||||
"""
|
||||
has_token = bool(getattr(target_server, "api_token_name", "") and getattr(target_server, "api_token_value", ""))
|
||||
if not has_token or getattr(target_server, "use_password", False):
|
||||
raise ValueError(
|
||||
f"Server '{target_server.name}' must be configured with an API token "
|
||||
f"(not password) for remote migration. Edit the target server settings."
|
||||
)
|
||||
|
||||
client = _get_proxmox_client(target_server)
|
||||
try:
|
||||
join_data = client.proxmox.cluster.config.join.get()
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to fetch target cluster fingerprint from '{target_server.name}': {e}") from e
|
||||
|
||||
fingerprint = _extract_fingerprint(join_data, target_ip=target_server.ip_address)
|
||||
if not fingerprint:
|
||||
raise ValueError(f"Could not determine SSL fingerprint of target server '{target_server.name}'")
|
||||
|
||||
token = f"{target_server.api_user}!{target_server.api_token_name}={target_server.api_token_value}"
|
||||
return (
|
||||
f"apitoken=PVEAPIToken={token},"
|
||||
f"host={target_server.ip_address}:{target_server.port},"
|
||||
f"fingerprint={fingerprint}"
|
||||
)
|
||||
|
||||
|
||||
def get_next_vmid(db: Session, server_id: int) -> int:
|
||||
"""Get next available VMID from Proxmox server (sequential)"""
|
||||
|
||||
|
||||
@@ -17,7 +17,8 @@ from ...db import SessionLocal
|
||||
from ...models import ProxmoxServer, VMInstance, OSTemplate, OSTemplateGroup, DeployTask, User
|
||||
from ...proxmox import ProxmoxClient
|
||||
from ...logging_service import LoggingService
|
||||
from ._helpers import _get_proxmox_client, get_next_vmid, save_vm_instance, save_vm_instance
|
||||
from ._helpers import (_get_proxmox_client, get_next_vmid, save_vm_instance,
|
||||
soft_delete_vm_instance, _build_remote_endpoint)
|
||||
|
||||
|
||||
# Re-use deploy executor and update helper
|
||||
@@ -603,6 +604,116 @@ def _do_migrate_sync(task_id: int, server_id: int, vmid: int, node: str, vm_type
|
||||
db.close()
|
||||
|
||||
|
||||
def _do_remote_migrate_sync(task_id: int, server_id: int, vmid: int, node: str, vm_type: str,
|
||||
target_server_id: int, target_node: str, target_vmid: int,
|
||||
target_storage: Optional[str], target_bridge: Optional[str],
|
||||
online: bool, delete_source: bool,
|
||||
user_id: int, username: str):
|
||||
"""Мигрировать VM/LXC на другой (независимый) Proxmox-кластер через remote_migrate
|
||||
API (фоновая задача с прогрессом). В отличие от _do_migrate_sync, источник и цель —
|
||||
разные ProxmoxServer с разными кредами; UPID задачи живёт на источнике."""
|
||||
db = SessionLocal()
|
||||
server = None
|
||||
is_lxc = (vm_type == 'lxc')
|
||||
try:
|
||||
_update_deploy_task(task_id, 'running', 'Подключение к Proxmox...', 10,
|
||||
vmid=vmid, node=node)
|
||||
server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
|
||||
target_server = db.query(ProxmoxServer).filter(ProxmoxServer.id == target_server_id).first()
|
||||
if not server or not target_server:
|
||||
_update_deploy_task(task_id, 'failed', 'Сервер не найден', 0, error='Proxmox сервер не найден')
|
||||
return
|
||||
client = _connect(server)
|
||||
if not client:
|
||||
_update_deploy_task(task_id, 'failed', 'Ошибка подключения к Proxmox', 10,
|
||||
error='Не удалось подключиться к исходному Proxmox серверу')
|
||||
return
|
||||
|
||||
_update_deploy_task(task_id, 'running', 'Подготовка remote-endpoint...', 20,
|
||||
vmid=vmid, node=node)
|
||||
try:
|
||||
target_endpoint = _build_remote_endpoint(target_server, target_node)
|
||||
except ValueError as e:
|
||||
_update_deploy_task(task_id, 'failed', 'Ошибка конфигурации цели', 20, error=str(e))
|
||||
return
|
||||
|
||||
_update_deploy_task(task_id, 'running', f'Миграция → {target_server.name}/{target_node}...', 30,
|
||||
vmid=vmid, node=node)
|
||||
try:
|
||||
if is_lxc:
|
||||
upid = client.remote_migrate_container(
|
||||
node, vmid, target_endpoint, target_vmid,
|
||||
target_storage=target_storage or None, target_bridge=target_bridge or None,
|
||||
online=online, delete_source=delete_source,
|
||||
)
|
||||
else:
|
||||
upid = client.remote_migrate_vm(
|
||||
node, vmid, target_endpoint, target_vmid,
|
||||
target_storage=target_storage or None, target_bridge=target_bridge or None,
|
||||
online=online, delete_source=delete_source,
|
||||
)
|
||||
except Exception as e:
|
||||
_update_deploy_task(task_id, 'failed', 'Ошибка миграции', 30, error=f'Remote migrate failed: {e}')
|
||||
return
|
||||
if not upid:
|
||||
_update_deploy_task(task_id, 'failed', 'Ошибка миграции', 30,
|
||||
error='Proxmox не вернул UPID задачи миграции')
|
||||
return
|
||||
|
||||
# Прогресс переноса маппится в 30..95 из лога задачи на источнике
|
||||
_update_deploy_task(task_id, 'running', 'Перенос данных...', 50, vmid=vmid, node=node)
|
||||
ok = _wait_migrate_with_progress(client, task_id, node, upid, timeout=3600)
|
||||
if not ok:
|
||||
_update_deploy_task(task_id, 'failed', 'Миграция не удалась', 50,
|
||||
error=f'Задача миграции завершилась с ошибкой. {_task_log_tail(client, node, upid)}'[:300])
|
||||
return
|
||||
|
||||
# Переносим локальную запись инстанса (владелец, метаданные) на целевой сервер
|
||||
try:
|
||||
cached = db.query(VMInstance).filter(
|
||||
VMInstance.server_id == server_id,
|
||||
VMInstance.vmid == vmid,
|
||||
VMInstance.deleted_at.is_(None),
|
||||
).first()
|
||||
if cached:
|
||||
save_vm_instance(
|
||||
db, target_server_id, target_vmid, target_node, vm_type, cached.name,
|
||||
cores=cached.cores, memory=cached.memory, disk_size=cached.disk_size,
|
||||
ip_address=cached.ip_address, ip_prefix=cached.ip_prefix, gateway=cached.gateway,
|
||||
nameserver=cached.nameserver, template_id=cached.template_id,
|
||||
template_name=cached.template_name, description=cached.description,
|
||||
extra_config=cached.extra_config, owner_id=cached.owner_id,
|
||||
)
|
||||
if delete_source:
|
||||
soft_delete_vm_instance(db, server_id, vmid)
|
||||
except Exception as reg_err:
|
||||
logger.warning(f"[REMOTE MIGRATE #{task_id}] failed to move instance cache record: {reg_err}")
|
||||
|
||||
LoggingService.log_proxmox_action(
|
||||
db=db, action='remote_migrate',
|
||||
resource_type='lxc' if is_lxc else 'vm', resource_id=vmid,
|
||||
username=username, server_id=server_id,
|
||||
server_name=server.name, node_name=node,
|
||||
details={
|
||||
'target_server_id': target_server_id, 'target_server_name': target_server.name,
|
||||
'target_node': target_node, 'target_vmid': target_vmid,
|
||||
'target_storage': target_storage, 'target_bridge': target_bridge,
|
||||
'online': online, 'delete_source': delete_source,
|
||||
},
|
||||
success=True,
|
||||
)
|
||||
|
||||
_update_deploy_task(task_id, 'completed', f'Мигрировано на {target_server.name}/{target_node}', 100,
|
||||
vmid=target_vmid, node=target_node)
|
||||
logger.info(f"[REMOTE MIGRATE #{task_id}] {vmid}@{server.name}/{node} -> {target_vmid}@{target_server.name}/{target_node} by {username}")
|
||||
except Exception as e:
|
||||
err = str(e)
|
||||
logger.error(f"[REMOTE MIGRATE #{task_id}] failed: {err}")
|
||||
_update_deploy_task(task_id, 'failed', f'Ошибка: {err[:150]}', 0, error=err)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _wait_migrate_with_progress(client, task_id: int, node: str, upid: str,
|
||||
timeout: int = 3600) -> bool:
|
||||
"""Дождаться задачи миграции, маппя % копирования диска из лога в 50..95."""
|
||||
|
||||
@@ -16,7 +16,7 @@ from ...db import get_db
|
||||
from ...models import ProxmoxServer, User
|
||||
from ...auth import PermissionChecker
|
||||
from ...proxmox import ProxmoxClient
|
||||
from ._helpers import _get_proxmox_client
|
||||
from ._helpers import _get_proxmox_client, _extract_fingerprint
|
||||
from ...proxmox import _run_in_executor
|
||||
|
||||
router = APIRouter()
|
||||
@@ -31,51 +31,6 @@ def _get_server_or_404(db: Session, server_id: int) -> ProxmoxServer:
|
||||
return server
|
||||
|
||||
|
||||
def _extract_fingerprint(join_data: dict, target_ip: str = None) -> Optional[str]:
|
||||
"""
|
||||
Достать SHA-256 fingerprint из ответа GET /cluster/config/join.
|
||||
|
||||
В Proxmox API нет поля `fingerprint` на верхнем уровне — оно лежит в
|
||||
nodelist[].pve_fp у каждой ноды. Выбираем запись целевой кластерной ноды
|
||||
по pve_addr/ring0_addr == target_ip, с фолбэком на preferred_node и на
|
||||
первую ноду с непустым pve_fp.
|
||||
"""
|
||||
if not isinstance(join_data, dict):
|
||||
return None
|
||||
|
||||
# Совместимость: если провайдер всё же вернул fingerprint сверху — берём его.
|
||||
top = join_data.get("fingerprint")
|
||||
if top:
|
||||
return top
|
||||
|
||||
nodelist = join_data.get("nodelist") or []
|
||||
|
||||
def _fp(entry: dict) -> Optional[str]:
|
||||
return entry.get("pve_fp") if isinstance(entry, dict) else None
|
||||
|
||||
# 1) По IP целевой ноды
|
||||
if target_ip:
|
||||
for entry in nodelist:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if target_ip in (entry.get("pve_addr"), entry.get("ring0_addr")) and _fp(entry):
|
||||
return entry["pve_fp"]
|
||||
|
||||
# 2) По preferred_node
|
||||
preferred = join_data.get("preferred_node")
|
||||
if preferred:
|
||||
for entry in nodelist:
|
||||
if isinstance(entry, dict) and entry.get("name") == preferred and _fp(entry):
|
||||
return entry["pve_fp"]
|
||||
|
||||
# 3) Первая нода с непустым pve_fp
|
||||
for entry in nodelist:
|
||||
if _fp(entry):
|
||||
return entry["pve_fp"]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _get_password_client(server: ProxmoxServer, override_password: str = None) -> ProxmoxClient:
|
||||
"""
|
||||
Build a ProxmoxClient that uses password auth (required for cluster ops).
|
||||
|
||||
@@ -421,6 +421,16 @@ class MigrateRequest(BaseModel):
|
||||
online: bool = False
|
||||
|
||||
|
||||
class RemoteMigrateRequest(BaseModel):
|
||||
target_server_id: int
|
||||
target_node: str
|
||||
target_vmid: Optional[int] = None
|
||||
target_storage: Optional[str] = None
|
||||
target_bridge: Optional[str] = None
|
||||
online: bool = False
|
||||
delete_source: bool = True
|
||||
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
username: str = "root"
|
||||
password: str
|
||||
@@ -576,6 +586,104 @@ def migrate_container_endpoint(
|
||||
return DeployTaskStartResponse(task_id=task.id, status='pending', name=task.name)
|
||||
|
||||
|
||||
# -------- Remote migrate (move VM/LXC to a different, independent cluster) --------
|
||||
|
||||
def _require_pve8_plus(client, server_name: str):
|
||||
"""Remote_migrate API существует только в PVE >= 8.0 — проверяем до сабмита задачи."""
|
||||
try:
|
||||
version = (client.proxmox.version.get() or {}).get('version', '')
|
||||
major = int(str(version).split('.')[0])
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Не удалось определить версию PVE сервера '{server_name}': {e}")
|
||||
if major < 8:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Remote-миграция требует Proxmox VE >= 8.0 (сервер '{server_name}': версия {version})",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/{server_id}/vm/{vmid}/remote-migrate", status_code=202)
|
||||
def remote_migrate_vm_endpoint(
|
||||
server_id: int,
|
||||
vmid: int,
|
||||
node: str,
|
||||
body: RemoteMigrateRequest,
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(PermissionChecker("vm:remote_migrate"))
|
||||
):
|
||||
"""Мигрировать VM (qemu) на другой (независимый) Proxmox-кластер — фоновая задача."""
|
||||
from ...models import DeployTask
|
||||
from ..templates import _deploy_executor, DeployTaskStartResponse
|
||||
from .async_ops import _do_remote_migrate_sync
|
||||
|
||||
require_vm_access(db, current_user, server_id, vmid)
|
||||
source_server = _resolve_server(db, server_id)
|
||||
if body.target_server_id == server_id:
|
||||
raise HTTPException(status_code=400, detail="Для миграции внутри одного кластера используйте обычную миграцию")
|
||||
target_server = _resolve_server(db, body.target_server_id)
|
||||
|
||||
_require_pve8_plus(_get_client_or_503(source_server), source_server.name)
|
||||
_require_pve8_plus(_get_client_or_503(target_server), target_server.name)
|
||||
|
||||
task = DeployTask(
|
||||
kind='remote_migrate',
|
||||
name=f'remote migrate {vmid} → {target_server.name}/{body.target_node}',
|
||||
status='pending', step='В очереди...', progress=0,
|
||||
template_id=None, server_id=server_id, target_server_id=body.target_server_id,
|
||||
user_id=current_user.id, node=node, vmid=vmid,
|
||||
)
|
||||
db.add(task); db.commit(); db.refresh(task)
|
||||
|
||||
_deploy_executor.submit(
|
||||
_do_remote_migrate_sync, task.id, server_id, vmid, node, 'qemu',
|
||||
body.target_server_id, body.target_node, body.target_vmid or vmid,
|
||||
body.target_storage, body.target_bridge, body.online, body.delete_source,
|
||||
current_user.id, current_user.username,
|
||||
)
|
||||
return DeployTaskStartResponse(task_id=task.id, status='pending', name=task.name)
|
||||
|
||||
|
||||
@router.post("/api/{server_id}/container/{vmid}/remote-migrate", status_code=202)
|
||||
def remote_migrate_container_endpoint(
|
||||
server_id: int,
|
||||
vmid: int,
|
||||
node: str,
|
||||
body: RemoteMigrateRequest,
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(PermissionChecker("lxc:remote_migrate"))
|
||||
):
|
||||
"""Мигрировать LXC контейнер на другой (независимый) Proxmox-кластер — фоновая задача."""
|
||||
from ...models import DeployTask
|
||||
from ..templates import _deploy_executor, DeployTaskStartResponse
|
||||
from .async_ops import _do_remote_migrate_sync
|
||||
|
||||
require_vm_access(db, current_user, server_id, vmid)
|
||||
source_server = _resolve_server(db, server_id)
|
||||
if body.target_server_id == server_id:
|
||||
raise HTTPException(status_code=400, detail="Для миграции внутри одного кластера используйте обычную миграцию")
|
||||
target_server = _resolve_server(db, body.target_server_id)
|
||||
|
||||
_require_pve8_plus(_get_client_or_503(source_server), source_server.name)
|
||||
_require_pve8_plus(_get_client_or_503(target_server), target_server.name)
|
||||
|
||||
task = DeployTask(
|
||||
kind='remote_migrate',
|
||||
name=f'remote migrate {vmid} → {target_server.name}/{body.target_node}',
|
||||
status='pending', step='В очереди...', progress=0,
|
||||
template_id=None, server_id=server_id, target_server_id=body.target_server_id,
|
||||
user_id=current_user.id, node=node, vmid=vmid,
|
||||
)
|
||||
db.add(task); db.commit(); db.refresh(task)
|
||||
|
||||
_deploy_executor.submit(
|
||||
_do_remote_migrate_sync, task.id, server_id, vmid, node, 'lxc',
|
||||
body.target_server_id, body.target_node, body.target_vmid or vmid,
|
||||
body.target_storage, body.target_bridge, body.online, body.delete_source,
|
||||
current_user.id, current_user.username,
|
||||
)
|
||||
return DeployTaskStartResponse(task_id=task.id, status='pending', name=task.name)
|
||||
|
||||
|
||||
# -------- Reinstall (re-clone from saved template) --------
|
||||
|
||||
@router.post("/api/{server_id}/vm/{vmid}/reinstall", status_code=202)
|
||||
|
||||
Reference in New Issue
Block a user