fix(appstore): DNS, per-server golden template, bind-mount perms, delete race
All checks were successful
Backend Tests / pytest (Python 3.12) (push) Successful in 1m1s
Deploy to Production / Trigger production update (push) Successful in 8s

App Store install/update pipeline reliability fixes:
- explicit nameserver (APPSTORE_NAMESERVER) on App Store LXCs and the golden
  template build CT, instead of inheriting a node resolver that may be
  unreachable from the container network (e.g. Tailscale-only DNS)
- golden template storage/resolution moved from one global setting to
  per-server_id (migration 37: app_operations.server_id) — building it on one
  Proxmox server no longer overwrites the template configured for another
- golden template automated build fixed: vzdump export step no longer fails
  on "unable to parse directory volume name 'vztmpl/probe'"
- bind-mount host directories are pre-created with permissive ownership
  before `docker compose up`, fixing Permission denied for non-root
  containers (e.g. MariaDB) writing to Docker-auto-created root-owned dirs
- delete/teardown now waits for Proxmox to actually confirm the destroy task
  finished (was silently proceeding after a short timeout, letting a new
  install reuse the VMID while the node was still wiping the old disk)
- pct create retries a few times on transient "got lock" config errors
- failed pipeline steps now report the tail of command output (4000 chars)
  instead of the head, so the real error isn't buried under pull/progress noise
- install error box in the UI wraps long text and scrolls instead of
  stretching the dialog

Also carries forward in-progress script catalog / instances / RBAC changes
that were already uncommitted in the working tree before this session.

Bump to v1.9.1, update WIKI.md/docs/CHANGELOG accordingly.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
markmorado
2026-07-17 04:49:40 +05:00
parent b8536ac2e8
commit 6a79dd2048
40 changed files with 1100 additions and 109 deletions

View File

@@ -3,7 +3,7 @@ from fastapi.responses import JSONResponse
from sqlalchemy.orm import Session
from sqlalchemy import func
from loguru import logger
from typing import List
from typing import List, Optional
import ssl
import asyncio
import httpx
@@ -93,6 +93,83 @@ def _get_proxmox_client(server: ProxmoxServer) -> ProxmoxClient:
return ProxmoxClient.from_server(server)
def _extract_fingerprint(join_data: dict, target_ip: str = None) -> Optional[str]:
"""
Достать SHA-256 fingerprint из ответа GET /cluster/config/join.
В Proxmox API нет поля `fingerprint` на верхнем уровне — оно лежит в
nodelist[].pve_fp у каждой ноды. Выбираем запись целевой кластерной ноды
по pve_addr/ring0_addr == target_ip, с фолбэком на preferred_node и на
первую ноду с непустым pve_fp.
"""
if not isinstance(join_data, dict):
return None
# Совместимость: если провайдер всё же вернул fingerprint сверху — берём его.
top = join_data.get("fingerprint")
if top:
return top
nodelist = join_data.get("nodelist") or []
def _fp(entry: dict) -> Optional[str]:
return entry.get("pve_fp") if isinstance(entry, dict) else None
# 1) По IP целевой ноды
if target_ip:
for entry in nodelist:
if not isinstance(entry, dict):
continue
if target_ip in (entry.get("pve_addr"), entry.get("ring0_addr")) and _fp(entry):
return entry["pve_fp"]
# 2) По preferred_node
preferred = join_data.get("preferred_node")
if preferred:
for entry in nodelist:
if isinstance(entry, dict) and entry.get("name") == preferred and _fp(entry):
return entry["pve_fp"]
# 3) Первая нода с непустым pve_fp
for entry in nodelist:
if _fp(entry):
return entry["pve_fp"]
return None
def _build_remote_endpoint(target_server: ProxmoxServer, target_node: str) -> str:
"""
Собрать `target-endpoint` строку для PVE `remote_migrate` API.
Формат: apitoken=PVEAPIToken={user}!{token}={secret},host={ip}:{port},fingerprint={fp}
Требует API-токен у целевого сервера (remote_migrate не принимает пароль).
"""
has_token = bool(getattr(target_server, "api_token_name", "") and getattr(target_server, "api_token_value", ""))
if not has_token or getattr(target_server, "use_password", False):
raise ValueError(
f"Server '{target_server.name}' must be configured with an API token "
f"(not password) for remote migration. Edit the target server settings."
)
client = _get_proxmox_client(target_server)
try:
join_data = client.proxmox.cluster.config.join.get()
except Exception as e:
raise ValueError(f"Failed to fetch target cluster fingerprint from '{target_server.name}': {e}") from e
fingerprint = _extract_fingerprint(join_data, target_ip=target_server.ip_address)
if not fingerprint:
raise ValueError(f"Could not determine SSL fingerprint of target server '{target_server.name}'")
token = f"{target_server.api_user}!{target_server.api_token_name}={target_server.api_token_value}"
return (
f"apitoken=PVEAPIToken={token},"
f"host={target_server.ip_address}:{target_server.port},"
f"fingerprint={fingerprint}"
)
def get_next_vmid(db: Session, server_id: int) -> int:
"""Get next available VMID from Proxmox server (sequential)"""

View File

@@ -17,7 +17,8 @@ from ...db import SessionLocal
from ...models import ProxmoxServer, VMInstance, OSTemplate, OSTemplateGroup, DeployTask, User
from ...proxmox import ProxmoxClient
from ...logging_service import LoggingService
from ._helpers import _get_proxmox_client, get_next_vmid, save_vm_instance, save_vm_instance
from ._helpers import (_get_proxmox_client, get_next_vmid, save_vm_instance,
soft_delete_vm_instance, _build_remote_endpoint)
# Re-use deploy executor and update helper
@@ -603,6 +604,116 @@ def _do_migrate_sync(task_id: int, server_id: int, vmid: int, node: str, vm_type
db.close()
def _do_remote_migrate_sync(task_id: int, server_id: int, vmid: int, node: str, vm_type: str,
target_server_id: int, target_node: str, target_vmid: int,
target_storage: Optional[str], target_bridge: Optional[str],
online: bool, delete_source: bool,
user_id: int, username: str):
"""Мигрировать VM/LXC на другой (независимый) Proxmox-кластер через remote_migrate
API (фоновая задача с прогрессом). В отличие от _do_migrate_sync, источник и цель —
разные ProxmoxServer с разными кредами; UPID задачи живёт на источнике."""
db = SessionLocal()
server = None
is_lxc = (vm_type == 'lxc')
try:
_update_deploy_task(task_id, 'running', 'Подключение к Proxmox...', 10,
vmid=vmid, node=node)
server = db.query(ProxmoxServer).filter(ProxmoxServer.id == server_id).first()
target_server = db.query(ProxmoxServer).filter(ProxmoxServer.id == target_server_id).first()
if not server or not target_server:
_update_deploy_task(task_id, 'failed', 'Сервер не найден', 0, error='Proxmox сервер не найден')
return
client = _connect(server)
if not client:
_update_deploy_task(task_id, 'failed', 'Ошибка подключения к Proxmox', 10,
error='Не удалось подключиться к исходному Proxmox серверу')
return
_update_deploy_task(task_id, 'running', 'Подготовка remote-endpoint...', 20,
vmid=vmid, node=node)
try:
target_endpoint = _build_remote_endpoint(target_server, target_node)
except ValueError as e:
_update_deploy_task(task_id, 'failed', 'Ошибка конфигурации цели', 20, error=str(e))
return
_update_deploy_task(task_id, 'running', f'Миграция → {target_server.name}/{target_node}...', 30,
vmid=vmid, node=node)
try:
if is_lxc:
upid = client.remote_migrate_container(
node, vmid, target_endpoint, target_vmid,
target_storage=target_storage or None, target_bridge=target_bridge or None,
online=online, delete_source=delete_source,
)
else:
upid = client.remote_migrate_vm(
node, vmid, target_endpoint, target_vmid,
target_storage=target_storage or None, target_bridge=target_bridge or None,
online=online, delete_source=delete_source,
)
except Exception as e:
_update_deploy_task(task_id, 'failed', 'Ошибка миграции', 30, error=f'Remote migrate failed: {e}')
return
if not upid:
_update_deploy_task(task_id, 'failed', 'Ошибка миграции', 30,
error='Proxmox не вернул UPID задачи миграции')
return
# Прогресс переноса маппится в 30..95 из лога задачи на источнике
_update_deploy_task(task_id, 'running', 'Перенос данных...', 50, vmid=vmid, node=node)
ok = _wait_migrate_with_progress(client, task_id, node, upid, timeout=3600)
if not ok:
_update_deploy_task(task_id, 'failed', 'Миграция не удалась', 50,
error=f'Задача миграции завершилась с ошибкой. {_task_log_tail(client, node, upid)}'[:300])
return
# Переносим локальную запись инстанса (владелец, метаданные) на целевой сервер
try:
cached = db.query(VMInstance).filter(
VMInstance.server_id == server_id,
VMInstance.vmid == vmid,
VMInstance.deleted_at.is_(None),
).first()
if cached:
save_vm_instance(
db, target_server_id, target_vmid, target_node, vm_type, cached.name,
cores=cached.cores, memory=cached.memory, disk_size=cached.disk_size,
ip_address=cached.ip_address, ip_prefix=cached.ip_prefix, gateway=cached.gateway,
nameserver=cached.nameserver, template_id=cached.template_id,
template_name=cached.template_name, description=cached.description,
extra_config=cached.extra_config, owner_id=cached.owner_id,
)
if delete_source:
soft_delete_vm_instance(db, server_id, vmid)
except Exception as reg_err:
logger.warning(f"[REMOTE MIGRATE #{task_id}] failed to move instance cache record: {reg_err}")
LoggingService.log_proxmox_action(
db=db, action='remote_migrate',
resource_type='lxc' if is_lxc else 'vm', resource_id=vmid,
username=username, server_id=server_id,
server_name=server.name, node_name=node,
details={
'target_server_id': target_server_id, 'target_server_name': target_server.name,
'target_node': target_node, 'target_vmid': target_vmid,
'target_storage': target_storage, 'target_bridge': target_bridge,
'online': online, 'delete_source': delete_source,
},
success=True,
)
_update_deploy_task(task_id, 'completed', f'Мигрировано на {target_server.name}/{target_node}', 100,
vmid=target_vmid, node=target_node)
logger.info(f"[REMOTE MIGRATE #{task_id}] {vmid}@{server.name}/{node} -> {target_vmid}@{target_server.name}/{target_node} by {username}")
except Exception as e:
err = str(e)
logger.error(f"[REMOTE MIGRATE #{task_id}] failed: {err}")
_update_deploy_task(task_id, 'failed', f'Ошибка: {err[:150]}', 0, error=err)
finally:
db.close()
def _wait_migrate_with_progress(client, task_id: int, node: str, upid: str,
timeout: int = 3600) -> bool:
"""Дождаться задачи миграции, маппя % копирования диска из лога в 50..95."""

View File

@@ -16,7 +16,7 @@ from ...db import get_db
from ...models import ProxmoxServer, User
from ...auth import PermissionChecker
from ...proxmox import ProxmoxClient
from ._helpers import _get_proxmox_client
from ._helpers import _get_proxmox_client, _extract_fingerprint
from ...proxmox import _run_in_executor
router = APIRouter()
@@ -31,51 +31,6 @@ def _get_server_or_404(db: Session, server_id: int) -> ProxmoxServer:
return server
def _extract_fingerprint(join_data: dict, target_ip: str = None) -> Optional[str]:
"""
Достать SHA-256 fingerprint из ответа GET /cluster/config/join.
В Proxmox API нет поля `fingerprint` на верхнем уровне — оно лежит в
nodelist[].pve_fp у каждой ноды. Выбираем запись целевой кластерной ноды
по pve_addr/ring0_addr == target_ip, с фолбэком на preferred_node и на
первую ноду с непустым pve_fp.
"""
if not isinstance(join_data, dict):
return None
# Совместимость: если провайдер всё же вернул fingerprint сверху — берём его.
top = join_data.get("fingerprint")
if top:
return top
nodelist = join_data.get("nodelist") or []
def _fp(entry: dict) -> Optional[str]:
return entry.get("pve_fp") if isinstance(entry, dict) else None
# 1) По IP целевой ноды
if target_ip:
for entry in nodelist:
if not isinstance(entry, dict):
continue
if target_ip in (entry.get("pve_addr"), entry.get("ring0_addr")) and _fp(entry):
return entry["pve_fp"]
# 2) По preferred_node
preferred = join_data.get("preferred_node")
if preferred:
for entry in nodelist:
if isinstance(entry, dict) and entry.get("name") == preferred and _fp(entry):
return entry["pve_fp"]
# 3) Первая нода с непустым pve_fp
for entry in nodelist:
if _fp(entry):
return entry["pve_fp"]
return None
def _get_password_client(server: ProxmoxServer, override_password: str = None) -> ProxmoxClient:
"""
Build a ProxmoxClient that uses password auth (required for cluster ops).

View File

@@ -421,6 +421,16 @@ class MigrateRequest(BaseModel):
online: bool = False
class RemoteMigrateRequest(BaseModel):
target_server_id: int
target_node: str
target_vmid: Optional[int] = None
target_storage: Optional[str] = None
target_bridge: Optional[str] = None
online: bool = False
delete_source: bool = True
class ChangePasswordRequest(BaseModel):
username: str = "root"
password: str
@@ -576,6 +586,104 @@ def migrate_container_endpoint(
return DeployTaskStartResponse(task_id=task.id, status='pending', name=task.name)
# -------- Remote migrate (move VM/LXC to a different, independent cluster) --------
def _require_pve8_plus(client, server_name: str):
"""Remote_migrate API существует только в PVE >= 8.0 — проверяем до сабмита задачи."""
try:
version = (client.proxmox.version.get() or {}).get('version', '')
major = int(str(version).split('.')[0])
except Exception as e:
raise HTTPException(status_code=502, detail=f"Не удалось определить версию PVE сервера '{server_name}': {e}")
if major < 8:
raise HTTPException(
status_code=400,
detail=f"Remote-миграция требует Proxmox VE >= 8.0 (сервер '{server_name}': версия {version})",
)
@router.post("/api/{server_id}/vm/{vmid}/remote-migrate", status_code=202)
def remote_migrate_vm_endpoint(
server_id: int,
vmid: int,
node: str,
body: RemoteMigrateRequest,
db: Session = Depends(get_db),
current_user: User = Depends(PermissionChecker("vm:remote_migrate"))
):
"""Мигрировать VM (qemu) на другой (независимый) Proxmox-кластер — фоновая задача."""
from ...models import DeployTask
from ..templates import _deploy_executor, DeployTaskStartResponse
from .async_ops import _do_remote_migrate_sync
require_vm_access(db, current_user, server_id, vmid)
source_server = _resolve_server(db, server_id)
if body.target_server_id == server_id:
raise HTTPException(status_code=400, detail="Для миграции внутри одного кластера используйте обычную миграцию")
target_server = _resolve_server(db, body.target_server_id)
_require_pve8_plus(_get_client_or_503(source_server), source_server.name)
_require_pve8_plus(_get_client_or_503(target_server), target_server.name)
task = DeployTask(
kind='remote_migrate',
name=f'remote migrate {vmid} → {target_server.name}/{body.target_node}',
status='pending', step='В очереди...', progress=0,
template_id=None, server_id=server_id, target_server_id=body.target_server_id,
user_id=current_user.id, node=node, vmid=vmid,
)
db.add(task); db.commit(); db.refresh(task)
_deploy_executor.submit(
_do_remote_migrate_sync, task.id, server_id, vmid, node, 'qemu',
body.target_server_id, body.target_node, body.target_vmid or vmid,
body.target_storage, body.target_bridge, body.online, body.delete_source,
current_user.id, current_user.username,
)
return DeployTaskStartResponse(task_id=task.id, status='pending', name=task.name)
@router.post("/api/{server_id}/container/{vmid}/remote-migrate", status_code=202)
def remote_migrate_container_endpoint(
server_id: int,
vmid: int,
node: str,
body: RemoteMigrateRequest,
db: Session = Depends(get_db),
current_user: User = Depends(PermissionChecker("lxc:remote_migrate"))
):
"""Мигрировать LXC контейнер на другой (независимый) Proxmox-кластер — фоновая задача."""
from ...models import DeployTask
from ..templates import _deploy_executor, DeployTaskStartResponse
from .async_ops import _do_remote_migrate_sync
require_vm_access(db, current_user, server_id, vmid)
source_server = _resolve_server(db, server_id)
if body.target_server_id == server_id:
raise HTTPException(status_code=400, detail="Для миграции внутри одного кластера используйте обычную миграцию")
target_server = _resolve_server(db, body.target_server_id)
_require_pve8_plus(_get_client_or_503(source_server), source_server.name)
_require_pve8_plus(_get_client_or_503(target_server), target_server.name)
task = DeployTask(
kind='remote_migrate',
name=f'remote migrate {vmid} → {target_server.name}/{body.target_node}',
status='pending', step='В очереди...', progress=0,
template_id=None, server_id=server_id, target_server_id=body.target_server_id,
user_id=current_user.id, node=node, vmid=vmid,
)
db.add(task); db.commit(); db.refresh(task)
_deploy_executor.submit(
_do_remote_migrate_sync, task.id, server_id, vmid, node, 'lxc',
body.target_server_id, body.target_node, body.target_vmid or vmid,
body.target_storage, body.target_bridge, body.online, body.delete_source,
current_user.id, current_user.username,
)
return DeployTaskStartResponse(task_id=task.id, status='pending', name=task.name)
# -------- Reinstall (re-clone from saved template) --------
@router.post("/api/{server_id}/vm/{vmid}/reinstall", status_code=202)