feat(ipam): several IP addresses per instance
A guest was implicitly limited to a single allocation: lookups were
"(server, vmid) -> one row", release_ip_by_vmid freed one address, and
the link tool skipped guests that already had one. An instance can now
hold a primary address plus any number of aliases, managed from the
Network tab of the instance and visible on the IPAM pages.
Schema and service:
- ipam_allocations gains is_primary, assignment_kind, target_interface
and apply_* columns; migration 43 marks existing rows as primary
- find_allocations_by_resource/set_primary; release_ip_by_vmid now frees
every address of a deleted guest instead of just the first one
- next-available search may scan the whole subnet when a network has no
auto-assign pool, skipping node and cached guest addresses so it never
hands out an address that is already live
Applying to the guest (services/guest_ip_service.py):
- reuses script_engine, so LXC goes through pct exec and VMs through the
guest agent; the shell script synchronises the whole alias list of an
interface, which makes add, remove and retry the same code path
- persists via ifupdown, netplan, NetworkManager or sysconfig and never
restarts guest networking; an unknown stack degrades to runtime_only
and an unreachable guest keeps the allocation as pending
API and UI:
- /ipam/api/guests/{server_id}/{vmid}/addresses: list, add, apply,
primary, delete; allocations list accepts guest filters
- instance list exposes ips[] alongside the primary ip
- addresses card on the Network tab, "+N" badge in the IP column,
primary badge in the allocations table
Fixes found along the way:
- link-allocations registers every address of a guest, carries the MAC
over, refuses to take an address away from another live guest and no
longer matches allocations by resource name (which leaked an address
between same-named guests on different servers)
- IPAM history no longer fails to serialise datetime values, which used
to abort re-linking with a 500
This commit is contained in:
@@ -5,6 +5,7 @@ Uses APScheduler for periodic tasks
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, Any, List, Optional
|
||||
from loguru import logger
|
||||
@@ -108,12 +109,17 @@ class MonitoringWorker:
|
||||
# задачу failed. Транзитный сетевой сбой не должен «убивать» живую задачу.
|
||||
UPID_FAIL_THRESHOLD = 3
|
||||
|
||||
# Как часто перепроверять IP гостя у Proxmox. Синк идёт каждые 10 секунд,
|
||||
# а адрес меняется редко — без троттлинга это сотни лишних запросов в минуту.
|
||||
IP_DETECT_TTL = 300
|
||||
|
||||
def __init__(self):
|
||||
self.last_vm_states: Dict[str, str] = {} # vm_id -> status
|
||||
self.last_resource_alerts: Dict[str, float] = {} # resource_id -> last_alert_time
|
||||
self.last_server_states: Dict[int, bool] = {} # server_id -> is_online
|
||||
self.last_server_alerts: Dict[int, float] = {} # server_id -> last_alert_time
|
||||
self.upid_fail_counts: Dict[int, int] = {} # ProxmoxTask.id -> consecutive fetch failures
|
||||
self.last_ip_detect: Dict[tuple, float] = {} # (server_id, vmid) -> monotonic time
|
||||
|
||||
def _create_proxmox_client(self, server: ProxmoxServer) -> ProxmoxClient:
|
||||
"""Create ProxmoxClient from server model, raising if auth is missing."""
|
||||
@@ -619,18 +625,20 @@ class MonitoringWorker:
|
||||
IPAMAllocation.status.in_(['allocated', 'reserved'])
|
||||
).all()
|
||||
|
||||
# Build IPAM lookups
|
||||
# Build IPAM lookups.
|
||||
# Только по идентификатору гостя: сопоставление по имени давало чужой
|
||||
# адрес одноимённым гостям на разных серверах (два разных "wireguard").
|
||||
# У гостя может быть несколько адресов — в кэш инстанса пишем
|
||||
# основной (is_primary); без флага берём самую раннюю запись.
|
||||
ipam_by_server_vmid = {} # (server_id, vmid) -> allocation
|
||||
ipam_by_name = {} # hostname/resource_name -> allocation
|
||||
|
||||
|
||||
for alloc in ipam_allocations:
|
||||
if alloc.proxmox_server_id and alloc.proxmox_vmid:
|
||||
ipam_by_server_vmid[(alloc.proxmox_server_id, alloc.proxmox_vmid)] = alloc
|
||||
if alloc.hostname:
|
||||
ipam_by_name[alloc.hostname.lower()] = alloc
|
||||
if alloc.resource_name:
|
||||
ipam_by_name[alloc.resource_name.lower()] = alloc
|
||||
|
||||
key = (alloc.proxmox_server_id, alloc.proxmox_vmid)
|
||||
current = ipam_by_server_vmid.get(key)
|
||||
if current is None or (alloc.is_primary and not current.is_primary):
|
||||
ipam_by_server_vmid[key] = alloc
|
||||
|
||||
# Detect clusters: servers that see multiple nodes are part of a cluster
|
||||
# Group servers by their node set
|
||||
server_nodes = {} # server_id -> set of node names
|
||||
@@ -749,26 +757,34 @@ class MonitoringWorker:
|
||||
all_seen.add((server_id, vmid))
|
||||
|
||||
# Get IP from IPAM
|
||||
ipam_alloc = (
|
||||
ipam_by_server_vmid.get((server_id, vmid)) or
|
||||
ipam_by_name.get(vm_name.lower())
|
||||
)
|
||||
ipam_alloc = ipam_by_server_vmid.get((server_id, vmid))
|
||||
ip_address = ipam_alloc.ip_address if ipam_alloc else None
|
||||
|
||||
# If no IPAM IP, try to detect from Proxmox (LXC config or QEMU guest agent)
|
||||
if not ip_address:
|
||||
# If no IPAM IP, try to detect from Proxmox (LXC config or QEMU guest agent).
|
||||
# Троттлим: адрес гостя меняется куда реже, чем идёт синк.
|
||||
detect_key = (server_id, vmid)
|
||||
detect_due = (time.monotonic() - self.last_ip_detect.get(detect_key, 0.0)) >= self.IP_DETECT_TTL
|
||||
if not ip_address and detect_due:
|
||||
self.last_ip_detect[detect_key] = time.monotonic()
|
||||
try:
|
||||
px_client = server_clients.get(server_id)
|
||||
if px_client and node_name:
|
||||
if vm_type == 'lxc':
|
||||
config = px_client.proxmox.nodes(node_name).lxc(vmid).config.get()
|
||||
for i in range(4):
|
||||
ipconfig = config.get(f'ipconfig{i}', '')
|
||||
if 'ip=' in str(ipconfig):
|
||||
ip_part = str(ipconfig).split('ip=')[1].split(',')[0]
|
||||
if ip_part and ip_part != 'dhcp':
|
||||
ip_address = ip_part.split('/')[0] if '/' in ip_part else ip_part
|
||||
# Статика из netN + живые адреса запущенного
|
||||
# контейнера (DHCP виден только там)
|
||||
for iface in px_client.get_container_interfaces(
|
||||
node_name, vmid,
|
||||
include_live=vm_data.get('status') == 'running',
|
||||
):
|
||||
for ip_info in iface.get('ips') or []:
|
||||
if ip_info.get('type') != 'ipv4':
|
||||
continue
|
||||
addr = ip_info.get('address', '')
|
||||
if addr:
|
||||
ip_address = addr
|
||||
break
|
||||
if ip_address:
|
||||
break
|
||||
elif vm_type == 'qemu' and vm_data.get('status') == 'running':
|
||||
result = px_client.proxmox.nodes(node_name).qemu(vmid).agent('network-get-interfaces').get()
|
||||
if result and 'result' in result:
|
||||
@@ -862,6 +878,11 @@ class MonitoringWorker:
|
||||
db.rollback()
|
||||
return
|
||||
|
||||
# Не копим троттлинг-метки по гостям, которых больше нет
|
||||
self.last_ip_detect = {
|
||||
key: seen_at for key, seen_at in self.last_ip_detect.items() if key in all_seen
|
||||
}
|
||||
|
||||
# Mark VMs that no longer exist as deleted
|
||||
# Get all active VMs from synced servers
|
||||
synced_server_ids = set(server.id for server in servers)
|
||||
|
||||
Reference in New Issue
Block a user