Interactive infrastructure map (panel -> cluster -> node -> guest) rendered with React Flow, backed by a single aggregated /api/network/topology endpoint instead of fanning out over servers/nodes/SDN/VMs. Adds a netN NIC parser for QEMU/LXC guest configs, status/threshold filters, canvas tools, and PNG/SVG export.
487 lines
20 KiB
Python
487 lines
20 KiB
Python
"""Aggregated network topology: panel -> cluster -> node -> guest.
|
|
|
|
One endpoint feeds the whole /topology page so the frontend makes a single
|
|
request instead of fanning out over servers/nodes/sdn/vms. Everything the graph
|
|
draws - plus the bridge/VLAN/MAC data the planned "group by VLAN" view will
|
|
need - is collected here.
|
|
|
|
Kept separate from ``cluster.py``'s ``/api/cluster/topology``, which answers a
|
|
different question (which servers form which cluster) and is consumed by the
|
|
Cluster page.
|
|
"""
|
|
|
|
import asyncio
|
|
import threading
|
|
import time as time_lib
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
|
from loguru import logger
|
|
from sqlalchemy.orm import Session, joinedload
|
|
|
|
from ...auth import PermissionChecker
|
|
from ...db import get_db
|
|
from ...models import IPAMAllocation, IPAMNetwork, ProxmoxServer, User, VMInstance
|
|
from ...proxmox import _run_in_executor, parse_guest_nics
|
|
from ...schemas import (NetworkTopologyResponse, TopologyBridge, TopologyCluster,
|
|
TopologyGuest, TopologyNic, TopologyNode,
|
|
TopologyPanelSummary, TopologyWarning)
|
|
from ._helpers import _get_proxmox_client, can_view_all_instances, get_visible_server_ids
|
|
|
|
router = APIRouter()
|
|
|
|
# Live cluster/resources + node networks, keyed by server id. Short TTL: the
|
|
# page auto-refreshes every 30s and several users may watch it at once.
|
|
_topo_cache: Dict[int, Dict[str, Any]] = {}
|
|
_topo_lock = threading.Lock()
|
|
TOPO_TTL = 15
|
|
|
|
# Guest configs change rarely, and reading them costs one HTTP call per guest.
|
|
_nic_cache: Dict[Tuple[int, int], Dict[str, Any]] = {}
|
|
_nic_lock = threading.Lock()
|
|
NIC_TTL = 300
|
|
NIC_MAX_WORKERS = 8
|
|
NIC_MAX_FETCHES = 300 # hard ceiling so a huge cluster cannot hang the request
|
|
|
|
|
|
def _truthy(value: Any) -> bool:
|
|
"""Proxmox returns flags as 1/0, "1"/"0" or bools depending on the field."""
|
|
return str(value).strip().lower() in ("1", "true", "yes", "on")
|
|
|
|
|
|
# ==================== Live collection (runs off the request thread) ====================
|
|
|
|
def _collect_server(server: ProxmoxServer, include_bridges: bool) -> Dict[str, Any]:
|
|
"""Everything one Proxmox connection can tell us, in as few calls as possible."""
|
|
client = _get_proxmox_client(server)
|
|
|
|
# A single /cluster/resources call carries both node and guest metrics.
|
|
resources = client.get_cluster_resources() or []
|
|
nodes = [r for r in resources if r.get("type") == "node"]
|
|
guests = [r for r in resources if r.get("type") in ("qemu", "lxc")]
|
|
|
|
bridges: Dict[str, List[Dict[str, Any]]] = {}
|
|
if include_bridges:
|
|
# Zones/vnets are cluster-wide; fetch once and filter per node below.
|
|
try:
|
|
zones = {z.get("zone"): z for z in (client.get_sdn_zones() or [])}
|
|
vnets = client.get_sdn_vnets() or []
|
|
except Exception as exc: # SDN is optional and older PVE lacks it
|
|
logger.debug(f"SDN unavailable on {server.name}: {exc}")
|
|
zones, vnets = {}, []
|
|
|
|
for node_res in nodes:
|
|
node_name = node_res.get("node")
|
|
if not node_name:
|
|
continue
|
|
entries: List[Dict[str, Any]] = []
|
|
try:
|
|
for iface in client.get_node_networks(node_name) or []:
|
|
name = iface.get("iface") or iface.get("name")
|
|
if not name:
|
|
continue
|
|
cidr = iface.get("cidr")
|
|
if not cidr and iface.get("address") and iface.get("netmask"):
|
|
cidr = "{0}/{1}".format(iface["address"], iface["netmask"])
|
|
entries.append({
|
|
"name": name,
|
|
"type": iface.get("type"),
|
|
"cidr": cidr,
|
|
"vlan_aware": _truthy(iface.get("bridge_vlan_aware")),
|
|
"ports": iface.get("bridge_ports") or iface.get("slaves"),
|
|
"active": _truthy(iface.get("active")),
|
|
})
|
|
except Exception as exc:
|
|
logger.debug(f"Could not read networks of {server.name}/{node_name}: {exc}")
|
|
|
|
# SDN vnets are attachable as bridges but never appear in /network.
|
|
for vnet in vnets:
|
|
vnet_name = vnet.get("vnet")
|
|
if not vnet_name:
|
|
continue
|
|
zone_name = vnet.get("zone", "")
|
|
zone_nodes = (zones.get(zone_name, {}).get("nodes") or "").replace(" ", "")
|
|
if zone_nodes and node_name not in zone_nodes.split(","):
|
|
continue
|
|
entries.append({
|
|
"name": vnet_name,
|
|
"type": "vnet",
|
|
"zone": zone_name,
|
|
"active": True,
|
|
})
|
|
bridges[node_name] = entries
|
|
|
|
return {"nodes": nodes, "guests": guests, "bridges": bridges}
|
|
|
|
|
|
def _fetch_nics(server: ProxmoxServer, targets: List[Tuple[str, int, str]]) -> Dict[int, List[Dict[str, Any]]]:
|
|
"""``netN`` cards for the given guests. ``targets`` = [(node, vmid, type)]."""
|
|
if not targets:
|
|
return {}
|
|
client = _get_proxmox_client(server)
|
|
|
|
def _one(node: str, vmid: int, vm_type: str) -> Tuple[int, List[Dict[str, Any]]]:
|
|
getter = client.get_vm_config if vm_type == "qemu" else client.get_container_config
|
|
config = getter(node, vmid) or {}
|
|
return vmid, [nic.to_dict() for nic in parse_guest_nics(config)]
|
|
|
|
result: Dict[int, List[Dict[str, Any]]] = {}
|
|
with ThreadPoolExecutor(max_workers=min(NIC_MAX_WORKERS, len(targets))) as pool:
|
|
futures = [pool.submit(_one, *target) for target in targets]
|
|
for future in as_completed(futures):
|
|
try:
|
|
vmid, nics = future.result()
|
|
except Exception as exc:
|
|
logger.debug(f"Could not read guest config on {server.name}: {exc}")
|
|
continue
|
|
result[vmid] = nics
|
|
return result
|
|
|
|
|
|
# ==================== Helpers ====================
|
|
|
|
def _split_tags(raw: Optional[str]) -> List[str]:
|
|
"""Proxmox stores tags as a ``;``-separated string; the graph wants a list."""
|
|
if not raw:
|
|
return []
|
|
return [t.strip() for t in str(raw).replace(",", ";").split(";") if t.strip()]
|
|
|
|
|
|
def _to_int(value: Any) -> Optional[int]:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _to_float(value: Any) -> Optional[float]:
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _empty_response() -> NetworkTopologyResponse:
|
|
return NetworkTopologyResponse(
|
|
generated_at=datetime.now(timezone.utc),
|
|
panel=TopologyPanelSummary(),
|
|
clusters=[],
|
|
warnings=[],
|
|
)
|
|
|
|
|
|
# ==================== Endpoint ====================
|
|
|
|
@router.get("/api/network/topology", response_model=NetworkTopologyResponse)
|
|
async def get_network_topology(
|
|
request: Request,
|
|
server_id: Optional[int] = Query(None, description="Limit to a single connection"),
|
|
include_stopped: bool = Query(True),
|
|
include_templates: bool = Query(False),
|
|
include_bridges: bool = Query(True),
|
|
include_nics: bool = Query(False, description="Read every guest config for bridge/VLAN/MAC - one call per guest"),
|
|
db: Session = Depends(get_db),
|
|
current_user: User = Depends(PermissionChecker("network:view")),
|
|
):
|
|
"""Infrastructure tree for the topology page.
|
|
|
|
Always answers 200 (bar a permission denial): an unreachable server
|
|
degrades to cached rows plus a warning rather than failing the page.
|
|
"""
|
|
warnings: List[TopologyWarning] = []
|
|
|
|
# -- Visibility ----------------------------------------------------------
|
|
# None means "no restriction" and is NOT the same as an empty set.
|
|
visible_ids = get_visible_server_ids(request, db, current_user)
|
|
if visible_ids is not None and not visible_ids:
|
|
return _empty_response()
|
|
|
|
servers_q = db.query(ProxmoxServer)
|
|
if visible_ids is not None:
|
|
servers_q = servers_q.filter(ProxmoxServer.id.in_(visible_ids))
|
|
if server_id is not None:
|
|
if visible_ids is not None and server_id not in visible_ids:
|
|
# 404 rather than 403 so the response cannot confirm the server exists.
|
|
raise HTTPException(status_code=404, detail="Proxmox server not found")
|
|
servers_q = servers_q.filter(ProxmoxServer.id == server_id)
|
|
servers = servers_q.order_by(ProxmoxServer.name).all()
|
|
if not servers:
|
|
return _empty_response()
|
|
server_map = {s.id: s for s in servers}
|
|
|
|
# -- Ownership: live Proxmox data carries no owner, the DB does ----------
|
|
see_all = can_view_all_instances(current_user)
|
|
|
|
db_q = (
|
|
db.query(VMInstance)
|
|
.options(joinedload(VMInstance.owner))
|
|
.filter(VMInstance.deleted_at.is_(None), VMInstance.server_id.in_(server_map.keys()))
|
|
)
|
|
if not see_all:
|
|
db_q = db_q.filter(VMInstance.owner_id == current_user.id)
|
|
db_guests = db_q.all()
|
|
db_by_key: Dict[Tuple[int, int], VMInstance] = {(g.server_id, g.vmid): g for g in db_guests}
|
|
# Pair-scoped, not vmid-scoped: the same vmid on two servers is two guests.
|
|
owned_keys = set(db_by_key.keys())
|
|
|
|
# -- IP addresses from IPAM, falling back to the cached instance row ------
|
|
ipam_by_key: Dict[Tuple[int, int], str] = {}
|
|
for alloc in (
|
|
db.query(IPAMAllocation)
|
|
.filter(IPAMAllocation.status.in_(["allocated", "reserved"]))
|
|
.all()
|
|
):
|
|
if alloc.proxmox_server_id and alloc.proxmox_vmid and alloc.ip_address:
|
|
ipam_by_key[(alloc.proxmox_server_id, alloc.proxmox_vmid)] = alloc.ip_address
|
|
|
|
ipam_nets = db.query(IPAMNetwork).filter(
|
|
IPAMNetwork.proxmox_server_id.in_(server_map.keys())
|
|
).all()
|
|
ipam_by_bridge: Dict[Tuple[int, str], IPAMNetwork] = {
|
|
(n.proxmox_server_id, n.proxmox_bridge): n for n in ipam_nets if n.proxmox_bridge
|
|
}
|
|
|
|
# -- Live data, in parallel: one dead cluster must not stall the rest -----
|
|
live: Dict[int, Dict[str, Any]] = {}
|
|
stale_servers: List[ProxmoxServer] = []
|
|
now = time_lib.time()
|
|
with _topo_lock: # guards the dict only, never a network call
|
|
for server in servers:
|
|
cached = _topo_cache.get(server.id)
|
|
if cached and (now - cached["time"]) < TOPO_TTL:
|
|
live[server.id] = cached["data"]
|
|
elif server.is_online is False:
|
|
# Offline servers contribute nothing but a connection timeout.
|
|
continue
|
|
else:
|
|
stale_servers.append(server)
|
|
|
|
if stale_servers:
|
|
tasks = [_run_in_executor(_collect_server, s, include_bridges) for s in stale_servers]
|
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
for server, result in zip(stale_servers, results):
|
|
if isinstance(result, HTTPException):
|
|
warnings.append(TopologyWarning(
|
|
server_id=server.id, server_name=server.name,
|
|
code="no_credentials", message=str(result.detail),
|
|
))
|
|
continue
|
|
if isinstance(result, BaseException):
|
|
logger.warning(f"Topology collection failed for {server.name}: {result}")
|
|
warnings.append(TopologyWarning(
|
|
server_id=server.id, server_name=server.name,
|
|
code="error", message=str(result),
|
|
))
|
|
continue
|
|
live[server.id] = result
|
|
with _topo_lock:
|
|
_topo_cache[server.id] = {"time": time_lib.time(), "data": result}
|
|
|
|
for server in servers:
|
|
if server.id not in live and not any(w.server_id == server.id for w in warnings):
|
|
warnings.append(TopologyWarning(
|
|
server_id=server.id, server_name=server.name,
|
|
code="offline", message=f"'{server.name}' is unreachable - showing cached data",
|
|
))
|
|
|
|
# -- NICs (opt-in): bridge / VLAN / MAC straight from the guest configs ---
|
|
nics_by_key: Dict[Tuple[int, int], List[Dict[str, Any]]] = {}
|
|
if include_nics:
|
|
pending: Dict[int, List[Tuple[str, int, str]]] = {}
|
|
budget = NIC_MAX_FETCHES
|
|
truncated = False
|
|
now = time_lib.time()
|
|
for sid, data in live.items():
|
|
for guest in data["guests"]:
|
|
vmid = _to_int(guest.get("vmid"))
|
|
node = guest.get("node")
|
|
if vmid is None or not node:
|
|
continue
|
|
if not see_all and (sid, vmid) not in owned_keys:
|
|
continue
|
|
with _nic_lock:
|
|
cached = _nic_cache.get((sid, vmid))
|
|
if cached and (now - cached["time"]) < NIC_TTL:
|
|
nics_by_key[(sid, vmid)] = cached["data"]
|
|
continue
|
|
if budget <= 0:
|
|
truncated = True
|
|
continue
|
|
budget -= 1
|
|
pending.setdefault(sid, []).append((node, vmid, guest.get("type", "qemu")))
|
|
|
|
if truncated:
|
|
warnings.append(TopologyWarning(
|
|
code="nics_truncated",
|
|
message=f"Network card details limited to {NIC_MAX_FETCHES} guests per request",
|
|
))
|
|
|
|
if pending:
|
|
server_ids = list(pending.keys())
|
|
tasks = [_run_in_executor(_fetch_nics, server_map[sid], pending[sid]) for sid in server_ids]
|
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
stamp = time_lib.time()
|
|
for sid, result in zip(server_ids, results):
|
|
if isinstance(result, BaseException):
|
|
logger.debug(f"NIC collection failed for server {sid}: {result}")
|
|
continue
|
|
for vmid, nics in result.items():
|
|
nics_by_key[(sid, vmid)] = nics
|
|
with _nic_lock:
|
|
_nic_cache[(sid, vmid)] = {"time": stamp, "data": nics}
|
|
|
|
# -- Assemble ------------------------------------------------------------
|
|
def _build_guest(sid: int, node_name: str, res: Optional[Dict[str, Any]],
|
|
row: Optional[VMInstance]) -> Optional[TopologyGuest]:
|
|
res = res or {}
|
|
vmid = _to_int(res.get("vmid"))
|
|
if vmid is None and row is not None:
|
|
vmid = row.vmid
|
|
if vmid is None:
|
|
return None
|
|
|
|
vm_type = res.get("type") or (row.vm_type if row else None) or "qemu"
|
|
is_template = _truthy(res.get("template")) or bool(row and row.is_template)
|
|
if is_template and not include_templates:
|
|
return None
|
|
status = res.get("status") or (row.status if row else None) or "unknown"
|
|
if not include_stopped and status != "running":
|
|
return None
|
|
|
|
tags = _split_tags(res.get("tags")) or _split_tags(row.tags if row else None)
|
|
name = res.get("name") or (row.name if row else None) or f"vm-{vmid}"
|
|
|
|
return TopologyGuest(
|
|
id=f"guest:{sid}:{vmid}",
|
|
server_id=sid,
|
|
node=node_name,
|
|
vmid=vmid,
|
|
type=vm_type,
|
|
name=name,
|
|
status=status,
|
|
is_template=is_template,
|
|
lock=res.get("lock"),
|
|
cpu=_to_float(res.get("cpu")),
|
|
cores=_to_int(res.get("maxcpu")) or (row.cores if row else None),
|
|
mem=_to_int(res.get("mem")),
|
|
maxmem=_to_int(res.get("maxmem")) or (row.memory if row else None),
|
|
disk=_to_int(res.get("disk")),
|
|
maxdisk=_to_int(res.get("maxdisk")),
|
|
uptime=_to_int(res.get("uptime")),
|
|
tags=tags,
|
|
owner_id=row.owner_id if row else None,
|
|
owner_username=(row.owner.username if row and row.owner else None),
|
|
ip=ipam_by_key.get((sid, vmid)) or (row.ip_address if row else None),
|
|
nics=[
|
|
TopologyNic(**{k: v for k, v in nic.items() if k in TopologyNic.model_fields})
|
|
for nic in nics_by_key.get((sid, vmid), [])
|
|
],
|
|
)
|
|
|
|
# cluster_name groups connections; a server without one stands alone.
|
|
cluster_buckets: Dict[str, Dict[str, Any]] = {}
|
|
for server in servers:
|
|
key = f"cluster:{server.cluster_name}" if server.cluster_name else f"cluster:server:{server.id}"
|
|
bucket = cluster_buckets.setdefault(key, {
|
|
"id": key,
|
|
"name": server.cluster_name or server.name,
|
|
"kind": "cluster" if server.cluster_name else "standalone",
|
|
"server_ids": [],
|
|
"online": False,
|
|
"nodes": {}, # node name -> TopologyNode, deduplicated per cluster
|
|
})
|
|
bucket["server_ids"].append(server.id)
|
|
if server.id in live:
|
|
bucket["online"] = True
|
|
|
|
data = live.get(server.id)
|
|
if data is None:
|
|
# Rebuild from the 10s VM cache so an offline server still shows up.
|
|
by_node: Dict[str, List[VMInstance]] = {}
|
|
for row in db_guests:
|
|
if row.server_id != server.id:
|
|
continue
|
|
by_node.setdefault(row.node or server.name, []).append(row)
|
|
for node_name, node_rows in by_node.items():
|
|
if node_name in bucket["nodes"]:
|
|
continue
|
|
guests = [g for g in (_build_guest(server.id, node_name, None, r) for r in node_rows) if g]
|
|
guests.sort(key=lambda g: g.vmid)
|
|
bucket["nodes"][node_name] = TopologyNode(
|
|
id=f"node:{server.id}:{node_name}",
|
|
server_id=server.id, server_name=server.name, node=node_name,
|
|
status="unknown", stale=True, guests=guests,
|
|
)
|
|
continue
|
|
|
|
guests_by_node: Dict[str, List[Dict[str, Any]]] = {}
|
|
for res in data["guests"]:
|
|
node_name = res.get("node")
|
|
vmid = _to_int(res.get("vmid"))
|
|
if not node_name or vmid is None:
|
|
continue
|
|
if not see_all and (server.id, vmid) not in owned_keys:
|
|
continue
|
|
guests_by_node.setdefault(node_name, []).append(res)
|
|
|
|
for node_res in data["nodes"]:
|
|
node_name = node_res.get("node")
|
|
if not node_name or node_name in bucket["nodes"]:
|
|
continue # same PVE cluster reachable through two connections
|
|
bridges = []
|
|
for entry in data["bridges"].get(node_name, []):
|
|
ipam_net = ipam_by_bridge.get((server.id, entry["name"]))
|
|
bridges.append(TopologyBridge(
|
|
**entry,
|
|
ipam_network_id=ipam_net.id if ipam_net else None,
|
|
ipam_cidr=ipam_net.network if ipam_net else None,
|
|
ipam_name=ipam_net.name if ipam_net else None,
|
|
))
|
|
guests = [
|
|
g for g in (
|
|
_build_guest(server.id, node_name, res,
|
|
db_by_key.get((server.id, _to_int(res.get("vmid")))))
|
|
for res in guests_by_node.get(node_name, [])
|
|
) if g
|
|
]
|
|
guests.sort(key=lambda g: g.vmid)
|
|
bucket["nodes"][node_name] = TopologyNode(
|
|
id=f"node:{server.id}:{node_name}",
|
|
server_id=server.id,
|
|
server_name=server.name,
|
|
node=node_name,
|
|
status=node_res.get("status") or "unknown",
|
|
stale=False,
|
|
cpu=_to_float(node_res.get("cpu")),
|
|
maxcpu=_to_int(node_res.get("maxcpu")),
|
|
mem=_to_int(node_res.get("mem")),
|
|
maxmem=_to_int(node_res.get("maxmem")),
|
|
uptime=_to_int(node_res.get("uptime")),
|
|
bridges=bridges,
|
|
guests=guests,
|
|
)
|
|
|
|
clusters = [
|
|
TopologyCluster(
|
|
id=b["id"], name=b["name"], kind=b["kind"],
|
|
server_ids=b["server_ids"], online=b["online"],
|
|
nodes=sorted(b["nodes"].values(), key=lambda n: n.node),
|
|
)
|
|
for b in cluster_buckets.values()
|
|
]
|
|
clusters.sort(key=lambda c: (c.kind != "cluster", c.name.lower()))
|
|
|
|
node_count = sum(len(c.nodes) for c in clusters)
|
|
guest_count = sum(len(n.guests) for c in clusters for n in c.nodes)
|
|
|
|
return NetworkTopologyResponse(
|
|
generated_at=datetime.now(timezone.utc),
|
|
panel=TopologyPanelSummary(
|
|
cluster_count=len(clusters), node_count=node_count, guest_count=guest_count,
|
|
),
|
|
clusters=clusters,
|
|
warnings=warnings,
|
|
)
|