Merge branch 'feature/metrics-history'
This commit is contained in:
@@ -21,6 +21,7 @@ from ...ipam_service import IPAMService
|
||||
from ._helpers import (check_vm_access, require_vm_access, _get_proxmox_client,
|
||||
get_next_vmid, archive_and_delete_snapshots,
|
||||
save_vm_instance, get_vm_instance, soft_delete_vm_instance)
|
||||
from ...services.metrics_history import query_instance_metrics, timeframe_to_range
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -1868,6 +1869,55 @@ def get_container_rrddata(
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
# ==================== Metrics History (DB-only) ====================
|
||||
|
||||
def _resolve_metrics_window(timeframe, from_ts, to_ts):
|
||||
from ...config import settings
|
||||
now = int(time_lib.time())
|
||||
if from_ts is not None and to_ts is not None:
|
||||
f, t = int(from_ts), int(to_ts)
|
||||
else:
|
||||
f, t = timeframe_to_range(timeframe, now)
|
||||
max_span = getattr(settings, "METRICS_RETENTION_DAYS", 30) * 86400
|
||||
if t - f > max_span:
|
||||
f = t - max_span
|
||||
return f, t
|
||||
|
||||
|
||||
@router.get("/api/{server_id}/vm/{vmid}/metrics")
|
||||
def get_vm_metrics(
|
||||
server_id: int,
|
||||
vmid: int,
|
||||
node: str = Query(None),
|
||||
timeframe: str = Query("hour", regex="^(hour|day|week|month)$"),
|
||||
from_ts: int = Query(None),
|
||||
to_ts: int = Query(None),
|
||||
nic: str = Query("all"),
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(PermissionChecker("vm:view")),
|
||||
):
|
||||
"""Retrieve VM metrics time-series from the local DB (no Proxmox call)."""
|
||||
f, t = _resolve_metrics_window(timeframe, from_ts, to_ts)
|
||||
return query_instance_metrics(db, server_id, vmid, f, t, nic)
|
||||
|
||||
|
||||
@router.get("/api/{server_id}/container/{vmid}/metrics")
|
||||
def get_container_metrics(
|
||||
server_id: int,
|
||||
vmid: int,
|
||||
node: str = Query(None),
|
||||
timeframe: str = Query("hour", regex="^(hour|day|week|month)$"),
|
||||
from_ts: int = Query(None),
|
||||
to_ts: int = Query(None),
|
||||
nic: str = Query("all"),
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(PermissionChecker("vm:view")),
|
||||
):
|
||||
"""Retrieve container metrics time-series from the local DB (no Proxmox call)."""
|
||||
f, t = _resolve_metrics_window(timeframe, from_ts, to_ts)
|
||||
return query_instance_metrics(db, server_id, vmid, f, t, nic)
|
||||
|
||||
|
||||
# ==================== VNC Console ====================
|
||||
|
||||
@router.get("/api/{server_id}/vm/{vmid}/vnc")
|
||||
@@ -2850,6 +2900,7 @@ def set_vm_owner(
|
||||
if not instance:
|
||||
raise HTTPException(status_code=404, detail="VM not found in cache")
|
||||
|
||||
target_user = None
|
||||
if body.user_id is not None:
|
||||
target_user = db.query(User).filter(User.id == body.user_id, User.is_active == True).first()
|
||||
if not target_user:
|
||||
@@ -2858,6 +2909,29 @@ def set_vm_owner(
|
||||
instance.owner_id = body.user_id
|
||||
db.commit()
|
||||
logger.info(f"Admin {current_user.username} set owner of VM {vmid} (server {server_id}) to user_id={body.user_id}")
|
||||
|
||||
# Push a real-time event so every connected client refreshes the owner
|
||||
# column without a manual page reload.
|
||||
try:
|
||||
from ...websocket_manager import broadcast_event
|
||||
owner_user = (
|
||||
{
|
||||
"username": target_user.username,
|
||||
"email": target_user.email,
|
||||
"full_name": target_user.full_name,
|
||||
}
|
||||
if target_user else None
|
||||
)
|
||||
broadcast_event(
|
||||
"vm_owner_changed",
|
||||
server_id=server_id,
|
||||
vmid=vmid,
|
||||
owner_id=body.user_id,
|
||||
owner_user=owner_user,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to broadcast vm_owner_changed: {e}")
|
||||
|
||||
return {"ok": True, "owner_id": body.user_id}
|
||||
|
||||
|
||||
|
||||
@@ -52,6 +52,10 @@ class Settings(BaseSettings):
|
||||
# once your servers' host keys are in known_hosts.
|
||||
SSH_REJECT_UNKNOWN_HOSTS: bool = Field(default=False, env="SSH_REJECT_UNKNOWN_HOSTS")
|
||||
|
||||
# Metrics history
|
||||
METRICS_COLLECT_INTERVAL: int = Field(default=15, env="METRICS_COLLECT_INTERVAL")
|
||||
METRICS_RETENTION_DAYS: int = Field(default=30, env="METRICS_RETENTION_DAYS")
|
||||
|
||||
# CORS — comma-separated list of allowed origins; "*" is insecure in production
|
||||
CORS_ORIGINS: str = Field(default="*", env="CORS_ORIGINS")
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from .auth import Role, user_servers, User, ActiveSession, LoginAttempt, Blocked
|
||||
from .proxmox import OSTemplateGroup, OSTemplate, ProxmoxServer, VMInstance, VMSnapshotArchive
|
||||
from .ipam import IPAMNetwork, IPAMPool, IPAMAllocation, IPAMHistory
|
||||
from .tasks import TaskQueue, ProxmoxTask, DeployTask
|
||||
from .metrics import InstanceMetric, InstanceNicMetric
|
||||
from .settings import PanelSettings, SecuritySetting, Workspace, WorkspaceServer, WorkspaceUser
|
||||
from .misc import Notification, NotificationPreference, BackupJob, AuditLog
|
||||
|
||||
@@ -12,6 +13,7 @@ __all__ = [
|
||||
'OSTemplateGroup', 'OSTemplate', 'ProxmoxServer', 'VMInstance', 'VMSnapshotArchive',
|
||||
'IPAMNetwork', 'IPAMPool', 'IPAMAllocation', 'IPAMHistory',
|
||||
'TaskQueue', 'ProxmoxTask', 'DeployTask',
|
||||
'InstanceMetric', 'InstanceNicMetric',
|
||||
'PanelSettings', 'SecuritySetting', 'Workspace', 'WorkspaceServer', 'WorkspaceUser',
|
||||
'Notification', 'NotificationPreference', 'BackupJob', 'AuditLog'
|
||||
]
|
||||
|
||||
39
backend/app/models/metrics.py
Normal file
39
backend/app/models/metrics.py
Normal file
@@ -0,0 +1,39 @@
|
||||
from sqlalchemy import Float
|
||||
from .base import Base, Column, Integer, BigInteger, String, DateTime, Index, func
|
||||
|
||||
|
||||
class InstanceMetric(Base):
|
||||
__tablename__ = "instance_metric"
|
||||
|
||||
id = Column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
|
||||
server_id = Column(Integer, nullable=False)
|
||||
vmid = Column(Integer, nullable=False)
|
||||
vm_type = Column(String(10), nullable=True)
|
||||
ts = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
cpu = Column(Float, nullable=True)
|
||||
mem = Column(BigInteger, nullable=True)
|
||||
maxmem = Column(BigInteger, nullable=True)
|
||||
disk_used = Column(BigInteger, nullable=True)
|
||||
disk_total = Column(BigInteger, nullable=True)
|
||||
netin_rate = Column(Float, nullable=True)
|
||||
netout_rate = Column(Float, nullable=True)
|
||||
diskread_rate = Column(Float, nullable=True)
|
||||
diskwrite_rate = Column(Float, nullable=True)
|
||||
iops_read = Column(Float, nullable=True)
|
||||
iops_write = Column(Float, nullable=True)
|
||||
|
||||
__table_args__ = (Index("ix_instance_metric_lookup", "server_id", "vmid", "ts"),)
|
||||
|
||||
|
||||
class InstanceNicMetric(Base):
|
||||
__tablename__ = "instance_nic_metric"
|
||||
|
||||
id = Column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
|
||||
server_id = Column(Integer, nullable=False)
|
||||
vmid = Column(Integer, nullable=False)
|
||||
ts = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
dev = Column(String(64), nullable=False)
|
||||
in_rate = Column(Float, nullable=True)
|
||||
out_rate = Column(Float, nullable=True)
|
||||
|
||||
__table_args__ = (Index("ix_instance_nic_metric_lookup", "server_id", "vmid", "ts"),)
|
||||
@@ -1310,6 +1310,29 @@ class ProxmoxClient(VmMixin, LxcMixin, ClusterMixin, StorageMixin, NetworkMixin,
|
||||
logger.debug(f"Не удалось получить сетевые интерфейсы VM {vmid} (guest agent может быть не установлен): {e}")
|
||||
return []
|
||||
|
||||
def get_vm_blockstats(self, node: str, vmid: int) -> str:
|
||||
"""Raw HMP 'info blockstats' text via the QEMU monitor (VM only).
|
||||
Returns '' if unavailable."""
|
||||
if not self.proxmox:
|
||||
return ""
|
||||
try:
|
||||
res = self.proxmox.nodes(node).qemu(vmid).monitor.post(command="info blockstats")
|
||||
return res if isinstance(res, str) else str(res or "")
|
||||
except Exception as e:
|
||||
logger.debug(f"blockstats unavailable for VM {vmid} on {node}: {e}")
|
||||
return ""
|
||||
|
||||
def get_node_netstat(self, node: str) -> list:
|
||||
"""Per-tap/veth interface counters for the node ([] on failure)."""
|
||||
if not self.proxmox:
|
||||
return []
|
||||
try:
|
||||
res = self.proxmox.nodes(node).netstat.get()
|
||||
return res if isinstance(res, list) else []
|
||||
except Exception as e:
|
||||
logger.debug(f"netstat unavailable for node {node}: {e}")
|
||||
return []
|
||||
|
||||
def get_container_interfaces(self, node: str, vmid: int) -> List[Dict]:
|
||||
"""
|
||||
Получить сетевые интерфейсы контейнера
|
||||
|
||||
@@ -606,4 +606,8 @@ def start_metrics_broadcaster(db_factory) -> None:
|
||||
asyncio.create_task(server_metrics_loop(db_factory), name="metrics:servers")
|
||||
asyncio.create_task(instance_metrics_loop(db_factory), name="metrics:instances")
|
||||
asyncio.create_task(instances_list_metrics_loop(db_factory), name="metrics:instances_list")
|
||||
# Import inside function to avoid circular imports
|
||||
from .metrics_collector import metrics_collector_loop, metrics_retention_loop
|
||||
asyncio.create_task(metrics_collector_loop(db_factory), name="metrics:collector")
|
||||
asyncio.create_task(metrics_retention_loop(db_factory), name="metrics:retention")
|
||||
logger.info("[metrics_broadcaster] Metrics broadcasting tasks started")
|
||||
|
||||
148
backend/app/services/metrics_collector.py
Normal file
148
backend/app/services/metrics_collector.py
Normal file
@@ -0,0 +1,148 @@
|
||||
"""Background collector that persists instance telemetry to our own tables."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from .metrics_history import (
|
||||
compute_rate, parse_blockstats, parse_netstat, aggregate_fsinfo,
|
||||
)
|
||||
|
||||
|
||||
def _status(client, inst_type, node, vmid):
|
||||
if inst_type == "qemu":
|
||||
return client.get_vm_status(node, vmid)
|
||||
return client.get_container_status(node, vmid)
|
||||
|
||||
|
||||
def collect_once(db, items, clients, prev, _now=None):
|
||||
"""One collection tick. Mutates `prev` to carry counters between ticks."""
|
||||
from ..models import InstanceMetric, InstanceNicMetric
|
||||
now = _now if _now is not None else time.monotonic()
|
||||
ts = datetime.now(tz=timezone.utc)
|
||||
|
||||
# netstat is one call per (server_id, node); cache per tick.
|
||||
netstat_cache: dict[tuple, list] = {}
|
||||
# Track which prev keys are touched this tick so stale entries can be evicted.
|
||||
seen: set = set()
|
||||
|
||||
for it in items:
|
||||
if it.get("status") != "running":
|
||||
continue
|
||||
sid, vmid = it["server_id"], it["vmid"]
|
||||
inst_type, node = it.get("type"), it.get("node")
|
||||
client = clients.get(sid)
|
||||
if client is None or not node:
|
||||
continue
|
||||
try:
|
||||
status = _status(client, inst_type, node, vmid)
|
||||
if not status:
|
||||
continue
|
||||
|
||||
# disk fill
|
||||
if inst_type == "qemu":
|
||||
used, total = aggregate_fsinfo(client.get_vm_fsinfo(node, vmid))
|
||||
else:
|
||||
used = int(status.get("disk", 0) or 0)
|
||||
total = int(status.get("maxdisk", 0) or 0)
|
||||
|
||||
# cumulative counters → rates
|
||||
key = f"{sid}:{vmid}"
|
||||
cur = {
|
||||
"netin": float(status.get("netin", 0) or 0),
|
||||
"netout": float(status.get("netout", 0) or 0),
|
||||
"diskread": float(status.get("diskread", 0) or 0),
|
||||
"diskwrite": float(status.get("diskwrite", 0) or 0),
|
||||
}
|
||||
if inst_type == "qemu":
|
||||
rd_ops, wr_ops = parse_blockstats(client.get_vm_blockstats(node, vmid))
|
||||
cur["rd_ops"], cur["wr_ops"] = float(rd_ops), float(wr_ops)
|
||||
|
||||
pkey = prev.get(key)
|
||||
dt = (now - pkey[0]) if pkey else 0
|
||||
pc = pkey[1] if pkey else {}
|
||||
|
||||
def rate(name):
|
||||
return compute_rate(pc.get(name), cur.get(name), dt)
|
||||
|
||||
db.add(InstanceMetric(
|
||||
server_id=sid, vmid=vmid, vm_type=inst_type, ts=ts,
|
||||
cpu=status.get("cpu"), mem=status.get("mem"), maxmem=status.get("maxmem"),
|
||||
disk_used=used, disk_total=total,
|
||||
netin_rate=rate("netin"), netout_rate=rate("netout"),
|
||||
diskread_rate=rate("diskread"), diskwrite_rate=rate("diskwrite"),
|
||||
iops_read=rate("rd_ops") if inst_type == "qemu" else None,
|
||||
iops_write=rate("wr_ops") if inst_type == "qemu" else None,
|
||||
))
|
||||
prev[key] = (now, cur)
|
||||
seen.add(key)
|
||||
|
||||
# per-NIC
|
||||
ck = (sid, node)
|
||||
if ck not in netstat_cache:
|
||||
netstat_cache[ck] = client.get_node_netstat(node)
|
||||
per_dev = parse_netstat(netstat_cache[ck], vmid)
|
||||
for dev, (din, dout) in per_dev.items():
|
||||
nkey = f"{sid}:{vmid}:{dev}"
|
||||
npkey = prev.get(nkey)
|
||||
ndt = (now - npkey[0]) if npkey else 0
|
||||
npc = npkey[1] if npkey else {}
|
||||
db.add(InstanceNicMetric(
|
||||
server_id=sid, vmid=vmid, ts=ts, dev=dev,
|
||||
in_rate=compute_rate(npc.get("in"), float(din), ndt),
|
||||
out_rate=compute_rate(npc.get("out"), float(dout), ndt),
|
||||
))
|
||||
prev[nkey] = (now, {"in": float(din), "out": float(dout)})
|
||||
seen.add(nkey)
|
||||
except Exception as e:
|
||||
logger.debug(f"[metrics_collector] {sid}:{vmid} error: {e}")
|
||||
|
||||
# Evict stale keys from prev (VMs/NICs not seen this tick).
|
||||
for k in list(prev):
|
||||
if k not in seen:
|
||||
del prev[k]
|
||||
|
||||
|
||||
async def metrics_collector_loop(db_factory) -> None:
|
||||
from ..config import settings
|
||||
from .metrics_broadcaster import _enumerate_instances_for_list
|
||||
prev: dict = {}
|
||||
while True:
|
||||
start = time.monotonic()
|
||||
try:
|
||||
items, clients = await asyncio.to_thread(_enumerate_instances_for_list, db_factory)
|
||||
db = db_factory()
|
||||
try:
|
||||
await asyncio.to_thread(collect_once, db, items, clients, prev)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"[metrics_collector] tick error: {e}")
|
||||
await asyncio.sleep(max(1.0, settings.METRICS_COLLECT_INTERVAL - (time.monotonic() - start)))
|
||||
|
||||
|
||||
async def metrics_retention_loop(db_factory) -> None:
|
||||
from ..config import settings
|
||||
from ..models import InstanceMetric, InstanceNicMetric
|
||||
from datetime import timedelta
|
||||
while True:
|
||||
try:
|
||||
cutoff = datetime.now(tz=timezone.utc) - timedelta(days=settings.METRICS_RETENTION_DAYS)
|
||||
db = db_factory()
|
||||
try:
|
||||
db.query(InstanceMetric).filter(InstanceMetric.ts < cutoff).delete()
|
||||
db.query(InstanceNicMetric).filter(InstanceNicMetric.ts < cutoff).delete()
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"[metrics_collector] retention error: {e}")
|
||||
await asyncio.sleep(3600)
|
||||
163
backend/app/services/metrics_history.py
Normal file
163
backend/app/services/metrics_history.py
Normal file
@@ -0,0 +1,163 @@
|
||||
"""Pure helpers + read query for instance metric history."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import timezone as _tz
|
||||
|
||||
_RD_OPS = re.compile(r"rd_operations=(\d+)")
|
||||
_WR_OPS = re.compile(r"wr_operations=(\d+)")
|
||||
|
||||
|
||||
def compute_rate(prev, cur, dt):
|
||||
"""Per-second rate from cumulative counters. None if unknowable; 0 on reset."""
|
||||
if prev is None or cur is None or not dt or dt <= 0:
|
||||
return None
|
||||
delta = float(cur) - float(prev)
|
||||
if delta < 0:
|
||||
return 0.0
|
||||
return delta / dt
|
||||
|
||||
|
||||
def parse_blockstats(text: str):
|
||||
"""Sum rd_operations / wr_operations across all block devices in HMP output."""
|
||||
rd = sum(int(m) for m in _RD_OPS.findall(text or ""))
|
||||
wr = sum(int(m) for m in _WR_OPS.findall(text or ""))
|
||||
return rd, wr
|
||||
|
||||
|
||||
def parse_netstat(rows, vmid: int):
|
||||
"""Map dev -> (in_bytes, out_bytes) for the given vmid from /nodes/{node}/netstat."""
|
||||
out: dict[str, tuple[int, int]] = {}
|
||||
for r in rows or []:
|
||||
try:
|
||||
if int(r.get("vmid")) != int(vmid):
|
||||
continue
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
dev = r.get("dev")
|
||||
if not dev:
|
||||
continue
|
||||
out[dev] = (int(float(r.get("in", 0) or 0)), int(float(r.get("out", 0) or 0)))
|
||||
return out
|
||||
|
||||
|
||||
def aggregate_fsinfo(disks):
|
||||
"""Sum used/total bytes across guest filesystems."""
|
||||
used = sum(int(d.get("used", 0) or 0) for d in disks or [])
|
||||
total = sum(int(d.get("total", 0) or 0) for d in disks or [])
|
||||
return used, total
|
||||
|
||||
|
||||
_TIMEFRAME_SECONDS = {
|
||||
"hour": 3600,
|
||||
"day": 86400,
|
||||
"week": 7 * 86400,
|
||||
"month": 30 * 86400,
|
||||
}
|
||||
|
||||
|
||||
def timeframe_to_range(timeframe, now_ts):
|
||||
"""Convert timeframe string to (from_ts, to_ts) tuple."""
|
||||
span = _TIMEFRAME_SECONDS.get(timeframe, 3600)
|
||||
return now_ts - span, now_ts
|
||||
|
||||
|
||||
def pick_bucket_seconds(from_ts, to_ts, target_points=150, floor=15):
|
||||
"""Calculate bucket size in seconds for time-series aggregation."""
|
||||
span = max(1, to_ts - from_ts)
|
||||
ideal = span // max(1, target_points)
|
||||
return max(floor, int(ideal))
|
||||
|
||||
|
||||
def aggregate_series(rows, from_ts, bucket, fields):
|
||||
"""Aggregate time-series data into buckets, averaging numeric fields."""
|
||||
if not rows:
|
||||
return []
|
||||
buckets: dict[int, dict] = {}
|
||||
order: list[int] = []
|
||||
for r in rows:
|
||||
b = from_ts + ((int(r["time"]) - from_ts) // bucket) * bucket
|
||||
if b not in buckets:
|
||||
buckets[b] = {f: [] for f in fields}
|
||||
order.append(b)
|
||||
for f in fields:
|
||||
v = r.get(f)
|
||||
if v is not None:
|
||||
buckets[b][f].append(float(v))
|
||||
out = []
|
||||
for b in order:
|
||||
point = {"time": b}
|
||||
for f in fields:
|
||||
vals = buckets[b][f]
|
||||
point[f] = (sum(vals) / len(vals)) if vals else None
|
||||
out.append(point)
|
||||
return out
|
||||
|
||||
|
||||
_AGG_FIELDS = ["cpu", "mem", "maxmem", "netin", "netout",
|
||||
"diskread", "diskwrite", "diskpct", "iops_read", "iops_write"]
|
||||
|
||||
|
||||
def _to_ts(dt):
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=_tz.utc)
|
||||
return int(dt.timestamp())
|
||||
|
||||
|
||||
def query_instance_metrics(db, server_id, vmid, from_ts, to_ts, nic="all"):
|
||||
from ..models import InstanceMetric, InstanceNicMetric
|
||||
from datetime import datetime
|
||||
|
||||
start = datetime.fromtimestamp(from_ts, tz=_tz.utc)
|
||||
end = datetime.fromtimestamp(to_ts, tz=_tz.utc)
|
||||
|
||||
base = (db.query(InstanceMetric)
|
||||
.filter(InstanceMetric.server_id == server_id,
|
||||
InstanceMetric.vmid == vmid,
|
||||
InstanceMetric.ts >= start, InstanceMetric.ts <= end)
|
||||
.order_by(InstanceMetric.ts.asc()).all())
|
||||
|
||||
# Distinct NICs seen in the window (for the selector).
|
||||
if nic == "all":
|
||||
# Only fetch distinct dev names — avoids loading all NIC rows.
|
||||
dev_rows = (db.query(InstanceNicMetric.dev)
|
||||
.filter(InstanceNicMetric.server_id == server_id,
|
||||
InstanceNicMetric.vmid == vmid,
|
||||
InstanceNicMetric.ts >= start, InstanceNicMetric.ts <= end)
|
||||
.distinct().all())
|
||||
nics = sorted({r.dev for r in dev_rows})
|
||||
nic_by_ts: dict[int, tuple] = {}
|
||||
else:
|
||||
# Load full NIC rows for the requested dev to build per-ts override map.
|
||||
nic_rows = (db.query(InstanceNicMetric)
|
||||
.filter(InstanceNicMetric.server_id == server_id,
|
||||
InstanceNicMetric.vmid == vmid,
|
||||
InstanceNicMetric.ts >= start, InstanceNicMetric.ts <= end)
|
||||
.order_by(InstanceNicMetric.ts.asc()).all())
|
||||
nics = sorted({r.dev for r in nic_rows})
|
||||
nic_by_ts = {}
|
||||
for r in nic_rows:
|
||||
if r.dev == nic:
|
||||
nic_by_ts[_to_ts(r.ts)] = (r.in_rate, r.out_rate)
|
||||
|
||||
rows = []
|
||||
for m in base:
|
||||
ts = _to_ts(m.ts)
|
||||
netin, netout = m.netin_rate, m.netout_rate
|
||||
if nic != "all":
|
||||
ov = nic_by_ts.get(ts)
|
||||
netin, netout = (ov if ov else (None, None))
|
||||
diskpct = (m.disk_used / m.disk_total * 100.0) if (m.disk_total or 0) > 0 else None
|
||||
rows.append({
|
||||
"time": ts,
|
||||
"cpu": (m.cpu * 100.0) if m.cpu is not None else None,
|
||||
"mem": m.mem, "maxmem": m.maxmem,
|
||||
"netin": netin, "netout": netout,
|
||||
"diskread": m.diskread_rate, "diskwrite": m.diskwrite_rate,
|
||||
"diskpct": diskpct,
|
||||
"iops_read": m.iops_read, "iops_write": m.iops_write,
|
||||
})
|
||||
|
||||
bucket = pick_bucket_seconds(from_ts, to_ts)
|
||||
data = aggregate_series(rows, from_ts, bucket, _AGG_FIELDS)
|
||||
return {"data": data, "from": from_ts, "to": to_ts, "meta": {"nics": nics}}
|
||||
74
backend/tests/test_api_metrics.py
Normal file
74
backend/tests/test_api_metrics.py
Normal file
@@ -0,0 +1,74 @@
|
||||
"""Tests for /metrics read endpoints (Task 6)."""
|
||||
from datetime import datetime, timezone
|
||||
import time
|
||||
|
||||
|
||||
def test_metrics_endpoint_returns_series(client, admin_headers, db_session):
|
||||
"""Seeds InstanceMetric rows then reads them via the API.
|
||||
The endpoint reads only the DB — no Proxmox call required."""
|
||||
from app.models import InstanceMetric
|
||||
|
||||
now = int(time.time())
|
||||
for i, cpu in enumerate((0.1, 0.3)):
|
||||
db_session.add(InstanceMetric(
|
||||
server_id=1, vmid=114, vm_type="qemu",
|
||||
ts=datetime.fromtimestamp(now - 10 + i, tz=timezone.utc),
|
||||
cpu=cpu, mem=100, maxmem=200, disk_used=10, disk_total=40,
|
||||
netin_rate=1.0, netout_rate=2.0, diskread_rate=0.0, diskwrite_rate=0.0,
|
||||
iops_read=5.0, iops_write=1.0))
|
||||
db_session.commit()
|
||||
|
||||
r = client.get(
|
||||
"/proxmox/api/1/vm/114/metrics?timeframe=hour&node=prod",
|
||||
headers=admin_headers,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert "data" in body and "meta" in body
|
||||
assert body["data"], "expected at least one bucket"
|
||||
assert "nics" in body["meta"]
|
||||
|
||||
|
||||
def test_container_metrics_endpoint_returns_series(client, admin_headers, db_session):
|
||||
"""Same flow for container endpoint."""
|
||||
from app.models import InstanceMetric
|
||||
|
||||
now = int(time.time())
|
||||
db_session.add(InstanceMetric(
|
||||
server_id=1, vmid=200, vm_type="lxc",
|
||||
ts=datetime.fromtimestamp(now - 5, tz=timezone.utc),
|
||||
cpu=0.2, mem=512, maxmem=1024, disk_used=20, disk_total=80,
|
||||
netin_rate=3.0, netout_rate=1.5, diskread_rate=0.0, diskwrite_rate=0.0,
|
||||
iops_read=2.0, iops_write=0.5))
|
||||
db_session.commit()
|
||||
|
||||
r = client.get(
|
||||
"/proxmox/api/1/container/200/metrics?timeframe=hour&node=prod",
|
||||
headers=admin_headers,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert "data" in body and "meta" in body
|
||||
assert "nics" in body["meta"]
|
||||
|
||||
|
||||
def test_metrics_explicit_from_to(client, admin_headers, db_session):
|
||||
"""Explicit from_ts/to_ts take precedence over timeframe."""
|
||||
from app.models import InstanceMetric
|
||||
|
||||
now = int(time.time())
|
||||
db_session.add(InstanceMetric(
|
||||
server_id=1, vmid=300, vm_type="qemu",
|
||||
ts=datetime.fromtimestamp(now - 30, tz=timezone.utc),
|
||||
cpu=0.5, mem=256, maxmem=512, disk_used=5, disk_total=20,
|
||||
netin_rate=0.5, netout_rate=0.5, diskread_rate=0.0, diskwrite_rate=0.0,
|
||||
iops_read=1.0, iops_write=1.0))
|
||||
db_session.commit()
|
||||
|
||||
r = client.get(
|
||||
f"/proxmox/api/1/vm/300/metrics?from_ts={now - 60}&to_ts={now}",
|
||||
headers=admin_headers,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert "data" in body and "meta" in body
|
||||
37
backend/tests/test_metrics_client.py
Normal file
37
backend/tests/test_metrics_client.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from unittest.mock import MagicMock
|
||||
from app.proxmox.client import ProxmoxClient
|
||||
|
||||
|
||||
def _client_with_mock(proxmox):
|
||||
c = ProxmoxClient.__new__(ProxmoxClient)
|
||||
c.proxmox = proxmox
|
||||
return c
|
||||
|
||||
|
||||
def test_get_vm_blockstats_returns_text():
|
||||
px = MagicMock()
|
||||
px.nodes.return_value.qemu.return_value.monitor.post.return_value = "drive-scsi0: rd_operations=5 wr_operations=2"
|
||||
c = _client_with_mock(px)
|
||||
assert "rd_operations=5" in c.get_vm_blockstats("prod", 114)
|
||||
px.nodes.return_value.qemu.return_value.monitor.post.assert_called_once_with(command="info blockstats")
|
||||
|
||||
|
||||
def test_get_vm_blockstats_swallows_errors():
|
||||
px = MagicMock()
|
||||
px.nodes.return_value.qemu.return_value.monitor.post.side_effect = RuntimeError("nope")
|
||||
c = _client_with_mock(px)
|
||||
assert c.get_vm_blockstats("prod", 114) == ""
|
||||
|
||||
|
||||
def test_get_node_netstat_returns_rows():
|
||||
px = MagicMock()
|
||||
px.nodes.return_value.netstat.get.return_value = [{"dev": "tap114i0", "vmid": "114", "in": "1", "out": "2"}]
|
||||
c = _client_with_mock(px)
|
||||
assert c.get_node_netstat("prod")[0]["dev"] == "tap114i0"
|
||||
|
||||
|
||||
def test_get_node_netstat_swallows_errors():
|
||||
px = MagicMock()
|
||||
px.nodes.return_value.netstat.get.side_effect = RuntimeError("nope")
|
||||
c = _client_with_mock(px)
|
||||
assert c.get_node_netstat("prod") == []
|
||||
83
backend/tests/test_metrics_collector.py
Normal file
83
backend/tests/test_metrics_collector.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""Tests for metrics_collector.collect_once (unit-testable core)."""
|
||||
from unittest.mock import MagicMock
|
||||
from app.services.metrics_collector import collect_once
|
||||
|
||||
|
||||
def _client():
|
||||
c = MagicMock()
|
||||
# status/current
|
||||
c.get_vm_status.return_value = {
|
||||
"cpu": 0.5, "mem": 100, "maxmem": 200,
|
||||
"netin": 1000, "netout": 2000, "diskread": 0, "diskwrite": 500,
|
||||
}
|
||||
c.get_vm_fsinfo.return_value = [{"used": 10, "total": 40}]
|
||||
c.get_vm_blockstats.return_value = "drive-scsi0: rd_operations=100 wr_operations=50"
|
||||
c.get_node_netstat.return_value = [
|
||||
{"dev": "tap114i0", "vmid": "114", "in": "1000", "out": "2000"}]
|
||||
return c
|
||||
|
||||
|
||||
def test_collect_once_first_tick_no_rates(db_session):
|
||||
from app.models import InstanceMetric
|
||||
items = [{"server_id": 1, "vmid": 114, "type": "qemu", "node": "prod", "status": "running"}]
|
||||
clients = {1: _client()}
|
||||
prev = {}
|
||||
collect_once(db_session, items, clients, prev, _now=100.0)
|
||||
db_session.commit()
|
||||
m = db_session.query(InstanceMetric).one()
|
||||
assert m.cpu == 0.5 and m.disk_used == 10 and m.disk_total == 40
|
||||
assert m.netin_rate is None # no previous counter yet
|
||||
assert m.iops_read is None
|
||||
|
||||
|
||||
def test_collect_once_second_tick_computes_rates(db_session):
|
||||
from app.models import InstanceMetric, InstanceNicMetric
|
||||
items = [{"server_id": 1, "vmid": 114, "type": "qemu", "node": "prod", "status": "running"}]
|
||||
c = _client()
|
||||
clients = {1: c}
|
||||
prev = {}
|
||||
collect_once(db_session, items, clients, prev, _now=100.0)
|
||||
# advance counters by 10 over 2 seconds
|
||||
c.get_vm_status.return_value = {"cpu": 0.5, "mem": 100, "maxmem": 200,
|
||||
"netin": 1020, "netout": 2040, "diskread": 0, "diskwrite": 500}
|
||||
c.get_vm_blockstats.return_value = "drive-scsi0: rd_operations=110 wr_operations=56"
|
||||
c.get_node_netstat.return_value = [{"dev": "tap114i0", "vmid": "114", "in": "1020", "out": "2040"}]
|
||||
collect_once(db_session, items, clients, prev, _now=102.0)
|
||||
db_session.commit()
|
||||
rows = db_session.query(InstanceMetric).order_by(InstanceMetric.id).all()
|
||||
assert rows[1].netin_rate == 10.0 # (1020-1000)/2
|
||||
assert rows[1].iops_read == 5.0 # (110-100)/2
|
||||
nic = db_session.query(InstanceNicMetric).order_by(InstanceNicMetric.id).all()[-1]
|
||||
assert nic.in_rate == 10.0 # (1020-1000)/2
|
||||
|
||||
|
||||
def test_collect_once_evicts_stale_prev(db_session):
|
||||
"""Keys for VMs absent from the current tick must be pruned from prev."""
|
||||
def _client_for(vmid_str):
|
||||
c = MagicMock()
|
||||
c.get_vm_status.return_value = {
|
||||
"cpu": 0.1, "mem": 50, "maxmem": 100,
|
||||
"netin": 0, "netout": 0, "diskread": 0, "diskwrite": 0,
|
||||
}
|
||||
c.get_vm_fsinfo.return_value = [{"used": 5, "total": 20}]
|
||||
c.get_vm_blockstats.return_value = ""
|
||||
c.get_node_netstat.return_value = [
|
||||
{"dev": f"tap{vmid_str}i0", "vmid": vmid_str, "in": "0", "out": "0"}
|
||||
]
|
||||
return c
|
||||
|
||||
prev = {}
|
||||
# Tick 1: only vmid 114 running
|
||||
items_114 = [{"server_id": 1, "vmid": 114, "type": "qemu", "node": "prod", "status": "running"}]
|
||||
clients_114 = {1: _client_for("114")}
|
||||
collect_once(db_session, items_114, clients_114, prev, _now=100.0)
|
||||
# After tick 1, prev must contain 114 keys
|
||||
assert any("114" in k for k in prev), "Expected 114 keys in prev after tick 1"
|
||||
|
||||
# Tick 2: only vmid 115 running (114 is gone)
|
||||
items_115 = [{"server_id": 1, "vmid": 115, "type": "qemu", "node": "prod", "status": "running"}]
|
||||
clients_115 = {1: _client_for("115")}
|
||||
collect_once(db_session, items_115, clients_115, prev, _now=102.0)
|
||||
# 114 keys must be evicted; 115 keys must be present
|
||||
assert not any("114" in k for k in prev), "Expected 114 keys evicted from prev after tick 2"
|
||||
assert any("115" in k for k in prev), "Expected 115 keys in prev after tick 2"
|
||||
71
backend/tests/test_metrics_history.py
Normal file
71
backend/tests/test_metrics_history.py
Normal file
@@ -0,0 +1,71 @@
|
||||
from app.services import metrics_history as mh
|
||||
|
||||
|
||||
def test_compute_rate_normal():
|
||||
assert mh.compute_rate(100, 160, 2.0) == 30.0
|
||||
|
||||
def test_compute_rate_no_prev():
|
||||
assert mh.compute_rate(None, 160, 2.0) is None
|
||||
|
||||
def test_compute_rate_counter_reset():
|
||||
assert mh.compute_rate(500, 10, 2.0) == 0.0
|
||||
|
||||
def test_compute_rate_zero_dt():
|
||||
assert mh.compute_rate(100, 200, 0) is None
|
||||
|
||||
def test_parse_blockstats_sums_devices():
|
||||
text = (
|
||||
"drive-scsi0: rd_bytes=10 wr_bytes=20 rd_operations=100 wr_operations=50 flush_operations=1\n"
|
||||
"drive-scsi1: rd_bytes=5 wr_bytes=5 rd_operations=7 wr_operations=3 flush_operations=0\n"
|
||||
)
|
||||
assert mh.parse_blockstats(text) == (107, 53)
|
||||
|
||||
def test_parse_blockstats_empty():
|
||||
assert mh.parse_blockstats("") == (0, 0)
|
||||
|
||||
def test_parse_netstat_filters_by_vmid():
|
||||
rows = [
|
||||
{"dev": "tap114i0", "vmid": "114", "in": "1000", "out": "2000"},
|
||||
{"dev": "tap114i1", "vmid": "114", "in": "10", "out": "20"},
|
||||
{"dev": "tap200i0", "vmid": "200", "in": "9", "out": "9"},
|
||||
]
|
||||
out = mh.parse_netstat(rows, 114)
|
||||
assert out == {"tap114i0": (1000, 2000), "tap114i1": (10, 20)}
|
||||
|
||||
def test_aggregate_fsinfo_sums():
|
||||
disks = [{"used": 10, "total": 40}, {"used": 5, "total": 20}]
|
||||
assert mh.aggregate_fsinfo(disks) == (15, 60)
|
||||
|
||||
def test_aggregate_fsinfo_empty():
|
||||
assert mh.aggregate_fsinfo([]) == (0, 0)
|
||||
|
||||
|
||||
def test_timeframe_to_range_hour():
|
||||
assert mh.timeframe_to_range("hour", 10_000) == (10_000 - 3600, 10_000)
|
||||
|
||||
def test_timeframe_to_range_month():
|
||||
assert mh.timeframe_to_range("month", 10_000_000) == (10_000_000 - 30 * 86400, 10_000_000)
|
||||
|
||||
def test_timeframe_to_range_unknown_defaults_hour():
|
||||
assert mh.timeframe_to_range("nonsense", 10_000) == (10_000 - 3600, 10_000)
|
||||
|
||||
def test_pick_bucket_respects_floor():
|
||||
# 1h window, target 150 -> 24s ideal, but floor 15 keeps >=15
|
||||
assert mh.pick_bucket_seconds(0, 3600, target_points=150, floor=15) == 24
|
||||
|
||||
def test_pick_bucket_floor_applied():
|
||||
assert mh.pick_bucket_seconds(0, 300, target_points=150, floor=15) == 15
|
||||
|
||||
def test_aggregate_series_averages_buckets():
|
||||
rows = [
|
||||
{"time": 0, "cpu": 10.0, "mem": 100},
|
||||
{"time": 5, "cpu": 20.0, "mem": 200},
|
||||
{"time": 20, "cpu": 40.0, "mem": None},
|
||||
]
|
||||
out = mh.aggregate_series(rows, from_ts=0, bucket=10, fields=["cpu", "mem"])
|
||||
# bucket 0: times 0,5 -> cpu 15, mem 150 ; bucket 20: time 20 -> cpu 40, mem None
|
||||
assert out[0] == {"time": 0, "cpu": 15.0, "mem": 150.0}
|
||||
assert out[-1] == {"time": 20, "cpu": 40.0, "mem": None}
|
||||
|
||||
def test_aggregate_series_empty():
|
||||
assert mh.aggregate_series([], 0, 10, ["cpu"]) == []
|
||||
20
backend/tests/test_metrics_models.py
Normal file
20
backend/tests/test_metrics_models.py
Normal file
@@ -0,0 +1,20 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
def test_instance_metric_roundtrip(db_session):
|
||||
from app.models import InstanceMetric, InstanceNicMetric
|
||||
ts = datetime(2026, 6, 21, 12, 0, tzinfo=timezone.utc)
|
||||
db_session.add(InstanceMetric(
|
||||
server_id=1, vmid=114, vm_type="qemu", ts=ts,
|
||||
cpu=0.25, mem=100, maxmem=200, disk_used=10, disk_total=40,
|
||||
netin_rate=1.0, netout_rate=2.0, diskread_rate=0.0, diskwrite_rate=3.0,
|
||||
iops_read=12.0, iops_write=4.0,
|
||||
))
|
||||
db_session.add(InstanceNicMetric(
|
||||
server_id=1, vmid=114, ts=ts, dev="tap114i0", in_rate=5.0, out_rate=6.0))
|
||||
db_session.commit()
|
||||
m = db_session.query(InstanceMetric).one()
|
||||
assert m.vmid == 114
|
||||
assert m.iops_read == 12.0
|
||||
n = db_session.query(InstanceNicMetric).one()
|
||||
assert n.dev == "tap114i0"
|
||||
assert n.out_rate == 6.0
|
||||
45
backend/tests/test_metrics_query.py
Normal file
45
backend/tests/test_metrics_query.py
Normal file
@@ -0,0 +1,45 @@
|
||||
# backend/tests/test_metrics_query.py
|
||||
from datetime import datetime, timezone
|
||||
from app.services.metrics_history import query_instance_metrics
|
||||
|
||||
|
||||
def _row(db, InstanceMetric, ts, **kw):
|
||||
db.add(InstanceMetric(server_id=1, vmid=114, vm_type="qemu",
|
||||
ts=datetime.fromtimestamp(ts, tz=timezone.utc), **kw))
|
||||
|
||||
|
||||
def test_query_aggregate_all_nic(db_session):
|
||||
from app.models import InstanceMetric
|
||||
_row(db_session, InstanceMetric, 1000, cpu=0.10, mem=100, maxmem=200,
|
||||
disk_used=10, disk_total=40, netin_rate=1.0, netout_rate=2.0,
|
||||
diskread_rate=0.0, diskwrite_rate=0.0, iops_read=5.0, iops_write=1.0)
|
||||
_row(db_session, InstanceMetric, 1005, cpu=0.30, mem=140, maxmem=200,
|
||||
disk_used=20, disk_total=40, netin_rate=3.0, netout_rate=4.0,
|
||||
diskread_rate=0.0, diskwrite_rate=0.0, iops_read=7.0, iops_write=3.0)
|
||||
db_session.commit()
|
||||
out = query_instance_metrics(db_session, 1, 114, from_ts=1000, to_ts=1010, nic="all")
|
||||
assert out["from"] == 1000 and out["to"] == 1010
|
||||
pt = out["data"][0]
|
||||
assert pt["cpu"] == 20.0 # avg(10%,30%)
|
||||
assert pt["netin"] == 2.0 # avg(1,3)
|
||||
assert pt["diskpct"] == 37.5 # avg(25%,50%)
|
||||
assert pt["iops_read"] == 6.0
|
||||
|
||||
|
||||
def test_query_specific_nic_overrides_net(db_session):
|
||||
from app.models import InstanceMetric, InstanceNicMetric
|
||||
_row(db_session, InstanceMetric, 1000, cpu=0.0, mem=0, maxmem=1,
|
||||
disk_used=0, disk_total=1, netin_rate=99.0, netout_rate=99.0,
|
||||
diskread_rate=0.0, diskwrite_rate=0.0, iops_read=0.0, iops_write=0.0)
|
||||
db_session.add(InstanceNicMetric(server_id=1, vmid=114,
|
||||
ts=datetime.fromtimestamp(1000, tz=timezone.utc), dev="tap114i0",
|
||||
in_rate=7.0, out_rate=8.0))
|
||||
db_session.commit()
|
||||
out = query_instance_metrics(db_session, 1, 114, 1000, 1010, nic="tap114i0")
|
||||
assert out["meta"]["nics"] == ["tap114i0"]
|
||||
assert out["data"][0]["netin"] == 7.0 and out["data"][0]["netout"] == 8.0
|
||||
|
||||
|
||||
def test_query_empty(db_session):
|
||||
out = query_instance_metrics(db_session, 1, 999, 0, 10, nic="all")
|
||||
assert out["data"] == [] and out["meta"]["nics"] == []
|
||||
Reference in New Issue
Block a user