Merge branch 'feature/metrics-history'
All checks were successful
Backend Tests / pytest (Python 3.12) (push) Successful in 50s
Deploy to Production / Trigger production update (push) Successful in 7s

This commit is contained in:
2026-06-22 02:47:45 +05:00
21 changed files with 1052 additions and 78 deletions

View File

@@ -21,6 +21,7 @@ from ...ipam_service import IPAMService
from ._helpers import (check_vm_access, require_vm_access, _get_proxmox_client,
get_next_vmid, archive_and_delete_snapshots,
save_vm_instance, get_vm_instance, soft_delete_vm_instance)
from ...services.metrics_history import query_instance_metrics, timeframe_to_range
router = APIRouter()
@@ -1868,6 +1869,55 @@ def get_container_rrddata(
raise HTTPException(status_code=500, detail=str(e))
# ==================== Metrics History (DB-only) ====================
def _resolve_metrics_window(timeframe, from_ts, to_ts):
from ...config import settings
now = int(time_lib.time())
if from_ts is not None and to_ts is not None:
f, t = int(from_ts), int(to_ts)
else:
f, t = timeframe_to_range(timeframe, now)
max_span = getattr(settings, "METRICS_RETENTION_DAYS", 30) * 86400
if t - f > max_span:
f = t - max_span
return f, t
@router.get("/api/{server_id}/vm/{vmid}/metrics")
def get_vm_metrics(
server_id: int,
vmid: int,
node: str = Query(None),
timeframe: str = Query("hour", regex="^(hour|day|week|month)$"),
from_ts: int = Query(None),
to_ts: int = Query(None),
nic: str = Query("all"),
db: Session = Depends(get_db),
current_user: User = Depends(PermissionChecker("vm:view")),
):
"""Retrieve VM metrics time-series from the local DB (no Proxmox call)."""
f, t = _resolve_metrics_window(timeframe, from_ts, to_ts)
return query_instance_metrics(db, server_id, vmid, f, t, nic)
@router.get("/api/{server_id}/container/{vmid}/metrics")
def get_container_metrics(
server_id: int,
vmid: int,
node: str = Query(None),
timeframe: str = Query("hour", regex="^(hour|day|week|month)$"),
from_ts: int = Query(None),
to_ts: int = Query(None),
nic: str = Query("all"),
db: Session = Depends(get_db),
current_user: User = Depends(PermissionChecker("vm:view")),
):
"""Retrieve container metrics time-series from the local DB (no Proxmox call)."""
f, t = _resolve_metrics_window(timeframe, from_ts, to_ts)
return query_instance_metrics(db, server_id, vmid, f, t, nic)
# ==================== VNC Console ====================
@router.get("/api/{server_id}/vm/{vmid}/vnc")
@@ -2850,6 +2900,7 @@ def set_vm_owner(
if not instance:
raise HTTPException(status_code=404, detail="VM not found in cache")
target_user = None
if body.user_id is not None:
target_user = db.query(User).filter(User.id == body.user_id, User.is_active == True).first()
if not target_user:
@@ -2858,6 +2909,29 @@ def set_vm_owner(
instance.owner_id = body.user_id
db.commit()
logger.info(f"Admin {current_user.username} set owner of VM {vmid} (server {server_id}) to user_id={body.user_id}")
# Push a real-time event so every connected client refreshes the owner
# column without a manual page reload.
try:
from ...websocket_manager import broadcast_event
owner_user = (
{
"username": target_user.username,
"email": target_user.email,
"full_name": target_user.full_name,
}
if target_user else None
)
broadcast_event(
"vm_owner_changed",
server_id=server_id,
vmid=vmid,
owner_id=body.user_id,
owner_user=owner_user,
)
except Exception as e:
logger.debug(f"Failed to broadcast vm_owner_changed: {e}")
return {"ok": True, "owner_id": body.user_id}

View File

@@ -52,6 +52,10 @@ class Settings(BaseSettings):
# once your servers' host keys are in known_hosts.
SSH_REJECT_UNKNOWN_HOSTS: bool = Field(default=False, env="SSH_REJECT_UNKNOWN_HOSTS")
# Metrics history
METRICS_COLLECT_INTERVAL: int = Field(default=15, env="METRICS_COLLECT_INTERVAL")
METRICS_RETENTION_DAYS: int = Field(default=30, env="METRICS_RETENTION_DAYS")
# CORS — comma-separated list of allowed origins; "*" is insecure in production
CORS_ORIGINS: str = Field(default="*", env="CORS_ORIGINS")

View File

@@ -3,6 +3,7 @@ from .auth import Role, user_servers, User, ActiveSession, LoginAttempt, Blocked
from .proxmox import OSTemplateGroup, OSTemplate, ProxmoxServer, VMInstance, VMSnapshotArchive
from .ipam import IPAMNetwork, IPAMPool, IPAMAllocation, IPAMHistory
from .tasks import TaskQueue, ProxmoxTask, DeployTask
from .metrics import InstanceMetric, InstanceNicMetric
from .settings import PanelSettings, SecuritySetting, Workspace, WorkspaceServer, WorkspaceUser
from .misc import Notification, NotificationPreference, BackupJob, AuditLog
@@ -12,6 +13,7 @@ __all__ = [
'OSTemplateGroup', 'OSTemplate', 'ProxmoxServer', 'VMInstance', 'VMSnapshotArchive',
'IPAMNetwork', 'IPAMPool', 'IPAMAllocation', 'IPAMHistory',
'TaskQueue', 'ProxmoxTask', 'DeployTask',
'InstanceMetric', 'InstanceNicMetric',
'PanelSettings', 'SecuritySetting', 'Workspace', 'WorkspaceServer', 'WorkspaceUser',
'Notification', 'NotificationPreference', 'BackupJob', 'AuditLog'
]

View File

@@ -0,0 +1,39 @@
from sqlalchemy import Float
from .base import Base, Column, Integer, BigInteger, String, DateTime, Index, func
class InstanceMetric(Base):
__tablename__ = "instance_metric"
id = Column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
server_id = Column(Integer, nullable=False)
vmid = Column(Integer, nullable=False)
vm_type = Column(String(10), nullable=True)
ts = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
cpu = Column(Float, nullable=True)
mem = Column(BigInteger, nullable=True)
maxmem = Column(BigInteger, nullable=True)
disk_used = Column(BigInteger, nullable=True)
disk_total = Column(BigInteger, nullable=True)
netin_rate = Column(Float, nullable=True)
netout_rate = Column(Float, nullable=True)
diskread_rate = Column(Float, nullable=True)
diskwrite_rate = Column(Float, nullable=True)
iops_read = Column(Float, nullable=True)
iops_write = Column(Float, nullable=True)
__table_args__ = (Index("ix_instance_metric_lookup", "server_id", "vmid", "ts"),)
class InstanceNicMetric(Base):
__tablename__ = "instance_nic_metric"
id = Column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
server_id = Column(Integer, nullable=False)
vmid = Column(Integer, nullable=False)
ts = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
dev = Column(String(64), nullable=False)
in_rate = Column(Float, nullable=True)
out_rate = Column(Float, nullable=True)
__table_args__ = (Index("ix_instance_nic_metric_lookup", "server_id", "vmid", "ts"),)

View File

@@ -1310,6 +1310,29 @@ class ProxmoxClient(VmMixin, LxcMixin, ClusterMixin, StorageMixin, NetworkMixin,
logger.debug(f"Не удалось получить сетевые интерфейсы VM {vmid} (guest agent может быть не установлен): {e}")
return []
def get_vm_blockstats(self, node: str, vmid: int) -> str:
"""Raw HMP 'info blockstats' text via the QEMU monitor (VM only).
Returns '' if unavailable."""
if not self.proxmox:
return ""
try:
res = self.proxmox.nodes(node).qemu(vmid).monitor.post(command="info blockstats")
return res if isinstance(res, str) else str(res or "")
except Exception as e:
logger.debug(f"blockstats unavailable for VM {vmid} on {node}: {e}")
return ""
def get_node_netstat(self, node: str) -> list:
"""Per-tap/veth interface counters for the node ([] on failure)."""
if not self.proxmox:
return []
try:
res = self.proxmox.nodes(node).netstat.get()
return res if isinstance(res, list) else []
except Exception as e:
logger.debug(f"netstat unavailable for node {node}: {e}")
return []
def get_container_interfaces(self, node: str, vmid: int) -> List[Dict]:
"""
Получить сетевые интерфейсы контейнера

View File

@@ -606,4 +606,8 @@ def start_metrics_broadcaster(db_factory) -> None:
asyncio.create_task(server_metrics_loop(db_factory), name="metrics:servers")
asyncio.create_task(instance_metrics_loop(db_factory), name="metrics:instances")
asyncio.create_task(instances_list_metrics_loop(db_factory), name="metrics:instances_list")
# Import inside function to avoid circular imports
from .metrics_collector import metrics_collector_loop, metrics_retention_loop
asyncio.create_task(metrics_collector_loop(db_factory), name="metrics:collector")
asyncio.create_task(metrics_retention_loop(db_factory), name="metrics:retention")
logger.info("[metrics_broadcaster] Metrics broadcasting tasks started")

View File

@@ -0,0 +1,148 @@
"""Background collector that persists instance telemetry to our own tables."""
from __future__ import annotations
import asyncio
import time
from datetime import datetime, timezone
from loguru import logger
from .metrics_history import (
compute_rate, parse_blockstats, parse_netstat, aggregate_fsinfo,
)
def _status(client, inst_type, node, vmid):
if inst_type == "qemu":
return client.get_vm_status(node, vmid)
return client.get_container_status(node, vmid)
def collect_once(db, items, clients, prev, _now=None):
"""One collection tick. Mutates `prev` to carry counters between ticks."""
from ..models import InstanceMetric, InstanceNicMetric
now = _now if _now is not None else time.monotonic()
ts = datetime.now(tz=timezone.utc)
# netstat is one call per (server_id, node); cache per tick.
netstat_cache: dict[tuple, list] = {}
# Track which prev keys are touched this tick so stale entries can be evicted.
seen: set = set()
for it in items:
if it.get("status") != "running":
continue
sid, vmid = it["server_id"], it["vmid"]
inst_type, node = it.get("type"), it.get("node")
client = clients.get(sid)
if client is None or not node:
continue
try:
status = _status(client, inst_type, node, vmid)
if not status:
continue
# disk fill
if inst_type == "qemu":
used, total = aggregate_fsinfo(client.get_vm_fsinfo(node, vmid))
else:
used = int(status.get("disk", 0) or 0)
total = int(status.get("maxdisk", 0) or 0)
# cumulative counters → rates
key = f"{sid}:{vmid}"
cur = {
"netin": float(status.get("netin", 0) or 0),
"netout": float(status.get("netout", 0) or 0),
"diskread": float(status.get("diskread", 0) or 0),
"diskwrite": float(status.get("diskwrite", 0) or 0),
}
if inst_type == "qemu":
rd_ops, wr_ops = parse_blockstats(client.get_vm_blockstats(node, vmid))
cur["rd_ops"], cur["wr_ops"] = float(rd_ops), float(wr_ops)
pkey = prev.get(key)
dt = (now - pkey[0]) if pkey else 0
pc = pkey[1] if pkey else {}
def rate(name):
return compute_rate(pc.get(name), cur.get(name), dt)
db.add(InstanceMetric(
server_id=sid, vmid=vmid, vm_type=inst_type, ts=ts,
cpu=status.get("cpu"), mem=status.get("mem"), maxmem=status.get("maxmem"),
disk_used=used, disk_total=total,
netin_rate=rate("netin"), netout_rate=rate("netout"),
diskread_rate=rate("diskread"), diskwrite_rate=rate("diskwrite"),
iops_read=rate("rd_ops") if inst_type == "qemu" else None,
iops_write=rate("wr_ops") if inst_type == "qemu" else None,
))
prev[key] = (now, cur)
seen.add(key)
# per-NIC
ck = (sid, node)
if ck not in netstat_cache:
netstat_cache[ck] = client.get_node_netstat(node)
per_dev = parse_netstat(netstat_cache[ck], vmid)
for dev, (din, dout) in per_dev.items():
nkey = f"{sid}:{vmid}:{dev}"
npkey = prev.get(nkey)
ndt = (now - npkey[0]) if npkey else 0
npc = npkey[1] if npkey else {}
db.add(InstanceNicMetric(
server_id=sid, vmid=vmid, ts=ts, dev=dev,
in_rate=compute_rate(npc.get("in"), float(din), ndt),
out_rate=compute_rate(npc.get("out"), float(dout), ndt),
))
prev[nkey] = (now, {"in": float(din), "out": float(dout)})
seen.add(nkey)
except Exception as e:
logger.debug(f"[metrics_collector] {sid}:{vmid} error: {e}")
# Evict stale keys from prev (VMs/NICs not seen this tick).
for k in list(prev):
if k not in seen:
del prev[k]
async def metrics_collector_loop(db_factory) -> None:
from ..config import settings
from .metrics_broadcaster import _enumerate_instances_for_list
prev: dict = {}
while True:
start = time.monotonic()
try:
items, clients = await asyncio.to_thread(_enumerate_instances_for_list, db_factory)
db = db_factory()
try:
await asyncio.to_thread(collect_once, db, items, clients, prev)
db.commit()
finally:
db.close()
except asyncio.CancelledError:
break
except Exception as e:
logger.warning(f"[metrics_collector] tick error: {e}")
await asyncio.sleep(max(1.0, settings.METRICS_COLLECT_INTERVAL - (time.monotonic() - start)))
async def metrics_retention_loop(db_factory) -> None:
from ..config import settings
from ..models import InstanceMetric, InstanceNicMetric
from datetime import timedelta
while True:
try:
cutoff = datetime.now(tz=timezone.utc) - timedelta(days=settings.METRICS_RETENTION_DAYS)
db = db_factory()
try:
db.query(InstanceMetric).filter(InstanceMetric.ts < cutoff).delete()
db.query(InstanceNicMetric).filter(InstanceNicMetric.ts < cutoff).delete()
db.commit()
finally:
db.close()
except asyncio.CancelledError:
break
except Exception as e:
logger.warning(f"[metrics_collector] retention error: {e}")
await asyncio.sleep(3600)

View File

@@ -0,0 +1,163 @@
"""Pure helpers + read query for instance metric history."""
from __future__ import annotations
import re
from datetime import timezone as _tz
_RD_OPS = re.compile(r"rd_operations=(\d+)")
_WR_OPS = re.compile(r"wr_operations=(\d+)")
def compute_rate(prev, cur, dt):
"""Per-second rate from cumulative counters. None if unknowable; 0 on reset."""
if prev is None or cur is None or not dt or dt <= 0:
return None
delta = float(cur) - float(prev)
if delta < 0:
return 0.0
return delta / dt
def parse_blockstats(text: str):
"""Sum rd_operations / wr_operations across all block devices in HMP output."""
rd = sum(int(m) for m in _RD_OPS.findall(text or ""))
wr = sum(int(m) for m in _WR_OPS.findall(text or ""))
return rd, wr
def parse_netstat(rows, vmid: int):
"""Map dev -> (in_bytes, out_bytes) for the given vmid from /nodes/{node}/netstat."""
out: dict[str, tuple[int, int]] = {}
for r in rows or []:
try:
if int(r.get("vmid")) != int(vmid):
continue
except (TypeError, ValueError):
continue
dev = r.get("dev")
if not dev:
continue
out[dev] = (int(float(r.get("in", 0) or 0)), int(float(r.get("out", 0) or 0)))
return out
def aggregate_fsinfo(disks):
"""Sum used/total bytes across guest filesystems."""
used = sum(int(d.get("used", 0) or 0) for d in disks or [])
total = sum(int(d.get("total", 0) or 0) for d in disks or [])
return used, total
_TIMEFRAME_SECONDS = {
"hour": 3600,
"day": 86400,
"week": 7 * 86400,
"month": 30 * 86400,
}
def timeframe_to_range(timeframe, now_ts):
"""Convert timeframe string to (from_ts, to_ts) tuple."""
span = _TIMEFRAME_SECONDS.get(timeframe, 3600)
return now_ts - span, now_ts
def pick_bucket_seconds(from_ts, to_ts, target_points=150, floor=15):
"""Calculate bucket size in seconds for time-series aggregation."""
span = max(1, to_ts - from_ts)
ideal = span // max(1, target_points)
return max(floor, int(ideal))
def aggregate_series(rows, from_ts, bucket, fields):
"""Aggregate time-series data into buckets, averaging numeric fields."""
if not rows:
return []
buckets: dict[int, dict] = {}
order: list[int] = []
for r in rows:
b = from_ts + ((int(r["time"]) - from_ts) // bucket) * bucket
if b not in buckets:
buckets[b] = {f: [] for f in fields}
order.append(b)
for f in fields:
v = r.get(f)
if v is not None:
buckets[b][f].append(float(v))
out = []
for b in order:
point = {"time": b}
for f in fields:
vals = buckets[b][f]
point[f] = (sum(vals) / len(vals)) if vals else None
out.append(point)
return out
_AGG_FIELDS = ["cpu", "mem", "maxmem", "netin", "netout",
"diskread", "diskwrite", "diskpct", "iops_read", "iops_write"]
def _to_ts(dt):
if dt.tzinfo is None:
dt = dt.replace(tzinfo=_tz.utc)
return int(dt.timestamp())
def query_instance_metrics(db, server_id, vmid, from_ts, to_ts, nic="all"):
from ..models import InstanceMetric, InstanceNicMetric
from datetime import datetime
start = datetime.fromtimestamp(from_ts, tz=_tz.utc)
end = datetime.fromtimestamp(to_ts, tz=_tz.utc)
base = (db.query(InstanceMetric)
.filter(InstanceMetric.server_id == server_id,
InstanceMetric.vmid == vmid,
InstanceMetric.ts >= start, InstanceMetric.ts <= end)
.order_by(InstanceMetric.ts.asc()).all())
# Distinct NICs seen in the window (for the selector).
if nic == "all":
# Only fetch distinct dev names — avoids loading all NIC rows.
dev_rows = (db.query(InstanceNicMetric.dev)
.filter(InstanceNicMetric.server_id == server_id,
InstanceNicMetric.vmid == vmid,
InstanceNicMetric.ts >= start, InstanceNicMetric.ts <= end)
.distinct().all())
nics = sorted({r.dev for r in dev_rows})
nic_by_ts: dict[int, tuple] = {}
else:
# Load full NIC rows for the requested dev to build per-ts override map.
nic_rows = (db.query(InstanceNicMetric)
.filter(InstanceNicMetric.server_id == server_id,
InstanceNicMetric.vmid == vmid,
InstanceNicMetric.ts >= start, InstanceNicMetric.ts <= end)
.order_by(InstanceNicMetric.ts.asc()).all())
nics = sorted({r.dev for r in nic_rows})
nic_by_ts = {}
for r in nic_rows:
if r.dev == nic:
nic_by_ts[_to_ts(r.ts)] = (r.in_rate, r.out_rate)
rows = []
for m in base:
ts = _to_ts(m.ts)
netin, netout = m.netin_rate, m.netout_rate
if nic != "all":
ov = nic_by_ts.get(ts)
netin, netout = (ov if ov else (None, None))
diskpct = (m.disk_used / m.disk_total * 100.0) if (m.disk_total or 0) > 0 else None
rows.append({
"time": ts,
"cpu": (m.cpu * 100.0) if m.cpu is not None else None,
"mem": m.mem, "maxmem": m.maxmem,
"netin": netin, "netout": netout,
"diskread": m.diskread_rate, "diskwrite": m.diskwrite_rate,
"diskpct": diskpct,
"iops_read": m.iops_read, "iops_write": m.iops_write,
})
bucket = pick_bucket_seconds(from_ts, to_ts)
data = aggregate_series(rows, from_ts, bucket, _AGG_FIELDS)
return {"data": data, "from": from_ts, "to": to_ts, "meta": {"nics": nics}}

View File

@@ -0,0 +1,74 @@
"""Tests for /metrics read endpoints (Task 6)."""
from datetime import datetime, timezone
import time
def test_metrics_endpoint_returns_series(client, admin_headers, db_session):
"""Seeds InstanceMetric rows then reads them via the API.
The endpoint reads only the DB — no Proxmox call required."""
from app.models import InstanceMetric
now = int(time.time())
for i, cpu in enumerate((0.1, 0.3)):
db_session.add(InstanceMetric(
server_id=1, vmid=114, vm_type="qemu",
ts=datetime.fromtimestamp(now - 10 + i, tz=timezone.utc),
cpu=cpu, mem=100, maxmem=200, disk_used=10, disk_total=40,
netin_rate=1.0, netout_rate=2.0, diskread_rate=0.0, diskwrite_rate=0.0,
iops_read=5.0, iops_write=1.0))
db_session.commit()
r = client.get(
"/proxmox/api/1/vm/114/metrics?timeframe=hour&node=prod",
headers=admin_headers,
)
assert r.status_code == 200
body = r.json()
assert "data" in body and "meta" in body
assert body["data"], "expected at least one bucket"
assert "nics" in body["meta"]
def test_container_metrics_endpoint_returns_series(client, admin_headers, db_session):
"""Same flow for container endpoint."""
from app.models import InstanceMetric
now = int(time.time())
db_session.add(InstanceMetric(
server_id=1, vmid=200, vm_type="lxc",
ts=datetime.fromtimestamp(now - 5, tz=timezone.utc),
cpu=0.2, mem=512, maxmem=1024, disk_used=20, disk_total=80,
netin_rate=3.0, netout_rate=1.5, diskread_rate=0.0, diskwrite_rate=0.0,
iops_read=2.0, iops_write=0.5))
db_session.commit()
r = client.get(
"/proxmox/api/1/container/200/metrics?timeframe=hour&node=prod",
headers=admin_headers,
)
assert r.status_code == 200
body = r.json()
assert "data" in body and "meta" in body
assert "nics" in body["meta"]
def test_metrics_explicit_from_to(client, admin_headers, db_session):
"""Explicit from_ts/to_ts take precedence over timeframe."""
from app.models import InstanceMetric
now = int(time.time())
db_session.add(InstanceMetric(
server_id=1, vmid=300, vm_type="qemu",
ts=datetime.fromtimestamp(now - 30, tz=timezone.utc),
cpu=0.5, mem=256, maxmem=512, disk_used=5, disk_total=20,
netin_rate=0.5, netout_rate=0.5, diskread_rate=0.0, diskwrite_rate=0.0,
iops_read=1.0, iops_write=1.0))
db_session.commit()
r = client.get(
f"/proxmox/api/1/vm/300/metrics?from_ts={now - 60}&to_ts={now}",
headers=admin_headers,
)
assert r.status_code == 200
body = r.json()
assert "data" in body and "meta" in body

View File

@@ -0,0 +1,37 @@
from unittest.mock import MagicMock
from app.proxmox.client import ProxmoxClient
def _client_with_mock(proxmox):
c = ProxmoxClient.__new__(ProxmoxClient)
c.proxmox = proxmox
return c
def test_get_vm_blockstats_returns_text():
px = MagicMock()
px.nodes.return_value.qemu.return_value.monitor.post.return_value = "drive-scsi0: rd_operations=5 wr_operations=2"
c = _client_with_mock(px)
assert "rd_operations=5" in c.get_vm_blockstats("prod", 114)
px.nodes.return_value.qemu.return_value.monitor.post.assert_called_once_with(command="info blockstats")
def test_get_vm_blockstats_swallows_errors():
px = MagicMock()
px.nodes.return_value.qemu.return_value.monitor.post.side_effect = RuntimeError("nope")
c = _client_with_mock(px)
assert c.get_vm_blockstats("prod", 114) == ""
def test_get_node_netstat_returns_rows():
px = MagicMock()
px.nodes.return_value.netstat.get.return_value = [{"dev": "tap114i0", "vmid": "114", "in": "1", "out": "2"}]
c = _client_with_mock(px)
assert c.get_node_netstat("prod")[0]["dev"] == "tap114i0"
def test_get_node_netstat_swallows_errors():
px = MagicMock()
px.nodes.return_value.netstat.get.side_effect = RuntimeError("nope")
c = _client_with_mock(px)
assert c.get_node_netstat("prod") == []

View File

@@ -0,0 +1,83 @@
"""Tests for metrics_collector.collect_once (unit-testable core)."""
from unittest.mock import MagicMock
from app.services.metrics_collector import collect_once
def _client():
c = MagicMock()
# status/current
c.get_vm_status.return_value = {
"cpu": 0.5, "mem": 100, "maxmem": 200,
"netin": 1000, "netout": 2000, "diskread": 0, "diskwrite": 500,
}
c.get_vm_fsinfo.return_value = [{"used": 10, "total": 40}]
c.get_vm_blockstats.return_value = "drive-scsi0: rd_operations=100 wr_operations=50"
c.get_node_netstat.return_value = [
{"dev": "tap114i0", "vmid": "114", "in": "1000", "out": "2000"}]
return c
def test_collect_once_first_tick_no_rates(db_session):
from app.models import InstanceMetric
items = [{"server_id": 1, "vmid": 114, "type": "qemu", "node": "prod", "status": "running"}]
clients = {1: _client()}
prev = {}
collect_once(db_session, items, clients, prev, _now=100.0)
db_session.commit()
m = db_session.query(InstanceMetric).one()
assert m.cpu == 0.5 and m.disk_used == 10 and m.disk_total == 40
assert m.netin_rate is None # no previous counter yet
assert m.iops_read is None
def test_collect_once_second_tick_computes_rates(db_session):
from app.models import InstanceMetric, InstanceNicMetric
items = [{"server_id": 1, "vmid": 114, "type": "qemu", "node": "prod", "status": "running"}]
c = _client()
clients = {1: c}
prev = {}
collect_once(db_session, items, clients, prev, _now=100.0)
# advance counters by 10 over 2 seconds
c.get_vm_status.return_value = {"cpu": 0.5, "mem": 100, "maxmem": 200,
"netin": 1020, "netout": 2040, "diskread": 0, "diskwrite": 500}
c.get_vm_blockstats.return_value = "drive-scsi0: rd_operations=110 wr_operations=56"
c.get_node_netstat.return_value = [{"dev": "tap114i0", "vmid": "114", "in": "1020", "out": "2040"}]
collect_once(db_session, items, clients, prev, _now=102.0)
db_session.commit()
rows = db_session.query(InstanceMetric).order_by(InstanceMetric.id).all()
assert rows[1].netin_rate == 10.0 # (1020-1000)/2
assert rows[1].iops_read == 5.0 # (110-100)/2
nic = db_session.query(InstanceNicMetric).order_by(InstanceNicMetric.id).all()[-1]
assert nic.in_rate == 10.0 # (1020-1000)/2
def test_collect_once_evicts_stale_prev(db_session):
"""Keys for VMs absent from the current tick must be pruned from prev."""
def _client_for(vmid_str):
c = MagicMock()
c.get_vm_status.return_value = {
"cpu": 0.1, "mem": 50, "maxmem": 100,
"netin": 0, "netout": 0, "diskread": 0, "diskwrite": 0,
}
c.get_vm_fsinfo.return_value = [{"used": 5, "total": 20}]
c.get_vm_blockstats.return_value = ""
c.get_node_netstat.return_value = [
{"dev": f"tap{vmid_str}i0", "vmid": vmid_str, "in": "0", "out": "0"}
]
return c
prev = {}
# Tick 1: only vmid 114 running
items_114 = [{"server_id": 1, "vmid": 114, "type": "qemu", "node": "prod", "status": "running"}]
clients_114 = {1: _client_for("114")}
collect_once(db_session, items_114, clients_114, prev, _now=100.0)
# After tick 1, prev must contain 114 keys
assert any("114" in k for k in prev), "Expected 114 keys in prev after tick 1"
# Tick 2: only vmid 115 running (114 is gone)
items_115 = [{"server_id": 1, "vmid": 115, "type": "qemu", "node": "prod", "status": "running"}]
clients_115 = {1: _client_for("115")}
collect_once(db_session, items_115, clients_115, prev, _now=102.0)
# 114 keys must be evicted; 115 keys must be present
assert not any("114" in k for k in prev), "Expected 114 keys evicted from prev after tick 2"
assert any("115" in k for k in prev), "Expected 115 keys in prev after tick 2"

View File

@@ -0,0 +1,71 @@
from app.services import metrics_history as mh
def test_compute_rate_normal():
assert mh.compute_rate(100, 160, 2.0) == 30.0
def test_compute_rate_no_prev():
assert mh.compute_rate(None, 160, 2.0) is None
def test_compute_rate_counter_reset():
assert mh.compute_rate(500, 10, 2.0) == 0.0
def test_compute_rate_zero_dt():
assert mh.compute_rate(100, 200, 0) is None
def test_parse_blockstats_sums_devices():
text = (
"drive-scsi0: rd_bytes=10 wr_bytes=20 rd_operations=100 wr_operations=50 flush_operations=1\n"
"drive-scsi1: rd_bytes=5 wr_bytes=5 rd_operations=7 wr_operations=3 flush_operations=0\n"
)
assert mh.parse_blockstats(text) == (107, 53)
def test_parse_blockstats_empty():
assert mh.parse_blockstats("") == (0, 0)
def test_parse_netstat_filters_by_vmid():
rows = [
{"dev": "tap114i0", "vmid": "114", "in": "1000", "out": "2000"},
{"dev": "tap114i1", "vmid": "114", "in": "10", "out": "20"},
{"dev": "tap200i0", "vmid": "200", "in": "9", "out": "9"},
]
out = mh.parse_netstat(rows, 114)
assert out == {"tap114i0": (1000, 2000), "tap114i1": (10, 20)}
def test_aggregate_fsinfo_sums():
disks = [{"used": 10, "total": 40}, {"used": 5, "total": 20}]
assert mh.aggregate_fsinfo(disks) == (15, 60)
def test_aggregate_fsinfo_empty():
assert mh.aggregate_fsinfo([]) == (0, 0)
def test_timeframe_to_range_hour():
assert mh.timeframe_to_range("hour", 10_000) == (10_000 - 3600, 10_000)
def test_timeframe_to_range_month():
assert mh.timeframe_to_range("month", 10_000_000) == (10_000_000 - 30 * 86400, 10_000_000)
def test_timeframe_to_range_unknown_defaults_hour():
assert mh.timeframe_to_range("nonsense", 10_000) == (10_000 - 3600, 10_000)
def test_pick_bucket_respects_floor():
# 1h window, target 150 -> 24s ideal, but floor 15 keeps >=15
assert mh.pick_bucket_seconds(0, 3600, target_points=150, floor=15) == 24
def test_pick_bucket_floor_applied():
assert mh.pick_bucket_seconds(0, 300, target_points=150, floor=15) == 15
def test_aggregate_series_averages_buckets():
rows = [
{"time": 0, "cpu": 10.0, "mem": 100},
{"time": 5, "cpu": 20.0, "mem": 200},
{"time": 20, "cpu": 40.0, "mem": None},
]
out = mh.aggregate_series(rows, from_ts=0, bucket=10, fields=["cpu", "mem"])
# bucket 0: times 0,5 -> cpu 15, mem 150 ; bucket 20: time 20 -> cpu 40, mem None
assert out[0] == {"time": 0, "cpu": 15.0, "mem": 150.0}
assert out[-1] == {"time": 20, "cpu": 40.0, "mem": None}
def test_aggregate_series_empty():
assert mh.aggregate_series([], 0, 10, ["cpu"]) == []

View File

@@ -0,0 +1,20 @@
from datetime import datetime, timezone
def test_instance_metric_roundtrip(db_session):
from app.models import InstanceMetric, InstanceNicMetric
ts = datetime(2026, 6, 21, 12, 0, tzinfo=timezone.utc)
db_session.add(InstanceMetric(
server_id=1, vmid=114, vm_type="qemu", ts=ts,
cpu=0.25, mem=100, maxmem=200, disk_used=10, disk_total=40,
netin_rate=1.0, netout_rate=2.0, diskread_rate=0.0, diskwrite_rate=3.0,
iops_read=12.0, iops_write=4.0,
))
db_session.add(InstanceNicMetric(
server_id=1, vmid=114, ts=ts, dev="tap114i0", in_rate=5.0, out_rate=6.0))
db_session.commit()
m = db_session.query(InstanceMetric).one()
assert m.vmid == 114
assert m.iops_read == 12.0
n = db_session.query(InstanceNicMetric).one()
assert n.dev == "tap114i0"
assert n.out_rate == 6.0

View File

@@ -0,0 +1,45 @@
# backend/tests/test_metrics_query.py
from datetime import datetime, timezone
from app.services.metrics_history import query_instance_metrics
def _row(db, InstanceMetric, ts, **kw):
db.add(InstanceMetric(server_id=1, vmid=114, vm_type="qemu",
ts=datetime.fromtimestamp(ts, tz=timezone.utc), **kw))
def test_query_aggregate_all_nic(db_session):
from app.models import InstanceMetric
_row(db_session, InstanceMetric, 1000, cpu=0.10, mem=100, maxmem=200,
disk_used=10, disk_total=40, netin_rate=1.0, netout_rate=2.0,
diskread_rate=0.0, diskwrite_rate=0.0, iops_read=5.0, iops_write=1.0)
_row(db_session, InstanceMetric, 1005, cpu=0.30, mem=140, maxmem=200,
disk_used=20, disk_total=40, netin_rate=3.0, netout_rate=4.0,
diskread_rate=0.0, diskwrite_rate=0.0, iops_read=7.0, iops_write=3.0)
db_session.commit()
out = query_instance_metrics(db_session, 1, 114, from_ts=1000, to_ts=1010, nic="all")
assert out["from"] == 1000 and out["to"] == 1010
pt = out["data"][0]
assert pt["cpu"] == 20.0 # avg(10%,30%)
assert pt["netin"] == 2.0 # avg(1,3)
assert pt["diskpct"] == 37.5 # avg(25%,50%)
assert pt["iops_read"] == 6.0
def test_query_specific_nic_overrides_net(db_session):
from app.models import InstanceMetric, InstanceNicMetric
_row(db_session, InstanceMetric, 1000, cpu=0.0, mem=0, maxmem=1,
disk_used=0, disk_total=1, netin_rate=99.0, netout_rate=99.0,
diskread_rate=0.0, diskwrite_rate=0.0, iops_read=0.0, iops_write=0.0)
db_session.add(InstanceNicMetric(server_id=1, vmid=114,
ts=datetime.fromtimestamp(1000, tz=timezone.utc), dev="tap114i0",
in_rate=7.0, out_rate=8.0))
db_session.commit()
out = query_instance_metrics(db_session, 1, 114, 1000, 1010, nic="tap114i0")
assert out["meta"]["nics"] == ["tap114i0"]
assert out["data"][0]["netin"] == 7.0 and out["data"][0]["netout"] == 8.0
def test_query_empty(db_session):
out = query_instance_metrics(db_session, 1, 999, 0, 10, nic="all")
assert out["data"] == [] and out["meta"]["nics"] == []