2 Commits

Author SHA1 Message Date
972d54d1f1 fix(ipam): keep extra addresses across a guest reboot
Some checks failed
Backend Tests / pytest (Python 3.12) (push) Failing after 1m44s
Deploy to Production / Trigger production update (push) Failing after 11s
SonarCloud / SonarCloud Analysis (push) Failing after 3m0s
Testing on a live AlmaLinux container showed the alias disappearing after
a restart even though it was reported as applied: Proxmox regenerates the
guest network configuration on every start (pve-container rewrites
/etc/network/interfaces, ifcfg or the NM keyfile from netN), wiping any
address added to the native stack.

    before reboot: address1=10.10.10.103/24 address2=10.10.10.106/24
    after reboot:  addresses=10.10.10.103/24     (file recreated by PVE)

Install a oneshot systemd unit that re-adds the recorded addresses after
network-online.target — /etc/systemd is not touched by Proxmox. The native
branches stay, so persist is now reported as e.g. "nm+systemd"; the unit
and its helper are removed once the last address is released.

Also fixed while testing on the two test instances:
- the alias script is ASCII-only now: it travels through the Proxmox API
  and its marker ends up in guest system configs
- a composite persist marker no longer falls through the parser, which
  used to report a successful apply as runtime_only
- allocate_ip marks the first address of a guest as primary on its own,
  so allocations coming from the creation wizard are flagged correctly
2026-08-22 01:36:10 +05:00
6ac505e481 fix(proxmox): send guest-agent scripts as bytes
Proxmox base64-encodes the input-data of an agent exec in Perl, which dies
with "Wide character in subroutine entry" as soon as the payload contains
non-ASCII — a Cyrillic comment in a script is enough, so running any such
script on a VM failed with a 500.

Hand the same UTF-8 bytes over as latin-1: the JSON carries plain bytes,
Perl is happy and the guest still receives valid UTF-8.
2026-08-22 01:35:56 +05:00
8 changed files with 291 additions and 44 deletions

View File

@@ -283,7 +283,7 @@ class IPAMService:
status: str = "allocated",
allocated_by: Optional[str] = None,
notes: Optional[str] = None,
is_primary: bool = False,
is_primary: Optional[bool] = None,
assignment_kind: str = "primary",
target_interface: Optional[str] = None,
apply_status: Optional[str] = None
@@ -344,6 +344,16 @@ class IPAMService:
# Build FQDN if not provided
if not fqdn and hostname and network.dns_domain:
fqdn = f"{hostname}.{network.dns_domain}"
# Первый адрес гостя становится основным сам: адрес выдаётся из шести
# разных мест (мастер создания, шаблоны, ISO, appstore, ручная выдача),
# и требовать флаг от каждого — значит рано или поздно его забыть.
if is_primary is None:
is_primary = (
assignment_kind == "primary"
and proxmox_vmid is not None
and not self._has_primary(proxmox_server_id, proxmox_vmid)
)
# Create allocation
allocation = IPAMAllocation(
@@ -785,6 +795,16 @@ class IPAMService:
# ==================== Sync & Scan Operations ====================
def _has_primary(self, server_id: Optional[int], vmid: Optional[int]) -> bool:
"""Есть ли у гостя адрес, помеченный основным."""
if vmid is None:
return False
return self.db.query(IPAMAllocation.id).filter(
IPAMAllocation.proxmox_server_id == server_id,
IPAMAllocation.proxmox_vmid == vmid,
IPAMAllocation.is_primary.is_(True)
).first() is not None
def _guest_exists(self, server_id: Optional[int], vmid: Optional[int]) -> bool:
"""Существует ли ещё гость, за которым числится аллокация."""
if server_id is None or vmid is None:
@@ -862,13 +882,9 @@ class IPAMService:
self.db.commit()
return existing, None
# Create new allocation. Основным делаем первый адрес гостя —
# именно он попадёт в колонку IP списка инстансов.
has_primary = self.db.query(IPAMAllocation).filter(
IPAMAllocation.proxmox_server_id == proxmox_server_id,
IPAMAllocation.proxmox_vmid == vmid,
IPAMAllocation.is_primary.is_(True)
).first() is not None
# Create new allocation. Основным первый адрес гостя пометит сам
# allocate_ip — именно он попадёт в колонку IP списка инстансов.
has_primary = self._has_primary(proxmox_server_id, vmid)
return self.allocate_ip(
ip_address=ip_address,

View File

@@ -1852,9 +1852,16 @@ class ProxmoxClient(VmMixin, LxcMixin, ClusterMixin, StorageMixin, NetworkMixin,
#
# Proxmox API сам кодирует input-data в base64 перед передачей
# guest agent'у, поэтому передаем сырое содержимое скрипта.
# Proxmox кодирует input-data в base64 силами Perl, а тот падает
# с "Wide character in subroutine entry" на любой строке с
# не-ASCII — например на комментариях кириллицей. Отдаём те же
# UTF-8 байты, представленные как latin-1: в JSON уходят байты,
# гость получает корректный UTF-8.
payload = script_content.encode('utf-8').decode('latin-1')
exec_result = self.proxmox.nodes(node).qemu(vmid).agent.exec.post(
command=[interpreter],
**{'input-data': script_content}
**{'input-data': payload}
)
if 'pid' not in exec_result:

View File

@@ -1,20 +1,20 @@
#!/bin/sh
# Привести набор дополнительных IPv4-адресов (alias) интерфейса к заданному.
# Bring the set of extra IPv4 aliases of an interface to the requested one.
#
# Параметры приходят переменными окружения (их подставляет
# script_engine.render_params, значения не конкатенируются в команды):
# IFACE — интерфейс гостя, напр. eth0
# ADDRESSES — список CIDR через пробел; пустая строка = снять все алиасы
# PVEM_ROOT — префикс путей; в госте всегда пуст, задаётся только тестами
# Parameters arrive as environment variables (rendered by
# script_engine.render_params, values are never concatenated into commands):
# IFACE - guest interface, e.g. eth0
# ADDRESSES - space separated CIDR list; empty string removes every alias
# PVEM_ROOT - path prefix; always empty in a guest, set by tests only
#
# Скрипт идемпотентен: он синхронизирует состояние, а не «добавляет».
# Под управлением находятся только адреса из своего state-файла — адреса,
# настроенные в госте вручную, не трогаются.
# The script is idempotent: it synchronises state instead of appending.
# Only addresses listed in its own state file are managed, so anything
# configured inside the guest by hand stays untouched.
#
# Сеть намеренно НЕ перезапускается: адрес поднимается через `ip addr add`
# здесь и сейчас, а конфиг ОС правится только ради переживания ребута.
# Никаких `netplan apply` / `ifup` / `nmcli con up` — они рвут связь с
# работающим гостем.
# Networking is deliberately NOT restarted: the address goes up via
# `ip addr add` right away, and the OS config is edited only so that the
# address survives a reboot. No `netplan apply` / `ifup` / `nmcli con up`:
# they would cut the connection to a running guest.
[ -n "$IFACE" ] || { echo "missing IFACE"; exit 2; }
@@ -27,7 +27,7 @@ mkdir -p "$STATE_DIR" 2>/dev/null || true
PREV=""
[ -f "$STATE" ] && PREV=$(cat "$STATE")
# 1. Снять адреса, ушедшие из списка (только наши, из state-файла)
# 1. Remove addresses dropped from the list (ours only, per state file)
for old in $PREV; do
keep=0
for new in $ADDRESSES; do
@@ -37,7 +37,7 @@ for old in $PREV; do
ip addr del "$old" dev "$IFACE" 2>/dev/null || true
done
# 2. Поднять недостающие
# 2. Bring up the missing ones
for a in $ADDRESSES; do
addr=${a%%/*}
if ip -4 addr show dev "$IFACE" 2>/dev/null | grep -qw "$addr"; then
@@ -51,10 +51,10 @@ done
printf '%s\n' "$ADDRESSES" > "$STATE"
# 3. Закрепить в конфиге ОС — первый распознанный стек выигрывает.
# Ветки идут последовательно, а не через elif: наличие инструмента ещё не
# значит, что он управляет этим интерфейсом (типичный случай — NetworkManager
# в LXC), и тогда управление должно уходить дальше по списку.
# 3. Persist in the OS config - the first recognised stack wins.
# Branches run in sequence rather than as elif: having a tool installed
# does not mean it manages this interface (NetworkManager inside an LXC
# is the usual case), and then control must fall through to the next one.
persist=none
MARKER="# managed by PVEmanager"
@@ -68,10 +68,10 @@ SYSCONFIG_DIR="$ROOT/etc/sysconfig/network-scripts"
# --- ifupdown (Debian/Ubuntu) ---
if [ "$persist" = none ] && [ -d "$IFUP_DIR" ] && [ -f "$IFACES_FILE" ] && \
command -v ifup >/dev/null 2>&1; then
# Шаблоны LXC от Proxmox создают interfaces.d, но не подключают его из
# /etc/network/interfaces — без source-строки файлы оттуда не читаются,
# поэтому добавляем её (идемпотентно). Список интерфейсов не трогаем:
# его перезаписывает Proxmox при правке NIC.
# Proxmox LXC templates create interfaces.d but never source it from
# /etc/network/interfaces, so files there are ignored; add the source
# line (idempotently). The interfaces file itself is left alone:
# Proxmox rewrites it whenever the NIC is edited.
if ! grep -qE '^[[:space:]]*(source|source-directory)' "$IFACES_FILE"; then
echo "source /etc/network/interfaces.d/*" >> "$IFACES_FILE"
fi
@@ -79,7 +79,7 @@ if [ "$persist" = none ] && [ -d "$IFUP_DIR" ] && [ -f "$IFACES_FILE" ] && \
rm -f "$IFUP_FILE"
else
{
echo "$MARKER — не редактировать вручную"
echo "$MARKER - do not edit by hand"
n=0
for a in $ADDRESSES; do
n=$((n + 1))
@@ -92,13 +92,13 @@ if [ "$persist" = none ] && [ -d "$IFUP_DIR" ] && [ -f "$IFACES_FILE" ] && \
persist=ifupdown
fi
# --- netplan (Ubuntu cloud-образы) ---
# --- netplan (Ubuntu cloud images) ---
if [ "$persist" = none ] && [ -d "$NETPLAN_DIR" ]; then
if [ -z "$ADDRESSES" ]; then
rm -f "$NETPLAN_FILE"
else
{
echo "$MARKER — не редактировать вручную"
echo "$MARKER - do not edit by hand"
echo "network:"
echo " version: 2"
echo " ethernets:"
@@ -113,9 +113,9 @@ if [ "$persist" = none ] && [ -d "$NETPLAN_DIR" ]; then
persist=netplan
fi
# --- NetworkManager (RHEL-based VM, современные десктопные образы) ---
# --- NetworkManager (RHEL-based VMs, modern desktop images) ---
if [ "$persist" = none ] && command -v nmcli >/dev/null 2>&1; then
# Пустой ответ = интерфейсом NM не управляет; тогда идём дальше.
# Empty answer means NM does not manage the interface - fall through.
CON=$(nmcli -g GENERAL.CONNECTION device show "$IFACE" 2>/dev/null)
if [ -n "$CON" ]; then
for old in $PREV; do
@@ -129,10 +129,10 @@ if [ "$persist" = none ] && command -v nmcli >/dev/null 2>&1; then
fi
# --- sysconfig / network-scripts (RHEL, CentOS, Rocky, Alma) ---
# Шаблоны LXC этих дистрибутивов Proxmox настраивает именно так
# (pve-container, модуль Redhat.pm), а NetworkManager в них обычно не стоит.
# Proxmox configures LXC templates of these distros exactly this way
# (pve-container, Redhat.pm), and NetworkManager is usually absent there.
if [ "$persist" = none ] && [ -d "$SYSCONFIG_DIR" ]; then
# Сносим прежние свои ifcfg-алиасы; чужие файлы без маркера не трогаем.
# Drop our previous ifcfg aliases; files without the marker stay.
for f in "$SYSCONFIG_DIR/ifcfg-$IFACE":*; do
[ -f "$f" ] || continue
if head -n 1 "$f" 2>/dev/null | grep -q "$MARKER"; then
@@ -146,7 +146,7 @@ if [ "$persist" = none ] && [ -d "$SYSCONFIG_DIR" ]; then
addr=${a%%/*}
plen=${a##*/}
{
echo "$MARKER — не редактировать вручную"
echo "$MARKER - do not edit by hand"
echo "DEVICE=$IFACE:$n"
echo "NAME=$IFACE:$n"
echo "ONPARENT=yes"
@@ -159,5 +159,62 @@ if [ "$persist" = none ] && [ -d "$SYSCONFIG_DIR" ]; then
persist=sysconfig
fi
# --- systemd unit: survives Proxmox rewriting the guest network config ---
# Proxmox regenerates the guest network configuration on every container
# start (pve-container writes /etc/network/interfaces, ifcfg or the NM
# keyfile from netN), which wipes any alias we added to the native stack.
# A tiny oneshot unit re-adds the addresses from the state file after boot,
# and /etc/systemd is not touched by Proxmox.
HELPER=$ROOT/usr/local/sbin/pvemanager-aliases
UNIT=$ROOT/etc/systemd/system/pvemanager-aliases.service
if command -v systemctl >/dev/null 2>&1 && [ -d "$ROOT/etc/systemd/system" ]; then
# Is there anything left to restore on any interface at all
remaining=$(cat "$STATE_DIR"/aliases.* 2>/dev/null | tr -d '[:space:]')
if [ -z "$remaining" ]; then
systemctl disable pvemanager-aliases.service >/dev/null 2>&1 || true
rm -f "$UNIT" "$HELPER"
systemctl daemon-reload >/dev/null 2>&1 || true
else
mkdir -p "$ROOT/usr/local/sbin" 2>/dev/null || true
cat > "$HELPER" <<'HELPER_EOF'
#!/bin/sh
# managed by PVEmanager - do not edit by hand
# Re-add the alias addresses recorded by PVEmanager for every interface.
for state in /var/lib/pvemanager/aliases.*; do
[ -f "$state" ] || continue
iface=${state##*/aliases.}
for cidr in $(cat "$state"); do
addr=${cidr%%/*}
ip -4 addr show dev "$iface" 2>/dev/null | grep -qw "$addr" && continue
ip addr add "$cidr" dev "$iface" 2>/dev/null || true
done
done
HELPER_EOF
chmod 755 "$HELPER"
cat > "$UNIT" <<'UNIT_EOF'
# managed by PVEmanager - do not edit by hand
[Unit]
Description=PVEmanager extra IP addresses
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/sbin/pvemanager-aliases
[Install]
WantedBy=multi-user.target
UNIT_EOF
systemctl daemon-reload >/dev/null 2>&1 || true
systemctl enable pvemanager-aliases.service >/dev/null 2>&1 || true
persist="${persist}+systemd"
[ "$persist" = "none+systemd" ] && persist=systemd
fi
fi
echo "PVEMANAGER_PERSIST=$persist"
echo "PVEMANAGER_OK=1"

View File

@@ -37,7 +37,9 @@ RUNTIME_ONLY = "runtime_only" # поднят, но стек не распозн
PENDING = "pending" # гость недоступен, попробуем позже
FAILED = "failed" # скрипт отработал с ошибкой
_PERSIST_RE = re.compile(r"^PVEMANAGER_PERSIST=(\w+)$", re.MULTILINE)
# Значение может быть составным ("nm+systemd"), когда родной стек
# дополнен подстраховкой systemd-юнитом.
_PERSIST_RE = re.compile(r"^PVEMANAGER_PERSIST=([\w+]+)$", re.MULTILINE)
_OK_RE = re.compile(r"^PVEMANAGER_OK=1$", re.MULTILINE)
_IFACE_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,15}$")

View File

@@ -103,3 +103,23 @@ def test_include_live_false_skips_live_endpoint():
# Stopped container: querying /interfaces would only produce a 500.
assert _client_with_mock(px).get_container_interfaces("pve1", 130, include_live=False) == []
lxc.interfaces.get.assert_not_called()
def test_execute_script_sends_bytes_not_wide_chars():
"""Скрипт с кириллицей не должен ронять Perl-обработчик Proxmox.
PVE кодирует input-data в base64 средствами Perl и падает с
"Wide character in subroutine entry" на любой строке с не-ASCII.
Отдаём UTF-8 байты, представленные как latin-1.
"""
px = MagicMock()
agent = px.nodes.return_value.qemu.return_value.agent
agent.exec.post.return_value = {"pid": 1}
agent("exec-status").get.return_value = {"exited": 1, "exitcode": 0, "out-data": "ok"}
script = "#!/bin/sh\n# комментарий с кириллицей\necho ok\n"
_client_with_mock(px).execute_script("prod", 102, script)
sent = agent.exec.post.call_args.kwargs["input-data"]
assert all(ord(ch) < 256 for ch in sent), "в API уходят широкие символы"
assert sent.encode("latin-1").decode("utf-8") == script

View File

@@ -31,6 +31,11 @@ FAKE_IFUP = """#!/bin/sh
exit 0
"""
FAKE_SYSTEMCTL = """#!/bin/sh
echo "$*" >> "$FAKE_LOG/systemctl.log"
exit 0
"""
@pytest.fixture
def sandbox(tmp_path):
@@ -41,7 +46,8 @@ def sandbox(tmp_path):
for path in (root, bindir, logdir):
path.mkdir(parents=True)
for name, body in (("ip", FAKE_IP), ("nmcli", FAKE_NMCLI), ("ifup", FAKE_IFUP)):
for name, body in (("ip", FAKE_IP), ("nmcli", FAKE_NMCLI), ("ifup", FAKE_IFUP),
("systemctl", FAKE_SYSTEMCTL)):
target = bindir / name
target.write_text(body)
target.chmod(0o755)
@@ -57,7 +63,7 @@ def sandbox(tmp_path):
"ADDRESSES": addresses,
}
# Прячем инструменты, которых в этом сценарии быть не должно
for name in ("ip", "nmcli", "ifup"):
for name in ("ip", "nmcli", "ifup", "systemctl"):
path = bindir / name
hidden = bindir / f"{name}.hidden"
if name in tools and hidden.exists():
@@ -296,3 +302,75 @@ def test_missing_iface_is_rejected(sandbox):
assert result.returncode == 2
assert "missing IFACE" in result.stdout
def test_script_is_ascii_only():
"""Скрипт уезжает в гостя через API Proxmox, который спотыкается на
не-ASCII, и его маркер попадает в системные конфиги — держим файл в ASCII."""
raw = SCRIPT.read_bytes()
non_ascii = [b for b in raw if b > 127]
assert not non_ascii, f"в скрипте {len(non_ascii)} не-ASCII байт"
# --- systemd-подстраховка ------------------------------------------------------
def test_systemd_unit_is_installed_alongside_native_stack(sandbox):
"""Proxmox переписывает сетевой конфиг гостя при каждом старте, поэтому
адреса восстанавливает ещё и oneshot-юнит — /etc/systemd он не трогает."""
root = sandbox["root"]
_mkdirs(root, "etc/sysconfig/network-scripts", "etc/systemd/system")
result = sandbox["run"](addresses="10.10.10.90/24", tools=("ip", "systemctl"))
assert sandbox["persist_of"](result) == "sysconfig+systemd"
unit = (root / "etc/systemd/system/pvemanager-aliases.service").read_text()
assert "After=network-online.target" in unit
assert "ExecStart=/usr/local/sbin/pvemanager-aliases" in unit
helper = root / "usr/local/sbin/pvemanager-aliases"
assert "ip addr add" in helper.read_text()
assert "enable pvemanager-aliases.service" in sandbox["log"]("systemctl")
def test_systemd_unit_alone_when_no_native_stack(sandbox):
root = sandbox["root"]
_mkdirs(root, "etc/systemd/system")
result = sandbox["run"](addresses="10.10.10.90/24", tools=("ip", "systemctl"))
assert sandbox["persist_of"](result) == "systemd"
assert (root / "etc/systemd/system/pvemanager-aliases.service").exists()
def test_systemd_unit_removed_when_last_address_goes(sandbox):
root = sandbox["root"]
_mkdirs(root, "etc/systemd/system")
sandbox["run"](addresses="10.10.10.90/24", tools=("ip", "systemctl"))
result = sandbox["run"](addresses="", tools=("ip", "systemctl"))
assert sandbox["persist_of"](result) == "none"
assert not (root / "etc/systemd/system/pvemanager-aliases.service").exists()
assert not (root / "usr/local/sbin/pvemanager-aliases").exists()
assert "disable pvemanager-aliases.service" in sandbox["log"]("systemctl")
def test_systemd_unit_stays_while_another_interface_has_addresses(sandbox):
"""Опустошили eth0, но на eth1 адрес остался — юнит нужен дальше."""
root = sandbox["root"]
_mkdirs(root, "etc/systemd/system")
sandbox["run"](addresses="10.10.10.90/24", iface="eth0", tools=("ip", "systemctl"))
sandbox["run"](addresses="10.10.10.91/24", iface="eth1", tools=("ip", "systemctl"))
sandbox["run"](addresses="", iface="eth0", tools=("ip", "systemctl"))
assert (root / "etc/systemd/system/pvemanager-aliases.service").exists()
def test_without_systemd_persist_has_no_suffix(sandbox):
root = sandbox["root"]
_mkdirs(root, "etc/sysconfig/network-scripts", "etc/systemd/system")
result = sandbox["run"](addresses="10.10.10.90/24", tools=("ip",))
assert sandbox["persist_of"](result) == "sysconfig"
assert not (root / "etc/systemd/system/pvemanager-aliases.service").exists()

View File

@@ -216,3 +216,25 @@ def test_unknown_guest_is_pending_not_crash(guest, monkeypatch):
assert result.status == guest_ip_service.PENDING
assert recorder.calls == []
def test_combined_persist_marker_is_understood(guest, monkeypatch):
"""Скрипт сообщает составной способ закрепления: родной стек + systemd."""
allocation = guest["add_alias"]("10.10.10.90")
_patch(monkeypatch, script_engine.ExecResult(
True, "PVEMANAGER_PERSIST=nm+systemd\nPVEMANAGER_OK=1\n", 0))
result = guest_ip_service.apply_address(guest["db"], allocation)
assert result.persist == "nm+systemd"
assert result.status == guest_ip_service.APPLIED
assert allocation.apply_status == guest_ip_service.APPLIED
def test_plain_systemd_persist_is_applied(guest, monkeypatch):
allocation = guest["add_alias"]("10.10.10.90")
_patch(monkeypatch, script_engine.ExecResult(
True, "PVEMANAGER_PERSIST=systemd\nPVEMANAGER_OK=1\n", 0))
assert guest_ip_service.apply_address(guest["db"], allocation).status == \
guest_ip_service.APPLIED

View File

@@ -255,3 +255,48 @@ def test_network_scan_skips_node_and_cached_guest_ips(env):
env["network"].id, None, allow_network_scan=True)
assert ip not in ("10.10.10.1", "10.10.10.2", "10.10.10.3")
def test_first_address_of_a_guest_becomes_primary_automatically(env):
"""Мастер создания инстанса не передаёт is_primary — флаг ставится сам."""
first, error = env["service"].allocate_ip(
ip_address="10.10.10.60", network_id=env["network"].id, resource_type="lxc",
resource_name="gitea", proxmox_server_id=env["server"].id, proxmox_vmid=109,
proxmox_node="dev", allocated_by="wizard",
)
assert error is None and first.is_primary is True
second, error = env["service"].allocate_ip(
ip_address="10.10.10.61", network_id=env["network"].id, resource_type="lxc",
resource_name="gitea", proxmox_server_id=env["server"].id, proxmox_vmid=109,
proxmox_node="dev", allocated_by="wizard",
)
assert error is None and second.is_primary is False
def test_alias_never_claims_primary_by_itself(env):
alias, error = env["service"].allocate_ip(
ip_address="10.10.10.62", network_id=env["network"].id,
proxmox_server_id=env["server"].id, proxmox_vmid=109, allocated_by="tester",
assignment_kind="alias", target_interface="eth0",
)
assert error is None and alias.is_primary is False
def test_explicit_flag_still_wins(env):
env["allocate"]("10.10.10.63", primary=True, kind="primary")
forced, error = env["service"].allocate_ip(
ip_address="10.10.10.64", network_id=env["network"].id,
proxmox_server_id=env["server"].id, proxmox_vmid=109, allocated_by="tester",
is_primary=False,
)
assert error is None and forced.is_primary is False
def test_reservation_without_a_guest_is_not_primary(env):
"""Ручная бронь без привязки к гостю основной быть не может."""
manual, error = env["service"].allocate_ip(
ip_address="10.10.10.65", network_id=env["network"].id,
resource_name="reserved by hand", allocated_by="admin",
)
assert error is None and manual.is_primary is False