mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-05-13 20:45:01 +00:00
MacRimi
469 lines
16 KiB
Bash
469 lines
16 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
if [[ -n "${__PROXMENUX_GPU_HOOK_GUARD_HELPERS__}" ]]; then
|
|
return 0
|
|
fi
|
|
__PROXMENUX_GPU_HOOK_GUARD_HELPERS__=1
|
|
|
|
# Issue #195: snippets used to live at the hard-coded `local:snippets/`
|
|
# path, which broke LXC/VM migration between cluster nodes — `local` is
|
|
# node-specific, so the hookscript reference was dangling on the target
|
|
# node. The path now resolves dynamically through
|
|
# `_resolve_snippets_storage` and is cached per-process. Callers should
|
|
# invoke `_compute_snippets_paths` (interactive flag optional) before
|
|
# referencing the two PROXMENUX_GPU_HOOK_* variables.
|
|
PROXMENUX_GPU_HOOK_FILENAME="proxmenux-gpu-guard.sh"
|
|
PROXMENUX_GPU_HOOK_STORAGE_REF=""
|
|
PROXMENUX_GPU_HOOK_ABS_PATH=""
|
|
|
|
PROXMENUX_CONFIG_JSON="${PROXMENUX_CONFIG_JSON:-/usr/local/share/proxmenux/config.json}"
|
|
|
|
_gpu_guard_msg_warn() {
|
|
if declare -F msg_warn >/dev/null 2>&1; then
|
|
msg_warn "$1"
|
|
else
|
|
echo "[WARN] $1" >&2
|
|
fi
|
|
}
|
|
|
|
_gpu_guard_msg_ok() {
|
|
if declare -F msg_ok >/dev/null 2>&1; then
|
|
msg_ok "$1"
|
|
else
|
|
echo "[OK] $1"
|
|
fi
|
|
}
|
|
|
|
# ────────────────────────────────────────────────────────────────────
|
|
# Snippets storage resolution (issue #195)
|
|
# ────────────────────────────────────────────────────────────────────
|
|
|
|
_save_snippets_storage_preference() {
|
|
local storage="$1"
|
|
command -v jq >/dev/null 2>&1 || return 0
|
|
mkdir -p "$(dirname "$PROXMENUX_CONFIG_JSON")" 2>/dev/null || true
|
|
[[ -f "$PROXMENUX_CONFIG_JSON" ]] || echo "{}" > "$PROXMENUX_CONFIG_JSON"
|
|
jq --arg s "$storage" '.snippets_storage = $s' "$PROXMENUX_CONFIG_JSON" \
|
|
> "${PROXMENUX_CONFIG_JSON}.tmp" 2>/dev/null \
|
|
&& mv "${PROXMENUX_CONFIG_JSON}.tmp" "$PROXMENUX_CONFIG_JSON"
|
|
}
|
|
|
|
# Decide which PVE storage backs ProxMenux snippets (hookscripts).
|
|
#
|
|
# Outcomes (in order):
|
|
# 1. Cached resolution in this shell → reuse, no work.
|
|
# 2. No active storage with content=snippets → fall back to "local".
|
|
# 3. Single candidate (standalone host with only `local`) → use it silently.
|
|
# 4. Multiple candidates + saved preference → use saved.
|
|
# 5. Multiple candidates, no preference, $1 == "interactive" + whiptail
|
|
# available → prompt the user, save the choice, use it.
|
|
# 6. Otherwise (non-interactive auto-call from sync_*, cron, etc.) →
|
|
# use the first listed candidate. Avoids blocking on a dialog from
|
|
# a non-tty context.
|
|
_list_snippets_candidates() {
|
|
pvesm status -content snippets 2>/dev/null \
|
|
| awk 'NR>1 && $3=="active" {print $1}'
|
|
}
|
|
|
|
# PVE 9 ships `local` without `snippets` in its content list, so a fresh
|
|
# install has zero candidates and ProxMenux can't write a hookscript
|
|
# anywhere. This silently appends `snippets` to local's content set so
|
|
# the GPU passthrough flow works out of the box. We only touch `local`
|
|
# (the always-present default storage) and only when there's nothing
|
|
# else to choose — never modifies a custom storage definition.
|
|
_ensure_local_supports_snippets() {
|
|
local current
|
|
current=$(pvesh get /storage/local --output-format json 2>/dev/null | jq -r '.content // empty' 2>/dev/null)
|
|
[[ -z "$current" ]] && return 1
|
|
echo "$current" | tr ',' '\n' | grep -qx 'snippets' && return 0
|
|
|
|
local new_content="${current},snippets"
|
|
if pvesm set local --content "$new_content" >/dev/null 2>&1; then
|
|
_gpu_guard_msg_ok "Enabled 'snippets' on the 'local' storage so ProxMenux can install hookscripts."
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
_resolve_snippets_storage() {
|
|
local interactive="${1:-}"
|
|
|
|
if [[ -n "${__PROXMENUX_RESOLVED_SNIPPETS_STORAGE:-}" ]]; then
|
|
echo "$__PROXMENUX_RESOLVED_SNIPPETS_STORAGE"
|
|
return 0
|
|
fi
|
|
|
|
local candidates
|
|
candidates=$(_list_snippets_candidates)
|
|
|
|
if [[ -z "$candidates" ]]; then
|
|
# Fresh PVE 9 host — `local` doesn't include `snippets` by default.
|
|
# Auto-enable it; if that succeeds, re-list and continue.
|
|
if _ensure_local_supports_snippets; then
|
|
candidates=$(_list_snippets_candidates)
|
|
fi
|
|
fi
|
|
|
|
if [[ -z "$candidates" ]]; then
|
|
# Still nothing usable — fall back to `local` and let the caller
|
|
# surface the error if writing actually fails.
|
|
__PROXMENUX_RESOLVED_SNIPPETS_STORAGE="local"
|
|
echo "local"
|
|
return 0
|
|
fi
|
|
|
|
local count
|
|
count=$(echo "$candidates" | wc -l)
|
|
|
|
if [[ "$count" -eq 1 ]]; then
|
|
__PROXMENUX_RESOLVED_SNIPPETS_STORAGE="$candidates"
|
|
echo "$candidates"
|
|
return 0
|
|
fi
|
|
|
|
if [[ -f "$PROXMENUX_CONFIG_JSON" ]] && command -v jq >/dev/null 2>&1; then
|
|
local pref
|
|
pref=$(jq -r '.snippets_storage // empty' "$PROXMENUX_CONFIG_JSON" 2>/dev/null)
|
|
if [[ -n "$pref" ]] && echo "$candidates" | grep -qFx "$pref"; then
|
|
__PROXMENUX_RESOLVED_SNIPPETS_STORAGE="$pref"
|
|
echo "$pref"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [[ "$interactive" == "interactive" ]] && command -v whiptail >/dev/null 2>&1; then
|
|
local options=()
|
|
local first_pick=1
|
|
while IFS= read -r s; do
|
|
[[ -z "$s" ]] && continue
|
|
if [[ $first_pick -eq 1 ]]; then
|
|
options+=("$s" "" "ON")
|
|
first_pick=0
|
|
else
|
|
options+=("$s" "" "OFF")
|
|
fi
|
|
done <<< "$candidates"
|
|
|
|
local choice
|
|
choice=$(whiptail --backtitle "ProxMenux" \
|
|
--title "Snippets storage (used by hookscripts)" \
|
|
--radiolist \
|
|
"Pick the storage where ProxMenux installs snippets/hookscripts.\n\nFor cluster setups, choose a shared NFS/CIFS storage so VMs and LXCs migrate cleanly between nodes — \`local\` is node-specific and breaks migration." \
|
|
20 78 8 \
|
|
"${options[@]}" 3>&1 1>&2 2>&3) || choice=""
|
|
|
|
if [[ -n "$choice" ]] && echo "$candidates" | grep -qFx "$choice"; then
|
|
_save_snippets_storage_preference "$choice"
|
|
__PROXMENUX_RESOLVED_SNIPPETS_STORAGE="$choice"
|
|
echo "$choice"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
local first
|
|
first=$(echo "$candidates" | head -n 1)
|
|
__PROXMENUX_RESOLVED_SNIPPETS_STORAGE="$first"
|
|
echo "$first"
|
|
}
|
|
|
|
# Populate the two PROXMENUX_GPU_HOOK_* variables from whichever storage
|
|
# `_resolve_snippets_storage` returns. Idempotent — safe to call multiple
|
|
# times, the resolver is cached per-process.
|
|
_compute_snippets_paths() {
|
|
local interactive="${1:-}"
|
|
local storage
|
|
storage=$(_resolve_snippets_storage "$interactive")
|
|
|
|
PROXMENUX_GPU_HOOK_STORAGE_REF="${storage}:snippets/${PROXMENUX_GPU_HOOK_FILENAME}"
|
|
|
|
# `pvesm path` understands the storage:content/file syntax for any
|
|
# registered storage and returns the absolute filesystem path — works
|
|
# for `local`, NFS, CIFS, dir, etc. Falls back to the conventional
|
|
# mount point if pvesm doesn't resolve (very old PVE / mid-mount
|
|
# transitions).
|
|
local abs
|
|
abs=$(pvesm path "$PROXMENUX_GPU_HOOK_STORAGE_REF" 2>/dev/null)
|
|
if [[ -n "$abs" ]]; then
|
|
PROXMENUX_GPU_HOOK_ABS_PATH="$abs"
|
|
elif [[ "$storage" == "local" ]]; then
|
|
PROXMENUX_GPU_HOOK_ABS_PATH="/var/lib/vz/snippets/${PROXMENUX_GPU_HOOK_FILENAME}"
|
|
else
|
|
PROXMENUX_GPU_HOOK_ABS_PATH="/mnt/pve/${storage}/snippets/${PROXMENUX_GPU_HOOK_FILENAME}"
|
|
fi
|
|
}
|
|
|
|
_gpu_guard_has_vm_gpu() {
|
|
local vmid="$1"
|
|
qm config "$vmid" 2>/dev/null | grep -qE '^hostpci[0-9]+:'
|
|
}
|
|
|
|
_gpu_guard_has_lxc_gpu() {
|
|
local ctid="$1"
|
|
local conf="/etc/pve/lxc/${ctid}.conf"
|
|
[[ -f "$conf" ]] || return 1
|
|
grep -qE 'dev[0-9]+:.*(/dev/dri|/dev/nvidia|/dev/kfd)|lxc\.mount\.entry:.*dev/dri' "$conf" 2>/dev/null
|
|
}
|
|
|
|
ensure_proxmenux_gpu_guard_hookscript() {
|
|
# Issue #195: resolve which snippets storage to write to (interactive
|
|
# — this function is called from the GPU passthrough flow which is
|
|
# always run from a tty). The resolver caches its answer for the rest
|
|
# of the bash session, so subsequent attach_* calls reuse it.
|
|
_compute_snippets_paths "interactive"
|
|
|
|
mkdir -p "$(dirname "$PROXMENUX_GPU_HOOK_ABS_PATH")" 2>/dev/null || true
|
|
|
|
cat >"$PROXMENUX_GPU_HOOK_ABS_PATH" <<'HOOKEOF'
|
|
#!/usr/bin/env bash
|
|
set -u
|
|
|
|
arg1="${1:-}"
|
|
arg2="${2:-}"
|
|
case "$arg1" in
|
|
pre-start|post-start|pre-stop|post-stop)
|
|
phase="$arg1"
|
|
guest_id="$arg2"
|
|
;;
|
|
*)
|
|
guest_id="$arg1"
|
|
phase="$arg2"
|
|
;;
|
|
esac
|
|
[[ "$phase" == "pre-start" ]] || exit 0
|
|
|
|
vm_conf="/etc/pve/qemu-server/${guest_id}.conf"
|
|
ct_conf="/etc/pve/lxc/${guest_id}.conf"
|
|
|
|
if [[ -f "$vm_conf" ]]; then
|
|
mapfile -t hostpci_lines < <(grep -E '^hostpci[0-9]+:' "$vm_conf" 2>/dev/null || true)
|
|
[[ ${#hostpci_lines[@]} -eq 0 ]] && exit 0
|
|
|
|
# Build slot list used by this VM and block if any running VM already uses same slot.
|
|
slot_keys=()
|
|
for line in "${hostpci_lines[@]}"; do
|
|
val="${line#*: }"
|
|
[[ "$val" == *"mapping="* ]] && continue
|
|
first_field="${val%%,*}"
|
|
IFS=';' read -r -a ids <<< "$first_field"
|
|
for id in "${ids[@]}"; do
|
|
id="${id#host=}"
|
|
id="${id// /}"
|
|
[[ -z "$id" ]] && continue
|
|
if [[ "$id" =~ ^[0-9a-fA-F]{2}:[0-9a-fA-F]{2}$ ]]; then
|
|
key="${id,,}"
|
|
else
|
|
[[ "$id" =~ ^0000: ]] || id="0000:${id}"
|
|
key="${id#0000:}"
|
|
key="${key%.*}"
|
|
key="${key,,}"
|
|
fi
|
|
dup=0
|
|
for existing in "${slot_keys[@]}"; do
|
|
[[ "$existing" == "$key" ]] && dup=1 && break
|
|
done
|
|
[[ "$dup" -eq 0 ]] && slot_keys+=("$key")
|
|
done
|
|
done
|
|
|
|
if [[ ${#slot_keys[@]} -gt 0 ]]; then
|
|
conflict_details=""
|
|
for other_conf in /etc/pve/qemu-server/*.conf; do
|
|
[[ -f "$other_conf" ]] || continue
|
|
other_vmid="$(basename "$other_conf" .conf)"
|
|
[[ "$other_vmid" == "$guest_id" ]] && continue
|
|
qm status "$other_vmid" 2>/dev/null | grep -q "status: running" || continue
|
|
|
|
for key in "${slot_keys[@]}"; do
|
|
if grep -qE "^hostpci[0-9]+:.*(0000:)?${key}(\\.[0-7])?([,[:space:]]|$)" "$other_conf" 2>/dev/null; then
|
|
other_name="$(awk '/^name:/ {print $2}' "$other_conf" 2>/dev/null)"
|
|
[[ -z "$other_name" ]] && other_name="VM-${other_vmid}"
|
|
conflict_details+=$'\n'"- ${key} in use by VM ${other_vmid} (${other_name})"
|
|
break
|
|
fi
|
|
done
|
|
done
|
|
|
|
if [[ -n "$conflict_details" ]]; then
|
|
echo "ProxMenux GPU Guard: VM ${guest_id} blocked at pre-start." >&2
|
|
echo "A hostpci device slot is already in use by another running VM." >&2
|
|
printf '%s\n' "$conflict_details" >&2
|
|
echo "Stop the source VM or remove/move the shared hostpci assignment." >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
failed=0
|
|
details=""
|
|
for line in "${hostpci_lines[@]}"; do
|
|
val="${line#*: }"
|
|
[[ "$val" == *"mapping="* ]] && continue
|
|
|
|
first_field="${val%%,*}"
|
|
IFS=';' read -r -a ids <<< "$first_field"
|
|
for id in "${ids[@]}"; do
|
|
id="${id#host=}"
|
|
id="${id// /}"
|
|
[[ -z "$id" ]] && continue
|
|
|
|
# Slot-only syntax (e.g. 01:00 or 0000:01:00) is accepted by Proxmox.
|
|
if [[ "$id" =~ ^([0-9a-fA-F]{4}:)?[0-9a-fA-F]{2}:[0-9a-fA-F]{2}$ ]]; then
|
|
slot="${id,,}"
|
|
slot="${slot#0000:}"
|
|
slot_has_gpu=false
|
|
for dev in /sys/bus/pci/devices/0000:${slot}.*; do
|
|
[[ -e "$dev" ]] || continue
|
|
# SR-IOV: skip Virtual Functions when iterating a whole slot.
|
|
# VFs share the slot with their PF but carry their own driver
|
|
# state; their vfio-pci rebind is handled by Proxmox at VM
|
|
# start. Pre-flighting them would falsely block SR-IOV setups
|
|
# where the PF legitimately stays on the native driver.
|
|
[[ -L "${dev}/physfn" ]] && continue
|
|
class_hex="$(cat "$dev/class" 2>/dev/null | sed 's/^0x//')"
|
|
[[ "${class_hex:0:2}" != "03" ]] && continue
|
|
slot_has_gpu=true
|
|
drv="$(basename "$(readlink "$dev/driver" 2>/dev/null)" 2>/dev/null)"
|
|
if [[ "$drv" != "vfio-pci" ]]; then
|
|
failed=1
|
|
details+=$'\n'"- ${dev##*/}: driver=${drv:-none}"
|
|
fi
|
|
done
|
|
# If this slot does not include a display/3D controller, it is not GPU-guarded.
|
|
[[ "$slot_has_gpu" == "true" ]] || true
|
|
continue
|
|
fi
|
|
|
|
[[ "$id" =~ ^0000: ]] || id="0000:${id}"
|
|
dev_path="/sys/bus/pci/devices/${id}"
|
|
if [[ ! -d "$dev_path" ]]; then
|
|
failed=1
|
|
details+=$'\n'"- ${id}: PCI device not found"
|
|
continue
|
|
fi
|
|
# SR-IOV VF: do not pre-flight the driver. Proxmox rebinds the VF
|
|
# to vfio-pci as part of VM start; at pre-start time the VF may
|
|
# still be on its native driver (i915, etc.) — that is normal,
|
|
# not an error. Blocking here would prevent every SR-IOV VF
|
|
# passthrough from starting.
|
|
if [[ -L "${dev_path}/physfn" ]]; then
|
|
continue
|
|
fi
|
|
class_hex="$(cat "$dev_path/class" 2>/dev/null | sed 's/^0x//')"
|
|
# Enforce vfio only for display/3D devices (PCI class 03xx).
|
|
[[ "${class_hex:0:2}" == "03" ]] || continue
|
|
drv="$(basename "$(readlink "$dev_path/driver" 2>/dev/null)" 2>/dev/null)"
|
|
if [[ "$drv" != "vfio-pci" ]]; then
|
|
failed=1
|
|
details+=$'\n'"- ${id}: driver=${drv:-none}"
|
|
fi
|
|
done
|
|
done
|
|
|
|
if [[ "$failed" -eq 1 ]]; then
|
|
echo "ProxMenux GPU Guard: VM ${guest_id} blocked at pre-start." >&2
|
|
echo "GPU passthrough device is not ready for VM mode (vfio-pci required)." >&2
|
|
printf '%s\n' "$details" >&2
|
|
echo "Switch mode to GPU -> VM from ProxMenux: GPUs and Coral-TPU Menu." >&2
|
|
exit 1
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -f "$ct_conf" ]]; then
|
|
mapfile -t gpu_dev_paths < <(
|
|
{
|
|
grep -E '^dev[0-9]+:' "$ct_conf" 2>/dev/null | sed -E 's/^dev[0-9]+:[[:space:]]*([^,[:space:]]+).*/\1/'
|
|
grep -E '^lxc\.mount\.entry:' "$ct_conf" 2>/dev/null | sed -E 's/^lxc\.mount\.entry:[[:space:]]*([^[:space:]]+).*/\1/'
|
|
} | grep -E '^/dev/(dri|nvidia|kfd)' | sort -u
|
|
)
|
|
|
|
[[ ${#gpu_dev_paths[@]} -eq 0 ]] && exit 0
|
|
|
|
missing=""
|
|
for dev in "${gpu_dev_paths[@]}"; do
|
|
[[ -e "$dev" ]] || missing+=$'\n'"- ${dev} unavailable"
|
|
done
|
|
|
|
if [[ -n "$missing" ]]; then
|
|
echo "ProxMenux GPU Guard: LXC ${guest_id} blocked at pre-start." >&2
|
|
echo "Configured GPU devices are unavailable in host device nodes." >&2
|
|
printf '%s\n' "$missing" >&2
|
|
echo "Switch mode to GPU -> LXC from ProxMenux: GPUs and Coral-TPU Menu." >&2
|
|
exit 1
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
exit 0
|
|
HOOKEOF
|
|
|
|
chmod 755 "$PROXMENUX_GPU_HOOK_ABS_PATH" 2>/dev/null || true
|
|
}
|
|
|
|
attach_proxmenux_gpu_guard_to_vm() {
|
|
local vmid="$1"
|
|
_gpu_guard_has_vm_gpu "$vmid" || return 0
|
|
|
|
# Resolver cache populated by ensure_* (or the first call here).
|
|
# Pass "interactive" so a sync done in isolation can still prompt;
|
|
# sync_proxmenux_gpu_guard_hooks pre-seeds the cache to suppress the
|
|
# dialog when running non-interactively.
|
|
_compute_snippets_paths "interactive"
|
|
|
|
local current
|
|
current=$(qm config "$vmid" 2>/dev/null | awk '/^hookscript:/ {print $2}')
|
|
if [[ "$current" == "$PROXMENUX_GPU_HOOK_STORAGE_REF" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if qm set "$vmid" --hookscript "$PROXMENUX_GPU_HOOK_STORAGE_REF" >/dev/null 2>&1; then
|
|
_gpu_guard_msg_ok "PCIe passthrough guard attached to VM ${vmid} (${PROXMENUX_GPU_HOOK_STORAGE_REF})"
|
|
else
|
|
_gpu_guard_msg_warn "Could not attach PCIe passthrough guard to VM ${vmid}. Verify ${__PROXMENUX_RESOLVED_SNIPPETS_STORAGE} storage supports snippets."
|
|
fi
|
|
}
|
|
|
|
attach_proxmenux_gpu_guard_to_lxc() {
|
|
local ctid="$1"
|
|
_gpu_guard_has_lxc_gpu "$ctid" || return 0
|
|
|
|
_compute_snippets_paths "interactive"
|
|
|
|
local current
|
|
current=$(pct config "$ctid" 2>/dev/null | awk '/^hookscript:/ {print $2}')
|
|
if [[ "$current" == "$PROXMENUX_GPU_HOOK_STORAGE_REF" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if pct set "$ctid" -hookscript "$PROXMENUX_GPU_HOOK_STORAGE_REF" >/dev/null 2>&1; then
|
|
_gpu_guard_msg_ok "PCIe passthrough guard attached to LXC ${ctid} (${PROXMENUX_GPU_HOOK_STORAGE_REF})"
|
|
else
|
|
_gpu_guard_msg_warn "Could not attach PCIe passthrough guard to LXC ${ctid}. Verify ${__PROXMENUX_RESOLVED_SNIPPETS_STORAGE} storage supports snippets."
|
|
fi
|
|
}
|
|
|
|
# Iterate every VM/LXC and reattach the guard if it has GPU passthrough
|
|
# but no current hookscript reference. Used for cluster-wide sync /
|
|
# upgrades. Runs non-interactively: pre-seeds the resolver cache so the
|
|
# inner attach_* calls don't pop a dialog from a possibly headless
|
|
# context.
|
|
sync_proxmenux_gpu_guard_hooks() {
|
|
if [[ -z "${__PROXMENUX_RESOLVED_SNIPPETS_STORAGE:-}" ]]; then
|
|
__PROXMENUX_RESOLVED_SNIPPETS_STORAGE=$(_resolve_snippets_storage "")
|
|
fi
|
|
|
|
ensure_proxmenux_gpu_guard_hookscript
|
|
|
|
local vmid ctid
|
|
for conf in /etc/pve/qemu-server/*.conf; do
|
|
[[ -f "$conf" ]] || continue
|
|
vmid=$(basename "$conf" .conf)
|
|
_gpu_guard_has_vm_gpu "$vmid" && attach_proxmenux_gpu_guard_to_vm "$vmid"
|
|
done
|
|
|
|
for conf in /etc/pve/lxc/*.conf; do
|
|
[[ -f "$conf" ]] || continue
|
|
ctid=$(basename "$conf" .conf)
|
|
_gpu_guard_has_lxc_gpu "$ctid" && attach_proxmenux_gpu_guard_to_lxc "$ctid"
|
|
done
|
|
}
|