mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-04-05 20:03:48 +00:00
733 lines
26 KiB
Bash
733 lines
26 KiB
Bash
#!/bin/bash
|
||
# ============================================================================
|
||
# ProxMenux - Real Proxmox Event Simulator
|
||
# ============================================================================
|
||
# This script triggers ACTUAL events on Proxmox so that PVE's notification
|
||
# system fires real webhooks through the full pipeline:
|
||
#
|
||
# PVE event -> PVE notification -> webhook POST -> our pipeline -> Telegram
|
||
#
|
||
# Unlike test_all_notifications.sh (which injects directly via API), this
|
||
# script makes Proxmox generate the events itself.
|
||
#
|
||
# Usage:
|
||
# chmod +x test_real_events.sh
|
||
# ./test_real_events.sh # interactive menu
|
||
# ./test_real_events.sh disk # run disk tests only
|
||
# ./test_real_events.sh backup # run backup tests only
|
||
# ./test_real_events.sh all # run all tests
|
||
# ============================================================================
|
||
|
||
set -euo pipefail
|
||
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
CYAN='\033[0;36m'
|
||
BOLD='\033[1m'
|
||
NC='\033[0m'
|
||
|
||
API="http://127.0.0.1:8008"
|
||
LOG_FILE="/tmp/proxmenux_real_test_$(date +%Y%m%d_%H%M%S).log"
|
||
|
||
# ── Helpers ─────────────────────────────────────────────────────
|
||
log() { echo -e "$1" | tee -a "$LOG_FILE"; }
|
||
header() {
|
||
echo "" | tee -a "$LOG_FILE"
|
||
echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━<EFBFBD><EFBFBD>━━━━━━━━━━━━━${NC}" | tee -a "$LOG_FILE"
|
||
echo -e "${BOLD} $1${NC}" | tee -a "$LOG_FILE"
|
||
echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" | tee -a "$LOG_FILE"
|
||
}
|
||
|
||
warn() { log "${YELLOW} [!] $1${NC}"; }
|
||
ok() { log "${GREEN} [OK] $1${NC}"; }
|
||
fail() { log "${RED} [FAIL] $1${NC}"; }
|
||
info() { log "${CYAN} [i] $1${NC}"; }
|
||
|
||
confirm() {
|
||
echo ""
|
||
echo -e "${YELLOW} $1${NC}"
|
||
echo -ne " Continue? [Y/n]: "
|
||
read -r ans
|
||
[[ -z "$ans" || "$ans" =~ ^[Yy] ]]
|
||
}
|
||
|
||
wait_webhook() {
|
||
local seconds=${1:-10}
|
||
log " Waiting ${seconds}s for webhook delivery..."
|
||
sleep "$seconds"
|
||
}
|
||
|
||
snapshot_history() {
|
||
curl -s "${API}/api/notifications/history?limit=200" 2>/dev/null | python3 -c "
|
||
import sys, json
|
||
try:
|
||
data = json.load(sys.stdin)
|
||
count = len(data.get('history', []))
|
||
print(count)
|
||
except:
|
||
print(0)
|
||
" 2>/dev/null || echo "0"
|
||
}
|
||
|
||
check_new_events() {
|
||
local before=$1
|
||
local after
|
||
after=$(snapshot_history)
|
||
local diff=$((after - before))
|
||
if [ "$diff" -gt 0 ]; then
|
||
ok "Received $diff new notification(s) via webhook"
|
||
# Show the latest events
|
||
curl -s "${API}/api/notifications/history?limit=$((diff + 2))" 2>/dev/null | python3 -c "
|
||
import sys, json
|
||
data = json.load(sys.stdin)
|
||
for h in data.get('history', [])[:$diff]:
|
||
sev = h.get('severity', '?')
|
||
icon = {'CRITICAL': ' RED', 'WARNING': ' YEL', 'INFO': ' BLU'}.get(sev, ' ???')
|
||
print(f'{icon} {h[\"event_type\"]:25s} {h.get(\"title\", \"\")[:60]}')
|
||
" 2>/dev/null | tee -a "$LOG_FILE"
|
||
else
|
||
warn "No new notifications detected (may need more time or check filters)"
|
||
fi
|
||
}
|
||
|
||
# ── Pre-flight checks ──────────────────────────────────────────
|
||
preflight() {
|
||
header "Pre-flight Checks"
|
||
|
||
# Check if running as root
|
||
if [ "$(id -u)" -ne 0 ]; then
|
||
fail "This script must be run as root"
|
||
exit 1
|
||
fi
|
||
ok "Running as root"
|
||
|
||
# Check ProxMenux is running
|
||
if curl -s "${API}/api/health" >/dev/null 2>&1; then
|
||
ok "ProxMenux Monitor is running"
|
||
else
|
||
fail "ProxMenux Monitor not reachable at ${API}"
|
||
exit 1
|
||
fi
|
||
|
||
# Check webhook is configured by querying PVE directly
|
||
if pvesh get /cluster/notifications/endpoints/webhook --output-format json 2>/dev/null | python3 -c "
|
||
import sys, json
|
||
endpoints = json.load(sys.stdin)
|
||
found = any('proxmenux' in e.get('name','').lower() for e in (endpoints if isinstance(endpoints, list) else [endpoints]))
|
||
exit(0 if found else 1)
|
||
" 2>/dev/null; then
|
||
ok "PVE webhook endpoint 'proxmenux-webhook' is configured"
|
||
else
|
||
warn "PVE webhook may not be configured. Run setup from the UI first."
|
||
if ! confirm "Continue anyway?"; then
|
||
exit 1
|
||
fi
|
||
fi
|
||
|
||
# Check notification config
|
||
# API returns { config: { enabled: true/false/'true'/'false', ... }, success: true }
|
||
if curl -s "${API}/api/notifications/settings" 2>/dev/null | python3 -c "
|
||
import sys, json
|
||
d = json.load(sys.stdin)
|
||
cfg = d.get('config', d)
|
||
enabled = cfg.get('enabled', False)
|
||
exit(0 if enabled is True or str(enabled).lower() == 'true' else 1)
|
||
" 2>/dev/null; then
|
||
ok "Notifications are enabled"
|
||
else
|
||
fail "Notifications are NOT enabled. Enable them in the UI first."
|
||
exit 1
|
||
fi
|
||
|
||
# Re-run webhook setup to ensure priv config and body template exist
|
||
info "Re-configuring PVE webhook (ensures priv config + body template)..."
|
||
local setup_result
|
||
setup_result=$(curl -s -X POST "${API}/api/notifications/proxmox/setup-webhook" 2>/dev/null)
|
||
if echo "$setup_result" | python3 -c "import sys,json; d=json.load(sys.stdin); exit(0 if d.get('configured') else 1)" 2>/dev/null; then
|
||
ok "PVE webhook re-configured successfully"
|
||
else
|
||
local setup_err
|
||
setup_err=$(echo "$setup_result" | python3 -c "import sys,json; print(json.load(sys.stdin).get('error','unknown'))" 2>/dev/null)
|
||
warn "Webhook setup returned: ${setup_err}"
|
||
warn "PVE webhook events may not work. Manual commands below:"
|
||
echo "$setup_result" | python3 -c "
|
||
import sys, json
|
||
d = json.load(sys.stdin)
|
||
for cmd in d.get('fallback_commands', []):
|
||
print(f' {cmd}')
|
||
" 2>/dev/null
|
||
if ! confirm "Continue anyway?"; then
|
||
exit 1
|
||
fi
|
||
fi
|
||
|
||
# Find a VM/CT for testing
|
||
VMID=""
|
||
VMNAME=""
|
||
VMTYPE=""
|
||
|
||
# Try to find a stopped CT first (safest)
|
||
local cts
|
||
cts=$(pvesh get /cluster/resources --type vm --output-format json 2>/dev/null || echo "[]")
|
||
|
||
# Look for a stopped container
|
||
VMID=$(echo "$cts" | python3 -c "
|
||
import sys, json
|
||
vms = json.load(sys.stdin)
|
||
# Prefer stopped CTs, then stopped VMs
|
||
for v in sorted(vms, key=lambda x: (0 if x.get('type')=='lxc' else 1, 0 if x.get('status')=='stopped' else 1)):
|
||
if v.get('status') == 'stopped':
|
||
print(v.get('vmid', ''))
|
||
break
|
||
" 2>/dev/null || echo "")
|
||
|
||
if [ -n "$VMID" ]; then
|
||
VMTYPE=$(echo "$cts" | python3 -c "
|
||
import sys, json
|
||
vms = json.load(sys.stdin)
|
||
for v in vms:
|
||
if str(v.get('vmid')) == '$VMID':
|
||
print(v.get('type', 'qemu'))
|
||
break
|
||
" 2>/dev/null)
|
||
VMNAME=$(echo "$cts" | python3 -c "
|
||
import sys, json
|
||
vms = json.load(sys.stdin)
|
||
for v in vms:
|
||
if str(v.get('vmid')) == '$VMID':
|
||
print(v.get('name', 'unknown'))
|
||
break
|
||
" 2>/dev/null)
|
||
ok "Found stopped ${VMTYPE} for testing: ${VMID} (${VMNAME})"
|
||
else
|
||
warn "No stopped VM/CT found. Backup tests will use ID 0 (host backup)."
|
||
fi
|
||
|
||
# List available storage
|
||
info "Available storage:"
|
||
pvesh get /storage --output-format json 2>/dev/null | python3 -c "
|
||
import sys, json
|
||
stores = json.load(sys.stdin)
|
||
for s in stores:
|
||
sid = s.get('storage', '?')
|
||
stype = s.get('type', '?')
|
||
content = s.get('content', '?')
|
||
print(f' {sid:20s} type={stype:10s} content={content}')
|
||
" 2>/dev/null | tee -a "$LOG_FILE" || warn "Could not list storage"
|
||
|
||
echo ""
|
||
log " Log file: ${LOG_FILE}"
|
||
}
|
||
|
||
# ============================================================================
|
||
# TEST CATEGORY: DISK ERRORS
|
||
# ============================================================================
|
||
test_disk() {
|
||
header "DISK ERROR TESTS"
|
||
|
||
# ── Test D1: SMART error injection ──
|
||
log ""
|
||
log "${BOLD} Test D1: SMART error log injection${NC}"
|
||
info "Writes a simulated SMART error to syslog so JournalWatcher catches it."
|
||
info "This tests the journal -> notification_events -> pipeline flow."
|
||
|
||
local before
|
||
before=$(snapshot_history)
|
||
|
||
# Inject a realistic SMART error into the system journal
|
||
logger -t kernel -p kern.err "ata1.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x6 frozen"
|
||
sleep 1
|
||
logger -t kernel -p kern.crit "ata1.00: failed command: READ FPDMA QUEUED"
|
||
sleep 1
|
||
logger -t smartd -p daemon.warning "Device: /dev/sda [SAT], 1 Currently unreadable (pending) sectors"
|
||
|
||
wait_webhook 8
|
||
check_new_events "$before"
|
||
|
||
# ── Test D2: ZFS error simulation ──
|
||
log ""
|
||
log "${BOLD} Test D2: ZFS scrub error simulation${NC}"
|
||
|
||
# Check if ZFS is available
|
||
if command -v zpool >/dev/null 2>&1; then
|
||
local zpools
|
||
zpools=$(zpool list -H -o name 2>/dev/null || echo "")
|
||
|
||
if [ -n "$zpools" ]; then
|
||
local pool
|
||
pool=$(echo "$zpools" | head -1)
|
||
info "ZFS pool found: ${pool}"
|
||
info "Injecting ZFS checksum error into syslog (non-destructive)."
|
||
|
||
before=$(snapshot_history)
|
||
|
||
# Simulate ZFS error events via syslog (non-destructive)
|
||
logger -t kernel -p kern.warning "ZFS: pool '${pool}' has experienced an error"
|
||
sleep 1
|
||
logger -t zfs-module -p daemon.err "CHECKSUM error on ${pool}:mirror-0/sda: zio error"
|
||
|
||
wait_webhook 8
|
||
check_new_events "$before"
|
||
else
|
||
warn "ZFS installed but no pools found. Skipping ZFS test."
|
||
fi
|
||
else
|
||
warn "ZFS not installed. Skipping ZFS test."
|
||
fi
|
||
|
||
# ── Test D3: Filesystem space pressure ──
|
||
log ""
|
||
log "${BOLD} Test D3: Disk space pressure simulation${NC}"
|
||
info "Creates a large temporary file to fill disk, triggering space warnings."
|
||
info "The Health Monitor should detect low disk space within ~60s."
|
||
|
||
# Check current free space on /
|
||
local free_pct
|
||
free_pct=$(df / | tail -1 | awk '{print 100-$5}' | tr -d '%')
|
||
info "Current free space on /: ${free_pct}%"
|
||
|
||
if [ "$free_pct" -gt 15 ]; then
|
||
info "Disk has ${free_pct}% free. Need to reduce below threshold for test."
|
||
|
||
# Calculate how much to fill (leave only 8% free)
|
||
local total_k free_k fill_k
|
||
total_k=$(df / | tail -1 | awk '{print $2}')
|
||
free_k=$(df / | tail -1 | awk '{print $4}')
|
||
fill_k=$((free_k - (total_k * 8 / 100)))
|
||
|
||
if [ "$fill_k" -gt 0 ] && [ "$fill_k" -lt 50000000 ]; then
|
||
info "Will create ${fill_k}KB temp file to simulate low space."
|
||
|
||
if confirm "This will temporarily fill disk to ~92% on /. Safe to proceed?"; then
|
||
before=$(snapshot_history)
|
||
|
||
dd if=/dev/zero of=/tmp/.proxmenux_disk_test bs=1024 count="$fill_k" 2>/dev/null || true
|
||
ok "Temp file created. Disk pressure active."
|
||
info "Waiting 90s for Health Monitor to detect low space..."
|
||
|
||
# Wait for health monitor polling cycle
|
||
for i in $(seq 1 9); do
|
||
echo -ne "\r Waiting... ${i}0/90s"
|
||
sleep 10
|
||
done
|
||
echo ""
|
||
|
||
# Clean up immediately
|
||
rm -f /tmp/.proxmenux_disk_test
|
||
ok "Temp file removed. Disk space restored."
|
||
|
||
check_new_events "$before"
|
||
else
|
||
warn "Skipped disk pressure test."
|
||
fi
|
||
else
|
||
warn "Cannot safely fill disk (would need ${fill_k}KB). Skipping."
|
||
fi
|
||
else
|
||
warn "Disk already at ${free_pct}% free. Health Monitor may already be alerting."
|
||
fi
|
||
|
||
# ── Test D4: I/O error in syslog ──
|
||
log ""
|
||
log "${BOLD} Test D4: Generic I/O error injection${NC}"
|
||
info "Injects I/O errors into syslog for JournalWatcher."
|
||
|
||
before=$(snapshot_history)
|
||
|
||
logger -t kernel -p kern.err "Buffer I/O error on dev sdb1, logical block 0, async page read"
|
||
sleep 1
|
||
logger -t kernel -p kern.err "EXT4-fs error (device sdb1): ext4_find_entry:1455: inode #2: comm ls: reading directory lblock 0"
|
||
|
||
wait_webhook 8
|
||
check_new_events "$before"
|
||
}
|
||
|
||
# ============================================================================
|
||
# TEST CATEGORY: BACKUP EVENTS
|
||
# ============================================================================
|
||
test_backup() {
|
||
header "BACKUP EVENT TESTS"
|
||
|
||
local backup_storage=""
|
||
|
||
# Find backup-capable storage
|
||
backup_storage=$(pvesh get /storage --output-format json 2>/dev/null | python3 -c "
|
||
import sys, json
|
||
stores = json.load(sys.stdin)
|
||
for s in stores:
|
||
content = s.get('content', '')
|
||
if 'backup' in content or 'vztmpl' in content:
|
||
print(s.get('storage', ''))
|
||
break
|
||
# Fallback: try 'local'
|
||
else:
|
||
for s in stores:
|
||
if s.get('storage') == 'local':
|
||
print('local')
|
||
break
|
||
" 2>/dev/null || echo "local")
|
||
|
||
info "Using backup storage: ${backup_storage}"
|
||
|
||
# ── Test B1: Successful vzdump backup ──
|
||
if [ -n "$VMID" ]; then
|
||
log ""
|
||
log "${BOLD} Test B1: Real vzdump backup (success)${NC}"
|
||
info "Running a real vzdump backup of ${VMTYPE} ${VMID} (${VMNAME})."
|
||
info "This triggers PVE's notification system with a real backup event."
|
||
|
||
if confirm "This will backup ${VMTYPE} ${VMID} to '${backup_storage}'. Proceed?"; then
|
||
local before
|
||
before=$(snapshot_history)
|
||
|
||
# Use snapshot mode for VMs (non-disruptive), stop mode for CTs
|
||
local bmode="snapshot"
|
||
if [ "$VMTYPE" = "lxc" ]; then
|
||
bmode="suspend"
|
||
fi
|
||
|
||
info "Starting vzdump (mode=${bmode}, compress=zstd)..."
|
||
if vzdump "$VMID" --storage "$backup_storage" --mode "$bmode" --compress zstd --notes-template "ProxMenux test backup" 2>&1 | tee -a "$LOG_FILE"; then
|
||
ok "vzdump completed successfully!"
|
||
else
|
||
warn "vzdump returned non-zero (check output above)"
|
||
fi
|
||
|
||
wait_webhook 12
|
||
check_new_events "$before"
|
||
|
||
# Clean up the test backup
|
||
info "Cleaning up test backup file..."
|
||
local latest_bak
|
||
latest_bak=$(find "/var/lib/vz/dump/" -name "vzdump-*-${VMID}-*" -type f -newer /tmp/.proxmenux_bak_marker 2>/dev/null | head -1 || echo "")
|
||
# Create a marker for cleanup
|
||
touch /tmp/.proxmenux_bak_marker 2>/dev/null || true
|
||
else
|
||
warn "Skipped backup success test."
|
||
fi
|
||
|
||
# ── Test B2: Failed vzdump backup ──
|
||
log ""
|
||
log "${BOLD} Test B2: vzdump backup failure (invalid storage)${NC}"
|
||
info "Attempting backup to non-existent storage to trigger a backup failure event."
|
||
|
||
before=$(snapshot_history)
|
||
|
||
# This WILL fail because the storage doesn't exist
|
||
info "Starting vzdump to fake storage (will fail intentionally)..."
|
||
vzdump "$VMID" --storage "nonexistent_storage_12345" --mode snapshot 2>&1 | tail -5 | tee -a "$LOG_FILE" || true
|
||
|
||
warn "vzdump failed as expected (this is intentional)."
|
||
|
||
wait_webhook 12
|
||
check_new_events "$before"
|
||
|
||
else
|
||
warn "No VM/CT available for backup tests."
|
||
info "You can create a minimal LXC container for testing:"
|
||
info " pct create 9999 local:vztmpl/debian-12-standard_12.2-1_amd64.tar.zst --storage local-lvm --memory 128 --cores 1"
|
||
fi
|
||
|
||
# ── Test B3: Snapshot create/delete ──
|
||
if [ -n "$VMID" ] && [ "$VMTYPE" = "qemu" ]; then
|
||
log ""
|
||
log "${BOLD} Test B3: VM Snapshot create & delete${NC}"
|
||
info "Creating a snapshot of VM ${VMID} to test snapshot events."
|
||
|
||
if confirm "Create snapshot 'proxmenux_test' on VM ${VMID}?"; then
|
||
local before
|
||
before=$(snapshot_history)
|
||
|
||
if qm snapshot "$VMID" proxmenux_test --description "ProxMenux test snapshot" 2>&1 | tee -a "$LOG_FILE"; then
|
||
ok "Snapshot created!"
|
||
else
|
||
warn "Snapshot creation returned non-zero"
|
||
fi
|
||
|
||
wait_webhook 10
|
||
check_new_events "$before"
|
||
|
||
# Clean up snapshot
|
||
info "Cleaning up test snapshot..."
|
||
qm delsnapshot "$VMID" proxmenux_test 2>/dev/null || true
|
||
ok "Snapshot removed."
|
||
fi
|
||
elif [ -n "$VMID" ] && [ "$VMTYPE" = "lxc" ]; then
|
||
log ""
|
||
log "${BOLD} Test B3: CT Snapshot create & delete${NC}"
|
||
info "Creating a snapshot of CT ${VMID}."
|
||
|
||
if confirm "Create snapshot 'proxmenux_test' on CT ${VMID}?"; then
|
||
local before
|
||
before=$(snapshot_history)
|
||
|
||
if pct snapshot "$VMID" proxmenux_test --description "ProxMenux test snapshot" 2>&1 | tee -a "$LOG_FILE"; then
|
||
ok "Snapshot created!"
|
||
else
|
||
warn "Snapshot creation returned non-zero"
|
||
fi
|
||
|
||
wait_webhook 10
|
||
check_new_events "$before"
|
||
|
||
# Clean up
|
||
info "Cleaning up test snapshot..."
|
||
pct delsnapshot "$VMID" proxmenux_test 2>/dev/null || true
|
||
ok "Snapshot removed."
|
||
fi
|
||
fi
|
||
|
||
# ── Test B4: PVE scheduled backup notification ──
|
||
log ""
|
||
log "${BOLD} Test B4: Trigger PVE notification system directly${NC}"
|
||
info "Using 'pvesh create /notifications/endpoints/...' to test PVE's own system."
|
||
info "This sends a test notification through PVE, which should hit our webhook."
|
||
|
||
local before
|
||
before=$(snapshot_history)
|
||
|
||
# PVE 8.x has a test endpoint for notifications
|
||
if pvesh create /notifications/targets/test --target proxmenux-webhook 2>&1 | tee -a "$LOG_FILE"; then
|
||
ok "PVE test notification sent!"
|
||
else
|
||
# Try alternative method
|
||
info "Direct test not available. Trying via API..."
|
||
pvesh set /notifications/endpoints/webhook/proxmenux-webhook --test 1 2>/dev/null || \
|
||
warn "Could not send PVE test notification (requires PVE 8.1+)"
|
||
fi
|
||
|
||
wait_webhook 8
|
||
check_new_events "$before"
|
||
}
|
||
|
||
# ============================================================================
|
||
# TEST CATEGORY: VM/CT LIFECYCLE
|
||
# ============================================================================
|
||
test_vmct() {
|
||
header "VM/CT LIFECYCLE TESTS"
|
||
|
||
if [ -z "$VMID" ]; then
|
||
warn "No stopped VM/CT found for lifecycle tests."
|
||
info "Create a minimal CT: pct create 9999 local:vztmpl/debian-12-standard_12.2-1_amd64.tar.zst --storage local-lvm --memory 128 --cores 1"
|
||
return
|
||
fi
|
||
|
||
log ""
|
||
log "${BOLD} Test V1: Start ${VMTYPE} ${VMID} (${VMNAME})${NC}"
|
||
|
||
if confirm "Start ${VMTYPE} ${VMID}? It will be stopped again after the test."; then
|
||
local before
|
||
before=$(snapshot_history)
|
||
|
||
if [ "$VMTYPE" = "lxc" ]; then
|
||
pct start "$VMID" 2>&1 | tee -a "$LOG_FILE" || true
|
||
else
|
||
qm start "$VMID" 2>&1 | tee -a "$LOG_FILE" || true
|
||
fi
|
||
|
||
ok "Start command sent."
|
||
wait_webhook 10
|
||
check_new_events "$before"
|
||
|
||
# Wait a moment
|
||
sleep 5
|
||
|
||
# ── Test V2: Stop ──
|
||
log ""
|
||
log "${BOLD} Test V2: Stop ${VMTYPE} ${VMID}${NC}"
|
||
|
||
before=$(snapshot_history)
|
||
|
||
if [ "$VMTYPE" = "lxc" ]; then
|
||
pct stop "$VMID" 2>&1 | tee -a "$LOG_FILE" || true
|
||
else
|
||
qm stop "$VMID" 2>&1 | tee -a "$LOG_FILE" || true
|
||
fi
|
||
|
||
ok "Stop command sent."
|
||
wait_webhook 10
|
||
check_new_events "$before"
|
||
fi
|
||
}
|
||
|
||
# ============================================================================
|
||
# TEST CATEGORY: SYSTEM EVENTS (via syslog injection)
|
||
# ============================================================================
|
||
test_system() {
|
||
header "SYSTEM EVENT TESTS (syslog injection)"
|
||
|
||
# ── Test S1: Authentication failures ──
|
||
log ""
|
||
log "${BOLD} Test S1: SSH auth failure injection${NC}"
|
||
info "Injecting SSH auth failure messages into syslog."
|
||
|
||
local before
|
||
before=$(snapshot_history)
|
||
|
||
logger -t sshd -p auth.warning "Failed password for root from 192.168.1.200 port 44312 ssh2"
|
||
sleep 2
|
||
logger -t sshd -p auth.warning "Failed password for invalid user admin from 10.0.0.50 port 55123 ssh2"
|
||
sleep 2
|
||
logger -t sshd -p auth.warning "Failed password for root from 192.168.1.200 port 44315 ssh2"
|
||
|
||
wait_webhook 8
|
||
check_new_events "$before"
|
||
|
||
# ── Test S2: Firewall event ──
|
||
log ""
|
||
log "${BOLD} Test S2: Firewall drop event${NC}"
|
||
|
||
before=$(snapshot_history)
|
||
|
||
logger -t kernel -p kern.warning "pve-fw-reject: IN=vmbr0 OUT= MAC=00:11:22:33:44:55 SRC=10.0.0.99 DST=192.168.1.1 PROTO=TCP DPT=22 REJECT"
|
||
sleep 2
|
||
logger -t pvefw -p daemon.warning "firewall: blocked incoming connection from 10.0.0.99:45678 to 192.168.1.1:8006"
|
||
|
||
wait_webhook 8
|
||
check_new_events "$before"
|
||
|
||
# ── Test S3: Service failure ──
|
||
log ""
|
||
log "${BOLD} Test S3: Service failure injection${NC}"
|
||
|
||
before=$(snapshot_history)
|
||
|
||
logger -t systemd -p daemon.err "pvedaemon.service: Main process exited, code=exited, status=1/FAILURE"
|
||
sleep 1
|
||
logger -t systemd -p daemon.err "Failed to start Proxmox VE API Daemon."
|
||
|
||
wait_webhook 8
|
||
check_new_events "$before"
|
||
}
|
||
|
||
# ============================================================================
|
||
# SUMMARY & REPORT
|
||
# ============================================================================
|
||
show_summary() {
|
||
header "TEST SUMMARY"
|
||
|
||
info "Fetching full notification history..."
|
||
echo ""
|
||
|
||
curl -s "${API}/api/notifications/history?limit=200" 2>/dev/null | python3 -c "
|
||
import sys, json
|
||
from collections import Counter
|
||
|
||
data = json.load(sys.stdin)
|
||
history = data.get('history', [])
|
||
|
||
if not history:
|
||
print(' No notifications in history.')
|
||
sys.exit(0)
|
||
|
||
# Group by event_type
|
||
by_type = Counter(h['event_type'] for h in history)
|
||
# Group by severity
|
||
by_sev = Counter(h.get('severity', '?') for h in history)
|
||
# Group by source
|
||
by_src = Counter(h.get('source', '?') for h in history)
|
||
|
||
print(f' Total notifications: {len(history)}')
|
||
print()
|
||
|
||
sev_icons = {'CRITICAL': '\033[0;31mCRITICAL\033[0m', 'WARNING': '\033[1;33mWARNING\033[0m', 'INFO': '\033[0;36mINFO\033[0m'}
|
||
print(' By severity:')
|
||
for sev, count in by_sev.most_common():
|
||
icon = sev_icons.get(sev, sev)
|
||
print(f' {icon}: {count}')
|
||
|
||
print()
|
||
print(' By source:')
|
||
for src, count in by_src.most_common():
|
||
print(f' {src:20s}: {count}')
|
||
|
||
print()
|
||
print(' By event type:')
|
||
for etype, count in by_type.most_common():
|
||
print(f' {etype:30s}: {count}')
|
||
|
||
print()
|
||
print(' Latest 15 events:')
|
||
for h in history[:15]:
|
||
sev = h.get('severity', '?')
|
||
icon = {'CRITICAL': ' \033[0;31mRED\033[0m', 'WARNING': ' \033[1;33mYEL\033[0m', 'INFO': ' \033[0;36mBLU\033[0m'}.get(sev, ' ???')
|
||
ts = h.get('sent_at', '?')[:19]
|
||
src = h.get('source', '?')[:12]
|
||
print(f' {icon} {ts} {src:12s} {h[\"event_type\"]:25s} {h.get(\"title\", \"\")[:50]}')
|
||
" 2>/dev/null | tee -a "$LOG_FILE"
|
||
|
||
echo ""
|
||
info "Full log saved to: ${LOG_FILE}"
|
||
echo ""
|
||
info "To see all history:"
|
||
echo -e " ${CYAN}curl -s '${API}/api/notifications/history?limit=200' | python3 -m json.tool${NC}"
|
||
echo ""
|
||
info "To check Telegram delivery, look at your Telegram bot chat."
|
||
}
|
||
|
||
# ============================================================================
|
||
# INTERACTIVE MENU
|
||
# ============================================================================
|
||
show_menu() {
|
||
echo ""
|
||
echo -e "${BOLD} ProxMenux Real Event Test Suite${NC}"
|
||
echo ""
|
||
echo -e " ${CYAN}1)${NC} Disk error tests (SMART, ZFS, I/O, space pressure)"
|
||
echo -e " ${CYAN}2)${NC} Backup tests (vzdump success/fail, snapshots)"
|
||
echo -e " ${CYAN}3)${NC} VM/CT lifecycle tests (start/stop real VMs)"
|
||
echo -e " ${CYAN}4)${NC} System event tests (auth, firewall, service failures)"
|
||
echo -e " ${CYAN}5)${NC} Run ALL tests"
|
||
echo -e " ${CYAN}6)${NC} Show summary report"
|
||
echo -e " ${CYAN}q)${NC} Exit"
|
||
echo ""
|
||
echo -ne " Select: "
|
||
}
|
||
|
||
# ── Main ────────────────────────────────────────────────────────
|
||
main() {
|
||
local mode="${1:-menu}"
|
||
|
||
echo ""
|
||
echo -e "${BOLD}============================================================${NC}"
|
||
echo -e "${BOLD} ProxMenux - Real Proxmox Event Simulator${NC}"
|
||
echo -e "${BOLD}============================================================${NC}"
|
||
echo -e " Tests REAL events through the full PVE -> webhook pipeline."
|
||
echo -e " Log file: ${CYAN}${LOG_FILE}${NC}"
|
||
echo ""
|
||
|
||
preflight
|
||
|
||
case "$mode" in
|
||
disk) test_disk; show_summary ;;
|
||
backup) test_backup; show_summary ;;
|
||
vmct) test_vmct; show_summary ;;
|
||
system) test_system; show_summary ;;
|
||
all)
|
||
test_disk
|
||
test_backup
|
||
test_vmct
|
||
test_system
|
||
show_summary
|
||
;;
|
||
menu|*)
|
||
while true; do
|
||
show_menu
|
||
read -r choice
|
||
case "$choice" in
|
||
1) test_disk ;;
|
||
2) test_backup ;;
|
||
3) test_vmct ;;
|
||
4) test_system ;;
|
||
5) test_disk; test_backup; test_vmct; test_system; show_summary; break ;;
|
||
6) show_summary ;;
|
||
q|Q) echo " Bye!"; break ;;
|
||
*) warn "Invalid option" ;;
|
||
esac
|
||
done
|
||
;;
|
||
esac
|
||
}
|
||
|
||
main "${1:-menu}"
|