diff --git a/AppImage/ProxMenux-1.0.1.AppImage b/AppImage/ProxMenux-1.0.1.AppImage deleted file mode 100755 index 2011a10e..00000000 Binary files a/AppImage/ProxMenux-1.0.1.AppImage and /dev/null differ diff --git a/AppImage/ProxMenux-Monitor.AppImage.sha256 b/AppImage/ProxMenux-Monitor.AppImage.sha256 deleted file mode 100644 index 8ae47f12..00000000 --- a/AppImage/ProxMenux-Monitor.AppImage.sha256 +++ /dev/null @@ -1 +0,0 @@ -f35de512c1a19843d15a9a3263a5104759d041ffc9d01249450babe0b0c3f889 ProxMenux-1.0.1.AppImage diff --git a/AppImage/README.md b/AppImage/README.md index f90c0a9d..28ba074b 100644 --- a/AppImage/README.md +++ b/AppImage/README.md @@ -730,6 +730,23 @@ entities: ![Home Assistant Integration Example](AppImage/public/images/docs/homeassistant-integration.png) +--- + +## License + +This project is licensed under the **Creative Commons Attribution-NonCommercial 4.0 International License (CC BY-NC 4.0)**. + +You are free to: +- Share — copy and redistribute the material in any medium or format +- Adapt — remix, transform, and build upon the material + +Under the following terms: +- Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made +- NonCommercial — You may not use the material for commercial purposes + +For more details, see the [full license](https://creativecommons.org/licenses/by-nc/4.0/). + + --- diff --git a/AppImage/components/latency-detail-modal.tsx b/AppImage/components/latency-detail-modal.tsx index 16a4076d..a51e107c 100644 --- a/AppImage/components/latency-detail-modal.tsx +++ b/AppImage/components/latency-detail-modal.tsx @@ -259,36 +259,29 @@ const generateLatencyReport = (report: ReportData) => { .rpt-footer { color: #4b5563; } } @media screen { - body { max-width: 1000px; margin: 0 auto; padding: 24px 32px; padding-top: 64px; } + body { max-width: 1000px; margin: 0 auto; padding: 24px 32px; padding-top: 80px; } } - /* Top bar for screen only */ + /* Top bar for screen only - uses larger sizes to be visible when scaled on mobile */ .top-bar { position: fixed; top: 0; left: 0; right: 0; background: #0f172a; color: #e2e8f0; - padding: 12px 24px; display: flex; align-items: center; justify-content: space-between; z-index: 100; - font-size: 13px; + padding: 16px 32px; display: flex; align-items: center; justify-content: space-between; z-index: 100; + font-size: 18px; } .top-bar button { - background: #06b6d4; color: #fff; border: none; padding: 8px 20px; border-radius: 6px; - font-size: 13px; font-weight: 600; cursor: pointer; + background: #06b6d4; color: #fff; border: none; padding: 14px 32px; border-radius: 8px; + font-size: 18px; font-weight: 600; cursor: pointer; } .top-bar button:hover { background: #0891b2; } .top-bar .close-btn { background: rgba(255,255,255,0.1); color: #fff; border: 1px solid rgba(255,255,255,0.2); - padding: 6px 12px; border-radius: 6px; display: flex; align-items: center; gap: 6px; - cursor: pointer; font-size: 13px; font-weight: 500; + padding: 14px 24px; border-radius: 8px; display: flex; align-items: center; gap: 8px; + cursor: pointer; font-size: 18px; font-weight: 500; } .top-bar .close-btn:hover { background: rgba(255,255,255,0.2); } - .top-bar .close-btn .close-text { display: none; } + .top-bar .close-btn .close-text { display: inline; } .hide-mobile { } @media print { .top-bar { display: none; } body { padding-top: 0; } } - @media screen and (max-width: 600px) { - .top-bar { padding: 10px 12px; } - .hide-mobile { display: none !important; } - .top-bar .close-btn { padding: 8px 16px; font-size: 14px; } - .top-bar .close-btn .close-text { display: inline; } - body { padding-top: 60px; } - } /* Header */ .rpt-header { diff --git a/AppImage/components/security.tsx b/AppImage/components/security.tsx index 8c933f76..7e1cc1e4 100644 --- a/AppImage/components/security.tsx +++ b/AppImage/components/security.tsx @@ -989,36 +989,29 @@ export function Security() { [style*="color:#0891b2"] { color: #0891b2 !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; } } @media screen { - body { max-width: 1000px; margin: 0 auto; padding: 24px 32px; padding-top: 64px; } + body { max-width: 1000px; margin: 0 auto; padding: 24px 32px; padding-top: 80px; } } - /* Top bar for screen only */ + /* Top bar for screen only - uses larger sizes to be visible when scaled on mobile */ .top-bar { position: fixed; top: 0; left: 0; right: 0; background: #0f172a; color: #e2e8f0; - padding: 12px 24px; display: flex; align-items: center; justify-content: space-between; z-index: 100; - font-size: 13px; + padding: 16px 32px; display: flex; align-items: center; justify-content: space-between; z-index: 100; + font-size: 18px; } .top-bar button { - background: #06b6d4; color: #fff; border: none; padding: 8px 20px; border-radius: 6px; - font-size: 13px; font-weight: 600; cursor: pointer; + background: #06b6d4; color: #fff; border: none; padding: 14px 32px; border-radius: 8px; + font-size: 18px; font-weight: 600; cursor: pointer; } .top-bar button:hover { background: #0891b2; } .top-bar .close-btn { background: rgba(255,255,255,0.1); color: #fff; border: 1px solid rgba(255,255,255,0.2); - padding: 6px 12px; border-radius: 6px; display: flex; align-items: center; gap: 6px; - cursor: pointer; font-size: 13px; font-weight: 500; + padding: 14px 24px; border-radius: 8px; display: flex; align-items: center; gap: 8px; + cursor: pointer; font-size: 18px; font-weight: 500; } .top-bar .close-btn:hover { background: rgba(255,255,255,0.2); } - .top-bar .close-btn .close-text { display: none; } + .top-bar .close-btn .close-text { display: inline; } .hide-mobile { } @media print { .top-bar { display: none; } body { padding-top: 0; } } - @media screen and (max-width: 600px) { - .top-bar { padding: 10px 12px; } - .hide-mobile { display: none !important; } - .top-bar .close-btn { padding: 8px 16px; font-size: 14px; } - .top-bar .close-btn .close-text { display: inline; } - body { padding-top: 60px; } - } /* Header */ .rpt-header { diff --git a/AppImage/scripts/test_all_notifications.sh b/AppImage/scripts/test_all_notifications.sh new file mode 100644 index 00000000..725ebc5d --- /dev/null +++ b/AppImage/scripts/test_all_notifications.sh @@ -0,0 +1,481 @@ +#!/bin/bash +# ============================================================================ +# ProxMenux Notification System - Complete Test Suite +# ============================================================================ +# +# Usage: +# chmod +x test_all_notifications.sh +# ./test_all_notifications.sh # Run ALL tests (with 3s pause between) +# ./test_all_notifications.sh system # Run only System category +# ./test_all_notifications.sh vm_ct # Run only VM/CT category +# ./test_all_notifications.sh backup # Run only Backup category +# ./test_all_notifications.sh resources # Run only Resources category +# ./test_all_notifications.sh storage # Run only Storage category +# ./test_all_notifications.sh network # Run only Network category +# ./test_all_notifications.sh security # Run only Security category +# ./test_all_notifications.sh cluster # Run only Cluster category +# ./test_all_notifications.sh burst # Run only Burst aggregation tests +# +# Each test sends a simulated webhook to the local notification endpoint. +# Check your Telegram/Gotify/Discord/Email for the notifications. +# ============================================================================ + +API="http://127.0.0.1:8008/api/notifications/webhook" +PAUSE=3 # seconds between tests + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color +BOLD='\033[1m' + +test_count=0 +pass_count=0 +fail_count=0 + +send_test() { + local name="$1" + local payload="$2" + test_count=$((test_count + 1)) + + echo -e "${CYAN} [$test_count] ${BOLD}$name${NC}" + + response=$(curl -s -w "\n%{http_code}" -X POST "$API" \ + -H "Content-Type: application/json" \ + -d "$payload" 2>&1) + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | head -n -1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "202" ]; then + echo -e " ${GREEN}HTTP $http_code${NC} - $body" + pass_count=$((pass_count + 1)) + else + echo -e " ${RED}HTTP $http_code${NC} - $body" + fail_count=$((fail_count + 1)) + fi + + sleep "$PAUSE" +} + +# ============================================================================ +# SYSTEM CATEGORY (group: system) +# ============================================================================ +test_system() { + echo "" + echo -e "${YELLOW}========================================${NC}" + echo -e "${YELLOW} SYSTEM - Startup, shutdown, kernel${NC}" + echo -e "${YELLOW}========================================${NC}" + echo "" + + # 1. state_change (disabled by default -- test to verify it does NOT arrive) + send_test "state_change (should NOT arrive - disabled by default)" \ + '{"type":"state_change","component":"health","severity":"warning","title":"overall changed to WARNING","body":"overall status changed from OK to WARNING."}' + + # 2. new_error + send_test "new_error" \ + '{"type":"new_error","component":"health","severity":"warning","title":"New WARNING - cpu","body":"CPU usage exceeds 90% for more than 5 minutes","category":"cpu"}' + + # 3. error_resolved + send_test "error_resolved" \ + '{"type":"error_resolved","component":"health","severity":"info","title":"Resolved - cpu","body":"CPU usage returned to normal.\nDuration: 15 minutes","category":"cpu","duration":"15 minutes"}' + + # 4. error_escalated + send_test "error_escalated" \ + '{"type":"error_escalated","component":"health","severity":"critical","title":"Escalated to CRITICAL - memory","body":"Memory usage exceeded 95% and swap is active","category":"memory"}' + + # 5. system_shutdown + send_test "system_shutdown" \ + '{"type":"system_shutdown","component":"system","severity":"warning","title":"System shutting down","body":"The system is shutting down.\nUser initiated shutdown."}' + + # 6. system_reboot + send_test "system_reboot" \ + '{"type":"system_reboot","component":"system","severity":"warning","title":"System rebooting","body":"The system is rebooting.\nKernel update applied."}' + + # 7. system_problem + send_test "system_problem" \ + '{"type":"system_problem","component":"system","severity":"critical","title":"System problem detected","body":"Kernel panic: Attempted to kill init! exitcode=0x00000009"}' + + # 8. service_fail + send_test "service_fail" \ + '{"type":"service_fail","component":"systemd","severity":"warning","title":"Service failed - pvedaemon","body":"Service pvedaemon has failed.\nUnit pvedaemon.service entered failed state.","service_name":"pvedaemon"}' + + # 9. update_available (legacy, superseded by update_summary) + send_test "update_available" \ + '{"type":"update_available","component":"apt","severity":"info","title":"Updates available","body":"Total updates: 12\nSecurity: 3\nProxmox: 5\nKernel: 1\nImportant: pve-manager (8.3.5 -> 8.4.1)","total_count":"12","security_count":"3","pve_count":"5","kernel_count":"1","important_list":"pve-manager (8.3.5 -> 8.4.1)"}' + + # 10. update_complete + send_test "update_complete" \ + '{"type":"update_complete","component":"apt","severity":"info","title":"Update completed","body":"12 packages updated successfully."}' + + # 11. unknown_persistent + send_test "unknown_persistent" \ + '{"type":"unknown_persistent","component":"health","severity":"warning","title":"Check unavailable - temperature","body":"Health check for temperature has been unavailable for 3+ cycles.\nSensor not responding.","category":"temperature"}' + + # 12. health_persistent + send_test "health_persistent" \ + '{"type":"health_persistent","component":"health","severity":"warning","title":"3 active health issue(s)","body":"The following health issues remain active:\n- CPU at 92%\n- Memory at 88%\n- Disk /dev/sda at 94%\n\nThis digest is sent once every 24 hours while issues persist.","count":"3"}' + + # 13. health_issue_new + send_test "health_issue_new" \ + '{"type":"health_issue_new","component":"health","severity":"warning","title":"New health issue - disk","body":"New WARNING issue detected:\nDisk /dev/sda usage at 94%","category":"disk"}' + + # 14. health_issue_resolved + send_test "health_issue_resolved" \ + '{"type":"health_issue_resolved","component":"health","severity":"info","title":"Resolved - disk","body":"disk issue has been resolved.\nDisk usage dropped to 72%.\nDuration: 3 hours","category":"disk","duration":"3 hours"}' + + # 15. update_summary + send_test "update_summary" \ + '{"type":"update_summary","component":"apt","severity":"info","title":"Updates available","body":"Total updates: 70\nSecurity updates: 9\nProxmox-related updates: 24\nKernel updates: 1\nImportant packages: pve-manager (8.3.5 -> 8.4.1), proxmox-ve (8.3.0 -> 8.4.0), qemu-server (8.3.8 -> 8.4.2)","total_count":"70","security_count":"9","pve_count":"24","kernel_count":"1","important_list":"pve-manager (8.3.5 -> 8.4.1), proxmox-ve (8.3.0 -> 8.4.0), qemu-server (8.3.8 -> 8.4.2)"}' + + # 16. pve_update + send_test "pve_update" \ + '{"type":"pve_update","component":"apt","severity":"info","title":"Proxmox VE 8.4.1 available","body":"Proxmox VE 8.3.5 -> 8.4.1\npve-manager 8.3.5 -> 8.4.1","current_version":"8.3.5","new_version":"8.4.1","version":"8.4.1","details":"pve-manager 8.3.5 -> 8.4.1"}' +} + +# ============================================================================ +# VM / CT CATEGORY (group: vm_ct) +# ============================================================================ +test_vm_ct() { + echo "" + echo -e "${YELLOW}========================================${NC}" + echo -e "${YELLOW} VM / CT - Start, stop, crash, migration${NC}" + echo -e "${YELLOW}========================================${NC}" + echo "" + + # 1. vm_start + send_test "vm_start" \ + '{"type":"vm_start","component":"qemu","severity":"info","title":"VM 100 started","body":"ubuntu-server (100) has been started.","vmid":"100","vmname":"ubuntu-server"}' + + # 2. vm_stop + send_test "vm_stop" \ + '{"type":"vm_stop","component":"qemu","severity":"info","title":"VM 100 stopped","body":"ubuntu-server (100) has been stopped.","vmid":"100","vmname":"ubuntu-server"}' + + # 3. vm_shutdown + send_test "vm_shutdown" \ + '{"type":"vm_shutdown","component":"qemu","severity":"info","title":"VM 100 shutdown","body":"ubuntu-server (100) has been shut down.","vmid":"100","vmname":"ubuntu-server"}' + + # 4. vm_fail + send_test "vm_fail" \ + '{"type":"vm_fail","component":"qemu","severity":"critical","title":"VM 100 FAILED","body":"ubuntu-server (100) has failed.\nKVM: internal error: unexpected exit to hypervisor","vmid":"100","vmname":"ubuntu-server","reason":"KVM: internal error: unexpected exit to hypervisor"}' + + # 5. vm_restart + send_test "vm_restart" \ + '{"type":"vm_restart","component":"qemu","severity":"info","title":"VM 100 restarted","body":"ubuntu-server (100) has been restarted.","vmid":"100","vmname":"ubuntu-server"}' + + # 6. ct_start + send_test "ct_start" \ + '{"type":"ct_start","component":"lxc","severity":"info","title":"CT 200 started","body":"nginx-proxy (200) has been started.","vmid":"200","vmname":"nginx-proxy"}' + + # 7. ct_stop + send_test "ct_stop" \ + '{"type":"ct_stop","component":"lxc","severity":"info","title":"CT 200 stopped","body":"nginx-proxy (200) has been stopped.","vmid":"200","vmname":"nginx-proxy"}' + + # 8. ct_fail + send_test "ct_fail" \ + '{"type":"ct_fail","component":"lxc","severity":"critical","title":"CT 200 FAILED","body":"nginx-proxy (200) has failed.\nContainer exited with error code 137","vmid":"200","vmname":"nginx-proxy","reason":"Container exited with error code 137"}' + + # 9. migration_start + send_test "migration_start" \ + '{"type":"migration_start","component":"qemu","severity":"info","title":"Migration started - 100","body":"ubuntu-server (100) migration to pve-node2 started.","vmid":"100","vmname":"ubuntu-server","target_node":"pve-node2"}' + + # 10. migration_complete + send_test "migration_complete" \ + '{"type":"migration_complete","component":"qemu","severity":"info","title":"Migration complete - 100","body":"ubuntu-server (100) migrated successfully to pve-node2.","vmid":"100","vmname":"ubuntu-server","target_node":"pve-node2"}' + + # 11. migration_fail + send_test "migration_fail" \ + '{"type":"migration_fail","component":"qemu","severity":"critical","title":"Migration FAILED - 100","body":"ubuntu-server (100) migration to pve-node2 failed.\nNetwork timeout during memory transfer","vmid":"100","vmname":"ubuntu-server","target_node":"pve-node2","reason":"Network timeout during memory transfer"}' + + # 12. replication_fail + send_test "replication_fail" \ + '{"type":"replication_fail","component":"replication","severity":"critical","title":"Replication FAILED - 100","body":"Replication of ubuntu-server (100) has failed.\nTarget storage unreachable","vmid":"100","vmname":"ubuntu-server","reason":"Target storage unreachable"}' + + # 13. replication_complete + send_test "replication_complete" \ + '{"type":"replication_complete","component":"replication","severity":"info","title":"Replication complete - 100","body":"Replication of ubuntu-server (100) completed successfully.","vmid":"100","vmname":"ubuntu-server"}' +} + +# ============================================================================ +# BACKUP CATEGORY (group: backup) +# ============================================================================ +test_backup() { + echo "" + echo -e "${YELLOW}========================================${NC}" + echo -e "${YELLOW} BACKUPS - Backup start, complete, fail${NC}" + echo -e "${YELLOW}========================================${NC}" + echo "" + + # 1. backup_start + send_test "backup_start" \ + '{"type":"backup_start","component":"vzdump","severity":"info","title":"Backup started - 100","body":"Backup of ubuntu-server (100) has started.","vmid":"100","vmname":"ubuntu-server"}' + + # 2. backup_complete + send_test "backup_complete" \ + '{"type":"backup_complete","component":"vzdump","severity":"info","title":"Backup complete - 100","body":"Backup of ubuntu-server (100) completed successfully.\nSize: 12.4 GB","vmid":"100","vmname":"ubuntu-server","size":"12.4 GB"}' + + # 3. backup_fail + send_test "backup_fail" \ + '{"type":"backup_fail","component":"vzdump","severity":"critical","title":"Backup FAILED - 100","body":"Backup of ubuntu-server (100) has failed.\nStorage local-lvm is full","vmid":"100","vmname":"ubuntu-server","reason":"Storage local-lvm is full"}' + + # 4. snapshot_complete + send_test "snapshot_complete" \ + '{"type":"snapshot_complete","component":"qemu","severity":"info","title":"Snapshot created - 100","body":"Snapshot of ubuntu-server (100) created: pre-upgrade-2026","vmid":"100","vmname":"ubuntu-server","snapshot_name":"pre-upgrade-2026"}' + + # 5. snapshot_fail + send_test "snapshot_fail" \ + '{"type":"snapshot_fail","component":"qemu","severity":"critical","title":"Snapshot FAILED - 100","body":"Snapshot of ubuntu-server (100) failed.\nInsufficient space on storage","vmid":"100","vmname":"ubuntu-server","reason":"Insufficient space on storage"}' +} + +# ============================================================================ +# RESOURCES CATEGORY (group: resources) +# ============================================================================ +test_resources() { + echo "" + echo -e "${YELLOW}========================================${NC}" + echo -e "${YELLOW} RESOURCES - CPU, memory, temperature${NC}" + echo -e "${YELLOW}========================================${NC}" + echo "" + + # 1. cpu_high + send_test "cpu_high" \ + '{"type":"cpu_high","component":"health","severity":"warning","title":"High CPU usage (94%)","body":"CPU usage is at 94% on 16 cores.\nTop process: kvm (VM 100)","value":"94","cores":"16","details":"Top process: kvm (VM 100)"}' + + # 2. ram_high + send_test "ram_high" \ + '{"type":"ram_high","component":"health","severity":"warning","title":"High memory usage (91%)","body":"Memory usage: 58.2 GB / 64 GB (91%).\n4 VMs running, swap at 2.1 GB","value":"91","used":"58.2 GB","total":"64 GB","details":"4 VMs running, swap at 2.1 GB"}' + + # 3. temp_high + send_test "temp_high" \ + '{"type":"temp_high","component":"health","severity":"critical","title":"High temperature (89C)","body":"CPU temperature: 89C (threshold: 80C).\nCheck cooling system immediately","value":"89","threshold":"80","details":"Check cooling system immediately"}' + + # 4. load_high + send_test "load_high" \ + '{"type":"load_high","component":"health","severity":"warning","title":"High system load (24.5)","body":"System load average: 24.5 on 16 cores.\nI/O wait: 35%","value":"24.5","cores":"16","details":"I/O wait: 35%"}' +} + +# ============================================================================ +# STORAGE CATEGORY (group: storage) +# ============================================================================ +test_storage() { + echo "" + echo -e "${YELLOW}========================================${NC}" + echo -e "${YELLOW} STORAGE - Disk space, I/O errors, SMART${NC}" + echo -e "${YELLOW}========================================${NC}" + echo "" + + # 1. disk_space_low + send_test "disk_space_low" \ + '{"type":"disk_space_low","component":"storage","severity":"warning","title":"Low disk space on /var","body":"/var: 93% used (4.2 GB available).","mount":"/var","used":"93","available":"4.2 GB"}' + + # 2. disk_io_error + send_test "disk_io_error" \ + '{"type":"disk_io_error","component":"smart","severity":"critical","title":"Disk I/O error","body":"I/O error detected on /dev/sdb.\nSMART error: Current Pending Sector Count = 8","device":"/dev/sdb","reason":"SMART error: Current Pending Sector Count = 8"}' + + # 3. burst_disk_io + send_test "burst_disk_io" \ + '{"type":"burst_disk_io","component":"storage","severity":"critical","title":"5 disk I/O errors on /dev/sdb, /dev/sdc","body":"5 I/O errors detected in 60s.\nDevices: /dev/sdb, /dev/sdc","count":"5","window":"60s","entity_list":"/dev/sdb, /dev/sdc"}' +} + +# ============================================================================ +# NETWORK CATEGORY (group: network) +# ============================================================================ +test_network() { + echo "" + echo -e "${YELLOW}========================================${NC}" + echo -e "${YELLOW} NETWORK - Connectivity, bond, latency${NC}" + echo -e "${YELLOW}========================================${NC}" + echo "" + + # 1. network_down + send_test "network_down" \ + '{"type":"network_down","component":"network","severity":"critical","title":"Network connectivity lost","body":"Network connectivity check failed.\nGateway 192.168.1.1 unreachable. Bond vmbr0 degraded.","reason":"Gateway 192.168.1.1 unreachable. Bond vmbr0 degraded."}' + + # 2. network_latency + send_test "network_latency" \ + '{"type":"network_latency","component":"network","severity":"warning","title":"High network latency (450ms)","body":"Latency to gateway: 450ms (threshold: 100ms).","value":"450","threshold":"100"}' +} + +# ============================================================================ +# SECURITY CATEGORY (group: security) +# ============================================================================ +test_security() { + echo "" + echo -e "${YELLOW}========================================${NC}" + echo -e "${YELLOW} SECURITY - Auth failures, fail2ban, firewall${NC}" + echo -e "${YELLOW}========================================${NC}" + echo "" + + # 1. auth_fail + send_test "auth_fail" \ + '{"type":"auth_fail","component":"auth","severity":"warning","title":"Authentication failure","body":"Failed login attempt from 203.0.113.42.\nUser: root\nService: sshd","source_ip":"203.0.113.42","username":"root","service":"sshd"}' + + # 2. ip_block + send_test "ip_block" \ + '{"type":"ip_block","component":"security","severity":"info","title":"IP blocked by Fail2Ban","body":"IP 203.0.113.42 has been banned.\nJail: sshd\nFailures: 5","source_ip":"203.0.113.42","jail":"sshd","failures":"5"}' + + # 3. firewall_issue + send_test "firewall_issue" \ + '{"type":"firewall_issue","component":"firewall","severity":"warning","title":"Firewall issue detected","body":"Firewall rule conflict detected on vmbr0.\nRule 15 overlaps with rule 23, potentially blocking cluster traffic.","reason":"Firewall rule conflict detected on vmbr0. Rule 15 overlaps with rule 23."}' + + # 4. user_permission_change + send_test "user_permission_change" \ + '{"type":"user_permission_change","component":"auth","severity":"info","title":"User permission changed","body":"User: admin@pam\nChange: Added PVEAdmin role on /vms/100","username":"admin@pam","change_details":"Added PVEAdmin role on /vms/100"}' + + # 5. burst_auth_fail + send_test "burst_auth_fail" \ + '{"type":"burst_auth_fail","component":"security","severity":"warning","title":"8 auth failures in 2m","body":"8 authentication failures detected in 2m.\nSources: 203.0.113.42, 198.51.100.7, 192.0.2.15","count":"8","window":"2m","entity_list":"203.0.113.42, 198.51.100.7, 192.0.2.15"}' + + # 6. burst_ip_block + send_test "burst_ip_block" \ + '{"type":"burst_ip_block","component":"security","severity":"info","title":"Fail2Ban banned 4 IPs in 5m","body":"4 IPs banned by Fail2Ban in 5m.\nIPs: 203.0.113.42, 198.51.100.7, 192.0.2.15, 10.0.0.99","count":"4","window":"5m","entity_list":"203.0.113.42, 198.51.100.7, 192.0.2.15, 10.0.0.99"}' +} + +# ============================================================================ +# CLUSTER CATEGORY (group: cluster) +# ============================================================================ +test_cluster() { + echo "" + echo -e "${YELLOW}========================================${NC}" + echo -e "${YELLOW} CLUSTER - Quorum, split-brain, HA fencing${NC}" + echo -e "${YELLOW}========================================${NC}" + echo "" + + # 1. split_brain + send_test "split_brain" \ + '{"type":"split_brain","component":"cluster","severity":"critical","title":"SPLIT-BRAIN detected","body":"Cluster split-brain condition detected.\nQuorum status: No quorum - 1/3 nodes visible","quorum":"No quorum - 1/3 nodes visible"}' + + # 2. node_disconnect + send_test "node_disconnect" \ + '{"type":"node_disconnect","component":"corosync","severity":"critical","title":"Node disconnected","body":"Node pve-node3 has disconnected from the cluster.","node_name":"pve-node3"}' + + # 3. node_reconnect + send_test "node_reconnect" \ + '{"type":"node_reconnect","component":"corosync","severity":"info","title":"Node reconnected","body":"Node pve-node3 has reconnected to the cluster.","node_name":"pve-node3"}' + + # 4. burst_cluster + send_test "burst_cluster" \ + '{"type":"burst_cluster","component":"cluster","severity":"critical","title":"Cluster flapping detected (6 changes)","body":"Cluster state changed 6 times in 5m.\nNodes: pve-node2, pve-node3","count":"6","window":"5m","entity_list":"pve-node2, pve-node3"}' +} + +# ============================================================================ +# BURST AGGREGATION TESTS (send rapid events to trigger burst detection) +# ============================================================================ +test_burst() { + echo "" + echo -e "${YELLOW}========================================${NC}" + echo -e "${YELLOW} BURST - Rapid events to trigger aggregation${NC}" + echo -e "${YELLOW}========================================${NC}" + echo "" + + echo -e "${BLUE} Sending 5 rapid auth_fail events (should trigger burst_auth_fail)...${NC}" + for i in $(seq 1 5); do + curl -s -X POST "$API" \ + -H "Content-Type: application/json" \ + -d "{\"type\":\"auth_fail\",\"component\":\"auth\",\"severity\":\"warning\",\"title\":\"Auth fail from 10.0.0.$i\",\"body\":\"Failed login from 10.0.0.$i\",\"source_ip\":\"10.0.0.$i\"}" > /dev/null + echo -e " ${CYAN}Sent auth_fail $i/5${NC}" + sleep 0.5 + done + echo -e " ${GREEN}Done. Wait ~10s for burst aggregation...${NC}" + sleep 10 + + echo "" + echo -e "${BLUE} Sending 4 rapid disk_io_error events (should trigger burst_disk_io)...${NC}" + for i in $(seq 1 4); do + curl -s -X POST "$API" \ + -H "Content-Type: application/json" \ + -d "{\"type\":\"disk_io_error\",\"component\":\"smart\",\"severity\":\"critical\",\"title\":\"I/O error on /dev/sd${i}\",\"body\":\"Error on device\",\"device\":\"/dev/sd${i}\"}" > /dev/null + echo -e " ${CYAN}Sent disk_io_error $i/4${NC}" + sleep 0.5 + done + echo -e " ${GREEN}Done. Wait ~10s for burst aggregation...${NC}" + sleep 10 + + echo "" + echo -e "${BLUE} Sending 3 rapid node_disconnect events (should trigger burst_cluster)...${NC}" + for i in $(seq 1 3); do + curl -s -X POST "$API" \ + -H "Content-Type: application/json" \ + -d "{\"type\":\"node_disconnect\",\"component\":\"corosync\",\"severity\":\"critical\",\"title\":\"Node pve-node$i disconnected\",\"body\":\"Node lost\",\"node_name\":\"pve-node$i\"}" > /dev/null + echo -e " ${CYAN}Sent node_disconnect $i/3${NC}" + sleep 0.5 + done + echo -e " ${GREEN}Done. Wait ~10s for burst aggregation...${NC}" + sleep 10 +} + +# ============================================================================ +# MAIN +# ============================================================================ + +echo "" +echo -e "${BOLD}============================================================${NC}" +echo -e "${BOLD} ProxMenux Notification System - Complete Test Suite${NC}" +echo -e "${BOLD}============================================================${NC}" +echo -e " API: $API" +echo -e " Pause: ${PAUSE}s between tests" +echo "" + +# Check that the service is reachable +status=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:8008/api/notifications/status" 2>/dev/null) +if [ "$status" != "200" ]; then + echo -e "${RED}ERROR: Notification service not reachable (HTTP $status)${NC}" + echo -e " Make sure ProxMenux Monitor is running." + exit 1 +fi +echo -e "${GREEN}Service is reachable.${NC}" + +# Parse argument +category="${1:-all}" + +case "$category" in + system) test_system ;; + vm_ct) test_vm_ct ;; + backup) test_backup ;; + resources) test_resources ;; + storage) test_storage ;; + network) test_network ;; + security) test_security ;; + cluster) test_cluster ;; + burst) test_burst ;; + all) + test_system + test_vm_ct + test_backup + test_resources + test_storage + test_network + test_security + test_cluster + test_burst + ;; + *) + echo -e "${RED}Unknown category: $category${NC}" + echo "Usage: $0 [system|vm_ct|backup|resources|storage|network|security|cluster|burst|all]" + exit 1 + ;; +esac + +# ============================================================================ +# SUMMARY +# ============================================================================ +echo "" +echo -e "${BOLD}============================================================${NC}" +echo -e "${BOLD} SUMMARY${NC}" +echo -e "${BOLD}============================================================${NC}" +echo -e " Total tests: $test_count" +echo -e " ${GREEN}Accepted:${NC} $pass_count" +echo -e " ${RED}Rejected:${NC} $fail_count" +echo "" +echo -e " Check your notification channels for the messages." +echo -e " Note: Some events may be filtered by your current settings" +echo -e " (severity filter, disabled categories, disabled individual events)." +echo "" +echo -e " To check notification history (all events):" +echo -e " ${CYAN}curl -s 'http://127.0.0.1:8008/api/notifications/history?limit=200' | python3 -m json.tool${NC}" +echo "" +echo -e " To count events by type:" +echo -e " ${CYAN}curl -s 'http://127.0.0.1:8008/api/notifications/history?limit=200' | python3 -c \"import sys,json; h=json.load(sys.stdin)['history']; [print(f' {t}: {c}') for t,c in sorted(dict((e['event_type'],sum(1 for x in h if x['event_type']==e['event_type'])) for e in h).items())]\"${NC} +echo "" diff --git a/AppImage/scripts/test_real_events.sh b/AppImage/scripts/test_real_events.sh new file mode 100644 index 00000000..8e377f76 --- /dev/null +++ b/AppImage/scripts/test_real_events.sh @@ -0,0 +1,732 @@ +#!/bin/bash +# ============================================================================ +# ProxMenux - Real Proxmox Event Simulator +# ============================================================================ +# This script triggers ACTUAL events on Proxmox so that PVE's notification +# system fires real webhooks through the full pipeline: +# +# PVE event -> PVE notification -> webhook POST -> our pipeline -> Telegram +# +# Unlike test_all_notifications.sh (which injects directly via API), this +# script makes Proxmox generate the events itself. +# +# Usage: +# chmod +x test_real_events.sh +# ./test_real_events.sh # interactive menu +# ./test_real_events.sh disk # run disk tests only +# ./test_real_events.sh backup # run backup tests only +# ./test_real_events.sh all # run all tests +# ============================================================================ + +set -euo pipefail + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +BOLD='\033[1m' +NC='\033[0m' + +API="http://127.0.0.1:8008" +LOG_FILE="/tmp/proxmenux_real_test_$(date +%Y%m%d_%H%M%S).log" + +# ── Helpers ───────────────────────────────────────────────────── +log() { echo -e "$1" | tee -a "$LOG_FILE"; } +header() { + echo "" | tee -a "$LOG_FILE" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" | tee -a "$LOG_FILE" + echo -e "${BOLD} $1${NC}" | tee -a "$LOG_FILE" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" | tee -a "$LOG_FILE" +} + +warn() { log "${YELLOW} [!] $1${NC}"; } +ok() { log "${GREEN} [OK] $1${NC}"; } +fail() { log "${RED} [FAIL] $1${NC}"; } +info() { log "${CYAN} [i] $1${NC}"; } + +confirm() { + echo "" + echo -e "${YELLOW} $1${NC}" + echo -ne " Continue? [Y/n]: " + read -r ans + [[ -z "$ans" || "$ans" =~ ^[Yy] ]] +} + +wait_webhook() { + local seconds=${1:-10} + log " Waiting ${seconds}s for webhook delivery..." + sleep "$seconds" +} + +snapshot_history() { + curl -s "${API}/api/notifications/history?limit=200" 2>/dev/null | python3 -c " +import sys, json +try: + data = json.load(sys.stdin) + count = len(data.get('history', [])) + print(count) +except: + print(0) +" 2>/dev/null || echo "0" +} + +check_new_events() { + local before=$1 + local after + after=$(snapshot_history) + local diff=$((after - before)) + if [ "$diff" -gt 0 ]; then + ok "Received $diff new notification(s) via webhook" + # Show the latest events + curl -s "${API}/api/notifications/history?limit=$((diff + 2))" 2>/dev/null | python3 -c " +import sys, json +data = json.load(sys.stdin) +for h in data.get('history', [])[:$diff]: + sev = h.get('severity', '?') + icon = {'CRITICAL': ' RED', 'WARNING': ' YEL', 'INFO': ' BLU'}.get(sev, ' ???') + print(f'{icon} {h[\"event_type\"]:25s} {h.get(\"title\", \"\")[:60]}') +" 2>/dev/null | tee -a "$LOG_FILE" + else + warn "No new notifications detected (may need more time or check filters)" + fi +} + +# ── Pre-flight checks ────────────────────────────────────────── +preflight() { + header "Pre-flight Checks" + + # Check if running as root + if [ "$(id -u)" -ne 0 ]; then + fail "This script must be run as root" + exit 1 + fi + ok "Running as root" + + # Check ProxMenux is running + if curl -s "${API}/api/health" >/dev/null 2>&1; then + ok "ProxMenux Monitor is running" + else + fail "ProxMenux Monitor not reachable at ${API}" + exit 1 + fi + + # Check webhook is configured by querying PVE directly + if pvesh get /cluster/notifications/endpoints/webhook --output-format json 2>/dev/null | python3 -c " +import sys, json +endpoints = json.load(sys.stdin) +found = any('proxmenux' in e.get('name','').lower() for e in (endpoints if isinstance(endpoints, list) else [endpoints])) +exit(0 if found else 1) +" 2>/dev/null; then + ok "PVE webhook endpoint 'proxmenux-webhook' is configured" + else + warn "PVE webhook may not be configured. Run setup from the UI first." + if ! confirm "Continue anyway?"; then + exit 1 + fi + fi + + # Check notification config + # API returns { config: { enabled: true/false/'true'/'false', ... }, success: true } + if curl -s "${API}/api/notifications/settings" 2>/dev/null | python3 -c " +import sys, json +d = json.load(sys.stdin) +cfg = d.get('config', d) +enabled = cfg.get('enabled', False) +exit(0 if enabled is True or str(enabled).lower() == 'true' else 1) +" 2>/dev/null; then + ok "Notifications are enabled" + else + fail "Notifications are NOT enabled. Enable them in the UI first." + exit 1 + fi + + # Re-run webhook setup to ensure priv config and body template exist + info "Re-configuring PVE webhook (ensures priv config + body template)..." + local setup_result + setup_result=$(curl -s -X POST "${API}/api/notifications/proxmox/setup-webhook" 2>/dev/null) + if echo "$setup_result" | python3 -c "import sys,json; d=json.load(sys.stdin); exit(0 if d.get('configured') else 1)" 2>/dev/null; then + ok "PVE webhook re-configured successfully" + else + local setup_err + setup_err=$(echo "$setup_result" | python3 -c "import sys,json; print(json.load(sys.stdin).get('error','unknown'))" 2>/dev/null) + warn "Webhook setup returned: ${setup_err}" + warn "PVE webhook events may not work. Manual commands below:" + echo "$setup_result" | python3 -c " +import sys, json +d = json.load(sys.stdin) +for cmd in d.get('fallback_commands', []): + print(f' {cmd}') +" 2>/dev/null + if ! confirm "Continue anyway?"; then + exit 1 + fi + fi + + # Find a VM/CT for testing + VMID="" + VMNAME="" + VMTYPE="" + + # Try to find a stopped CT first (safest) + local cts + cts=$(pvesh get /cluster/resources --type vm --output-format json 2>/dev/null || echo "[]") + + # Look for a stopped container + VMID=$(echo "$cts" | python3 -c " +import sys, json +vms = json.load(sys.stdin) +# Prefer stopped CTs, then stopped VMs +for v in sorted(vms, key=lambda x: (0 if x.get('type')=='lxc' else 1, 0 if x.get('status')=='stopped' else 1)): + if v.get('status') == 'stopped': + print(v.get('vmid', '')) + break +" 2>/dev/null || echo "") + + if [ -n "$VMID" ]; then + VMTYPE=$(echo "$cts" | python3 -c " +import sys, json +vms = json.load(sys.stdin) +for v in vms: + if str(v.get('vmid')) == '$VMID': + print(v.get('type', 'qemu')) + break +" 2>/dev/null) + VMNAME=$(echo "$cts" | python3 -c " +import sys, json +vms = json.load(sys.stdin) +for v in vms: + if str(v.get('vmid')) == '$VMID': + print(v.get('name', 'unknown')) + break +" 2>/dev/null) + ok "Found stopped ${VMTYPE} for testing: ${VMID} (${VMNAME})" + else + warn "No stopped VM/CT found. Backup tests will use ID 0 (host backup)." + fi + + # List available storage + info "Available storage:" + pvesh get /storage --output-format json 2>/dev/null | python3 -c " +import sys, json +stores = json.load(sys.stdin) +for s in stores: + sid = s.get('storage', '?') + stype = s.get('type', '?') + content = s.get('content', '?') + print(f' {sid:20s} type={stype:10s} content={content}') +" 2>/dev/null | tee -a "$LOG_FILE" || warn "Could not list storage" + + echo "" + log " Log file: ${LOG_FILE}" +} + +# ============================================================================ +# TEST CATEGORY: DISK ERRORS +# ============================================================================ +test_disk() { + header "DISK ERROR TESTS" + + # ── Test D1: SMART error injection ── + log "" + log "${BOLD} Test D1: SMART error log injection${NC}" + info "Writes a simulated SMART error to syslog so JournalWatcher catches it." + info "This tests the journal -> notification_events -> pipeline flow." + + local before + before=$(snapshot_history) + + # Inject a realistic SMART error into the system journal + logger -t kernel -p kern.err "ata1.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x6 frozen" + sleep 1 + logger -t kernel -p kern.crit "ata1.00: failed command: READ FPDMA QUEUED" + sleep 1 + logger -t smartd -p daemon.warning "Device: /dev/sda [SAT], 1 Currently unreadable (pending) sectors" + + wait_webhook 8 + check_new_events "$before" + + # ── Test D2: ZFS error simulation ── + log "" + log "${BOLD} Test D2: ZFS scrub error simulation${NC}" + + # Check if ZFS is available + if command -v zpool >/dev/null 2>&1; then + local zpools + zpools=$(zpool list -H -o name 2>/dev/null || echo "") + + if [ -n "$zpools" ]; then + local pool + pool=$(echo "$zpools" | head -1) + info "ZFS pool found: ${pool}" + info "Injecting ZFS checksum error into syslog (non-destructive)." + + before=$(snapshot_history) + + # Simulate ZFS error events via syslog (non-destructive) + logger -t kernel -p kern.warning "ZFS: pool '${pool}' has experienced an error" + sleep 1 + logger -t zfs-module -p daemon.err "CHECKSUM error on ${pool}:mirror-0/sda: zio error" + + wait_webhook 8 + check_new_events "$before" + else + warn "ZFS installed but no pools found. Skipping ZFS test." + fi + else + warn "ZFS not installed. Skipping ZFS test." + fi + + # ── Test D3: Filesystem space pressure ── + log "" + log "${BOLD} Test D3: Disk space pressure simulation${NC}" + info "Creates a large temporary file to fill disk, triggering space warnings." + info "The Health Monitor should detect low disk space within ~60s." + + # Check current free space on / + local free_pct + free_pct=$(df / | tail -1 | awk '{print 100-$5}' | tr -d '%') + info "Current free space on /: ${free_pct}%" + + if [ "$free_pct" -gt 15 ]; then + info "Disk has ${free_pct}% free. Need to reduce below threshold for test." + + # Calculate how much to fill (leave only 8% free) + local total_k free_k fill_k + total_k=$(df / | tail -1 | awk '{print $2}') + free_k=$(df / | tail -1 | awk '{print $4}') + fill_k=$((free_k - (total_k * 8 / 100))) + + if [ "$fill_k" -gt 0 ] && [ "$fill_k" -lt 50000000 ]; then + info "Will create ${fill_k}KB temp file to simulate low space." + + if confirm "This will temporarily fill disk to ~92% on /. Safe to proceed?"; then + before=$(snapshot_history) + + dd if=/dev/zero of=/tmp/.proxmenux_disk_test bs=1024 count="$fill_k" 2>/dev/null || true + ok "Temp file created. Disk pressure active." + info "Waiting 90s for Health Monitor to detect low space..." + + # Wait for health monitor polling cycle + for i in $(seq 1 9); do + echo -ne "\r Waiting... ${i}0/90s" + sleep 10 + done + echo "" + + # Clean up immediately + rm -f /tmp/.proxmenux_disk_test + ok "Temp file removed. Disk space restored." + + check_new_events "$before" + else + warn "Skipped disk pressure test." + fi + else + warn "Cannot safely fill disk (would need ${fill_k}KB). Skipping." + fi + else + warn "Disk already at ${free_pct}% free. Health Monitor may already be alerting." + fi + + # ── Test D4: I/O error in syslog ── + log "" + log "${BOLD} Test D4: Generic I/O error injection${NC}" + info "Injects I/O errors into syslog for JournalWatcher." + + before=$(snapshot_history) + + logger -t kernel -p kern.err "Buffer I/O error on dev sdb1, logical block 0, async page read" + sleep 1 + logger -t kernel -p kern.err "EXT4-fs error (device sdb1): ext4_find_entry:1455: inode #2: comm ls: reading directory lblock 0" + + wait_webhook 8 + check_new_events "$before" +} + +# ============================================================================ +# TEST CATEGORY: BACKUP EVENTS +# ============================================================================ +test_backup() { + header "BACKUP EVENT TESTS" + + local backup_storage="" + + # Find backup-capable storage + backup_storage=$(pvesh get /storage --output-format json 2>/dev/null | python3 -c " +import sys, json +stores = json.load(sys.stdin) +for s in stores: + content = s.get('content', '') + if 'backup' in content or 'vztmpl' in content: + print(s.get('storage', '')) + break +# Fallback: try 'local' +else: + for s in stores: + if s.get('storage') == 'local': + print('local') + break +" 2>/dev/null || echo "local") + + info "Using backup storage: ${backup_storage}" + + # ── Test B1: Successful vzdump backup ── + if [ -n "$VMID" ]; then + log "" + log "${BOLD} Test B1: Real vzdump backup (success)${NC}" + info "Running a real vzdump backup of ${VMTYPE} ${VMID} (${VMNAME})." + info "This triggers PVE's notification system with a real backup event." + + if confirm "This will backup ${VMTYPE} ${VMID} to '${backup_storage}'. Proceed?"; then + local before + before=$(snapshot_history) + + # Use snapshot mode for VMs (non-disruptive), stop mode for CTs + local bmode="snapshot" + if [ "$VMTYPE" = "lxc" ]; then + bmode="suspend" + fi + + info "Starting vzdump (mode=${bmode}, compress=zstd)..." + if vzdump "$VMID" --storage "$backup_storage" --mode "$bmode" --compress zstd --notes-template "ProxMenux test backup" 2>&1 | tee -a "$LOG_FILE"; then + ok "vzdump completed successfully!" + else + warn "vzdump returned non-zero (check output above)" + fi + + wait_webhook 12 + check_new_events "$before" + + # Clean up the test backup + info "Cleaning up test backup file..." + local latest_bak + latest_bak=$(find "/var/lib/vz/dump/" -name "vzdump-*-${VMID}-*" -type f -newer /tmp/.proxmenux_bak_marker 2>/dev/null | head -1 || echo "") + # Create a marker for cleanup + touch /tmp/.proxmenux_bak_marker 2>/dev/null || true + else + warn "Skipped backup success test." + fi + + # ── Test B2: Failed vzdump backup ── + log "" + log "${BOLD} Test B2: vzdump backup failure (invalid storage)${NC}" + info "Attempting backup to non-existent storage to trigger a backup failure event." + + before=$(snapshot_history) + + # This WILL fail because the storage doesn't exist + info "Starting vzdump to fake storage (will fail intentionally)..." + vzdump "$VMID" --storage "nonexistent_storage_12345" --mode snapshot 2>&1 | tail -5 | tee -a "$LOG_FILE" || true + + warn "vzdump failed as expected (this is intentional)." + + wait_webhook 12 + check_new_events "$before" + + else + warn "No VM/CT available for backup tests." + info "You can create a minimal LXC container for testing:" + info " pct create 9999 local:vztmpl/debian-12-standard_12.2-1_amd64.tar.zst --storage local-lvm --memory 128 --cores 1" + fi + + # ── Test B3: Snapshot create/delete ── + if [ -n "$VMID" ] && [ "$VMTYPE" = "qemu" ]; then + log "" + log "${BOLD} Test B3: VM Snapshot create & delete${NC}" + info "Creating a snapshot of VM ${VMID} to test snapshot events." + + if confirm "Create snapshot 'proxmenux_test' on VM ${VMID}?"; then + local before + before=$(snapshot_history) + + if qm snapshot "$VMID" proxmenux_test --description "ProxMenux test snapshot" 2>&1 | tee -a "$LOG_FILE"; then + ok "Snapshot created!" + else + warn "Snapshot creation returned non-zero" + fi + + wait_webhook 10 + check_new_events "$before" + + # Clean up snapshot + info "Cleaning up test snapshot..." + qm delsnapshot "$VMID" proxmenux_test 2>/dev/null || true + ok "Snapshot removed." + fi + elif [ -n "$VMID" ] && [ "$VMTYPE" = "lxc" ]; then + log "" + log "${BOLD} Test B3: CT Snapshot create & delete${NC}" + info "Creating a snapshot of CT ${VMID}." + + if confirm "Create snapshot 'proxmenux_test' on CT ${VMID}?"; then + local before + before=$(snapshot_history) + + if pct snapshot "$VMID" proxmenux_test --description "ProxMenux test snapshot" 2>&1 | tee -a "$LOG_FILE"; then + ok "Snapshot created!" + else + warn "Snapshot creation returned non-zero" + fi + + wait_webhook 10 + check_new_events "$before" + + # Clean up + info "Cleaning up test snapshot..." + pct delsnapshot "$VMID" proxmenux_test 2>/dev/null || true + ok "Snapshot removed." + fi + fi + + # ── Test B4: PVE scheduled backup notification ── + log "" + log "${BOLD} Test B4: Trigger PVE notification system directly${NC}" + info "Using 'pvesh create /notifications/endpoints/...' to test PVE's own system." + info "This sends a test notification through PVE, which should hit our webhook." + + local before + before=$(snapshot_history) + + # PVE 8.x has a test endpoint for notifications + if pvesh create /notifications/targets/test --target proxmenux-webhook 2>&1 | tee -a "$LOG_FILE"; then + ok "PVE test notification sent!" + else + # Try alternative method + info "Direct test not available. Trying via API..." + pvesh set /notifications/endpoints/webhook/proxmenux-webhook --test 1 2>/dev/null || \ + warn "Could not send PVE test notification (requires PVE 8.1+)" + fi + + wait_webhook 8 + check_new_events "$before" +} + +# ============================================================================ +# TEST CATEGORY: VM/CT LIFECYCLE +# ============================================================================ +test_vmct() { + header "VM/CT LIFECYCLE TESTS" + + if [ -z "$VMID" ]; then + warn "No stopped VM/CT found for lifecycle tests." + info "Create a minimal CT: pct create 9999 local:vztmpl/debian-12-standard_12.2-1_amd64.tar.zst --storage local-lvm --memory 128 --cores 1" + return + fi + + log "" + log "${BOLD} Test V1: Start ${VMTYPE} ${VMID} (${VMNAME})${NC}" + + if confirm "Start ${VMTYPE} ${VMID}? It will be stopped again after the test."; then + local before + before=$(snapshot_history) + + if [ "$VMTYPE" = "lxc" ]; then + pct start "$VMID" 2>&1 | tee -a "$LOG_FILE" || true + else + qm start "$VMID" 2>&1 | tee -a "$LOG_FILE" || true + fi + + ok "Start command sent." + wait_webhook 10 + check_new_events "$before" + + # Wait a moment + sleep 5 + + # ── Test V2: Stop ── + log "" + log "${BOLD} Test V2: Stop ${VMTYPE} ${VMID}${NC}" + + before=$(snapshot_history) + + if [ "$VMTYPE" = "lxc" ]; then + pct stop "$VMID" 2>&1 | tee -a "$LOG_FILE" || true + else + qm stop "$VMID" 2>&1 | tee -a "$LOG_FILE" || true + fi + + ok "Stop command sent." + wait_webhook 10 + check_new_events "$before" + fi +} + +# ============================================================================ +# TEST CATEGORY: SYSTEM EVENTS (via syslog injection) +# ============================================================================ +test_system() { + header "SYSTEM EVENT TESTS (syslog injection)" + + # ── Test S1: Authentication failures ── + log "" + log "${BOLD} Test S1: SSH auth failure injection${NC}" + info "Injecting SSH auth failure messages into syslog." + + local before + before=$(snapshot_history) + + logger -t sshd -p auth.warning "Failed password for root from 192.168.1.200 port 44312 ssh2" + sleep 2 + logger -t sshd -p auth.warning "Failed password for invalid user admin from 10.0.0.50 port 55123 ssh2" + sleep 2 + logger -t sshd -p auth.warning "Failed password for root from 192.168.1.200 port 44315 ssh2" + + wait_webhook 8 + check_new_events "$before" + + # ── Test S2: Firewall event ── + log "" + log "${BOLD} Test S2: Firewall drop event${NC}" + + before=$(snapshot_history) + + logger -t kernel -p kern.warning "pve-fw-reject: IN=vmbr0 OUT= MAC=00:11:22:33:44:55 SRC=10.0.0.99 DST=192.168.1.1 PROTO=TCP DPT=22 REJECT" + sleep 2 + logger -t pvefw -p daemon.warning "firewall: blocked incoming connection from 10.0.0.99:45678 to 192.168.1.1:8006" + + wait_webhook 8 + check_new_events "$before" + + # ── Test S3: Service failure ── + log "" + log "${BOLD} Test S3: Service failure injection${NC}" + + before=$(snapshot_history) + + logger -t systemd -p daemon.err "pvedaemon.service: Main process exited, code=exited, status=1/FAILURE" + sleep 1 + logger -t systemd -p daemon.err "Failed to start Proxmox VE API Daemon." + + wait_webhook 8 + check_new_events "$before" +} + +# ============================================================================ +# SUMMARY & REPORT +# ============================================================================ +show_summary() { + header "TEST SUMMARY" + + info "Fetching full notification history..." + echo "" + + curl -s "${API}/api/notifications/history?limit=200" 2>/dev/null | python3 -c " +import sys, json +from collections import Counter + +data = json.load(sys.stdin) +history = data.get('history', []) + +if not history: + print(' No notifications in history.') + sys.exit(0) + +# Group by event_type +by_type = Counter(h['event_type'] for h in history) +# Group by severity +by_sev = Counter(h.get('severity', '?') for h in history) +# Group by source +by_src = Counter(h.get('source', '?') for h in history) + +print(f' Total notifications: {len(history)}') +print() + +sev_icons = {'CRITICAL': '\033[0;31mCRITICAL\033[0m', 'WARNING': '\033[1;33mWARNING\033[0m', 'INFO': '\033[0;36mINFO\033[0m'} +print(' By severity:') +for sev, count in by_sev.most_common(): + icon = sev_icons.get(sev, sev) + print(f' {icon}: {count}') + +print() +print(' By source:') +for src, count in by_src.most_common(): + print(f' {src:20s}: {count}') + +print() +print(' By event type:') +for etype, count in by_type.most_common(): + print(f' {etype:30s}: {count}') + +print() +print(' Latest 15 events:') +for h in history[:15]: + sev = h.get('severity', '?') + icon = {'CRITICAL': ' \033[0;31mRED\033[0m', 'WARNING': ' \033[1;33mYEL\033[0m', 'INFO': ' \033[0;36mBLU\033[0m'}.get(sev, ' ???') + ts = h.get('sent_at', '?')[:19] + src = h.get('source', '?')[:12] + print(f' {icon} {ts} {src:12s} {h[\"event_type\"]:25s} {h.get(\"title\", \"\")[:50]}') +" 2>/dev/null | tee -a "$LOG_FILE" + + echo "" + info "Full log saved to: ${LOG_FILE}" + echo "" + info "To see all history:" + echo -e " ${CYAN}curl -s '${API}/api/notifications/history?limit=200' | python3 -m json.tool${NC}" + echo "" + info "To check Telegram delivery, look at your Telegram bot chat." +} + +# ============================================================================ +# INTERACTIVE MENU +# ============================================================================ +show_menu() { + echo "" + echo -e "${BOLD} ProxMenux Real Event Test Suite${NC}" + echo "" + echo -e " ${CYAN}1)${NC} Disk error tests (SMART, ZFS, I/O, space pressure)" + echo -e " ${CYAN}2)${NC} Backup tests (vzdump success/fail, snapshots)" + echo -e " ${CYAN}3)${NC} VM/CT lifecycle tests (start/stop real VMs)" + echo -e " ${CYAN}4)${NC} System event tests (auth, firewall, service failures)" + echo -e " ${CYAN}5)${NC} Run ALL tests" + echo -e " ${CYAN}6)${NC} Show summary report" + echo -e " ${CYAN}q)${NC} Exit" + echo "" + echo -ne " Select: " +} + +# ── Main ──────────────────────────────────────────────────────── +main() { + local mode="${1:-menu}" + + echo "" + echo -e "${BOLD}============================================================${NC}" + echo -e "${BOLD} ProxMenux - Real Proxmox Event Simulator${NC}" + echo -e "${BOLD}============================================================${NC}" + echo -e " Tests REAL events through the full PVE -> webhook pipeline." + echo -e " Log file: ${CYAN}${LOG_FILE}${NC}" + echo "" + + preflight + + case "$mode" in + disk) test_disk; show_summary ;; + backup) test_backup; show_summary ;; + vmct) test_vmct; show_summary ;; + system) test_system; show_summary ;; + all) + test_disk + test_backup + test_vmct + test_system + show_summary + ;; + menu|*) + while true; do + show_menu + read -r choice + case "$choice" in + 1) test_disk ;; + 2) test_backup ;; + 3) test_vmct ;; + 4) test_system ;; + 5) test_disk; test_backup; test_vmct; test_system; show_summary; break ;; + 6) show_summary ;; + q|Q) echo " Bye!"; break ;; + *) warn "Invalid option" ;; + esac + done + ;; + esac +} + +main "${1:-menu}"