Update notification service

This commit is contained in:
MacRimi
2026-03-30 19:55:19 +02:00
parent 261b2bfb3c
commit 2fc5e2865d
7 changed files with 526 additions and 84 deletions
+142
View File
@@ -456,3 +456,145 @@ def delete_storage_exclusion(storage_name):
return jsonify({'error': 'Storage not found in exclusions'}), 404
except Exception as e:
return jsonify({'error': str(e)}), 500
# ═══════════════════════════════════════════════════════════════════════════
# NETWORK INTERFACE EXCLUSION ROUTES
# ═══════════════════════════════════════════════════════════════════════════
@health_bp.route('/api/health/interfaces', methods=['GET'])
def get_network_interfaces():
"""Get all network interfaces with their exclusion status."""
try:
import psutil
# Get all interfaces
net_if_stats = psutil.net_if_stats()
net_if_addrs = psutil.net_if_addrs()
# Get current exclusions
exclusions = {e['interface_name']: e for e in health_persistence.get_excluded_interfaces()}
result = []
for iface, stats in net_if_stats.items():
if iface == 'lo':
continue
# Determine interface type
if iface.startswith('vmbr'):
iface_type = 'bridge'
elif iface.startswith('bond'):
iface_type = 'bond'
elif iface.startswith(('vlan', 'veth')):
iface_type = 'vlan'
elif iface.startswith(('eth', 'ens', 'enp', 'eno')):
iface_type = 'physical'
else:
iface_type = 'other'
# Get IP address if any
ip_addr = None
if iface in net_if_addrs:
for addr in net_if_addrs[iface]:
if addr.family == 2: # IPv4
ip_addr = addr.address
break
exclusion = exclusions.get(iface, {})
result.append({
'name': iface,
'type': iface_type,
'is_up': stats.isup,
'speed': stats.speed,
'ip_address': ip_addr,
'exclude_health': exclusion.get('exclude_health', 0) == 1,
'exclude_notifications': exclusion.get('exclude_notifications', 0) == 1,
'excluded_at': exclusion.get('excluded_at'),
'reason': exclusion.get('reason')
})
# Sort: bridges first, then physical, then others
type_order = {'bridge': 0, 'bond': 1, 'physical': 2, 'vlan': 3, 'other': 4}
result.sort(key=lambda x: (type_order.get(x['type'], 5), x['name']))
return jsonify({'interfaces': result})
except Exception as e:
return jsonify({'error': str(e)}), 500
@health_bp.route('/api/health/interface-exclusions', methods=['GET'])
def get_interface_exclusions():
"""Get all interface exclusions."""
try:
exclusions = health_persistence.get_excluded_interfaces()
return jsonify({'exclusions': exclusions})
except Exception as e:
return jsonify({'error': str(e)}), 500
@health_bp.route('/api/health/interface-exclusions', methods=['POST'])
def save_interface_exclusion():
"""
Add or update an interface exclusion.
Request body:
{
"interface_name": "vmbr0",
"interface_type": "bridge",
"exclude_health": true,
"exclude_notifications": true,
"reason": "Intentionally disabled bridge"
}
"""
try:
data = request.get_json()
if not data or 'interface_name' not in data:
return jsonify({'error': 'interface_name is required'}), 400
interface_name = data['interface_name']
interface_type = data.get('interface_type', 'unknown')
exclude_health = data.get('exclude_health', True)
exclude_notifications = data.get('exclude_notifications', True)
reason = data.get('reason')
# Check if already excluded
existing = health_persistence.get_excluded_interfaces()
exists = any(e['interface_name'] == interface_name for e in existing)
if exists:
# Update existing
success = health_persistence.update_interface_exclusion(
interface_name, exclude_health, exclude_notifications
)
else:
# Add new
success = health_persistence.exclude_interface(
interface_name, interface_type, exclude_health, exclude_notifications, reason
)
if success:
return jsonify({
'success': True,
'message': f'Interface {interface_name} exclusion saved',
'interface_name': interface_name
})
else:
return jsonify({'error': 'Failed to save exclusion'}), 500
except Exception as e:
return jsonify({'error': str(e)}), 500
@health_bp.route('/api/health/interface-exclusions/<interface_name>', methods=['DELETE'])
def delete_interface_exclusion(interface_name):
"""Remove an interface from the exclusion list."""
try:
success = health_persistence.remove_interface_exclusion(interface_name)
if success:
return jsonify({
'success': True,
'message': f'Interface {interface_name} removed from exclusions'
})
else:
return jsonify({'error': 'Interface not found in exclusions'}), 404
except Exception as e:
return jsonify({'error': str(e)}), 500
+15 -1
View File
@@ -2335,6 +2335,7 @@ class HealthMonitor:
"""
Optimized network check - only alerts for interfaces that are actually in use.
Avoids false positives for unused physical interfaces.
Respects interface exclusions configured by the user.
"""
try:
issues = []
@@ -2352,12 +2353,25 @@ class HealthMonitor:
except Exception:
net_if_addrs = {}
# Get excluded interfaces (for health checks)
excluded_interfaces = health_persistence.get_excluded_interface_names('health')
active_interfaces = set()
for interface, stats in net_if_stats.items():
if interface == 'lo':
continue
# Skip excluded interfaces
if interface in excluded_interfaces:
interface_details[interface] = {
'status': 'EXCLUDED',
'reason': 'Excluded from monitoring',
'is_up': stats.isup,
'dismissable': True
}
continue
# Check if important interface is down
if not stats.isup:
should_alert = False
@@ -3870,7 +3884,7 @@ class HealthMonitor:
status = 'WARNING'
reason = 'Failed to check for updates (apt-get error)'
# ── Build checks dict ────────────────────────────────
# ── Build checks dict ────────────────────────────────
age_dismissed = bool(age_result and age_result.get('type') == 'skipped_acknowledged')
update_age_status = 'CRITICAL' if (last_update_days and last_update_days >= 548) else (
'INFO' if age_dismissed else ('WARNING' if (last_update_days and last_update_days >= 365) else 'OK'))
+149
View File
@@ -251,6 +251,21 @@ class HealthPersistence:
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_excluded_storage ON excluded_storages(storage_name)')
# Table for excluded network interfaces - allows users to exclude interfaces
# (like intentionally disabled bridges) from health monitoring and notifications
cursor.execute('''
CREATE TABLE IF NOT EXISTS excluded_interfaces (
id INTEGER PRIMARY KEY AUTOINCREMENT,
interface_name TEXT UNIQUE NOT NULL,
interface_type TEXT NOT NULL,
excluded_at TEXT NOT NULL,
exclude_health INTEGER DEFAULT 1,
exclude_notifications INTEGER DEFAULT 1,
reason TEXT
)
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_excluded_interface ON excluded_interfaces(interface_name)')
conn.commit()
conn.close()
@@ -2328,6 +2343,140 @@ class HealthPersistence:
return {row[0] for row in cursor.fetchall()}
except Exception:
return set()
# ═══════════════════════════════════════════════════════════════════════════
# NETWORK INTERFACE EXCLUSION MANAGEMENT
# ═══════════════════════════════════════════════════════════════════════════
def get_excluded_interfaces(self) -> List[Dict[str, Any]]:
"""Get list of all excluded network interfaces."""
try:
with self._db_connection(row_factory=True) as conn:
cursor = conn.cursor()
cursor.execute('''
SELECT interface_name, interface_type, excluded_at,
exclude_health, exclude_notifications, reason
FROM excluded_interfaces
''')
return [dict(row) for row in cursor.fetchall()]
except Exception as e:
print(f"[HealthPersistence] Error getting excluded interfaces: {e}")
return []
def is_interface_excluded(self, interface_name: str, check_type: str = 'health') -> bool:
"""
Check if a network interface is excluded from monitoring.
Args:
interface_name: Name of the interface (e.g., 'vmbr0', 'eth0')
check_type: 'health' or 'notifications'
Returns:
True if the interface is excluded for the given check type
"""
try:
with self._db_connection() as conn:
cursor = conn.cursor()
column = 'exclude_health' if check_type == 'health' else 'exclude_notifications'
cursor.execute(f'''
SELECT 1 FROM excluded_interfaces
WHERE interface_name = ? AND {column} = 1
''', (interface_name,))
return cursor.fetchone() is not None
except Exception:
return False
def exclude_interface(self, interface_name: str, interface_type: str,
exclude_health: bool = True, exclude_notifications: bool = True,
reason: str = None) -> bool:
"""
Add a network interface to the exclusion list.
Args:
interface_name: Name of the interface (e.g., 'vmbr0')
interface_type: Type of interface ('bridge', 'physical', 'bond', 'vlan')
exclude_health: Whether to exclude from health monitoring
exclude_notifications: Whether to exclude from notifications
reason: Optional reason for exclusion
Returns:
True if successful
"""
try:
with self._db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
INSERT OR REPLACE INTO excluded_interfaces
(interface_name, interface_type, excluded_at, exclude_health, exclude_notifications, reason)
VALUES (?, ?, ?, ?, ?, ?)
''', (
interface_name,
interface_type,
datetime.now().isoformat(),
1 if exclude_health else 0,
1 if exclude_notifications else 0,
reason
))
conn.commit()
print(f"[HealthPersistence] Interface {interface_name} added to exclusions")
return True
except Exception as e:
print(f"[HealthPersistence] Error excluding interface: {e}")
return False
def update_interface_exclusion(self, interface_name: str,
exclude_health: bool, exclude_notifications: bool) -> bool:
"""Update exclusion settings for an interface."""
try:
with self._db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
UPDATE excluded_interfaces
SET exclude_health = ?, exclude_notifications = ?
WHERE interface_name = ?
''', (1 if exclude_health else 0, 1 if exclude_notifications else 0, interface_name))
conn.commit()
return cursor.rowcount > 0
except Exception as e:
print(f"[HealthPersistence] Error updating interface exclusion: {e}")
return False
def remove_interface_exclusion(self, interface_name: str) -> bool:
"""Remove an interface from the exclusion list."""
try:
with self._db_connection() as conn:
cursor = conn.cursor()
cursor.execute('DELETE FROM excluded_interfaces WHERE interface_name = ?', (interface_name,))
conn.commit()
removed = cursor.rowcount > 0
if removed:
print(f"[HealthPersistence] Interface {interface_name} removed from exclusions")
return removed
except Exception as e:
print(f"[HealthPersistence] Error removing interface exclusion: {e}")
return False
def get_excluded_interface_names(self, check_type: str = 'health') -> set:
"""
Get set of interface names excluded for a specific check type.
Args:
check_type: 'health' or 'notifications'
Returns:
Set of excluded interface names
"""
try:
with self._db_connection() as conn:
cursor = conn.cursor()
column = 'exclude_health' if check_type == 'health' else 'exclude_notifications'
cursor.execute(f'''
SELECT interface_name FROM excluded_interfaces
WHERE {column} = 1
''')
return {row[0] for row in cursor.fetchall()}
except Exception:
return set()
# Global instance
+2 -2
View File
@@ -28,7 +28,7 @@ from pathlib import Path
# ─── Shared State for Cross-Watcher Coordination ──────────────────
# ─── Startup Grace Period ───────────────────────────────────────────────────
# ─── Startup Grace Period ───────────────────────────────────────────────────
# Import centralized startup grace management
# This provides a single source of truth for all grace period logic
import startup_grace
@@ -2610,7 +2610,7 @@ class PollingCollector:
pass
# ─── Proxmox Webhook Receiver ──────────────────────────────────
# ─── Proxmox Webhook Receiver ──────────────────────────────────
class ProxmoxHookWatcher:
"""Receives native Proxmox VE notifications via local webhook endpoint.
+1 -1
View File
@@ -385,7 +385,7 @@ class BurstAggregator:
return etype
# ─── Notification Manager ────────────────────────────────────────
# ─── Notification Manager ────────────────────────────────────────
class NotificationManager:
"""Central notification orchestrator.
+24 -72
View File
@@ -1500,83 +1500,35 @@ Rules for the tip:
# Emoji instructions injected into AI_SYSTEM_PROMPT for rich channels (Telegram, Discord, Pushover)
AI_EMOJI_INSTRUCTIONS = """
═══ EMOJI RULES ═══
Use 1-2 emojis at START of lines where they add clarity. Combine when meaningful (💾✅ backup ok).
Not every line needs emoji — use them to highlight, not as filler. Blank lines = completely empty.
═══ EMOJI ENRICHMENT (VISUAL CLARITY) ═══
Your goal is to maintain the original structure of the message while using emojis to add visual clarity,
ESPECIALLY when adding new context, formatting technical data, or writing tips.
TITLE: ✅success ❌failed 💥crash 🆘critical 📦updates 🆕pve-update 🚚migration ⏹️stop
🔽shutdown ⚠️warning 💢split-brain 🔌disconnect 🚨auth-fail 🚷banned 📋digest
🚀 = something STARTS (VM/CT start, backup start, server boot, task begin)
Combine: 💾🚀backup-start 🖥️🚀system-boot 🚀VM/CT-start
RULES:
1. PRESERVE BASE STRUCTURE: Respect the original fields and layout provided in the input message.
2. ENHANCE WITH ICONS: Place emojis at the START of a line to identify the data type.
3. NEW CONTEXT: When adding journal info, SMART data, or known errors, use appropriate icons to make it readable.
4. NO SPAM: Do not put emojis in the middle or end of sentences. Use 1-3 emojis at START of lines where they add clarity. Combine when meaningful (💾✅ backup ok).
5. HIGHLIGHT ONLY: Not every line needs emoji — use them to highlight, not as filler. Blank lines = completely empty.
BODY: 🏷️VM/CT name ✔️ok ❌error 💽size 💾total ⏱️duration 🗄️storage 📊summary
📦updates 🔒security 🔄proxmox ⚙️kernel 🗂️packages 💿disk 📝reason
🌐IP 👤user 🌡️temp 🔥CPU 💧RAM 🎯target 🔹current 🟢new 📌item
TITLE EMOJIS:
✅ success ❌ failed 💥 crash 🆘 critical 📦 updates 🆕 pve-update 🚚 migration
⏹️ stop 🔽 shutdown ⚠️ warning 💢 split-brain 🔌 disconnect 🚨 auth-fail 🚷 banned 📋 digest
🚀 = something STARTS (VM/CT start, backup start, server boot, task begin)
Combine: 💾🚀 backup-start 🖥️🚀 system-boot 🚀 VM/CT-start
BODY EMOJIS:
🏷️ VM/CT name ✔️ ok ❌ error 💽 size 💾 total ⏱️ duration 🗄️ storage 📊 summary
📦 updates 🔒 security 🔄 proxmox ⚙️ kernel 🗂️ packages 💿 disk 📝 reason/log
🌐 IP 👤 user 🌡️ temp 🔥 CPU 💧 RAM 🎯 target 🔹 current 🟢 new 📌 item
BLANK LINES: Insert between logical sections (VM entries, before summary, before packages block).
═══ EXAMPLES (follow these formats) ═══
BACKUP START:
[TITLE]
💾🚀 pve01: Backup started
[BODY]
Backup job starting on storage PBS.
🏷️ VMs: web01 (100), db (101)
BACKUP COMPLETE:
[TITLE]
💾✅ pve01: Backup complete
[BODY]
Backup job finished on storage local-bak.
🏷️ VM web01 (ID: 100)
✔️ Status: ok
💽 Size: 12.3 GiB
⏱️ Duration: 00:04:21
🗄️ Storage: vm/100/2026-03-17T22:00:08Z
📊 Total: 1 backup | 💾 12.3 GiB | ⏱️ 00:04:21
BACKUP PARTIAL FAIL:
[TITLE]
💾❌ pve01: Backup partially failed
[BODY]
Backup job finished with errors.
🏷️ VM web01 (ID: 100)
✔️ Status: ok
💽 Size: 12.3 GiB
🏷️ VM broken (ID: 102)
❌ Status: error
📊 Total: 2 backups | ❌ 1 failed
UPDATES:
[TITLE]
📦 amd: Updates available
[BODY]
📦 Total updates: 24
🔒 Security updates: 6
🔄 Proxmox updates: 0
🗂️ Important packages:
• none
VM/CT START:
[TITLE]
🚀 pve01: VM arch-linux (100) started
[BODY]
🏷️ Virtual machine arch-linux (ID: 100)
✔️ Now running
HEALTH DEGRADED:
[TITLE]
⚠️ amd: Health warning — Disk I/O
[BODY]
💿 Device: /dev/sda
⚠️ 1 sector unreadable (pending)"""
NEW CONTEXT formatting (use when adding journal/SMART/enriched data):
📝 Logs indicate process crashed (exit-code 255)
💿 Device /dev/sdb: SMART Health FAILED
⚠️ Recurring issue: 5 occurrences in last 24h
💡 Tip: Run 'systemctl status pvedaemon' to verify"""
# No emoji instructions for email/plain text channels