# HTS Investigation Suite - Système Monitoring Complet

# HIGH TECH SECURITY - Surveillance et alertes automatisées

import asyncio
import json
import logging
import smtplib
import time
from datetime import datetime, timedelta
from email.mime.text import MimeText
from email.mime.multipart import MimeMultipart
from typing import Dict, List, Optional
import psutil
import redis
import requests
from sqlalchemy import create_engine, text
from prometheus_client import start_http_server, Gauge, Counter, Histogram, Info
import structlog

# Configuration
class MonitoringConfig:
# Métriques Prometheus
PROMETHEUS_PORT = 8001

# Seuils d'alerte
THRESHOLDS = {
'cpu_usage': 80.0,
'memory_usage': 85.0,
'disk_usage': 90.0,
'response_time': 5.0, # secondes
'error_rate': 10.0, # pourcentage
'failed_analyses': 5 # nombre
}

# Configuration email
SMTP_CONFIG = {
'server': 'smtp.gmail.com',
'port': 587,
'username': 'hts-monitoring@gmail.com',
'password': 'HTS_Monitor_2025!',
'from_email': 'hts-monitoring@hts-investigation.com'
}

# Destinataires alertes
ALERT_RECIPIENTS = [
'admin@hts-investigation.com',
'contact@hts-investigation.com'
]

# URLs à surveiller
ENDPOINTS_TO_MONITOR = [
'http://localhost:8000/health',
'http://localhost:8000/',
'http://localhost:8000/api/stats/global'
]

# Configuration logging
logger = structlog.get_logger()

# Métriques Prometheus
system_cpu_usage = Gauge('hts_system_cpu_usage_percent', 'CPU usage percentage')
system_memory_usage = Gauge('hts_system_memory_usage_percent', 'Memory usage percentage')
system_disk_usage = Gauge('hts_system_disk_usage_percent', 'Disk usage percentage')
api_response_time = Histogram('hts_api_response_time_seconds', 'API response time')
api_requests_total = Counter('hts_api_requests_total', 'Total API requests', ['method', 'endpoint', 'status'])
active_investigations = Gauge('hts_active_investigations_total', 'Number of active investigations')
failed_analyses = Counter('hts_failed_analyses_total', 'Number of failed analyses')
threat_alerts = Counter('hts_threat_alerts_total', 'Number of threat alerts', ['severity'])
system_info = Info('hts_system_info', 'System information')

class HTSMonitoringSystem:
def __init__(self):
self.redis_client = redis.Redis(host='localhost', port=6379, password='HTS_Redis_2025!', db=0)
self.db_engine = create_engine('postgresql://hts_user:HTS_Investigation_2025!@localhost/hts_investigation')
self.alerts_sent = {}
self.last_health_check = {}

# Initialisation métriques système
system_info.info({
'version': '2.1.0',
'company': 'HIGH TECH SECURITY',
'address': '203-205 The Vale, London W3 7QS, United Kingdom',
'company_number': '7849187',
'policy_number': '550.232.804'
})

async def start_monitoring(self):
"""Démarrage du système de monitoring"""
logger.info("🛡️ HIGH TECH SECURITY - Démarrage monitoring")

# Démarrage serveur Prometheus
start_http_server(MonitoringConfig.PROMETHEUS_PORT)
logger.info(f"Prometheus metrics server démarré sur le port {MonitoringConfig.PROMETHEUS_PORT}")

# Boucle principale de monitoring
while True:
try:
await self.run_monitoring_cycle()
await asyncio.sleep(30) # Cycle toutes les 30 secondes
except Exception as e:
logger.error(f"Erreur dans le cycle de monitoring: {e}")
await asyncio.sleep(60)

async def run_monitoring_cycle(self):
"""Cycle complet de monitoring"""
tasks = [
self.monitor_system_resources(),
self.monitor_api_health(),
self.monitor_database_health(),
self.monitor_business_metrics(),
self.check_security_alerts(),
self.cleanup_old_data()
]

await asyncio.gather(*tasks, return_exceptions=True)

async def monitor_system_resources(self):
"""Surveillance des ressources système"""
try:
# CPU
cpu_percent = psutil.cpu_percent(interval=1)
system_cpu_usage.set(cpu_percent)

# Mémoire
memory = psutil.virtual_memory()
memory_percent = memory.percent
system_memory_usage.set(memory_percent)

# Disque
disk = psutil.disk_usage('/')
disk_percent = (disk.used / disk.total) * 100
system_disk_usage.set(disk_percent)

# Vérification seuils
if cpu_percent > MonitoringConfig.THRESHOLDS['cpu_usage']:
await self.send_alert('system', f'CPU usage élevé: {cpu_percent:.1f}%', 'high')

if memory_percent > MonitoringConfig.THRESHOLDS['memory_usage']:
await self.send_alert('system', f'Mémoire usage élevé: {memory_percent:.1f}%', 'high')

if disk_percent > MonitoringConfig.THRESHOLDS['disk_usage']:
await self.send_alert('system', f'Disque usage élevé: {disk_percent:.1f}%', 'critical')

logger.debug(f"Ressources: CPU {cpu_percent:.1f}%, Mémoire {memory_percent:.1f}%, Disque {disk_percent:.1f}%")

except Exception as e:
logger.error(f"Erreur monitoring ressources système: {e}")

async def monitor_api_health(self):
"""Surveillance santé API"""
for endpoint in MonitoringConfig.ENDPOINTS_TO_MONITOR:
try:
start_time = time.time()
response = requests.get(endpoint, timeout=10)
response_time = time.time() - start_time

# Métriques Prometheus
api_response_time.observe(response_time)
api_requests_total.labels(
method='GET',
endpoint=endpoint,
status=response.status_code
).inc()

# Vérification santé
if response.status_code != 200:
await self.send_alert('api', f'Endpoint {endpoint} retourne {response.status_code}', 'critical')

if response_time > MonitoringConfig.THRESHOLDS['response_time']:
await self.send_alert('api', f'Temps de réponse élevé pour {endpoint}: {response_time:.2f}s', 'warning')

self.last_health_check[endpoint] = {
'status': response.status_code,
'response_time': response_time,
'timestamp': datetime.now()
}

except requests.exceptions.RequestException as e:
await self.send_alert('api', f'Endpoint {endpoint} inaccessible: {str(e)}', 'critical')
api_requests_total.labels(method='GET', endpoint=endpoint, status='timeout').inc()

async def monitor_database_health(self):
"""Surveillance santé base de données"""
try:
with self.db_engine.connect() as conn:
# Test connexion
start_time = time.time()
result = conn.execute(text("SELECT 1"))
db_response_time = time.time() - start_time

if db_response_time > 2.0:
await self.send_alert('database', f'Base de données lente: {db_response_time:.2f}s', 'warning')

# Vérification taille base
size_result = conn.execute(text("""
SELECT pg_size_pretty(pg_database_size('hts_investigation')) as size,
pg_database_size('hts_investigation') as size_bytes
"""))
size_info = size_result.fetchone()

# Vérification connexions actives
conn_result = conn.execute(text("""
SELECT count(*) as active_connections
FROM pg_stat_activity
WHERE state = 'active'
"""))
active_connections = conn_result.fetchone()[0]

if active_connections > 50:
await self.send_alert('database', f'Trop de connexions actives: {active_connections}', 'warning')

logger.debug(f"DB: {size_info[0]}, {active_connections} connexions actives")

except Exception as e:
await self.send_alert('database', f'Erreur base de données: {str(e)}', 'critical')
logger.error(f"Erreur monitoring base de données: {e}")

async def monitor_business_metrics(self):
"""Surveillance métriques business"""
try:
with self.db_engine.connect() as conn:
# Investigations actives
investigations_result = conn.execute(text("""
SELECT COUNT(*) FROM investigations WHERE status = 'active'
"""))
active_inv_count = investigations_result.fetchone()[0]
active_investigations.set(active_inv_count)

# Analyses échouées (dernières 24h)
failed_result = conn.execute(text("""
SELECT COUNT(*) FROM analysis_results
WHERE created_at > NOW() - INTERVAL '24 hours'
AND confidence_score < 0.5
"""))
failed_count = failed_result.fetchone()[0]

if failed_count > MonitoringConfig.THRESHOLDS['failed_analyses']:
await self.send_alert('business', f'Trop d\'analyses échouées: {failed_count}', 'warning')

# Alertes par sévérité (dernières 24h)
alerts_result = conn.execute(text("""
SELECT sev

monitoring.html

@@ -0,0 +1,349 @@
+```html
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Monitoring | HTS Investigation Suite</title>
+    <link rel="icon" type="image/x-icon" href="/static/favicon.ico">
+    <script src="https://cdn.tailwindcss.com"></script>
+    <script src="https://unpkg.com/feather-icons"></script>
+    <script src="https://cdn.jsdelivr.net/npm/vanta@latest/dist/vanta.globe.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+    <style>
+        @import url('https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;500;700&family=Rubik:wght@300;400;500;600;700&display=swap');
+        body {
+            font-family: 'Rubik', sans-serif;
+        }
+        .vanta-canvas {
+            position: absolute;
+            top: 0;
+            left: 0;
+            width: 100%;
+            height: 100%;
+            z-index: 0;
+            opacity: 0.15;
+        }
+        .alert-critical {
+            background: linear-gradient(90deg, rgba(239,68,68,0.1) 0%, rgba(239,68,68,0.3) 100%);
+            border-left: 4px solid #ef4444;
+        }
+        .alert-high {
+            background: linear-gradient(90deg, rgba(249,115,22,0.1) 0%, rgba(249,115,22,0.3) 100%);
+            border-left: 4px solid #f97316;
+        }
+        .alert-medium {
+            background: linear-gradient(90deg, rgba(234,179,8,0.1) 0%, rgba(234,179,8,0.3) 100%);
+            border-left: 4px solid #eab308;
+        }
+        .alert-low {
+            background: linear-gradient(90deg, rgba(59,130,246,0.1) 0%, rgba(59,130,246,0.3) 100%);
+            border-left: 4px solid #3b82f6;
+        }
+        .animate-pulse-slow {
+            animation: pulse 3s cubic-bezier(0.4, 0, 0.6, 1) infinite;
+        }
+        @keyframes pulse {
+            0%, 100% { opacity: 1; }
+            50% { opacity: 0.5; }
+        }
+    </style>
+</head>
+<body class="bg-gray-50">
+    <div id="vanta-bg" class="vanta-canvas"></div>
+    
+    <!-- Header -->
+    <header class="relative z-10 bg-white shadow-lg border-b border-gray-200">
+        <div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-4">
+            <div class="flex items-center justify-between">
+                <div class="flex items-center space-x-4">
+                    <div class="bg-blue-600 p-2 rounded-lg">
+                        <i data-feather="shield" class="text-white w-6 h-6"></i>
+                    </div>
+                    <div>
+                        <h1 class="text-2xl font-bold text-gray-900">HIGH TECH SECURITY</h1>
+                        <p class="text-sm text-gray-600">HTS Investigation Suite v2.1</p>
+                    </div>
+                </div>
+                
+                <div class="hidden md:flex items-center space-x-6">
+                    <div class="text-right">
+                        <div class="text-gray-600">203-205 The Vale, London W3 7QS</div>
+                        <div class="text-gray-600">UK: +44 7591 665201 | Company: 7849187</div>
+                    </div>
+                    <div class="flex items-center space-x-2">
+                        <i data-feather="lock" class="w-5 h-5 text-green-500"></i>
+                        <span class="text-sm text-green-600 font-medium">Sécurisé</span>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </header>
+
+    <!-- Navigation -->
+    <nav class="relative z-10 bg-gray-800 text-white">
+        <div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
+            <div class="flex space-x-1 overflow-x-auto py-2 hide-scrollbar">
+                <a href="/" class="flex items-center space-x-2 px-4 py-3 border-b-2 border-transparent hover:border-gray-600 transition-colors">
+                    <i data-feather="trending-up" class="w-4 h-4"></i>
+                    <span>Dashboard</span>
+                </a>
+                <a href="/darkbert" class="flex items-center space-x-2 px-4 py-3 border-b-2 border-transparent hover:border-gray-600 transition-colors">
+                    <i data-feather="brain" class="w-4 h-4"></i>
+                    <span>DarkBERT Analysis</span>
+                </a>
+                <a href="/investigations" class="flex items-center space-x-2 px-4 py-3 border-b-2 border-transparent hover:border-gray-600 transition-colors">
+                    <i data-feather="search" class="w-4 h-4"></i>
+                    <span>Investigations</span>
+                </a>
+                <a href="/monitoring" class="flex items-center space-x-2 px-4 py-3 border-b-2 border-blue-400 text-blue-400 transition-colors">
+                    <i data-feather="eye" class="w-4 h-4"></i>
+                    <span>Monitoring</span>
+                </a>
+            </div>
+        </div>
+    </nav>
+
+    <!-- Main Content -->
+    <main class="relative z-10 max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
+        <!-- System Status Overview -->
+        <div class="bg-white rounded-xl shadow-lg p-6 mb-8">
+            <div class="flex items-center justify-between mb-6">
+                <h2 class="text-2xl font-bold text-gray-900">System Monitoring Dashboard</h2>
+                <div class="flex items-center space-x-2">
+                    <div class="w-2 h-2 rounded-full bg-green-500 animate-pulse-slow"></div>
+                    <span class="text-sm text-gray-600">Real-time monitoring active</span>
+                </div>
+            </div>
+
+            <div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6 mb-8">
+                <!-- CPU Usage -->
+                <div class="bg-white rounded-xl shadow p-4 border border-gray-200">
+                    <div class="flex items-center justify-between mb-2">
+                        <div class="flex items-center space-x-2">
+                            <i data-feather="cpu" class="w-5 h-5 text-blue-600"></i>
+                            <span class="font-medium text-gray-700">CPU Usage</span>
+                        </div>
+                        <span class="text-sm font-bold" id="cpu-percent">--%</span>
+                    </div>
+                    <div class="h-2 bg-gray-200 rounded-full overflow-hidden">
+                        <div id="cpu-bar" class="h-full bg-blue-600 rounded-full" style="width: 0%"></div>
+                    </div>
+                </div>
+                
+                <!-- Memory Usage -->
+                <div class="bg-white rounded-xl shadow p-4 border border-gray-200">
+                    <div class="flex items-center justify-between mb-2">
+                        <div class="flex items-center space-x-2">
+                            <i data-feather="hard-drive" class="w-5 h-5 text-purple-600"></i>
+                            <span class="font-medium text-gray-700">Memory Usage</span>
+                        </div>
+                        <span class="text-sm font-bold" id="memory-percent">--%</span>
+                    </div>
+                    <div class="h-2 bg-gray-200 rounded-full overflow-hidden">
+                        <div id="memory-bar" class="h-full bg-purple-600 rounded-full" style="width: 0%"></div>
+                    </div>
+                </div>
+                
+                <!-- Disk Usage -->
+                <div class="bg-white rounded-xl shadow p-4 border border-gray-200">
+                    <div class="flex items-center justify-between mb-2">
+                        <div class="flex items-center space-x-2">
+                            <i data-feather="database" class="w-5 h-5 text-orange-600"></i>
+                            <span class="font-medium text-gray-700">Disk Usage</span>
+                        </div>
+                        <span class="text-sm font-bold" id="disk-percent">--%</span>
+                    </div>
+                    <div class="h-2 bg-gray-200 rounded-full overflow-hidden">
+                        <div id="disk-bar" class="h-full bg-orange-600 rounded-full" style="width: 0%"></div>
+                    </div>
+                </div>
+                
+                <!-- Active Investigations -->
+                <div class="bg-white rounded-xl shadow p-4 border border-gray-200">
+                    <div class="flex items-center justify-between mb-2">
+                        <div class="flex items-center space-x-2">
+                            <i data-feather="file-text" class="w-5 h-5 text-green-600"></i>
+                            <span class="font-medium text-gray-700">Active Investigations</span>
+                        </div>
+                        <span class="text-sm font-bold" id="active-investigations">--</span>
+                    </div>
+                    <div class="h-2 bg-gray-200 rounded-full overflow-hidden">
+                        <div id="investigations-bar" class="h-full bg-green-600 rounded-full" style="width: 0%"></div>
+                    </div>
+                </div>
+            </div>
+
+            <!-- Resource Usage Charts -->
+            <div class="grid grid-cols-1 lg:grid-cols-2 gap-6 mb-8">
+                <div class="bg-white rounded-xl shadow-lg p-4 border border-gray-200">
+                    <h3 class="text-lg font-semibold text-gray-900 mb-4">CPU & Memory Usage (Last 24h)</h3>
+                    <canvas id="system-usage-chart" height="200"></canvas>
+                </div>
+                <div class="bg-white rounded-xl shadow-lg p-4 border border-gray-200">
+                    <h3 class="text-lg font-semibold text-gray-900 mb-4">API Response Times (Last 24h)</h3>
+                    <canvas id="api-response-chart" height="200"></canvas>
+                </div>
+            </div>
+
+            <!-- Alerts Section -->
+            <div class="bg-white rounded-xl shadow-lg p-6 border border-gray-200">
+                <div class="flex items-center justify-between mb-4">
+                    <h3 class="text-lg font-semibold text-gray-900">Recent Alerts</h3>
+                    <div class="flex space-x-2">
+                        <button class="px-3 py-1 text-xs bg-blue-100 text-blue-800 rounded-full">All</button>
+                        <button class="px-3 py-1 text-xs bg-red-100 text-red-800 rounded-full">Critical</button>
+                        <button class="px-3 py-1 text-xs bg-orange-100 text-orange-800 rounded-full">High</button>
+                        <button class="px-3 py-1 text-xs bg-yellow-100 text-yellow-800 rounded-full">Medium</button>
+                        <button class="px-3 py-1 text-xs bg-blue-100 text-blue-800 rounded-full">Low</button>
+                    </div>
+                </div>
+                
+                <div class="space-y-3 max-h-96 overflow-y-auto">
+                    <!-- Sample Alert Items -->
+                    <div class="p-4 rounded-lg alert-critical">
+                        <div class="flex justify-between items-start">
+                            <div>
+                                <span class="font-medium">Critical CPU Usage (95.7%)</span>
+                                <span class="text-xs ml-2 px-2 py-1 bg-red-600 text-white rounded-full">Critical</span>
+                            </div>
+                            <span class="text-xs text-gray-500">2 min ago</span>
+                        </div>
+                        <p class="text-sm mt-1">CPU usage exceeds critical threshold of 90%</p>
+                    </div>
+                    
+                    <div class="p-4 rounded-lg alert-high">
+                        <div class="flex justify-between items-start">
+                            <div>
+                                <span class="font-medium">High Memory Usage (87.2%)</span>
+                                <span class="text-xs ml-2 px-2 py-1 bg-orange-600 text-white rounded-full">High</span>
+                            </div>
+                            <span class="text-xs text-gray-500">15 min ago</span>
+                        </div>
+                        <p class="text-sm mt-1">Memory usage exceeds warning threshold of 85%</p>
+                    </div>
+                    
+                    <div class="p-4 rounded-lg alert-medium">
+                        <div class="flex justify-between items-start">
+                            <div>
+                                <span class="font-medium">Slow API Response (5.2s)</span>
+                                <span class="text-xs ml-2 px-2 py-1 bg-yellow-600 text-white rounded-full">Medium</span>
+                            </div>
+                            <span class="text-xs text-gray-500">1 hour ago</span>
+                        </div>
+                        <p class="text-sm mt-1">API endpoint /health responded slowly</p>
+                    </div>
+                    
+                    <div class="p-4 rounded-lg alert-low">
+                        <div class="flex justify-between items-start">
+                            <div>
+                                <span class="font-medium">Database Connection Warning</span>
+                                <span class="text-xs ml-2 px-2 py-1 bg-blue-600 text-white rounded-full">Low</span>
+                            </div>
+                            <span class="text-xs text-gray-500">3 hours ago</span>
+                        </div>
+                        <p class="text-sm mt-1">Database response time exceeded 2 seconds</p>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </main>
+
+    <script>
+        // Initialize Vanta.js globe effect
+        feather.replace();
+        VANTA.GLOBE({
+            el: "#vanta-bg",
+            mouseControls: true,
+            touchControls: true,
+            gyroControls: false,
+            minHeight: 200.00,
+            minWidth: 200.00,
+            scale: 1.00,
+            scaleMobile: 1.00,
+            color: 0x3b82f6,
+            backgroundColor: 0xf8fafc,
+            size: 0.8
+        });
+
+        // Initialize charts
+        const systemUsageCtx = document.getElementById('system-usage-chart').getContext('2d');
+        const apiResponseCtx = document.getElementById('api-response-chart').getContext('2d');
+
+        // Sample data for charts
+        const labels = Array.from({length: 24}, (_, i) => `${i}:00`);
+        
+        const systemUsageChart = new Chart(systemUsageCtx, {
+            type: 'line',
+            data: {
+                labels: labels,
+                datasets: [
+                    {
+                        label: 'CPU Usage (%)',
+                        data: [45, 48, 52, 55, 60, 65, 70, 75, 80, 85, 90, 95, 92, 88, 85, 80, 75, 70, 65, 60, 55, 50, 45, 40],
+                        borderColor: '#3b82f6',
+                        backgroundColor: 'rgba(59, 130, 246, 0.1)',
+                        tension: 0.3,
+                        fill: true
+                    },
+                    {
+                        label: 'Memory Usage (%)',
+                        data: [60, 62, 65, 68, 70, 72, 75, 78, 80, 82, 85, 87, 85, 82, 80, 78, 75, 72, 70, 68, 65, 62, 60, 58],
+                        borderColor: '#8b5cf6',
+                        backgroundColor: 'rgba(139, 92, 246, 0.1)',
+                        tension: 0.3,
+                        fill: true
+                    }
+                ]
+            },
+            options: {
+                responsive: true,
+                plugins: {
+                    legend: {
+                        position: 'top',
+                    }
+                },
+                scales: {
+                    y: {
+                        beginAtZero: true,
+                        max: 100
+                    }
+                }
+            }
+        });
+
+        const apiResponseChart = new Chart(apiResponseCtx, {
+            type: 'bar',
+            data: {
+                labels: labels,
+                datasets: [
+                    {
+                        label: 'API Response Time (ms)',
+                        data: [120, 150, 180, 200, 220, 250, 300, 350, 400, 450, 500, 550, 500, 450, 400, 350, 300, 250, 200, 150, 120, 100, 90, 80],
+                        backgroundColor: '#10b981',
+                        borderColor: '#10b981',
+                        borderWidth: 1
+                    }
+                ]
+            },
+            options: {
+                responsive: true,
+                plugins: {
+                    legend: {
+                        position: 'top',
+                    }
+                },
+                scales: {
+                    y: {
+                        beginAtZero: true
+                    }
+                }
+            }
+        });
+
+        // Simulate real-time updates
+        function updateMetrics() {
+            // Simulate random values
+            const cpu = Math.floor(Math.random() * 30) + 60;
+            const memory = Math.floor(Math.random() * 25) + 60;
+            const disk = Math.floor(Math.random()