import paramiko import time import re from collections import defaultdict import logging logger = logging.getLogger(__name__) # Глобальный словарь для хранения предыдущих значений RX/TX prev_net_stats = {} def get_ssh_client(host_config): """Создает SSH подключение с детальным логированием""" client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: logger.info(f"🔌 Подключаюсь к {host_config['ip']}:{host_config['port']} как {host_config['user']}") client.connect( hostname=host_config['ip'], port=host_config['port'], username=host_config['user'], key_filename=host_config['key_path'], timeout=10 ) logger.info(f"✅ Подключено к {host_config['ip']}") return client except paramiko.AuthenticationException: error_msg = f"❌ Ошибка аутентификации для {host_config['ip']}! Проверь логин или ключ." logger.error(error_msg) return error_msg except paramiko.SSHException as e: error_msg = f"❌ SSH ошибка для {host_config['ip']}: {str(e)}" logger.error(error_msg) return error_msg except Exception as e: error_msg = f"❌ Неизвестная ошибка для {host_config['ip']}: {str(e)}" logger.error(error_msg) return error_msg def exec_cmd(client, command): """Выполняет команду и возвращает вывод""" try: stdin, stdout, stderr = client.exec_command(command, timeout=10) return stdout.read().decode('utf-8').strip() except Exception as e: return f"CMD Error: {str(e)}" def detect_host_type(client): """Определяет тип хоста: physical, vm, lxc, container""" # Проверка на LXC lxc_check = exec_cmd(client, "cat /proc/1/environ 2>/dev/null | tr '\\0' '\\n' | grep -i lxc || echo ''") if lxc_check: return "lxc" # Проверка на контейнер (Docker и др) container_check = exec_cmd(client, "cat /proc/1/environ 2>/dev/null | tr '\\0' '\\n' | grep -i container || echo ''") if container_check: return "container" # Проверка на виртуальную машину virt_check = exec_cmd(client, "systemd-detect-virt 2>/dev/null || echo 'none'") if virt_check and virt_check not in ['none', '']: return "vm" # Проверка через dmidecode (если есть) virt_check2 = exec_cmd(client, "dmidecode -s system-manufacturer 2>/dev/null || echo 'Unknown'") if virt_check2: virt_vendors = ['vmware', 'virtualbox', 'kvm', 'qemu', 'xen', 'microsoft', 'parallels'] if any(vendor in virt_check2.lower() for vendor in virt_vendors): return "vm" # Если ничего не найдено - считаем физическим железом return "physical" def check_alerts(metrics, host_type): """Проверяет критические метрики и возвращает список алертов""" alerts = [] # CPU алерт (>80%) cpu = metrics.get('cpu', 0) if cpu > 80: alerts.append(f"🔴 ВЫСОКАЯ ЗАГРУЗКА CPU: {cpu:.1f}%") elif cpu > 60: alerts.append(f"🟡 ПОВЫШЕННАЯ ЗАГРУЗКА CPU: {cpu:.1f}%") # RAM алерт (>85%) mem_percent = metrics.get('mem_percent', 0) if mem_percent > 85: alerts.append(f"🔴 НЕХВАТКА ОЗУ: {mem_percent:.1f}% (использовано {metrics.get('mem_used_mb', 0)} MB)") elif mem_percent > 70: alerts.append(f"🟡 ПОВЫШЕННОЕ ПОТРЕБЛЕНИЕ ОЗУ: {mem_percent:.1f}%") # Диск алерт (>80%) disk_usage = metrics.get('disk_usage', 0) if disk_usage > 80: alerts.append(f"🔴 МАЛО МЕСТА НА ДИСКЕ: {disk_usage:.1f}%") elif disk_usage > 70: alerts.append(f"🟡 ЗАКАНЧИВАЕТСЯ МЕСТО НА ДИСКЕ: {disk_usage:.1f}%") # OOM алерт oom = metrics.get('oom', '') if oom and "Нет OOM" not in oom: alerts.append(f"💀 OOM KILLER СРАБОТАЛ! Проверь логи!") # Load Average алерт (если > количество ядер) load_1 = float(metrics.get('load_1', 0)) if load_1 > 4: alerts.append(f"🔴 ВЫСОКИЙ LOAD AVERAGE: {load_1}") elif load_1 > 2: alerts.append(f"🟡 ПОВЫШЕННЫЙ LOAD AVERAGE: {load_1}") return alerts def collect_metrics(host_config): """Сбор всех метрик с одного хоста""" client = get_ssh_client(host_config) if isinstance(client, str): return {"error": client, "host": host_config['name'], "ip": host_config['ip']} result = { "host": host_config['name'], "ip": host_config['ip'], "metrics": {} } # 1. Определяем тип хоста host_type = detect_host_type(client) result["host_type"] = host_type result["metrics"]["host_type"] = host_type # 2. CPU загрузка (%) cpu_output = exec_cmd(client, "top -bn1 | grep 'Cpu(s)' | awk '{print $2}' | cut -d'%' -f1") try: result["metrics"]["cpu"] = float(cpu_output.replace(',', '.')) except: result["metrics"]["cpu"] = 0.0 # 3. ОЗУ (используется в MB и %) mem_output = exec_cmd(client, "free -m | grep Mem") parts = re.split(r'\s+', mem_output) if len(parts) >= 3: total_mem = int(parts[1]) used_mem = int(parts[2]) result["metrics"]["mem_used_mb"] = used_mem result["metrics"]["mem_percent"] = round((used_mem / total_mem) * 100, 1) result["metrics"]["mem_total_mb"] = total_mem else: result["metrics"]["mem_used_mb"] = 0 result["metrics"]["mem_percent"] = 0 result["metrics"]["mem_total_mb"] = 0 # 4. Uptime и Load Average uptime_output = exec_cmd(client, "uptime") load_avg = re.search(r'load average:\s+([\d.]+),\s+([\d.]+),\s+([\d.]+)', uptime_output) if load_avg: result["metrics"]["load_1"] = load_avg.group(1) result["metrics"]["load_5"] = load_avg.group(2) result["metrics"]["load_15"] = load_avg.group(3) uptime_seconds = exec_cmd(client, "cat /proc/uptime | awk '{print $1}'") try: uptime_seconds = float(uptime_seconds.split('.')[0]) days = uptime_seconds // 86400 hours = (uptime_seconds % 86400) // 3600 minutes = (uptime_seconds % 3600) // 60 result["metrics"]["uptime"] = f"{int(days)}d {int(hours)}h {int(minutes)}m" except: result["metrics"]["uptime"] = "N/A" # 5. Сеть (трафик за последний час) net_output = exec_cmd(client, "cat /proc/net/dev | tail -n +3 | grep -v lo") current_net = {} for line in net_output.split('\n'): if not line.strip(): continue parts = re.split(r'\s+', line.strip()) iface = parts[0].replace(':', '') rx_bytes = int(parts[1]) tx_bytes = int(parts[9]) current_net[iface] = {'rx': rx_bytes, 'tx': tx_bytes} global prev_net_stats host_key = host_config['ip'] total_rx_hour = 0 total_tx_hour = 0 if host_key in prev_net_stats: old_data = prev_net_stats[host_key] for iface, vals in current_net.items(): if iface in old_data: delta_rx = vals['rx'] - old_data[iface]['rx'] delta_tx = vals['tx'] - old_data[iface]['tx'] if delta_rx > 0: total_rx_hour += delta_rx if delta_tx > 0: total_tx_hour += delta_tx result["metrics"]["net_rx_mb"] = round(total_rx_hour / (1024 * 1024), 2) result["metrics"]["net_tx_mb"] = round(total_tx_hour / (1024 * 1024), 2) prev_net_stats[host_key] = current_net # 6. Диски (iostat %util) iostat_output = exec_cmd(client, "iostat -x -k 1 2 | grep -E 'sd[a-z]|vd[a-z]|nvme' | tail -n +3") disk_stats = [] for line in iostat_output.split('\n'): if not line.strip(): continue parts = re.split(r'\s+', line.strip()) if len(parts) >= 14: disk_name = parts[0] try: util = float(parts[-1].replace(',', '.')) rps = float(parts[3].replace(',', '.')) wps = float(parts[4].replace(',', '.')) disk_stats.append({ "name": disk_name, "util": util, "rps": rps, "wps": wps }) except: pass disk_stats.sort(key=lambda x: x['util'], reverse=True) result["metrics"]["disks"] = disk_stats[:3] # 7. Свободное место на диске (%) disk_usage_output = exec_cmd(client, "df -h / | tail -1 | awk '{print $5}' | sed 's/%//'") try: result["metrics"]["disk_usage"] = float(disk_usage_output.replace(',', '.')) except: result["metrics"]["disk_usage"] = 0 # 8. Температуры (ТОЛЬКО ДЛЯ ФИЗИЧЕСКИХ ХОСТОВ!) temps = [] if host_type == "physical": temp_output = exec_cmd(client, "sensors -u 2>/dev/null | grep -E 'temp[0-9]+_input|Core' | head -5") if temp_output and "Error" not in temp_output and "command not found" not in temp_output: for line in temp_output.split('\n'): if '_input:' in line: val = re.search(r'([\d.]+)', line) if val: temps.append(float(val.group(1))) else: # Если sensors не установлен - пробуем через /sys/class/thermal temp_sys = exec_cmd(client, "cat /sys/class/thermal/thermal_zone*/temp 2>/dev/null | head -3") if temp_sys and "No such file" not in temp_sys: for line in temp_sys.split('\n'): try: temps.append(float(line) / 1000) except: pass result["metrics"]["temps"] = temps # 9. Топ-5 процессов по CPU и RAM top_cpu = exec_cmd(client, "ps aux --sort=-%cpu | head -6 | tail -5") top_mem = exec_cmd(client, "ps aux --sort=-%mem | head -6 | tail -5") result["metrics"]["top_cpu"] = top_cpu result["metrics"]["top_mem"] = top_mem # 10. Логи (ошибки) last_logs = exec_cmd(client, "journalctl -p 3 -xn 5 --no-pager 2>/dev/null || dmesg | tail -5") result["metrics"]["logs_errors"] = last_logs # 11. Fail2ban fail2ban_status = exec_cmd(client, "fail2ban-client status 2>/dev/null") if not fail2ban_status or "not found" in fail2ban_status: ps_fail2ban = exec_cmd(client, "ps aux | grep -v grep | grep fail2ban-server") if ps_fail2ban: fail2ban_status = "✅ Fail2ban запущен" else: fail2ban_status = "❌ Fail2ban не установлен или не запущен" else: lines = fail2ban_status.split('\n') jails_info = [] for line in lines: if '|- Number of jail:' in line or '`- Number of jail:' in line: jails_info.append(line.strip()) if 'Currently banned:' in line: jails_info.append(line.strip()) if jails_info: fail2ban_status = "✅ Fail2ban активен\n" + "\n".join(jails_info) fail2ban_logs = exec_cmd(client, "tail -20 /var/log/fail2ban.log 2>/dev/null | grep -E 'BAN|UNBAN' | tail -5 || echo 'Нет записей о банах'") active_bans = exec_cmd(client, "fail2ban-client status 2>/dev/null | grep -A 5 'Currently banned:'") if active_bans and "Currently banned:" in active_bans: fail2ban_logs += "\n\n🔒 Активные баны:\n" + active_bans result["metrics"]["fail2ban_status"] = fail2ban_status result["metrics"]["fail2ban_logs"] = fail2ban_logs # 12. OOM oom_check = exec_cmd(client, "dmesg | grep -i 'out of memory' | tail -3 || echo 'Нет OOM-событий'") if oom_check and "Нет OOM-событий" not in oom_check: result["metrics"]["oom"] = oom_check else: result["metrics"]["oom"] = "✅ Нет OOM-событий" # 13. Проверка алертов result["metrics"]["alerts"] = check_alerts(result["metrics"], host_type) client.close() return result