#!/bin/bash
# vps_failover.sh - Mavis 22/07 2026
# Failover notturno (00:00-06:59 Europe/Rome) per Mattia Trading Engine.
# Eseguito ogni 1min via cron.
#
# Cosa fa:
#  1. Check mabest.tail2b1710.ts.net (HTTP GET /healthz)
#  2. Se giù per 3+ check consecutivi (3min), manda alert email + log
#  3. Se pc_locale DOWN per 10+ minuti consecutivi, manda SECALATION
#  4. Check salute VPS: processi Python proliferati, log size, RAM libera
#
# STATO file: /opt/charter-live/logs/vps_failover_state.json
# LOG file: /opt/charter-live/logs/vps_failover.log

set -e

MABEST_URL="https://mabest.tail2b1710.ts.net/healthz"
STATE_FILE="/opt/charter-live/logs/vps_failover_state.json"
LOG_FILE="/opt/charter-live/logs/vps_failover.log"
ALERT_PY="/opt/charter-live/alert.py"
TIMEOUT=8

# Soglie alert salute VPS
MAX_UPTIME_CHECK_PROCS=5   # alert se > 5 uptime_check.py (proliferazione)
MAX_CHARTER_PROCS=10       # alert se > 10 processi charter (esclusi uptime_check)
MAX_LOGS_SIZE_MB=100       # alert se log totali > 100 MB
MIN_FREE_RAM_MB=500        # alert se RAM libera < 500 MB

mkdir -p "$(dirname "$LOG_FILE")"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >> "$LOG_FILE"
}

# Stato corrente (default: pc_up=true, down_streak=0)
if [ -f "$STATE_FILE" ]; then
    down_streak=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('down_streak', 0))" 2>/dev/null || echo 0)
    last_alert=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('last_alert_ts', 0))" 2>/dev/null || echo 0)
else
    down_streak=0
    last_alert=0
fi

now=$(date +%s)

# ============ CHECK 1: mabest.tail2b1710.ts.net ============
http_code=$(curl -sk --max-time $TIMEOUT -o /dev/null -w "%{http_code}" "$MABEST_URL" 2>/dev/null || echo "000")

if [ "$http_code" = "200" ]; then
    if [ "$down_streak" -gt 0 ]; then
        log "[RECOVER] mabest tornato UP dopo $down_streak check falliti (HTTP $http_code)"
        if [ "$down_streak" -ge 5 ] && [ $((now - last_alert)) -gt 300 ]; then
            $ALERT_PY "RECOVERED: mabest.tail2b1710.ts.net tornato UP" "VPS Hetzner conferma: mabest tornato UP dopo $down_streak minuti DOWN. Timestamp: $(date)" "INFO" 2>/dev/null || true
        fi
        down_streak=0
    else
        log "[ok] mabest UP"
    fi
else
    down_streak=$((down_streak + 1))
    log "[ALERT] mabest DOWN (HTTP $http_code), down_streak=$down_streak"
    if [ "$down_streak" -eq 3 ] && [ $((now - last_alert)) -gt 300 ]; then
        $ALERT_PY "ALERT: mabest.tail2b1710.ts.net DOWN da 3 min" "VPS Hetzner: mabest DOWN 3+ minuti (HTTP $http_code). Possibile PC locale spento/freeze/Windows Update. Mattia: intervenire manualmente al mattino. Verificare posizioni Bybit 1° account. Timestamp: $(date)" "CRITICAL" 2>/dev/null || true
        last_alert=$now
    fi
    if [ "$down_streak" -eq 10 ] && [ $((now - last_alert)) -gt 300 ]; then
        $ALERT_PY "ESCALATION: mabest DOWN da 10 min" "VPS Hetzner: mabest DOWN 10+ minuti. Intervento manuale richiesto. Verificare: (1) PC acceso, (2) supervisore attivo, (3) posizioni Bybit 1° account senza SL/TP. Timestamp: $(date)" "CRITICAL" 2>/dev/null || true
        last_alert=$now
    fi
fi

# ============ CHECK 2: Salute VPS (processi, log, RAM) ============
# Solo ogni 5 minuti (5 check su 60) per non spammare alert
minute=$(date +%M)
if [ $((minute % 5)) -eq 0 ]; then
    # Conta SOLO uptime_check.py (vera causa proliferazione)
    uptime_count=$(pgrep -fc "uptime_check.py" 2>/dev/null || echo 0)
    # Conta processi charter (esclusi uptime_check)
    charter_count=$(ps -ef | grep "python3" | grep -v grep | grep "/opt/charter-live/" | grep -v "uptime_check" | wc -l)
    # Size log totali
    logs_size_mb=$(du -sm /opt/charter-live/logs /opt/charter-live/live_deploy/logs /opt/charter-live/live_deploy/webhook_listener/logs 2>/dev/null | tail -1 | awk '{print $1}')
    logs_size_mb=${logs_size_mb:-0}
    # RAM libera
    free_ram_mb=$(free -m | awk '/^Mem:/{print $7}')
    free_ram_mb=${free_ram_mb:-0}

    log "[HEALTH] uptime_check=$uptime_count charter=$charter_count logs=${logs_size_mb}MB free_RAM=${free_ram_mb}MB"

    # Alert se uptime_check.py proliferato
    if [ "$uptime_count" -gt "$MAX_UPTIME_CHECK_PROCS" ]; then
        if [ $((now - last_alert)) -gt 1800 ]; then
            $ALERT_PY "VPS ALERT: $uptime_count processi uptime_check.py (>${MAX_UPTIME_CHECK_PROCS})" "VPS Hetzner: PROLIFERAZIONE uptime_check.py. Cron */5 non termina i processi (memory o hang). Mattia: ssh root@167.233.133.244 'pkill -9 -f uptime_check.py; crontab -l | grep uptime'. Timestamp: $(date)" "CRITICAL" 2>/dev/null || true
            last_alert=$now
        fi
    fi
    # Alert se troppi processi charter
    if [ "$charter_count" -gt "$MAX_CHARTER_PROCS" ]; then
        if [ $((now - last_alert)) -gt 1800 ]; then
            $ALERT_PY "VPS ALERT: $charter_count processi charter (>${MAX_CHARTER_PROCS})" "VPS Hetzner: troppi processi charter. Verificare ps -ef | grep python. Timestamp: $(date)" "WARNING" 2>/dev/null || true
            last_alert=$now
        fi
    fi
    if [ "$logs_size_mb" -gt "$MAX_LOGS_SIZE_MB" ]; then
        if [ $((now - last_alert)) -gt 1800 ]; then
            $ALERT_PY "VPS ALERT: log totali ${logs_size_mb}MB (>${MAX_LOGS_SIZE_MB}MB)" "VPS Hetzner: log files troppo grandi. Timestamp: $(date). Comando utile: ssh root@167.233.133.244 'find /opt/charter-live -name \"*.log\" -size +50M -exec ls -lh {} \;'" "WARNING" 2>/dev/null || true
            last_alert=$now
        fi
    fi
    if [ "$free_ram_mb" -lt "$MIN_FREE_RAM_MB" ]; then
        if [ $((now - last_alert)) -gt 1800 ]; then
            $ALERT_PY "VPS ALERT: RAM libera ${free_ram_mb}MB (<${MIN_FREE_RAM_MB}MB)" "VPS Hetzner: poca RAM. Sistema in sofferenza. Timestamp: $(date). Comando utile: ssh root@167.233.133.244 'ps aux --sort=-%mem | head -10'" "CRITICAL" 2>/dev/null || true
            last_alert=$now
        fi
    fi
fi

# Salva stato
python3 -c "
import json
with open('$STATE_FILE', 'w') as f:
    json.dump({'down_streak': $down_streak, 'last_alert_ts': $last_alert, 'last_check_ts': $now, 'last_http_code': '$http_code'}, f)
"