#!/bin/bash
# lib/health.sh - Health check enrichi pour Smart-Deploy
# Vérifie le status d'un service et récupère les infos enrichies
# Charger common.sh si pas déjà fait
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
[[ -z "${LOG_PREFIX:-}" ]] && source "$SCRIPT_DIR/common.sh"
# Configuration
HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-30}" # Timeout par requête
HEALTH_CHECK_RETRIES="${HEALTH_CHECK_RETRIES:-10}" # Nombre de tentatives
HEALTH_CHECK_INTERVAL="${HEALTH_CHECK_INTERVAL:-3}" # Secondes entre tentatives
HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_ENDPOINT:-/health}"
# Configuration granulaire par étape
PORT_CHECK_RETRIES="${PORT_CHECK_RETRIES:-10}" # Tentatives port readiness
PORT_CHECK_INTERVAL="${PORT_CHECK_INTERVAL:-2}" # Secondes entre tentatives port
HTTP_CHECK_RETRIES="${HTTP_CHECK_RETRIES:-5}" # Tentatives HTTP health
HTTP_CHECK_INTERVAL="${HTTP_CHECK_INTERVAL:-3}" # Secondes entre tentatives HTTP
# Fichier pour stocker les infos de santé
HEALTH_DATA_DIR="/stock_8to/33800-stack/monitoring/health-data"
# ============================================================
# check_container_running - Vérifie qu'un container Docker est up
# ============================================================
# Arguments:
# $1 - Container name
# $2 - Target host (IP ou hostname SSH)
# $3 - Target user SSH
# Retourne:
# 0 - Container running
# 1 - Container not running / not found
# ============================================================
check_container_running() {
local container_name="$1"
local target_host="${2:-}"
local target_user="${3:-gouroubleu}"
local docker_cmd="docker inspect --format '{{.State.Status}}' ${container_name} 2>/dev/null"
local status=""
if [[ -n "$target_host" ]]; then
status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${target_user}@${target_host}" "$docker_cmd" 2>/dev/null) || true
else
status=$(eval "$docker_cmd" 2>/dev/null) || true
fi
if [[ "$status" == "running" ]]; then
# Récupérer l'uptime pour le log
local started_at_cmd="docker inspect --format '{{.State.StartedAt}}' ${container_name} 2>/dev/null"
local started_at=""
if [[ -n "$target_host" ]]; then
started_at=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${target_user}@${target_host}" "$started_at_cmd" 2>/dev/null) || true
else
started_at=$(eval "$started_at_cmd" 2>/dev/null) || true
fi
log_success "Container ${container_name} is running (started: ${started_at:-unknown})"
return 0
elif [[ -n "$status" ]]; then
log_error "Container ${container_name} status: ${status} (expected: running)"
return 1
else
log_error "Container ${container_name} not found"
return 1
fi
}
# ============================================================
# check_port_ready - Vérifie qu'un port TCP accepte des connexions
# ============================================================
# Arguments:
# $1 - Port
# $2 - Target host (IP)
# $3 - Target user SSH
# $4 - Max retries (optionnel, défaut: PORT_CHECK_RETRIES)
# $5 - Interval (optionnel, défaut: PORT_CHECK_INTERVAL)
# Retourne:
# 0 - Port accepting connections
# 1 - Port not ready after all retries
# ============================================================
check_port_ready() {
local port="$1"
local target_host="${2:-}"
local target_user="${3:-gouroubleu}"
local max_retries="${4:-$PORT_CHECK_RETRIES}"
local interval="${5:-$PORT_CHECK_INTERVAL}"
local attempt=1
local check_cmd="timeout 2 bash -c 'echo > /dev/tcp/localhost/${port}' 2>/dev/null"
while [[ $attempt -le $max_retries ]]; do
local result=0
if [[ -n "$target_host" ]]; then
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${target_user}@${target_host}" "$check_cmd" 2>/dev/null || result=$?
else
eval "$check_cmd" 2>/dev/null || result=$?
fi
if [[ $result -eq 0 ]]; then
log_success "Port ${port} is accepting connections"
return 0
fi
if [[ $attempt -lt $max_retries ]]; then
log_info " Attempt ${attempt}/${max_retries}: port ${port} not ready, waiting ${interval}s..."
sleep "$interval"
fi
((attempt++))
done
log_error "Port ${port} not ready after ${max_retries} attempts"
return 1
}
# ============================================================
# check_http_health - Vérifie /health en HTTP interne (sans retry loop)
# ============================================================
# Arguments:
# $1 - URL complète (ex: http://192.168.1.12:5501/health)
# $2 - Service name
# $3 - Max retries (optionnel, défaut: HTTP_CHECK_RETRIES)
# $4 - Interval (optionnel, défaut: HTTP_CHECK_INTERVAL)
# Retourne:
# 0 - Healthy
# 1 - Unhealthy (HTTP 503 ou status != healthy)
# 2 - Unreachable after retries
# ============================================================
check_http_health() {
local url="$1"
local service_name="${2:-unknown}"
local max_retries="${3:-$HTTP_CHECK_RETRIES}"
local interval="${4:-$HTTP_CHECK_INTERVAL}"
local attempt=1
while [[ $attempt -le $max_retries ]]; do
local response=""
response=$(curl -sf --max-time 10 -w "\n%{http_code}" "$url" 2>/dev/null) || true
local http_code=$(echo "$response" | tail -n1)
local body=$(echo "$response" | sed '$d')
if [[ "$http_code" == "200" ]]; then
local status=""
if echo "$body" | jq -e . >/dev/null 2>&1; then
status=$(echo "$body" | jq -r '.status // "unknown"' 2>/dev/null)
else
status="ok"
fi
if [[ "$status" == "healthy" || "$status" == "ok" || "$status" == "unknown" || -z "$status" ]]; then
log_success "Service ${service_name} is healthy (HTTP 200, status: ${status:-ok})"
store_health_data "$service_name" "$body"
return 0
else
log_info " Attempt ${attempt}/${max_retries}: status=${status}, waiting ${interval}s..."
fi
elif [[ "$http_code" == "503" ]]; then
log_error "Service ${service_name} returned 503 (unhealthy)"
store_health_data "$service_name" "$body"
return 1
else
log_info " Attempt ${attempt}/${max_retries}: HTTP ${http_code:-timeout}, waiting ${interval}s..."
fi
if [[ $attempt -lt $max_retries ]]; then
sleep "$interval"
fi
((attempt++))
done
log_error "HTTP health check failed after ${max_retries} attempts"
return 2
}
# ============================================================
# check_https_external - Vérifie via domaine HTTPS (1 tentative)
# ============================================================
# Arguments:
# $1 - URL complète HTTPS (ex: https://mon-app.33800.nowhere84.com/health)
# $2 - Service name
# Retourne:
# 0 - OK
# 1 - Failed
# ============================================================
check_https_external() {
local url="$1"
local service_name="${2:-unknown}"
local response=""
response=$(curl -sf --max-time 10 -w "\n%{http_code}" "$url" 2>/dev/null) || true
local http_code=$(echo "$response" | tail -n1)
if [[ "$http_code" == "200" ]]; then
log_success "External HTTPS OK: ${url}"
return 0
else
log_error "External HTTPS failed: ${url} (HTTP ${http_code:-timeout})"
return 1
fi
}
# ============================================================
# health_check - Vérifie qu'un service répond et est healthy
# ============================================================
# Arguments:
# $1 - URL de base du service (ex: http://192.168.1.12:3050)
# $2 - Endpoint health (optionnel, défaut: /health)
# $3 - Service name (pour les logs et stockage)
# Retourne:
# 0 - Service healthy
# 1 - Service unhealthy (status != healthy)
# 2 - Service unreachable (timeout/connection error)
# ============================================================
health_check() {
local base_url="$1"
local endpoint="${2:-$HEALTH_CHECK_ENDPOINT}"
local service_name="${3:-unknown}"
local url="${base_url}${endpoint}"
log_info "Health check: $url"
local attempt=1
local response=""
local http_code=""
while [[ $attempt -le $HEALTH_CHECK_RETRIES ]]; do
log_debug "Attempt $attempt/$HEALTH_CHECK_RETRIES..."
# Faire la requête avec timeout
response=$(curl -sf --max-time "$HEALTH_CHECK_TIMEOUT" -w "\n%{http_code}" "$url" 2>/dev/null)
local curl_exit=$?
if [[ $curl_exit -eq 0 ]]; then
# Extraire le code HTTP (dernière ligne)
http_code=$(echo "$response" | tail -n1)
# Extraire le body JSON (tout sauf la dernière ligne)
local body=$(echo "$response" | sed '$d')
if [[ "$http_code" == "200" ]]; then
# Vérifier le status dans le JSON (si c'est du JSON)
local status=""
if echo "$body" | jq -e . >/dev/null 2>&1; then
# C'est du JSON valide, extraire le status
status=$(echo "$body" | jq -r '.status // "unknown"' 2>/dev/null)
else
# Pas du JSON (HTML, texte, etc.) - HTTP 200 suffit
status="ok"
fi
if [[ "$status" == "healthy" || "$status" == "ok" || "$status" == "unknown" || -z "$status" ]]; then
log_success "Service $service_name is healthy (HTTP 200, status: ${status:-ok})"
# Stocker les infos enrichies
store_health_data "$service_name" "$body"
return 0
else
log_warn "Service $service_name status: $status (expected: healthy)"
# Attendre avant de réessayer (le service démarre peut-être)
if [[ $attempt -lt $HEALTH_CHECK_RETRIES ]]; then
sleep "$HEALTH_CHECK_INTERVAL"
fi
fi
elif [[ "$http_code" == "503" ]]; then
# Service Unavailable - explicitement unhealthy
log_warn "Service $service_name returned 503 (unhealthy)"
local body=$(echo "$response" | sed '$d')
store_health_data "$service_name" "$body"
# On retourne unhealthy immédiatement sur 503
return 1
else
log_warn "Unexpected HTTP code: $http_code"
fi
else
log_debug "Connection failed (curl exit: $curl_exit)"
fi
((attempt++))
[[ $attempt -le $HEALTH_CHECK_RETRIES ]] && sleep "$HEALTH_CHECK_INTERVAL"
done
log_error "Health check failed after $HEALTH_CHECK_RETRIES attempts"
return 2
}
# ============================================================
# health_check_with_rollback - Health check avec rollback auto
# ============================================================
# Arguments:
# $1 - URL de base du service
# $2 - Endpoint health
# $3 - Service name
# $4 - Strategy (docker/files/remote)
# $5 - Config file path
# Retourne:
# 0 - Service healthy
# 1 - Service unhealthy, rollback effectué
# 2 - Service unreachable, rollback effectué
# 3 - Rollback échoué
# ============================================================
health_check_with_rollback() {
local base_url="$1"
local endpoint="$2"
local service_name="$3"
local strategy="$4"
local config_file="$5"
health_check "$base_url" "$endpoint" "$service_name"
local health_result=$?
if [[ $health_result -ne 0 ]]; then
log_error "Health check failed for $service_name (code: $health_result)"
log_warn "Initiating automatic rollback..."
# Notification avant rollback
if type notify_health_failure &>/dev/null; then
notify_health_failure "$service_name" "target" "$endpoint"
fi
# Exécuter le rollback selon la stratégie
case "$strategy" in
docker)
rollback_docker "$service_name" "$config_file"
;;
files)
rollback_files "$service_name" "$config_file"
;;
remote)
rollback_remote "$service_name" "$config_file"
;;
*)
log_error "Unknown strategy for rollback: $strategy"
return 3
;;
esac
local rollback_result=$?
if [[ $rollback_result -eq 0 ]]; then
log_success "Rollback completed successfully"
# Notification rollback réussi
if type notify_rollback &>/dev/null; then
notify_rollback "$service_name" "auto"
fi
# Re-vérifier la santé après rollback
log_info "Verifying health after rollback..."
sleep 5
health_check "$base_url" "$endpoint" "$service_name"
if [[ $? -eq 0 ]]; then
log_success "Service healthy after rollback"
return 1 # Unhealthy mais rollback OK
else
log_error "Service still unhealthy after rollback!"
return 3
fi
else
log_error "Rollback failed!"
return 3
fi
fi
return 0
}
# ============================================================
# store_health_data - Stocke les infos de santé enrichies
# ============================================================
store_health_data() {
local service_name="$1"
local health_json="$2"
# Créer le dossier si nécessaire
mkdir -p "$HEALTH_DATA_DIR" || true
local output_file="$HEALTH_DATA_DIR/${service_name}.json"
# Vérifier si c'est du JSON valide avant d'enrichir
if echo "$health_json" | jq -e . >/dev/null 2>&1; then
# JSON valide: enrichir avec timestamp et service name
echo "$health_json" | jq --arg name "$service_name" --arg ts "$(date -Iseconds)" \
'. + {service_name: $name, checked_at: $ts}' > "$output_file" 2>/dev/null || true
log_debug "Health data stored: $output_file"
else
# Pas du JSON (HTML, texte) - stocker un status simple
echo "{\"service_name\": \"$service_name\", \"status\": \"ok\", \"checked_at\": \"$(date -Iseconds)\"}" > "$output_file" || true
log_debug "Health status stored (non-JSON response): $output_file"
fi
}
# ============================================================
# get_health_data - Récupère les infos de santé stockées
# ============================================================
get_health_data() {
local service_name="$1"
local health_file="$HEALTH_DATA_DIR/${service_name}.json"
if [[ -f "$health_file" ]]; then
cat "$health_file"
else
echo "{}"
fi
}
# ============================================================
# list_health_data - Liste tous les services avec leur santé
# ============================================================
list_health_data() {
if [[ ! -d "$HEALTH_DATA_DIR" ]]; then
echo "[]"
return
fi
local result="["
local first=true
for file in "$HEALTH_DATA_DIR"/*.json; do
[[ ! -f "$file" ]] && continue
if [[ "$first" == "true" ]]; then
first=false
else
result+=","
fi
result+=$(cat "$file")
done
result+="]"
echo "$result"
}
# ============================================================
# extract_crons_from_health - Extrait les crons du health check
# ============================================================
extract_crons_from_health() {
local service_name="$1"
local health_data=$(get_health_data "$service_name")
echo "$health_data" | jq -r '.scheduledJobs // []' 2>/dev/null
}
# ============================================================
# build_health_url - Construit l'URL de health check
# ============================================================
# Arguments:
# $1 - Target machine (prod-portainer, nginx, etc.)
# $2 - Port
# $3 - Domain (optionnel, pour HTTPS externe)
# $4 - Endpoint (optionnel, défaut: /health)
# ============================================================
build_health_url() {
local target="$1"
local port="$2"
local domain="$3"
local endpoint="${4:-/health}"
# Si on a un domaine HTTPS, l'utiliser
if [[ -n "$domain" && "$domain" != "null" ]]; then
echo "https://${domain}${endpoint}"
return
fi
# Sinon, utiliser l'IP interne
local ip=""
case "$target" in
prod-portainer)
ip="192.168.1.12"
;;
dev-portainer)
ip="192.168.1.51"
;;
nginx)
ip="192.168.1.104"
;;
pve)
ip="192.168.1.4"
;;
*)
# Essayer de résoudre via machines-registry
ip=$(get_machine_ip "$target" 2>/dev/null)
;;
esac
if [[ -n "$ip" ]]; then
echo "http://${ip}:${port}${endpoint}"
else
log_error "Cannot build health URL for target: $target"
return 1
fi
}
# ============================================================
# rollback_docker - Rollback pour stratégie Docker
# ============================================================
rollback_docker() {
local service_name="$1"
local config_file="$2"
log_info "Docker rollback for $service_name"
# Récupérer l'image précédente depuis le registre ou tags
# Pour l'instant, on redéploie la version "previous"
local registry="registry.33800.nowhere84.com"
local previous_tag="${registry}/gouroubleu/${service_name}:previous"
# Vérifier si le tag previous existe
if docker manifest inspect "$previous_tag" &>/dev/null; then
log_info "Rolling back to $previous_tag"
# Mettre à jour le container
ssh gouroubleu@prod-portainer.local \
"cd /mnt/stock_8to/33800-stack/docker/stacks/${service_name} && \
docker compose pull && \
docker compose up -d"
return $?
else
log_warn "No previous tag found, cannot rollback"
return 1
fi
}
# Note: rollback_files et rollback_remote sont définis dans leurs stratégies respectives
# (strategies/files.sh et strategies/remote.sh)
# Ils sont appelés via source dans smart-deploy.sh
log_debug "lib/health.sh loaded"