FruitServer/deploy/watchdog.ps1

# Author: fruit.com.cn
#
# 服务看门狗：每 2 分钟检查一次，挂了自动拉起，连续失败发飞书告警。
# 注册为 Windows 计划任务后无人值守运行。

param(
    [string]$ComposeDir = "C:\FruitServer",
    [string]$LogDir = "C:\FruitServer\logs",
    [string]$FeishuWebhook = "",
    [int]$MaxRestarts = 3,
    [string]$FrontendTask = "FruitShorts-Web-3005"
)

$ErrorActionPreference = "Continue"
$ComposeFile = Join-Path $ComposeDir "docker-compose.yml"
$LogFile = Join-Path $LogDir "watchdog-$(Get-Date -Format 'yyyy-MM-dd').log"

function Write-Log($msg) {
    $line = "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $msg"
    Add-Content -Path $LogFile -Value $line -ErrorAction SilentlyContinue
}

function Send-Alert($title, $msg) {
    if (-not $FeishuWebhook) { return }
    $body = @{
        msg_type = "text"
        content  = @{ text = "[$title] $msg" }
    } | ConvertTo-Json -Depth 3
    try {
        Invoke-RestMethod -Uri $FeishuWebhook -Method Post -Body $body -ContentType "application/json" -TimeoutSec 10
    } catch {
        Write-Log "飞书告警发送失败: $_"
    }
}

function Test-Docker {
    docker info --format "{{.ServerVersion}}" 2>$null | Out-Null
    return ($LASTEXITCODE -eq 0)
}

function Test-Http([string]$url) {
    try {
        $response = Invoke-WebRequest -Uri $url -TimeoutSec 10 -UseBasicParsing -ErrorAction Stop
        return ($response.StatusCode -ge 200 -and $response.StatusCode -lt 400)
    } catch {
        return $false
    }
}

function Test-Container([string]$container) {
    $status = docker inspect -f "{{.State.Status}}" $container 2>$null
    return ($status -eq "running")
}

function Invoke-Compose([string[]]$composeArgs) {
    & docker compose -f $ComposeFile @composeArgs 2>&1 | Out-Null
    return $LASTEXITCODE
}

# 确保日志目录存在
New-Item -ItemType Directory -Force -Path $LogDir | Out-Null

# Docker Desktop 登录后偶尔尚未就绪，先等待并尝试拉起。
if (-not (Test-Docker)) {
    Write-Log "Docker 未就绪，尝试启动 Docker Desktop"
    $dockerExe = "C:\Program Files\Docker\Docker\Docker Desktop.exe"
    if (Test-Path $dockerExe) {
        Start-Process -FilePath $dockerExe
    }

    for ($i = 0; $i -lt 18; $i++) {
        Start-Sleep -Seconds 5
        if (Test-Docker) { break }
    }
}

if (-not (Test-Docker)) {
    Write-Log "Docker 在 90 秒内未就绪，跳过本轮检查"
    Send-Alert "服务异常" "Docker Desktop 无法启动，请人工检查桌面机。"
    exit 1
}

# 健康检查定义：名称 → 检查方式
$checks = @(
    @{ Name = "app-api";    Type = "http";     Url = "http://127.0.0.1:8081/healthz"; Service = "app-api"; Container = "fruitshorts-app-api" },
    @{ Name = "admin-api";  Type = "http";     Url = "http://127.0.0.1:8080/healthz"; Service = "admin-api"; Container = "fruitshorts-admin-api" },
    @{ Name = "frontend";   Type = "frontend"; Url = "http://127.0.0.1:3005/dist-dev/" },
    @{ Name = "Grafana";    Type = "http";     Url = "http://127.0.0.1:3000/api/health"; Service = "grafana"; Container = "fruitshorts-grafana" },
    @{ Name = "PostgreSQL"; Type = "docker";   Service = "db"; Container = "fruitshorts-db" },
    @{ Name = "Redis";      Type = "docker";   Service = "redis"; Container = "fruitshorts-redis" },
    @{ Name = "Prometheus"; Type = "http";     Url = "http://127.0.0.1:9090/-/healthy"; Service = "prometheus"; Container = "fruitshorts-prometheus" }
)

# 跟踪每个服务的连续失败次数。脚本由计划任务每 2 分钟拉起一次，
# 进程内变量会跨轮丢失，因此计数持久化到状态文件，否则「连续 3 次告警」永远不触发。
$StateFile = Join-Path $LogDir "watchdog-state.json"
$failCounts = @{}
if (Test-Path $StateFile) {
    try {
        (Get-Content $StateFile -Raw | ConvertFrom-Json).PSObject.Properties |
            ForEach-Object { $failCounts[$_.Name] = [int]$_.Value }
    } catch {
        Write-Log "状态文件损坏，重置连续失败计数"
    }
}

foreach ($check in $checks) {
    $name = $check.Name
    $healthy = $false

    if ($check.Type -eq "http") {
        $healthy = Test-Http $check.Url
    } elseif ($check.Type -eq "frontend") {
        $healthy = Test-Http $check.Url
    } else {
        $healthy = Test-Container $check.Container
    }

    if ($healthy) {
        if ($failCounts[$name] -gt 0) {
            Write-Log "$name 恢复正常（此前连续失败 $($failCounts[$name]) 次）"
            Send-Alert "服务恢复" "$name 已恢复正常"
        }
        $failCounts[$name] = 0
    } else {
        if ($null -eq $failCounts[$name]) {
            $failCounts[$name] = 0
        }
        $failCounts[$name] = $failCounts[$name] + 1
        $count = $failCounts[$name]
        Write-Log "$name 不健康（第 $count 次）"

        if ($check.Type -eq "frontend") {
            Write-Log "$name 重启计划任务 $FrontendTask"
            Stop-ScheduledTask -TaskName $FrontendTask -ErrorAction SilentlyContinue
            Start-Sleep -Seconds 2
            Start-ScheduledTask -TaskName $FrontendTask -ErrorAction SilentlyContinue
            Start-Sleep -Seconds 6
            $healthy = Test-Http $check.Url
        } elseif ($check.Type -eq "http") {
            Write-Log "$name 重启 Compose 服务 $($check.Service)"
            Invoke-Compose @("restart", $check.Service) | Out-Null
            Start-Sleep -Seconds 12
            $healthy = Test-Http $check.Url

            if (-not $healthy) {
                Write-Log "$name 重启后仍不健康，执行 compose up -d"
                Invoke-Compose @("up", "-d", $check.Service) | Out-Null
            }
        } else {
            Write-Log "$name 拉起容器 $($check.Container)"
            docker start $check.Container 2>&1 | Out-Null
            Start-Sleep -Seconds 5
            $healthy = Test-Container $check.Container

            if (-not $healthy) {
                Write-Log "$name 启动失败，执行 compose up -d"
                Invoke-Compose @("up", "-d", $check.Service) | Out-Null
            }
        }

        # 连续失败达到上限 → 告警
        if (-not $healthy -and $count -ge $MaxRestarts) {
            Send-Alert "服务异常" "$name 已连续 $count 次不健康，已尝试重启。请人工介入检查。"
            Write-Log "$name 连续 $count 次失败，已发告警"
        } elseif ($healthy) {
            Write-Log "$name 已自动恢复"
        }
    }
}

# 磁盘检查：超过 85% 告警
$drive = (Get-PSDrive C -ErrorAction SilentlyContinue)
if ($drive) {
    $pct = [math]::Round(($drive.Used / ($drive.Used + $drive.Free)) * 100)
    if ($pct -gt 85) {
        Write-Log "磁盘使用率 ${pct}%（超过 85%）"
        Send-Alert "磁盘告警" "C 盘使用率 ${pct}%，请清理日志或扩容"
    }
}

# 落盘本轮结束时的连续失败计数，供下一轮接着累计
try {
    $failCounts | ConvertTo-Json | Set-Content -Path $StateFile -Encoding UTF8
} catch {
    Write-Log "状态文件写入失败: $_"
}