diff --git a/deploy/backend-watchdog-task.xml b/deploy/backend-watchdog-task.xml new file mode 100644 index 0000000..b58a9cf --- /dev/null +++ b/deploy/backend-watchdog-task.xml @@ -0,0 +1,32 @@ + + + + FruitShorts backend watchdog: health check every 2 minutes and restart unhealthy services. + + + + S-1-5-18 + HighestAvailable + + + + IgnoreNew + false + false + true + PT10M + + + + + PT2M + + + + + + powershell.exe + -NoProfile -ExecutionPolicy Bypass -File "C:\Users\Administrator\Desktop\ReelShorts\FruitServer\deploy\watchdog.ps1" -ComposeDir "C:\Users\Administrator\Desktop\ReelShorts\FruitServer" -LogDir "C:\Users\Administrator\Desktop\ReelShorts\FruitServer\logs" + + + diff --git a/deploy/watchdog.ps1 b/deploy/watchdog.ps1 new file mode 100644 index 0000000..404211f --- /dev/null +++ b/deploy/watchdog.ps1 @@ -0,0 +1,170 @@ +# Author: fruit.com.cn +# +# 服务看门狗：每 2 分钟检查一次，挂了自动拉起，连续失败发飞书告警。 +# 注册为 Windows 计划任务后无人值守运行。 + +param( + [string]$ComposeDir = "C:\FruitServer", + [string]$LogDir = "C:\FruitServer\logs", + [string]$FeishuWebhook = "", + [int]$MaxRestarts = 3, + [string]$FrontendTask = "FruitShorts-Web-3005" +) + +$ErrorActionPreference = "Continue" +$ComposeFile = Join-Path $ComposeDir "docker-compose.yml" +$LogFile = Join-Path $LogDir "watchdog-$(Get-Date -Format 'yyyy-MM-dd').log" + +function Write-Log($msg) { + $line = "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $msg" + Add-Content -Path $LogFile -Value $line -ErrorAction SilentlyContinue +} + +function Send-Alert($title, $msg) { + if (-not $FeishuWebhook) { return } + $body = @{ + msg_type = "text" + content = @{ text = "[$title] $msg" } + } | ConvertTo-Json -Depth 3 + try { + Invoke-RestMethod -Uri $FeishuWebhook -Method Post -Body $body -ContentType "application/json" -TimeoutSec 10 + } catch { + Write-Log "飞书告警发送失败: $_" + } +} + +function Test-Docker { + docker info --format "{{.ServerVersion}}" 2>$null | Out-Null + return ($LASTEXITCODE -eq 0) +} + +function Test-Http([string]$url) { + try { + $response = Invoke-WebRequest -Uri $url -TimeoutSec 10 -UseBasicParsing -ErrorAction Stop + return ($response.StatusCode -ge 200 -and $response.StatusCode -lt 400) + } catch { + return $false + } +} + +function Test-Container([string]$container) { + $status = docker inspect -f "{{.State.Status}}" $container 2>$null + return ($status -eq "running") +} + +function Invoke-Compose([string[]]$composeArgs) { + & docker compose -f $ComposeFile @composeArgs 2>&1 | Out-Null + return $LASTEXITCODE +} + +# 确保日志目录存在 +New-Item -ItemType Directory -Force -Path $LogDir | Out-Null + +# Docker Desktop 登录后偶尔尚未就绪，先等待并尝试拉起。 +if (-not (Test-Docker)) { + Write-Log "Docker 未就绪，尝试启动 Docker Desktop" + $dockerExe = "C:\Program Files\Docker\Docker\Docker Desktop.exe" + if (Test-Path $dockerExe) { + Start-Process -FilePath $dockerExe + } + + for ($i = 0; $i -lt 18; $i++) { + Start-Sleep -Seconds 5 + if (Test-Docker) { break } + } +} + +if (-not (Test-Docker)) { + Write-Log "Docker 在 90 秒内未就绪，跳过本轮检查" + Send-Alert "服务异常" "Docker Desktop 无法启动，请人工检查桌面机。" + exit 1 +} + +# 健康检查定义：名称 → 检查方式 +$checks = @( + @{ Name = "app-api"; Type = "http"; Url = "http://127.0.0.1:8081/healthz"; Service = "app-api"; Container = "fruitshorts-app-api" }, + @{ Name = "admin-api"; Type = "http"; Url = "http://127.0.0.1:8080/healthz"; Service = "admin-api"; Container = "fruitshorts-admin-api" }, + @{ Name = "frontend"; Type = "frontend"; Url = "http://127.0.0.1:3005/dist-dev/" }, + @{ Name = "Grafana"; Type = "http"; Url = "http://127.0.0.1:3000/api/health"; Service = "grafana"; Container = "fruitshorts-grafana" }, + @{ Name = "PostgreSQL"; Type = "docker"; Service = "db"; Container = "fruitshorts-db" }, + @{ Name = "Redis"; Type = "docker"; Service = "redis"; Container = "fruitshorts-redis" }, + @{ Name = "Prometheus"; Type = "http"; Url = "http://127.0.0.1:9090/-/healthy"; Service = "prometheus"; Container = "fruitshorts-prometheus" } +) + +# 跟踪每个服务的连续失败次数 +$failCounts = @{} + +foreach ($check in $checks) { + $name = $check.Name + $healthy = $false + + if ($check.Type -eq "http") { + $healthy = Test-Http $check.Url + } elseif ($check.Type -eq "frontend") { + $healthy = Test-Http $check.Url + } else { + $healthy = Test-Container $check.Container + } + + if ($healthy) { + if ($failCounts[$name] -gt 0) { + Write-Log "$name 恢复正常（此前连续失败 $($failCounts[$name]) 次）" + Send-Alert "服务恢复" "$name 已恢复正常" + } + $failCounts[$name] = 0 + } else { + if ($null -eq $failCounts[$name]) { + $failCounts[$name] = 0 + } + $failCounts[$name] = $failCounts[$name] + 1 + $count = $failCounts[$name] + Write-Log "$name 不健康（第 $count 次）" + + if ($check.Type -eq "frontend") { + Write-Log "$name 重启计划任务 $FrontendTask" + Stop-ScheduledTask -TaskName $FrontendTask -ErrorAction SilentlyContinue + Start-Sleep -Seconds 2 + Start-ScheduledTask -TaskName $FrontendTask -ErrorAction SilentlyContinue + Start-Sleep -Seconds 6 + $healthy = Test-Http $check.Url + } elseif ($check.Type -eq "http") { + Write-Log "$name 重启 Compose 服务 $($check.Service)" + Invoke-Compose @("restart", $check.Service) | Out-Null + Start-Sleep -Seconds 12 + $healthy = Test-Http $check.Url + + if (-not $healthy) { + Write-Log "$name 重启后仍不健康，执行 compose up -d" + Invoke-Compose @("up", "-d", $check.Service) | Out-Null + } + } else { + Write-Log "$name 拉起容器 $($check.Container)" + docker start $check.Container 2>&1 | Out-Null + Start-Sleep -Seconds 5 + $healthy = Test-Container $check.Container + + if (-not $healthy) { + Write-Log "$name 启动失败，执行 compose up -d" + Invoke-Compose @("up", "-d", $check.Service) | Out-Null + } + } + + # 连续失败达到上限 → 告警 + if (-not $healthy -and $count -ge $MaxRestarts) { + Send-Alert "服务异常" "$name 已连续 $count 次不健康，已尝试重启。请人工介入检查。" + Write-Log "$name 连续 $count 次失败，已发告警" + } elseif ($healthy) { + Write-Log "$name 已自动恢复" + } + } +} + +# 磁盘检查：超过 85% 告警 +$drive = (Get-PSDrive C -ErrorAction SilentlyContinue) +if ($drive) { + $pct = [math]::Round(($drive.Used / ($drive.Used + $drive.Free)) * 100) + if ($pct -gt 85) { + Write-Log "磁盘使用率 ${pct}%（超过 85%）" + Send-Alert "磁盘告警" "C 盘使用率 ${pct}%，请清理日志或扩容" + } +}