Zhou Weidong
09b0f89e8d
计划任务每 2 分钟起新进程，进程内计数永远到不了阈值，告警静默哑火；计数改 JSON 落盘跨进程累加，达阈值后只在首达与每 30 轮提醒。Webhook 支持参数、环境 变量、deploy/feishu-webhook.txt 三级兜底，密钥文件不入库。
216 lines
8.2 KiB
PowerShell
216 lines
8.2 KiB
PowerShell
# Author: fruit.com.cn
|
||
#
|
||
# 服务看门狗：每 2 分钟检查一次，挂了自动拉起，连续失败发飞书告警。
|
||
# 注册为 Windows 计划任务后无人值守运行。
|
||
|
||
param(
|
||
[string]$ComposeDir = "C:\FruitServer",
|
||
[string]$LogDir = "C:\FruitServer\logs",
|
||
[string]$FeishuWebhook = "",
|
||
[int]$MaxRestarts = 3,
|
||
[string]$FrontendTask = "FruitShorts-Web-3005"
|
||
)
|
||
|
||
$ErrorActionPreference = "Continue"
|
||
$ComposeFile = Join-Path $ComposeDir "docker-compose.yml"
|
||
$LogFile = Join-Path $LogDir "watchdog-$(Get-Date -Format 'yyyy-MM-dd').log"
|
||
# 连续失败计数落盘文件：计划任务每 2 分钟起一个新进程，计数只放进程内永远累积不到
|
||
# $MaxRestarts，告警会静默哑火。
|
||
$StateFile = Join-Path $LogDir "watchdog-state.json"
|
||
|
||
function Write-Log($msg) {
|
||
$line = "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $msg"
|
||
Add-Content -Path $LogFile -Value $line -ErrorAction SilentlyContinue
|
||
}
|
||
|
||
function Send-Alert($title, $msg) {
|
||
if (-not $FeishuWebhook) { return }
|
||
$body = @{
|
||
msg_type = "text"
|
||
content = @{ text = "[$title] $msg" }
|
||
} | ConvertTo-Json -Depth 3
|
||
try {
|
||
Invoke-RestMethod -Uri $FeishuWebhook -Method Post -Body $body -ContentType "application/json" -TimeoutSec 10
|
||
} catch {
|
||
Write-Log "飞书告警发送失败: $_"
|
||
}
|
||
}
|
||
|
||
# 连续失败计数的读写：JSON 落盘，跨进程保留
|
||
function Import-FailCounts {
|
||
$counts = @{}
|
||
if (-not (Test-Path $StateFile)) { return $counts }
|
||
try {
|
||
$saved = Get-Content -Path $StateFile -Raw | ConvertFrom-Json
|
||
foreach ($prop in $saved.PSObject.Properties) {
|
||
$counts[$prop.Name] = [int]$prop.Value
|
||
}
|
||
} catch {
|
||
Write-Log "看门狗计数文件解析失败，各服务按 0 重新计数: $_"
|
||
}
|
||
return $counts
|
||
}
|
||
|
||
function Export-FailCounts($counts) {
|
||
try {
|
||
$counts | ConvertTo-Json | Set-Content -Path $StateFile -Encoding UTF8
|
||
} catch {
|
||
Write-Log "看门狗计数文件写入失败: $_"
|
||
}
|
||
}
|
||
|
||
function Test-Docker {
|
||
docker info --format "{{.ServerVersion}}" 2>$null | Out-Null
|
||
return ($LASTEXITCODE -eq 0)
|
||
}
|
||
|
||
function Test-Http([string]$url) {
|
||
try {
|
||
$response = Invoke-WebRequest -Uri $url -TimeoutSec 10 -UseBasicParsing -ErrorAction Stop
|
||
return ($response.StatusCode -ge 200 -and $response.StatusCode -lt 400)
|
||
} catch {
|
||
return $false
|
||
}
|
||
}
|
||
|
||
function Test-Container([string]$container) {
|
||
$status = docker inspect -f "{{.State.Status}}" $container 2>$null
|
||
return ($status -eq "running")
|
||
}
|
||
|
||
function Invoke-Compose([string[]]$composeArgs) {
|
||
& docker compose -f $ComposeFile @composeArgs 2>&1 | Out-Null
|
||
return $LASTEXITCODE
|
||
}
|
||
|
||
# 确保日志目录存在
|
||
New-Item -ItemType Directory -Force -Path $LogDir | Out-Null
|
||
|
||
# 告警 Webhook：命令行参数 > 环境变量 > deploy 目录下的 feishu-webhook.txt。
|
||
# 计划任务注册命令里不写密钥（密钥不该进版本库），由部署机本地放文件或配环境变量。
|
||
if (-not $FeishuWebhook) { $FeishuWebhook = $env:FRUITSHORTS_FEISHU_WEBHOOK }
|
||
if (-not $FeishuWebhook) {
|
||
$webhookFile = Join-Path $PSScriptRoot "feishu-webhook.txt"
|
||
if (Test-Path $webhookFile) {
|
||
$FeishuWebhook = (Get-Content -Path $webhookFile -Raw -ErrorAction SilentlyContinue)
|
||
if ($FeishuWebhook) { $FeishuWebhook = $FeishuWebhook.Trim() }
|
||
}
|
||
}
|
||
if (-not $FeishuWebhook) {
|
||
Write-Log "未配置飞书 Webhook（参数/环境变量/feishu-webhook.txt 均为空），告警只写本地日志"
|
||
}
|
||
|
||
# Docker Desktop 登录后偶尔尚未就绪，先等待并尝试拉起。
|
||
if (-not (Test-Docker)) {
|
||
Write-Log "Docker 未就绪，尝试启动 Docker Desktop"
|
||
$dockerExe = "C:\Program Files\Docker\Docker\Docker Desktop.exe"
|
||
if (Test-Path $dockerExe) {
|
||
Start-Process -FilePath $dockerExe
|
||
}
|
||
|
||
for ($i = 0; $i -lt 18; $i++) {
|
||
Start-Sleep -Seconds 5
|
||
if (Test-Docker) { break }
|
||
}
|
||
}
|
||
|
||
if (-not (Test-Docker)) {
|
||
Write-Log "Docker 在 90 秒内未就绪，跳过本轮检查"
|
||
Send-Alert "服务异常" "Docker Desktop 无法启动，请人工检查桌面机。"
|
||
exit 1
|
||
}
|
||
|
||
# 健康检查定义：名称 → 检查方式
|
||
$checks = @(
|
||
@{ Name = "app-api"; Type = "http"; Url = "http://127.0.0.1:8081/healthz"; Service = "app-api"; Container = "fruitshorts-app-api" },
|
||
@{ Name = "admin-api"; Type = "http"; Url = "http://127.0.0.1:8080/healthz"; Service = "admin-api"; Container = "fruitshorts-admin-api" },
|
||
@{ Name = "frontend"; Type = "frontend"; Url = "http://127.0.0.1:3005/dist-dev/" },
|
||
@{ Name = "Grafana"; Type = "http"; Url = "http://127.0.0.1:3000/api/health"; Service = "grafana"; Container = "fruitshorts-grafana" },
|
||
@{ Name = "PostgreSQL"; Type = "docker"; Service = "db"; Container = "fruitshorts-db" },
|
||
@{ Name = "Redis"; Type = "docker"; Service = "redis"; Container = "fruitshorts-redis" },
|
||
@{ Name = "Prometheus"; Type = "http"; Url = "http://127.0.0.1:9090/-/healthy"; Service = "prometheus"; Container = "fruitshorts-prometheus" }
|
||
)
|
||
|
||
# 跟踪每个服务的连续失败次数（跨进程从状态文件恢复）
|
||
$failCounts = Import-FailCounts
|
||
|
||
foreach ($check in $checks) {
|
||
$name = $check.Name
|
||
$healthy = $false
|
||
|
||
if ($check.Type -eq "http") {
|
||
$healthy = Test-Http $check.Url
|
||
} elseif ($check.Type -eq "frontend") {
|
||
$healthy = Test-Http $check.Url
|
||
} else {
|
||
$healthy = Test-Container $check.Container
|
||
}
|
||
|
||
if ($healthy) {
|
||
if ($failCounts[$name] -gt 0) {
|
||
Write-Log "$name 恢复正常（此前连续失败 $($failCounts[$name]) 次）"
|
||
Send-Alert "服务恢复" "$name 已恢复正常"
|
||
}
|
||
$failCounts[$name] = 0
|
||
} else {
|
||
if ($null -eq $failCounts[$name]) {
|
||
$failCounts[$name] = 0
|
||
}
|
||
$failCounts[$name] = $failCounts[$name] + 1
|
||
$count = $failCounts[$name]
|
||
Write-Log "$name 不健康（第 $count 次）"
|
||
|
||
if ($check.Type -eq "frontend") {
|
||
Write-Log "$name 重启计划任务 $FrontendTask"
|
||
Stop-ScheduledTask -TaskName $FrontendTask -ErrorAction SilentlyContinue
|
||
Start-Sleep -Seconds 2
|
||
Start-ScheduledTask -TaskName $FrontendTask -ErrorAction SilentlyContinue
|
||
Start-Sleep -Seconds 6
|
||
$healthy = Test-Http $check.Url
|
||
} elseif ($check.Type -eq "http") {
|
||
Write-Log "$name 重启 Compose 服务 $($check.Service)"
|
||
Invoke-Compose @("restart", $check.Service) | Out-Null
|
||
Start-Sleep -Seconds 12
|
||
$healthy = Test-Http $check.Url
|
||
|
||
if (-not $healthy) {
|
||
Write-Log "$name 重启后仍不健康，执行 compose up -d"
|
||
Invoke-Compose @("up", "-d", $check.Service) | Out-Null
|
||
}
|
||
} else {
|
||
Write-Log "$name 拉起容器 $($check.Container)"
|
||
docker start $check.Container 2>&1 | Out-Null
|
||
Start-Sleep -Seconds 5
|
||
$healthy = Test-Container $check.Container
|
||
|
||
if (-not $healthy) {
|
||
Write-Log "$name 启动失败，执行 compose up -d"
|
||
Invoke-Compose @("up", "-d", $check.Service) | Out-Null
|
||
}
|
||
}
|
||
|
||
# 连续失败达到上限 → 告警。计数会跨进程累积，所以只在首次达阈值和
|
||
# 之后每 30 轮（约 1 小时）提醒一次，避免每 2 分钟刷屏。
|
||
if (-not $healthy -and $count -ge $MaxRestarts -and
|
||
($count -eq $MaxRestarts -or $count % 30 -eq 0)) {
|
||
Send-Alert "服务异常" "$name 已连续 $count 次不健康，已尝试重启。请人工介入检查。"
|
||
Write-Log "$name 连续 $count 次失败，已发告警"
|
||
} elseif ($healthy) {
|
||
Write-Log "$name 已自动恢复"
|
||
}
|
||
}
|
||
}
|
||
|
||
# 磁盘检查：超过 85% 告警
|
||
$drive = (Get-PSDrive C -ErrorAction SilentlyContinue)
|
||
if ($drive) {
|
||
$pct = [math]::Round(($drive.Used / ($drive.Used + $drive.Free)) * 100)
|
||
if ($pct -gt 85) {
|
||
Write-Log "磁盘使用率 ${pct}%（超过 85%）"
|
||
Send-Alert "磁盘告警" "C 盘使用率 ${pct}%，请清理日志或扩容"
|
||
}
|
||
}
|
||
|
||
# 本轮结果落盘，供下一轮（2 分钟后的新进程）继续累加
|
||
Export-FailCounts $failCounts
|