chore(deploy): 加入服务看门狗脚本与 Windows 计划任务注册文件
- watchdog.ps1：每 2 分钟健康检查，Docker/DB/Redis/API/Grafana 任一 不健康自动重启，连续 3 次失败发飞书告警，C 盘超 85% 也告警 - backend-watchdog-task.xml：开机触发、2 分钟循环的计划任务配置
This commit is contained in:
Zhou Weidong authored and trae-solo-agent committed
1 parent
3e2d6a06d9
commit
657190d5a8
2 files changed
+202
No files matched your search
@@ -0,0 +1,32 @@
|
||||
<?xml version="1.0" encoding="UTF-16"?>
|
||||
<Task version="1.3" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">
|
||||
<RegistrationInfo>
|
||||
<Description>FruitShorts backend watchdog: health check every 2 minutes and restart unhealthy services.</Description>
|
||||
</RegistrationInfo>
|
||||
<Principals>
|
||||
<Principal id="Author">
|
||||
<UserId>S-1-5-18</UserId>
|
||||
<RunLevel>HighestAvailable</RunLevel>
|
||||
</Principal>
|
||||
</Principals>
|
||||
<Settings>
|
||||
<MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy>
|
||||
<DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries>
|
||||
<StopIfGoingOnBatteries>false</StopIfGoingOnBatteries>
|
||||
<StartWhenAvailable>true</StartWhenAvailable>
|
||||
<ExecutionTimeLimit>PT10M</ExecutionTimeLimit>
|
||||
</Settings>
|
||||
<Triggers>
|
||||
<BootTrigger>
|
||||
<Repetition>
|
||||
<Interval>PT2M</Interval>
|
||||
</Repetition>
|
||||
</BootTrigger>
|
||||
</Triggers>
|
||||
<Actions Context="Author">
|
||||
<Exec>
|
||||
<Command>powershell.exe</Command>
|
||||
<Arguments>-NoProfile -ExecutionPolicy Bypass -File "C:\Users\Administrator\Desktop\ReelShorts\FruitServer\deploy\watchdog.ps1" -ComposeDir "C:\Users\Administrator\Desktop\ReelShorts\FruitServer" -LogDir "C:\Users\Administrator\Desktop\ReelShorts\FruitServer\logs"</Arguments>
|
||||
</Exec>
|
||||
</Actions>
|
||||
</Task>
|
||||
@@ -0,0 +1,170 @@
|
||||
# Author: fruit.com.cn
|
||||
#
|
||||
# 服务看门狗：每 2 分钟检查一次，挂了自动拉起，连续失败发飞书告警。
|
||||
# 注册为 Windows 计划任务后无人值守运行。
|
||||
|
||||
param(
|
||||
[string]$ComposeDir = "C:\FruitServer",
|
||||
[string]$LogDir = "C:\FruitServer\logs",
|
||||
[string]$FeishuWebhook = "",
|
||||
[int]$MaxRestarts = 3,
|
||||
[string]$FrontendTask = "FruitShorts-Web-3005"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Continue"
|
||||
$ComposeFile = Join-Path $ComposeDir "docker-compose.yml"
|
||||
$LogFile = Join-Path $LogDir "watchdog-$(Get-Date -Format 'yyyy-MM-dd').log"
|
||||
|
||||
function Write-Log($msg) {
|
||||
$line = "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $msg"
|
||||
Add-Content -Path $LogFile -Value $line -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
function Send-Alert($title, $msg) {
|
||||
if (-not $FeishuWebhook) { return }
|
||||
$body = @{
|
||||
msg_type = "text"
|
||||
content = @{ text = "[$title] $msg" }
|
||||
} | ConvertTo-Json -Depth 3
|
||||
try {
|
||||
Invoke-RestMethod -Uri $FeishuWebhook -Method Post -Body $body -ContentType "application/json" -TimeoutSec 10
|
||||
} catch {
|
||||
Write-Log "飞书告警发送失败: $_"
|
||||
}
|
||||
}
|
||||
|
||||
function Test-Docker {
|
||||
docker info --format "{{.ServerVersion}}" 2>$null | Out-Null
|
||||
return ($LASTEXITCODE -eq 0)
|
||||
}
|
||||
|
||||
function Test-Http([string]$url) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -Uri $url -TimeoutSec 10 -UseBasicParsing -ErrorAction Stop
|
||||
return ($response.StatusCode -ge 200 -and $response.StatusCode -lt 400)
|
||||
} catch {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
function Test-Container([string]$container) {
|
||||
$status = docker inspect -f "{{.State.Status}}" $container 2>$null
|
||||
return ($status -eq "running")
|
||||
}
|
||||
|
||||
function Invoke-Compose([string[]]$composeArgs) {
|
||||
& docker compose -f $ComposeFile @composeArgs 2>&1 | Out-Null
|
||||
return $LASTEXITCODE
|
||||
}
|
||||
|
||||
# 确保日志目录存在
|
||||
New-Item -ItemType Directory -Force -Path $LogDir | Out-Null
|
||||
|
||||
# Docker Desktop 登录后偶尔尚未就绪，先等待并尝试拉起。
|
||||
if (-not (Test-Docker)) {
|
||||
Write-Log "Docker 未就绪，尝试启动 Docker Desktop"
|
||||
$dockerExe = "C:\Program Files\Docker\Docker\Docker Desktop.exe"
|
||||
if (Test-Path $dockerExe) {
|
||||
Start-Process -FilePath $dockerExe
|
||||
}
|
||||
|
||||
for ($i = 0; $i -lt 18; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
if (Test-Docker) { break }
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Docker)) {
|
||||
Write-Log "Docker 在 90 秒内未就绪，跳过本轮检查"
|
||||
Send-Alert "服务异常" "Docker Desktop 无法启动，请人工检查桌面机。"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# 健康检查定义：名称 → 检查方式
|
||||
$checks = @(
|
||||
@{ Name = "app-api"; Type = "http"; Url = "http://127.0.0.1:8081/healthz"; Service = "app-api"; Container = "fruitshorts-app-api" },
|
||||
@{ Name = "admin-api"; Type = "http"; Url = "http://127.0.0.1:8080/healthz"; Service = "admin-api"; Container = "fruitshorts-admin-api" },
|
||||
@{ Name = "frontend"; Type = "frontend"; Url = "http://127.0.0.1:3005/dist-dev/" },
|
||||
@{ Name = "Grafana"; Type = "http"; Url = "http://127.0.0.1:3000/api/health"; Service = "grafana"; Container = "fruitshorts-grafana" },
|
||||
@{ Name = "PostgreSQL"; Type = "docker"; Service = "db"; Container = "fruitshorts-db" },
|
||||
@{ Name = "Redis"; Type = "docker"; Service = "redis"; Container = "fruitshorts-redis" },
|
||||
@{ Name = "Prometheus"; Type = "http"; Url = "http://127.0.0.1:9090/-/healthy"; Service = "prometheus"; Container = "fruitshorts-prometheus" }
|
||||
)
|
||||
|
||||
# 跟踪每个服务的连续失败次数
|
||||
$failCounts = @{}
|
||||
|
||||
foreach ($check in $checks) {
|
||||
$name = $check.Name
|
||||
$healthy = $false
|
||||
|
||||
if ($check.Type -eq "http") {
|
||||
$healthy = Test-Http $check.Url
|
||||
} elseif ($check.Type -eq "frontend") {
|
||||
$healthy = Test-Http $check.Url
|
||||
} else {
|
||||
$healthy = Test-Container $check.Container
|
||||
}
|
||||
|
||||
if ($healthy) {
|
||||
if ($failCounts[$name] -gt 0) {
|
||||
Write-Log "$name 恢复正常（此前连续失败 $($failCounts[$name]) 次）"
|
||||
Send-Alert "服务恢复" "$name 已恢复正常"
|
||||
}
|
||||
$failCounts[$name] = 0
|
||||
} else {
|
||||
if ($null -eq $failCounts[$name]) {
|
||||
$failCounts[$name] = 0
|
||||
}
|
||||
$failCounts[$name] = $failCounts[$name] + 1
|
||||
$count = $failCounts[$name]
|
||||
Write-Log "$name 不健康（第 $count 次）"
|
||||
|
||||
if ($check.Type -eq "frontend") {
|
||||
Write-Log "$name 重启计划任务 $FrontendTask"
|
||||
Stop-ScheduledTask -TaskName $FrontendTask -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 2
|
||||
Start-ScheduledTask -TaskName $FrontendTask -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 6
|
||||
$healthy = Test-Http $check.Url
|
||||
} elseif ($check.Type -eq "http") {
|
||||
Write-Log "$name 重启 Compose 服务 $($check.Service)"
|
||||
Invoke-Compose @("restart", $check.Service) | Out-Null
|
||||
Start-Sleep -Seconds 12
|
||||
$healthy = Test-Http $check.Url
|
||||
|
||||
if (-not $healthy) {
|
||||
Write-Log "$name 重启后仍不健康，执行 compose up -d"
|
||||
Invoke-Compose @("up", "-d", $check.Service) | Out-Null
|
||||
}
|
||||
} else {
|
||||
Write-Log "$name 拉起容器 $($check.Container)"
|
||||
docker start $check.Container 2>&1 | Out-Null
|
||||
Start-Sleep -Seconds 5
|
||||
$healthy = Test-Container $check.Container
|
||||
|
||||
if (-not $healthy) {
|
||||
Write-Log "$name 启动失败，执行 compose up -d"
|
||||
Invoke-Compose @("up", "-d", $check.Service) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
# 连续失败达到上限 → 告警
|
||||
if (-not $healthy -and $count -ge $MaxRestarts) {
|
||||
Send-Alert "服务异常" "$name 已连续 $count 次不健康，已尝试重启。请人工介入检查。"
|
||||
Write-Log "$name 连续 $count 次失败，已发告警"
|
||||
} elseif ($healthy) {
|
||||
Write-Log "$name 已自动恢复"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# 磁盘检查：超过 85% 告警
|
||||
$drive = (Get-PSDrive C -ErrorAction SilentlyContinue)
|
||||
if ($drive) {
|
||||
$pct = [math]::Round(($drive.Used / ($drive.Used + $drive.Free)) * 100)
|
||||
if ($pct -gt 85) {
|
||||
Write-Log "磁盘使用率 ${pct}%（超过 85%）"
|
||||
Send-Alert "磁盘告警" "C 盘使用率 ${pct}%，请清理日志或扩容"
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user