fix(deploy): 看门狗失败计数落盘并补齐告警 Webhook 兜底
计划任务每 2 分钟起新进程,进程内计数永远到不了阈值,告警静默哑火;计数改 JSON 落盘跨进程累加,达阈值后只在首达与每 30 轮提醒。Webhook 支持参数、环境 变量、deploy/feishu-webhook.txt 三级兜底,密钥文件不入库。
This commit is contained in:
1 parent
8fb1f52777
commit
09b0f89e8d
2 files changed
+52
-4
No files matched your search
+49
-4
@@ -14,6 +14,9 @@ param(
|
||||
$ErrorActionPreference = "Continue"
|
||||
$ComposeFile = Join-Path $ComposeDir "docker-compose.yml"
|
||||
$LogFile = Join-Path $LogDir "watchdog-$(Get-Date -Format 'yyyy-MM-dd').log"
|
||||
# 连续失败计数落盘文件:计划任务每 2 分钟起一个新进程,计数只放进程内永远累积不到
|
||||
# $MaxRestarts,告警会静默哑火。
|
||||
$StateFile = Join-Path $LogDir "watchdog-state.json"
|
||||
|
||||
function Write-Log($msg) {
|
||||
$line = "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $msg"
|
||||
@@ -33,6 +36,29 @@ function Send-Alert($title, $msg) {
|
||||
}
|
||||
}
|
||||
|
||||
# 连续失败计数的读写:JSON 落盘,跨进程保留
|
||||
function Import-FailCounts {
|
||||
$counts = @{}
|
||||
if (-not (Test-Path $StateFile)) { return $counts }
|
||||
try {
|
||||
$saved = Get-Content -Path $StateFile -Raw | ConvertFrom-Json
|
||||
foreach ($prop in $saved.PSObject.Properties) {
|
||||
$counts[$prop.Name] = [int]$prop.Value
|
||||
}
|
||||
} catch {
|
||||
Write-Log "看门狗计数文件解析失败,各服务按 0 重新计数: $_"
|
||||
}
|
||||
return $counts
|
||||
}
|
||||
|
||||
function Export-FailCounts($counts) {
|
||||
try {
|
||||
$counts | ConvertTo-Json | Set-Content -Path $StateFile -Encoding UTF8
|
||||
} catch {
|
||||
Write-Log "看门狗计数文件写入失败: $_"
|
||||
}
|
||||
}
|
||||
|
||||
function Test-Docker {
|
||||
docker info --format "{{.ServerVersion}}" 2>$null | Out-Null
|
||||
return ($LASTEXITCODE -eq 0)
|
||||
@@ -60,6 +86,20 @@ function Invoke-Compose([string[]]$composeArgs) {
|
||||
# 确保日志目录存在
|
||||
New-Item -ItemType Directory -Force -Path $LogDir | Out-Null
|
||||
|
||||
# 告警 Webhook:命令行参数 > 环境变量 > deploy 目录下的 feishu-webhook.txt。
|
||||
# 计划任务注册命令里不写密钥(密钥不该进版本库),由部署机本地放文件或配环境变量。
|
||||
if (-not $FeishuWebhook) { $FeishuWebhook = $env:FRUITSHORTS_FEISHU_WEBHOOK }
|
||||
if (-not $FeishuWebhook) {
|
||||
$webhookFile = Join-Path $PSScriptRoot "feishu-webhook.txt"
|
||||
if (Test-Path $webhookFile) {
|
||||
$FeishuWebhook = (Get-Content -Path $webhookFile -Raw -ErrorAction SilentlyContinue)
|
||||
if ($FeishuWebhook) { $FeishuWebhook = $FeishuWebhook.Trim() }
|
||||
}
|
||||
}
|
||||
if (-not $FeishuWebhook) {
|
||||
Write-Log "未配置飞书 Webhook(参数/环境变量/feishu-webhook.txt 均为空),告警只写本地日志"
|
||||
}
|
||||
|
||||
# Docker Desktop 登录后偶尔尚未就绪,先等待并尝试拉起。
|
||||
if (-not (Test-Docker)) {
|
||||
Write-Log "Docker 未就绪,尝试启动 Docker Desktop"
|
||||
@@ -91,8 +131,8 @@ $checks = @(
|
||||
@{ Name = "Prometheus"; Type = "http"; Url = "http://127.0.0.1:9090/-/healthy"; Service = "prometheus"; Container = "fruitshorts-prometheus" }
|
||||
)
|
||||
|
||||
# 跟踪每个服务的连续失败次数
|
||||
$failCounts = @{}
|
||||
# 跟踪每个服务的连续失败次数(跨进程从状态文件恢复)
|
||||
$failCounts = Import-FailCounts
|
||||
|
||||
foreach ($check in $checks) {
|
||||
$name = $check.Name
|
||||
@@ -149,8 +189,10 @@ foreach ($check in $checks) {
|
||||
}
|
||||
}
|
||||
|
||||
# 连续失败达到上限 → 告警
|
||||
if (-not $healthy -and $count -ge $MaxRestarts) {
|
||||
# 连续失败达到上限 → 告警。计数会跨进程累积,所以只在首次达阈值和
|
||||
# 之后每 30 轮(约 1 小时)提醒一次,避免每 2 分钟刷屏。
|
||||
if (-not $healthy -and $count -ge $MaxRestarts -and
|
||||
($count -eq $MaxRestarts -or $count % 30 -eq 0)) {
|
||||
Send-Alert "服务异常" "$name 已连续 $count 次不健康,已尝试重启。请人工介入检查。"
|
||||
Write-Log "$name 连续 $count 次失败,已发告警"
|
||||
} elseif ($healthy) {
|
||||
@@ -168,3 +210,6 @@ if ($drive) {
|
||||
Send-Alert "磁盘告警" "C 盘使用率 ${pct}%,请清理日志或扩容"
|
||||
}
|
||||
}
|
||||
|
||||
# 本轮结果落盘,供下一轮(2 分钟后的新进程)继续累加
|
||||
Export-FailCounts $failCounts
|
||||
Reference in new issue
Block a user