fix(deploy): 看门狗失败计数落盘并补齐告警 Webhook 兜底
计划任务每 2 分钟起新进程,进程内计数永远到不了阈值,告警静默哑火;计数改 JSON 落盘跨进程累加,达阈值后只在首达与每 30 轮提醒。Webhook 支持参数、环境 变量、deploy/feishu-webhook.txt 三级兜底,密钥文件不入库。
This commit is contained in:
1 parent
8fb1f52777
commit
09b0f89e8d
2 files changed
+52
-4
No files matched your search
@@ -20,3 +20,6 @@ data/uploads/*
|
|||||||
# 部署用的真实环境变量(含密钥),只提交 .env.example
|
# 部署用的真实环境变量(含密钥),只提交 .env.example
|
||||||
.env
|
.env
|
||||||
dump.rdb
|
dump.rdb
|
||||||
|
|
||||||
|
# 看门狗飞书告警 Webhook,只存在于部署机
|
||||||
|
deploy/feishu-webhook.txt
|
||||||
+49
-4
@@ -14,6 +14,9 @@ param(
|
|||||||
$ErrorActionPreference = "Continue"
|
$ErrorActionPreference = "Continue"
|
||||||
$ComposeFile = Join-Path $ComposeDir "docker-compose.yml"
|
$ComposeFile = Join-Path $ComposeDir "docker-compose.yml"
|
||||||
$LogFile = Join-Path $LogDir "watchdog-$(Get-Date -Format 'yyyy-MM-dd').log"
|
$LogFile = Join-Path $LogDir "watchdog-$(Get-Date -Format 'yyyy-MM-dd').log"
|
||||||
|
# 连续失败计数落盘文件:计划任务每 2 分钟起一个新进程,计数只放进程内永远累积不到
|
||||||
|
# $MaxRestarts,告警会静默哑火。
|
||||||
|
$StateFile = Join-Path $LogDir "watchdog-state.json"
|
||||||
|
|
||||||
function Write-Log($msg) {
|
function Write-Log($msg) {
|
||||||
$line = "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $msg"
|
$line = "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $msg"
|
||||||
@@ -33,6 +36,29 @@ function Send-Alert($title, $msg) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# 连续失败计数的读写:JSON 落盘,跨进程保留
|
||||||
|
function Import-FailCounts {
|
||||||
|
$counts = @{}
|
||||||
|
if (-not (Test-Path $StateFile)) { return $counts }
|
||||||
|
try {
|
||||||
|
$saved = Get-Content -Path $StateFile -Raw | ConvertFrom-Json
|
||||||
|
foreach ($prop in $saved.PSObject.Properties) {
|
||||||
|
$counts[$prop.Name] = [int]$prop.Value
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
Write-Log "看门狗计数文件解析失败,各服务按 0 重新计数: $_"
|
||||||
|
}
|
||||||
|
return $counts
|
||||||
|
}
|
||||||
|
|
||||||
|
function Export-FailCounts($counts) {
|
||||||
|
try {
|
||||||
|
$counts | ConvertTo-Json | Set-Content -Path $StateFile -Encoding UTF8
|
||||||
|
} catch {
|
||||||
|
Write-Log "看门狗计数文件写入失败: $_"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function Test-Docker {
|
function Test-Docker {
|
||||||
docker info --format "{{.ServerVersion}}" 2>$null | Out-Null
|
docker info --format "{{.ServerVersion}}" 2>$null | Out-Null
|
||||||
return ($LASTEXITCODE -eq 0)
|
return ($LASTEXITCODE -eq 0)
|
||||||
@@ -60,6 +86,20 @@ function Invoke-Compose([string[]]$composeArgs) {
|
|||||||
# 确保日志目录存在
|
# 确保日志目录存在
|
||||||
New-Item -ItemType Directory -Force -Path $LogDir | Out-Null
|
New-Item -ItemType Directory -Force -Path $LogDir | Out-Null
|
||||||
|
|
||||||
|
# 告警 Webhook:命令行参数 > 环境变量 > deploy 目录下的 feishu-webhook.txt。
|
||||||
|
# 计划任务注册命令里不写密钥(密钥不该进版本库),由部署机本地放文件或配环境变量。
|
||||||
|
if (-not $FeishuWebhook) { $FeishuWebhook = $env:FRUITSHORTS_FEISHU_WEBHOOK }
|
||||||
|
if (-not $FeishuWebhook) {
|
||||||
|
$webhookFile = Join-Path $PSScriptRoot "feishu-webhook.txt"
|
||||||
|
if (Test-Path $webhookFile) {
|
||||||
|
$FeishuWebhook = (Get-Content -Path $webhookFile -Raw -ErrorAction SilentlyContinue)
|
||||||
|
if ($FeishuWebhook) { $FeishuWebhook = $FeishuWebhook.Trim() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (-not $FeishuWebhook) {
|
||||||
|
Write-Log "未配置飞书 Webhook(参数/环境变量/feishu-webhook.txt 均为空),告警只写本地日志"
|
||||||
|
}
|
||||||
|
|
||||||
# Docker Desktop 登录后偶尔尚未就绪,先等待并尝试拉起。
|
# Docker Desktop 登录后偶尔尚未就绪,先等待并尝试拉起。
|
||||||
if (-not (Test-Docker)) {
|
if (-not (Test-Docker)) {
|
||||||
Write-Log "Docker 未就绪,尝试启动 Docker Desktop"
|
Write-Log "Docker 未就绪,尝试启动 Docker Desktop"
|
||||||
@@ -91,8 +131,8 @@ $checks = @(
|
|||||||
@{ Name = "Prometheus"; Type = "http"; Url = "http://127.0.0.1:9090/-/healthy"; Service = "prometheus"; Container = "fruitshorts-prometheus" }
|
@{ Name = "Prometheus"; Type = "http"; Url = "http://127.0.0.1:9090/-/healthy"; Service = "prometheus"; Container = "fruitshorts-prometheus" }
|
||||||
)
|
)
|
||||||
|
|
||||||
# 跟踪每个服务的连续失败次数
|
# 跟踪每个服务的连续失败次数(跨进程从状态文件恢复)
|
||||||
$failCounts = @{}
|
$failCounts = Import-FailCounts
|
||||||
|
|
||||||
foreach ($check in $checks) {
|
foreach ($check in $checks) {
|
||||||
$name = $check.Name
|
$name = $check.Name
|
||||||
@@ -149,8 +189,10 @@ foreach ($check in $checks) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# 连续失败达到上限 → 告警
|
# 连续失败达到上限 → 告警。计数会跨进程累积,所以只在首次达阈值和
|
||||||
if (-not $healthy -and $count -ge $MaxRestarts) {
|
# 之后每 30 轮(约 1 小时)提醒一次,避免每 2 分钟刷屏。
|
||||||
|
if (-not $healthy -and $count -ge $MaxRestarts -and
|
||||||
|
($count -eq $MaxRestarts -or $count % 30 -eq 0)) {
|
||||||
Send-Alert "服务异常" "$name 已连续 $count 次不健康,已尝试重启。请人工介入检查。"
|
Send-Alert "服务异常" "$name 已连续 $count 次不健康,已尝试重启。请人工介入检查。"
|
||||||
Write-Log "$name 连续 $count 次失败,已发告警"
|
Write-Log "$name 连续 $count 次失败,已发告警"
|
||||||
} elseif ($healthy) {
|
} elseif ($healthy) {
|
||||||
@@ -168,3 +210,6 @@ if ($drive) {
|
|||||||
Send-Alert "磁盘告警" "C 盘使用率 ${pct}%,请清理日志或扩容"
|
Send-Alert "磁盘告警" "C 盘使用率 ${pct}%,请清理日志或扩容"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# 本轮结果落盘,供下一轮(2 分钟后的新进程)继续累加
|
||||||
|
Export-FailCounts $failCounts
|
||||||
Reference in new issue
Block a user