From 09b0f89e8d8799441820cda19fe5634717a8fbf8 Mon Sep 17 00:00:00 2001
From: Zhou Weidong <502333560@qq.com>
Date: Wed, 28 Oct 2026 18:51:52 +0800
Subject: [PATCH] =?UTF-8?q?fix(deploy):=20=E7=9C=8B=E9=97=A8=E7=8B=97?=
 =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E8=AE=A1=E6=95=B0=E8=90=BD=E7=9B=98=E5=B9=B6?=
 =?UTF-8?q?=E8=A1=A5=E9=BD=90=E5=91=8A=E8=AD=A6=20Webhook=20=E5=85=9C?=
 =?UTF-8?q?=E5=BA=95?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

计划任务每 2 分钟起新进程，进程内计数永远到不了阈值，告警静默哑火；计数改
JSON 落盘跨进程累加，达阈值后只在首达与每 30 轮提醒。Webhook 支持参数、环境
变量、deploy/feishu-webhook.txt 三级兜底，密钥文件不入库。
---
 .gitignore          |  3 +++
 deploy/watchdog.ps1 | 53 +++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 74cb655..bd8c463 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,3 +20,6 @@ data/uploads/*
 # 部署用的真实环境变量（含密钥），只提交 .env.example
 .env
 dump.rdb
+
+# 看门狗飞书告警 Webhook，只存在于部署机
+deploy/feishu-webhook.txt
diff --git a/deploy/watchdog.ps1 b/deploy/watchdog.ps1
index 404211f..84c62db 100644
--- a/deploy/watchdog.ps1
+++ b/deploy/watchdog.ps1
@@ -14,6 +14,9 @@ param(
 $ErrorActionPreference = "Continue"
 $ComposeFile = Join-Path $ComposeDir "docker-compose.yml"
 $LogFile = Join-Path $LogDir "watchdog-$(Get-Date -Format 'yyyy-MM-dd').log"
+# 连续失败计数落盘文件：计划任务每 2 分钟起一个新进程，计数只放进程内永远累积不到
+# $MaxRestarts，告警会静默哑火。
+$StateFile = Join-Path $LogDir "watchdog-state.json"
 
 function Write-Log($msg) {
     $line = "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') $msg"
@@ -33,6 +36,29 @@ function Send-Alert($title, $msg) {
     }
 }
 
+# 连续失败计数的读写：JSON 落盘，跨进程保留
+function Import-FailCounts {
+    $counts = @{}
+    if (-not (Test-Path $StateFile)) { return $counts }
+    try {
+        $saved = Get-Content -Path $StateFile -Raw | ConvertFrom-Json
+        foreach ($prop in $saved.PSObject.Properties) {
+            $counts[$prop.Name] = [int]$prop.Value
+        }
+    } catch {
+        Write-Log "看门狗计数文件解析失败，各服务按 0 重新计数: $_"
+    }
+    return $counts
+}
+
+function Export-FailCounts($counts) {
+    try {
+        $counts | ConvertTo-Json | Set-Content -Path $StateFile -Encoding UTF8
+    } catch {
+        Write-Log "看门狗计数文件写入失败: $_"
+    }
+}
+
 function Test-Docker {
     docker info --format "{{.ServerVersion}}" 2>$null | Out-Null
     return ($LASTEXITCODE -eq 0)
@@ -60,6 +86,20 @@ function Invoke-Compose([string[]]$composeArgs) {
 # 确保日志目录存在
 New-Item -ItemType Directory -Force -Path $LogDir | Out-Null
 
+# 告警 Webhook：命令行参数 > 环境变量 > deploy 目录下的 feishu-webhook.txt。
+# 计划任务注册命令里不写密钥（密钥不该进版本库），由部署机本地放文件或配环境变量。
+if (-not $FeishuWebhook) { $FeishuWebhook = $env:FRUITSHORTS_FEISHU_WEBHOOK }
+if (-not $FeishuWebhook) {
+    $webhookFile = Join-Path $PSScriptRoot "feishu-webhook.txt"
+    if (Test-Path $webhookFile) {
+        $FeishuWebhook = (Get-Content -Path $webhookFile -Raw -ErrorAction SilentlyContinue)
+        if ($FeishuWebhook) { $FeishuWebhook = $FeishuWebhook.Trim() }
+    }
+}
+if (-not $FeishuWebhook) {
+    Write-Log "未配置飞书 Webhook（参数/环境变量/feishu-webhook.txt 均为空），告警只写本地日志"
+}
+
 # Docker Desktop 登录后偶尔尚未就绪，先等待并尝试拉起。
 if (-not (Test-Docker)) {
     Write-Log "Docker 未就绪，尝试启动 Docker Desktop"
@@ -91,8 +131,8 @@ $checks = @(
     @{ Name = "Prometheus"; Type = "http";     Url = "http://127.0.0.1:9090/-/healthy"; Service = "prometheus"; Container = "fruitshorts-prometheus" }
 )
 
-# 跟踪每个服务的连续失败次数
-$failCounts = @{}
+# 跟踪每个服务的连续失败次数（跨进程从状态文件恢复）
+$failCounts = Import-FailCounts
 
 foreach ($check in $checks) {
     $name = $check.Name
@@ -149,8 +189,10 @@ foreach ($check in $checks) {
             }
         }
 
-        # 连续失败达到上限 → 告警
-        if (-not $healthy -and $count -ge $MaxRestarts) {
+        # 连续失败达到上限 → 告警。计数会跨进程累积，所以只在首次达阈值和
+        # 之后每 30 轮（约 1 小时）提醒一次，避免每 2 分钟刷屏。
+        if (-not $healthy -and $count -ge $MaxRestarts -and
+                ($count -eq $MaxRestarts -or $count % 30 -eq 0)) {
             Send-Alert "服务异常" "$name 已连续 $count 次不健康，已尝试重启。请人工介入检查。"
             Write-Log "$name 连续 $count 次失败，已发告警"
         } elseif ($healthy) {
@@ -168,3 +210,6 @@ if ($drive) {
         Send-Alert "磁盘告警" "C 盘使用率 ${pct}%，请清理日志或扩容"
     }
 }
+
+# 本轮结果落盘，供下一轮（2 分钟后的新进程）继续累加
+Export-FailCounts $failCounts