129 lines
4.0 KiB
YAML
129 lines
4.0 KiB
YAML
# Author: fruit.com.cn
|
|
# Grafana 告警规则：服务下线 / 响应慢 / 磁盘满 自动通知。
|
|
# 需要在 Grafana UI 里配一个 Contact Point（飞书 webhook）并绑定到这些规则。
|
|
|
|
apiVersion: 1
|
|
|
|
groups:
|
|
- orgId: 1
|
|
name: fruitshorts-health
|
|
folder: FruitShorts
|
|
interval: 30s
|
|
rules:
|
|
# ── 服务下线 ──
|
|
- uid: fruitshorts-app-down
|
|
title: "app-api 下线"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 120, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: 'probe_success{instance=~"http://app-api:8081/.*"} == 0'
|
|
refId: A
|
|
- refId: C
|
|
relativeTimeRange: { from: 0, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator: { params: [0.5], type: gt }
|
|
operator: { type: and }
|
|
query: { params: [A] }
|
|
reducer: { type: last }
|
|
type: query
|
|
refId: C
|
|
noDataState: Alerting
|
|
for: 2m
|
|
annotations:
|
|
summary: "app-api (8081) 已下线超过 2 分钟"
|
|
labels:
|
|
severity: critical
|
|
team: backend
|
|
|
|
- uid: fruitshorts-admin-down
|
|
title: "admin-api 下线"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 120, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: 'probe_success{instance=~"http://admin-api:8080/.*"} == 0'
|
|
refId: A
|
|
- refId: C
|
|
relativeTimeRange: { from: 0, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator: { params: [0.5], type: gt }
|
|
operator: { type: and }
|
|
query: { params: [A] }
|
|
reducer: { type: last }
|
|
type: query
|
|
refId: C
|
|
noDataState: Alerting
|
|
for: 2m
|
|
annotations:
|
|
summary: "admin-api (8080) 已下线超过 2 分钟"
|
|
labels:
|
|
severity: critical
|
|
team: backend
|
|
|
|
# ── 响应慢 ──
|
|
- uid: fruitshorts-slow-response
|
|
title: "接口响应过慢"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: 'histogram_quantile(0.95, rate(probe_duration_seconds_bucket[5m])) > 2'
|
|
refId: A
|
|
- refId: C
|
|
relativeTimeRange: { from: 0, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator: { params: [0.5], type: gt }
|
|
operator: { type: and }
|
|
query: { params: [A] }
|
|
reducer: { type: last }
|
|
type: query
|
|
refId: C
|
|
for: 5m
|
|
annotations:
|
|
summary: "接口 P95 响应时间超过 2 秒"
|
|
labels:
|
|
severity: warning
|
|
team: backend
|
|
|
|
# ── 容器重启频繁 ──
|
|
- uid: fruitshorts-container-restart
|
|
title: "容器频繁重启"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: 'increase(container_start_time_seconds[10m]) > 3'
|
|
refId: A
|
|
- refId: C
|
|
relativeTimeRange: { from: 0, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator: { params: [0.5], type: gt }
|
|
operator: { type: and }
|
|
query: { params: [A] }
|
|
reducer: { type: last }
|
|
type: query
|
|
refId: C
|
|
for: 1m
|
|
annotations:
|
|
summary: "有容器在 10 分钟内重启超过 3 次"
|
|
labels:
|
|
severity: warning
|
|
team: infra
|