jacy/dukang
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(ops): WeCom webhook alerts for pay/redeem anomalies

Browse Source 
Add outbound group-bot alerts, rate/amount rules, stuck-order cron, HQ test button, and health db/redis checks.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
jacy
2026-08-02 16:01:30 +08:00
parent cb6ecaaba6
commit 3328c52cb4
27 changed files with 1036 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/dukang-api/src/common/alert/alert.constants.ts
+30
View File
@@ -0,0 +1,30 @@
/** 告警级别 */
export type AlertLevel = 'P0' | 'P1' | 'P2';
/** 支付 / 核销行为异常阈值（金额单位：元） */
export const ALERT_THRESHOLDS = {
payAmountMinYuan: 1,
payAmountMaxYuan: 5000,
payAttemptPerMinute: 3,
payFailPerMinute: 5,
redeemAmountMinYuan: 1,
redeemAmountMaxYuan: 1000,
redeemAttemptPerMinute: 3,
redeemFailPerMinute: 5,
} as const;
/** Redis 分钟桶计数 TTL（秒） */
export const ALERT_RATE_BUCKET_TTL_SEC = 120;
/** 默认去重 TTL */
export const ALERT_DEDUPE_TTL_SEC = 600;
/** 频次类告警去重 TTL（同一分钟桶只推一次） */
export const ALERT_RATE_DEDUPE_TTL_SEC = 90;
/** 卡住订单扫描 */
export const ALERT_STUCK_ORDER = {
pendingShipHours: 24,
inDeliveryHours: 48,
sampleLimit: 5,
} as const;
server/dukang-api/src/common/alert/alert.module.ts
+17
View File
@@ -0,0 +1,17 @@
import { Global, Module } from '@nestjs/common';
import { RedisModule } from '../redis/redis.module';
import { AlertService } from './alert.service';
import { PayRedeemAnomalyService } from './pay-redeem-anomaly.service';
import { WecomWebhookAlertService } from '../../integrations/wecom/wecom-webhook-alert.service';
/**
* 运营告警（企微 Webhook）。Global 以便 Filter / 各业务 Module 注入。
* Webhook 发送器在本 Module 注册，避免与 WecomModule↔Common 循环依赖。
*/
@Global()
@Module({
imports: [RedisModule],
providers: [WecomWebhookAlertService, AlertService, PayRedeemAnomalyService],
exports: [WecomWebhookAlertService, AlertService, PayRedeemAnomalyService],
})
export class AlertModule {}
server/dukang-api/src/common/alert/alert.service.ts
+95
View File
@@ -0,0 +1,95 @@
import { Injectable, Logger } from '@nestjs/common';
import { RedisService } from '../redis/redis.service';
import { WecomWebhookAlertService } from '../../integrations/wecom/wecom-webhook-alert.service';
import { ALERT_DEDUPE_TTL_SEC, type AlertLevel } from './alert.constants';
export type AlertNotifyInput = {
level: AlertLevel;
category: string;
title: string;
detail: string;
dedupeKey: string;
dedupeTtlSec?: number;
};
@Injectable()
export class AlertService {
private readonly logger = new Logger(AlertService.name);
constructor(
private readonly redis: RedisService,
private readonly wecomWebhook: WecomWebhookAlertService,
) {}
/** 异步告警，不阻塞调用方 */
notify(input: AlertNotifyInput): void {
void this.notifyAsync(input).catch((e) => {
this.logger.warn(`alert notify failed: ${e instanceof Error ? e.message : String(e)}`);
});
}
async notifyAsync(input: AlertNotifyInput): Promise<boolean> {
if (!this.wecomWebhook.isEnabled()) return false;
const ttl = input.dedupeTtlSec ?? ALERT_DEDUPE_TTL_SEC;
const dedupeRedisKey = `alert:dedupe:${input.dedupeKey}`;
try {
const ok = await this.redis.client.set(dedupeRedisKey, '1', 'EX', ttl, 'NX');
if (ok !== 'OK') return false;
} catch (e) {
this.logger.warn(
`alert dedupe redis error, send anyway: ${e instanceof Error ? e.message : String(e)}`,
);
}
return this.sendMarkdownNow(input);
}
/** HQ 测试推送：不去重，返回明确结果 */
async sendTestAlert(): Promise<{ ok: boolean; message: string }> {
const url = (process.env.WECOM_ALERT_WEBHOOK_URL || '').trim();
if (!url) {
return {
ok: false,
message: '未配置 WECOM_ALERT_WEBHOOK_URL，请在服务器 .env 中填写群机器人 Webhook',
};
}
if (process.env.WECOM_ALERT_ENABLED !== 'true') {
return {
ok: false,
message: '请先开启「启用企微运营告警 Webhook」并保存后再测试',
};
}
const sent = await this.sendMarkdownNow({
level: 'P2',
category: 'ops',
title: '告警测试',
detail: '你好，这是一条告警测试消息',
dedupeKey: `test|${Date.now()}`,
});
return sent
? { ok: true, message: '已发送测试告警，请查看企微群' }
: { ok: false, message: 'Webhook 调用失败，请检查 URL 或 API 日志' };
}
private async sendMarkdownNow(input: AlertNotifyInput): Promise<boolean> {
const envLabel = (process.env.WECOM_ALERT_ENV_LABEL || process.env.NODE_ENV || 'local').trim();
const now = new Date().toLocaleString('zh-CN', { timeZone: 'Asia/Shanghai' });
const content = [
`## [${input.level}] ${escapeMd(input.title)}`,
`> 环境：<font color="comment">${escapeMd(envLabel)}</font>`,
`> 类别：<font color="comment">${escapeMd(input.category)}</font>`,
`> 时间：${escapeMd(now)}`,
'',
escapeMd(input.detail).slice(0, 3500),
].join('\n');
return this.wecomWebhook.sendMarkdown(content);
}
}
/** 轻量转义，保留企微 markdown 的 <font> 标签可用 */
function escapeMd(s: string): string {
return s.replace(/([\\`*_[\]])/g, '\\$1');
}
server/dukang-api/src/common/alert/pay-redeem-anomaly.service.ts
+213
View File
@@ -0,0 +1,213 @@
import { Injectable, Logger } from '@nestjs/common';
import { RedisService } from '../redis/redis.service';
import { AlertService } from './alert.service';
import {
ALERT_RATE_BUCKET_TTL_SEC,
ALERT_RATE_DEDUPE_TTL_SEC,
ALERT_THRESHOLDS,
} from './alert.constants';
export type PayAnomalyMeta = {
orderNo?: string;
userId?: string | number | bigint;
reason?: string;
};
export type RedeemAnomalyMeta = {
storeId?: string | number | bigint;
userId?: string | number | bigint;
recordId?: string | number | bigint;
reason?: string;
};
@Injectable()
export class PayRedeemAnomalyService {
private readonly logger = new Logger(PayRedeemAnomalyService.name);
constructor(
private readonly redis: RedisService,
private readonly alert: AlertService,
) {}
onPayAttempt(amountYuan: number, meta: PayAnomalyMeta = {}): void {
this.checkPayAmount(amountYuan, meta);
void this.bumpAndMaybeAlert(
'pay:attempt',
ALERT_THRESHOLDS.payAttemptPerMinute,
'P0',
'pay',
'支付频率异常',
`1 分钟内支付尝试超过 ${ALERT_THRESHOLDS.payAttemptPerMinute} 次`,
meta,
);
}
onPaySuccess(amountYuan: number, meta: PayAnomalyMeta = {}): void {
this.checkPayAmount(amountYuan, meta);
}
onPayFail(reason: string, meta: PayAnomalyMeta = {}): void {
void this.bumpAndMaybeAlert(
'pay:fail',
ALERT_THRESHOLDS.payFailPerMinute,
'P0',
'pay',
'支付失败频率异常',
`1 分钟内支付失败超过 ${ALERT_THRESHOLDS.payFailPerMinute} 次；最近原因：${reason}`,
{ ...meta, reason },
);
}
onRedeemAttempt(amountYuan: number, meta: RedeemAnomalyMeta = {}): void {
this.checkRedeemAmount(amountYuan, meta);
void this.bumpAndMaybeAlert(
'redeem:attempt',
ALERT_THRESHOLDS.redeemAttemptPerMinute,
'P0',
'redeem',
'核销频率异常',
`1 分钟内核销尝试超过 ${ALERT_THRESHOLDS.redeemAttemptPerMinute} 次`,
meta,
);
}
onRedeemFail(reason: string, meta: RedeemAnomalyMeta = {}): void {
void this.bumpAndMaybeAlert(
'redeem:fail',
ALERT_THRESHOLDS.redeemFailPerMinute,
'P0',
'redeem',
'核销失败频率异常',
`1 分钟内核销失败超过 ${ALERT_THRESHOLDS.redeemFailPerMinute} 次；最近原因：${reason}`,
{ ...meta, reason },
);
}
/** 弱网阈值 / 补核销待办等业务信号 */
notifyRedeemOps(title: string, detail: string, dedupeKey: string): void {
this.alert.notify({
level: 'P1',
category: 'redeem',
title,
detail,
dedupeKey,
});
}
private checkPayAmount(amountYuan: number, meta: PayAnomalyMeta): void {
if (!Number.isFinite(amountYuan)) return;
if (amountYuan < ALERT_THRESHOLDS.payAmountMinYuan) {
this.alert.notify({
level: 'P1',
category: 'pay',
title: '支付金额过低',
detail: formatPayDetail(
`支付金额 ${amountYuan} 元 < ${ALERT_THRESHOLDS.payAmountMinYuan} 元`,
meta,
),
dedupeKey: `pay_amount_low|${meta.orderNo ?? amountYuan}`,
});
} else if (amountYuan > ALERT_THRESHOLDS.payAmountMaxYuan) {
this.alert.notify({
level: 'P1',
category: 'pay',
title: '支付金额过高',
detail: formatPayDetail(
`支付金额 ${amountYuan} 元 > ${ALERT_THRESHOLDS.payAmountMaxYuan} 元`,
meta,
),
dedupeKey: `pay_amount_high|${meta.orderNo ?? amountYuan}`,
});
}
}
private checkRedeemAmount(amountYuan: number, meta: RedeemAnomalyMeta): void {
if (!Number.isFinite(amountYuan)) return;
if (amountYuan < ALERT_THRESHOLDS.redeemAmountMinYuan) {
this.alert.notify({
level: 'P1',
category: 'redeem',
title: '核销金额过低',
detail: formatRedeemDetail(
`核销金额 ${amountYuan} 元 < ${ALERT_THRESHOLDS.redeemAmountMinYuan} 元`,
meta,
),
dedupeKey: `redeem_amount_low|${meta.recordId ?? meta.storeId ?? amountYuan}`,
});
} else if (amountYuan > ALERT_THRESHOLDS.redeemAmountMaxYuan) {
this.alert.notify({
level: 'P1',
category: 'redeem',
title: '核销金额过高',
detail: formatRedeemDetail(
`核销金额 ${amountYuan} 元 > ${ALERT_THRESHOLDS.redeemAmountMaxYuan} 元`,
meta,
),
dedupeKey: `redeem_amount_high|${meta.recordId ?? meta.storeId ?? amountYuan}`,
});
}
}
private async bumpAndMaybeAlert(
counterKind: string,
threshold: number,
level: 'P0' | 'P1' | 'P2',
category: string,
title: string,
baseDetail: string,
meta: PayAnomalyMeta & RedeemAnomalyMeta,
): Promise<void> {
try {
const bucket = Math.floor(Date.now() / 60_000);
const key = `alert:cnt:${counterKind}:${bucket}`;
const count = await this.redis.incr(key, ALERT_RATE_BUCKET_TTL_SEC);
if (count <= threshold) return;
const extra = [
`当前分钟计数：${count}`,
meta.orderNo ? `订单：${meta.orderNo}` : null,
meta.storeId != null ? `门店：${String(meta.storeId)}` : null,
meta.userId != null ? `用户：${String(meta.userId)}` : null,
meta.reason ? `原因：${meta.reason}` : null,
]
.filter(Boolean)
.join('\n');
this.alert.notify({
level,
category,
title,
detail: `${baseDetail}\n${extra}`,
dedupeKey: `${counterKind}|${bucket}`,
dedupeTtlSec: ALERT_RATE_DEDUPE_TTL_SEC,
});
} catch (e) {
this.logger.warn(
`anomaly counter failed: ${e instanceof Error ? e.message : String(e)}`,
);
}
}
}
function formatPayDetail(line: string, meta: PayAnomalyMeta): string {
return [
line,
meta.orderNo ? `订单：${meta.orderNo}` : null,
meta.userId != null ? `用户：${String(meta.userId)}` : null,
meta.reason ? `原因：${meta.reason}` : null,
]
.filter(Boolean)
.join('\n');
}
function formatRedeemDetail(line: string, meta: RedeemAnomalyMeta): string {
return [
line,
meta.storeId != null ? `门店：${String(meta.storeId)}` : null,
meta.userId != null ? `用户：${String(meta.userId)}` : null,
meta.recordId != null ? `记录：${String(meta.recordId)}` : null,
meta.reason ? `原因：${meta.reason}` : null,
]
.filter(Boolean)
.join('\n');
}
server/dukang-api/src/common/filters/http-exception.filter.ts
+38 -2
View File
@@ -4,8 +4,10 @@ import {
ExceptionFilter,
HttpException,
HttpStatus,
Injectable,
} from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { AlertService } from '../alert/alert.service';
function prismaErrorMessage(exception: Prisma.PrismaClientKnownRequestError): string {
switch (exception.code) {
@@ -24,10 +26,16 @@ function prismaErrorMessage(exception: Prisma.PrismaClientKnownRequestError): st
}
@Catch()
@Injectable()
export class HttpExceptionFilter implements ExceptionFilter {
constructor(private readonly alert: AlertService) {}
catch(exception: unknown, host: ArgumentsHost) {
const ctx = host.switchToHttp();
const response = ctx.getResponse();
const request = ctx.getRequest<{ method?: string; url?: string }>();
const method = request?.method ?? '?';
const path = request?.url ?? '?';
if (exception instanceof HttpException) {
const status = exception.getStatus();
@@ -36,9 +44,20 @@ export class HttpExceptionFilter implements ExceptionFilter {
typeof res === 'string'
? res
: (res as { message?: string | string[] }).message || exception.message;
const msgText = Array.isArray(message) ? message.join(', ') : message;
if (status >= 500) {
this.alert.notify({
level: 'P0',
category: 'api_error',
title: `API ${status}`,
detail: `${method} ${path}\n${msgText}`,
dedupeKey: `http_${status}|${method}|${path}|${String(msgText).slice(0, 80)}`,
dedupeTtlSec: 120,
});
}
response.status(status).json({
code: status,
message: Array.isArray(message) ? message.join(', ') : message,
message: msgText,
data: null,
});
return;
@@ -46,6 +65,14 @@ export class HttpExceptionFilter implements ExceptionFilter {
if (exception instanceof Prisma.PrismaClientKnownRequestError) {
console.error(exception);
this.alert.notify({
level: 'P0',
category: 'api_error',
title: `Prisma ${exception.code}`,
detail: `${method} ${path}\n${exception.message}`,
dedupeKey: `prisma|${exception.code}|${method}|${path}`,
dedupeTtlSec: 120,
});
response.status(HttpStatus.BAD_REQUEST).json({
code: 400,
message: prismaErrorMessage(exception),
@@ -64,9 +91,18 @@ export class HttpExceptionFilter implements ExceptionFilter {
}
console.error(exception);
const errMsg = exception instanceof Error ? exception.message : 'Internal server error';
this.alert.notify({
level: 'P0',
category: 'api_error',
title: '未捕获异常 500',
detail: `${method} ${path}\n${errMsg}`,
dedupeKey: `uncaught|${method}|${path}|${errMsg.slice(0, 80)}`,
dedupeTtlSec: 120,
});
response.status(HttpStatus.INTERNAL_SERVER_ERROR).json({
code: 500,
message: exception instanceof Error ? exception.message : 'Internal server error',
message: errMsg,
data: null,
});
}
server/dukang-api/src/common/hq-operation/hq-operation.constants.ts
+2
View File
@@ -97,6 +97,7 @@ export const HqOperationAction = {
SYSTEM_CONFIG_UPDATE: 'SYSTEM_CONFIG_UPDATE',
SYSTEM_CONFIG_SYNC_ENV: 'SYSTEM_CONFIG_SYNC_ENV',
SYSTEM_CONFIG_IMPORT_ENV: 'SYSTEM_CONFIG_IMPORT_ENV',
WECOM_ALERT_TEST: 'WECOM_ALERT_TEST',
} as const;
export type HqOperationActionCode = (typeof HqOperationAction)[keyof typeof HqOperationAction];
@@ -199,6 +200,7 @@ export const HQ_OPERATION_ACTION_LABELS: Record<string, string> = {
[HqOperationAction.SYSTEM_CONFIG_UPDATE]: '更新系统配置',
[HqOperationAction.SYSTEM_CONFIG_SYNC_ENV]: '同步系统配置到 env 文件',
[HqOperationAction.SYSTEM_CONFIG_IMPORT_ENV]: '从当前环境导入配置',
[HqOperationAction.WECOM_ALERT_TEST]: '测试企微运营告警',
STORE_PAYOUT: '门店打款确认',
};
server/dukang-api/src/common/system-config/system-config.registry.ts
+8
View File
@@ -59,6 +59,14 @@ export const SYSTEM_CONFIG_FIELDS: SystemConfigFieldMeta[] = [
requiresRestart: false,
description: '总开关。开启后连接 HQ「企微机器人」模块中已启用且配置完整的 Bot（每 Bot 同时仅 1 条长连接）',
},
{
key: 'WECOM_ALERT_ENABLED',
label: '启用企微运营告警 Webhook',
group: G.feature,
type: 'boolean',
requiresRestart: false,
description: '开启后向群机器人 Webhook 推送异常告警；Webhook URL 仅在服务器 .env 配置（WECOM_ALERT_WEBHOOK_URL）',
},
{ key: 'ALIYUN_SMS_SIGN_NAME', label: '短信签名', group: G.sms, type: 'string', requiresRestart: false },
{ key: 'ALIYUN_SMS_TEMPLATE_CODE', label: '默认短信模板', group: G.sms, type: 'string', requiresRestart: false },