feat: multi-module iteration

2026-08-04 21:38:49 +08:00
parent 9d96c73246
commit 71f508e02b
1366 changed files with 202004 additions and 0 deletions
@@ -0,0 +1,30 @@
+/** 告警级别 */
+export type AlertLevel = 'P0' | 'P1' | 'P2';
+
+/** 支付 / 核销行为异常阈值（金额单位：元） */
+export const ALERT_THRESHOLDS = {
+  payAmountMinYuan: 1,
+  payAmountMaxYuan: 5000,
+  payAttemptPerMinute: 3,
+  payFailPerMinute: 5,
+  redeemAmountMinYuan: 1,
+  redeemAmountMaxYuan: 1000,
+  redeemAttemptPerMinute: 3,
+  redeemFailPerMinute: 5,
+} as const;
+
+/** Redis 分钟桶计数 TTL（秒） */
+export const ALERT_RATE_BUCKET_TTL_SEC = 120;
+
+/** 默认去重 TTL */
+export const ALERT_DEDUPE_TTL_SEC = 600;
+
+/** 频次类告警去重 TTL（同一分钟桶只推一次） */
+export const ALERT_RATE_DEDUPE_TTL_SEC = 90;
+
+/** 卡住订单扫描 */
+export const ALERT_STUCK_ORDER = {
+  pendingShipHours: 24,
+  inDeliveryHours: 48,
+  sampleLimit: 5,
+} as const;
@@ -0,0 +1,16 @@
+import { Global, Module } from '@nestjs/common';
+import { RedisModule } from '../redis/redis.module';
+import { AlertService } from './alert.service';
+import { PayRedeemAnomalyService } from './pay-redeem-anomaly.service';
+import { WecomMessagePushService } from '../../integrations/wecom/wecom-message-push.service';
+
+/**
+ * 运营告警（企微 Webhook 多实例）。Global 以便 Filter / 各业务 Module 注入。
+ */
+@Global()
+@Module({
+  imports: [RedisModule],
+  providers: [WecomMessagePushService, AlertService, PayRedeemAnomalyService],
+  exports: [WecomMessagePushService, AlertService, PayRedeemAnomalyService],
+})
+export class AlertModule {}
@@ -0,0 +1,88 @@
+import { Injectable, Logger } from '@nestjs/common';
+import type { WecomPushCondition } from '@dukang/shared-types';
+import { RedisService } from '../redis/redis.service';
+import { WecomMessagePushService } from '../../integrations/wecom/wecom-message-push.service';
+import { ALERT_DEDUPE_TTL_SEC, type AlertLevel } from './alert.constants';
+
+export type AlertNotifyInput = {
+  level: AlertLevel;
+  category: string;
+  title: string;
+  detail: string;
+  dedupeKey: string;
+  dedupeTtlSec?: number;
+  /** 覆盖默认事件 key 映射 */
+  eventKeys?: WecomPushCondition[];
+};
+
+@Injectable()
+export class AlertService {
+  private readonly logger = new Logger(AlertService.name);
+
+  constructor(
+    private readonly redis: RedisService,
+    private readonly wecomPush: WecomMessagePushService,
+  ) {}
+
+  /** 异步告警，不阻塞调用方 */
+  notify(input: AlertNotifyInput): void {
+    void this.notifyAsync(input).catch((e) => {
+      this.logger.warn(`alert notify failed: ${e instanceof Error ? e.message : String(e)}`);
+    });
+  }
+
+  async notifyAsync(input: AlertNotifyInput): Promise<boolean> {
+    const eventKeys = resolveAlertEventKeys(input);
+    const hasAny = await Promise.all(eventKeys.map((k) => this.wecomPush.hasEnabledPushes(k)));
+    if (!hasAny.some(Boolean)) return false;
+
+    const ttl = input.dedupeTtlSec ?? ALERT_DEDUPE_TTL_SEC;
+    const dedupeRedisKey = `alert:dedupe:${input.dedupeKey}`;
+    try {
+      const ok = await this.redis.client.set(dedupeRedisKey, '1', 'EX', ttl, 'NX');
+      if (ok !== 'OK') return false;
+    } catch (e) {
+      this.logger.warn(
+        `alert dedupe redis error, send anyway: ${e instanceof Error ? e.message : String(e)}`,
+      );
+    }
+
+    return this.sendMarkdownNow(input, eventKeys);
+  }
+
+  private async sendMarkdownNow(
+    input: AlertNotifyInput,
+    eventKeys: WecomPushCondition[],
+  ): Promise<boolean> {
+    const envLabel = (process.env.WECOM_ALERT_ENV_LABEL || process.env.NODE_ENV || 'local').trim();
+    const now = new Date().toLocaleString('zh-CN', { timeZone: 'Asia/Shanghai' });
+    const content = [
+      `## [${input.level}] ${escapeMd(input.title)}`,
+      `> 环境：<font color="comment">${escapeMd(envLabel)}</font>`,
+      `> 类别：<font color="comment">${escapeMd(input.category)}</font>`,
+      `> 时间：${escapeMd(now)}`,
+      '',
+      escapeMd(input.detail).slice(0, 3500),
+    ].join('\n');
+
+    let sent = 0;
+    for (const eventKey of eventKeys) {
+      sent += await this.wecomPush.dispatchMarkdown(eventKey, content, { applyMention: false });
+    }
+    return sent > 0;
+  }
+}
+
+function resolveAlertEventKeys(input: AlertNotifyInput): WecomPushCondition[] {
+  if (input.eventKeys?.length) return [...new Set(input.eventKeys)];
+  if (input.category === 'pay') return ['alert.pay'];
+  if (input.category === 'redeem') return ['alert.redeem'];
+  if (input.category === 'settlement') return ['alert.settlement'];
+  if (input.category === 'ops') return ['alert.ops'];
+  return ['alert.system'];
+}
+
+/** 轻量转义，保留企微 markdown 的 <font> 标签可用 */
+function escapeMd(s: string): string {
+  return s.replace(/([\\`*_[\]])/g, '\\$1');
+}
@@ -0,0 +1,213 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { RedisService } from '../redis/redis.service';
+import { AlertService } from './alert.service';
+import {
+  ALERT_RATE_BUCKET_TTL_SEC,
+  ALERT_RATE_DEDUPE_TTL_SEC,
+  ALERT_THRESHOLDS,
+} from './alert.constants';
+
+export type PayAnomalyMeta = {
+  orderNo?: string;
+  userId?: string | number | bigint;
+  reason?: string;
+};
+
+export type RedeemAnomalyMeta = {
+  storeId?: string | number | bigint;
+  userId?: string | number | bigint;
+  recordId?: string | number | bigint;
+  reason?: string;
+};
+
+@Injectable()
+export class PayRedeemAnomalyService {
+  private readonly logger = new Logger(PayRedeemAnomalyService.name);
+
+  constructor(
+    private readonly redis: RedisService,
+    private readonly alert: AlertService,
+  ) {}
+
+  onPayAttempt(amountYuan: number, meta: PayAnomalyMeta = {}): void {
+    this.checkPayAmount(amountYuan, meta);
+    void this.bumpAndMaybeAlert(
+      'pay:attempt',
+      ALERT_THRESHOLDS.payAttemptPerMinute,
+      'P0',
+      'pay',
+      '支付频率异常',
+      `1 分钟内支付尝试超过 ${ALERT_THRESHOLDS.payAttemptPerMinute} 次`,
+      meta,
+    );
+  }
+
+  onPaySuccess(amountYuan: number, meta: PayAnomalyMeta = {}): void {
+    this.checkPayAmount(amountYuan, meta);
+  }
+
+  onPayFail(reason: string, meta: PayAnomalyMeta = {}): void {
+    void this.bumpAndMaybeAlert(
+      'pay:fail',
+      ALERT_THRESHOLDS.payFailPerMinute,
+      'P0',
+      'pay',
+      '支付失败频率异常',
+      `1 分钟内支付失败超过 ${ALERT_THRESHOLDS.payFailPerMinute} 次；最近原因：${reason}`,
+      { ...meta, reason },
+    );
+  }
+
+  onRedeemAttempt(amountYuan: number, meta: RedeemAnomalyMeta = {}): void {
+    this.checkRedeemAmount(amountYuan, meta);
+    void this.bumpAndMaybeAlert(
+      'redeem:attempt',
+      ALERT_THRESHOLDS.redeemAttemptPerMinute,
+      'P0',
+      'redeem',
+      '核销频率异常',
+      `1 分钟内核销尝试超过 ${ALERT_THRESHOLDS.redeemAttemptPerMinute} 次`,
+      meta,
+    );
+  }
+
+  onRedeemFail(reason: string, meta: RedeemAnomalyMeta = {}): void {
+    void this.bumpAndMaybeAlert(
+      'redeem:fail',
+      ALERT_THRESHOLDS.redeemFailPerMinute,
+      'P0',
+      'redeem',
+      '核销失败频率异常',
+      `1 分钟内核销失败超过 ${ALERT_THRESHOLDS.redeemFailPerMinute} 次；最近原因：${reason}`,
+      { ...meta, reason },
+    );
+  }
+
+  /** 弱网阈值 / 补核销待办等业务信号 */
+  notifyRedeemOps(title: string, detail: string, dedupeKey: string): void {
+    this.alert.notify({
+      level: 'P1',
+      category: 'redeem',
+      title,
+      detail,
+      dedupeKey,
+    });
+  }
+
+  private checkPayAmount(amountYuan: number, meta: PayAnomalyMeta): void {
+    if (!Number.isFinite(amountYuan)) return;
+    if (amountYuan < ALERT_THRESHOLDS.payAmountMinYuan) {
+      this.alert.notify({
+        level: 'P1',
+        category: 'pay',
+        title: '支付金额过低',
+        detail: formatPayDetail(
+          `支付金额 ${amountYuan} 元 < ${ALERT_THRESHOLDS.payAmountMinYuan} 元`,
+          meta,
+        ),
+        dedupeKey: `pay_amount_low|${meta.orderNo ?? amountYuan}`,
+      });
+    } else if (amountYuan > ALERT_THRESHOLDS.payAmountMaxYuan) {
+      this.alert.notify({
+        level: 'P1',
+        category: 'pay',
+        title: '支付金额过高',
+        detail: formatPayDetail(
+          `支付金额 ${amountYuan} 元 > ${ALERT_THRESHOLDS.payAmountMaxYuan} 元`,
+          meta,
+        ),
+        dedupeKey: `pay_amount_high|${meta.orderNo ?? amountYuan}`,
+      });
+    }
+  }
+
+  private checkRedeemAmount(amountYuan: number, meta: RedeemAnomalyMeta): void {
+    if (!Number.isFinite(amountYuan)) return;
+    if (amountYuan < ALERT_THRESHOLDS.redeemAmountMinYuan) {
+      this.alert.notify({
+        level: 'P1',
+        category: 'redeem',
+        title: '核销金额过低',
+        detail: formatRedeemDetail(
+          `核销金额 ${amountYuan} 元 < ${ALERT_THRESHOLDS.redeemAmountMinYuan} 元`,
+          meta,
+        ),
+        dedupeKey: `redeem_amount_low|${meta.recordId ?? meta.storeId ?? amountYuan}`,
+      });
+    } else if (amountYuan > ALERT_THRESHOLDS.redeemAmountMaxYuan) {
+      this.alert.notify({
+        level: 'P1',
+        category: 'redeem',
+        title: '核销金额过高',
+        detail: formatRedeemDetail(
+          `核销金额 ${amountYuan} 元 > ${ALERT_THRESHOLDS.redeemAmountMaxYuan} 元`,
+          meta,
+        ),
+        dedupeKey: `redeem_amount_high|${meta.recordId ?? meta.storeId ?? amountYuan}`,
+      });
+    }
+  }
+
+  private async bumpAndMaybeAlert(
+    counterKind: string,
+    threshold: number,
+    level: 'P0' | 'P1' | 'P2',
+    category: string,
+    title: string,
+    baseDetail: string,
+    meta: PayAnomalyMeta & RedeemAnomalyMeta,
+  ): Promise<void> {
+    try {
+      const bucket = Math.floor(Date.now() / 60_000);
+      const key = `alert:cnt:${counterKind}:${bucket}`;
+      const count = await this.redis.incr(key, ALERT_RATE_BUCKET_TTL_SEC);
+      if (count <= threshold) return;
+
+      const extra = [
+        `当前分钟计数：${count}`,
+        meta.orderNo ? `订单：${meta.orderNo}` : null,
+        meta.storeId != null ? `门店：${String(meta.storeId)}` : null,
+        meta.userId != null ? `用户：${String(meta.userId)}` : null,
+        meta.reason ? `原因：${meta.reason}` : null,
+      ]
+        .filter(Boolean)
+        .join('\n');
+
+      this.alert.notify({
+        level,
+        category,
+        title,
+        detail: `${baseDetail}\n${extra}`,
+        dedupeKey: `${counterKind}|${bucket}`,
+        dedupeTtlSec: ALERT_RATE_DEDUPE_TTL_SEC,
+      });
+    } catch (e) {
+      this.logger.warn(
+        `anomaly counter failed: ${e instanceof Error ? e.message : String(e)}`,
+      );
+    }
+  }
+}
+
+function formatPayDetail(line: string, meta: PayAnomalyMeta): string {
+  return [
+    line,
+    meta.orderNo ? `订单：${meta.orderNo}` : null,
+    meta.userId != null ? `用户：${String(meta.userId)}` : null,
+    meta.reason ? `原因：${meta.reason}` : null,
+  ]
+    .filter(Boolean)
+    .join('\n');
+}
+
+function formatRedeemDetail(line: string, meta: RedeemAnomalyMeta): string {
+  return [
+    line,
+    meta.storeId != null ? `门店：${String(meta.storeId)}` : null,
+    meta.userId != null ? `用户：${String(meta.userId)}` : null,
+    meta.recordId != null ? `记录：${String(meta.recordId)}` : null,
+    meta.reason ? `原因：${meta.reason}` : null,
+  ]
+    .filter(Boolean)
+    .join('\n');
+}