feat(ops): WeCom webhook alerts for pay/redeem anomalies

Add outbound group-bot alerts, rate/amount rules, stuck-order cron, HQ test button, and health db/redis checks.

Co-authored-by: Cursor <cursoragent@cursor.com>

server/dukang-api/src/jobs/jobs.module.ts

@@ -5,6 +5,7 @@ import { TradeModule } from '../modules/trade/trade.module';
 import { SettlementModule } from '../modules/settlement/settlement.module';
 import { DeliveryProcessor } from './delivery.processor';
 import { SettlementScheduler } from './settlement.scheduler';
+import { MonitorScheduler } from './monitor.scheduler';
 import { DELIVERY_QUEUE } from './jobs.constants';
 
 @Module({
@@ -14,6 +15,6 @@ import { DELIVERY_QUEUE } from './jobs.constants';
     TradeModule,
     SettlementModule,
   ],
-  providers: [DeliveryProcessor, SettlementScheduler],
+  providers: [DeliveryProcessor, SettlementScheduler, MonitorScheduler],
 })
 export class JobsModule {}

server/dukang-api/src/jobs/monitor.scheduler.ts

@@ -0,0 +1,149 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { Cron } from '@nestjs/schedule';
+import { PrismaService } from '../common/prisma/prisma.module';
+import { RedisService } from '../common/redis/redis.service';
+import { AlertService } from '../common/alert/alert.service';
+import { ALERT_STUCK_ORDER } from '../common/alert/alert.constants';
+
+@Injectable()
+export class MonitorScheduler {
+  private readonly logger = new Logger(MonitorScheduler.name);
+
+  constructor(
+    private readonly prisma: PrismaService,
+    private readonly redis: RedisService,
+    private readonly alert: AlertService,
+  ) {}
+
+  /** 每分钟：MySQL / Redis 探活 */
+  @Cron('*/1 * * * *', { timeZone: 'Asia/Shanghai' })
+  async checkInfra() {
+    try {
+      await this.prisma.$queryRaw`SELECT 1`;
+    } catch (e) {
+      this.logger.error('MySQL ping failed', e instanceof Error ? e.stack : e);
+      this.alert.notify({
+        level: 'P0',
+        category: 'infra',
+        title: 'MySQL 不可用',
+        detail: e instanceof Error ? e.message : String(e),
+        dedupeKey: 'infra_mysql_down',
+        dedupeTtlSec: 120,
+      });
+    }
+
+    try {
+      const pong = await this.redis.client.ping();
+      if (pong !== 'PONG') {
+        throw new Error(`unexpected ping reply: ${pong}`);
+      }
+    } catch (e) {
+      this.logger.error('Redis ping failed', e instanceof Error ? e.stack : e);
+      this.alert.notify({
+        level: 'P0',
+        category: 'infra',
+        title: 'Redis 不可用',
+        detail: e instanceof Error ? e.message : String(e),
+        dedupeKey: 'infra_redis_down',
+        dedupeTtlSec: 120,
+      });
+    }
+  }
+
+  /** 每 5 分钟：卡住订单扫描 */
+  @Cron('*/5 * * * *', { timeZone: 'Asia/Shanghai' })
+  async scanStuckOrders() {
+    const now = new Date();
+    const hourBucket = Math.floor(now.getTime() / 3_600_000);
+    const sample = ALERT_STUCK_ORDER.sampleLimit;
+
+    try {
+      const expiredUnpaid = await this.prisma.order.findMany({
+        where: {
+          status: 'PENDING_PAY',
+          payExpireAt: { lt: now },
+        },
+        select: { orderNo: true },
+        take: sample,
+        orderBy: { payExpireAt: 'asc' },
+      });
+      const expiredUnpaidTotal = await this.prisma.order.count({
+        where: { status: 'PENDING_PAY', payExpireAt: { lt: now } },
+      });
+      if (expiredUnpaidTotal > 0) {
+        this.alert.notify({
+          level: 'P1',
+          category: 'order',
+          title: '超时未关待付款订单',
+          detail: `共 ${expiredUnpaidTotal} 单\n样例：${expiredUnpaid.map((o) => o.orderNo).join(', ')}`,
+          dedupeKey: `stuck_pending_pay|${hourBucket}`,
+        });
+      }
+
+      const shipBefore = new Date(
+        now.getTime() - ALERT_STUCK_ORDER.pendingShipHours * 3600_000,
+      );
+      const stuckShip = await this.prisma.order.findMany({
+        where: {
+          status: 'PENDING_SHIP',
+          paidAt: { lt: shipBefore },
+        },
+        select: { orderNo: true },
+        take: sample,
+        orderBy: { paidAt: 'asc' },
+      });
+      const stuckShipTotal = await this.prisma.order.count({
+        where: { status: 'PENDING_SHIP', paidAt: { lt: shipBefore } },
+      });
+      if (stuckShipTotal > 0) {
+        this.alert.notify({
+          level: 'P1',
+          category: 'order',
+          title: `待发货超过 ${ALERT_STUCK_ORDER.pendingShipHours}h`,
+          detail: `共 ${stuckShipTotal} 单\n样例：${stuckShip.map((o) => o.orderNo).join(', ')}`,
+          dedupeKey: `stuck_pending_ship|${hourBucket}`,
+        });
+      }
+
+      const deliveryBefore = new Date(
+        now.getTime() - ALERT_STUCK_ORDER.inDeliveryHours * 3600_000,
+      );
+      const stuckDelivery = await this.prisma.order.findMany({
+        where: {
+          status: { in: ['OUT_WAREHOUSE', 'PENDING_RECEIVE'] },
+          updatedAt: { lt: deliveryBefore },
+        },
+        select: { orderNo: true, status: true },
+        take: sample,
+        orderBy: { updatedAt: 'asc' },
+      });
+      const stuckDeliveryTotal = await this.prisma.order.count({
+        where: {
+          status: { in: ['OUT_WAREHOUSE', 'PENDING_RECEIVE'] },
+          updatedAt: { lt: deliveryBefore },
+        },
+      });
+      if (stuckDeliveryTotal > 0) {
+        this.alert.notify({
+          level: 'P1',
+          category: 'order',
+          title: `配送中超过 ${ALERT_STUCK_ORDER.inDeliveryHours}h`,
+          detail: `共 ${stuckDeliveryTotal} 单\n样例：${stuckDelivery
+            .map((o) => `${o.orderNo}(${o.status})`)
+            .join(', ')}`,
+          dedupeKey: `stuck_in_delivery|${hourBucket}`,
+        });
+      }
+    } catch (e) {
+      this.logger.error('stuck order scan failed', e instanceof Error ? e.stack : e);
+      this.alert.notify({
+        level: 'P0',
+        category: 'job',
+        title: '卡住订单扫描失败',
+        detail: e instanceof Error ? e.message : String(e),
+        dedupeKey: 'stuck_scan_fail',
+        dedupeTtlSec: 300,
+      });
+    }
+  }
+}

server/dukang-api/src/jobs/settlement.scheduler.ts

@@ -1,6 +1,7 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { Cron } from '@nestjs/schedule';
 import { SettlementService } from '../modules/settlement/settlement.service';
+import { AlertService } from '../common/alert/alert.service';
 
 /**
  * 财务对账单定时任务（Asia/Shanghai）
@@ -11,7 +12,10 @@ import { SettlementService } from '../modules/settlement/settlement.service';
 export class SettlementScheduler {
   private readonly logger = new Logger(SettlementScheduler.name);
 
-  constructor(private readonly settlementService: SettlementService) {}
+  constructor(
+    private readonly settlementService: SettlementService,
+    private readonly alert: AlertService,
+  ) {}
 
   @Cron('0 8 * * *', { timeZone: 'Asia/Shanghai' })
   async handleDailyBills() {
@@ -21,12 +25,26 @@ export class SettlementScheduler {
       this.logger.log(`Winery bill: ${JSON.stringify(winery)}`);
     } catch (e) {
       this.logger.error('Winery bill job failed', e instanceof Error ? e.stack : e);
+      this.alert.notify({
+        level: 'P0',
+        category: 'job',
+        title: '酒厂日账单任务失败',
+        detail: e instanceof Error ? e.message : String(e),
+        dedupeKey: `job_winery_bill|${new Date().toISOString().slice(0, 10)}`,
+      });
     }
     try {
       const store = await this.settlementService.generateStoreBillsForDay();
       this.logger.log(`Store bills: ${JSON.stringify(store)}`);
     } catch (e) {
       this.logger.error('Store bill job failed', e instanceof Error ? e.stack : e);
+      this.alert.notify({
+        level: 'P0',
+        category: 'job',
+        title: '门店日账单任务失败',
+        detail: e instanceof Error ? e.message : String(e),
+        dedupeKey: `job_store_bill|${new Date().toISOString().slice(0, 10)}`,
+      });
     }
   }
 
@@ -38,16 +56,48 @@ export class SettlementScheduler {
       this.logger.log(
         `Partner bills: total=${result.total} success=${result.success} failed=${result.failed}`,
       );
+      if (result.failed > 0) {
+        this.alert.notify({
+          level: 'P0',
+          category: 'job',
+          title: '合伙人月账单部分失败',
+          detail: `total=${result.total} success=${result.success} failed=${result.failed}`,
+          dedupeKey: `job_partner_bill|${new Date().toISOString().slice(0, 7)}`,
+        });
+      }
     } catch (e) {
       this.logger.error('Partner bill job failed', e instanceof Error ? e.stack : e);
+      this.alert.notify({
+        level: 'P0',
+        category: 'job',
+        title: '合伙人月账单任务失败',
+        detail: e instanceof Error ? e.message : String(e),
+        dedupeKey: `job_partner_bill|${new Date().toISOString().slice(0, 7)}`,
+      });
     }
     try {
       const logistics = await this.settlementService.generatePreviousMonthLogisticsBills();
       this.logger.log(
         `Logistics bills: total=${logistics.total} success=${logistics.success} failed=${logistics.failed}`,
       );
+      if (logistics.failed > 0) {
+        this.alert.notify({
+          level: 'P0',
+          category: 'job',
+          title: '物流月对账部分失败',
+          detail: `total=${logistics.total} success=${logistics.success} failed=${logistics.failed}`,
+          dedupeKey: `job_logistics_bill|${new Date().toISOString().slice(0, 7)}`,
+        });
+      }
     } catch (e) {
       this.logger.error('Logistics bill job failed', e instanceof Error ? e.stack : e);
+      this.alert.notify({
+        level: 'P0',
+        category: 'job',
+        title: '物流月对账任务失败',
+        detail: e instanceof Error ? e.message : String(e),
+        dedupeKey: `job_logistics_bill|${new Date().toISOString().slice(0, 7)}`,
+      });
     }
   }
 }