feat: DB pool saturation as autoscale signal (#712)

docs/autoscale-db-pool-signal.md

@@ -0,0 +1,160 @@
+# DB Pool Saturation as Autoscaling Signal
+
+**Issue:** [#712](https://github.com/RevoraOrg/Revora-Backend/issues/712) — Autoscaling triggers: DB pool saturation as horizontal-scale signal  
+**Status:** Implemented
+
+---
+
+## Overview
+
+CPU-based HPA misses the real bottleneck when the Node event loop is fine but
+the PostgreSQL connection pool is saturated (clients queued on `pool.connect()`).
+This feature exports two gauges on every `/metrics` scrape:
+
+| Metric | Type | Meaning |
+|--------|------|---------|
+| `db.pool.waiters` | gauge | Clients waiting for a free pool connection |
+| `db.pool.utilization` | gauge | `totalCount / maxConnections` in `[0, 1]` |
+
+Both series are **always defined**, including when the pool is idle
+(`waiters=0`, `utilization=0`), so the autoscaler never sees a missing metric.
+
+Scrapes are guarded by metrics auth (`METRICS_TOKEN` / internal token middleware).
+
+---
+
+## Architecture
+
+```
+GET /metrics  ──(auth)──►  createPrometheusHandler(metrics, pool)
+                              │
+                              ├─ metrics.updatePoolSaturationMetrics(pool)
+                              │     db.pool.waiters
+                              │     db.pool.utilization
+                              └─ metrics.exportPrometheus()
+```
+
+---
+
+## Configuration
+
+| Variable | Role |
+|----------|------|
+| `METRICS_TOKEN` | Required in production for scrape auth |
+| Pool `max` (pg `Pool` option) | Denominator for utilization (default 10 in `src/db/pool.ts`) |
+
+---
+
+## HPA guidance
+
+Target **pool waiters** (or utilization) rather than (or in addition to) CPU.
+
+### Suggested thresholds
+
+| Signal | Warning | Scale-out |
+|--------|---------|-----------|
+| `db.pool.waiters` | `> 0` for 2m | `> 2` for 1m |
+| `db.pool.utilization` | `> 0.7` for 5m | `> 0.85` for 2m |
+
+### Example Kubernetes HPA (custom metrics / Prometheus adapter)
+
+```yaml
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: revora-api
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: revora-api
+  minReplicas: 2
+  maxReplicas: 20
+  metrics:
+    - type: Pods
+      pods:
+        metric:
+          name: db_pool_waiters
+        target:
+          type: AverageValue
+          averageValue: "1"
+    - type: Pods
+      pods:
+        metric:
+          name: db_pool_utilization
+        target:
+          type: AverageValue
+          # 0.75 → scale before hard saturation
+          averageValue: "750m"
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 60
+    scaleDown:
+      stabilizationWindowSeconds: 300
+```
+
+> Note: utilization is a ratio in `[0,1]`. Some adapters prefer milliratio
+> (`750m` = 0.75). Confirm your metrics-adapter scaling.
+
+### Example Prometheus alert rules
+
+```yaml
+groups:
+  - name: revora-db-pool
+    rules:
+      - alert: DbPoolWaiters
+        expr: db_pool_waiters > 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "DB pool has waiting clients"
+          description: "{{ $value }} clients waiting for a connection"
+
+      - alert: DbPoolSaturation
+        expr: db_pool_utilization > 0.85
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "DB pool utilization above 85%"
+```
+
+---
+
+## Security assumptions
+
+1. `/metrics` is behind scrape auth — pool gauges are not publicly readable.
+2. Metric labels contain no PII (no connection strings, user IDs, or SQL).
+3. Utilization is a closed ratio; waiters is a non-negative integer.
+
+---
+
+## Edge cases
+
+| Scenario | Behaviour |
+|----------|-----------|
+| Pool idle | Both gauges published as `0` |
+| Pool not passed to handler | Both gauges published as `0` |
+| `totalCount > max` | Utilization clamped to `1` |
+| `max == 0` | Utilization = `0` |
+
+---
+
+## Testing
+
+```bash
+npx jest src/lib/metrics.test.ts --testNamePattern="updatePoolSaturationMetrics" --forceExit
+```
+
+---
+
+## Related files
+
+| File | Role |
+|------|------|
+| `src/lib/metrics.ts` | `updatePoolSaturationMetrics()` |
+| `src/middleware/metricsMiddleware.ts` | Refresh gauges on scrape |
+| `src/db/pool.ts` | Primary pool source |
+| `src/app.ts` | Wires pool into `/metrics` |
+| `docs/autoscale-db-pool-signal.md` | This runbook |

src/app.ts

@@ -228,8 +228,9 @@ export function createApp() {
   app.use(createChangePasswordRouter({ requireAuth, db: pool }));
   app.use('/api/v1/health', createHealthRouter(pool, dbHealth, metrics));
 
-  // Metrics endpoint (Prometheus format) - secured with internal token
-  app.get('/metrics', createMetricsAuthMiddleware(), createPrometheusHandler(metrics));
+  // Metrics endpoint (Prometheus / OpenMetrics-compatible) - secured with internal token.
+  // Passes the primary pool so db.pool.waiters / db.pool.utilization are refreshed on scrape.
+  app.get('/metrics', createMetricsAuthMiddleware(), createPrometheusHandler(metrics, pool));
 
   // Reconciliation metrics endpoint (OpenMetrics format) - same auth guard
   app.get(

src/db/pool.ts

@@ -25,6 +25,10 @@
  *   - Connection strings are consumed by pg and never logged.
  *   - Metric labels contain no PII.
  *
+ * Autoscaling (#712): `/metrics` scrapes refresh `db.pool.waiters` and
+ * `db.pool.utilization` from this primary pool (see
+ * `MetricsCollector.updatePoolSaturationMetrics`).
+ *
  * @module db/pool
  */
 

src/lib/metrics.test.ts

@@ -697,4 +697,85 @@ describe('MetricsCollector', () => {
       expect(output).toContain('beta');
     });
   });
+
+  describe('updatePoolSaturationMetrics (issue #712)', () => {
+    let metrics: MetricsCollector;
+
+    beforeEach(() => {
+      metrics = new MetricsCollector({ enabled: true, enablePIIDetection: false });
+    });
+
+    afterEach(() => {
+      metrics.reset();
+    });
+
+    it('emits db.pool.waiters and db.pool.utilization when pool is busy', async () => {
+      const mockPool = {
+        totalCount: 8,
+        waitingCount: 3,
+        options: { max: 10 },
+      } as unknown as Pool;
+
+      metrics.updatePoolSaturationMetrics(mockPool);
+
+      const snapshot = await metrics.getSnapshot();
+      const waiters = snapshot.custom.find((m) => m.name === 'db_pool_waiters');
+      const util = snapshot.custom.find((m) => m.name === 'db_pool_utilization');
+      expect(waiters?.value).toBe(3);
+      expect(util?.value).toBeCloseTo(0.8);
+    });
+
+    it('keeps metrics defined when the pool is idle (waiters=0, utilization=0)', async () => {
+      const idlePool = {
+        totalCount: 0,
+        waitingCount: 0,
+        options: { max: 10 },
+      } as unknown as Pool;
+
+      metrics.updatePoolSaturationMetrics(idlePool);
+
+      const snapshot = await metrics.getSnapshot();
+      const waiters = snapshot.custom.find((m) => m.name === 'db_pool_waiters');
+      const util = snapshot.custom.find((m) => m.name === 'db_pool_utilization');
+      expect(waiters).toBeDefined();
+      expect(waiters?.value).toBe(0);
+      expect(util).toBeDefined();
+      expect(util?.value).toBe(0);
+    });
+
+    it('keeps metrics defined when no pool is provided', async () => {
+      metrics.updatePoolSaturationMetrics(null);
+
+      const snapshot = await metrics.getSnapshot();
+      expect(snapshot.custom.find((m) => m.name === 'db_pool_waiters')?.value).toBe(0);
+      expect(snapshot.custom.find((m) => m.name === 'db_pool_utilization')?.value).toBe(0);
+    });
+
+    it('clamps utilization to 1 when totalCount exceeds max', async () => {
+      metrics.updatePoolSaturationMetrics({
+        totalCount: 15,
+        waitingCount: 5,
+        options: { max: 10 },
+      } as unknown as Pool);
+
+      const snapshot = await metrics.getSnapshot();
+      expect(snapshot.custom.find((m) => m.name === 'db_pool_utilization')?.value).toBe(1);
+    });
+
+    it('exports pool gauges in OpenMetrics / Prometheus text', () => {
+      metrics.updatePoolSaturationMetrics({
+        totalCount: 5,
+        waitingCount: 2,
+        options: { max: 10 },
+      } as unknown as Pool);
+
+      const prom = metrics.exportPrometheus();
+      expect(prom).toContain('db_pool_waiters');
+      expect(prom).toContain('db_pool_utilization');
+
+      const om = metrics.exportOpenMetrics();
+      expect(om).toContain('db_pool_waiters');
+      expect(om).toContain('db_pool_utilization');
+    });
+  });
 });

src/lib/metrics.ts

@@ -454,6 +454,42 @@ export class MetricsCollector {
     };
   }
 
+  /**
+   * @notice Publish DB pool saturation gauges for horizontal autoscaling (#712).
+   *
+   * @dev Always writes both gauges — even when the pool is idle (waiters=0,
+   *      utilization=0) — so the autoscaler and alert rules never see a missing
+   *      series.  Call this on every metrics scrape (see
+   *      `createPrometheusHandler`).
+   *
+   * Metrics:
+   *  - `db.pool.waiters`     — clients waiting for a connection
+   *  - `db.pool.utilization` — totalCount / maxConnections in [0, 1]
+   *
+   * Security: labels contain no PII; values are pool counters only.
+   */
+  updatePoolSaturationMetrics(
+    pool?: Pick<Pool, 'totalCount' | 'waitingCount'> & { options?: { max?: number } } | null,
+  ): void {
+    const waiters = pool?.waitingCount ?? 0;
+    const total = pool?.totalCount ?? 0;
+    const max = pool?.options?.max ?? 0;
+    const utilization = max > 0 ? Math.min(1, Math.max(0, total / max)) : 0;
+
+    this.setGauge(
+      'db.pool.waiters',
+      waiters,
+      undefined,
+      'Number of clients waiting for a free DB pool connection',
+    );
+    this.setGauge(
+      'db.pool.utilization',
+      utilization,
+      undefined,
+      'DB pool utilization ratio (totalCount / maxConnections) in [0, 1]',
+    );
+  }
+
   /**
    * Collect application-level metrics
    * @returns Application metrics snapshot

src/middleware/metricsMiddleware.ts

@@ -192,20 +192,31 @@ export function createMetricsHandler(metrics: MetricsCollector, pool?: any) {
 
 /**
  * Create Prometheus-format metrics endpoint handler
- * 
- * Exposes metrics in Prometheus text format for scraping.
- * 
+ *
+ * Exposes metrics in Prometheus / OpenMetrics-compatible text format for
+ * scraping.  When a DB pool is supplied, refreshes `db.pool.waiters` and
+ * `db.pool.utilization` on every scrape so the autoscaler always sees a
+ * defined series (issue #712).
+ *
+ * MUST be mounted behind metrics scrape auth (`createMetricsAuthMiddleware`).
+ *
  * Usage:
  * ```typescript
- * app.get('/metrics/prometheus', createPrometheusHandler(globalMetrics));
+ * app.get('/metrics', createMetricsAuthMiddleware(), createPrometheusHandler(metrics, pool));
  * ```
- * 
+ *
  * @param metrics Metrics collector instance
+ * @param pool Optional pg Pool used to publish saturation gauges
  * @returns Express route handler
  */
-export function createPrometheusHandler(metrics: MetricsCollector) {
+export function createPrometheusHandler(
+  metrics: MetricsCollector,
+  pool?: Parameters<MetricsCollector['updatePoolSaturationMetrics']>[0],
+) {
   return (_req: Request, res: Response, next: NextFunction): void => {
     try {
+      // Always refresh pool gauges — even when idle — before export.
+      metrics.updatePoolSaturationMetrics(pool ?? null);
       const output = metrics.exportPrometheus();
       res.set('Content-Type', 'text/plain; version=0.0.4');
       res.status(200).send(output);