feat: Redis opportunity cache, CW API retry/logging, adaptive TTLs

- Add Redis-backed opportunity cache with background refresh (30s interval) - Fix concurrency bug: use lazy thunks instead of eager promises for batching - Add withCwRetry utility with exponential backoff for transient CW errors - Add adaptive TTL algorithms (primary, sub-resource, products) based on opportunity activity - Add include query param on GET /sales/opportunities/:id (notes,contacts,products) - Add opt-in CW API logger (LOG_CW_API env var) with timestamped files in cw-api-logs/ - Add debug-scripts/analyze-cw-calls.py for API call analysis - Add computeSubResourceCacheTTL and computeProductsCacheTTL algorithms with tests - Increase CW API timeout from 15s to 30s - Unblock cache refresh from startup chain (remove await) - Prioritize recently updated opportunities in refresh cycle - Add CACHING.md documentation - Update API_ROUTES.md with caching details and include param - Update copilot instructions to require CACHING.md sync - Add dev:log script for CW API call logging during development
2026-03-02 23:23:24 -06:00
parent fe71248e88
commit 6d935e7180
33 changed files with 2634 additions and 176 deletions
@@ -18,6 +18,20 @@ import {
 import { resolveMember } from "../modules/cw-utils/members/memberCache";
 import { ForecastProductController } from "./ForecastProductController";
 import GenericError from "../Errors/GenericError";
+import { computeSubResourceCacheTTL } from "../modules/algorithms/computeSubResourceCacheTTL";
+import { computeProductsCacheTTL } from "../modules/algorithms/computeProductsCacheTTL";
+import {
+  getCachedNotes,
+  getCachedContacts,
+  getCachedProducts,
+  getCachedSite,
+  fetchAndCacheNotes,
+  fetchAndCacheContacts,
+  fetchAndCacheProducts,
+  fetchAndCacheSite,
+  invalidateNotesCache,
+  invalidateProductsCache,
+} from "../modules/cache/opportunityCache";

 /**
 * Opportunity Controller
@@ -91,6 +105,27 @@ export class OpportunityController {
  private _customFields: CWCustomField[] | null = null;
  private _activities: ActivityController[] | null = null;

+  /** Compute the sub-resource cache TTL from this opportunity's fields. */
+  private _subResourceTTL(): number | null {
+    return computeSubResourceCacheTTL({
+      closedFlag: this.closedFlag,
+      closedDate: this.closedDate,
+      expectedCloseDate: this.expectedCloseDate,
+      lastUpdated: this.cwLastUpdated,
+    });
+  }
+
+  /** Compute the products-specific cache TTL from this opportunity's fields. */
+  private _productsTTL(): number | null {
+    return computeProductsCacheTTL({
+      closedFlag: this.closedFlag,
+      closedDate: this.closedDate,
+      expectedCloseDate: this.expectedCloseDate,
+      lastUpdated: this.cwLastUpdated,
+      statusCwId: this.statusCwId,
+    });
+  }
+
  constructor(
    data: Opportunity & { company?: Company | null },
    opts?: {
@@ -288,6 +323,7 @@ export class OpportunityController {
   *
   * Fetches the full site details (address, phone, flags) from ConnectWise
   * for the site associated with this opportunity.
+   * Checks the Redis cache first (30-min TTL); on miss, calls CW and caches.
   * Requires both companyCwId and siteCwId to be set.
   *
   * @returns Serialized site object or null
@@ -296,7 +332,17 @@ export class OpportunityController {
    if (this._siteData) return this._siteData;
    if (!this.companyCwId || !this.siteCwId) return null;

-    const cwSite = await fetchCompanySite(this.companyCwId, this.siteCwId);
+    // Try cache first
+    const cached = await getCachedSite(this.companyCwId, this.siteCwId);
+    if (cached) {
+      this._siteData = serializeCwSite(cached);
+      return this._siteData;
+    }
+
+    // Cache miss — fetch from CW and cache
+    const cwSite = await fetchAndCacheSite(this.companyCwId, this.siteCwId);
+    if (!cwSite) return null;
+
    this._siteData = serializeCwSite(cwSite);
    return this._siteData;
  }
@@ -304,13 +350,37 @@ export class OpportunityController {
  /**
   * Fetch Contacts
   *
-   * Fetches contacts associated with this opportunity from ConnectWise
-   * and returns a serialized array.
+   * Fetches contacts associated with this opportunity. Checks the Redis
+   * cache first; on miss, calls ConnectWise and caches the raw response.
+   *
+   * @param opts.fresh - Bypass cache and fetch directly from CW.
   */
-  public async fetchContacts() {
-    const contacts = await opportunityCw.fetchContacts(this.cwOpportunityId);
+  public async fetchContacts(opts?: { fresh?: boolean }) {
+    const ttl = this._subResourceTTL();

-    return contacts.map((ct) => ({
+    // Try cache first (unless forced fresh)
+    if (!opts?.fresh && ttl !== null) {
+      const cached = await getCachedContacts(this.cwOpportunityId);
+      if (cached) return this._serializeContacts(cached);
+    }
+
+    // Fetch from CW (fetchAndCache* handles 404 internally)
+    try {
+      const contacts =
+        ttl !== null
+          ? await fetchAndCacheContacts(this.cwOpportunityId, ttl)
+          : await opportunityCw.fetchContacts(this.cwOpportunityId);
+
+      return this._serializeContacts(contacts);
+    } catch (err: any) {
+      if (err?.isAxiosError && err?.response?.status === 404) return [];
+      throw err;
+    }
+  }
+
+  /** Serialize raw CW contact data into the API response shape. */
+  private _serializeContacts(contacts: any[]) {
+    return contacts.map((ct: any) => ({
      id: ct.id,
      contact: ct.contact ? { id: ct.contact.id, name: ct.contact.name } : null,
      company: ct.company
@@ -329,14 +399,38 @@ export class OpportunityController {
  /**
   * Fetch Notes
   *
-   * Fetches notes associated with this opportunity from ConnectWise
-   * and returns a serialized array.
+   * Fetches notes associated with this opportunity. Checks the Redis
+   * cache first; on miss, calls ConnectWise and caches the raw response.
+   *
+   * @param opts.fresh - Bypass cache and fetch directly from CW.
   */
-  public async fetchNotes() {
-    const notes = await opportunityCw.fetchNotes(this.cwOpportunityId);
+  public async fetchNotes(opts?: { fresh?: boolean }) {
+    const ttl = this._subResourceTTL();

+    // Try cache first (unless forced fresh)
+    if (!opts?.fresh && ttl !== null) {
+      const cached = await getCachedNotes(this.cwOpportunityId);
+      if (cached) return this._serializeNotes(cached);
+    }
+
+    // Fetch from CW (fetchAndCache* handles 404 internally)
+    try {
+      const notes =
+        ttl !== null
+          ? await fetchAndCacheNotes(this.cwOpportunityId, ttl)
+          : await opportunityCw.fetchNotes(this.cwOpportunityId);
+
+      return this._serializeNotes(notes);
+    } catch (err: any) {
+      if (err?.isAxiosError && err?.response?.status === 404) return [];
+      throw err;
+    }
+  }
+
+  /** Serialize raw CW note data into the API response shape. */
+  private async _serializeNotes(notes: any[]) {
    return Promise.all(
-      notes.map(async (n) => ({
+      notes.map(async (n: any) => ({
        id: n.id,
        text: n.text,
        type: n.type ? { id: n.type.id, name: n.type.name } : null,
@@ -388,15 +482,58 @@ export class OpportunityController {
  /**
   * Fetch Products
   *
-   * Fetches products (forecast/revenue items) for this opportunity from
-   * ConnectWise and returns ForecastProductController instances.
+   * Fetches products (forecast/revenue items) for this opportunity.
+   * Checks the Redis cache first; on miss, calls ConnectWise and
+   * caches the raw response using the products-specific TTL algorithm.
+   *
+   * @param opts.fresh - Bypass cache and fetch directly from CW.
   */
-  public async fetchProducts(): Promise<ForecastProductController[]> {
-    const [forecast, procProducts] = await Promise.all([
-      opportunityCw.fetchProducts(this.cwOpportunityId),
-      opportunityCw.fetchProcurementProducts(this.cwOpportunityId),
-    ]);
+  public async fetchProducts(opts?: {
+    fresh?: boolean;
+  }): Promise<ForecastProductController[]> {
+    const ttl = this._productsTTL();

+    let forecast: any;
+    let procProducts: any[];
+
+    // Try cache first (unless forced fresh)
+    if (!opts?.fresh && ttl !== null) {
+      const cached = await getCachedProducts(this.cwOpportunityId);
+      if (cached) {
+        forecast = cached.forecast;
+        procProducts = cached.procProducts;
+      } else {
+        // Cache miss — fetch from CW and cache
+        const blob = await fetchAndCacheProducts(this.cwOpportunityId, ttl);
+        forecast = blob.forecast;
+        procProducts = blob.procProducts;
+      }
+    } else {
+      // No caching (won/lost/pending or forced fresh) — fetch directly
+      try {
+        [forecast, procProducts] = await Promise.all([
+          opportunityCw.fetchProducts(this.cwOpportunityId),
+          opportunityCw.fetchProcurementProducts(this.cwOpportunityId),
+        ]);
+      } catch (err: any) {
+        if (err?.isAxiosError && err?.response?.status === 404) return [];
+        throw err;
+      }
+    }
+
+    return this._buildProductControllers(forecast, procProducts);
+  }
+
+  /**
+   * Build ForecastProductController[] from raw CW data.
+   *
+   * Extracted from fetchProducts() so both cached and fresh paths
+   * share the same ordering + enrichment logic.
+   */
+  private async _buildProductControllers(
+    forecast: any,
+    procProducts: any[],
+  ): Promise<ForecastProductController[]> {
    // Build a map of forecastDetailId → procurement product cancellation data
    const cancellationMap = new Map<number, Record<string, unknown>>();
    for (const pp of procProducts) {
@@ -412,30 +549,32 @@ export class OpportunityController {
    let ordered: typeof forecastItems;

    if (this.productSequence.length > 0) {
-      const itemById = new Map(forecastItems.map((fi) => [fi.id, fi]));
+      const itemById = new Map(forecastItems.map((fi: any) => [fi.id, fi]));
      // Items in the specified order first, then any new items not yet sequenced
      const sequenced = this.productSequence
        .map((id) => itemById.get(id))
-        .filter((fi): fi is NonNullable<typeof fi> => fi !== undefined);
+        .filter((fi: any): fi is NonNullable<typeof fi> => fi !== undefined);
      const sequencedIds = new Set(this.productSequence);
      const unsequenced = forecastItems
-        .filter((fi) => !sequencedIds.has(fi.id))
-        .sort((a, b) => a.sequenceNumber - b.sequenceNumber);
+        .filter((fi: any) => !sequencedIds.has(fi.id))
+        .sort((a: any, b: any) => a.sequenceNumber - b.sequenceNumber);
      ordered = [...sequenced, ...unsequenced];
    } else {
      ordered = [...forecastItems].sort(
-        (a, b) => a.sequenceNumber - b.sequenceNumber,
+        (a: any, b: any) => a.sequenceNumber - b.sequenceNumber,
      );
    }

-    const controllers = ordered.map((item) => {
-      const ctrl = new ForecastProductController(item);
-      const procData = cancellationMap.get(item.id);
-      if (procData) {
-        ctrl.applyCancellationData(procData as any);
-      }
-      return ctrl;
-    });
+    const controllers: ForecastProductController[] = ordered.map(
+      (item: any) => {
+        const ctrl = new ForecastProductController(item);
+        const procData = cancellationMap.get(item.id);
+        if (procData) {
+          ctrl.applyCancellationData(procData as any);
+        }
+        return ctrl;
+      },
+    );

    // Enrich with internal inventory data from local CatalogItem DB
    const catalogCwIds = controllers
@@ -559,6 +698,7 @@ export class OpportunityController {
        forecastItemId,
        data,
      );
+      await invalidateProductsCache(this.cwOpportunityId);
      return new ForecastProductController(updated);
    } catch (err: any) {
      console.error(
@@ -613,6 +753,9 @@ export class OpportunityController {
    });
    this.productSequence = orderedIds;

+    // Invalidate cached products since ordering changed
+    await invalidateProductsCache(this.cwOpportunityId);
+
    // Return items in the new order
    return this.fetchProducts();
  }
@@ -635,6 +778,7 @@ export class OpportunityController {
        this.cwOpportunityId,
        data,
      );
+      await invalidateProductsCache(this.cwOpportunityId);
      return created.map((item) => new ForecastProductController(item));
    } catch (err: any) {
      console.error(
@@ -680,6 +824,7 @@ export class OpportunityController {
      text: note,
      flagged: opts?.flagged ?? false,
    });
+    await invalidateNotesCache(this.cwOpportunityId);
    return created;
  }

@@ -700,6 +845,7 @@ export class OpportunityController {
      noteId,
      data,
    );
+    await invalidateNotesCache(this.cwOpportunityId);
    return updated;
  }

@@ -712,6 +858,7 @@ export class OpportunityController {
   */
  public async deleteNote(noteId: number): Promise<void> {
    await opportunityCw.deleteNote(this.cwOpportunityId, noteId);
+    await invalidateNotesCache(this.cwOpportunityId);
  }

  /**