feat: Redis opportunity cache, CW API retry/logging, adaptive TTLs
- Add Redis-backed opportunity cache with background refresh (30s interval) - Fix concurrency bug: use lazy thunks instead of eager promises for batching - Add withCwRetry utility with exponential backoff for transient CW errors - Add adaptive TTL algorithms (primary, sub-resource, products) based on opportunity activity - Add include query param on GET /sales/opportunities/:id (notes,contacts,products) - Add opt-in CW API logger (LOG_CW_API env var) with timestamped files in cw-api-logs/ - Add debug-scripts/analyze-cw-calls.py for API call analysis - Add computeSubResourceCacheTTL and computeProductsCacheTTL algorithms with tests - Increase CW API timeout from 15s to 30s - Unblock cache refresh from startup chain (remove await) - Prioritize recently updated opportunities in refresh cycle - Add CACHING.md documentation - Update API_ROUTES.md with caching details and include param - Update copilot instructions to require CACHING.md sync - Add dev:log script for CW API call logging during development
This commit is contained in:
+546
-49
@@ -13,6 +13,10 @@
|
||||
* `CWActivity[]` array fetched from `activityCw.fetchByOpportunity()`.
|
||||
* - **Company CW data** (`opp:company-cw:{cw_CompanyId}`) — the hydrated
|
||||
* company / contacts blob set by `CompanyController.hydrateCwData()`.
|
||||
* - **Notes** (`opp:notes:{cwOpportunityId}`) — raw CW notes array.
|
||||
* - **Contacts** (`opp:contacts:{cwOpportunityId}`) — raw CW contacts array.
|
||||
* - **Products** (`opp:products:{cwOpportunityId}`) — raw CW forecast +
|
||||
* procurement products blob.
|
||||
*
|
||||
* TTLs are computed dynamically via {@link computeCacheTTL} so hot
|
||||
* opportunities refresh every 30 s while stale ones live for 15 min.
|
||||
@@ -28,8 +32,16 @@
|
||||
import { prisma, redis } from "../../constants";
|
||||
import { activityCw } from "../cw-utils/activities/activities";
|
||||
import { computeCacheTTL } from "../algorithms/computeCacheTTL";
|
||||
import { computeSubResourceCacheTTL } from "../algorithms/computeSubResourceCacheTTL";
|
||||
import {
|
||||
computeProductsCacheTTL,
|
||||
PRODUCTS_TTL_HOT,
|
||||
} from "../algorithms/computeProductsCacheTTL";
|
||||
import { connectWiseApi } from "../../constants";
|
||||
import { fetchCwCompanyById } from "../cw-utils/fetchCompany";
|
||||
import { fetchCompanySite } from "../cw-utils/sites/companySites";
|
||||
import { opportunityCw } from "../cw-utils/opportunities/opportunities";
|
||||
import { withCwRetry } from "../cw-utils/withCwRetry";
|
||||
import { events } from "../globalEvents";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -38,6 +50,11 @@ import { events } from "../globalEvents";
|
||||
|
||||
const ACTIVITY_PREFIX = "opp:activities:";
|
||||
const COMPANY_CW_PREFIX = "opp:company-cw:";
|
||||
const NOTES_PREFIX = "opp:notes:";
|
||||
const CONTACTS_PREFIX = "opp:contacts:";
|
||||
const PRODUCTS_PREFIX = "opp:products:";
|
||||
const SITE_PREFIX = "opp:site:";
|
||||
const OPP_CW_PREFIX = "opp:cw-data:";
|
||||
|
||||
/** Redis key for cached activities by CW opportunity ID. */
|
||||
export const activityCacheKey = (cwOppId: number) =>
|
||||
@@ -47,6 +64,25 @@ export const activityCacheKey = (cwOppId: number) =>
|
||||
export const companyCwCacheKey = (cwCompanyId: number) =>
|
||||
`${COMPANY_CW_PREFIX}${cwCompanyId}`;
|
||||
|
||||
/** Redis key for cached opportunity notes by CW opportunity ID. */
|
||||
export const notesCacheKey = (cwOppId: number) => `${NOTES_PREFIX}${cwOppId}`;
|
||||
|
||||
/** Redis key for cached opportunity contacts by CW opportunity ID. */
|
||||
export const contactsCacheKey = (cwOppId: number) =>
|
||||
`${CONTACTS_PREFIX}${cwOppId}`;
|
||||
|
||||
/** Redis key for cached opportunity products by CW opportunity ID. */
|
||||
export const productsCacheKey = (cwOppId: number) =>
|
||||
`${PRODUCTS_PREFIX}${cwOppId}`;
|
||||
|
||||
/** Redis key for cached company site by CW company ID + site ID. */
|
||||
export const siteCacheKey = (cwCompanyId: number, cwSiteId: number) =>
|
||||
`${SITE_PREFIX}${cwCompanyId}:${cwSiteId}`;
|
||||
|
||||
/** Redis key for cached CW opportunity response by CW opportunity ID. */
|
||||
export const oppCwDataCacheKey = (cwOppId: number) =>
|
||||
`${OPP_CW_PREFIX}${cwOppId}`;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Read helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -85,6 +121,147 @@ export async function getCachedCompanyCwData(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve cached opportunity notes (raw CW data).
|
||||
*
|
||||
* @returns The parsed raw CW notes array or `null` on cache miss.
|
||||
*/
|
||||
export async function getCachedNotes(
|
||||
cwOpportunityId: number,
|
||||
): Promise<any[] | null> {
|
||||
const raw = await redis.get(notesCacheKey(cwOpportunityId));
|
||||
if (!raw) return null;
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve cached opportunity contacts (raw CW data).
|
||||
*
|
||||
* @returns The parsed raw CW contacts array or `null` on cache miss.
|
||||
*/
|
||||
export async function getCachedContacts(
|
||||
cwOpportunityId: number,
|
||||
): Promise<any[] | null> {
|
||||
const raw = await redis.get(contactsCacheKey(cwOpportunityId));
|
||||
if (!raw) return null;
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve cached opportunity products (raw CW forecast + procurement blob).
|
||||
*
|
||||
* @returns `{ forecast, procProducts }` or `null` on cache miss.
|
||||
*/
|
||||
export async function getCachedProducts(
|
||||
cwOpportunityId: number,
|
||||
): Promise<{ forecast: any; procProducts: any[] } | null> {
|
||||
const raw = await redis.get(productsCacheKey(cwOpportunityId));
|
||||
if (!raw) return null;
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve cached CW site data for a company/site pair.
|
||||
*
|
||||
* @returns Parsed site data or `null` on cache miss.
|
||||
*/
|
||||
export async function getCachedSite(
|
||||
cwCompanyId: number,
|
||||
cwSiteId: number,
|
||||
): Promise<any | null> {
|
||||
const raw = await redis.get(siteCacheKey(cwCompanyId, cwSiteId));
|
||||
if (!raw) return null;
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve cached CW opportunity response data.
|
||||
*
|
||||
* @returns Parsed CW opportunity object or `null` on cache miss.
|
||||
*/
|
||||
export async function getCachedOppCwData(
|
||||
cwOpportunityId: number,
|
||||
): Promise<any | null> {
|
||||
const raw = await redis.get(oppCwDataCacheKey(cwOpportunityId));
|
||||
if (!raw) return null;
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Check whether an error is an Axios 404 (resource not found in CW). */
|
||||
function isNotFoundError(err: unknown): boolean {
|
||||
if (typeof err !== "object" || err === null) return false;
|
||||
const e = err as Record<string, any>;
|
||||
return e.isAxiosError === true && e.response?.status === 404;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether an error is a transient network / timeout error.
|
||||
*
|
||||
* These are safe to swallow in background refresh tasks — CW will be
|
||||
* retried on the next refresh cycle. Logs a concise one-line warning
|
||||
* instead of dumping the full Axios error object.
|
||||
*/
|
||||
function isTransientError(err: unknown): boolean {
|
||||
if (typeof err !== "object" || err === null) return false;
|
||||
const e = err as Record<string, any>;
|
||||
if (!e.isAxiosError) return false;
|
||||
const code = e.code as string | undefined;
|
||||
return (
|
||||
code === "ECONNABORTED" ||
|
||||
code === "ECONNREFUSED" ||
|
||||
code === "ECONNRESET" ||
|
||||
code === "ETIMEDOUT" ||
|
||||
code === "ERR_NETWORK" ||
|
||||
code === "ENETUNREACH" ||
|
||||
code === "ERR_BAD_RESPONSE"
|
||||
);
|
||||
}
|
||||
|
||||
/** Build a concise error description for logging (avoids dumping entire Axios objects). */
|
||||
function describeError(err: unknown): string {
|
||||
if (typeof err !== "object" || err === null) return String(err);
|
||||
const e = err as Record<string, any>;
|
||||
if (e.isAxiosError) {
|
||||
const method = (e.config?.method ?? "?").toUpperCase();
|
||||
const url = e.config?.url ?? "unknown";
|
||||
const code = e.code ?? "";
|
||||
const status = e.response?.status ?? "";
|
||||
return `${method} ${url} → ${code || `HTTP ${status}`} (${e.message})`;
|
||||
}
|
||||
return e.message ?? String(err);
|
||||
}
|
||||
|
||||
/**
|
||||
* When true, transient-error warnings inside fetchAndCache* are suppressed.
|
||||
* Used during background refresh to avoid flooding the terminal — the
|
||||
* refresh function prints a single summary line instead.
|
||||
*/
|
||||
let _suppressTransientWarnings = false;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Write helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -92,21 +269,35 @@ export async function getCachedCompanyCwData(
|
||||
/**
|
||||
* Fetch activities from CW and cache them with the appropriate TTL.
|
||||
*
|
||||
* Returns an empty array if CW responds with 404 (opportunity doesn't
|
||||
* exist or was deleted upstream).
|
||||
*
|
||||
* @returns The raw `CWActivity[]` collection (as plain array).
|
||||
*/
|
||||
export async function fetchAndCacheActivities(
|
||||
cwOpportunityId: number,
|
||||
ttlMs: number,
|
||||
): Promise<any[]> {
|
||||
const collection = await activityCw.fetchByOpportunity(cwOpportunityId);
|
||||
const arr = collection.map((item) => item);
|
||||
await redis.set(
|
||||
activityCacheKey(cwOpportunityId),
|
||||
JSON.stringify(arr),
|
||||
"PX",
|
||||
ttlMs,
|
||||
);
|
||||
return arr;
|
||||
try {
|
||||
// Use the direct (single-call) variant to avoid the extra count request
|
||||
const arr = await activityCw.fetchByOpportunityDirect(cwOpportunityId);
|
||||
await redis.set(
|
||||
activityCacheKey(cwOpportunityId),
|
||||
JSON.stringify(arr),
|
||||
"PX",
|
||||
ttlMs,
|
||||
);
|
||||
return arr;
|
||||
} catch (err) {
|
||||
if (isNotFoundError(err)) return [];
|
||||
if (isTransientError(err)) {
|
||||
console.warn(
|
||||
`[cache] activities opp#${cwOpportunityId}: ${describeError(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -118,32 +309,255 @@ export async function fetchAndCacheCompanyCwData(
|
||||
cwCompanyId: number,
|
||||
ttlMs: number,
|
||||
): Promise<{ company: any; defaultContact: any; allContacts: any[] } | null> {
|
||||
const cwCompany = await fetchCwCompanyById(cwCompanyId);
|
||||
if (!cwCompany) return null;
|
||||
try {
|
||||
const cwCompany = await fetchCwCompanyById(cwCompanyId);
|
||||
if (!cwCompany) return null;
|
||||
|
||||
const contactHref = cwCompany.defaultContact?._info?.contact_href;
|
||||
const defaultContactData = contactHref
|
||||
? await connectWiseApi.get(contactHref)
|
||||
: undefined;
|
||||
const contactHref = cwCompany.defaultContact?._info?.contact_href;
|
||||
const defaultContactData = contactHref
|
||||
? await withCwRetry(() => connectWiseApi.get(contactHref), {
|
||||
label: `company#${cwCompanyId}/defaultContact`,
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const allContactsData = await connectWiseApi.get(
|
||||
`${cwCompany._info.contacts_href}&pageSize=1000`,
|
||||
);
|
||||
const allContactsData = await withCwRetry(
|
||||
() =>
|
||||
connectWiseApi.get(`${cwCompany._info.contacts_href}&pageSize=1000`),
|
||||
{ label: `company#${cwCompanyId}/allContacts` },
|
||||
);
|
||||
|
||||
const blob = {
|
||||
company: cwCompany,
|
||||
defaultContact: defaultContactData?.data ?? null,
|
||||
allContacts: allContactsData.data,
|
||||
};
|
||||
const blob = {
|
||||
company: cwCompany,
|
||||
defaultContact: defaultContactData?.data ?? null,
|
||||
allContacts: allContactsData.data,
|
||||
};
|
||||
|
||||
await redis.set(
|
||||
companyCwCacheKey(cwCompanyId),
|
||||
JSON.stringify(blob),
|
||||
"PX",
|
||||
ttlMs,
|
||||
);
|
||||
await redis.set(
|
||||
companyCwCacheKey(cwCompanyId),
|
||||
JSON.stringify(blob),
|
||||
"PX",
|
||||
ttlMs,
|
||||
);
|
||||
|
||||
return blob;
|
||||
return blob;
|
||||
} catch (err) {
|
||||
if (isNotFoundError(err)) return null;
|
||||
if (isTransientError(err)) {
|
||||
console.warn(`[cache] company#${cwCompanyId}: ${describeError(err)}`);
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch opportunity notes from CW and cache the raw response.
|
||||
*
|
||||
* Returns an empty array if CW responds with 404.
|
||||
*
|
||||
* @returns The raw CW notes array.
|
||||
*/
|
||||
export async function fetchAndCacheNotes(
|
||||
cwOpportunityId: number,
|
||||
ttlMs: number,
|
||||
): Promise<any[]> {
|
||||
try {
|
||||
const notes = await opportunityCw.fetchNotes(cwOpportunityId);
|
||||
await redis.set(
|
||||
notesCacheKey(cwOpportunityId),
|
||||
JSON.stringify(notes),
|
||||
"PX",
|
||||
ttlMs,
|
||||
);
|
||||
return notes;
|
||||
} catch (err) {
|
||||
if (isNotFoundError(err)) return [];
|
||||
if (isTransientError(err)) {
|
||||
console.warn(
|
||||
`[cache] notes opp#${cwOpportunityId}: ${describeError(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch opportunity contacts from CW and cache the raw response.
|
||||
*
|
||||
* Returns an empty array if CW responds with 404.
|
||||
*
|
||||
* @returns The raw CW contacts array.
|
||||
*/
|
||||
export async function fetchAndCacheContacts(
|
||||
cwOpportunityId: number,
|
||||
ttlMs: number,
|
||||
): Promise<any[]> {
|
||||
try {
|
||||
const contacts = await opportunityCw.fetchContacts(cwOpportunityId);
|
||||
await redis.set(
|
||||
contactsCacheKey(cwOpportunityId),
|
||||
JSON.stringify(contacts),
|
||||
"PX",
|
||||
ttlMs,
|
||||
);
|
||||
return contacts;
|
||||
} catch (err) {
|
||||
if (isNotFoundError(err)) return [];
|
||||
if (isTransientError(err)) {
|
||||
console.warn(
|
||||
`[cache] contacts opp#${cwOpportunityId}: ${describeError(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate cached notes for an opportunity.
|
||||
*
|
||||
* Call this after any note mutation (create, update, delete) so the
|
||||
* next read refreshes from ConnectWise.
|
||||
*/
|
||||
export async function invalidateNotesCache(
|
||||
cwOpportunityId: number,
|
||||
): Promise<void> {
|
||||
await redis.del(notesCacheKey(cwOpportunityId));
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate cached contacts for an opportunity.
|
||||
*
|
||||
* Call this after any contact mutation so the next read refreshes
|
||||
* from ConnectWise.
|
||||
*/
|
||||
export async function invalidateContactsCache(
|
||||
cwOpportunityId: number,
|
||||
): Promise<void> {
|
||||
await redis.del(contactsCacheKey(cwOpportunityId));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch opportunity products (forecast + procurement) from CW and cache.
|
||||
*
|
||||
* Stores both the forecast response and procurement products together
|
||||
* so that `fetchProducts()` can reconstruct ForecastProductControllers
|
||||
* from a single cache hit.
|
||||
*
|
||||
* @returns `{ forecast, procProducts }` blob.
|
||||
*/
|
||||
export async function fetchAndCacheProducts(
|
||||
cwOpportunityId: number,
|
||||
ttlMs: number,
|
||||
): Promise<{ forecast: any; procProducts: any[] }> {
|
||||
try {
|
||||
const [forecast, procProducts] = await Promise.all([
|
||||
opportunityCw.fetchProducts(cwOpportunityId),
|
||||
opportunityCw.fetchProcurementProducts(cwOpportunityId),
|
||||
]);
|
||||
|
||||
const blob = { forecast, procProducts };
|
||||
await redis.set(
|
||||
productsCacheKey(cwOpportunityId),
|
||||
JSON.stringify(blob),
|
||||
"PX",
|
||||
ttlMs,
|
||||
);
|
||||
return blob;
|
||||
} catch (err) {
|
||||
if (isNotFoundError(err))
|
||||
return { forecast: { forecastItems: [] }, procProducts: [] };
|
||||
if (isTransientError(err)) {
|
||||
console.warn(
|
||||
`[cache] products opp#${cwOpportunityId}: ${describeError(err)}`,
|
||||
);
|
||||
return { forecast: { forecastItems: [] }, procProducts: [] };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate cached products for an opportunity.
|
||||
*
|
||||
* Call this after any product mutation (add, update, resequence) so the
|
||||
* next read refreshes from ConnectWise.
|
||||
*/
|
||||
export async function invalidateProductsCache(
|
||||
cwOpportunityId: number,
|
||||
): Promise<void> {
|
||||
await redis.del(productsCacheKey(cwOpportunityId));
|
||||
}
|
||||
|
||||
/**
|
||||
* Site TTL — 30 minutes. Site/address data rarely changes so we cache
|
||||
* aggressively. The background refresh does NOT proactively warm site keys;
|
||||
* they are populated lazily on the first detail-view request.
|
||||
*/
|
||||
const SITE_TTL_MS = 1_800_000;
|
||||
|
||||
/**
|
||||
* Fetch a CW company site from ConnectWise and cache the result.
|
||||
*
|
||||
* @returns The raw CW site object.
|
||||
*/
|
||||
export async function fetchAndCacheSite(
|
||||
cwCompanyId: number,
|
||||
cwSiteId: number,
|
||||
): Promise<any> {
|
||||
try {
|
||||
const site = await fetchCompanySite(cwCompanyId, cwSiteId);
|
||||
await redis.set(
|
||||
siteCacheKey(cwCompanyId, cwSiteId),
|
||||
JSON.stringify(site),
|
||||
"PX",
|
||||
SITE_TTL_MS,
|
||||
);
|
||||
return site;
|
||||
} catch (err) {
|
||||
if (isNotFoundError(err)) return null;
|
||||
if (isTransientError(err)) {
|
||||
console.warn(
|
||||
`[cache] site company#${cwCompanyId}/site#${cwSiteId}: ${describeError(err)}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the raw CW opportunity response from ConnectWise and cache it.
|
||||
*
|
||||
* Used by `fetchItem()` in the manager to avoid a CW roundtrip when
|
||||
* the detail view is reloaded within the cache TTL window.
|
||||
*
|
||||
* @param cwOpportunityId - The CW opportunity ID
|
||||
* @param ttlMs - Cache TTL in milliseconds
|
||||
* @returns The raw CW opportunity response object.
|
||||
*/
|
||||
export async function fetchAndCacheOppCwData(
|
||||
cwOpportunityId: number,
|
||||
ttlMs: number,
|
||||
): Promise<any> {
|
||||
try {
|
||||
const cwData = await opportunityCw.fetch(cwOpportunityId);
|
||||
await redis.set(
|
||||
oppCwDataCacheKey(cwOpportunityId),
|
||||
JSON.stringify(cwData),
|
||||
"PX",
|
||||
ttlMs,
|
||||
);
|
||||
return cwData;
|
||||
} catch (err) {
|
||||
if (isNotFoundError(err)) return null;
|
||||
if (isTransientError(err)) {
|
||||
console.warn(`[cache] opp#${cwOpportunityId}: ${describeError(err)}`);
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -176,8 +590,10 @@ export async function refreshOpportunityCache(): Promise<void> {
|
||||
closedDate: true,
|
||||
expectedCloseDate: true,
|
||||
cwLastUpdated: true,
|
||||
statusCwId: true,
|
||||
company: { select: { cw_CompanyId: true } },
|
||||
},
|
||||
orderBy: { cwLastUpdated: "desc" },
|
||||
});
|
||||
|
||||
events.emit("cache:opportunities:refresh:started", {
|
||||
@@ -186,25 +602,41 @@ export async function refreshOpportunityCache(): Promise<void> {
|
||||
|
||||
let activitiesRefreshed = 0;
|
||||
let companiesRefreshed = 0;
|
||||
let notesRefreshed = 0;
|
||||
let contactsRefreshed = 0;
|
||||
let productsRefreshed = 0;
|
||||
let oppCwDataRefreshed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
// Batch-check which activity keys already exist via a pipeline
|
||||
// Batch-check which keys already exist via a single pipeline
|
||||
// (5 EXISTS per opportunity: oppCwData, activities, notes, contacts, products).
|
||||
const pipeline = redis.pipeline();
|
||||
for (const opp of opportunities) {
|
||||
pipeline.exists(oppCwDataCacheKey(opp.cwOpportunityId));
|
||||
pipeline.exists(activityCacheKey(opp.cwOpportunityId));
|
||||
pipeline.exists(notesCacheKey(opp.cwOpportunityId));
|
||||
pipeline.exists(contactsCacheKey(opp.cwOpportunityId));
|
||||
pipeline.exists(productsCacheKey(opp.cwOpportunityId));
|
||||
}
|
||||
const existsResults = await pipeline.exec();
|
||||
|
||||
const refreshTasks: Promise<void>[] = [];
|
||||
const refreshTasks: (() => Promise<void>)[] = [];
|
||||
|
||||
for (let i = 0; i < opportunities.length; i++) {
|
||||
const opp = opportunities[i]!;
|
||||
|
||||
const ttl = computeCacheTTL({
|
||||
const cacheTTLInput = {
|
||||
closedFlag: opp.closedFlag,
|
||||
closedDate: opp.closedDate,
|
||||
expectedCloseDate: opp.expectedCloseDate,
|
||||
lastUpdated: opp.cwLastUpdated,
|
||||
};
|
||||
|
||||
const ttl = computeCacheTTL(cacheTTLInput);
|
||||
const subTTL = computeSubResourceCacheTTL(cacheTTLInput);
|
||||
const productsTTL = computeProductsCacheTTL({
|
||||
...cacheTTLInput,
|
||||
statusCwId: opp.statusCwId,
|
||||
});
|
||||
|
||||
// Skip closed (ttl === null) — should not happen because of the query filter,
|
||||
@@ -215,43 +647,108 @@ export async function refreshOpportunityCache(): Promise<void> {
|
||||
}
|
||||
|
||||
// existsResults entries are [error, result] tuples
|
||||
const activityExists = existsResults?.[i]?.[1] === 1;
|
||||
// Pipeline order per opportunity: oppCwData, activities, notes, contacts, products
|
||||
const baseIdx = i * 5;
|
||||
const oppCwDataExists = existsResults?.[baseIdx]?.[1] === 1;
|
||||
const activityExists = existsResults?.[baseIdx + 1]?.[1] === 1;
|
||||
const notesExist = existsResults?.[baseIdx + 2]?.[1] === 1;
|
||||
const contactsExist = existsResults?.[baseIdx + 3]?.[1] === 1;
|
||||
const productsExist = existsResults?.[baseIdx + 4]?.[1] === 1;
|
||||
|
||||
// Proactively cache the CW opportunity response itself
|
||||
if (!oppCwDataExists) {
|
||||
refreshTasks.push(() =>
|
||||
fetchAndCacheOppCwData(opp.cwOpportunityId, ttl).then(() => {
|
||||
oppCwDataRefreshed++;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (!activityExists) {
|
||||
refreshTasks.push(
|
||||
refreshTasks.push(() =>
|
||||
fetchAndCacheActivities(opp.cwOpportunityId, ttl).then(() => {
|
||||
activitiesRefreshed++;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Refresh notes/contacts if sub-resource TTL applies and key is missing
|
||||
if (subTTL !== null) {
|
||||
if (!notesExist) {
|
||||
refreshTasks.push(() =>
|
||||
fetchAndCacheNotes(opp.cwOpportunityId, subTTL).then(() => {
|
||||
notesRefreshed++;
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (!contactsExist) {
|
||||
refreshTasks.push(() =>
|
||||
fetchAndCacheContacts(opp.cwOpportunityId, subTTL).then(() => {
|
||||
contactsRefreshed++;
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Proactively refresh products only for hot opps (updated within 3 days).
|
||||
// 30-minute lazy-cached products are filled on-demand by the endpoint
|
||||
// and do not need background refresh.
|
||||
if (productsTTL === PRODUCTS_TTL_HOT && !productsExist) {
|
||||
refreshTasks.push(() =>
|
||||
fetchAndCacheProducts(opp.cwOpportunityId, productsTTL).then(() => {
|
||||
productsRefreshed++;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Also refresh company CW data if the key is missing
|
||||
if (opp.company?.cw_CompanyId) {
|
||||
const cwCompanyId = opp.company.cw_CompanyId;
|
||||
refreshTasks.push(
|
||||
(async () => {
|
||||
const companyExists = await redis.exists(
|
||||
companyCwCacheKey(cwCompanyId),
|
||||
);
|
||||
if (!companyExists) {
|
||||
await fetchAndCacheCompanyCwData(cwCompanyId, ttl);
|
||||
companiesRefreshed++;
|
||||
}
|
||||
})(),
|
||||
);
|
||||
refreshTasks.push(async () => {
|
||||
const companyExists = await redis.exists(
|
||||
companyCwCacheKey(cwCompanyId),
|
||||
);
|
||||
if (!companyExists) {
|
||||
await fetchAndCacheCompanyCwData(cwCompanyId, ttl);
|
||||
companiesRefreshed++;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Run all refresh tasks concurrently with bounded concurrency
|
||||
const CONCURRENCY = 10;
|
||||
// Run refresh thunks with bounded concurrency and inter-batch delay.
|
||||
// Each thunk is only invoked here — no requests fire until we call them.
|
||||
// CW rate-limits aggressively so we keep this conservative.
|
||||
const CONCURRENCY = 6;
|
||||
const BATCH_DELAY_MS = 250;
|
||||
let timeoutCount = 0;
|
||||
|
||||
for (let i = 0; i < refreshTasks.length; i += CONCURRENCY) {
|
||||
await Promise.allSettled(refreshTasks.slice(i, i + CONCURRENCY));
|
||||
const batch = refreshTasks.slice(i, i + CONCURRENCY);
|
||||
const results = await Promise.allSettled(batch.map((fn) => fn()));
|
||||
for (const r of results) {
|
||||
if (r.status === "rejected") timeoutCount++;
|
||||
}
|
||||
// Small delay between batches to avoid overwhelming CW
|
||||
if (i + CONCURRENCY < refreshTasks.length) {
|
||||
await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY_MS));
|
||||
}
|
||||
}
|
||||
|
||||
if (timeoutCount > 0) {
|
||||
console.warn(
|
||||
`[cache] refresh: ${timeoutCount} task(s) failed (likely CW timeouts) — will retry next cycle`,
|
||||
);
|
||||
}
|
||||
|
||||
events.emit("cache:opportunities:refresh:completed", {
|
||||
totalOpportunities: opportunities.length,
|
||||
activitiesRefreshed,
|
||||
companiesRefreshed,
|
||||
notesRefreshed,
|
||||
contactsRefreshed,
|
||||
productsRefreshed,
|
||||
oppCwDataRefreshed,
|
||||
skipped,
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user