From fe71248e883ddf10ed8ebd69f2b1656565f5dcc5 Mon Sep 17 00:00:00 2001 From: Jackson Roberts Date: Mon, 2 Mar 2026 21:12:44 -0600 Subject: [PATCH] perf: cache-only strategy for list views, cache-then-cw for single fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add data-source hierarchy to opportunity manager (cache-only, cache-then-cw, cw-first) - fetchPages/search/fetchByCompany use cache-only: Redis → DB (no CW calls) - fetchItem uses cache-then-cw by default, cw-first when fresh=true - Add idleTimeout: 255 to Bun.serve to prevent request timeouts - Map CW status 57 (04. Confirmed Quote) to Active equivalency - Add computeCacheTTL algorithm and opportunityCache module --- src/index.ts | 10 + src/managers/opportunities.ts | 198 +++++++-- src/modules/algorithms/computeCacheTTL.ts | 166 ++++++++ src/modules/cache/opportunityCache.ts | 257 ++++++++++++ src/modules/globalEvents.ts | 12 + src/types/QuoteStatuses.ts | 1 + tests/unit/computeCacheTTL.test.ts | 477 ++++++++++++++++++++++ 7 files changed, 1092 insertions(+), 29 deletions(-) create mode 100644 src/modules/algorithms/computeCacheTTL.ts create mode 100644 src/modules/cache/opportunityCache.ts create mode 100644 tests/unit/computeCacheTTL.test.ts diff --git a/src/index.ts b/src/index.ts index 9649f6e..415521f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ import { refreshCompanies } from "./modules/cw-utils/refreshCompanies"; import { refreshCatalog } from "./modules/cw-utils/procurement/refreshCatalog"; import { refreshInventory } from "./modules/cw-utils/procurement/refreshInventory"; import { refreshOpportunities } from "./modules/cw-utils/opportunities/refreshOpportunities"; +import { refreshOpportunityCache } from "./modules/cache/opportunityCache"; import { refreshCwIdentifiers } from "./modules/cw-utils/members/refreshCwIdentifiers"; import { userDefinedFieldsCw } from "./modules/cw-utils/userDefinedFields"; import { events, setupEventDebugger } from "./modules/globalEvents"; @@ -41,6 +42,7 @@ const safeStartup = async (label: string, fn: () => Promise) => { // --------------------------------------------------------------------------- Bun.serve({ port: PORT, + idleTimeout: 255, websocket: engine.handler().websocket, fetch: (req, server) => { const url = new URL(req.url); @@ -120,6 +122,14 @@ setInterval(() => { ); }, 60 * 1000); +// Refresh opportunity CW cache every 30 seconds (activities + company hydration) +await safeStartup("refreshOpportunityCache", refreshOpportunityCache); +setInterval(() => { + return refreshOpportunityCache().catch((err) => + console.error("[interval] refreshOpportunityCache failed", err), + ); +}, 30 * 1000); + // Refresh User Defined Fields every 5 minutes await safeStartup("refreshUDFs", () => userDefinedFieldsCw.refresh()); setInterval( diff --git a/src/managers/opportunities.ts b/src/managers/opportunities.ts index 96341ac..1a51dfa 100644 --- a/src/managers/opportunities.ts +++ b/src/managers/opportunities.ts @@ -6,26 +6,121 @@ import { OpportunityController } from "../controllers/OpportunityController"; import GenericError from "../Errors/GenericError"; import { activityCw } from "../modules/cw-utils/activities/activities"; import { opportunityCw } from "../modules/cw-utils/opportunities/opportunities"; +import { computeCacheTTL } from "../modules/algorithms/computeCacheTTL"; +import { + getCachedActivities, + getCachedCompanyCwData, + fetchAndCacheActivities, + fetchAndCacheCompanyCwData, +} from "../modules/cache/opportunityCache"; + +// --------------------------------------------------------------------------- +// Data-source hierarchy helpers +// --------------------------------------------------------------------------- /** * Build a CompanyController with hydrated CW data from a Prisma Company record. + * + * Data-source hierarchy (controlled by `strategy`): + * + * - `"cache-only"` — Redis cache → bare DB record (no CW call). + * Ideal for list views where latency matters and the background + * refresh job is responsible for keeping the cache warm. + * + * - `"cache-then-cw"` (default) — Redis cache → CW API → cache result. + * On a cold cache, calls CW to ensure the caller gets full data. + * + * - `"cw-first"` — CW API (always) → cache result. + * Forces a fresh fetch regardless of cache state. */ async function buildCompanyController( company: Company, + opts?: { + strategy?: "cache-only" | "cache-then-cw" | "cw-first"; + ttlMs?: number; + }, ): Promise { + const strategy = opts?.strategy ?? "cache-then-cw"; const ctrl = new CompanyController(company); + + // ── cw-first: always fetch from CW ────────────────────────────────── + if (strategy === "cw-first") { + await ctrl.hydrateCwData(); + if (ctrl.cw_Data && opts?.ttlMs) { + await fetchAndCacheCompanyCwData(company.cw_CompanyId, opts.ttlMs).catch( + () => {}, + ); + } + return ctrl; + } + + // ── cache-only / cache-then-cw: try Redis first ───────────────────── + const cached = await getCachedCompanyCwData(company.cw_CompanyId); + if (cached) { + ctrl.cw_Data = cached; + return ctrl; + } + + // cache-only stops here — return the bare DB-backed controller + if (strategy === "cache-only") return ctrl; + + // cache-then-cw: cache miss — fall through to CW await ctrl.hydrateCwData(); + if (ctrl.cw_Data && opts?.ttlMs) { + await fetchAndCacheCompanyCwData(company.cw_CompanyId, opts.ttlMs).catch( + () => {}, + ); + } + return ctrl; } /** - * Fetch ActivityController[] for an opportunity from ConnectWise. + * Fetch ActivityController[] for an opportunity. + * + * Same three strategies as {@link buildCompanyController}: + * + * - `"cache-only"` — Redis → empty array (no CW call). + * - `"cache-then-cw"` (default) — Redis → CW API → cache result. + * - `"cw-first"` — CW API (always) → cache result. */ async function buildActivities( cwOpportunityId: number, + opts?: { + strategy?: "cache-only" | "cache-then-cw" | "cw-first"; + ttlMs?: number; + }, ): Promise { + const strategy = opts?.strategy ?? "cache-then-cw"; + + // ── cw-first: always fetch from CW ────────────────────────────────── + if (strategy === "cw-first") { + const collection = await activityCw.fetchByOpportunity(cwOpportunityId); + const arr = collection.map((item) => item); + if (opts?.ttlMs) { + await fetchAndCacheActivities(cwOpportunityId, opts.ttlMs).catch( + () => {}, + ); + } + return arr.map((item) => new ActivityController(item)); + } + + // ── cache-only / cache-then-cw: try Redis first ───────────────────── + const cached = await getCachedActivities(cwOpportunityId); + if (cached) { + return cached.map((item) => new ActivityController(item)); + } + + // cache-only stops here — return empty (background job will fill it) + if (strategy === "cache-only") return []; + + // cache-then-cw: cache miss — fall through to CW const collection = await activityCw.fetchByOpportunity(cwOpportunityId); - return collection.map((item) => new ActivityController(item)); + const arr = collection.map((item) => item); + if (opts?.ttlMs) { + await fetchAndCacheActivities(cwOpportunityId, opts.ttlMs).catch(() => {}); + } + return arr.map((item) => new ActivityController(item)); } export const opportunities = { @@ -35,10 +130,26 @@ export const opportunities = { * Fetch an opportunity by its internal ID or ConnectWise opportunity ID * and return an OpportunityController instance. * + * **Data-source strategy:** + * - `fresh: true` → `"cw-first"` — always fetches from CW, updates DB, caches result. + * - `fresh: false` (default) → `"cache-then-cw"` — tries Redis first, falls back to CW on miss. + * + * Individual fetches always contact CW to update the DB record with + * the latest data from ConnectWise, regardless of the cache strategy + * for activities/company hydration. + * * @param identifier - The internal ID (string) or CW opportunity ID (number) + * @param opts - Optional flags + * @param opts.fresh - When `true`, bypass the cache and pull directly from CW. * @returns {Promise} */ - async fetchItem(identifier: string | number): Promise { + async fetchItem( + identifier: string | number, + opts?: { fresh?: boolean }, + ): Promise { + const strategy: "cache-only" | "cache-then-cw" | "cw-first" = opts?.fresh + ? "cw-first" + : "cache-then-cw"; const isNumeric = typeof identifier === "number" || /^\d+$/.test(String(identifier)); @@ -81,11 +192,22 @@ export const opportunities = { include: { company: true }, }); - const activities = await buildActivities(updated.cwOpportunityId); + const ttlMs = + computeCacheTTL({ + closedFlag: updated.closedFlag, + closedDate: updated.closedDate, + expectedCloseDate: updated.expectedCloseDate, + lastUpdated: updated.cwLastUpdated, + }) ?? undefined; + + const activities = await buildActivities(updated.cwOpportunityId, { + strategy, + ttlMs, + }); return new OpportunityController(updated, { company: updated.company - ? await buildCompanyController(updated.company) + ? await buildCompanyController(updated.company, { strategy, ttlMs }) : undefined, customFields: cwData.customFields ?? [], activities, @@ -95,6 +217,11 @@ export const opportunities = { /** * Fetch All Opportunities (Paginated) * + * Uses the **cache-only** strategy: Redis → bare DB data. + * Activities and company hydration come from the Redis cache if + * available; otherwise the controller is returned with DB-only data. + * The background refresh job is responsible for keeping Redis warm. + * * @param page - Page number (1-based) * @param rpp - Records per page * @param opts - Optional filters @@ -116,15 +243,18 @@ export const opportunities = { }); return Promise.all( - items.map( - async (item) => - new OpportunityController(item, { - company: item.company - ? await buildCompanyController(item.company) - : undefined, - activities: await buildActivities(item.cwOpportunityId), + items.map(async (item) => { + return new OpportunityController(item, { + company: item.company + ? await buildCompanyController(item.company, { + strategy: "cache-only", + }) + : undefined, + activities: await buildActivities(item.cwOpportunityId, { + strategy: "cache-only", }), - ), + }); + }), ); }, @@ -134,6 +264,8 @@ export const opportunities = { * Search opportunities by name, company name, contact name, notes, * sales rep, or status with pagination support. * + * Uses the **cache-only** strategy (same as `fetchPages`). + * * @param query - Search query string * @param page - Page number (1-based) * @param rpp - Records per page @@ -174,15 +306,18 @@ export const opportunities = { }); return Promise.all( - items.map( - async (item) => - new OpportunityController(item, { - company: item.company - ? await buildCompanyController(item.company) - : undefined, - activities: await buildActivities(item.cwOpportunityId), + items.map(async (item) => { + return new OpportunityController(item, { + company: item.company + ? await buildCompanyController(item.company, { + strategy: "cache-only", + }) + : undefined, + activities: await buildActivities(item.cwOpportunityId, { + strategy: "cache-only", }), - ), + }); + }), ); }, @@ -240,6 +375,8 @@ export const opportunities = { * * Fetch all opportunities for a company by its internal company ID. * + * Uses the **cache-only** strategy (same as `fetchPages`). + * * @param companyId - The internal company ID * @param opts - Optional filters * @returns {Promise} @@ -258,15 +395,18 @@ export const opportunities = { }); return Promise.all( - items.map( - async (item) => - new OpportunityController(item, { - company: item.company - ? await buildCompanyController(item.company) - : undefined, - activities: await buildActivities(item.cwOpportunityId), + items.map(async (item) => { + return new OpportunityController(item, { + company: item.company + ? await buildCompanyController(item.company, { + strategy: "cache-only", + }) + : undefined, + activities: await buildActivities(item.cwOpportunityId, { + strategy: "cache-only", }), - ), + }); + }), ); }, }; diff --git a/src/modules/algorithms/computeCacheTTL.ts b/src/modules/algorithms/computeCacheTTL.ts new file mode 100644 index 0000000..636dcf4 --- /dev/null +++ b/src/modules/algorithms/computeCacheTTL.ts @@ -0,0 +1,166 @@ +/** + * @module computeCacheTTL + * + * Adaptive Cache TTL Algorithm + * ============================ + * + * Determines how long a cached record should live before it must be + * re-fetched from the upstream source (e.g. ConnectWise API). + * + * The algorithm prioritises freshness for records that are actively + * being worked on, while avoiding unnecessary API calls for stale or + * inactive data. + * + * ## Spec + * + * | # | Condition | TTL (ms) | TTL (human) | Rationale | + * |---|------------------------------------------------------------------|----------|-------------|--------------------------------------------------------------------| + * | 1 | `closedFlag` is `true` | `null` | Do not cache| Closed records are rarely accessed; caching wastes memory. | + * | 2 | `expectedCloseDate` OR `lastUpdated` is within the last **5 days**| 30 000 | 30 seconds | High-activity window — data changes frequently and must stay fresh.| + * | 3 | `expectedCloseDate` OR `lastUpdated` is within the last **14 days**| 60 000 | 60 seconds | Moderate activity — still relevant, but changes less often. | + * | 4 | Everything else (older than 14 days) | 900 000 | 15 minutes | Low activity — safe to serve from cache for longer. | + * + * ## Evaluation order + * + * Rules are evaluated **top-to-bottom**; the first matching rule wins. + * Rule 2 (5-day window) is a subset of Rule 3 (14-day window), so it + * must be checked first. + * + * ## Inputs + * + * | Field | Type | Description | + * |--------------------|------------------|--------------------------------------------------------------------| + * | `closedFlag` | `boolean` | Whether the record is closed / inactive. | + * | `expectedCloseDate`| `Date \| null` | The projected close date (future-looking relevance signal). | + * | `lastUpdated` | `Date \| null` | The last time the upstream record was modified (backward-looking). | + * | `now` | `Date` (optional)| Override for the current timestamp; defaults to `new Date()`. | + * + * ## Output + * + * Returns `number | null`: + * - A positive integer representing the TTL in **milliseconds**, or + * - `null` when the record should **not** be cached at all. + * + * ## Usage + * + * ```ts + * import { computeCacheTTL } from "../modules/algorithms/computeCacheTTL"; + * + * const ttl = computeCacheTTL({ + * closedFlag: opportunity.closedFlag, + * expectedCloseDate: opportunity.expectedCloseDate, + * lastUpdated: opportunity.cwLastUpdated, + * }); + * + * if (ttl !== null) { + * await redis.set(key, serialised, "PX", ttl); + * } + * ``` + */ + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** 30 seconds – TTL for high-activity records (within 5 days). */ +export const TTL_HIGH_ACTIVITY = 30_000; + +/** 60 seconds – TTL for moderate-activity records (within 14 days). */ +export const TTL_MODERATE_ACTIVITY = 60_000; + +/** 15 minutes – TTL for low-activity / stale records. */ +export const TTL_LOW_ACTIVITY = 900_000; + +/** 30 days in milliseconds. */ +const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; + +/** 5 days in milliseconds. */ +const FIVE_DAYS_MS = 5 * 24 * 60 * 60 * 1000; + +/** 14 days in milliseconds. */ +const FOURTEEN_DAYS_MS = 14 * 24 * 60 * 60 * 1000; + +// --------------------------------------------------------------------------- +// Input type +// --------------------------------------------------------------------------- + +export interface CacheTTLInput { + /** Whether the record is closed / inactive. */ + closedFlag: boolean; + /** When the record was closed — used for recently-closed caching (within 30 days). */ + closedDate: Date | null; + /** The projected close date — serves as a forward-looking relevance signal. */ + expectedCloseDate: Date | null; + /** The date the upstream record was last modified — backward-looking signal. */ + lastUpdated: Date | null; + /** + * Override for the current timestamp. + * Useful for deterministic testing. Defaults to `new Date()`. + */ + now?: Date; +} + +// --------------------------------------------------------------------------- +// Algorithm +// --------------------------------------------------------------------------- + +/** + * Compute the cache TTL for a record based on its activity signals. + * + * @param input - The record's activity signals. See {@link CacheTTLInput}. + * @returns The TTL in milliseconds, or `null` if the record should not be cached. + * + * @see Module-level JSDoc for the full spec table and evaluation rules. + */ +export function computeCacheTTL(input: CacheTTLInput): number | null { + const { + closedFlag, + closedDate, + expectedCloseDate, + lastUpdated, + now = new Date(), + } = input; + + const nowMs = now.getTime(); + + /** + * Check whether a date falls within a window around `now`. + * + * "Within" means the date is between `now - windowMs` and `now + windowMs`, + * allowing both past updates and future-scheduled dates to qualify. + */ + const isWithinWindow = (date: Date | null, windowMs: number): boolean => { + if (!date) return false; + const diff = Math.abs(nowMs - date.getTime()); + return diff <= windowMs; + }; + + // Rule 1 — Closed records + if (closedFlag) { + // Rule 1b — Recently closed (within 30 days) → cache at low-activity TTL + if (isWithinWindow(closedDate, THIRTY_DAYS_MS)) { + return TTL_LOW_ACTIVITY; + } + // Rule 1a — Closed longer than 30 days → do not cache + return null; + } + + // Rule 2 — High activity (5 days) + if ( + isWithinWindow(expectedCloseDate, FIVE_DAYS_MS) || + isWithinWindow(lastUpdated, FIVE_DAYS_MS) + ) { + return TTL_HIGH_ACTIVITY; + } + + // Rule 3 — Moderate activity (14 days) + if ( + isWithinWindow(expectedCloseDate, FOURTEEN_DAYS_MS) || + isWithinWindow(lastUpdated, FOURTEEN_DAYS_MS) + ) { + return TTL_MODERATE_ACTIVITY; + } + + // Rule 4 — Low activity / stale + return TTL_LOW_ACTIVITY; +} diff --git a/src/modules/cache/opportunityCache.ts b/src/modules/cache/opportunityCache.ts new file mode 100644 index 0000000..503ad39 --- /dev/null +++ b/src/modules/cache/opportunityCache.ts @@ -0,0 +1,257 @@ +/** + * @module opportunityCache + * + * Redis-backed cache for expensive ConnectWise API data associated + * with opportunities. + * + * ## What is cached + * + * Each non-closed opportunity may have two cached payloads keyed by + * its `cwOpportunityId`: + * + * - **Activities** (`opp:activities:{cwOpportunityId}`) — the raw + * `CWActivity[]` array fetched from `activityCw.fetchByOpportunity()`. + * - **Company CW data** (`opp:company-cw:{cw_CompanyId}`) — the hydrated + * company / contacts blob set by `CompanyController.hydrateCwData()`. + * + * TTLs are computed dynamically via {@link computeCacheTTL} so hot + * opportunities refresh every 30 s while stale ones live for 15 min. + * + * ## Background refresh + * + * {@link refreshOpportunityCache} is designed to be called on a + * 30-second interval from `src/index.ts`. It scans all non-closed + * DB opportunities, checks which cache keys have expired, and + * re-fetches only those from ConnectWise. + */ + +import { prisma, redis } from "../../constants"; +import { activityCw } from "../cw-utils/activities/activities"; +import { computeCacheTTL } from "../algorithms/computeCacheTTL"; +import { connectWiseApi } from "../../constants"; +import { fetchCwCompanyById } from "../cw-utils/fetchCompany"; +import { events } from "../globalEvents"; + +// --------------------------------------------------------------------------- +// Key helpers +// --------------------------------------------------------------------------- + +const ACTIVITY_PREFIX = "opp:activities:"; +const COMPANY_CW_PREFIX = "opp:company-cw:"; + +/** Redis key for cached activities by CW opportunity ID. */ +export const activityCacheKey = (cwOppId: number) => + `${ACTIVITY_PREFIX}${cwOppId}`; + +/** Redis key for cached company CW hydration data by CW company ID. */ +export const companyCwCacheKey = (cwCompanyId: number) => + `${COMPANY_CW_PREFIX}${cwCompanyId}`; + +// --------------------------------------------------------------------------- +// Read helpers +// --------------------------------------------------------------------------- + +/** + * Retrieve cached CW activities for an opportunity. + * + * @returns The parsed `CWActivity[]` or `null` on cache miss. + */ +export async function getCachedActivities( + cwOpportunityId: number, +): Promise { + const raw = await redis.get(activityCacheKey(cwOpportunityId)); + if (!raw) return null; + try { + return JSON.parse(raw); + } catch { + return null; + } +} + +/** + * Retrieve cached company CW hydration data. + * + * @returns `{ company, defaultContact, allContacts }` or `null` on cache miss. + */ +export async function getCachedCompanyCwData( + cwCompanyId: number, +): Promise<{ company: any; defaultContact: any; allContacts: any[] } | null> { + const raw = await redis.get(companyCwCacheKey(cwCompanyId)); + if (!raw) return null; + try { + return JSON.parse(raw); + } catch { + return null; + } +} + +// --------------------------------------------------------------------------- +// Write helpers +// --------------------------------------------------------------------------- + +/** + * Fetch activities from CW and cache them with the appropriate TTL. + * + * @returns The raw `CWActivity[]` collection (as plain array). + */ +export async function fetchAndCacheActivities( + cwOpportunityId: number, + ttlMs: number, +): Promise { + const collection = await activityCw.fetchByOpportunity(cwOpportunityId); + const arr = collection.map((item) => item); + await redis.set( + activityCacheKey(cwOpportunityId), + JSON.stringify(arr), + "PX", + ttlMs, + ); + return arr; +} + +/** + * Fetch company CW data (company, contacts) and cache with the given TTL. + * + * @returns The hydration blob or `null` if the company doesn't exist in CW. + */ +export async function fetchAndCacheCompanyCwData( + cwCompanyId: number, + ttlMs: number, +): Promise<{ company: any; defaultContact: any; allContacts: any[] } | null> { + const cwCompany = await fetchCwCompanyById(cwCompanyId); + if (!cwCompany) return null; + + const contactHref = cwCompany.defaultContact?._info?.contact_href; + const defaultContactData = contactHref + ? await connectWiseApi.get(contactHref) + : undefined; + + const allContactsData = await connectWiseApi.get( + `${cwCompany._info.contacts_href}&pageSize=1000`, + ); + + const blob = { + company: cwCompany, + defaultContact: defaultContactData?.data ?? null, + allContacts: allContactsData.data, + }; + + await redis.set( + companyCwCacheKey(cwCompanyId), + JSON.stringify(blob), + "PX", + ttlMs, + ); + + return blob; +} + +// --------------------------------------------------------------------------- +// Background refresh +// --------------------------------------------------------------------------- + +/** + * Refresh the opportunity cache. + * + * Scans all non-closed opportunities in the database, computes a TTL for each, + * checks whether the cache key still exists, and re-fetches from ConnectWise + * only for entries that have expired. + * + * Designed to be called every **30 seconds** from the process entry point. + */ +export async function refreshOpportunityCache(): Promise { + // Include non-closed AND recently-closed (within 30 days) opportunities + const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); + + const opportunities = await prisma.opportunity.findMany({ + where: { + OR: [ + { closedFlag: false }, + { closedFlag: true, closedDate: { gte: thirtyDaysAgo } }, + ], + }, + select: { + cwOpportunityId: true, + closedFlag: true, + closedDate: true, + expectedCloseDate: true, + cwLastUpdated: true, + company: { select: { cw_CompanyId: true } }, + }, + }); + + events.emit("cache:opportunities:refresh:started", { + totalOpportunities: opportunities.length, + }); + + let activitiesRefreshed = 0; + let companiesRefreshed = 0; + let skipped = 0; + + // Batch-check which activity keys already exist via a pipeline + const pipeline = redis.pipeline(); + for (const opp of opportunities) { + pipeline.exists(activityCacheKey(opp.cwOpportunityId)); + } + const existsResults = await pipeline.exec(); + + const refreshTasks: Promise[] = []; + + for (let i = 0; i < opportunities.length; i++) { + const opp = opportunities[i]!; + + const ttl = computeCacheTTL({ + closedFlag: opp.closedFlag, + closedDate: opp.closedDate, + expectedCloseDate: opp.expectedCloseDate, + lastUpdated: opp.cwLastUpdated, + }); + + // Skip closed (ttl === null) — should not happen because of the query filter, + // but guard anyway. + if (ttl === null) { + skipped++; + continue; + } + + // existsResults entries are [error, result] tuples + const activityExists = existsResults?.[i]?.[1] === 1; + + if (!activityExists) { + refreshTasks.push( + fetchAndCacheActivities(opp.cwOpportunityId, ttl).then(() => { + activitiesRefreshed++; + }), + ); + } + + // Also refresh company CW data if the key is missing + if (opp.company?.cw_CompanyId) { + const cwCompanyId = opp.company.cw_CompanyId; + refreshTasks.push( + (async () => { + const companyExists = await redis.exists( + companyCwCacheKey(cwCompanyId), + ); + if (!companyExists) { + await fetchAndCacheCompanyCwData(cwCompanyId, ttl); + companiesRefreshed++; + } + })(), + ); + } + } + + // Run all refresh tasks concurrently with bounded concurrency + const CONCURRENCY = 10; + for (let i = 0; i < refreshTasks.length; i += CONCURRENCY) { + await Promise.allSettled(refreshTasks.slice(i, i + CONCURRENCY)); + } + + events.emit("cache:opportunities:refresh:completed", { + totalOpportunities: opportunities.length, + activitiesRefreshed, + companiesRefreshed, + skipped, + }); +} diff --git a/src/modules/globalEvents.ts b/src/modules/globalEvents.ts index e610b63..7353f18 100644 --- a/src/modules/globalEvents.ts +++ b/src/modules/globalEvents.ts @@ -178,6 +178,18 @@ interface EventTypes { staleCount: number; }) => void; + // Cache Events + "cache:opportunities:refresh:started": (data: { + totalOpportunities: number; + }) => void; + "cache:opportunities:refresh:completed": (data: { + totalOpportunities: number; + activitiesRefreshed: number; + companiesRefreshed: number; + skipped: number; + }) => void; + "cache:opportunities:refresh:error": (data: { error: unknown }) => void; + // ConnectWise User Defined Fields Events "cw:udf:refresh:started": () => void; "cw:udf:refresh:completed": (data: { count: number }) => void; diff --git a/src/types/QuoteStatuses.ts b/src/types/QuoteStatuses.ts index 339fbdc..f436dce 100644 --- a/src/types/QuoteStatuses.ts +++ b/src/types/QuoteStatuses.ts @@ -135,6 +135,7 @@ export const QUOTE_STATUSES: QuoteStatus[] = [ 48, // PRE2413. Follow-Up Extended 52, // PRE2489. Overdue 55, // PRE24_70. Quote Sent - Sell + 57, // 04. Confirmed Quote ], }, diff --git a/tests/unit/computeCacheTTL.test.ts b/tests/unit/computeCacheTTL.test.ts new file mode 100644 index 0000000..8099998 --- /dev/null +++ b/tests/unit/computeCacheTTL.test.ts @@ -0,0 +1,477 @@ +import { describe, test, expect } from "bun:test"; +import { + computeCacheTTL, + TTL_HIGH_ACTIVITY, + TTL_MODERATE_ACTIVITY, + TTL_LOW_ACTIVITY, +} from "../../src/modules/algorithms/computeCacheTTL"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Fixed reference point so tests are deterministic. */ +const NOW = new Date("2026-03-02T12:00:00Z"); + +/** Return a Date offset from NOW by `days` (negative = past, positive = future). */ +const daysFromNow = (days: number): Date => + new Date(NOW.getTime() + days * 24 * 60 * 60 * 1000); + +// --------------------------------------------------------------------------- +// Rule 1a — Closed records older than 30 days should not be cached +// --------------------------------------------------------------------------- +describe("computeCacheTTL — Rule 1a: Closed records (>30 days)", () => { + test("returns null when closedFlag is true and closedDate is null", () => { + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: null, + expectedCloseDate: null, + lastUpdated: null, + now: NOW, + }), + ).toBeNull(); + }); + + test("returns null when closedFlag is true and closedDate is 60 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: daysFromNow(-60), + expectedCloseDate: daysFromNow(-1), + lastUpdated: daysFromNow(-1), + now: NOW, + }), + ).toBeNull(); + }); + + test("returns null when closedFlag is true and closedDate is 31 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: new Date(NOW.getTime() - 31 * 24 * 60 * 60 * 1000), + expectedCloseDate: daysFromNow(2), + lastUpdated: null, + now: NOW, + }), + ).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// Rule 1b — Recently closed (within 30 days) → 15 minutes +// --------------------------------------------------------------------------- +describe("computeCacheTTL — Rule 1b: Recently closed (≤30 days)", () => { + test("returns 15min when closed 1 day ago", () => { + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: daysFromNow(-1), + expectedCloseDate: null, + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); + + test("returns 15min when closed 15 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: daysFromNow(-15), + expectedCloseDate: null, + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); + + test("returns 15min when closed exactly 30 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: daysFromNow(-30), + expectedCloseDate: null, + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); + + test("returns 15min when closed today even with recent activity dates", () => { + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: NOW, + expectedCloseDate: daysFromNow(-1), + lastUpdated: NOW, + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); + + test("just past 30-day boundary returns null", () => { + const justPast30Days = new Date( + NOW.getTime() - 30 * 24 * 60 * 60 * 1000 - 1, + ); + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: justPast30Days, + expectedCloseDate: null, + lastUpdated: null, + now: NOW, + }), + ).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// Rule 2 — High activity (within 5 days) → 30 seconds +// --------------------------------------------------------------------------- +describe("computeCacheTTL — Rule 2: High activity (≤5 days)", () => { + test("returns 30s when lastUpdated is today", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: NOW, + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); + + test("returns 30s when lastUpdated is 3 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: daysFromNow(-3), + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); + + test("returns 30s when lastUpdated is exactly 5 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: daysFromNow(-5), + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); + + test("returns 30s when expectedCloseDate is 2 days in the future", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(2), + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); + + test("returns 30s when expectedCloseDate is 5 days in the future", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(5), + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); + + test("returns 30s when expectedCloseDate is 4 days ago (recently passed)", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(-4), + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); + + test("returns 30s when either date is within 5 days (lastUpdated wins)", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(-30), + lastUpdated: daysFromNow(-2), + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); + + test("returns 30s when either date is within 5 days (expectedCloseDate wins)", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(3), + lastUpdated: daysFromNow(-30), + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); +}); + +// --------------------------------------------------------------------------- +// Rule 3 — Moderate activity (within 14 days but > 5 days) → 60 seconds +// --------------------------------------------------------------------------- +describe("computeCacheTTL — Rule 3: Moderate activity (6–14 days)", () => { + test("returns 60s when lastUpdated is 6 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: daysFromNow(-6), + now: NOW, + }), + ).toBe(TTL_MODERATE_ACTIVITY); + }); + + test("returns 60s when lastUpdated is 10 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: daysFromNow(-10), + now: NOW, + }), + ).toBe(TTL_MODERATE_ACTIVITY); + }); + + test("returns 60s when lastUpdated is exactly 14 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: daysFromNow(-14), + now: NOW, + }), + ).toBe(TTL_MODERATE_ACTIVITY); + }); + + test("returns 60s when expectedCloseDate is 8 days in the future", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(8), + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_MODERATE_ACTIVITY); + }); + + test("returns 60s when expectedCloseDate is 14 days in the future", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(14), + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_MODERATE_ACTIVITY); + }); + + test("returns 60s when expectedCloseDate is 12 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(-12), + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_MODERATE_ACTIVITY); + }); +}); + +// --------------------------------------------------------------------------- +// Rule 4 — Low activity (older than 14 days) → 15 minutes +// --------------------------------------------------------------------------- +describe("computeCacheTTL — Rule 4: Low activity (>14 days)", () => { + test("returns 15min when lastUpdated is 15 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: daysFromNow(-15), + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); + + test("returns 15min when lastUpdated is 60 days ago", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: daysFromNow(-60), + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); + + test("returns 15min when expectedCloseDate is 20 days in the future", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(20), + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); + + test("returns 15min when both dates are null", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: null, + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); + + test("returns 15min when both dates are far in the past", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(-100), + lastUpdated: daysFromNow(-90), + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); +}); + +// --------------------------------------------------------------------------- +// Edge cases +// --------------------------------------------------------------------------- +describe("computeCacheTTL — edge cases", () => { + test("defaults `now` to current time when omitted", () => { + // Open, no dates → should return LOW_ACTIVITY (15min) + const result = computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: null, + }); + expect(result).toBe(TTL_LOW_ACTIVITY); + }); + + test("5-day boundary is inclusive", () => { + // Exactly 5 days should match high activity + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: daysFromNow(-5), + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); + + test("just past 5-day boundary falls to moderate", () => { + // 5 days + 1 millisecond past → moderate + const justPast5Days = new Date(NOW.getTime() - 5 * 24 * 60 * 60 * 1000 - 1); + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: justPast5Days, + now: NOW, + }), + ).toBe(TTL_MODERATE_ACTIVITY); + }); + + test("14-day boundary is inclusive", () => { + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: daysFromNow(-14), + now: NOW, + }), + ).toBe(TTL_MODERATE_ACTIVITY); + }); + + test("just past 14-day boundary falls to low activity", () => { + const justPast14Days = new Date( + NOW.getTime() - 14 * 24 * 60 * 60 * 1000 - 1, + ); + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: null, + lastUpdated: justPast14Days, + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); + + test("higher-priority rule wins when both dates span different tiers", () => { + // expectedCloseDate in 5-day window, lastUpdated in 14-day window → 30s + expect( + computeCacheTTL({ + closedFlag: false, + closedDate: null, + expectedCloseDate: daysFromNow(3), + lastUpdated: daysFromNow(-10), + now: NOW, + }), + ).toBe(TTL_HIGH_ACTIVITY); + }); + + test("closed >30 days always returns null regardless of other dates", () => { + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: daysFromNow(-60), + expectedCloseDate: NOW, + lastUpdated: NOW, + now: NOW, + }), + ).toBeNull(); + }); + + test("recently closed always returns 15min regardless of activity dates", () => { + expect( + computeCacheTTL({ + closedFlag: true, + closedDate: daysFromNow(-5), + expectedCloseDate: NOW, + lastUpdated: NOW, + now: NOW, + }), + ).toBe(TTL_LOW_ACTIVITY); + }); +});