feat: add CW callback route and optimize cache refresh workflows
This commit is contained in:
@@ -0,0 +1,164 @@
|
||||
import { createRoute } from "../../modules/api-utils/createRoute";
|
||||
import { apiResponse } from "../../modules/api-utils/apiResponse";
|
||||
import { ContentfulStatusCode } from "hono/utils/http-status";
|
||||
import { z } from "zod";
|
||||
import GenericError from "../../Errors/GenericError";
|
||||
|
||||
type ParsedJson = Record<string, unknown> | unknown[];
|
||||
|
||||
const callbackResource = z.enum([
|
||||
"opportunity",
|
||||
"ticket",
|
||||
"company",
|
||||
"activity",
|
||||
]);
|
||||
|
||||
const safeParseJson = (value: string): ParsedJson | null => {
|
||||
try {
|
||||
const parsed = JSON.parse(value);
|
||||
const isObject = typeof parsed === "object" && parsed !== null;
|
||||
|
||||
return isObject ? (parsed as ParsedJson) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const asObject = (value: ParsedJson | null): Record<string, unknown> | null => {
|
||||
if (!value) return null;
|
||||
if (Array.isArray(value)) return null;
|
||||
|
||||
return value;
|
||||
};
|
||||
|
||||
const parseJsonStringFields = (
|
||||
value: Record<string, unknown> | null,
|
||||
): Record<string, unknown> | null => {
|
||||
if (!value) return null;
|
||||
|
||||
return Object.entries(value).reduce<Record<string, unknown>>(
|
||||
(acc, [key, current]) => {
|
||||
if (typeof current !== "string") {
|
||||
acc[key] = current;
|
||||
|
||||
return acc;
|
||||
}
|
||||
|
||||
const looksLikeJson = current.startsWith("{") || current.startsWith("[");
|
||||
if (!looksLikeJson) {
|
||||
acc[key] = current;
|
||||
|
||||
return acc;
|
||||
}
|
||||
|
||||
const parsed = safeParseJson(current);
|
||||
acc[key] = parsed ?? current;
|
||||
|
||||
return acc;
|
||||
},
|
||||
{},
|
||||
);
|
||||
};
|
||||
|
||||
const parseEntity = (value: unknown): ParsedJson | null => {
|
||||
if (typeof value === "string") return safeParseJson(value);
|
||||
if (typeof value !== "object" || value === null) return null;
|
||||
|
||||
return value as ParsedJson;
|
||||
};
|
||||
|
||||
const buildSummary = (
|
||||
resource: z.infer<typeof callbackResource>,
|
||||
parsedBody: Record<string, unknown> | null,
|
||||
parsedEntity: Record<string, unknown> | null,
|
||||
) => {
|
||||
if (!parsedBody) return null;
|
||||
|
||||
return {
|
||||
resource,
|
||||
messageId: parsedBody.MessageId ?? null,
|
||||
action: parsedBody.Action ?? null,
|
||||
type: parsedBody.Type ?? null,
|
||||
id: parsedBody.ID ?? null,
|
||||
memberId: parsedBody.MemberId ?? null,
|
||||
entityStatus:
|
||||
parsedEntity?.StatusName ??
|
||||
parsedEntity?.TicketStatus ??
|
||||
parsedEntity?.Status ??
|
||||
null,
|
||||
entitySummary: parsedEntity?.Summary ?? parsedEntity?.CompanyName ?? null,
|
||||
entityUpdatedBy: parsedEntity?.UpdatedBy ?? null,
|
||||
entityLastUpdated:
|
||||
parsedEntity?.LastUpdatedUTC ?? parsedEntity?.LastUpdated ?? null,
|
||||
};
|
||||
};
|
||||
|
||||
const parseHeaders = (headers: Headers): Record<string, string> =>
|
||||
Object.fromEntries(headers.entries());
|
||||
|
||||
const callbackHeaderSummary = (headers: Record<string, string>) => ({
|
||||
contentType: headers["content-type"] ?? null,
|
||||
userAgent: headers["user-agent"] ?? null,
|
||||
host: headers.host ?? null,
|
||||
forwardedFor: headers["x-forwarded-for"] ?? null,
|
||||
callbackId:
|
||||
headers["x-cw-request-id"] ??
|
||||
headers["x-request-id"] ??
|
||||
headers["x-correlation-id"] ??
|
||||
null,
|
||||
});
|
||||
|
||||
/* /v1/cw/callback/:resource */
|
||||
export default createRoute("post", ["/callback/:secret/:resource"], async (c) => {
|
||||
const suppliedSecret = c.req.param("secret");
|
||||
const expectedSecret = process.env.CW_CALLBACK_SECRET;
|
||||
|
||||
if (expectedSecret && suppliedSecret !== expectedSecret) {
|
||||
throw new GenericError({
|
||||
name: "Unauthorized",
|
||||
message: "Invalid callback secret.",
|
||||
cause: "Path secret mismatch",
|
||||
status: 401,
|
||||
});
|
||||
}
|
||||
|
||||
if (!expectedSecret) {
|
||||
console.warn(
|
||||
"[cw-callback] CW_CALLBACK_SECRET is not configured; accepting path secret without verification",
|
||||
);
|
||||
}
|
||||
|
||||
const resource = callbackResource.parse(c.req.param("resource"));
|
||||
const headers = parseHeaders(c.req.raw.headers);
|
||||
const headerSummary = callbackHeaderSummary(headers);
|
||||
const rawBody = await c.req.text();
|
||||
const parsedJson = safeParseJson(rawBody);
|
||||
const parsedBody = asObject(parsedJson);
|
||||
const parsedBodyExpanded = parseJsonStringFields(parsedBody);
|
||||
const parsedEntity = asObject(parseEntity(parsedBodyExpanded?.Entity));
|
||||
const summary = buildSummary(resource, parsedBodyExpanded, parsedEntity);
|
||||
|
||||
const line = [
|
||||
`[cw-callback] resource=${resource}`,
|
||||
`action=${String(summary?.action ?? "-")}`,
|
||||
`type=${String(summary?.type ?? "-")}`,
|
||||
`id=${String(summary?.id ?? "-")}`,
|
||||
`by=${String(summary?.entityUpdatedBy ?? summary?.memberId ?? "-")}`,
|
||||
`requestId=${String(headerSummary.callbackId ?? "-")}`,
|
||||
`status=${String(summary?.entityStatus ?? "-")}`,
|
||||
`summary=${String(summary?.entitySummary ?? "-")}`,
|
||||
].join(" ");
|
||||
console.log(line);
|
||||
|
||||
const response = apiResponse.successful("CW callback received.", {
|
||||
resource,
|
||||
secretValidated: Boolean(expectedSecret),
|
||||
summary,
|
||||
headers,
|
||||
headerSummary,
|
||||
bodyParsed: parsedBodyExpanded,
|
||||
receivedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
return c.json(response, response.status as ContentfulStatusCode);
|
||||
});
|
||||
@@ -0,0 +1,3 @@
|
||||
import { default as callback } from "./callback";
|
||||
|
||||
export { callback };
|
||||
@@ -0,0 +1,7 @@
|
||||
import { Hono } from "hono";
|
||||
import * as cwRoutes from "../cw";
|
||||
|
||||
const cwRouter = new Hono();
|
||||
Object.values(cwRoutes).map((r) => cwRouter.route("/", r));
|
||||
|
||||
export default cwRouter;
|
||||
+169
-25
@@ -4,12 +4,27 @@ import { apiResponse } from "../../../modules/api-utils/apiResponse";
|
||||
import { ContentfulStatusCode } from "hono/utils/http-status";
|
||||
import { authMiddleware } from "../../middleware/authorization";
|
||||
import { processObjectValuePerms } from "../../../modules/permission-utils/processObjectPermissions";
|
||||
import GenericError from "../../../Errors/GenericError";
|
||||
import { prisma } from "../../../constants";
|
||||
import { computeSubResourceCacheTTL } from "../../../modules/algorithms/computeSubResourceCacheTTL";
|
||||
import { computeProductsCacheTTL } from "../../../modules/algorithms/computeProductsCacheTTL";
|
||||
import {
|
||||
getCachedSite,
|
||||
getCachedNotes,
|
||||
getCachedContacts,
|
||||
getCachedProducts,
|
||||
fetchAndCacheNotes,
|
||||
fetchAndCacheContacts,
|
||||
fetchAndCacheProducts,
|
||||
fetchAndCacheSite,
|
||||
} from "../../../modules/cache/opportunityCache";
|
||||
|
||||
/* GET /v1/sales/opportunities/:identifier?include=notes,contacts,products */
|
||||
export default createRoute(
|
||||
"get",
|
||||
["/opportunities/:identifier"],
|
||||
async (c) => {
|
||||
const t0 = performance.now();
|
||||
const identifier = c.req.param("identifier");
|
||||
const includeParam = c.req.query("include") ?? "";
|
||||
const includes = new Set(
|
||||
@@ -19,46 +34,175 @@ export default createRoute(
|
||||
.filter(Boolean),
|
||||
);
|
||||
|
||||
const item = await opportunities.fetchItem(identifier);
|
||||
// ── Quick DB lookup (≈3ms) to get cwOpportunityId for pre-warming ──
|
||||
const isNumeric = /^\d+$/.test(identifier);
|
||||
const dbRecord = await prisma.opportunity.findFirst({
|
||||
where: isNumeric
|
||||
? { cwOpportunityId: Number(identifier) }
|
||||
: { id: identifier },
|
||||
select: {
|
||||
cwOpportunityId: true,
|
||||
companyCwId: true,
|
||||
siteCwId: true,
|
||||
closedFlag: true,
|
||||
closedDate: true,
|
||||
expectedCloseDate: true,
|
||||
cwLastUpdated: true,
|
||||
statusCwId: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Eagerly load site data so toJson() includes full site info
|
||||
await item.fetchSite();
|
||||
if (!dbRecord) {
|
||||
throw new GenericError({
|
||||
message: "Opportunity not found",
|
||||
name: "OpportunityNotFound",
|
||||
cause: `No opportunity exists with identifier '${identifier}'`,
|
||||
status: 404,
|
||||
});
|
||||
}
|
||||
|
||||
// Compute TTLs from DB state
|
||||
const subTtl = computeSubResourceCacheTTL({
|
||||
closedFlag: dbRecord.closedFlag,
|
||||
closedDate: dbRecord.closedDate,
|
||||
expectedCloseDate: dbRecord.expectedCloseDate,
|
||||
lastUpdated: dbRecord.cwLastUpdated,
|
||||
});
|
||||
const prodTtl = computeProductsCacheTTL({
|
||||
closedFlag: dbRecord.closedFlag,
|
||||
closedDate: dbRecord.closedDate,
|
||||
expectedCloseDate: dbRecord.expectedCloseDate,
|
||||
lastUpdated: dbRecord.cwLastUpdated,
|
||||
statusCwId: dbRecord.statusCwId,
|
||||
});
|
||||
|
||||
// ── Pre-warm sub-resources only on cache miss ───────────────────────
|
||||
// Check Redis first — if the background refresh has kept the keys warm,
|
||||
// skip the CW calls entirely. Only fetch-and-cache on a miss.
|
||||
const cwOppId = dbRecord.cwOpportunityId;
|
||||
const _pw0 = performance.now();
|
||||
const _wrapPw = (label: string, p: Promise<any>) =>
|
||||
p
|
||||
.then((r) => {
|
||||
console.log(
|
||||
`[PERF:prewarm] ${label}: ${(performance.now() - _pw0).toFixed(0)}ms`,
|
||||
);
|
||||
return r;
|
||||
})
|
||||
.catch(() => {});
|
||||
|
||||
const prewarmPromises: Promise<any>[] = [];
|
||||
if (dbRecord.companyCwId && dbRecord.siteCwId) {
|
||||
const compId = dbRecord.companyCwId,
|
||||
siteId = dbRecord.siteCwId;
|
||||
prewarmPromises.push(
|
||||
_wrapPw(
|
||||
"site",
|
||||
getCachedSite(compId, siteId).then(
|
||||
(c) => c ?? fetchAndCacheSite(compId, siteId),
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
if (includes.has("notes") && subTtl)
|
||||
prewarmPromises.push(
|
||||
_wrapPw(
|
||||
"notes",
|
||||
getCachedNotes(cwOppId).then(
|
||||
(c) => c ?? fetchAndCacheNotes(cwOppId, subTtl),
|
||||
),
|
||||
),
|
||||
);
|
||||
if (includes.has("contacts") && subTtl)
|
||||
prewarmPromises.push(
|
||||
_wrapPw(
|
||||
"contacts",
|
||||
getCachedContacts(cwOppId).then(
|
||||
(c) => c ?? fetchAndCacheContacts(cwOppId, subTtl),
|
||||
),
|
||||
),
|
||||
);
|
||||
if (includes.has("products") && prodTtl)
|
||||
prewarmPromises.push(
|
||||
_wrapPw(
|
||||
"products",
|
||||
getCachedProducts(cwOppId).then(
|
||||
(c) => c ?? fetchAndCacheProducts(cwOppId, prodTtl),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// fetchItem runs its own CW calls (opp, activities, company) —
|
||||
// these execute concurrently with the sub-resource pre-warming above.
|
||||
const [item] = await Promise.all([
|
||||
opportunities.fetchItem(identifier),
|
||||
...prewarmPromises,
|
||||
]);
|
||||
const t1 = performance.now();
|
||||
console.log(`[PERF] fetchItem + prewarm: ${(t1 - t0).toFixed(0)}ms`);
|
||||
|
||||
// Sub-resources now hit warm Redis cache (near-instant)
|
||||
const _st = performance.now();
|
||||
const _wrapTimed = (label: string, p: Promise<any>) =>
|
||||
p.then((r) => {
|
||||
console.log(
|
||||
`[PERF:sub] ${label}: ${(performance.now() - _st).toFixed(0)}ms`,
|
||||
);
|
||||
return r;
|
||||
});
|
||||
|
||||
const subResourcePromises: Record<string, Promise<any>> = {
|
||||
_site: _wrapTimed("site", item.fetchSite()),
|
||||
};
|
||||
if (includes.has("notes")) {
|
||||
subResourcePromises.notes = _wrapTimed("notes", item.fetchNotes());
|
||||
}
|
||||
if (includes.has("contacts")) {
|
||||
subResourcePromises.contacts = _wrapTimed(
|
||||
"contacts",
|
||||
item.fetchContacts(),
|
||||
);
|
||||
}
|
||||
if (includes.has("products")) {
|
||||
subResourcePromises.products = _wrapTimed(
|
||||
"products",
|
||||
item
|
||||
.fetchProducts()
|
||||
.then((products) => products.map((p) => p.toJson())),
|
||||
);
|
||||
}
|
||||
|
||||
const keys = Object.keys(subResourcePromises);
|
||||
const results = await Promise.all(keys.map((k) => subResourcePromises[k]));
|
||||
const t2 = performance.now();
|
||||
console.log(
|
||||
`[PERF] sub-resources (${keys.join(",")}): ${(t2 - t1).toFixed(0)}ms`,
|
||||
);
|
||||
|
||||
// Apply toJson after site is hydrated (side-effect from fetchSite)
|
||||
const gatedData = await processObjectValuePerms(
|
||||
item.toJson(),
|
||||
"obj.opportunity",
|
||||
c.get("user"),
|
||||
);
|
||||
const t3 = performance.now();
|
||||
console.log(`[PERF] processObjectValuePerms: ${(t3 - t2).toFixed(0)}ms`);
|
||||
|
||||
// Fetch requested sub-resources in parallel
|
||||
const subResourcePromises: Record<string, Promise<any>> = {};
|
||||
if (includes.has("notes")) {
|
||||
subResourcePromises.notes = item.fetchNotes();
|
||||
}
|
||||
if (includes.has("contacts")) {
|
||||
subResourcePromises.contacts = item.fetchContacts();
|
||||
}
|
||||
if (includes.has("products")) {
|
||||
subResourcePromises.products = item
|
||||
.fetchProducts()
|
||||
.then((products) => products.map((p) => p.toJson()));
|
||||
}
|
||||
|
||||
const keys = Object.keys(subResourcePromises);
|
||||
if (keys.length > 0) {
|
||||
const results = await Promise.all(
|
||||
keys.map((k) => subResourcePromises[k]),
|
||||
);
|
||||
keys.forEach((k, i) => {
|
||||
// Attach sub-resources (skip the internal _site key)
|
||||
keys.forEach((k, i) => {
|
||||
if (k !== "_site") {
|
||||
(gatedData as any)[k] = results[i];
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const response = apiResponse.successful(
|
||||
"Opportunity fetched successfully!",
|
||||
gatedData,
|
||||
);
|
||||
|
||||
console.log(
|
||||
`[PERF] total handler: ${(performance.now() - t0).toFixed(0)}ms (includes=${includeParam || "none"})`,
|
||||
);
|
||||
return c.json(response, response.status as ContentfulStatusCode);
|
||||
},
|
||||
authMiddleware({ permissions: ["sales.opportunity.fetch"] }),
|
||||
|
||||
@@ -57,6 +57,7 @@ v1.route("/permissions", require("./routers/permissionRouter").default);
|
||||
v1.route("/unifi", require("./routers/unifiRouter").default);
|
||||
v1.route("/procurement", require("./routers/procurementRouter").default);
|
||||
v1.route("/sales", require("./routers/salesRouter").default);
|
||||
v1.route("/cw", require("./routers/cwRouter").default);
|
||||
app.route("/v1", v1);
|
||||
|
||||
export default app;
|
||||
|
||||
@@ -50,18 +50,21 @@ export class CompanyController {
|
||||
const cwCompany = await fetchCwCompanyById(this.cw_CompanyId);
|
||||
if (!cwCompany) return this;
|
||||
|
||||
const contactHref = cwCompany.defaultContact?._info?.contact_href;
|
||||
const defaultContactData = contactHref
|
||||
? await connectWiseApi.get(contactHref)
|
||||
: undefined;
|
||||
|
||||
const allContactsData = await connectWiseApi.get(
|
||||
`${cwCompany._info.contacts_href}&pageSize=1000`,
|
||||
);
|
||||
|
||||
// Derive default contact from allContacts instead of a separate CW call
|
||||
const defaultContactId = cwCompany.defaultContact?.id;
|
||||
const defaultContactData = defaultContactId
|
||||
? ((allContactsData.data as any[]).find(
|
||||
(c: any) => c.id === defaultContactId,
|
||||
) ?? null)
|
||||
: null;
|
||||
|
||||
this.cw_Data = {
|
||||
company: cwCompany,
|
||||
defaultContact: defaultContactData?.data ?? null,
|
||||
defaultContact: defaultContactData,
|
||||
allContacts: allContactsData.data,
|
||||
};
|
||||
|
||||
|
||||
@@ -15,7 +15,10 @@ import {
|
||||
CWOpportunity,
|
||||
CWOpportunityNote,
|
||||
} from "../modules/cw-utils/opportunities/opportunity.types";
|
||||
import { resolveMember } from "../modules/cw-utils/members/memberCache";
|
||||
import {
|
||||
resolveMember,
|
||||
resolveMembers,
|
||||
} from "../modules/cw-utils/members/memberCache";
|
||||
import { ForecastProductController } from "./ForecastProductController";
|
||||
import GenericError from "../Errors/GenericError";
|
||||
import { computeSubResourceCacheTTL } from "../modules/algorithms/computeSubResourceCacheTTL";
|
||||
@@ -429,18 +432,25 @@ export class OpportunityController {
|
||||
|
||||
/** Serialize raw CW note data into the API response shape. */
|
||||
private async _serializeNotes(notes: any[]) {
|
||||
return Promise.all(
|
||||
notes.map(async (n: any) => ({
|
||||
id: n.id,
|
||||
text: n.text,
|
||||
type: n.type ? { id: n.type.id, name: n.type.name } : null,
|
||||
flagged: n.flagged,
|
||||
dateEntered: n._info?.lastUpdated
|
||||
? new Date(n._info.lastUpdated)
|
||||
: null,
|
||||
enteredBy: await resolveMember(n.enteredBy),
|
||||
})),
|
||||
);
|
||||
// Batch-resolve all member identifiers in a single DB query
|
||||
const identifiers = notes
|
||||
.map((n: any) => n.enteredBy as string)
|
||||
.filter(Boolean);
|
||||
const memberMap = await resolveMembers(identifiers);
|
||||
|
||||
return notes.map((n: any) => ({
|
||||
id: n.id,
|
||||
text: n.text,
|
||||
type: n.type ? { id: n.type.id, name: n.type.name } : null,
|
||||
flagged: n.flagged,
|
||||
dateEntered: n._info?.lastUpdated ? new Date(n._info.lastUpdated) : null,
|
||||
enteredBy: memberMap.get(n.enteredBy) ?? {
|
||||
id: null,
|
||||
identifier: n.enteredBy,
|
||||
name: n.enteredBy,
|
||||
cwMemberId: null,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+24
-11
@@ -12,6 +12,7 @@ import { unifiSites } from "./managers/unifiSites";
|
||||
import { refreshCompanies } from "./modules/cw-utils/refreshCompanies";
|
||||
import { refreshCatalog } from "./modules/cw-utils/procurement/refreshCatalog";
|
||||
import { refreshInventory } from "./modules/cw-utils/procurement/refreshInventory";
|
||||
import { listenInventoryAdjustments } from "./modules/cw-utils/procurement/listenInventoryAdjustments";
|
||||
import { refreshOpportunities } from "./modules/cw-utils/opportunities/refreshOpportunities";
|
||||
import { refreshOpportunityCache } from "./modules/cache/opportunityCache";
|
||||
import { refreshCwIdentifiers } from "./modules/cw-utils/members/refreshCwIdentifiers";
|
||||
@@ -105,25 +106,37 @@ setInterval(() => {
|
||||
);
|
||||
}, 60 * 1000);
|
||||
|
||||
// Refresh the internal catalog every minute
|
||||
// Refresh the internal catalog every 30 minutes
|
||||
await safeStartup("refreshCatalog", refreshCatalog);
|
||||
setInterval(() => {
|
||||
return refreshCatalog().catch((err) =>
|
||||
console.error(`[interval] refreshCatalog failed: ${briefErr(err)}`),
|
||||
);
|
||||
}, 60 * 1000);
|
||||
setInterval(
|
||||
() => {
|
||||
return refreshCatalog().catch((err) =>
|
||||
console.error(`[interval] refreshCatalog failed: ${briefErr(err)}`),
|
||||
);
|
||||
},
|
||||
30 * 60 * 1000,
|
||||
);
|
||||
|
||||
// Refresh inventory on hand every 2 minutes
|
||||
await safeStartup("refreshInventory", refreshInventory);
|
||||
// Fallback full inventory sweep every 6 hours (listener handles real-time deltas)
|
||||
setInterval(
|
||||
() => {
|
||||
return refreshInventory().catch((err) =>
|
||||
console.error(`[interval] refreshInventory failed: ${briefErr(err)}`),
|
||||
);
|
||||
},
|
||||
2 * 60 * 1000,
|
||||
6 * 60 * 60 * 1000,
|
||||
);
|
||||
|
||||
// Listen for procurement adjustment changes and sync changed products to DB + cache
|
||||
await safeStartup("listenInventoryAdjustments", listenInventoryAdjustments);
|
||||
setInterval(() => {
|
||||
return listenInventoryAdjustments().catch((err) =>
|
||||
console.error(
|
||||
`[interval] listenInventoryAdjustments failed: ${briefErr(err)}`,
|
||||
),
|
||||
);
|
||||
}, 60 * 1000);
|
||||
|
||||
// Refresh opportunities every minute
|
||||
await safeStartup("refreshOpportunities", refreshOpportunities);
|
||||
setInterval(() => {
|
||||
@@ -132,7 +145,7 @@ setInterval(() => {
|
||||
);
|
||||
}, 60 * 1000);
|
||||
|
||||
// Refresh opportunity CW cache every 30 seconds (activities + company hydration)
|
||||
// Refresh opportunity CW cache every 20 minutes (activities + company hydration)
|
||||
// NOTE: Do NOT await — register the interval immediately so the cache refresh
|
||||
// is never blocked by a slow/stuck startup task above.
|
||||
safeStartup("refreshOpportunityCache", refreshOpportunityCache);
|
||||
@@ -142,7 +155,7 @@ setInterval(() => {
|
||||
`[interval] refreshOpportunityCache failed: ${briefErr(err)}`,
|
||||
);
|
||||
});
|
||||
}, 30 * 1000);
|
||||
}, 20 * 60 * 1000);
|
||||
|
||||
// Refresh User Defined Fields every 5 minutes
|
||||
await safeStartup("refreshUDFs", () => userDefinedFieldsCw.refresh());
|
||||
|
||||
@@ -42,16 +42,22 @@ async function buildCompanyController(
|
||||
ttlMs?: number;
|
||||
},
|
||||
): Promise<CompanyController> {
|
||||
const _ct0 = performance.now();
|
||||
const strategy = opts?.strategy ?? "cache-then-cw";
|
||||
const ctrl = new CompanyController(company);
|
||||
|
||||
// ── cw-first: always fetch from CW ──────────────────────────────────
|
||||
// ── cw-first: always fetch from CW (and cache the result) ──────────
|
||||
if (strategy === "cw-first") {
|
||||
await ctrl.hydrateCwData();
|
||||
if (ctrl.cw_Data && opts?.ttlMs) {
|
||||
await fetchAndCacheCompanyCwData(company.cw_CompanyId, opts.ttlMs).catch(
|
||||
() => {},
|
||||
);
|
||||
const blob = opts?.ttlMs
|
||||
? await fetchAndCacheCompanyCwData(
|
||||
company.cw_CompanyId,
|
||||
opts.ttlMs,
|
||||
).catch(() => null)
|
||||
: null;
|
||||
if (blob) {
|
||||
ctrl.cw_Data = blob;
|
||||
} else {
|
||||
await ctrl.hydrateCwData();
|
||||
}
|
||||
return ctrl;
|
||||
}
|
||||
@@ -66,14 +72,20 @@ async function buildCompanyController(
|
||||
// cache-only stops here — return the bare DB-backed controller
|
||||
if (strategy === "cache-only") return ctrl;
|
||||
|
||||
// cache-then-cw: cache miss — fall through to CW
|
||||
await ctrl.hydrateCwData();
|
||||
if (ctrl.cw_Data && opts?.ttlMs) {
|
||||
await fetchAndCacheCompanyCwData(company.cw_CompanyId, opts.ttlMs).catch(
|
||||
() => {},
|
||||
);
|
||||
// cache-then-cw: cache miss — fetch from CW once and cache in one pass
|
||||
if (opts?.ttlMs) {
|
||||
const blob = await fetchAndCacheCompanyCwData(
|
||||
company.cw_CompanyId,
|
||||
opts.ttlMs,
|
||||
).catch(() => null);
|
||||
if (blob) ctrl.cw_Data = blob;
|
||||
} else {
|
||||
await ctrl.hydrateCwData();
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[PERF:buildCompany] ${(performance.now() - _ct0).toFixed(0)}ms (strategy=${strategy}, hit=miss)`,
|
||||
);
|
||||
return ctrl;
|
||||
}
|
||||
|
||||
@@ -93,17 +105,14 @@ async function buildActivities(
|
||||
ttlMs?: number;
|
||||
},
|
||||
): Promise<ActivityController[]> {
|
||||
const _at0 = performance.now();
|
||||
const strategy = opts?.strategy ?? "cache-then-cw";
|
||||
|
||||
// ── cw-first: always fetch from CW ──────────────────────────────────
|
||||
// ── cw-first: always fetch from CW (and cache the result) ──────────
|
||||
if (strategy === "cw-first") {
|
||||
const collection = await activityCw.fetchByOpportunity(cwOpportunityId);
|
||||
const arr = collection.map((item) => item);
|
||||
if (opts?.ttlMs) {
|
||||
await fetchAndCacheActivities(cwOpportunityId, opts.ttlMs).catch(
|
||||
() => {},
|
||||
);
|
||||
}
|
||||
const arr = opts?.ttlMs
|
||||
? await fetchAndCacheActivities(cwOpportunityId, opts.ttlMs)
|
||||
: await activityCw.fetchByOpportunityDirect(cwOpportunityId);
|
||||
return arr.map((item) => new ActivityController(item));
|
||||
}
|
||||
|
||||
@@ -116,12 +125,13 @@ async function buildActivities(
|
||||
// cache-only stops here — return empty (background job will fill it)
|
||||
if (strategy === "cache-only") return [];
|
||||
|
||||
// cache-then-cw: cache miss — fall through to CW
|
||||
const collection = await activityCw.fetchByOpportunity(cwOpportunityId);
|
||||
const arr = collection.map((item) => item);
|
||||
if (opts?.ttlMs) {
|
||||
await fetchAndCacheActivities(cwOpportunityId, opts.ttlMs).catch(() => {});
|
||||
}
|
||||
// cache-then-cw: cache miss — fetch once and cache in one pass
|
||||
const arr = opts?.ttlMs
|
||||
? await fetchAndCacheActivities(cwOpportunityId, opts.ttlMs)
|
||||
: await activityCw.fetchByOpportunityDirect(cwOpportunityId);
|
||||
console.log(
|
||||
`[PERF:buildActivities] ${(performance.now() - _at0).toFixed(0)}ms (strategy=${strategy}, hit=miss, count=${arr.length})`,
|
||||
);
|
||||
return arr.map((item) => new ActivityController(item));
|
||||
}
|
||||
|
||||
@@ -192,6 +202,7 @@ export const opportunities = {
|
||||
identifier: string | number,
|
||||
opts?: { fresh?: boolean },
|
||||
): Promise<OpportunityController> {
|
||||
const _t0 = performance.now();
|
||||
const strategy: "cache-only" | "cache-then-cw" | "cw-first" = opts?.fresh
|
||||
? "cw-first"
|
||||
: "cache-then-cw";
|
||||
@@ -205,6 +216,8 @@ export const opportunities = {
|
||||
: { id: identifier as string },
|
||||
include: { company: true },
|
||||
});
|
||||
const _t1 = performance.now();
|
||||
console.log(`[PERF:fetchItem] DB lookup: ${(_t1 - _t0).toFixed(0)}ms`);
|
||||
|
||||
if (!existing) {
|
||||
throw new GenericError({
|
||||
@@ -232,12 +245,26 @@ export const opportunities = {
|
||||
// Try the Redis cache first
|
||||
cwData = await getCachedOppCwData(existing.cwOpportunityId);
|
||||
}
|
||||
const _t2 = performance.now();
|
||||
console.log(
|
||||
`[PERF:fetchItem] Redis cache check: ${(_t2 - _t1).toFixed(0)}ms (hit=${!!cwData})`,
|
||||
);
|
||||
|
||||
// ── Parallel block: CW opp fetch + activities + company ────────────
|
||||
// Activities and company hydration only need existing.cwOpportunityId
|
||||
// and existing.company — both available from the initial DB lookup —
|
||||
// so they can run concurrently with the CW opp fetch + DB update.
|
||||
|
||||
const cwOppPromise = (async () => {
|
||||
if (cwData) return; // cache hit — nothing to do
|
||||
|
||||
if (!cwData) {
|
||||
// Cache miss or forced fresh — fetch from CW and cache
|
||||
cwData = ttlMs
|
||||
? await fetchAndCacheOppCwData(existing.cwOpportunityId, ttlMs)
|
||||
: await opportunityCw.fetch(existing.cwOpportunityId);
|
||||
const _t2b = performance.now();
|
||||
console.log(
|
||||
`[PERF:fetchItem] CW opp fetch: ${(_t2b - _t2).toFixed(0)}ms`,
|
||||
);
|
||||
|
||||
if (!cwData) {
|
||||
throw new GenericError({
|
||||
@@ -264,15 +291,27 @@ export const opportunities = {
|
||||
data: { ...mapped, companyId },
|
||||
include: { company: true },
|
||||
});
|
||||
}
|
||||
console.log(
|
||||
`[PERF:fetchItem] DB update: ${(performance.now() - _t2b).toFixed(0)}ms`,
|
||||
);
|
||||
})();
|
||||
|
||||
// Hydrate activities and company in parallel
|
||||
const [activities, company] = await Promise.all([
|
||||
buildActivities(record.cwOpportunityId, { strategy, ttlMs }),
|
||||
record.company
|
||||
? buildCompanyController(record.company, { strategy, ttlMs })
|
||||
const _t3 = performance.now();
|
||||
// Hydrate activities and company in parallel with CW opp fetch
|
||||
const [, activities, company] = await Promise.all([
|
||||
cwOppPromise,
|
||||
buildActivities(existing.cwOpportunityId, { strategy, ttlMs }),
|
||||
existing.company
|
||||
? buildCompanyController(existing.company, { strategy, ttlMs })
|
||||
: Promise.resolve(undefined),
|
||||
]);
|
||||
const _t4 = performance.now();
|
||||
console.log(
|
||||
`[PERF:fetchItem] parallel block (cw+activities+company): ${(_t4 - _t3).toFixed(0)}ms`,
|
||||
);
|
||||
console.log(
|
||||
`[PERF:fetchItem] TOTAL: ${(_t4 - _t0).toFixed(0)}ms (strategy=${strategy}, ttl=${ttlMs}ms)`,
|
||||
);
|
||||
|
||||
return new OpportunityController(record, {
|
||||
company,
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
* | # | Condition | TTL (ms) | TTL (human) | Rationale |
|
||||
* |---|------------------------------------------------------------------|----------|-------------|--------------------------------------------------------------------|
|
||||
* | 1 | `closedFlag` is `true` | `null` | Do not cache| Closed records are rarely accessed; caching wastes memory. |
|
||||
* | 2 | `expectedCloseDate` OR `lastUpdated` is within the last **5 days**| 30 000 | 30 seconds | High-activity window — data changes frequently and must stay fresh.|
|
||||
* | 3 | `expectedCloseDate` OR `lastUpdated` is within the last **14 days**| 60 000 | 60 seconds | Moderate activity — still relevant, but changes less often. |
|
||||
* | 2 | `expectedCloseDate` OR `lastUpdated` is within the last **5 days**| 60 000 | 60 seconds | High-activity window — data changes frequently and must stay fresh.|
|
||||
* | 3 | `expectedCloseDate` OR `lastUpdated` is within the last **14 days**| 90 000 | 90 seconds | Moderate activity — still relevant, but changes less often. |
|
||||
* | 4 | Everything else (older than 14 days) | 900 000 | 15 minutes | Low activity — safe to serve from cache for longer. |
|
||||
*
|
||||
* ## Evaluation order
|
||||
@@ -62,11 +62,13 @@
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 30 seconds – TTL for high-activity records (within 5 days). */
|
||||
export const TTL_HIGH_ACTIVITY = 30_000;
|
||||
/** 60 seconds – TTL for high-activity records (within 5 days).
|
||||
* Must exceed the 30-second background refresh interval so the cache
|
||||
* stays warm between cycles. */
|
||||
export const TTL_HIGH_ACTIVITY = 60_000;
|
||||
|
||||
/** 60 seconds – TTL for moderate-activity records (within 14 days). */
|
||||
export const TTL_MODERATE_ACTIVITY = 60_000;
|
||||
/** 90 seconds – TTL for moderate-activity records (within 14 days). */
|
||||
export const TTL_MODERATE_ACTIVITY = 90_000;
|
||||
|
||||
/** 15 minutes – TTL for low-activity / stale records. */
|
||||
export const TTL_LOW_ACTIVITY = 900_000;
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
* | 1 | Status is **Won**, **Lost**, **Pending Won**, or **Pending Lost** | `null` | No cache | Products on terminal / near-terminal opps are static; no need to keep them warm. |
|
||||
* | 2 | Opportunity is **not cacheable** (main cache TTL is `null`) | `null` | No cache | If the opp itself is evicted, sub-resources follow suit. |
|
||||
* | 3 | `lastUpdated` is within the last **3 days** | 15 000 | 15 seconds | Actively-worked deals — products are being edited and need near-real-time freshness. |
|
||||
* | 4 | Everything else | 1 800 000 | 30 minutes | Lazy on-demand cache: fetched when requested, expires after 30 min without refresh. |
|
||||
* | 4 | Everything else | 1 200 000 | 20 minutes | Lazy on-demand cache: fetched when requested, expires after 20 min without refresh. |
|
||||
*
|
||||
* ## Evaluation order
|
||||
*
|
||||
@@ -44,11 +44,13 @@ import { QUOTE_STATUSES } from "../../types/QuoteStatuses";
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 15 seconds — TTL for hot products (opportunity updated within 3 days). */
|
||||
export const PRODUCTS_TTL_HOT = 15_000;
|
||||
/** 45 seconds — TTL for hot products (opportunity updated within 3 days).
|
||||
* Must exceed the 30-second background refresh interval so the cache
|
||||
* stays warm between cycles. */
|
||||
export const PRODUCTS_TTL_HOT = 45_000;
|
||||
|
||||
/** 30 minutes — TTL for on-demand product cache (lazy fallback). */
|
||||
export const PRODUCTS_TTL_LAZY = 1_800_000;
|
||||
/** 20 minutes — TTL for on-demand product cache (lazy fallback). */
|
||||
export const PRODUCTS_TTL_LAZY = 1_200_000;
|
||||
|
||||
/** 3 days in milliseconds. */
|
||||
const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000;
|
||||
|
||||
+25
-16
@@ -310,25 +310,34 @@ export async function fetchAndCacheCompanyCwData(
|
||||
ttlMs: number,
|
||||
): Promise<{ company: any; defaultContact: any; allContacts: any[] } | null> {
|
||||
try {
|
||||
const cwCompany = await fetchCwCompanyById(cwCompanyId);
|
||||
// Fetch company and all-contacts in parallel — the allContacts URL
|
||||
// can be constructed directly without the company response.
|
||||
const [cwCompany, allContactsData] = await Promise.all([
|
||||
fetchCwCompanyById(cwCompanyId),
|
||||
withCwRetry(
|
||||
() =>
|
||||
connectWiseApi.get(
|
||||
`/company/companies/${cwCompanyId}/contacts?pageSize=1000`,
|
||||
),
|
||||
{ label: `company#${cwCompanyId}/allContacts` },
|
||||
),
|
||||
]);
|
||||
|
||||
if (!cwCompany) return null;
|
||||
|
||||
const contactHref = cwCompany.defaultContact?._info?.contact_href;
|
||||
const defaultContactData = contactHref
|
||||
? await withCwRetry(() => connectWiseApi.get(contactHref), {
|
||||
label: `company#${cwCompanyId}/defaultContact`,
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const allContactsData = await withCwRetry(
|
||||
() =>
|
||||
connectWiseApi.get(`${cwCompany._info.contacts_href}&pageSize=1000`),
|
||||
{ label: `company#${cwCompanyId}/allContacts` },
|
||||
);
|
||||
// Default contact: derive from allContacts instead of making an
|
||||
// extra serial CW call. The company object carries the default
|
||||
// contact's ID, so we can pull it from the list we already fetched.
|
||||
const defaultContactId = cwCompany.defaultContact?.id;
|
||||
const defaultContactData = defaultContactId
|
||||
? ((allContactsData.data as any[]).find(
|
||||
(c: any) => c.id === defaultContactId,
|
||||
) ?? null)
|
||||
: null;
|
||||
|
||||
const blob = {
|
||||
company: cwCompany,
|
||||
defaultContact: defaultContactData?.data ?? null,
|
||||
defaultContact: defaultContactData,
|
||||
allContacts: allContactsData.data,
|
||||
};
|
||||
|
||||
@@ -491,11 +500,11 @@ export async function invalidateProductsCache(
|
||||
}
|
||||
|
||||
/**
|
||||
* Site TTL — 30 minutes. Site/address data rarely changes so we cache
|
||||
* Site TTL — 20 minutes. Site/address data rarely changes so we cache
|
||||
* aggressively. The background refresh does NOT proactively warm site keys;
|
||||
* they are populated lazily on the first detail-view request.
|
||||
*/
|
||||
const SITE_TTL_MS = 1_800_000;
|
||||
const SITE_TTL_MS = 1_200_000;
|
||||
|
||||
/**
|
||||
* Fetch a CW company site from ConnectWise and cache the result.
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* CW API Concurrency Limiter
|
||||
*
|
||||
* Limits the number of simultaneous in-flight requests to the ConnectWise
|
||||
* API. CW responds significantly slower under high concurrency (observed
|
||||
* ~3× slower at 9 concurrent vs 5–6 concurrent), so bounding the
|
||||
* parallelism actually reduces total wall-clock time.
|
||||
*
|
||||
* Implemented as an Axios request interceptor that gates on a simple
|
||||
* counting semaphore. When the limit is reached, new requests queue and
|
||||
* resolve in FIFO order as earlier requests complete.
|
||||
*/
|
||||
|
||||
import type { AxiosInstance, InternalAxiosRequestConfig } from "axios";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Semaphore
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class Semaphore {
|
||||
private _current = 0;
|
||||
private _queue: (() => void)[] = [];
|
||||
|
||||
constructor(private _max: number) {}
|
||||
|
||||
/** Acquire a slot — resolves immediately if under the limit, else waits. */
|
||||
acquire(): Promise<void> {
|
||||
if (this._current < this._max) {
|
||||
this._current++;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
this._queue.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
/** Release a slot — wakes the next queued caller, if any. */
|
||||
release(): void {
|
||||
const next = this._queue.shift();
|
||||
if (next) {
|
||||
// Hand the slot directly to the next waiter (don't decrement)
|
||||
next();
|
||||
} else {
|
||||
this._current--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Interceptor attachment
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Attach a concurrency-limiting interceptor to an Axios instance.
|
||||
*
|
||||
* @param api - The Axios instance to limit.
|
||||
* @param max - Maximum concurrent in-flight requests (default: 6).
|
||||
*/
|
||||
export function attachCwConcurrencyLimiter(api: AxiosInstance, max = 6): void {
|
||||
const sem = new Semaphore(max);
|
||||
|
||||
// Request interceptor: wait for a slot before the request fires
|
||||
api.interceptors.request.use(async (config: InternalAxiosRequestConfig) => {
|
||||
await sem.acquire();
|
||||
return config;
|
||||
});
|
||||
|
||||
// Response interceptor: release the slot on success or failure
|
||||
api.interceptors.response.use(
|
||||
(response) => {
|
||||
sem.release();
|
||||
return response;
|
||||
},
|
||||
(error) => {
|
||||
sem.release();
|
||||
return Promise.reject(error);
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -102,3 +102,40 @@ export const resolveMember = async (
|
||||
cwMemberId: cwMember?.id ?? null,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve Multiple CW Identifiers in a Single Batch
|
||||
*
|
||||
* Same as `resolveMember` but batches the DB query so that N identifiers
|
||||
* require only **one** `findMany` instead of N `findFirst` calls.
|
||||
*
|
||||
* @param identifiers - Array of CW member identifiers
|
||||
* @returns Map of identifier → ResolvedMember
|
||||
*/
|
||||
export const resolveMembers = async (
|
||||
identifiers: string[],
|
||||
): Promise<Map<string, ResolvedMember>> => {
|
||||
const unique = [...new Set(identifiers)];
|
||||
|
||||
// Single batched DB query for all identifiers
|
||||
const localUsers = await prisma.user.findMany({
|
||||
where: { cwIdentifier: { in: unique } },
|
||||
select: { id: true, cwIdentifier: true },
|
||||
});
|
||||
const userMap = new Map(localUsers.map((u) => [u.cwIdentifier, u.id]));
|
||||
|
||||
const result = new Map<string, ResolvedMember>();
|
||||
for (const identifier of unique) {
|
||||
const cwMember = memberCache.get(identifier);
|
||||
const name = cwMember
|
||||
? `${cwMember.firstName} ${cwMember.lastName}`.trim() || identifier
|
||||
: identifier;
|
||||
result.set(identifier, {
|
||||
id: userMap.get(identifier) ?? null,
|
||||
identifier,
|
||||
name,
|
||||
cwMemberId: cwMember?.id ?? null,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
@@ -66,10 +66,28 @@ export const catalogCw = {
|
||||
|
||||
return allItems;
|
||||
},
|
||||
fetchByCatalogId: async (cwCatalogId: number): Promise<CatalogItem> => {
|
||||
try {
|
||||
const response = await connectWiseApi.get(
|
||||
`/procurement/catalog/${cwCatalogId}`,
|
||||
);
|
||||
return response.data;
|
||||
} catch {
|
||||
const fallback = await connectWiseApi.get(
|
||||
`/procurement/catalog/items/${cwCatalogId}`,
|
||||
);
|
||||
return fallback.data;
|
||||
}
|
||||
},
|
||||
fetch: async (id: string): Promise<CatalogItem> => {
|
||||
const response = await connectWiseApi.get(
|
||||
`/procurement/catalog/items/${id}`,
|
||||
);
|
||||
return response.data;
|
||||
const numericId = Number(id);
|
||||
if (!Number.isFinite(numericId)) {
|
||||
const response = await connectWiseApi.get(
|
||||
`/procurement/catalog/items/${id}`,
|
||||
);
|
||||
return response.data;
|
||||
}
|
||||
|
||||
return catalogCw.fetchByCatalogId(numericId);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,469 @@
|
||||
import { prisma, redis, connectWiseApi } from "../../../constants";
|
||||
import { withCwRetry } from "../withCwRetry";
|
||||
import { catalogCw } from "./catalog";
|
||||
import { CatalogItem } from "./catalog.types";
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
type TrackedProduct = {
|
||||
cwCatalogId: number;
|
||||
product: string;
|
||||
onHand: string;
|
||||
inventory: string;
|
||||
key: string;
|
||||
};
|
||||
|
||||
type AdjustmentSnapshot = {
|
||||
key: string;
|
||||
trackedRows: TrackedProduct[];
|
||||
signature: string;
|
||||
};
|
||||
|
||||
const ADJUSTMENTS_ENDPOINT = "/procurement/adjustments?pageSize=1000";
|
||||
const CATALOG_ITEM_CACHE_PREFIX = "catalog:item:cw:";
|
||||
const CATALOG_ITEM_CACHE_TTL_SECONDS = 20 * 60;
|
||||
const MAX_SYNC_PER_CYCLE = Number(
|
||||
process.env.CW_ADJUSTMENT_SYNC_MAX_PER_CYCLE ?? "50",
|
||||
);
|
||||
const SYNC_COOLDOWN_MS = Number(
|
||||
process.env.CW_ADJUSTMENT_SYNC_COOLDOWN_MS ?? `${10 * 60 * 1000}`,
|
||||
);
|
||||
|
||||
let previous = new Map<string, AdjustmentSnapshot>();
|
||||
let previousProductState = new Map<number, string>();
|
||||
const lastSyncedAt = new Map<number, number>();
|
||||
let inFlight = false;
|
||||
|
||||
const isObject = (value: unknown): value is JsonObject =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
const toObject = (value: unknown): JsonObject => {
|
||||
if (!isObject(value)) return {};
|
||||
|
||||
return value;
|
||||
};
|
||||
|
||||
const stableStringify = (value: unknown): string => {
|
||||
if (Array.isArray(value)) {
|
||||
const entries = value.map((entry) => stableStringify(entry)).sort();
|
||||
|
||||
return `[${entries.join(",")}]`;
|
||||
}
|
||||
|
||||
if (isObject(value)) {
|
||||
const keys = Object.keys(value).sort();
|
||||
const pairs = keys.map(
|
||||
(key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`,
|
||||
);
|
||||
|
||||
return `{${pairs.join(",")}}`;
|
||||
}
|
||||
|
||||
return JSON.stringify(value);
|
||||
};
|
||||
|
||||
const readPathValue = (obj: JsonObject, path: string): unknown => {
|
||||
const parts = path.split(".");
|
||||
let current: unknown = obj;
|
||||
|
||||
for (const part of parts) {
|
||||
if (!isObject(current)) return null;
|
||||
current = current[part];
|
||||
}
|
||||
|
||||
return current;
|
||||
};
|
||||
|
||||
const firstValue = (obj: JsonObject, paths: string[]): unknown => {
|
||||
for (const path of paths) {
|
||||
const value = readPathValue(obj, path);
|
||||
if (value === null || value === undefined || value === "") continue;
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
const asNumber = (value: unknown): number | null => {
|
||||
if (typeof value === "number" && Number.isFinite(value)) return value;
|
||||
if (typeof value === "string" && value.length > 0) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isFinite(parsed)) return parsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
const asText = (value: unknown): string => {
|
||||
if (value === null || value === undefined || value === "") return "-";
|
||||
if (
|
||||
typeof value === "string" ||
|
||||
typeof value === "number" ||
|
||||
typeof value === "boolean"
|
||||
) {
|
||||
return String(value);
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map((entry) => asText(entry)).join(",")}]`;
|
||||
}
|
||||
|
||||
if (!isObject(value)) return String(value);
|
||||
|
||||
const preferredFields = ["name", "identifier", "id", "code", "value"];
|
||||
for (const field of preferredFields) {
|
||||
const fieldValue = readPathValue(value, field);
|
||||
if (fieldValue === null || fieldValue === undefined || fieldValue === "")
|
||||
continue;
|
||||
if (typeof fieldValue === "object") continue;
|
||||
|
||||
return String(fieldValue);
|
||||
}
|
||||
|
||||
return stableStringify(value);
|
||||
};
|
||||
|
||||
const adjustmentKey = (adjustment: JsonObject): string => {
|
||||
const keyPaths = [
|
||||
"id",
|
||||
"adjustmentId",
|
||||
"procurementAdjustmentId",
|
||||
"recordId",
|
||||
"recId",
|
||||
"_info.id",
|
||||
"_info.href",
|
||||
];
|
||||
|
||||
for (const path of keyPaths) {
|
||||
const key = firstValue(adjustment, [path]);
|
||||
const keyText = asText(key);
|
||||
if (keyText !== "-") return keyText;
|
||||
}
|
||||
|
||||
return `anon:${stableStringify(adjustment)}`;
|
||||
};
|
||||
|
||||
const trackedRow = (detail: JsonObject): TrackedProduct | null => {
|
||||
const cwCatalogId = asNumber(
|
||||
firstValue(detail, [
|
||||
"catalogItem.id",
|
||||
"catalogItemId",
|
||||
"catalog.id",
|
||||
"catalogId",
|
||||
"item.id",
|
||||
"itemId",
|
||||
"product.id",
|
||||
"productId",
|
||||
"id",
|
||||
]),
|
||||
);
|
||||
if (!cwCatalogId) return null;
|
||||
|
||||
const onHand = asText(
|
||||
firstValue(detail, [
|
||||
"onHand",
|
||||
"onHandQty",
|
||||
"onHandQuantity",
|
||||
"qtyOnHand",
|
||||
"quantityOnHand",
|
||||
"quantity.onHand",
|
||||
]),
|
||||
);
|
||||
const inventory = asText(
|
||||
firstValue(detail, [
|
||||
"inventory",
|
||||
"inventoryQty",
|
||||
"inventoryLevel",
|
||||
"quantity",
|
||||
"qty",
|
||||
]),
|
||||
);
|
||||
if (onHand === "-" && inventory === "-") return null;
|
||||
|
||||
const product = asText(
|
||||
firstValue(detail, [
|
||||
"product.name",
|
||||
"product.identifier",
|
||||
"item.name",
|
||||
"item.identifier",
|
||||
"catalogItem.name",
|
||||
"catalogItem.identifier",
|
||||
"productName",
|
||||
"productIdentifier",
|
||||
"sku",
|
||||
"identifier",
|
||||
]),
|
||||
);
|
||||
|
||||
return {
|
||||
cwCatalogId,
|
||||
product,
|
||||
onHand,
|
||||
inventory,
|
||||
key: `${cwCatalogId}|${product}|${onHand}|${inventory}`,
|
||||
};
|
||||
};
|
||||
|
||||
const trackedRows = (adjustment: JsonObject): TrackedProduct[] => {
|
||||
const detailCandidates = [
|
||||
readPathValue(adjustment, "adjustmentDetails"),
|
||||
readPathValue(adjustment, "details"),
|
||||
readPathValue(adjustment, "lineItems"),
|
||||
];
|
||||
|
||||
for (const candidate of detailCandidates) {
|
||||
if (!Array.isArray(candidate)) continue;
|
||||
|
||||
const rows = candidate
|
||||
.map((entry) => trackedRow(toObject(entry)))
|
||||
.filter((entry): entry is TrackedProduct => entry !== null)
|
||||
.sort((a, b) => a.key.localeCompare(b.key));
|
||||
|
||||
if (rows.length > 0) return rows;
|
||||
}
|
||||
|
||||
const root = trackedRow(adjustment);
|
||||
if (!root) return [];
|
||||
|
||||
return [root];
|
||||
};
|
||||
|
||||
const snapshot = (rows: unknown[]): Map<string, AdjustmentSnapshot> => {
|
||||
const out = new Map<string, AdjustmentSnapshot>();
|
||||
|
||||
for (const entry of rows) {
|
||||
const adjustment = toObject(entry);
|
||||
const key = adjustmentKey(adjustment);
|
||||
const rowsTracked = trackedRows(adjustment);
|
||||
const signature = stableStringify(rowsTracked);
|
||||
out.set(key, {
|
||||
key,
|
||||
trackedRows: rowsTracked,
|
||||
signature,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
};
|
||||
|
||||
const changedCatalogIds = (
|
||||
before: Map<number, string>,
|
||||
after: Map<number, string>,
|
||||
): Set<number> => {
|
||||
const changed = new Set<number>();
|
||||
|
||||
for (const [cwCatalogId, nextSignature] of after) {
|
||||
const prevSignature = before.get(cwCatalogId);
|
||||
if (!prevSignature) {
|
||||
changed.add(cwCatalogId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (prevSignature === nextSignature) continue;
|
||||
changed.add(cwCatalogId);
|
||||
}
|
||||
|
||||
return changed;
|
||||
};
|
||||
|
||||
const productState = (
|
||||
adjustments: Map<string, AdjustmentSnapshot>,
|
||||
): Map<number, string> => {
|
||||
const grouped = new Map<number, Set<string>>();
|
||||
|
||||
for (const snapshot of adjustments.values()) {
|
||||
for (const row of snapshot.trackedRows) {
|
||||
const rows = grouped.get(row.cwCatalogId) ?? new Set<string>();
|
||||
rows.add(row.key);
|
||||
grouped.set(row.cwCatalogId, rows);
|
||||
}
|
||||
}
|
||||
|
||||
const state = new Map<number, string>();
|
||||
for (const [cwCatalogId, rows] of grouped) {
|
||||
state.set(cwCatalogId, stableStringify([...rows].sort()));
|
||||
}
|
||||
|
||||
return state;
|
||||
};
|
||||
|
||||
const applySyncGuards = (ids: number[]): number[] => {
|
||||
const now = Date.now();
|
||||
const cooledIds = ids.filter((cwCatalogId) => {
|
||||
const last = lastSyncedAt.get(cwCatalogId);
|
||||
if (!last) return true;
|
||||
|
||||
return now - last >= SYNC_COOLDOWN_MS;
|
||||
});
|
||||
|
||||
if (cooledIds.length <= MAX_SYNC_PER_CYCLE) return cooledIds;
|
||||
return cooledIds.slice(0, MAX_SYNC_PER_CYCLE);
|
||||
};
|
||||
|
||||
const fetchAdjustments = async (): Promise<unknown[]> => {
|
||||
const response = await withCwRetry(
|
||||
() => connectWiseApi.get(ADJUSTMENTS_ENDPOINT),
|
||||
{
|
||||
label: "inventory-adjustments",
|
||||
maxAttempts: 3,
|
||||
},
|
||||
);
|
||||
const payload = response.data;
|
||||
|
||||
if (Array.isArray(payload)) return payload;
|
||||
if (isObject(payload) && Array.isArray(payload.data)) return payload.data;
|
||||
|
||||
return [];
|
||||
};
|
||||
|
||||
const cacheKey = (cwCatalogId: number) =>
|
||||
`${CATALOG_ITEM_CACHE_PREFIX}${cwCatalogId}`;
|
||||
|
||||
const cwLastUpdated = (item: CatalogItem): Date => {
|
||||
const value = item._info?.lastUpdated;
|
||||
if (!value) return new Date();
|
||||
|
||||
const parsed = new Date(value);
|
||||
const invalidDate = Number.isNaN(parsed.getTime());
|
||||
if (invalidDate) return new Date();
|
||||
|
||||
return parsed;
|
||||
};
|
||||
|
||||
const syncCatalogItem = async (cwCatalogId: number): Promise<boolean> => {
|
||||
try {
|
||||
const item = await withCwRetry(
|
||||
() => catalogCw.fetchByCatalogId(cwCatalogId),
|
||||
{
|
||||
label: `catalog-item:${cwCatalogId}`,
|
||||
maxAttempts: 3,
|
||||
},
|
||||
);
|
||||
const onHand = await withCwRetry(
|
||||
() => catalogCw.fetchInventoryOnHand(cwCatalogId),
|
||||
{
|
||||
label: `catalog-onhand:${cwCatalogId}`,
|
||||
maxAttempts: 3,
|
||||
},
|
||||
);
|
||||
|
||||
const persisted = await prisma.catalogItem.upsert({
|
||||
where: { cwCatalogId },
|
||||
create: {
|
||||
cwCatalogId,
|
||||
identifier: item.identifier,
|
||||
name: item.description,
|
||||
description: item.description,
|
||||
customerDescription: item.customerDescription,
|
||||
internalNotes: item.notes,
|
||||
category: item.category?.name,
|
||||
categoryCwId: item.category?.id,
|
||||
subcategory: item.subcategory?.name,
|
||||
subcategoryCwId: item.subcategory?.id,
|
||||
manufacturer: item.manufacturer?.name,
|
||||
manufactureCwId: item.manufacturer?.id,
|
||||
partNumber: item.manufacturerPartNumber,
|
||||
vendorName: item.vendor?.name,
|
||||
vendorSku: item.vendorSku,
|
||||
vendorCwId: item.vendor?.id,
|
||||
price: item.price,
|
||||
cost: item.cost,
|
||||
inactive: item.inactiveFlag,
|
||||
salesTaxable: item.taxableFlag,
|
||||
onHand,
|
||||
cwLastUpdated: cwLastUpdated(item),
|
||||
},
|
||||
update: {
|
||||
identifier: item.identifier,
|
||||
name: item.description,
|
||||
description: item.description,
|
||||
customerDescription: item.customerDescription,
|
||||
internalNotes: item.notes,
|
||||
category: item.category?.name,
|
||||
categoryCwId: item.category?.id,
|
||||
subcategory: item.subcategory?.name,
|
||||
subcategoryCwId: item.subcategory?.id,
|
||||
manufacturer: item.manufacturer?.name,
|
||||
manufactureCwId: item.manufacturer?.id,
|
||||
partNumber: item.manufacturerPartNumber,
|
||||
vendorName: item.vendor?.name,
|
||||
vendorSku: item.vendorSku,
|
||||
vendorCwId: item.vendor?.id,
|
||||
price: item.price,
|
||||
cost: item.cost,
|
||||
inactive: item.inactiveFlag,
|
||||
salesTaxable: item.taxableFlag,
|
||||
onHand,
|
||||
cwLastUpdated: cwLastUpdated(item),
|
||||
},
|
||||
});
|
||||
|
||||
await redis.set(
|
||||
cacheKey(cwCatalogId),
|
||||
JSON.stringify({
|
||||
cwCatalogId,
|
||||
onHand,
|
||||
cwItem: item,
|
||||
dbItem: persisted,
|
||||
syncedAt: new Date().toISOString(),
|
||||
}),
|
||||
"EX",
|
||||
CATALOG_ITEM_CACHE_TTL_SECONDS,
|
||||
);
|
||||
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[inventory-adjustments] failed to sync catalog item ${cwCatalogId}`,
|
||||
err,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export const listenInventoryAdjustments = async (): Promise<void> => {
|
||||
if (inFlight) return;
|
||||
inFlight = true;
|
||||
|
||||
try {
|
||||
const rows = await fetchAdjustments();
|
||||
const current = snapshot(rows);
|
||||
const currentProductState = productState(current);
|
||||
|
||||
if (previous.size === 0) {
|
||||
previous = current;
|
||||
previousProductState = currentProductState;
|
||||
console.log(
|
||||
`[inventory-adjustments] baseline captured (${current.size} adjustments, ${currentProductState.size} products)`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const changedIds = [
|
||||
...changedCatalogIds(previousProductState, currentProductState),
|
||||
].sort((a, b) => a - b);
|
||||
const guardedIds = applySyncGuards(changedIds);
|
||||
previous = current;
|
||||
previousProductState = currentProductState;
|
||||
if (guardedIds.length === 0) return;
|
||||
|
||||
let successCount = 0;
|
||||
for (const cwCatalogId of guardedIds) {
|
||||
const ok = await syncCatalogItem(cwCatalogId);
|
||||
if (!ok) continue;
|
||||
lastSyncedAt.set(cwCatalogId, Date.now());
|
||||
successCount += 1;
|
||||
}
|
||||
|
||||
const skippedByCooldown = changedIds.length - guardedIds.length;
|
||||
|
||||
console.log(
|
||||
`[inventory-adjustments] inventory changed for ${changedIds.length} products, queued ${guardedIds.length}, synced ${successCount}, cooldown/cap skipped ${skippedByCooldown}`,
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("[inventory-adjustments] listener failed", err);
|
||||
} finally {
|
||||
inFlight = false;
|
||||
}
|
||||
};
|
||||
@@ -2,6 +2,31 @@ import { prisma } from "../../../constants";
|
||||
import { events } from "../../globalEvents";
|
||||
import { catalogCw } from "./catalog";
|
||||
|
||||
const CONCURRENCY = 6;
|
||||
const BATCH_DELAY_MS = 250;
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
const runSlowParallel = async (
|
||||
tasks: Array<() => Promise<void>>,
|
||||
): Promise<number> => {
|
||||
let failureCount = 0;
|
||||
|
||||
for (let i = 0; i < tasks.length; i += CONCURRENCY) {
|
||||
const batch = tasks.slice(i, i + CONCURRENCY);
|
||||
const results = await Promise.allSettled(batch.map((task) => task()));
|
||||
|
||||
for (const result of results) {
|
||||
if (result.status === "rejected") failureCount += 1;
|
||||
}
|
||||
|
||||
if (i + CONCURRENCY >= tasks.length) continue;
|
||||
await sleep(BATCH_DELAY_MS);
|
||||
}
|
||||
|
||||
return failureCount;
|
||||
};
|
||||
|
||||
export const refreshCatalog = async () => {
|
||||
events.emit("cw:catalog:refresh:check");
|
||||
|
||||
@@ -46,101 +71,104 @@ export const refreshCatalog = async () => {
|
||||
staleCount: staleIds.length,
|
||||
});
|
||||
|
||||
// 4. Fetch full catalog data, then filter to only stale items
|
||||
const staleIdSet = new Set(staleIds);
|
||||
const allCwItems = await catalogCw.fetchAllItemsFromCw();
|
||||
const allStaleItems = new Map<number, any>();
|
||||
// 4. Fetch full CW item data for stale IDs using slow, bounded concurrency
|
||||
const cwItemMap = new Map<number, any>();
|
||||
const itemFetchTasks: Array<() => Promise<void>> = staleIds.map(
|
||||
(cwId) => async () => {
|
||||
const item = await catalogCw.fetchByCatalogId(cwId);
|
||||
cwItemMap.set(cwId, item);
|
||||
},
|
||||
);
|
||||
const itemFetchFailures = await runSlowParallel(itemFetchTasks);
|
||||
|
||||
for (const [id, item] of allCwItems) {
|
||||
if (staleIdSet.has(id)) {
|
||||
allStaleItems.set(id, item);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Batch fetch inventory onHand for stale items (50 concurrent)
|
||||
// 5. Fetch inventory onHand for stale IDs using the same slow parallel strategy
|
||||
const onHandMap = new Map<number, number>();
|
||||
const batchSize = 50;
|
||||
const inventoryTasks: Array<() => Promise<void>> = staleIds.map(
|
||||
(cwId) => async () => {
|
||||
try {
|
||||
const onHand = await catalogCw.fetchInventoryOnHand(cwId);
|
||||
onHandMap.set(cwId, onHand);
|
||||
} catch {
|
||||
onHandMap.set(cwId, 0);
|
||||
}
|
||||
},
|
||||
);
|
||||
const inventoryFailures = await runSlowParallel(inventoryTasks);
|
||||
|
||||
for (let i = 0; i < staleIds.length; i += batchSize) {
|
||||
const batch = staleIds.slice(i, i + batchSize);
|
||||
await Promise.all(
|
||||
batch.map(async (cwId) => {
|
||||
try {
|
||||
const onHand = await catalogCw.fetchInventoryOnHand(cwId);
|
||||
onHandMap.set(cwId, onHand);
|
||||
} catch {
|
||||
onHandMap.set(cwId, 0);
|
||||
}
|
||||
}),
|
||||
// 6. Upsert stale/new items with bounded slow parallel execution
|
||||
let updatedCount = 0;
|
||||
const upsertTasks: Array<() => Promise<void>> = staleIds.map(
|
||||
(cwId) => async () => {
|
||||
const item = cwItemMap.get(cwId);
|
||||
if (!item) return;
|
||||
|
||||
const cwLastUpdated = item._info?.lastUpdated
|
||||
? new Date(item._info.lastUpdated)
|
||||
: new Date();
|
||||
const onHand = onHandMap.get(cwId) ?? 0;
|
||||
|
||||
await prisma.catalogItem.upsert({
|
||||
where: { cwCatalogId: cwId },
|
||||
create: {
|
||||
cwCatalogId: cwId,
|
||||
identifier: item.identifier,
|
||||
name: item.description,
|
||||
description: item.description,
|
||||
customerDescription: item.customerDescription,
|
||||
internalNotes: item.notes,
|
||||
category: item.category?.name,
|
||||
categoryCwId: item.category?.id,
|
||||
subcategory: item.subcategory?.name,
|
||||
subcategoryCwId: item.subcategory?.id,
|
||||
manufacturer: item.manufacturer?.name,
|
||||
manufactureCwId: item.manufacturer?.id,
|
||||
partNumber: item.manufacturerPartNumber,
|
||||
vendorName: item.vendor?.name,
|
||||
vendorSku: item.vendorSku,
|
||||
vendorCwId: item.vendor?.id,
|
||||
price: item.price,
|
||||
cost: item.cost,
|
||||
inactive: item.inactiveFlag,
|
||||
salesTaxable: item.taxableFlag,
|
||||
onHand,
|
||||
cwLastUpdated,
|
||||
},
|
||||
update: {
|
||||
name: item.description,
|
||||
identifier: item.identifier,
|
||||
description: item.description,
|
||||
customerDescription: item.customerDescription,
|
||||
internalNotes: item.notes,
|
||||
category: item.category?.name,
|
||||
categoryCwId: item.category?.id,
|
||||
subcategory: item.subcategory?.name,
|
||||
subcategoryCwId: item.subcategory?.id,
|
||||
manufacturer: item.manufacturer?.name,
|
||||
manufactureCwId: item.manufacturer?.id,
|
||||
partNumber: item.manufacturerPartNumber,
|
||||
vendorName: item.vendor?.name,
|
||||
vendorSku: item.vendorSku,
|
||||
vendorCwId: item.vendor?.id,
|
||||
price: item.price,
|
||||
cost: item.cost,
|
||||
inactive: item.inactiveFlag,
|
||||
salesTaxable: item.taxableFlag,
|
||||
onHand,
|
||||
cwLastUpdated,
|
||||
},
|
||||
});
|
||||
updatedCount += 1;
|
||||
},
|
||||
);
|
||||
const upsertFailures = await runSlowParallel(upsertTasks);
|
||||
|
||||
const failedTasks = itemFetchFailures + inventoryFailures + upsertFailures;
|
||||
if (failedTasks > 0) {
|
||||
console.warn(
|
||||
`[catalog-refresh] ${failedTasks} slow-parallel task(s) failed; remaining items will retry next cycle`,
|
||||
);
|
||||
}
|
||||
|
||||
// 6. Upsert only the stale/new items
|
||||
const updatedCount = (
|
||||
await Promise.all(
|
||||
staleIds.map(async (cwId) => {
|
||||
const item = allStaleItems.get(cwId);
|
||||
if (!item) return null;
|
||||
|
||||
const cwLastUpdated = item._info?.lastUpdated
|
||||
? new Date(item._info.lastUpdated)
|
||||
: new Date();
|
||||
const onHand = onHandMap.get(cwId) ?? 0;
|
||||
|
||||
return await prisma.catalogItem.upsert({
|
||||
where: { cwCatalogId: cwId },
|
||||
create: {
|
||||
cwCatalogId: cwId,
|
||||
identifier: item.identifier,
|
||||
name: item.description,
|
||||
description: item.description,
|
||||
customerDescription: item.customerDescription,
|
||||
internalNotes: item.notes,
|
||||
category: item.category?.name,
|
||||
categoryCwId: item.category?.id,
|
||||
subcategory: item.subcategory?.name,
|
||||
subcategoryCwId: item.subcategory?.id,
|
||||
manufacturer: item.manufacturer?.name,
|
||||
manufactureCwId: item.manufacturer?.id,
|
||||
partNumber: item.manufacturerPartNumber,
|
||||
vendorName: item.vendor?.name,
|
||||
vendorSku: item.vendorSku,
|
||||
vendorCwId: item.vendor?.id,
|
||||
price: item.price,
|
||||
cost: item.cost,
|
||||
inactive: item.inactiveFlag,
|
||||
salesTaxable: item.taxableFlag,
|
||||
onHand,
|
||||
cwLastUpdated,
|
||||
},
|
||||
update: {
|
||||
name: item.description,
|
||||
identifier: item.identifier,
|
||||
description: item.description,
|
||||
customerDescription: item.customerDescription,
|
||||
internalNotes: item.notes,
|
||||
category: item.category?.name,
|
||||
categoryCwId: item.category?.id,
|
||||
subcategory: item.subcategory?.name,
|
||||
subcategoryCwId: item.subcategory?.id,
|
||||
manufacturer: item.manufacturer?.name,
|
||||
manufactureCwId: item.manufacturer?.id,
|
||||
partNumber: item.manufacturerPartNumber,
|
||||
vendorName: item.vendor?.name,
|
||||
vendorSku: item.vendorSku,
|
||||
vendorCwId: item.vendor?.id,
|
||||
price: item.price,
|
||||
cost: item.cost,
|
||||
inactive: item.inactiveFlag,
|
||||
salesTaxable: item.taxableFlag,
|
||||
onHand,
|
||||
cwLastUpdated,
|
||||
},
|
||||
});
|
||||
}),
|
||||
)
|
||||
).filter(Boolean).length;
|
||||
|
||||
events.emit("cw:catalog:refresh:completed", {
|
||||
totalCw: cwSummaries.size,
|
||||
totalDb: dbItems.length,
|
||||
|
||||
@@ -2,6 +2,11 @@ import { prisma } from "../../../constants";
|
||||
import { events } from "../../globalEvents";
|
||||
import { catalogCw } from "./catalog";
|
||||
|
||||
const CONCURRENCY = 6;
|
||||
const BATCH_DELAY_MS = 250;
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export const refreshInventory = async () => {
|
||||
events.emit("cw:inventory:refresh:check");
|
||||
|
||||
@@ -23,13 +28,13 @@ export const refreshInventory = async () => {
|
||||
totalItems: dbItems.length,
|
||||
});
|
||||
|
||||
// 2. Batch fetch inventory onHand for all items (50 concurrent)
|
||||
// 2. Slow-parallel fetch inventory onHand for all items
|
||||
const onHandMap = new Map<number, number>();
|
||||
const batchSize = 150;
|
||||
let failedCount = 0;
|
||||
|
||||
for (let i = 0; i < dbItems.length; i += batchSize) {
|
||||
const batch = dbItems.slice(i, i + batchSize);
|
||||
await Promise.all(
|
||||
for (let i = 0; i < dbItems.length; i += CONCURRENCY) {
|
||||
const batch = dbItems.slice(i, i + CONCURRENCY);
|
||||
const results = await Promise.allSettled(
|
||||
batch.map(async (item) => {
|
||||
try {
|
||||
const onHand = await catalogCw.fetchInventoryOnHand(item.cwCatalogId);
|
||||
@@ -39,6 +44,13 @@ export const refreshInventory = async () => {
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
for (const result of results) {
|
||||
if (result.status === "rejected") failedCount += 1;
|
||||
}
|
||||
|
||||
if (i + CONCURRENCY >= dbItems.length) continue;
|
||||
await sleep(BATCH_DELAY_MS);
|
||||
}
|
||||
|
||||
// 3. Only update items where onHand has changed
|
||||
@@ -71,4 +83,10 @@ export const refreshInventory = async () => {
|
||||
totalItems: dbItems.length,
|
||||
updatedCount,
|
||||
});
|
||||
|
||||
if (failedCount > 0) {
|
||||
console.warn(
|
||||
`[inventory-refresh] ${failedCount} task(s) failed; fallback values were used and will retry next sweep`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -383,6 +383,13 @@ export const PERMISSION_NODES = {
|
||||
],
|
||||
},
|
||||
|
||||
cwCallbacks: {
|
||||
name: "ConnectWise Callback Routes",
|
||||
description:
|
||||
"Inbound ConnectWise callback endpoints. These routes are intentionally unauthenticated and do not require permission nodes.",
|
||||
permissions: [],
|
||||
},
|
||||
|
||||
sales: {
|
||||
name: "Sales Permissions",
|
||||
description: "Permissions for accessing and managing sales opportunities",
|
||||
|
||||
Reference in New Issue
Block a user