all the haul

api/src/index.ts

@@ -1,61 +1,32 @@
-import { refresh } from "./api/auth";
 import app from "./api/server";
 import { setupSockets } from "./api/sockets";
-import {
-  COLLECTOR_WS_URL,
-  connectCollectorSocket,
-  collectorSocket,
-  engine,
-  PORT,
-  prisma,
-  unifi,
-  unifiPassword,
-  unifiUsername,
-} from "./constants";
+import { engine, PORT, prisma } from "./constants";
 import { unifiSites } from "./managers/unifiSites";
-import { refreshCompanies } from "./modules/cw-utils/refreshCompanies";
-import { refreshCatalog } from "./modules/cw-utils/procurement/refreshCatalog";
-import { refreshInventory } from "./modules/cw-utils/procurement/refreshInventory";
-import { listenInventoryAdjustments } from "./modules/cw-utils/procurement/listenInventoryAdjustments";
-import { refreshSalesOpportunityMetricsCache } from "./modules/cache/salesOpportunityMetricsCache";
-import { refreshCwIdentifiers } from "./modules/cw-utils/members/refreshCwIdentifiers";
-import { refreshCwMembers } from "./modules/cw-utils/members/refreshCwMembers";
-import { userDefinedFieldsCw } from "./modules/cw-utils/userDefinedFields";
 import { events } from "./modules/globalEvents";
 import { setupEventDebugger } from "./modules/logging/eventDebugger";
 import { signPermissions } from "./modules/permission-utils/signPermissions";
 import { RoleController } from "./controllers/RoleController";
-import cuid from "cuid";
+import { initializeWorkerSystem, getBoss } from "./workert";
+import { WorkerQueue } from "./modules/workers/queues";
+import { enqueueIncrementalSync } from "./modules/workers/incremental-sync";
 import { startCommsServer } from "./modules/workers/coms";
-import {
-  enqueueActiveOpportunityRefreshJob,
-  startOpportunityCacheWorkers,
-} from "./workert";
+import cuid from "cuid";
+
+const startupArgs = new Set(Bun.argv.slice(2));
+const simpleTerminalMode =
+  startupArgs.has("-st") || startupArgs.has("--simple-terminal");
 
 // Setup global event debugger in non-production environments
-if (Bun.env.NODE_ENV == "development") {
+if (Bun.env.NODE_ENV == "development" && !simpleTerminalMode) {
   setupEventDebugger({ processLabel: "API" });
 }
 
-/** Concise error message for interval logs — avoids dumping full Axios error objects. */
-const briefErr = (err: any): string => {
-  if (err?.isAxiosError) {
-    const method = (err.config?.method ?? "?").toUpperCase();
-    const url = err.config?.url ?? "?";
-    return `${method} ${url} → ${err.code ?? `HTTP ${err.response?.status}`}`;
-  }
-  return err?.message ?? String(err);
-};
-
-// Helper to run a startup sync safely — failures are logged but never crash the process.
+// Helper to run a startup task safely — failures are logged but never crash the process.
 const safeStartup = async (label: string, fn: () => Promise<void>) => {
   try {
     await fn();
   } catch (err) {
-    console.error(
-      `[startup] ${label} failed — will retry on next interval`,
-      err,
-    );
+    console.error(`[startup] ${label} failed`, err);
   }
 };
 
@@ -83,33 +54,30 @@ console.log(`[startup] Server listening on port ${PORT}`);
 setupSockets();
 console.log("[startup] Socket namespaces initialized");
 
-collectorSocket.on("connect", () => {
-  console.log(
-    `[startup] Collector socket connected: ${COLLECTOR_WS_URL} (${collectorSocket.id})`,
-  );
-});
-
-collectorSocket.on("connect_error", (err) => {
-  console.error(`[startup] Collector socket connect_error: ${err.message}`);
-});
-
-connectCollectorSocket();
-console.log("[startup] Collector socket initialization started");
+// Initialize worker system (PgBoss connection)
+await safeStartup("initializeWorkerSystem", () => initializeWorkerSystem());
 
+// Start the inter-process comms server so the worker can connect on :8671
 startCommsServer();
-console.log("[startup] Worker comms server initialized");
+console.log("[startup] Comms server listening on :8671");
 
-const embeddedWorkersEnabled = Bun.env.START_EMBEDDED_WORKERS === "true";
-if (embeddedWorkersEnabled) {
-  await safeStartup(
-    "startOpportunityCacheWorkers",
-    startOpportunityCacheWorkers,
+// Enqueue a full dalpuri sync on startup
+await safeStartup("enqueueDalpuriFullSync", async () => {
+  const jobId = await getBoss().send(WorkerQueue.DALPURI_FULL_SYNC, {}, { singletonKey: `startup-${Date.now()}` });
+  if (jobId) {
+    console.log(`[startup] Dalpuri full sync enqueued: ${jobId}`);
+  } else {
+    console.warn("[startup] Dalpuri full sync send returned null — job may already be pending or PgBoss not ready");
+  }
+});
+
+// Broadcast incremental sync jobs from the API process every 5s so the
+// interval survives worker restarts.
+setInterval(() => {
+  enqueueIncrementalSync().catch((err) =>
+    console.error(`[interval] enqueueIncrementalSync failed: ${err?.message ?? err}`)
   );
-} else {
-  console.log(
-    "[startup] Embedded opportunity workers disabled on API node (set START_EMBEDDED_WORKERS=true to override)",
-  );
-}
+}, 5_000);
 
 // ---------------------------------------------------------------------------
 // Background initialisation — none of this blocks the server.
@@ -141,114 +109,30 @@ await safeStartup("ensureAdminRole", async () => {
   }
 });
 
-// Refresh the internal list of companies every minute
-await safeStartup("refreshCompanies", refreshCompanies);
-setInterval(() => {
-  return refreshCompanies().catch((err) =>
-    console.error(`[interval] refreshCompanies failed: ${briefErr(err)}`),
+// Enqueue an initial cold-load metrics refresh on startup
+await safeStartup("enqueueSalesMetricsRefresh", async () => {
+  const jobId = await getBoss().send(
+    WorkerQueue.REFRESH_SALES_METRICS,
+    { forceColdLoad: true },
+    { singletonKey: `startup-metrics-${Date.now()}` }
   );
-}, 60 * 1000);
+  if (jobId) {
+    console.log(`[startup] Sales metrics refresh enqueued: ${jobId}`);
+  } else {
+    console.warn("[startup] Sales metrics refresh send returned null — job may already be pending");
+  }
+});
 
-// Refresh the internal catalog every 30 minutes
-await safeStartup("refreshCatalog", refreshCatalog);
-setInterval(
-  () => {
-    return refreshCatalog().catch((err) =>
-      console.error(`[interval] refreshCatalog failed: ${briefErr(err)}`),
-    );
-  },
-  30 * 60 * 1000,
-);
-
-// Fallback full inventory sweep every 6 hours (listener handles real-time deltas)
-setInterval(
-  () => {
-    return refreshInventory().catch((err) =>
-      console.error(`[interval] refreshInventory failed: ${briefErr(err)}`),
-    );
-  },
-  6 * 60 * 60 * 1000,
-);
-
-// Listen for procurement adjustment changes and sync changed products to DB + cache
-await safeStartup("listenInventoryAdjustments", listenInventoryAdjustments);
+// Enqueue a metrics refresh every 5 minutes
 setInterval(() => {
-  return listenInventoryAdjustments().catch((err) =>
-    console.error(
-      `[interval] listenInventoryAdjustments failed: ${briefErr(err)}`,
-    ),
-  );
-}, 60 * 1000);
-
-// Refresh opportunity CW cache every 20 minutes (activities + company hydration)
-// NOTE: Do NOT await — register the interval immediately so the cache refresh
-// is never blocked by a slow/stuck startup task above.
-safeStartup("enqueueActiveOpportunityRefreshJob", async () => {
-  await enqueueActiveOpportunityRefreshJob();
-});
-setInterval(
-  () => {
-    return enqueueActiveOpportunityRefreshJob().catch((err) => {
-      console.error(
-        `[interval] enqueueActiveOpportunityRefreshJob failed: ${briefErr(err)}`,
-      );
-    });
-  },
-  20 * 60 * 1000,
-);
-
-// Refresh User Defined Fields every 5 minutes
-await safeStartup("refreshUDFs", async () => {
-  await userDefinedFieldsCw.refresh();
-});
-setInterval(
-  () => {
-    return userDefinedFieldsCw
-      .refresh()
-      .catch((err) =>
-        console.error(`[interval] refreshUDFs failed: ${briefErr(err)}`),
-      );
-  },
-  5 * 60 * 1000,
-);
-
-// Refresh sales opportunity metrics cache for active CW members every 5 minutes
-await safeStartup("refreshSalesOpportunityMetricsCache", () =>
-  refreshSalesOpportunityMetricsCache({ forceColdLoad: true }),
-);
-setInterval(
-  () => {
-    return refreshSalesOpportunityMetricsCache().catch((err) =>
-      console.error(
-        `[interval] refreshSalesOpportunityMetricsCache failed: ${briefErr(err)}`,
-      ),
+  getBoss()
+    .send(WorkerQueue.REFRESH_SALES_METRICS, {}, { singletonKey: "metrics-interval" })
+    .catch((err) =>
+      console.error(`[interval] REFRESH_SALES_METRICS enqueue failed: ${err?.message ?? err}`)
     );
-  },
-  5 * 60 * 1000,
-);
-
-// Refresh CW identifiers for all users every 30 minutes
-await safeStartup("refreshCwIdentifiers", refreshCwIdentifiers);
-setInterval(
-  () => {
-    return refreshCwIdentifiers().catch((err) =>
-      console.error(`[interval] refreshCwIdentifiers failed: ${briefErr(err)}`),
-    );
-  },
-  30 * 60 * 1000,
-);
-
-// Refresh CW members DB table every hour
-await safeStartup("refreshCwMembers", refreshCwMembers);
-setInterval(
-  () => {
-    return refreshCwMembers().catch((err) =>
-      console.error(`[interval] refreshCwMembers failed: ${briefErr(err)}`),
-    );
-  },
-  60 * 60 * 1000,
-);
+}, 5 * 60 * 1000);
 
+// Sync UniFi sites
 await safeStartup("syncSites", async () => {
   await unifiSites.syncSites();
 });
@@ -256,6 +140,6 @@ setInterval(() => {
   return unifiSites
     .syncSites()
     .catch((err) =>
-      console.error(`[interval] syncSites failed: ${briefErr(err)}`),
+      console.error(`[interval] syncSites failed: ${err?.message ?? err}`)
     );
 }, 60 * 1000);