import { PrismaMssql } from "@prisma/adapter-mssql"; import { PrismaPg } from "@prisma/adapter-pg"; import { readFileSync } from "node:fs"; import { resolve } from "node:path"; import { PrismaClient as CwPrismaClient } from "../generated/prisma/client"; import { PrismaClient as ApiPrismaClient } from "../../api/generated/prisma/client"; import { catalogCategoryTranslation, catalogItemTranslation, catalogItemTypeTranslation, catalogManufacturerTranslation, catalogSubcategoryTranslation, companyAddressTranslation, companyTranslation, contactTranslation, corporateLocationTranslation, createTranslationContext, cwMemberTranslation, internalDepartmentTranslation, opportunityStatusTranslation, opportunityTranslation, opportunityTypeTranslation, productDataTranslation, productInventoryTranslation, serviceTicketBoardTranslation, serviceTicketImpactTranslation, serviceTicketLocationTranslation, serviceTicketNoteTranslation, serviceTicketPriorityTranslation, serviceTicketSeverityTranslation, serviceTicketSourceTranslation, serviceTicketTranslation, serviceTicketTypeTranslation, scheduleStatusTranslation, scheduleTypeTranslation, scheduleSpanTranslation, scheduleTranslation, userTranslation, warehouseBinTranslation, type TranslationContext, } from "./index"; import { Translation, SkipRowError } from "./translations/types"; type Row = Record; type AnyTranslation = Translation; type SyncTableConfig = { sourceModel: string; targetModel: string; translation: AnyTranslation; uniqueField: string; lastUpdatedField: string; sourceArgs?: Record; }; type SyncResult = { insertedOrUpdated: number; skipped: number; failed: number; }; const parseEnvFile = (path: string): Record => { const envData = readFileSync(path, "utf8"); const out: Record = {}; for (const rawLine of envData.split(/\r?\n/)) { const line = rawLine.trim(); if (!line || line.startsWith("#")) continue; const index = line.indexOf("="); if (index <= 0) continue; const key = line.slice(0, index).trim(); let value = line.slice(index + 1).trim(); if ( (value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'")) ) { value = value.slice(1, -1); } out[key] = value; } return out; }; const resolveApiDatabaseUrl = (): string => { if (process.env.API_DATABASE_URL) return process.env.API_DATABASE_URL; if (process.env.OPTIMA_API_DATABASE_URL) return process.env.OPTIMA_API_DATABASE_URL; const candidates = [ resolve(import.meta.dir, "../../api/.env"), resolve(process.cwd(), "../api/.env"), ]; for (const apiEnvPath of candidates) { try { const apiEnv = parseEnvFile(apiEnvPath); if (apiEnv.DATABASE_URL) { return apiEnv.DATABASE_URL; } } catch { // Try next path candidate. } } return ""; }; const parseJsonMapFromEnv = (envKey: string): Map => { const raw = process.env[envKey]; if (!raw) return new Map(); try { const parsed = JSON.parse(raw) as Record; const entries = Object.entries(parsed) .map(([key, value]) => [Number.parseInt(key, 10), value] as const) .filter( ([key, value]) => Number.isInteger(key) && typeof value === "string" ); return new Map(entries); } catch (error) { console.warn(`Could not parse ${envKey} as JSON map. Ignoring.`, error); return new Map(); } }; const normalizeData = (data: Row): Row => { const normalized: Row = {}; for (const [key, value] of Object.entries(data)) { if (value !== undefined) { normalized[key] = value; } } return normalized; }; const sanitizeUserForeignKeys = ( data: Row, context: TranslationContext, targetModel: string ): Row => { const sanitized = { ...data }; if (targetModel === "serviceTicket" && typeof sanitized.userId === "string") { const id = context.usersByIdentifier.get(sanitized.userId as string); sanitized.userId = id || null; } if (targetModel === "opportunity" && typeof sanitized.ownerId === "string") { const id = context.usersByIdentifier.get(sanitized.ownerId as string); sanitized.ownerId = id || null; } return sanitized; }; const translateRow = ( row: Row, translation: AnyTranslation, context: TranslationContext ): Row => { const out: Row = {}; for (const entry of translation.values) { const fromKey = entry.from as string; const toKey = entry.to as string; const input = row[fromKey]; if (input === undefined) { continue; } const output = entry.process ? entry.process(input as never, context, row) : input; out[toKey] = output as unknown; } return normalizeData(out); }; const refreshContextFromApi = async ( apiPrisma: ApiPrismaClient, context: TranslationContext ): Promise => { context.userIds.clear(); context.serviceTicketIds.clear(); context.opportunityIds.clear(); context.catalogItemIds.clear(); context.corporateLocationIds.clear(); context.usersByMemberRecId.clear(); context.userIdentifiersByMemberRecId.clear(); context.usersByIdentifier.clear(); context.serviceTicketBoardUidsById.clear(); context.opportunityStatusIds.clear(); context.scheduleStatusIds.clear(); context.scheduleTypeIds.clear(); context.scheduleSpanIds.clear(); const [ users, boards, serviceTickets, opportunities, catalogItems, locations, opportunityStatuses, scheduleStatuses, scheduleTypes, scheduleSpans, ] = await Promise.all([ apiPrisma.user.findMany({ select: { id: true, cwMemberId: true, cwIdentifier: true, }, }), apiPrisma.serviceTicketBoard.findMany({ select: { id: true, uid: true, }, }), apiPrisma.serviceTicket.findMany({ select: { id: true, }, }), apiPrisma.opportunity.findMany({ select: { id: true, }, }), apiPrisma.catalogItem.findMany({ select: { id: true, }, }), apiPrisma.corporateLocation.findMany({ select: { id: true, }, }), apiPrisma.opportunityStatus.findMany({ select: { id: true, }, }), apiPrisma.scheduleStatus.findMany({ select: { id: true, }, }), apiPrisma.scheduleType.findMany({ select: { id: true, }, }), apiPrisma.scheduleSpan.findMany({ select: { id: true, }, }), ]); const defaultOpportunityType = await apiPrisma.opportunityType.findFirst({ orderBy: { id: "asc" }, select: { id: true }, }); for (const user of users) { context.userIds.add(user.id); if (user.cwMemberId != null) { context.usersByMemberRecId.set(user.cwMemberId, user.id); if (user.cwIdentifier) { context.userIdentifiersByMemberRecId.set( user.cwMemberId, user.cwIdentifier ); } } if (user.cwIdentifier) { context.usersByIdentifier.set(user.cwIdentifier, user.id); } } const cwMembers = await apiPrisma.cwMember.findMany({ select: { cwMemberId: true, identifier: true }, }); for (const member of cwMembers) { if ( member.cwMemberId != null && member.identifier && !context.userIdentifiersByMemberRecId.has(member.cwMemberId) ) { context.userIdentifiersByMemberRecId.set( member.cwMemberId, member.identifier ); } } for (const board of boards) { context.serviceTicketBoardUidsById.set(board.id, board.uid); } for (const serviceTicket of serviceTickets) { context.serviceTicketIds.add(serviceTicket.id); } for (const opportunity of opportunities) { context.opportunityIds.add(opportunity.id); } for (const catalogItem of catalogItems) { context.catalogItemIds.add(catalogItem.id); } for (const location of locations) { context.corporateLocationIds.add(location.id); } for (const status of opportunityStatuses) { context.opportunityStatusIds.add(status.id); } for (const status of scheduleStatuses) { context.scheduleStatusIds.add(status.id); } for (const type of scheduleTypes) { context.scheduleTypeIds.add(type.id); } for (const span of scheduleSpans) { context.scheduleSpanIds.add(span.id); } context.defaultOpportunityTypeId = defaultOpportunityType?.id ?? null; // Optional context fed from env-provided maps. context.billingTypeByTicketId = parseJsonMapFromEnv( "BILLING_TYPE_CONTEXT_JSON" ); context.billingInstructionsByTicketId = parseJsonMapFromEnv( "BILLING_INSTRUCTIONS_CONTEXT_JSON" ); }; const refreshCustomFieldContextFromCw = async ( cwPrisma: CwPrismaClient, context: TranslationContext ): Promise => { context.opportunityNarrativeByOpportunityId.clear(); context.productCustomByIvProductId.clear(); const [productCustomRows, opportunityCustomRows] = await Promise.all([ cwPrisma.$queryRawUnsafe>>(` SELECT iv.RecID, iv.ConfigUIDF as fieldName, iv.ConfigValue as fieldValue FROM dbo.iV_Product iv `), cwPrisma.$queryRawUnsafe>>(` SELECT so.RecID, so.SalesPipelineUIDF as fieldName, so.OpportunitySalesNotes as fieldValue FROM dbo.opportunity so `), ]); for (const row of productCustomRows) { const productId = row.RecID; const fieldName = row.fieldName; const fieldValue = row.fieldValue; if ( typeof productId === "number" && typeof fieldName === "string" && typeof fieldValue === "string" ) { if (!context.productCustomByIvProductId.has(productId)) { context.productCustomByIvProductId.set(productId, new Map()); } context.productCustomByIvProductId .get(productId) ?.set(fieldName.toLowerCase().trim(), fieldValue.trim()); } } for (const row of opportunityCustomRows) { const opportunityId = row.RecID; const fieldName = row.fieldName; const fieldValue = row.fieldValue; if ( typeof opportunityId === "number" && typeof fieldName === "string" && typeof fieldValue === "string" ) { if (fieldName.toLowerCase().includes("narrative")) { context.opportunityNarrativeByOpportunityId.set( opportunityId, fieldValue ); } } } }; const sanitizeModelData = ( data: Row, targetModel: string, context: TranslationContext ): Row => { const sanitized = { ...data }; if (targetModel === "warehouseBin") { if (sanitized.minQuantity == null) sanitized.minQuantity = 0; if (sanitized.maxQuantity == null) sanitized.maxQuantity = 0; } if (targetModel === "serviceTicket") { const boardId = sanitized.serviceTicketBoardId; if (typeof boardId === "string") { const parsed = Number.parseInt(boardId, 10); sanitized.serviceTicketBoardId = Number.isNaN(parsed) ? null : parsed; } } if (targetModel === "opportunity") { if (sanitized.typeId == null && context.defaultOpportunityTypeId != null) { sanitized.typeId = context.defaultOpportunityTypeId; } if ( sanitized.statusId != null && !context.opportunityStatusIds.has(sanitized.statusId as number) ) { sanitized.statusId = null; } if ( sanitized.locationId != null && !context.corporateLocationIds.has(sanitized.locationId as number) ) { sanitized.locationId = null; } } if (targetModel === "schedule") { // Nullify statusId if the referenced ScheduleStatus hasn't synced yet if ( sanitized.statusId != null && !context.scheduleStatusIds.has(sanitized.statusId as number) ) { sanitized.statusId = null; } // Nullify typeId if the referenced ScheduleType hasn't synced yet if ( sanitized.typeId != null && !context.scheduleTypeIds.has(sanitized.typeId as number) ) { sanitized.typeId = null; } // Nullify scheduleSpanId if the referenced ScheduleSpan hasn't synced yet if ( sanitized.scheduleSpanId != null && !context.scheduleSpanIds.has(sanitized.scheduleSpanId as number) ) { sanitized.scheduleSpanId = null; } } return sanitized; }; const asRecord = (value: unknown): Record | null => { if (typeof value !== "object" || value == null) { return null; } return value as Record; }; const parseFieldsFromErrorMessage = (message: string): string[] => { const fields = new Set(); // Handles patterns like (`"cwIdentifier"`) and (`login`) for (const match of message.matchAll(/`\"([^\"]+)\"`|`([^`]+)`/g)) { const candidate = (match[1] ?? match[2] ?? "").trim(); if (!candidate) continue; // Skip common non-field tokens that can appear in error messages. if (candidate === "targetDelegate.upsert()" || candidate === "invocation") { continue; } fields.add(candidate); } return [...fields]; }; const getUniqueConstraintFields = (error: unknown): string[] => { const errorRecord = asRecord(error); if (!errorRecord || errorRecord.code !== "P2002") { return []; } const meta = asRecord(errorRecord.meta); if (!meta) { return []; } const target = meta.target; if (Array.isArray(target)) { return target.filter((field): field is string => typeof field === "string"); } if (typeof target === "string") { return [target]; } if (typeof errorRecord.message === "string") { return parseFieldsFromErrorMessage(errorRecord.message); } return []; }; const formatValue = (value: unknown): string => { if (value instanceof Date) { return value.toISOString(); } if (typeof value === "string") { return JSON.stringify(value); } if ( typeof value === "number" || typeof value === "boolean" || value == null ) { return String(value); } try { return JSON.stringify(value); } catch { return "[unserializable]"; } }; const formatUniqueConstraintError = ( error: unknown, data: Row | null ): string => { const fields = getUniqueConstraintFields(error); if (fields.length === 0) { return "Unique constraint failed."; } const fieldValues = fields.map((field) => { const value = data ? data[field] : undefined; return `${field}=${formatValue(value)}`; }); return `Unique constraint failed on field(s): ${fields.join( ", " )}. Values: ${fieldValues.join(", ")}`; }; const getConfigForTable = (table: string): SyncTableConfig | null => { const configMap: Record = { member: { sourceModel: "member", targetModel: "user", translation: userTranslation as unknown as AnyTranslation, uniqueField: "cwMemberId", lastUpdatedField: "lastUpdatedUtc", }, cwMember: { sourceModel: "member", targetModel: "cwMember", translation: cwMemberTranslation as unknown as AnyTranslation, uniqueField: "cwMemberId", lastUpdatedField: "lastUpdatedUtc", }, company: { sourceModel: "company", targetModel: "company", translation: companyTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdate", }, companyAddress: { sourceModel: "companyAddress", targetModel: "companyAddress", translation: companyAddressTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, contact: { sourceModel: "contact", targetModel: "contact", translation: contactTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, ownerLevel: { sourceModel: "ownerLevel", targetModel: "corporateLocation", translation: corporateLocationTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, department: { sourceModel: "department", targetModel: "internalDepartment", translation: internalDepartmentTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUTC", }, productType: { sourceModel: "productType", targetModel: "catalogItemType", translation: catalogItemTypeTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, productCategory: { sourceModel: "productCategory", targetModel: "catalogCategory", translation: catalogCategoryTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, productSubcategory: { sourceModel: "productSubcategory", targetModel: "catalogSubcategory", translation: catalogSubcategoryTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, manufacturer: { sourceModel: "manufacturer", targetModel: "catalogManufacturer", translation: catalogManufacturerTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, warehouseBin: { sourceModel: "warehouseBin", targetModel: "warehouseBin", translation: warehouseBinTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, productCatalog: { sourceModel: "productCatalog", targetModel: "catalogItem", translation: catalogItemTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, productInventory: { sourceModel: "productInventory", targetModel: "productInventory", translation: productInventoryTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, srType: { sourceModel: "srType", targetModel: "serviceTicketType", translation: serviceTicketTypeTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, srLocation: { sourceModel: "srLocation", targetModel: "serviceTicketLocation", translation: serviceTicketLocationTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, srSource: { sourceModel: "srSource", targetModel: "serviceTicketSource", translation: serviceTicketSourceTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, srImpact: { sourceModel: "srImpact", targetModel: "serviceTicketImpact", translation: serviceTicketImpactTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, srSeverity: { sourceModel: "srSeverity", targetModel: "serviceTicketSeverity", translation: serviceTicketSeverityTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, srUrgency: { sourceModel: "srUrgency", targetModel: "serviceTicketPriority", translation: serviceTicketPriorityTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, srBoard: { sourceModel: "srBoard", targetModel: "serviceTicketBoard", translation: serviceTicketBoardTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, soType: { sourceModel: "soType", targetModel: "opportunityType", translation: opportunityTypeTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, soOppStatus: { sourceModel: "soOppStatus", targetModel: "opportunityStatus", translation: opportunityStatusTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, opportunity: { sourceModel: "opportunity", targetModel: "opportunity", translation: opportunityTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", sourceArgs: { include: { members: { select: { memberRecId: true, primarySalesFlag: true, secondarySalesFlag: true, }, }, soOppStatus: { select: { closedFlag: true, }, }, }, }, }, srService: { sourceModel: "srService", targetModel: "serviceTicket", translation: serviceTicketTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, iV_Product: { sourceModel: "iV_Product", targetModel: "productData", translation: productDataTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUTC", }, ticketNote: { sourceModel: "ticketNote", targetModel: "serviceTicketNote", translation: serviceTicketNoteTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdatedUtc", }, scheduleStatus: { sourceModel: "scheduleStatus", targetModel: "scheduleStatus", translation: scheduleStatusTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdateUtc", }, scheduleType: { sourceModel: "scheduleType", targetModel: "scheduleType", translation: scheduleTypeTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdateUtc", }, scheduleSpan: { sourceModel: "scheduleSpan", targetModel: "scheduleSpan", translation: scheduleSpanTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "scheduleSpanRecId", }, schedule: { sourceModel: "schedule", targetModel: "schedule", translation: scheduleTranslation as unknown as AnyTranslation, uniqueField: "id", lastUpdatedField: "lastUpdateUtc", }, }; return configMap[table] ?? null; }; let _cwPrisma: CwPrismaClient | null = null; let _apiPrisma: ApiPrismaClient | null = null; const getClients = (): { cwPrisma: CwPrismaClient; apiPrisma: ApiPrismaClient; } => { if (!_cwPrisma || !_apiPrisma) { const cwDatabaseUrl = process.env.CW_DATABASE_URL || process.env.DATABASE_URL; const apiDatabaseUrl = resolveApiDatabaseUrl(); if (!cwDatabaseUrl) { throw new Error( "Missing CW DB connection string. Set CW_DATABASE_URL or DATABASE_URL." ); } if (!apiDatabaseUrl) { throw new Error( "Missing API DB connection string. Set API_DATABASE_URL/OPTIMA_API_DATABASE_URL or ensure ../api/.env has DATABASE_URL." ); } _cwPrisma = new CwPrismaClient({ adapter: new PrismaMssql(cwDatabaseUrl) }); _apiPrisma = new ApiPrismaClient({ adapter: new PrismaPg({ connectionString: apiDatabaseUrl }), }); } return { cwPrisma: _cwPrisma, apiPrisma: _apiPrisma }; }; export async function disconnectSyncClients(): Promise { await Promise.all([_cwPrisma?.$disconnect(), _apiPrisma?.$disconnect()]); _cwPrisma = null; _apiPrisma = null; } /** * Syncs a specific number of records from a CW table to the API database. * Records are pulled ordered by lastUpdatedUTC in descending order (most recent first). * * @param table - The name of the source table (e.g., 'opportunity', 'srService', 'company') * @param numRec - The number of records to pull and sync * @returns SyncResult containing counts of inserted/updated, skipped, and failed records */ export async function syncTableUpdates( table: string, numRec: number ): Promise { const config = getConfigForTable(table); if (!config) { throw new Error( `Unsupported table: ${table}. No sync configuration found.` ); } const { cwPrisma, apiPrisma } = getClients(); const context = createTranslationContext(); // Refresh context needed for translations await refreshContextFromApi(apiPrisma, context); if ( config.targetModel === "opportunity" || config.targetModel === "productData" ) { await refreshCustomFieldContextFromCw(cwPrisma, context); } // Build the source args with lastUpdatedField ordering and take limit const sourceArgs = { ...config.sourceArgs, take: numRec, orderBy: { [config.lastUpdatedField]: "desc", }, }; const sourceDelegate = ( cwPrisma as unknown as Record )[config.sourceModel]; const targetDelegate = ( apiPrisma as unknown as Record )[config.targetModel]; if (!sourceDelegate) { throw new Error(`CW delegate not found: ${config.sourceModel}`); } if (!targetDelegate) { throw new Error(`API delegate not found: ${config.targetModel}`); } // Fetch the records const rows = (await sourceDelegate.findMany(sourceArgs)) as Row[]; let insertedOrUpdated = 0; let skipped = 0; let failed = 0; let sampleErrorsPrinted = 0; // Process each row for (const row of rows) { let translatedData: Row | null = null; try { const translated = translateRow(row, config.translation, context); const userSanitized = sanitizeUserForeignKeys( translated, context, config.targetModel ); const data = sanitizeModelData( userSanitized, config.targetModel, context ); translatedData = data; const uniqueValue = data[config.uniqueField]; if ( uniqueValue === undefined || uniqueValue === null || uniqueValue === "" ) { skipped += 1; continue; } await targetDelegate.upsert({ where: { [config.uniqueField]: uniqueValue, }, create: data, update: data, }); insertedOrUpdated += 1; } catch (error) { if (error instanceof SkipRowError) { skipped += 1; continue; } failed += 1; if (sampleErrorsPrinted < 5) { sampleErrorsPrinted += 1; const uniqueFields = getUniqueConstraintFields(error); const message = uniqueFields.length > 0 ? formatUniqueConstraintError(error, translatedData) : error instanceof Error ? error.message : "Unknown row sync error"; console.error( `Failed row in ${config.sourceModel} -> ${config.targetModel}:`, message ); } } } if (failed > sampleErrorsPrinted) { console.error( `${config.sourceModel}: suppressed ${ failed - sampleErrorsPrinted } additional row errors` ); } return { insertedOrUpdated, skipped, failed }; }