all the haul
This commit is contained in:
@@ -0,0 +1,334 @@
|
||||
import { PrismaMssql } from "@prisma/adapter-mssql";
|
||||
import { PrismaPg } from "@prisma/adapter-pg";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { PrismaClient as CwPrismaClient } from "./generated/prisma/client";
|
||||
import { PrismaClient as ApiPrismaClient } from "../api/generated/prisma/client";
|
||||
|
||||
type EnvMap = Record<string, string>;
|
||||
|
||||
type Summary = {
|
||||
cwTotal: number;
|
||||
apiTotal: number;
|
||||
eligibleForSync: number;
|
||||
missingSrServiceRecId: number;
|
||||
missingMemberRecId: number;
|
||||
missingParentTicketInApi: number;
|
||||
missingAuthorMappingInApi: number;
|
||||
eligibleButMissingInApi: number;
|
||||
sampleMissingParentTicketIds: number[];
|
||||
sampleMissingAuthorMemberRecIds: number[];
|
||||
sampleEligibleButMissingNoteIds: number[];
|
||||
topMissingAuthorMemberRecIds: Array<{ memberRecId: number; count: number }>;
|
||||
topMissingParentTicketIds: Array<{ srServiceRecId: number; count: number }>;
|
||||
automateApi: {
|
||||
total: number;
|
||||
eligibleForSync: number;
|
||||
missingSrServiceRecId: number;
|
||||
missingMemberRecId: number;
|
||||
missingParentTicketInApi: number;
|
||||
missingAuthorMappingInApi: number;
|
||||
eligibleButMissingInApi: number;
|
||||
sampleMemberRecIds: number[];
|
||||
sampleNoteIdsMissingInApi: number[];
|
||||
};
|
||||
};
|
||||
|
||||
const isAutomateApiAuthor = (
|
||||
createdBy: string | null,
|
||||
originalAuthor: string | null
|
||||
): boolean => {
|
||||
const normalizedCreatedBy = createdBy?.trim().toLowerCase() ?? "";
|
||||
const normalizedOriginalAuthor = originalAuthor?.trim().toLowerCase() ?? "";
|
||||
return (
|
||||
normalizedCreatedBy.includes("automateapi") ||
|
||||
normalizedOriginalAuthor.includes("automateapi")
|
||||
);
|
||||
};
|
||||
|
||||
const parseEnvFile = (path: string): EnvMap => {
|
||||
const envData = readFileSync(path, "utf8");
|
||||
const out: EnvMap = {};
|
||||
|
||||
for (const rawLine of envData.split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
|
||||
const index = line.indexOf("=");
|
||||
if (index <= 0) continue;
|
||||
|
||||
const key = line.slice(0, index).trim();
|
||||
let value = line.slice(index + 1).trim();
|
||||
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
|
||||
out[key] = value;
|
||||
}
|
||||
|
||||
return out;
|
||||
};
|
||||
|
||||
const readApiEnv = (): EnvMap => {
|
||||
const candidates = [
|
||||
resolve(import.meta.dir, "../api/.env"),
|
||||
resolve(process.cwd(), "../api/.env"),
|
||||
resolve(process.cwd(), "api/.env"),
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
return parseEnvFile(candidate);
|
||||
} catch {
|
||||
// Try next
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
};
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const apiEnv = readApiEnv();
|
||||
|
||||
const cwDatabaseUrl =
|
||||
process.env.CW_DATABASE_URL ||
|
||||
process.env.DATABASE_URL ||
|
||||
apiEnv.CW_DATABASE_URL;
|
||||
const apiDatabaseUrl =
|
||||
process.env.API_DATABASE_URL ||
|
||||
process.env.OPTIMA_API_DATABASE_URL ||
|
||||
apiEnv.API_DATABASE_URL ||
|
||||
apiEnv.OPTIMA_API_DATABASE_URL ||
|
||||
apiEnv.DATABASE_URL;
|
||||
|
||||
if (!cwDatabaseUrl) {
|
||||
throw new Error("Missing CW DB URL. Set CW_DATABASE_URL or DATABASE_URL.");
|
||||
}
|
||||
|
||||
if (!apiDatabaseUrl) {
|
||||
throw new Error(
|
||||
"Missing API DB URL. Set API_DATABASE_URL/OPTIMA_API_DATABASE_URL or provide api/.env DATABASE_URL."
|
||||
);
|
||||
}
|
||||
|
||||
const cwPrisma = new CwPrismaClient({
|
||||
adapter: new PrismaMssql(cwDatabaseUrl),
|
||||
});
|
||||
const apiPrisma = new ApiPrismaClient({
|
||||
adapter: new PrismaPg({ connectionString: apiDatabaseUrl }),
|
||||
});
|
||||
|
||||
try {
|
||||
console.log("[diag] Loading API reference sets...");
|
||||
const [apiNotes, apiTickets, apiUsers] = await Promise.all([
|
||||
apiPrisma.serviceTicketNote.findMany({ select: { id: true } }),
|
||||
apiPrisma.serviceTicket.findMany({ select: { id: true } }),
|
||||
apiPrisma.user.findMany({ select: { cwMemberId: true } }),
|
||||
]);
|
||||
|
||||
const apiNoteIds = new Set<number>(apiNotes.map((r) => r.id));
|
||||
const apiTicketIds = new Set<number>(apiTickets.map((r) => r.id));
|
||||
const apiUserMemberIds = new Set<number>(
|
||||
apiUsers
|
||||
.map((r) => r.cwMemberId)
|
||||
.filter((v): v is number => Number.isInteger(v))
|
||||
);
|
||||
|
||||
console.log(
|
||||
`[diag] API sets: notes=${apiNoteIds.size} tickets=${apiTicketIds.size} usersWithCwMemberId=${apiUserMemberIds.size}`
|
||||
);
|
||||
|
||||
const cwTotal = await cwPrisma.ticketNote.count();
|
||||
const apiTotal = apiNoteIds.size;
|
||||
|
||||
let missingSrServiceRecId = 0;
|
||||
let missingMemberRecId = 0;
|
||||
let missingParentTicketInApi = 0;
|
||||
let missingAuthorMappingInApi = 0;
|
||||
let eligibleForSync = 0;
|
||||
let eligibleButMissingInApi = 0;
|
||||
|
||||
let automateTotal = 0;
|
||||
let automateEligibleForSync = 0;
|
||||
let automateMissingSrServiceRecId = 0;
|
||||
let automateMissingMemberRecId = 0;
|
||||
let automateMissingParentTicketInApi = 0;
|
||||
let automateMissingAuthorMappingInApi = 0;
|
||||
let automateEligibleButMissingInApi = 0;
|
||||
|
||||
const sampleMissingParentTicketIds: number[] = [];
|
||||
const sampleMissingAuthorMemberRecIds: number[] = [];
|
||||
const sampleEligibleButMissingNoteIds: number[] = [];
|
||||
const automateSampleMemberRecIds: number[] = [];
|
||||
const automateSampleNoteIdsMissingInApi: number[] = [];
|
||||
const missingAuthorCounts = new Map<number, number>();
|
||||
const missingParentCounts = new Map<number, number>();
|
||||
|
||||
let cursor = 0;
|
||||
const batchSize = 5000;
|
||||
|
||||
console.log(
|
||||
`[diag] Scanning CW TicketNote rows in batches of ${batchSize}...`
|
||||
);
|
||||
|
||||
while (true) {
|
||||
const batch = await cwPrisma.ticketNote.findMany({
|
||||
where: {
|
||||
ticketNoteRecId: {
|
||||
gt: cursor,
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
ticketNoteRecId: "asc",
|
||||
},
|
||||
select: {
|
||||
ticketNoteRecId: true,
|
||||
srServiceRecId: true,
|
||||
memberRecId: true,
|
||||
createdBy: true,
|
||||
originalAuthor: true,
|
||||
},
|
||||
take: batchSize,
|
||||
});
|
||||
|
||||
if (batch.length === 0) break;
|
||||
|
||||
for (const row of batch) {
|
||||
const noteId = row.ticketNoteRecId;
|
||||
const srServiceRecId = row.srServiceRecId;
|
||||
const memberRecId = row.memberRecId;
|
||||
const isAutomate = isAutomateApiAuthor(
|
||||
row.createdBy,
|
||||
row.originalAuthor
|
||||
);
|
||||
|
||||
if (isAutomate) {
|
||||
automateTotal++;
|
||||
if (
|
||||
memberRecId &&
|
||||
automateSampleMemberRecIds.length < 20 &&
|
||||
!automateSampleMemberRecIds.includes(memberRecId)
|
||||
) {
|
||||
automateSampleMemberRecIds.push(memberRecId);
|
||||
}
|
||||
}
|
||||
|
||||
let blocked = false;
|
||||
|
||||
if (!srServiceRecId) {
|
||||
missingSrServiceRecId++;
|
||||
if (isAutomate) {
|
||||
automateMissingSrServiceRecId++;
|
||||
}
|
||||
blocked = true;
|
||||
} else if (!apiTicketIds.has(srServiceRecId)) {
|
||||
missingParentTicketInApi++;
|
||||
if (isAutomate) {
|
||||
automateMissingParentTicketInApi++;
|
||||
}
|
||||
blocked = true;
|
||||
missingParentCounts.set(
|
||||
srServiceRecId,
|
||||
(missingParentCounts.get(srServiceRecId) ?? 0) + 1
|
||||
);
|
||||
if (sampleMissingParentTicketIds.length < 20) {
|
||||
sampleMissingParentTicketIds.push(srServiceRecId);
|
||||
}
|
||||
}
|
||||
|
||||
if (!memberRecId) {
|
||||
missingMemberRecId++;
|
||||
if (isAutomate) {
|
||||
automateMissingMemberRecId++;
|
||||
}
|
||||
blocked = true;
|
||||
} else if (!apiUserMemberIds.has(memberRecId)) {
|
||||
missingAuthorMappingInApi++;
|
||||
if (isAutomate) {
|
||||
automateMissingAuthorMappingInApi++;
|
||||
}
|
||||
blocked = true;
|
||||
missingAuthorCounts.set(
|
||||
memberRecId,
|
||||
(missingAuthorCounts.get(memberRecId) ?? 0) + 1
|
||||
);
|
||||
if (sampleMissingAuthorMemberRecIds.length < 20) {
|
||||
sampleMissingAuthorMemberRecIds.push(memberRecId);
|
||||
}
|
||||
}
|
||||
|
||||
if (!blocked) {
|
||||
eligibleForSync++;
|
||||
if (isAutomate) {
|
||||
automateEligibleForSync++;
|
||||
}
|
||||
if (!apiNoteIds.has(noteId)) {
|
||||
eligibleButMissingInApi++;
|
||||
if (isAutomate) {
|
||||
automateEligibleButMissingInApi++;
|
||||
if (automateSampleNoteIdsMissingInApi.length < 20) {
|
||||
automateSampleNoteIdsMissingInApi.push(noteId);
|
||||
}
|
||||
}
|
||||
if (sampleEligibleButMissingNoteIds.length < 50) {
|
||||
sampleEligibleButMissingNoteIds.push(noteId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cursor = batch[batch.length - 1]!.ticketNoteRecId;
|
||||
if (cursor % 50000 < batchSize) {
|
||||
console.log(`[diag] Progress cursor=${cursor}`);
|
||||
}
|
||||
}
|
||||
|
||||
const summary: Summary = {
|
||||
cwTotal,
|
||||
apiTotal,
|
||||
eligibleForSync,
|
||||
missingSrServiceRecId,
|
||||
missingMemberRecId,
|
||||
missingParentTicketInApi,
|
||||
missingAuthorMappingInApi,
|
||||
eligibleButMissingInApi,
|
||||
sampleMissingParentTicketIds,
|
||||
sampleMissingAuthorMemberRecIds,
|
||||
sampleEligibleButMissingNoteIds,
|
||||
topMissingAuthorMemberRecIds: [...missingAuthorCounts.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 20)
|
||||
.map(([memberRecId, count]) => ({ memberRecId, count })),
|
||||
topMissingParentTicketIds: [...missingParentCounts.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 20)
|
||||
.map(([srServiceRecId, count]) => ({ srServiceRecId, count })),
|
||||
automateApi: {
|
||||
total: automateTotal,
|
||||
eligibleForSync: automateEligibleForSync,
|
||||
missingSrServiceRecId: automateMissingSrServiceRecId,
|
||||
missingMemberRecId: automateMissingMemberRecId,
|
||||
missingParentTicketInApi: automateMissingParentTicketInApi,
|
||||
missingAuthorMappingInApi: automateMissingAuthorMappingInApi,
|
||||
eligibleButMissingInApi: automateEligibleButMissingInApi,
|
||||
sampleMemberRecIds: automateSampleMemberRecIds,
|
||||
sampleNoteIdsMissingInApi: automateSampleNoteIdsMissingInApi,
|
||||
},
|
||||
};
|
||||
|
||||
console.log("[diag] TicketNote sync gap summary:");
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
} finally {
|
||||
await Promise.all([cwPrisma.$disconnect(), apiPrisma.$disconnect()]);
|
||||
}
|
||||
};
|
||||
|
||||
main().catch((error) => {
|
||||
console.error("[diag] Failed:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user