perf(csv-loader): optimize reverse reference resolution
Implement a reverse lookup cache to avoid re-filtering the entire referenced table for every row during reverse reference resolution. This improves performance from O(N*M) to O(N+M) where N is the number of rows in the current table and M is the number of rows in the referenced table. Also update documentation to reflect the new union resolution behavior and migration notes for version 2.0.0.
This commit is contained in:
@@ -12,6 +12,16 @@ import { parseCsv } from "./loader.js";
|
||||
/** Cache for loaded referenced tables */
|
||||
const referenceTableCache = new Map<string, Record<string, unknown>[]>();
|
||||
|
||||
/**
|
||||
* Cache for reverse-reference lookups: filePath -> (foreignKey -> (fkValue -> rows[])).
|
||||
* Built once per referenced table + foreign key, mirroring what module-gen emits,
|
||||
* so resolveReverseReference doesn't re-filter the whole table per row.
|
||||
*/
|
||||
const reverseLookupCache = new Map<
|
||||
string,
|
||||
Map<string, Map<string, Record<string, unknown>[]>>
|
||||
>();
|
||||
|
||||
/** Set of file paths currently being loaded (to detect circular references) */
|
||||
const loadingFiles = new Set<string>();
|
||||
|
||||
@@ -39,6 +49,7 @@ export function loadReferenceTable(
|
||||
): {
|
||||
lookup: Map<string, Record<string, unknown>>;
|
||||
refTable: Record<string, unknown>[];
|
||||
refFilePath: string;
|
||||
} {
|
||||
const baseDir =
|
||||
refBaseDir ||
|
||||
@@ -83,7 +94,51 @@ export function loadReferenceTable(
|
||||
}
|
||||
});
|
||||
|
||||
return { lookup, refTable };
|
||||
return { lookup, refTable, refFilePath };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build (and cache) a reverse lookup for a referenced table + foreign key:
|
||||
* a map from the foreign-key value to the rows that reference it.
|
||||
*/
|
||||
function getReverseLookup(
|
||||
schema: ReverseReferenceSchema,
|
||||
refBaseDir: string | undefined,
|
||||
defaultPrimaryKey: string,
|
||||
currentFilePath: string | undefined,
|
||||
): Map<string, Record<string, unknown>[]> {
|
||||
const { refTable, refFilePath } = loadReferenceTable(
|
||||
schema,
|
||||
refBaseDir,
|
||||
defaultPrimaryKey,
|
||||
currentFilePath,
|
||||
);
|
||||
|
||||
let byForeignKey = reverseLookupCache.get(refFilePath);
|
||||
if (!byForeignKey) {
|
||||
byForeignKey = new Map();
|
||||
reverseLookupCache.set(refFilePath, byForeignKey);
|
||||
}
|
||||
|
||||
const cached = byForeignKey.get(schema.foreignKey);
|
||||
if (cached) return cached;
|
||||
|
||||
const lookup = new Map<string, Record<string, unknown>[]>();
|
||||
for (const row of refTable) {
|
||||
const fkValue = row[schema.foreignKey];
|
||||
const fkStr =
|
||||
fkValue !== null && fkValue !== undefined && typeof fkValue === "object"
|
||||
? String((fkValue as Record<string, unknown>)[defaultPrimaryKey])
|
||||
: String(fkValue);
|
||||
const bucket = lookup.get(fkStr);
|
||||
if (bucket) {
|
||||
bucket.push(row);
|
||||
} else {
|
||||
lookup.set(fkStr, [row]);
|
||||
}
|
||||
}
|
||||
byForeignKey.set(schema.foreignKey, lookup);
|
||||
return lookup;
|
||||
}
|
||||
|
||||
export function resolveReferenceId(
|
||||
@@ -333,21 +388,13 @@ export function resolveReverseReference(
|
||||
defaultPrimaryKey: string,
|
||||
currentFilePath: string | undefined,
|
||||
): Record<string, unknown>[] {
|
||||
const { refTable } = loadReferenceTable(
|
||||
const lookup = getReverseLookup(
|
||||
schema,
|
||||
refBaseDir,
|
||||
defaultPrimaryKey,
|
||||
currentFilePath,
|
||||
);
|
||||
const pkStr = String(pkValue);
|
||||
return refTable.filter((row) => {
|
||||
const fkValue = row[schema.foreignKey];
|
||||
const fkStr =
|
||||
fkValue !== null && fkValue !== undefined && typeof fkValue === "object"
|
||||
? String((fkValue as Record<string, unknown>)[defaultPrimaryKey])
|
||||
: String(fkValue);
|
||||
return fkStr === pkStr;
|
||||
});
|
||||
return lookup.get(String(pkValue)) ?? [];
|
||||
}
|
||||
|
||||
export function resolveNestedReferences(
|
||||
|
||||
@@ -43,6 +43,40 @@ describe("parseCsv - reverse reference resolution", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("should resolve reverse reference when the foreign key is an object reference", () => {
|
||||
// The referenced table's foreign key column is itself a reference, so the
|
||||
// stored value is an object and the reverse lookup must key on its primary key.
|
||||
const ordersCsvPath = path.join(fixturesDir, "rev_orders.csv");
|
||||
const ordersContent = [
|
||||
"id,customer,total",
|
||||
"string,@users,number",
|
||||
"1,1,100",
|
||||
"2,1,50",
|
||||
].join("\n");
|
||||
fs.writeFileSync(ordersCsvPath, ordersContent);
|
||||
|
||||
try {
|
||||
const csv = [
|
||||
"id,name",
|
||||
"string,string",
|
||||
"# inject orders = ~rev_orders(customer)",
|
||||
"1,Alice",
|
||||
].join("\n");
|
||||
|
||||
const result = parseCsv(csv, {
|
||||
emitTypes: false,
|
||||
currentFilePath: path.join(fixturesDir, "test.csv"),
|
||||
});
|
||||
|
||||
const orders = result.data[0].orders as Record<string, unknown>[];
|
||||
expect(orders).toHaveLength(2);
|
||||
expect(orders[0].id).toBe("1");
|
||||
expect(orders[1].id).toBe("2");
|
||||
} finally {
|
||||
fs.unlinkSync(ordersCsvPath);
|
||||
}
|
||||
});
|
||||
|
||||
it("should return empty array for reverse reference with no matches", () => {
|
||||
const ordersCsvPath = path.join(fixturesDir, "rev_orders.csv");
|
||||
const ordersContent = [
|
||||
|
||||
Reference in New Issue
Block a user