perf(csv-loader): optimize reverse reference resolution

Implement a reverse lookup cache to avoid re-filtering the entire
referenced table for every row during reverse reference resolution.
This improves performance from O(N*M) to O(N+M) where N is the number
of rows in the current table and M is the number of rows in the
referenced table.

Also update documentation to reflect the new union resolution
behavior and migration notes for version 2.0.0.
This commit is contained in:
2026-08-06 10:27:18 +08:00
parent 37e3514c0c
commit 641af7341a
5 changed files with 115 additions and 12 deletions
+58 -11
View File
@@ -12,6 +12,16 @@ import { parseCsv } from "./loader.js";
/** Cache for loaded referenced tables */
const referenceTableCache = new Map<string, Record<string, unknown>[]>();
/**
* Cache for reverse-reference lookups: filePath -> (foreignKey -> (fkValue -> rows[])).
* Built once per referenced table + foreign key, mirroring what module-gen emits,
* so resolveReverseReference doesn't re-filter the whole table per row.
*/
const reverseLookupCache = new Map<
string,
Map<string, Map<string, Record<string, unknown>[]>>
>();
/** Set of file paths currently being loaded (to detect circular references) */
const loadingFiles = new Set<string>();
@@ -39,6 +49,7 @@ export function loadReferenceTable(
): {
lookup: Map<string, Record<string, unknown>>;
refTable: Record<string, unknown>[];
refFilePath: string;
} {
const baseDir =
refBaseDir ||
@@ -83,7 +94,51 @@ export function loadReferenceTable(
}
});
return { lookup, refTable };
return { lookup, refTable, refFilePath };
}
/**
* Build (and cache) a reverse lookup for a referenced table + foreign key:
* a map from the foreign-key value to the rows that reference it.
*/
function getReverseLookup(
schema: ReverseReferenceSchema,
refBaseDir: string | undefined,
defaultPrimaryKey: string,
currentFilePath: string | undefined,
): Map<string, Record<string, unknown>[]> {
const { refTable, refFilePath } = loadReferenceTable(
schema,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
);
let byForeignKey = reverseLookupCache.get(refFilePath);
if (!byForeignKey) {
byForeignKey = new Map();
reverseLookupCache.set(refFilePath, byForeignKey);
}
const cached = byForeignKey.get(schema.foreignKey);
if (cached) return cached;
const lookup = new Map<string, Record<string, unknown>[]>();
for (const row of refTable) {
const fkValue = row[schema.foreignKey];
const fkStr =
fkValue !== null && fkValue !== undefined && typeof fkValue === "object"
? String((fkValue as Record<string, unknown>)[defaultPrimaryKey])
: String(fkValue);
const bucket = lookup.get(fkStr);
if (bucket) {
bucket.push(row);
} else {
lookup.set(fkStr, [row]);
}
}
byForeignKey.set(schema.foreignKey, lookup);
return lookup;
}
export function resolveReferenceId(
@@ -333,21 +388,13 @@ export function resolveReverseReference(
defaultPrimaryKey: string,
currentFilePath: string | undefined,
): Record<string, unknown>[] {
const { refTable } = loadReferenceTable(
const lookup = getReverseLookup(
schema,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
);
const pkStr = String(pkValue);
return refTable.filter((row) => {
const fkValue = row[schema.foreignKey];
const fkStr =
fkValue !== null && fkValue !== undefined && typeof fkValue === "object"
? String((fkValue as Record<string, unknown>)[defaultPrimaryKey])
: String(fkValue);
return fkStr === pkStr;
});
return lookup.get(String(pkValue)) ?? [];
}
export function resolveNestedReferences(