refactor(csv-loader): decouple reference resolution and module

generation

Extract reference resolution logic, type generation, and module
generation into dedicated modules to improve maintainability and
clean up the core loader.
This commit is contained in:
hypercross 2026-04-20 00:48:01 +08:00
parent eeaac92e39
commit f94e9b68e4
9 changed files with 1061 additions and 1012 deletions

View File

@ -1,8 +1,8 @@
import type { CsvLoaderOptions } from './loader.js';
import { csvToModule } from './loader.js';
import * as path from 'path';
import * as fs from 'fs';
import type { Plugin, OnLoadResult } from 'esbuild';
import * as path from "path";
import * as fs from "fs";
import type { Plugin, OnLoadResult } from "esbuild";
import { csvToModule } from "./module-gen";
import { CsvLoaderOptions } from "./types";
export interface CsvEsbuildOptions extends CsvLoaderOptions {
/** Include pattern for CSV files (default: /\.csv$/) */
@ -16,7 +16,7 @@ export interface CsvEsbuildOptions extends CsvLoaderOptions {
}
function createFilter(
pattern: RegExp | string | Array<RegExp | string>
pattern: RegExp | string | Array<RegExp | string>,
): RegExp {
if (pattern instanceof RegExp) return pattern;
if (Array.isArray(pattern)) {
@ -28,7 +28,7 @@ function createFilter(
function matchesPattern(
id: string,
pattern: RegExp | string | Array<RegExp | string> | undefined
pattern: RegExp | string | Array<RegExp | string> | undefined,
): boolean {
if (!pattern) return true;
const patterns = Array.isArray(pattern) ? pattern : [pattern];
@ -46,7 +46,7 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin {
include = /\.csv$/,
exclude,
emitTypes = true,
typesOutputDir = '',
typesOutputDir = "",
writeToDisk = false,
...parseOptions
} = options;
@ -54,7 +54,7 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin {
const includeFilter = createFilter(include);
return {
name: 'inline-schema-csv',
name: "inline-schema-csv",
setup(build) {
build.onLoad({ filter: includeFilter }, async (args) => {
@ -66,7 +66,7 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin {
// Read the file content
let content: string;
try {
content = await fs.promises.readFile(args.path, 'utf-8');
content = await fs.promises.readFile(args.path, "utf-8");
} catch (error) {
return {
errors: [{ text: `Failed to read file: ${args.path}` }],
@ -74,9 +74,11 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin {
}
// Infer resource name from filename
const fileName = path.basename(args.path, '.csv').split('.')[0];
const fileName = path.basename(args.path, ".csv").split(".")[0];
const resourceName = fileName
.replace(/[-_\s]+(.)?/g, (_, char) => char ? char.toUpperCase() : '')
.replace(/[-_\s]+(.)?/g, (_, char) =>
char ? char.toUpperCase() : "",
)
.replace(/^(.)/, (_, char) => char.toUpperCase());
const result = csvToModule(content, {
@ -91,8 +93,8 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin {
// Emit type definition file if enabled
if (emitTypes && result.dts) {
const dtsPath = typesOutputDir
? path.join(typesOutputDir, path.basename(args.path) + '.d.ts')
: args.path + '.d.ts';
? path.join(typesOutputDir, path.basename(args.path) + ".d.ts")
: args.path + ".d.ts";
if (writeToDisk) {
const absolutePath = path.isAbsolute(dtsPath)
@ -105,7 +107,7 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin {
return {
contents: result.js,
loader: 'js' as const,
loader: "js" as const,
};
});
},

View File

@ -1,7 +1,8 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { parseCsv, csvToModule } from "./loader";
import { parseCsv } from "./loader";
import * as path from "path";
import * as fs from "fs";
import { csvToModule } from "./module-gen";
const fixturesDir = path.join(__dirname, "fixtures");

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,250 @@
import * as path from "path";
import { parseCsv } from "./loader.js";
import { hasNestedReferences } from "./reference-resolver.js";
import type { Schema } from "../types.js";
import type { CsvLoaderOptions, PropertyConfig } from "./types.js";
/**
* Generate runtime reference resolution code for a schema.
* Returns a JS expression string that resolves references using lookup maps.
*/
function generateSchemaResolutionCode(
schema: Schema,
valueExpr: string,
lookupVar: (tableName: string) => string,
pkField: string,
reverseLookupVar?: (tableName: string, foreignKey: string) => string,
): string {
switch (schema.type) {
case "reference": {
const lookup = lookupVar(schema.tableName);
if (schema.isOptional) {
if (schema.isArray) {
return `(${valueExpr} === null || ${valueExpr} === undefined ? ${valueExpr} : (Array.isArray(${valueExpr}) ? ${valueExpr}.map(id => ${lookup}.get(String(id))) : ${lookup}.get(String(${valueExpr}))))`;
}
return `(${valueExpr} === null || ${valueExpr} === undefined ? ${valueExpr} : ${lookup}.get(String(${valueExpr})))`;
}
if (schema.isArray) {
return `(Array.isArray(${valueExpr}) ? ${valueExpr}.map(id => ${lookup}.get(String(id))) : ${valueExpr})`;
}
return `${lookup}.get(String(${valueExpr}))`;
}
case "reverseReference": {
if (!reverseLookupVar) return valueExpr;
const reverseLookup = reverseLookupVar(
schema.tableName,
schema.foreignKey,
);
if (schema.isOptional) {
return `(${reverseLookup}.get(String(row.${pkField})) || null)`;
}
return `(${reverseLookup}.get(String(row.${pkField})) || [])`;
}
case "tuple": {
const elementResolvers = schema.elements.map((el, i) => {
if (hasNestedReferences(el.schema)) {
return generateSchemaResolutionCode(
el.schema,
`${valueExpr}[${i}]`,
lookupVar,
pkField,
reverseLookupVar,
);
}
return `${valueExpr}[${i}]`;
});
return `[${elementResolvers.join(", ")}]`;
}
case "array": {
if (hasNestedReferences(schema.element)) {
const itemResolve = generateSchemaResolutionCode(
schema.element,
"item",
lookupVar,
pkField,
reverseLookupVar,
);
return `(${valueExpr}).map(item => ${itemResolve})`;
}
return valueExpr;
}
case "union": {
const refMembers = schema.members.filter((m) => hasNestedReferences(m));
const nonRefMembers = schema.members.filter(
(m) => !hasNestedReferences(m),
);
const resolveParts: string[] = [];
for (const member of refMembers) {
const resolveCode = generateSchemaResolutionCode(
member,
valueExpr,
lookupVar,
pkField,
reverseLookupVar,
);
resolveParts.push(resolveCode);
}
if (nonRefMembers.length > 0) {
resolveParts.push(valueExpr);
}
if (resolveParts.length === 0) return valueExpr;
if (resolveParts.length === 1) return resolveParts[0];
return `(${resolveParts.join(" ?? ")})`;
}
default:
return valueExpr;
}
}
/**
* Generate JavaScript module code from CSV content.
* Emits an accessor function for tables with references (lazy resolution),
* or static JSON for tables without references.
*/
export function csvToModule(
content: string,
options: CsvLoaderOptions & { resourceName?: string } = {},
): { js: string; dts?: string } {
const result = parseCsv(content, { ...options, resolveReferences: false });
const hasRefs =
result.referenceFields.length > 0 || result.reverseReferences.length > 0;
const defaultPrimaryKey = options.defaultPrimaryKey ?? "id";
const imports: string[] = [];
const lookupInits: string[] = [];
const lookupVarMap = new Map<string, string>();
// Reverse lookup maps: grouped by (tableName, foreignKey)
const reverseLookupInits: string[] = [];
const reverseLookupVarMap = new Map<string, string>();
const currentTableName = options.currentFilePath
? path.basename(
options.currentFilePath,
path.extname(options.currentFilePath),
)
: undefined;
// Build forward lookup maps for referenced tables
const uniqueTables = new Set(result.referenceFields.map((f) => f.tableName));
// Also include tables from reverse references
for (const decl of result.reverseReferences) {
uniqueTables.add(decl.tableName);
}
uniqueTables.forEach((tableName) => {
const lookupVar = `_${tableName}Lookup`;
lookupVarMap.set(tableName, lookupVar);
if (tableName === currentTableName) {
lookupInits.push(
`const ${lookupVar} = new Map(_raw.map(p => [String(p.${defaultPrimaryKey}), p]));`,
);
} else {
const varName = `_${tableName}`;
imports.push(`import ${varName} from './${tableName}.csv';`);
lookupInits.push(
`const ${lookupVar} = new Map(${varName}().map(p => [String(p.${defaultPrimaryKey}), p]));`,
);
}
});
// Build reverse lookup maps for reverse references
for (const decl of result.reverseReferences) {
const key = `${decl.tableName}:${decl.foreignKey}`;
if (reverseLookupVarMap.has(key)) continue;
const revLookupVar = `_${decl.tableName}By_${decl.foreignKey}`;
reverseLookupVarMap.set(key, revLookupVar);
if (decl.tableName === currentTableName) {
reverseLookupInits.push(
`const ${revLookupVar} = new Map();`,
`for (const r of _raw) {`,
` const kv = r.${decl.foreignKey};`,
` const k = String(typeof kv === "object" && "${defaultPrimaryKey}" in kv ? kv.${defaultPrimaryKey} : kv);`,
` if (!${revLookupVar}.has(k)) ${revLookupVar}.set(k, []);`,
` ${revLookupVar}.get(k).push(r);`,
`}`,
);
} else {
const varName = `_${decl.tableName}`;
reverseLookupInits.push(
`const ${revLookupVar} = new Map();`,
`for (const r of ${varName}()) {`,
` const kv = r.${decl.foreignKey};`,
` const k = String(typeof kv === "object" && "${defaultPrimaryKey}" in kv ? kv.${defaultPrimaryKey} : kv);`,
` if (!${revLookupVar}.has(k)) ${revLookupVar}.set(k, []);`,
` ${revLookupVar}.get(k).push(r);`,
`}`,
);
}
}
const lookupVar = (tableName: string) => lookupVarMap.get(tableName)!;
const reverseLookupVar = (tableName: string, foreignKey: string) =>
reverseLookupVarMap.get(`${tableName}:${foreignKey}`)!;
const rowResolvers: string[] = [];
for (const config of result.propertyConfigs) {
if (config.isReverseReference) {
// Reverse reference resolution
const decl = result.reverseReferences.find(
(d) => d.fieldName === config.name,
);
if (decl) {
const revLookup = reverseLookupVar(decl.tableName, decl.foreignKey);
if (decl.isOptional) {
rowResolvers.push(
` ${config.name}: (${revLookup}.get(String(row.${defaultPrimaryKey})) || null),`,
);
} else {
rowResolvers.push(
` ${config.name}: (${revLookup}.get(String(row.${defaultPrimaryKey})) || []),`,
);
}
}
} else if (hasNestedReferences(config.schema)) {
const resolveCode = generateSchemaResolutionCode(
config.schema,
`row.${config.name}`,
lookupVar,
defaultPrimaryKey,
reverseLookupVar,
);
rowResolvers.push(` ${config.name}: ${resolveCode},`);
}
}
const rawJson = JSON.stringify(result.data, null, 2);
const js = [
...imports,
"",
`const _raw = ${rawJson};`,
"",
"let _resolved = null;",
"",
"export default function getData() {",
" if (_resolved) return _resolved;",
" _resolved = _raw;",
...lookupInits.map((l) => ` ${l}`),
...reverseLookupInits.map((l) => ` ${l}`),
...(rowResolvers.length > 0
? [
" _resolved = _raw.map(row => ({",
" ...row,",
...rowResolvers,
" }));",
]
: []),
" return _resolved;",
"}",
].join("\n");
return {
js,
dts: result.typeDefinition,
};
}

View File

@ -0,0 +1,577 @@
import * as fs from "fs";
import * as path from "path";
import {
parseValue,
} from "../index.js";
import type {
Schema,
ReferenceSchema,
ReverseReferenceSchema,
} from "../types.js";
import type {
ReferenceFieldInfo,
} from "./types.js";
import { parseCsv } from "./loader.js";
/** Cache for loaded referenced tables */
const referenceTableCache = new Map<string, Record<string, unknown>[]>();
/** Set of file paths currently being loaded (to detect circular references) */
const loadingFiles = new Set<string>();
export function hasNestedReferences(schema: Schema): boolean {
switch (schema.type) {
case "reference":
case "reverseReference":
return true;
case "tuple":
return schema.elements.some((el) => hasNestedReferences(el.schema));
case "array":
return hasNestedReferences(schema.element);
case "union":
return schema.members.some((m) => hasNestedReferences(m));
default:
return false;
}
}
export function loadReferenceTable(
schema: ReferenceSchema | ReverseReferenceSchema,
refBaseDir: string | undefined,
defaultPrimaryKey: string,
currentFilePath: string | undefined,
): {
lookup: Map<string, Record<string, unknown>>;
refTable: Record<string, unknown>[];
} {
const baseDir =
refBaseDir ||
(currentFilePath ? path.dirname(currentFilePath) : process.cwd());
const fileName = `${schema.tableName}.csv`;
const refFilePath = path.isAbsolute(fileName)
? fileName
: path.join(baseDir, fileName);
let refTable: Record<string, unknown>[];
if (referenceTableCache.has(refFilePath)) {
refTable = referenceTableCache.get(refFilePath)!;
} else {
if (loadingFiles.has(refFilePath)) {
throw new Error(
`Circular reference detected: table "${schema.tableName}" (${refFilePath}) is already being loaded`,
);
}
loadingFiles.add(refFilePath);
try {
const refContent = fs.readFileSync(refFilePath, "utf-8");
const refResult = parseCsv(refContent, {
currentFilePath: refFilePath,
emitTypes: false,
});
refTable = refResult.data;
referenceTableCache.set(refFilePath, refTable);
} catch (error) {
throw new Error(
`Failed to load referenced table "${schema.tableName}" from ${refFilePath}: ${error instanceof Error ? error.message : String(error)}`,
);
} finally {
loadingFiles.delete(refFilePath);
}
}
const lookup = new Map<string, Record<string, unknown>>();
refTable.forEach((row) => {
const pkValue = row[defaultPrimaryKey];
if (pkValue !== undefined) {
lookup.set(String(pkValue), row);
}
});
return { lookup, refTable };
}
export function resolveReferenceId(
id: string,
lookup: Map<string, Record<string, unknown>>,
tableName: string,
): Record<string, unknown> {
const obj = lookup.get(id);
if (!obj) {
throw new Error(`Reference to "${tableName}" with id="${id}" not found`);
}
return obj;
}
export function parseReferenceIds(
schema: ReferenceSchema,
valueString: string,
): unknown {
const trimmed = valueString.trim();
if (schema.isOptional && trimmed === "") {
return null;
}
const valueParser = new ReferenceValueParser(trimmed);
const ids = valueParser.parseIds(schema.isArray);
if (schema.isArray) {
return ids;
}
return ids[0];
}
export function parseValueWithReferenceIds(
valueString: string,
schema: Schema,
): unknown {
if (!hasNestedReferences(schema)) {
return parseValue(schema, valueString);
}
switch (schema.type) {
case "reference":
return parseReferenceIds(schema, valueString);
case "reverseReference":
// Reverse references don't store IDs; they're derived at resolution time
return null;
case "tuple": {
const parsed = parseValue(schema, valueString) as unknown[];
return schema.elements.map((el, i) =>
hasNestedReferences(el.schema)
? extractNestedReferenceIds(parsed[i], el.schema)
: parsed[i],
);
}
case "array": {
const parsed = parseValue(schema, valueString) as unknown[];
return parsed.map((item) =>
hasNestedReferences(schema.element)
? extractNestedReferenceIds(item, schema.element)
: item,
);
}
case "union": {
for (const member of schema.members) {
if (hasNestedReferences(member)) {
try {
const parsed = parseValue(member, valueString);
return extractNestedReferenceIds(parsed, member);
} catch {}
}
}
return parseValue(schema, valueString);
}
default:
return parseValue(schema, valueString);
}
}
export function extractNestedReferenceIds(value: unknown, schema: Schema): unknown {
switch (schema.type) {
case "reference":
if (value === null || value === undefined) return value;
if (schema.isArray) {
const ids = Array.isArray(value) ? value : [value];
return ids.map((id) => String(id));
}
return String(value);
case "reverseReference":
// Reverse references don't store IDs; return null placeholder
return null;
case "tuple": {
if (!Array.isArray(value)) return value;
return schema.elements.map((el, i) =>
hasNestedReferences(el.schema)
? extractNestedReferenceIds(value[i], el.schema)
: value[i],
);
}
case "array": {
if (!Array.isArray(value)) return value;
return value.map((item) =>
hasNestedReferences(schema.element)
? extractNestedReferenceIds(item, schema.element)
: item,
);
}
case "union": {
for (const member of schema.members) {
if (hasNestedReferences(member)) {
try {
return extractNestedReferenceIds(value, member);
} catch {}
}
}
return value;
}
default:
return value;
}
}
export function collectReferenceFields(
schema: Schema,
name: string,
): ReferenceFieldInfo[] {
const fields: ReferenceFieldInfo[] = [];
switch (schema.type) {
case "reference":
fields.push({
name,
tableName: schema.tableName,
isArray: schema.isArray,
schema,
});
break;
case "reverseReference":
fields.push({
name,
tableName: schema.tableName,
isArray: true,
foreignKey: schema.foreignKey,
schema,
});
break;
case "tuple":
for (const el of schema.elements) {
fields.push(...collectReferenceFields(el.schema, name));
}
break;
case "array":
fields.push(...collectReferenceFields(schema.element, name));
break;
case "union":
for (const member of schema.members) {
fields.push(...collectReferenceFields(member, name));
}
break;
}
return fields;
}
export function parseValueWithReferences(
valueString: string,
schema: Schema,
refBaseDir: string | undefined,
defaultPrimaryKey: string,
currentFilePath: string | undefined,
currentRowPk?: unknown,
): unknown {
if (!hasNestedReferences(schema)) {
return parseValue(schema, valueString);
}
switch (schema.type) {
case "reference":
return parseReferenceValue(
schema,
valueString,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
);
case "reverseReference": {
if (currentRowPk === undefined) return [];
return resolveReverseReference(
schema,
currentRowPk,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
);
}
case "tuple": {
const parsed = parseValue(schema, valueString) as unknown[];
return schema.elements.map((el, i) =>
resolveNestedReferences(
parsed[i],
el.schema,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
currentRowPk,
),
);
}
case "array": {
const parsed = parseValue(schema, valueString) as unknown[];
return parsed.map((item) =>
resolveNestedReferences(
item,
schema.element,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
currentRowPk,
),
);
}
case "union": {
const errors: Error[] = [];
for (const member of schema.members) {
if (hasNestedReferences(member)) {
try {
const parsed = parseValue(member, valueString);
return resolveNestedReferences(
parsed,
member,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
currentRowPk,
);
} catch (e) {
errors.push(e instanceof Error ? e : new Error(String(e)));
}
}
}
if (
errors.length > 0 &&
errors.every((e) =>
/not found|Circular reference|Failed to load/.test(e.message),
)
) {
for (const member of schema.members) {
if (!hasNestedReferences(member)) {
try {
return parseValue(member, valueString);
} catch {}
}
}
}
return parseValue(schema, valueString);
}
default:
return parseValue(schema, valueString);
}
}
export function resolveReverseReference(
schema: ReverseReferenceSchema,
pkValue: unknown,
refBaseDir: string | undefined,
defaultPrimaryKey: string,
currentFilePath: string | undefined,
): Record<string, unknown>[] {
const { refTable } = loadReferenceTable(
schema,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
);
const pkStr = String(pkValue);
return refTable.filter((row) => {
const fkValue = row[schema.foreignKey];
const fkStr =
fkValue !== null && fkValue !== undefined && typeof fkValue === "object"
? String((fkValue as Record<string, unknown>)[defaultPrimaryKey])
: String(fkValue);
return fkStr === pkStr;
});
}
export function resolveNestedReferences(
value: unknown,
schema: Schema,
refBaseDir: string | undefined,
defaultPrimaryKey: string,
currentFilePath: string | undefined,
currentRowPk?: unknown,
): unknown {
switch (schema.type) {
case "reference": {
if (value === null || value === undefined) return value;
const { lookup } = loadReferenceTable(
schema,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
);
if (schema.isArray) {
const ids = Array.isArray(value) ? value : [value];
return ids.map((id) =>
resolveReferenceId(String(id), lookup, schema.tableName),
);
}
return resolveReferenceId(String(value), lookup, schema.tableName);
}
case "reverseReference": {
if (currentRowPk === undefined) return [];
const results = resolveReverseReference(
schema,
currentRowPk,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
);
return results;
}
case "tuple": {
if (!Array.isArray(value)) return value;
return schema.elements.map((el, i) =>
resolveNestedReferences(
value[i],
el.schema,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
currentRowPk,
),
);
}
case "array": {
if (!Array.isArray(value)) return value;
return value.map((item) =>
resolveNestedReferences(
item,
schema.element,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
currentRowPk,
),
);
}
case "union": {
const errors: Error[] = [];
for (const member of schema.members) {
if (hasNestedReferences(member)) {
try {
return resolveNestedReferences(
value,
member,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
currentRowPk,
);
} catch (e) {
errors.push(e instanceof Error ? e : new Error(String(e)));
}
}
}
if (errors.length > 0) {
throw errors[0];
}
return value;
}
default:
return value;
}
}
export function parseReferenceValue(
schema: ReferenceSchema,
valueString: string,
refBaseDir: string | undefined,
defaultPrimaryKey: string,
currentFilePath: string | undefined,
): unknown {
const trimmed = valueString.trim();
if (schema.isOptional && trimmed === "") {
return null;
}
const { lookup } = loadReferenceTable(
schema,
refBaseDir,
defaultPrimaryKey,
currentFilePath,
);
const valueParser = new ReferenceValueParser(trimmed);
const ids = valueParser.parseIds(schema.isArray);
if (schema.isArray) {
return ids.map((id) => resolveReferenceId(id, lookup, schema.tableName));
}
return resolveReferenceId(ids[0], lookup, schema.tableName);
}
class ReferenceValueParser {
private input: string;
private pos: number = 0;
constructor(input: string) {
this.input = input;
}
private peek(): string {
return this.input[this.pos] || "";
}
private consume(): string {
return this.input[this.pos++] || "";
}
private skipWhitespace(): void {
while (this.pos < this.input.length && /\s/.test(this.input[this.pos])) {
this.pos++;
}
}
private consumeStr(str: string): boolean {
if (this.input.slice(this.pos, this.pos + str.length) === str) {
this.pos += str.length;
return true;
}
return false;
}
parseIds(isArray: boolean): string[] {
this.skipWhitespace();
if (isArray) {
// Parse array format: [id1; id2; id3]
if (this.peek() === "[") {
this.consume();
}
this.skipWhitespace();
if (this.peek() === "]") {
this.consume();
return [];
}
const ids: string[] = [];
while (true) {
this.skipWhitespace();
let id = "";
while (
this.pos < this.input.length &&
this.peek() !== ";" &&
this.peek() !== "]"
) {
id += this.consume();
}
const trimmedId = id.trim();
if (trimmedId) {
ids.push(trimmedId);
}
this.skipWhitespace();
if (!this.consumeStr(";")) {
break;
}
}
this.skipWhitespace();
if (this.peek() === "]") {
this.consume();
}
return ids;
} else {
// Parse single ID
let id = "";
while (this.pos < this.input.length) {
const char = this.peek();
if (char === ";" || char === "]" || char === ",") {
break;
}
id += this.consume();
}
return [id.trim()];
}
}
}

View File

@ -1,7 +1,7 @@
import type { CsvLoaderOptions } from './loader.js';
import { csvToModule } from './loader.js';
import * as path from 'path';
import * as fs from 'fs';
import * as path from "path";
import * as fs from "fs";
import { csvToModule } from "./module-gen";
import { CsvLoaderOptions } from "./types";
export interface CsvRollupOptions extends CsvLoaderOptions {
/** Include pattern for CSV files (default: /\.csv$/) */
@ -16,7 +16,7 @@ export interface CsvRollupOptions extends CsvLoaderOptions {
function matchesPattern(
id: string,
pattern: RegExp | string | Array<RegExp | string> | undefined
pattern: RegExp | string | Array<RegExp | string> | undefined,
): boolean {
if (!pattern) return true;
const patterns = Array.isArray(pattern) ? pattern : [pattern];
@ -34,7 +34,7 @@ interface TransformResult {
}
interface EmitFileOptions {
type: 'asset';
type: "asset";
fileName: string;
source: string;
}
@ -48,7 +48,7 @@ interface RollupPlugin {
transform: (
this: PluginContext,
code: string,
id: string
id: string,
) => TransformResult | null;
}
@ -61,13 +61,13 @@ export function csvLoader(options: CsvRollupOptions = {}): RollupPlugin {
include = /\.csv$/,
exclude,
emitTypes = true,
typesOutputDir = '',
typesOutputDir = "",
writeToDisk = false,
...parseOptions
} = options;
return {
name: 'inline-schema-csv',
name: "inline-schema-csv",
transform(code: string, id: string) {
// Check if file matches the include/exclude patterns
@ -75,12 +75,12 @@ export function csvLoader(options: CsvRollupOptions = {}): RollupPlugin {
if (exclude && matchesPattern(id, exclude)) return null;
// Only process .csv files
if (!id.endsWith('.csv')) return null;
if (!id.endsWith(".csv")) return null;
// Infer resource name from filename
const fileName = path.basename(id, '.csv').split('.')[0];
const fileName = path.basename(id, ".csv").split(".")[0];
const resourceName = fileName
.replace(/[-_\s]+(.)?/g, (_, char) => char ? char.toUpperCase() : '')
.replace(/[-_\s]+(.)?/g, (_, char) => (char ? char.toUpperCase() : ""))
.replace(/^(.)/, (_, char) => char.toUpperCase());
const result = csvToModule(code, {
@ -95,8 +95,8 @@ export function csvLoader(options: CsvRollupOptions = {}): RollupPlugin {
// Emit type definition file if enabled
if (emitTypes && result.dts) {
const dtsPath = typesOutputDir
? path.join(typesOutputDir, path.basename(id) + '.d.ts')
: id + '.d.ts';
? path.join(typesOutputDir, path.basename(id) + ".d.ts")
: id + ".d.ts";
if (writeToDisk) {
// Write directly to disk
@ -108,7 +108,7 @@ export function csvLoader(options: CsvRollupOptions = {}): RollupPlugin {
} else {
// Emit to Rollup's virtual module system
this.emitFile({
type: 'asset',
type: "asset",
fileName: dtsPath,
source: result.dts,
});

View File

@ -0,0 +1,69 @@
import * as path from "path";
import { schemaToTypeString } from "../index.js";
import type { PropertyConfig } from "./types.js";
/**
* Generate TypeScript interface for the CSV data
*/
export function generateTypeDefinition(
resourceName: string,
propertyConfigs: PropertyConfig[],
references: Set<string>,
currentFilePath?: string,
): string {
const typeName = resourceName ? `${resourceName}Table` : "Table";
const currentTableName = currentFilePath
? path.basename(currentFilePath, path.extname(currentFilePath))
: undefined;
const singularType = resourceName
? resourceName.charAt(0).toUpperCase() + resourceName.slice(1)
: `${typeName}[number]`;
// Generate import statements for referenced tables
const imports: string[] = [];
const resourceNames = new Map<string, string>();
references.forEach((tableName) => {
if (tableName === currentTableName) {
resourceNames.set(tableName, singularType);
return;
}
// Convert table name to type name by capitalizing
const typeBase = tableName.charAt(0).toUpperCase() + tableName.slice(1);
resourceNames.set(tableName, typeBase);
// Generate import path based on current file path
let importPath: string;
if (currentFilePath) {
importPath = `./${tableName}.csv`;
} else {
importPath = `../${tableName}.csv`;
}
imports.push(`import type { ${typeBase} } from '${importPath}';`);
});
const importSection = imports.length > 0 ? imports.join("\n") + "\n\n" : "";
const properties = propertyConfigs
.map(
(config) =>
` readonly ${config.name}: ${schemaToTypeString(config.schema, resourceNames)};`,
)
.join("\n");
let exportAlias = "";
if (resourceName) {
const singularType =
resourceName.charAt(0).toUpperCase() + resourceName.slice(1);
exportAlias = `\nexport type ${singularType} = ${typeName}[number];`;
}
return `${importSection}type ${typeName} = readonly {
${properties}
}[];
${exportAlias}
declare function getData(): ${typeName};
export default getData;
`;
}

91
src/csv-loader/types.ts Normal file
View File

@ -0,0 +1,91 @@
import type {
Schema,
ReferenceSchema,
ReverseReferenceSchema,
} from "../types.js";
export interface CsvLoaderOptions {
delimiter?: string;
quote?: string;
escape?: string;
bom?: boolean;
comment?: string | false;
trim?: boolean;
/** Generate TypeScript declaration file (.d.ts) */
emitTypes?: boolean;
/** Output directory for generated type files (relative to output path) */
typesOutputDir?: string;
/** Write .d.ts files to disk (useful for dev server) */
writeToDisk?: boolean;
/** Base directory for resolving referenced CSV files (default: directory of current file) */
refBaseDir?: string;
/** Primary key field name for referenced tables (default: 'id') */
defaultPrimaryKey?: string;
/** Current file path (used to resolve relative references) */
currentFilePath?: string;
/**
* When false, reference fields store parsed IDs instead of resolved objects.
* Used by csvToModule to emit accessor-based code with lazy resolution.
* Default: true (resolves references eagerly by loading referenced CSV files).
*/
resolveReferences?: boolean;
}
export interface ReferenceFieldInfo {
/** Column name in the CSV */
name: string;
/** Referenced table name */
tableName: string;
/** Whether it's an array reference */
isArray: boolean;
/** The schema of this field (for nested references) */
schema: Schema;
/** For reverse references: the foreign key field name in the referenced table */
foreignKey?: string;
}
export interface CsvParseResult {
/** Parsed CSV data as array of objects */
data: Record<string, unknown>[];
/** Generated TypeScript type definition string (if emitTypes is true) */
typeDefinition?: string;
/** Property configurations for the CSV columns */
propertyConfigs: PropertyConfig[];
/** Referenced table names */
references: Set<string>;
/** Reference field metadata (populated when resolveReferences is false) */
referenceFields: ReferenceFieldInfo[];
/** Reverse reference declarations parsed from comment lines */
reverseReferences: ReverseReferenceDeclaration[];
}
export interface PropertyConfig {
name: string;
schema: any;
validator: (value: unknown) => boolean;
parser: (valueString: string) => unknown;
/** Whether this property is a reference to another table */
isReference?: boolean;
/** Referenced table name (if isReference is true) */
referenceTableName?: string;
/** Whether it's an array reference */
referenceIsArray?: boolean;
/** Whether this is a reverse reference (one-to-many) */
isReverseReference?: boolean;
/** Foreign key field name for reverse references */
reverseReferenceForeignKey?: string;
}
/** Parsed reverse reference declaration from a comment line */
export interface ReverseReferenceDeclaration {
/** Field name in the current table */
fieldName: string;
/** Referenced table name */
tableName: string;
/** Foreign key field name in the referenced table */
foreignKey: string;
/** Whether it's optional */
isOptional: boolean;
/** The parsed schema */
schema: ReverseReferenceSchema;
}

View File

@ -1,8 +1,8 @@
import type { LoaderContext } from '@rspack/core';
import type { CsvLoaderOptions } from './loader.js';
import { csvToModule } from './loader.js';
import * as path from 'path';
import * as fs from 'fs';
import type { LoaderContext } from "@rspack/core";
import * as path from "path";
import * as fs from "fs";
import { csvToModule } from "./module-gen";
import { CsvLoaderOptions } from "./types";
export interface CsvWebpackLoaderOptions extends CsvLoaderOptions {
/** Output directory for generated type files (relative to output path) */
@ -17,17 +17,17 @@ export interface CsvWebpackLoaderOptions extends CsvLoaderOptions {
export default function csvLoader(
this: LoaderContext<CsvWebpackLoaderOptions>,
content: string
content: string,
): string | Buffer {
const options = this.getOptions() as CsvWebpackLoaderOptions | undefined;
const emitTypes = options?.emitTypes ?? true;
const typesOutputDir = options?.typesOutputDir ?? '';
const typesOutputDir = options?.typesOutputDir ?? "";
const writeToDisk = options?.writeToDisk ?? false;
// Infer resource name from filename
const fileName = path.basename(this.resourcePath, '.csv').split('.')[0];
const fileName = path.basename(this.resourcePath, ".csv").split(".")[0];
const resourceName = fileName
.replace(/[-_\s]+(.)?/g, (_, char) => char ? char.toUpperCase() : '')
.replace(/[-_\s]+(.)?/g, (_, char) => (char ? char.toUpperCase() : ""))
.replace(/^(.)/, (_, char) => char.toUpperCase());
const result = csvToModule(content, {
@ -38,13 +38,13 @@ export default function csvLoader(
// Emit type definition file if enabled
if (emitTypes && result.dts) {
const context = this.context || '';
const context = this.context || "";
// Get relative path from context, normalize to forward slashes
let relativePath = this.resourcePath.replace(context, '');
if (relativePath.startsWith('\\') || relativePath.startsWith('/')) {
let relativePath = this.resourcePath.replace(context, "");
if (relativePath.startsWith("\\") || relativePath.startsWith("/")) {
relativePath = relativePath.substring(1);
}
relativePath = relativePath.replace(/\\/g, '/');
relativePath = relativePath.replace(/\\/g, "/");
// Replace .csv with .csv.d.ts for the output filename
const dtsFileName = `${relativePath}.d.ts`;
@ -54,7 +54,11 @@ export default function csvLoader(
if (writeToDisk) {
// Write directly to disk (useful for dev server)
const absolutePath = path.join(this.context || process.cwd(), typesOutputDir || '', dtsFileName);
const absolutePath = path.join(
this.context || process.cwd(),
typesOutputDir || "",
dtsFileName,
);
fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
fs.writeFileSync(absolutePath, result.dts);
} else {