From f94e9b68e4f6b6d110b138ba2352c39bdfa3c8c3 Mon Sep 17 00:00:00 2001 From: hypercross Date: Mon, 20 Apr 2026 00:48:01 +0800 Subject: [PATCH] refactor(csv-loader): decouple reference resolution and module generation Extract reference resolution logic, type generation, and module generation into dedicated modules to improve maintainability and clean up the core loader. --- src/csv-loader/esbuild.ts | 32 +- src/csv-loader/loader.test.ts | 3 +- src/csv-loader/loader.ts | 989 +-------------------------- src/csv-loader/module-gen.ts | 250 +++++++ src/csv-loader/reference-resolver.ts | 577 ++++++++++++++++ src/csv-loader/rollup.ts | 30 +- src/csv-loader/type-gen.ts | 69 ++ src/csv-loader/types.ts | 91 +++ src/csv-loader/webpack.ts | 32 +- 9 files changed, 1061 insertions(+), 1012 deletions(-) create mode 100644 src/csv-loader/module-gen.ts create mode 100644 src/csv-loader/reference-resolver.ts create mode 100644 src/csv-loader/type-gen.ts create mode 100644 src/csv-loader/types.ts diff --git a/src/csv-loader/esbuild.ts b/src/csv-loader/esbuild.ts index 585c61e..f776fbd 100644 --- a/src/csv-loader/esbuild.ts +++ b/src/csv-loader/esbuild.ts @@ -1,8 +1,8 @@ -import type { CsvLoaderOptions } from './loader.js'; -import { csvToModule } from './loader.js'; -import * as path from 'path'; -import * as fs from 'fs'; -import type { Plugin, OnLoadResult } from 'esbuild'; +import * as path from "path"; +import * as fs from "fs"; +import type { Plugin, OnLoadResult } from "esbuild"; +import { csvToModule } from "./module-gen"; +import { CsvLoaderOptions } from "./types"; export interface CsvEsbuildOptions extends CsvLoaderOptions { /** Include pattern for CSV files (default: /\.csv$/) */ @@ -16,7 +16,7 @@ export interface CsvEsbuildOptions extends CsvLoaderOptions { } function createFilter( - pattern: RegExp | string | Array + pattern: RegExp | string | Array, ): RegExp { if (pattern instanceof RegExp) return pattern; if (Array.isArray(pattern)) { @@ -28,7 +28,7 @@ function createFilter( function matchesPattern( id: string, - pattern: RegExp | string | Array | undefined + pattern: RegExp | string | Array | undefined, ): boolean { if (!pattern) return true; const patterns = Array.isArray(pattern) ? pattern : [pattern]; @@ -46,7 +46,7 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin { include = /\.csv$/, exclude, emitTypes = true, - typesOutputDir = '', + typesOutputDir = "", writeToDisk = false, ...parseOptions } = options; @@ -54,7 +54,7 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin { const includeFilter = createFilter(include); return { - name: 'inline-schema-csv', + name: "inline-schema-csv", setup(build) { build.onLoad({ filter: includeFilter }, async (args) => { @@ -66,7 +66,7 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin { // Read the file content let content: string; try { - content = await fs.promises.readFile(args.path, 'utf-8'); + content = await fs.promises.readFile(args.path, "utf-8"); } catch (error) { return { errors: [{ text: `Failed to read file: ${args.path}` }], @@ -74,9 +74,11 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin { } // Infer resource name from filename - const fileName = path.basename(args.path, '.csv').split('.')[0]; + const fileName = path.basename(args.path, ".csv").split(".")[0]; const resourceName = fileName - .replace(/[-_\s]+(.)?/g, (_, char) => char ? char.toUpperCase() : '') + .replace(/[-_\s]+(.)?/g, (_, char) => + char ? char.toUpperCase() : "", + ) .replace(/^(.)/, (_, char) => char.toUpperCase()); const result = csvToModule(content, { @@ -91,8 +93,8 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin { // Emit type definition file if enabled if (emitTypes && result.dts) { const dtsPath = typesOutputDir - ? path.join(typesOutputDir, path.basename(args.path) + '.d.ts') - : args.path + '.d.ts'; + ? path.join(typesOutputDir, path.basename(args.path) + ".d.ts") + : args.path + ".d.ts"; if (writeToDisk) { const absolutePath = path.isAbsolute(dtsPath) @@ -105,7 +107,7 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin { return { contents: result.js, - loader: 'js' as const, + loader: "js" as const, }; }); }, diff --git a/src/csv-loader/loader.test.ts b/src/csv-loader/loader.test.ts index 906c9d7..3d80816 100644 --- a/src/csv-loader/loader.test.ts +++ b/src/csv-loader/loader.test.ts @@ -1,7 +1,8 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { parseCsv, csvToModule } from "./loader"; +import { parseCsv } from "./loader"; import * as path from "path"; import * as fs from "fs"; +import { csvToModule } from "./module-gen"; const fixturesDir = path.join(__dirname, "fixtures"); diff --git a/src/csv-loader/loader.ts b/src/csv-loader/loader.ts index 1c213d2..c5bf19b 100644 --- a/src/csv-loader/loader.ts +++ b/src/csv-loader/loader.ts @@ -10,534 +10,31 @@ import type { ReferenceSchema, ReverseReferenceSchema, } from "../types.js"; +import type { + CsvLoaderOptions, + ReferenceFieldInfo, + CsvParseResult, + PropertyConfig, + ReverseReferenceDeclaration, +} from "./types.js"; +import { + hasNestedReferences, + loadReferenceTable, + resolveReferenceId, + parseReferenceIds, + parseValueWithReferenceIds, + extractNestedReferenceIds, + collectReferenceFields, + parseValueWithReferences, + resolveReverseReference, + resolveNestedReferences, + parseReferenceValue, +} from "./reference-resolver.js"; +import { generateTypeDefinition } from "./type-gen.js"; +import { csvToModule } from "./module-gen.js"; import * as fs from "fs"; import * as path from "path"; -function hasNestedReferences(schema: Schema): boolean { - switch (schema.type) { - case "reference": - case "reverseReference": - return true; - case "tuple": - return schema.elements.some((el) => hasNestedReferences(el.schema)); - case "array": - return hasNestedReferences(schema.element); - case "union": - return schema.members.some((m) => hasNestedReferences(m)); - default: - return false; - } -} - -function loadReferenceTable( - schema: ReferenceSchema | ReverseReferenceSchema, - refBaseDir: string | undefined, - defaultPrimaryKey: string, - currentFilePath: string | undefined, -): { - lookup: Map>; - refTable: Record[]; -} { - const baseDir = - refBaseDir || - (currentFilePath ? path.dirname(currentFilePath) : process.cwd()); - const fileName = `${schema.tableName}.csv`; - const refFilePath = path.isAbsolute(fileName) - ? fileName - : path.join(baseDir, fileName); - - let refTable: Record[]; - if (referenceTableCache.has(refFilePath)) { - refTable = referenceTableCache.get(refFilePath)!; - } else { - if (loadingFiles.has(refFilePath)) { - throw new Error( - `Circular reference detected: table "${schema.tableName}" (${refFilePath}) is already being loaded`, - ); - } - loadingFiles.add(refFilePath); - try { - const refContent = fs.readFileSync(refFilePath, "utf-8"); - const refResult = parseCsv(refContent, { - currentFilePath: refFilePath, - emitTypes: false, - }); - refTable = refResult.data; - referenceTableCache.set(refFilePath, refTable); - } catch (error) { - throw new Error( - `Failed to load referenced table "${schema.tableName}" from ${refFilePath}: ${error instanceof Error ? error.message : String(error)}`, - ); - } finally { - loadingFiles.delete(refFilePath); - } - } - - const lookup = new Map>(); - refTable.forEach((row) => { - const pkValue = row[defaultPrimaryKey]; - if (pkValue !== undefined) { - lookup.set(String(pkValue), row); - } - }); - - return { lookup, refTable }; -} - -function resolveReferenceId( - id: string, - lookup: Map>, - tableName: string, -): Record { - const obj = lookup.get(id); - if (!obj) { - throw new Error(`Reference to "${tableName}" with id="${id}" not found`); - } - return obj; -} - -function parseReferenceIds( - schema: ReferenceSchema, - valueString: string, -): unknown { - const trimmed = valueString.trim(); - if (schema.isOptional && trimmed === "") { - return null; - } - const valueParser = new ReferenceValueParser(trimmed); - const ids = valueParser.parseIds(schema.isArray); - if (schema.isArray) { - return ids; - } - return ids[0]; -} - -function parseValueWithReferenceIds( - valueString: string, - schema: Schema, -): unknown { - if (!hasNestedReferences(schema)) { - return parseValue(schema, valueString); - } - - switch (schema.type) { - case "reference": - return parseReferenceIds(schema, valueString); - case "reverseReference": - // Reverse references don't store IDs; they're derived at resolution time - return null; - case "tuple": { - const parsed = parseValue(schema, valueString) as unknown[]; - return schema.elements.map((el, i) => - hasNestedReferences(el.schema) - ? extractNestedReferenceIds(parsed[i], el.schema) - : parsed[i], - ); - } - case "array": { - const parsed = parseValue(schema, valueString) as unknown[]; - return parsed.map((item) => - hasNestedReferences(schema.element) - ? extractNestedReferenceIds(item, schema.element) - : item, - ); - } - case "union": { - for (const member of schema.members) { - if (hasNestedReferences(member)) { - try { - const parsed = parseValue(member, valueString); - return extractNestedReferenceIds(parsed, member); - } catch {} - } - } - return parseValue(schema, valueString); - } - default: - return parseValue(schema, valueString); - } -} - -function extractNestedReferenceIds(value: unknown, schema: Schema): unknown { - switch (schema.type) { - case "reference": - if (value === null || value === undefined) return value; - if (schema.isArray) { - const ids = Array.isArray(value) ? value : [value]; - return ids.map((id) => String(id)); - } - return String(value); - case "reverseReference": - // Reverse references don't store IDs; return null placeholder - return null; - case "tuple": { - if (!Array.isArray(value)) return value; - return schema.elements.map((el, i) => - hasNestedReferences(el.schema) - ? extractNestedReferenceIds(value[i], el.schema) - : value[i], - ); - } - case "array": { - if (!Array.isArray(value)) return value; - return value.map((item) => - hasNestedReferences(schema.element) - ? extractNestedReferenceIds(item, schema.element) - : item, - ); - } - case "union": { - for (const member of schema.members) { - if (hasNestedReferences(member)) { - try { - return extractNestedReferenceIds(value, member); - } catch {} - } - } - return value; - } - default: - return value; - } -} - -function collectReferenceFields( - schema: Schema, - name: string, -): ReferenceFieldInfo[] { - const fields: ReferenceFieldInfo[] = []; - switch (schema.type) { - case "reference": - fields.push({ - name, - tableName: schema.tableName, - isArray: schema.isArray, - schema, - }); - break; - case "reverseReference": - fields.push({ - name, - tableName: schema.tableName, - isArray: true, - foreignKey: schema.foreignKey, - schema, - }); - break; - case "tuple": - for (const el of schema.elements) { - fields.push(...collectReferenceFields(el.schema, name)); - } - break; - case "array": - fields.push(...collectReferenceFields(schema.element, name)); - break; - case "union": - for (const member of schema.members) { - fields.push(...collectReferenceFields(member, name)); - } - break; - } - return fields; -} - -function parseValueWithReferences( - valueString: string, - schema: Schema, - refBaseDir: string | undefined, - defaultPrimaryKey: string, - currentFilePath: string | undefined, - currentRowPk?: unknown, -): unknown { - if (!hasNestedReferences(schema)) { - return parseValue(schema, valueString); - } - - switch (schema.type) { - case "reference": - return parseReferenceValue( - schema, - valueString, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - ); - case "reverseReference": { - if (currentRowPk === undefined) return []; - return resolveReverseReference( - schema, - currentRowPk, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - ); - } - case "tuple": { - const parsed = parseValue(schema, valueString) as unknown[]; - return schema.elements.map((el, i) => - resolveNestedReferences( - parsed[i], - el.schema, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - currentRowPk, - ), - ); - } - case "array": { - const parsed = parseValue(schema, valueString) as unknown[]; - return parsed.map((item) => - resolveNestedReferences( - item, - schema.element, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - currentRowPk, - ), - ); - } - case "union": { - const errors: Error[] = []; - for (const member of schema.members) { - if (hasNestedReferences(member)) { - try { - const parsed = parseValue(member, valueString); - return resolveNestedReferences( - parsed, - member, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - currentRowPk, - ); - } catch (e) { - errors.push(e instanceof Error ? e : new Error(String(e))); - } - } - } - if ( - errors.length > 0 && - errors.every((e) => - /not found|Circular reference|Failed to load/.test(e.message), - ) - ) { - for (const member of schema.members) { - if (!hasNestedReferences(member)) { - try { - return parseValue(member, valueString); - } catch {} - } - } - } - return parseValue(schema, valueString); - } - default: - return parseValue(schema, valueString); - } -} - -function resolveReverseReference( - schema: ReverseReferenceSchema, - pkValue: unknown, - refBaseDir: string | undefined, - defaultPrimaryKey: string, - currentFilePath: string | undefined, -): Record[] { - const { refTable } = loadReferenceTable( - schema, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - ); - const pkStr = String(pkValue); - return refTable.filter((row) => { - const fkValue = row[schema.foreignKey]; - const fkStr = - fkValue !== null && fkValue !== undefined && typeof fkValue === "object" - ? String((fkValue as Record)[defaultPrimaryKey]) - : String(fkValue); - return fkStr === pkStr; - }); -} - -function resolveNestedReferences( - value: unknown, - schema: Schema, - refBaseDir: string | undefined, - defaultPrimaryKey: string, - currentFilePath: string | undefined, - currentRowPk?: unknown, -): unknown { - switch (schema.type) { - case "reference": { - if (value === null || value === undefined) return value; - const { lookup } = loadReferenceTable( - schema, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - ); - if (schema.isArray) { - const ids = Array.isArray(value) ? value : [value]; - return ids.map((id) => - resolveReferenceId(String(id), lookup, schema.tableName), - ); - } - return resolveReferenceId(String(value), lookup, schema.tableName); - } - case "reverseReference": { - if (currentRowPk === undefined) return []; - const results = resolveReverseReference( - schema, - currentRowPk, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - ); - return results; - } - case "tuple": { - if (!Array.isArray(value)) return value; - return schema.elements.map((el, i) => - resolveNestedReferences( - value[i], - el.schema, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - currentRowPk, - ), - ); - } - case "array": { - if (!Array.isArray(value)) return value; - return value.map((item) => - resolveNestedReferences( - item, - schema.element, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - currentRowPk, - ), - ); - } - case "union": { - const errors: Error[] = []; - for (const member of schema.members) { - if (hasNestedReferences(member)) { - try { - return resolveNestedReferences( - value, - member, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - currentRowPk, - ); - } catch (e) { - errors.push(e instanceof Error ? e : new Error(String(e))); - } - } - } - if (errors.length > 0) { - throw errors[0]; - } - return value; - } - default: - return value; - } -} - -export interface CsvLoaderOptions { - delimiter?: string; - quote?: string; - escape?: string; - bom?: boolean; - comment?: string | false; - trim?: boolean; - /** Generate TypeScript declaration file (.d.ts) */ - emitTypes?: boolean; - /** Output directory for generated type files (relative to output path) */ - typesOutputDir?: string; - /** Write .d.ts files to disk (useful for dev server) */ - writeToDisk?: boolean; - /** Base directory for resolving referenced CSV files (default: directory of current file) */ - refBaseDir?: string; - /** Primary key field name for referenced tables (default: 'id') */ - defaultPrimaryKey?: string; - /** Current file path (used to resolve relative references) */ - currentFilePath?: string; - /** - * When false, reference fields store parsed IDs instead of resolved objects. - * Used by csvToModule to emit accessor-based code with lazy resolution. - * Default: true (resolves references eagerly by loading referenced CSV files). - */ - resolveReferences?: boolean; -} - -export interface ReferenceFieldInfo { - /** Column name in the CSV */ - name: string; - /** Referenced table name */ - tableName: string; - /** Whether it's an array reference */ - isArray: boolean; - /** The schema of this field (for nested references) */ - schema: Schema; - /** For reverse references: the foreign key field name in the referenced table */ - foreignKey?: string; -} - -export interface CsvParseResult { - /** Parsed CSV data as array of objects */ - data: Record[]; - /** Generated TypeScript type definition string (if emitTypes is true) */ - typeDefinition?: string; - /** Property configurations for the CSV columns */ - propertyConfigs: PropertyConfig[]; - /** Referenced table names */ - references: Set; - /** Reference field metadata (populated when resolveReferences is false) */ - referenceFields: ReferenceFieldInfo[]; - /** Reverse reference declarations parsed from comment lines */ - reverseReferences: ReverseReferenceDeclaration[]; -} - -interface PropertyConfig { - name: string; - schema: any; - validator: (value: unknown) => boolean; - parser: (valueString: string) => unknown; - /** Whether this property is a reference to another table */ - isReference?: boolean; - /** Referenced table name (if isReference is true) */ - referenceTableName?: string; - /** Whether it's an array reference */ - referenceIsArray?: boolean; - /** Whether this is a reverse reference (one-to-many) */ - isReverseReference?: boolean; - /** Foreign key field name for reverse references */ - reverseReferenceForeignKey?: string; -} - -/** Parsed reverse reference declaration from a comment line */ -export interface ReverseReferenceDeclaration { - /** Field name in the current table */ - fieldName: string; - /** Referenced table name */ - tableName: string; - /** Foreign key field name in the referenced table */ - foreignKey: string; - /** Whether it's optional */ - isOptional: boolean; - /** The parsed schema */ - schema: ReverseReferenceSchema; -} - /** * Parse a reverse reference declaration from a comment line. * Format: # fieldName := ~tableName(foreignKey) @@ -576,203 +73,6 @@ function parseReverseReferenceDeclaration( }; } -/** Cache for loaded referenced tables */ -const referenceTableCache = new Map[]>(); - -/** Set of file paths currently being loaded (to detect circular references) */ -const loadingFiles = new Set(); - -/** - * Parse and resolve a reference value. - * Loads the referenced table and replaces IDs with actual objects. - */ -function parseReferenceValue( - schema: ReferenceSchema, - valueString: string, - refBaseDir: string | undefined, - defaultPrimaryKey: string, - currentFilePath: string | undefined, -): unknown { - const trimmed = valueString.trim(); - if (schema.isOptional && trimmed === "") { - return null; - } - - const { lookup } = loadReferenceTable( - schema, - refBaseDir, - defaultPrimaryKey, - currentFilePath, - ); - - const valueParser = new ReferenceValueParser(trimmed); - const ids = valueParser.parseIds(schema.isArray); - - if (schema.isArray) { - return ids.map((id) => resolveReferenceId(id, lookup, schema.tableName)); - } - - return resolveReferenceId(ids[0], lookup, schema.tableName); -} - -/** - * Generate TypeScript interface for the CSV data - */ -class ReferenceValueParser { - private input: string; - private pos: number = 0; - - constructor(input: string) { - this.input = input; - } - - private peek(): string { - return this.input[this.pos] || ""; - } - - private consume(): string { - return this.input[this.pos++] || ""; - } - - private skipWhitespace(): void { - while (this.pos < this.input.length && /\s/.test(this.input[this.pos])) { - this.pos++; - } - } - - private consumeStr(str: string): boolean { - if (this.input.slice(this.pos, this.pos + str.length) === str) { - this.pos += str.length; - return true; - } - return false; - } - - parseIds(isArray: boolean): string[] { - this.skipWhitespace(); - - if (isArray) { - // Parse array format: [id1; id2; id3] - if (this.peek() === "[") { - this.consume(); - } - - this.skipWhitespace(); - - if (this.peek() === "]") { - this.consume(); - return []; - } - - const ids: string[] = []; - while (true) { - this.skipWhitespace(); - let id = ""; - while ( - this.pos < this.input.length && - this.peek() !== ";" && - this.peek() !== "]" - ) { - id += this.consume(); - } - const trimmedId = id.trim(); - if (trimmedId) { - ids.push(trimmedId); - } - this.skipWhitespace(); - - if (!this.consumeStr(";")) { - break; - } - } - - this.skipWhitespace(); - if (this.peek() === "]") { - this.consume(); - } - - return ids; - } else { - // Parse single ID - let id = ""; - while (this.pos < this.input.length) { - const char = this.peek(); - if (char === ";" || char === "]" || char === ",") { - break; - } - id += this.consume(); - } - return [id.trim()]; - } - } -} - -/** - * Generate TypeScript interface for the CSV data - */ -function generateTypeDefinition( - resourceName: string, - propertyConfigs: PropertyConfig[], - references: Set, - currentFilePath?: string, -): string { - const typeName = resourceName ? `${resourceName}Table` : "Table"; - const currentTableName = currentFilePath - ? path.basename(currentFilePath, path.extname(currentFilePath)) - : undefined; - const singularType = resourceName - ? resourceName.charAt(0).toUpperCase() + resourceName.slice(1) - : `${typeName}[number]`; - - // Generate import statements for referenced tables - const imports: string[] = []; - const resourceNames = new Map(); - - references.forEach((tableName) => { - if (tableName === currentTableName) { - resourceNames.set(tableName, singularType); - return; - } - // Convert table name to type name by capitalizing - const typeBase = tableName.charAt(0).toUpperCase() + tableName.slice(1); - resourceNames.set(tableName, typeBase); - - // Generate import path based on current file path - let importPath: string; - if (currentFilePath) { - importPath = `./${tableName}.csv`; - } else { - importPath = `../${tableName}.csv`; - } - imports.push(`import type { ${typeBase} } from '${importPath}';`); - }); - - const importSection = imports.length > 0 ? imports.join("\n") + "\n\n" : ""; - - const properties = propertyConfigs - .map( - (config) => - ` readonly ${config.name}: ${schemaToTypeString(config.schema, resourceNames)};`, - ) - .join("\n"); - - let exportAlias = ""; - if (resourceName) { - const singularType = - resourceName.charAt(0).toUpperCase() + resourceName.slice(1); - exportAlias = `\nexport type ${singularType} = ${typeName}[number];`; - } - - return `${importSection}type ${typeName} = readonly { -${properties} -}[]; -${exportAlias} - -declare function getData(): ${typeName}; -export default getData; -`; -} - /** * Parse CSV content string into structured data with schema validation. * This is a standalone function that doesn't depend on webpack/rspack LoaderContext. @@ -1040,248 +340,3 @@ export function parseCsv( return result; } - -/** - * Generate runtime reference resolution code for a schema. - * Returns a JS expression string that resolves references using lookup maps. - */ -function generateSchemaResolutionCode( - schema: Schema, - valueExpr: string, - lookupVar: (tableName: string) => string, - pkField: string, - reverseLookupVar?: (tableName: string, foreignKey: string) => string, -): string { - switch (schema.type) { - case "reference": { - const lookup = lookupVar(schema.tableName); - if (schema.isOptional) { - if (schema.isArray) { - return `(${valueExpr} === null || ${valueExpr} === undefined ? ${valueExpr} : (Array.isArray(${valueExpr}) ? ${valueExpr}.map(id => ${lookup}.get(String(id))) : ${lookup}.get(String(${valueExpr}))))`; - } - return `(${valueExpr} === null || ${valueExpr} === undefined ? ${valueExpr} : ${lookup}.get(String(${valueExpr})))`; - } - if (schema.isArray) { - return `(Array.isArray(${valueExpr}) ? ${valueExpr}.map(id => ${lookup}.get(String(id))) : ${valueExpr})`; - } - return `${lookup}.get(String(${valueExpr}))`; - } - case "reverseReference": { - if (!reverseLookupVar) return valueExpr; - const reverseLookup = reverseLookupVar( - schema.tableName, - schema.foreignKey, - ); - if (schema.isOptional) { - return `(${reverseLookup}.get(String(row.${pkField})) || null)`; - } - return `(${reverseLookup}.get(String(row.${pkField})) || [])`; - } - case "tuple": { - const elementResolvers = schema.elements.map((el, i) => { - if (hasNestedReferences(el.schema)) { - return generateSchemaResolutionCode( - el.schema, - `${valueExpr}[${i}]`, - lookupVar, - pkField, - reverseLookupVar, - ); - } - return `${valueExpr}[${i}]`; - }); - return `[${elementResolvers.join(", ")}]`; - } - case "array": { - if (hasNestedReferences(schema.element)) { - const itemResolve = generateSchemaResolutionCode( - schema.element, - "item", - lookupVar, - pkField, - reverseLookupVar, - ); - return `(${valueExpr}).map(item => ${itemResolve})`; - } - return valueExpr; - } - case "union": { - const refMembers = schema.members.filter((m) => hasNestedReferences(m)); - const nonRefMembers = schema.members.filter( - (m) => !hasNestedReferences(m), - ); - const resolveParts: string[] = []; - for (const member of refMembers) { - const resolveCode = generateSchemaResolutionCode( - member, - valueExpr, - lookupVar, - pkField, - reverseLookupVar, - ); - resolveParts.push(resolveCode); - } - if (nonRefMembers.length > 0) { - resolveParts.push(valueExpr); - } - if (resolveParts.length === 0) return valueExpr; - if (resolveParts.length === 1) return resolveParts[0]; - return `(${resolveParts.join(" ?? ")})`; - } - default: - return valueExpr; - } -} - -/** - * Generate JavaScript module code from CSV content. - * Emits an accessor function for tables with references (lazy resolution), - * or static JSON for tables without references. - */ -export function csvToModule( - content: string, - options: CsvLoaderOptions & { resourceName?: string } = {}, -): { js: string; dts?: string } { - const result = parseCsv(content, { ...options, resolveReferences: false }); - - const hasRefs = - result.referenceFields.length > 0 || result.reverseReferences.length > 0; - const defaultPrimaryKey = options.defaultPrimaryKey ?? "id"; - - const imports: string[] = []; - const lookupInits: string[] = []; - const lookupVarMap = new Map(); - - // Reverse lookup maps: grouped by (tableName, foreignKey) - const reverseLookupInits: string[] = []; - const reverseLookupVarMap = new Map(); - - const currentTableName = options.currentFilePath - ? path.basename( - options.currentFilePath, - path.extname(options.currentFilePath), - ) - : undefined; - - // Build forward lookup maps for referenced tables - const uniqueTables = new Set(result.referenceFields.map((f) => f.tableName)); - // Also include tables from reverse references - for (const decl of result.reverseReferences) { - uniqueTables.add(decl.tableName); - } - - uniqueTables.forEach((tableName) => { - const lookupVar = `_${tableName}Lookup`; - lookupVarMap.set(tableName, lookupVar); - - if (tableName === currentTableName) { - lookupInits.push( - `const ${lookupVar} = new Map(_raw.map(p => [String(p.${defaultPrimaryKey}), p]));`, - ); - } else { - const varName = `_${tableName}`; - imports.push(`import ${varName} from './${tableName}.csv';`); - lookupInits.push( - `const ${lookupVar} = new Map(${varName}().map(p => [String(p.${defaultPrimaryKey}), p]));`, - ); - } - }); - - // Build reverse lookup maps for reverse references - for (const decl of result.reverseReferences) { - const key = `${decl.tableName}:${decl.foreignKey}`; - if (reverseLookupVarMap.has(key)) continue; - - const revLookupVar = `_${decl.tableName}By_${decl.foreignKey}`; - reverseLookupVarMap.set(key, revLookupVar); - - if (decl.tableName === currentTableName) { - reverseLookupInits.push( - `const ${revLookupVar} = new Map();`, - `for (const r of _raw) {`, - ` const kv = r.${decl.foreignKey};`, - ` const k = String(typeof kv === "object" && "${defaultPrimaryKey}" in kv ? kv.${defaultPrimaryKey} : kv);`, - ` if (!${revLookupVar}.has(k)) ${revLookupVar}.set(k, []);`, - ` ${revLookupVar}.get(k).push(r);`, - `}`, - ); - } else { - const varName = `_${decl.tableName}`; - reverseLookupInits.push( - `const ${revLookupVar} = new Map();`, - `for (const r of ${varName}()) {`, - ` const kv = r.${decl.foreignKey};`, - ` const k = String(typeof kv === "object" && "${defaultPrimaryKey}" in kv ? kv.${defaultPrimaryKey} : kv);`, - ` if (!${revLookupVar}.has(k)) ${revLookupVar}.set(k, []);`, - ` ${revLookupVar}.get(k).push(r);`, - `}`, - ); - } - } - - const lookupVar = (tableName: string) => lookupVarMap.get(tableName)!; - const reverseLookupVar = (tableName: string, foreignKey: string) => - reverseLookupVarMap.get(`${tableName}:${foreignKey}`)!; - - const rowResolvers: string[] = []; - for (const config of result.propertyConfigs) { - if (config.isReverseReference) { - // Reverse reference resolution - const decl = result.reverseReferences.find( - (d) => d.fieldName === config.name, - ); - if (decl) { - const revLookup = reverseLookupVar(decl.tableName, decl.foreignKey); - if (decl.isOptional) { - rowResolvers.push( - ` ${config.name}: (${revLookup}.get(String(row.${defaultPrimaryKey})) || null),`, - ); - } else { - rowResolvers.push( - ` ${config.name}: (${revLookup}.get(String(row.${defaultPrimaryKey})) || []),`, - ); - } - } - } else if (hasNestedReferences(config.schema)) { - const resolveCode = generateSchemaResolutionCode( - config.schema, - `row.${config.name}`, - lookupVar, - defaultPrimaryKey, - reverseLookupVar, - ); - rowResolvers.push(` ${config.name}: ${resolveCode},`); - } - } - - const rawJson = JSON.stringify(result.data, null, 2); - - const js = [ - ...imports, - "", - `const _raw = ${rawJson};`, - "", - "let _resolved = null;", - "", - "export default function getData() {", - " if (_resolved) return _resolved;", - " _resolved = _raw;", - ...lookupInits.map((l) => ` ${l}`), - ...reverseLookupInits.map((l) => ` ${l}`), - ...(rowResolvers.length > 0 - ? [ - " _resolved = _raw.map(row => ({", - " ...row,", - ...rowResolvers, - " }));", - ] - : []), - " return _resolved;", - "}", - ].join("\n"); - - return { - js, - dts: result.typeDefinition, - }; -} diff --git a/src/csv-loader/module-gen.ts b/src/csv-loader/module-gen.ts new file mode 100644 index 0000000..e06c9a5 --- /dev/null +++ b/src/csv-loader/module-gen.ts @@ -0,0 +1,250 @@ +import * as path from "path"; +import { parseCsv } from "./loader.js"; +import { hasNestedReferences } from "./reference-resolver.js"; +import type { Schema } from "../types.js"; +import type { CsvLoaderOptions, PropertyConfig } from "./types.js"; + +/** + * Generate runtime reference resolution code for a schema. + * Returns a JS expression string that resolves references using lookup maps. + */ +function generateSchemaResolutionCode( + schema: Schema, + valueExpr: string, + lookupVar: (tableName: string) => string, + pkField: string, + reverseLookupVar?: (tableName: string, foreignKey: string) => string, +): string { + switch (schema.type) { + case "reference": { + const lookup = lookupVar(schema.tableName); + if (schema.isOptional) { + if (schema.isArray) { + return `(${valueExpr} === null || ${valueExpr} === undefined ? ${valueExpr} : (Array.isArray(${valueExpr}) ? ${valueExpr}.map(id => ${lookup}.get(String(id))) : ${lookup}.get(String(${valueExpr}))))`; + } + return `(${valueExpr} === null || ${valueExpr} === undefined ? ${valueExpr} : ${lookup}.get(String(${valueExpr})))`; + } + if (schema.isArray) { + return `(Array.isArray(${valueExpr}) ? ${valueExpr}.map(id => ${lookup}.get(String(id))) : ${valueExpr})`; + } + return `${lookup}.get(String(${valueExpr}))`; + } + case "reverseReference": { + if (!reverseLookupVar) return valueExpr; + const reverseLookup = reverseLookupVar( + schema.tableName, + schema.foreignKey, + ); + if (schema.isOptional) { + return `(${reverseLookup}.get(String(row.${pkField})) || null)`; + } + return `(${reverseLookup}.get(String(row.${pkField})) || [])`; + } + case "tuple": { + const elementResolvers = schema.elements.map((el, i) => { + if (hasNestedReferences(el.schema)) { + return generateSchemaResolutionCode( + el.schema, + `${valueExpr}[${i}]`, + lookupVar, + pkField, + reverseLookupVar, + ); + } + return `${valueExpr}[${i}]`; + }); + return `[${elementResolvers.join(", ")}]`; + } + case "array": { + if (hasNestedReferences(schema.element)) { + const itemResolve = generateSchemaResolutionCode( + schema.element, + "item", + lookupVar, + pkField, + reverseLookupVar, + ); + return `(${valueExpr}).map(item => ${itemResolve})`; + } + return valueExpr; + } + case "union": { + const refMembers = schema.members.filter((m) => hasNestedReferences(m)); + const nonRefMembers = schema.members.filter( + (m) => !hasNestedReferences(m), + ); + const resolveParts: string[] = []; + for (const member of refMembers) { + const resolveCode = generateSchemaResolutionCode( + member, + valueExpr, + lookupVar, + pkField, + reverseLookupVar, + ); + resolveParts.push(resolveCode); + } + if (nonRefMembers.length > 0) { + resolveParts.push(valueExpr); + } + if (resolveParts.length === 0) return valueExpr; + if (resolveParts.length === 1) return resolveParts[0]; + return `(${resolveParts.join(" ?? ")})`; + } + default: + return valueExpr; + } +} + +/** + * Generate JavaScript module code from CSV content. + * Emits an accessor function for tables with references (lazy resolution), + * or static JSON for tables without references. + */ +export function csvToModule( + content: string, + options: CsvLoaderOptions & { resourceName?: string } = {}, +): { js: string; dts?: string } { + const result = parseCsv(content, { ...options, resolveReferences: false }); + + const hasRefs = + result.referenceFields.length > 0 || result.reverseReferences.length > 0; + const defaultPrimaryKey = options.defaultPrimaryKey ?? "id"; + + const imports: string[] = []; + const lookupInits: string[] = []; + const lookupVarMap = new Map(); + + // Reverse lookup maps: grouped by (tableName, foreignKey) + const reverseLookupInits: string[] = []; + const reverseLookupVarMap = new Map(); + + const currentTableName = options.currentFilePath + ? path.basename( + options.currentFilePath, + path.extname(options.currentFilePath), + ) + : undefined; + + // Build forward lookup maps for referenced tables + const uniqueTables = new Set(result.referenceFields.map((f) => f.tableName)); + // Also include tables from reverse references + for (const decl of result.reverseReferences) { + uniqueTables.add(decl.tableName); + } + + uniqueTables.forEach((tableName) => { + const lookupVar = `_${tableName}Lookup`; + lookupVarMap.set(tableName, lookupVar); + + if (tableName === currentTableName) { + lookupInits.push( + `const ${lookupVar} = new Map(_raw.map(p => [String(p.${defaultPrimaryKey}), p]));`, + ); + } else { + const varName = `_${tableName}`; + imports.push(`import ${varName} from './${tableName}.csv';`); + lookupInits.push( + `const ${lookupVar} = new Map(${varName}().map(p => [String(p.${defaultPrimaryKey}), p]));`, + ); + } + }); + + // Build reverse lookup maps for reverse references + for (const decl of result.reverseReferences) { + const key = `${decl.tableName}:${decl.foreignKey}`; + if (reverseLookupVarMap.has(key)) continue; + + const revLookupVar = `_${decl.tableName}By_${decl.foreignKey}`; + reverseLookupVarMap.set(key, revLookupVar); + + if (decl.tableName === currentTableName) { + reverseLookupInits.push( + `const ${revLookupVar} = new Map();`, + `for (const r of _raw) {`, + ` const kv = r.${decl.foreignKey};`, + ` const k = String(typeof kv === "object" && "${defaultPrimaryKey}" in kv ? kv.${defaultPrimaryKey} : kv);`, + ` if (!${revLookupVar}.has(k)) ${revLookupVar}.set(k, []);`, + ` ${revLookupVar}.get(k).push(r);`, + `}`, + ); + } else { + const varName = `_${decl.tableName}`; + reverseLookupInits.push( + `const ${revLookupVar} = new Map();`, + `for (const r of ${varName}()) {`, + ` const kv = r.${decl.foreignKey};`, + ` const k = String(typeof kv === "object" && "${defaultPrimaryKey}" in kv ? kv.${defaultPrimaryKey} : kv);`, + ` if (!${revLookupVar}.has(k)) ${revLookupVar}.set(k, []);`, + ` ${revLookupVar}.get(k).push(r);`, + `}`, + ); + } + } + + const lookupVar = (tableName: string) => lookupVarMap.get(tableName)!; + const reverseLookupVar = (tableName: string, foreignKey: string) => + reverseLookupVarMap.get(`${tableName}:${foreignKey}`)!; + + const rowResolvers: string[] = []; + for (const config of result.propertyConfigs) { + if (config.isReverseReference) { + // Reverse reference resolution + const decl = result.reverseReferences.find( + (d) => d.fieldName === config.name, + ); + if (decl) { + const revLookup = reverseLookupVar(decl.tableName, decl.foreignKey); + if (decl.isOptional) { + rowResolvers.push( + ` ${config.name}: (${revLookup}.get(String(row.${defaultPrimaryKey})) || null),`, + ); + } else { + rowResolvers.push( + ` ${config.name}: (${revLookup}.get(String(row.${defaultPrimaryKey})) || []),`, + ); + } + } + } else if (hasNestedReferences(config.schema)) { + const resolveCode = generateSchemaResolutionCode( + config.schema, + `row.${config.name}`, + lookupVar, + defaultPrimaryKey, + reverseLookupVar, + ); + rowResolvers.push(` ${config.name}: ${resolveCode},`); + } + } + + const rawJson = JSON.stringify(result.data, null, 2); + + const js = [ + ...imports, + "", + `const _raw = ${rawJson};`, + "", + "let _resolved = null;", + "", + "export default function getData() {", + " if (_resolved) return _resolved;", + " _resolved = _raw;", + ...lookupInits.map((l) => ` ${l}`), + ...reverseLookupInits.map((l) => ` ${l}`), + ...(rowResolvers.length > 0 + ? [ + " _resolved = _raw.map(row => ({", + " ...row,", + ...rowResolvers, + " }));", + ] + : []), + " return _resolved;", + "}", + ].join("\n"); + + return { + js, + dts: result.typeDefinition, + }; +} diff --git a/src/csv-loader/reference-resolver.ts b/src/csv-loader/reference-resolver.ts new file mode 100644 index 0000000..5fa5be5 --- /dev/null +++ b/src/csv-loader/reference-resolver.ts @@ -0,0 +1,577 @@ +import * as fs from "fs"; +import * as path from "path"; +import { + parseValue, +} from "../index.js"; +import type { + Schema, + ReferenceSchema, + ReverseReferenceSchema, +} from "../types.js"; +import type { + ReferenceFieldInfo, +} from "./types.js"; +import { parseCsv } from "./loader.js"; + +/** Cache for loaded referenced tables */ +const referenceTableCache = new Map[]>(); + +/** Set of file paths currently being loaded (to detect circular references) */ +const loadingFiles = new Set(); + +export function hasNestedReferences(schema: Schema): boolean { + switch (schema.type) { + case "reference": + case "reverseReference": + return true; + case "tuple": + return schema.elements.some((el) => hasNestedReferences(el.schema)); + case "array": + return hasNestedReferences(schema.element); + case "union": + return schema.members.some((m) => hasNestedReferences(m)); + default: + return false; + } +} + +export function loadReferenceTable( + schema: ReferenceSchema | ReverseReferenceSchema, + refBaseDir: string | undefined, + defaultPrimaryKey: string, + currentFilePath: string | undefined, +): { + lookup: Map>; + refTable: Record[]; +} { + const baseDir = + refBaseDir || + (currentFilePath ? path.dirname(currentFilePath) : process.cwd()); + const fileName = `${schema.tableName}.csv`; + const refFilePath = path.isAbsolute(fileName) + ? fileName + : path.join(baseDir, fileName); + + let refTable: Record[]; + if (referenceTableCache.has(refFilePath)) { + refTable = referenceTableCache.get(refFilePath)!; + } else { + if (loadingFiles.has(refFilePath)) { + throw new Error( + `Circular reference detected: table "${schema.tableName}" (${refFilePath}) is already being loaded`, + ); + } + loadingFiles.add(refFilePath); + try { + const refContent = fs.readFileSync(refFilePath, "utf-8"); + const refResult = parseCsv(refContent, { + currentFilePath: refFilePath, + emitTypes: false, + }); + refTable = refResult.data; + referenceTableCache.set(refFilePath, refTable); + } catch (error) { + throw new Error( + `Failed to load referenced table "${schema.tableName}" from ${refFilePath}: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + loadingFiles.delete(refFilePath); + } + } + + const lookup = new Map>(); + refTable.forEach((row) => { + const pkValue = row[defaultPrimaryKey]; + if (pkValue !== undefined) { + lookup.set(String(pkValue), row); + } + }); + + return { lookup, refTable }; +} + +export function resolveReferenceId( + id: string, + lookup: Map>, + tableName: string, +): Record { + const obj = lookup.get(id); + if (!obj) { + throw new Error(`Reference to "${tableName}" with id="${id}" not found`); + } + return obj; +} + +export function parseReferenceIds( + schema: ReferenceSchema, + valueString: string, +): unknown { + const trimmed = valueString.trim(); + if (schema.isOptional && trimmed === "") { + return null; + } + const valueParser = new ReferenceValueParser(trimmed); + const ids = valueParser.parseIds(schema.isArray); + if (schema.isArray) { + return ids; + } + return ids[0]; +} + +export function parseValueWithReferenceIds( + valueString: string, + schema: Schema, +): unknown { + if (!hasNestedReferences(schema)) { + return parseValue(schema, valueString); + } + + switch (schema.type) { + case "reference": + return parseReferenceIds(schema, valueString); + case "reverseReference": + // Reverse references don't store IDs; they're derived at resolution time + return null; + case "tuple": { + const parsed = parseValue(schema, valueString) as unknown[]; + return schema.elements.map((el, i) => + hasNestedReferences(el.schema) + ? extractNestedReferenceIds(parsed[i], el.schema) + : parsed[i], + ); + } + case "array": { + const parsed = parseValue(schema, valueString) as unknown[]; + return parsed.map((item) => + hasNestedReferences(schema.element) + ? extractNestedReferenceIds(item, schema.element) + : item, + ); + } + case "union": { + for (const member of schema.members) { + if (hasNestedReferences(member)) { + try { + const parsed = parseValue(member, valueString); + return extractNestedReferenceIds(parsed, member); + } catch {} + } + } + return parseValue(schema, valueString); + } + default: + return parseValue(schema, valueString); + } +} + +export function extractNestedReferenceIds(value: unknown, schema: Schema): unknown { + switch (schema.type) { + case "reference": + if (value === null || value === undefined) return value; + if (schema.isArray) { + const ids = Array.isArray(value) ? value : [value]; + return ids.map((id) => String(id)); + } + return String(value); + case "reverseReference": + // Reverse references don't store IDs; return null placeholder + return null; + case "tuple": { + if (!Array.isArray(value)) return value; + return schema.elements.map((el, i) => + hasNestedReferences(el.schema) + ? extractNestedReferenceIds(value[i], el.schema) + : value[i], + ); + } + case "array": { + if (!Array.isArray(value)) return value; + return value.map((item) => + hasNestedReferences(schema.element) + ? extractNestedReferenceIds(item, schema.element) + : item, + ); + } + case "union": { + for (const member of schema.members) { + if (hasNestedReferences(member)) { + try { + return extractNestedReferenceIds(value, member); + } catch {} + } + } + return value; + } + default: + return value; + } +} + +export function collectReferenceFields( + schema: Schema, + name: string, +): ReferenceFieldInfo[] { + const fields: ReferenceFieldInfo[] = []; + switch (schema.type) { + case "reference": + fields.push({ + name, + tableName: schema.tableName, + isArray: schema.isArray, + schema, + }); + break; + case "reverseReference": + fields.push({ + name, + tableName: schema.tableName, + isArray: true, + foreignKey: schema.foreignKey, + schema, + }); + break; + case "tuple": + for (const el of schema.elements) { + fields.push(...collectReferenceFields(el.schema, name)); + } + break; + case "array": + fields.push(...collectReferenceFields(schema.element, name)); + break; + case "union": + for (const member of schema.members) { + fields.push(...collectReferenceFields(member, name)); + } + break; + } + return fields; +} + +export function parseValueWithReferences( + valueString: string, + schema: Schema, + refBaseDir: string | undefined, + defaultPrimaryKey: string, + currentFilePath: string | undefined, + currentRowPk?: unknown, +): unknown { + if (!hasNestedReferences(schema)) { + return parseValue(schema, valueString); + } + + switch (schema.type) { + case "reference": + return parseReferenceValue( + schema, + valueString, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + ); + case "reverseReference": { + if (currentRowPk === undefined) return []; + return resolveReverseReference( + schema, + currentRowPk, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + ); + } + case "tuple": { + const parsed = parseValue(schema, valueString) as unknown[]; + return schema.elements.map((el, i) => + resolveNestedReferences( + parsed[i], + el.schema, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + currentRowPk, + ), + ); + } + case "array": { + const parsed = parseValue(schema, valueString) as unknown[]; + return parsed.map((item) => + resolveNestedReferences( + item, + schema.element, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + currentRowPk, + ), + ); + } + case "union": { + const errors: Error[] = []; + for (const member of schema.members) { + if (hasNestedReferences(member)) { + try { + const parsed = parseValue(member, valueString); + return resolveNestedReferences( + parsed, + member, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + currentRowPk, + ); + } catch (e) { + errors.push(e instanceof Error ? e : new Error(String(e))); + } + } + } + if ( + errors.length > 0 && + errors.every((e) => + /not found|Circular reference|Failed to load/.test(e.message), + ) + ) { + for (const member of schema.members) { + if (!hasNestedReferences(member)) { + try { + return parseValue(member, valueString); + } catch {} + } + } + } + return parseValue(schema, valueString); + } + default: + return parseValue(schema, valueString); + } +} + +export function resolveReverseReference( + schema: ReverseReferenceSchema, + pkValue: unknown, + refBaseDir: string | undefined, + defaultPrimaryKey: string, + currentFilePath: string | undefined, +): Record[] { + const { refTable } = loadReferenceTable( + schema, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + ); + const pkStr = String(pkValue); + return refTable.filter((row) => { + const fkValue = row[schema.foreignKey]; + const fkStr = + fkValue !== null && fkValue !== undefined && typeof fkValue === "object" + ? String((fkValue as Record)[defaultPrimaryKey]) + : String(fkValue); + return fkStr === pkStr; + }); +} + +export function resolveNestedReferences( + value: unknown, + schema: Schema, + refBaseDir: string | undefined, + defaultPrimaryKey: string, + currentFilePath: string | undefined, + currentRowPk?: unknown, +): unknown { + switch (schema.type) { + case "reference": { + if (value === null || value === undefined) return value; + const { lookup } = loadReferenceTable( + schema, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + ); + if (schema.isArray) { + const ids = Array.isArray(value) ? value : [value]; + return ids.map((id) => + resolveReferenceId(String(id), lookup, schema.tableName), + ); + } + return resolveReferenceId(String(value), lookup, schema.tableName); + } + case "reverseReference": { + if (currentRowPk === undefined) return []; + const results = resolveReverseReference( + schema, + currentRowPk, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + ); + return results; + } + case "tuple": { + if (!Array.isArray(value)) return value; + return schema.elements.map((el, i) => + resolveNestedReferences( + value[i], + el.schema, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + currentRowPk, + ), + ); + } + case "array": { + if (!Array.isArray(value)) return value; + return value.map((item) => + resolveNestedReferences( + item, + schema.element, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + currentRowPk, + ), + ); + } + case "union": { + const errors: Error[] = []; + for (const member of schema.members) { + if (hasNestedReferences(member)) { + try { + return resolveNestedReferences( + value, + member, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + currentRowPk, + ); + } catch (e) { + errors.push(e instanceof Error ? e : new Error(String(e))); + } + } + } + if (errors.length > 0) { + throw errors[0]; + } + return value; + } + default: + return value; + } +} + +export function parseReferenceValue( + schema: ReferenceSchema, + valueString: string, + refBaseDir: string | undefined, + defaultPrimaryKey: string, + currentFilePath: string | undefined, +): unknown { + const trimmed = valueString.trim(); + if (schema.isOptional && trimmed === "") { + return null; + } + + const { lookup } = loadReferenceTable( + schema, + refBaseDir, + defaultPrimaryKey, + currentFilePath, + ); + + const valueParser = new ReferenceValueParser(trimmed); + const ids = valueParser.parseIds(schema.isArray); + + if (schema.isArray) { + return ids.map((id) => resolveReferenceId(id, lookup, schema.tableName)); + } + + return resolveReferenceId(ids[0], lookup, schema.tableName); +} + +class ReferenceValueParser { + private input: string; + private pos: number = 0; + + constructor(input: string) { + this.input = input; + } + + private peek(): string { + return this.input[this.pos] || ""; + } + + private consume(): string { + return this.input[this.pos++] || ""; + } + + private skipWhitespace(): void { + while (this.pos < this.input.length && /\s/.test(this.input[this.pos])) { + this.pos++; + } + } + + private consumeStr(str: string): boolean { + if (this.input.slice(this.pos, this.pos + str.length) === str) { + this.pos += str.length; + return true; + } + return false; + } + + parseIds(isArray: boolean): string[] { + this.skipWhitespace(); + + if (isArray) { + // Parse array format: [id1; id2; id3] + if (this.peek() === "[") { + this.consume(); + } + + this.skipWhitespace(); + + if (this.peek() === "]") { + this.consume(); + return []; + } + + const ids: string[] = []; + while (true) { + this.skipWhitespace(); + let id = ""; + while ( + this.pos < this.input.length && + this.peek() !== ";" && + this.peek() !== "]" + ) { + id += this.consume(); + } + const trimmedId = id.trim(); + if (trimmedId) { + ids.push(trimmedId); + } + this.skipWhitespace(); + + if (!this.consumeStr(";")) { + break; + } + } + + this.skipWhitespace(); + if (this.peek() === "]") { + this.consume(); + } + + return ids; + } else { + // Parse single ID + let id = ""; + while (this.pos < this.input.length) { + const char = this.peek(); + if (char === ";" || char === "]" || char === ",") { + break; + } + id += this.consume(); + } + return [id.trim()]; + } + } +} diff --git a/src/csv-loader/rollup.ts b/src/csv-loader/rollup.ts index c7a0e73..2eaad9e 100644 --- a/src/csv-loader/rollup.ts +++ b/src/csv-loader/rollup.ts @@ -1,7 +1,7 @@ -import type { CsvLoaderOptions } from './loader.js'; -import { csvToModule } from './loader.js'; -import * as path from 'path'; -import * as fs from 'fs'; +import * as path from "path"; +import * as fs from "fs"; +import { csvToModule } from "./module-gen"; +import { CsvLoaderOptions } from "./types"; export interface CsvRollupOptions extends CsvLoaderOptions { /** Include pattern for CSV files (default: /\.csv$/) */ @@ -16,7 +16,7 @@ export interface CsvRollupOptions extends CsvLoaderOptions { function matchesPattern( id: string, - pattern: RegExp | string | Array | undefined + pattern: RegExp | string | Array | undefined, ): boolean { if (!pattern) return true; const patterns = Array.isArray(pattern) ? pattern : [pattern]; @@ -34,7 +34,7 @@ interface TransformResult { } interface EmitFileOptions { - type: 'asset'; + type: "asset"; fileName: string; source: string; } @@ -48,7 +48,7 @@ interface RollupPlugin { transform: ( this: PluginContext, code: string, - id: string + id: string, ) => TransformResult | null; } @@ -61,13 +61,13 @@ export function csvLoader(options: CsvRollupOptions = {}): RollupPlugin { include = /\.csv$/, exclude, emitTypes = true, - typesOutputDir = '', + typesOutputDir = "", writeToDisk = false, ...parseOptions } = options; return { - name: 'inline-schema-csv', + name: "inline-schema-csv", transform(code: string, id: string) { // Check if file matches the include/exclude patterns @@ -75,12 +75,12 @@ export function csvLoader(options: CsvRollupOptions = {}): RollupPlugin { if (exclude && matchesPattern(id, exclude)) return null; // Only process .csv files - if (!id.endsWith('.csv')) return null; + if (!id.endsWith(".csv")) return null; // Infer resource name from filename - const fileName = path.basename(id, '.csv').split('.')[0]; + const fileName = path.basename(id, ".csv").split(".")[0]; const resourceName = fileName - .replace(/[-_\s]+(.)?/g, (_, char) => char ? char.toUpperCase() : '') + .replace(/[-_\s]+(.)?/g, (_, char) => (char ? char.toUpperCase() : "")) .replace(/^(.)/, (_, char) => char.toUpperCase()); const result = csvToModule(code, { @@ -95,8 +95,8 @@ export function csvLoader(options: CsvRollupOptions = {}): RollupPlugin { // Emit type definition file if enabled if (emitTypes && result.dts) { const dtsPath = typesOutputDir - ? path.join(typesOutputDir, path.basename(id) + '.d.ts') - : id + '.d.ts'; + ? path.join(typesOutputDir, path.basename(id) + ".d.ts") + : id + ".d.ts"; if (writeToDisk) { // Write directly to disk @@ -108,7 +108,7 @@ export function csvLoader(options: CsvRollupOptions = {}): RollupPlugin { } else { // Emit to Rollup's virtual module system this.emitFile({ - type: 'asset', + type: "asset", fileName: dtsPath, source: result.dts, }); diff --git a/src/csv-loader/type-gen.ts b/src/csv-loader/type-gen.ts new file mode 100644 index 0000000..012a5ed --- /dev/null +++ b/src/csv-loader/type-gen.ts @@ -0,0 +1,69 @@ +import * as path from "path"; +import { schemaToTypeString } from "../index.js"; +import type { PropertyConfig } from "./types.js"; + +/** + * Generate TypeScript interface for the CSV data + */ +export function generateTypeDefinition( + resourceName: string, + propertyConfigs: PropertyConfig[], + references: Set, + currentFilePath?: string, +): string { + const typeName = resourceName ? `${resourceName}Table` : "Table"; + const currentTableName = currentFilePath + ? path.basename(currentFilePath, path.extname(currentFilePath)) + : undefined; + const singularType = resourceName + ? resourceName.charAt(0).toUpperCase() + resourceName.slice(1) + : `${typeName}[number]`; + + // Generate import statements for referenced tables + const imports: string[] = []; + const resourceNames = new Map(); + + references.forEach((tableName) => { + if (tableName === currentTableName) { + resourceNames.set(tableName, singularType); + return; + } + // Convert table name to type name by capitalizing + const typeBase = tableName.charAt(0).toUpperCase() + tableName.slice(1); + resourceNames.set(tableName, typeBase); + + // Generate import path based on current file path + let importPath: string; + if (currentFilePath) { + importPath = `./${tableName}.csv`; + } else { + importPath = `../${tableName}.csv`; + } + imports.push(`import type { ${typeBase} } from '${importPath}';`); + }); + + const importSection = imports.length > 0 ? imports.join("\n") + "\n\n" : ""; + + const properties = propertyConfigs + .map( + (config) => + ` readonly ${config.name}: ${schemaToTypeString(config.schema, resourceNames)};`, + ) + .join("\n"); + + let exportAlias = ""; + if (resourceName) { + const singularType = + resourceName.charAt(0).toUpperCase() + resourceName.slice(1); + exportAlias = `\nexport type ${singularType} = ${typeName}[number];`; + } + + return `${importSection}type ${typeName} = readonly { +${properties} +}[]; +${exportAlias} + +declare function getData(): ${typeName}; +export default getData; +`; +} diff --git a/src/csv-loader/types.ts b/src/csv-loader/types.ts new file mode 100644 index 0000000..0d49a9f --- /dev/null +++ b/src/csv-loader/types.ts @@ -0,0 +1,91 @@ +import type { + Schema, + ReferenceSchema, + ReverseReferenceSchema, +} from "../types.js"; + +export interface CsvLoaderOptions { + delimiter?: string; + quote?: string; + escape?: string; + bom?: boolean; + comment?: string | false; + trim?: boolean; + /** Generate TypeScript declaration file (.d.ts) */ + emitTypes?: boolean; + /** Output directory for generated type files (relative to output path) */ + typesOutputDir?: string; + /** Write .d.ts files to disk (useful for dev server) */ + writeToDisk?: boolean; + /** Base directory for resolving referenced CSV files (default: directory of current file) */ + refBaseDir?: string; + /** Primary key field name for referenced tables (default: 'id') */ + defaultPrimaryKey?: string; + /** Current file path (used to resolve relative references) */ + currentFilePath?: string; + /** + * When false, reference fields store parsed IDs instead of resolved objects. + * Used by csvToModule to emit accessor-based code with lazy resolution. + * Default: true (resolves references eagerly by loading referenced CSV files). + */ + resolveReferences?: boolean; +} + +export interface ReferenceFieldInfo { + /** Column name in the CSV */ + name: string; + /** Referenced table name */ + tableName: string; + /** Whether it's an array reference */ + isArray: boolean; + /** The schema of this field (for nested references) */ + schema: Schema; + /** For reverse references: the foreign key field name in the referenced table */ + foreignKey?: string; +} + +export interface CsvParseResult { + /** Parsed CSV data as array of objects */ + data: Record[]; + /** Generated TypeScript type definition string (if emitTypes is true) */ + typeDefinition?: string; + /** Property configurations for the CSV columns */ + propertyConfigs: PropertyConfig[]; + /** Referenced table names */ + references: Set; + /** Reference field metadata (populated when resolveReferences is false) */ + referenceFields: ReferenceFieldInfo[]; + /** Reverse reference declarations parsed from comment lines */ + reverseReferences: ReverseReferenceDeclaration[]; +} + +export interface PropertyConfig { + name: string; + schema: any; + validator: (value: unknown) => boolean; + parser: (valueString: string) => unknown; + /** Whether this property is a reference to another table */ + isReference?: boolean; + /** Referenced table name (if isReference is true) */ + referenceTableName?: string; + /** Whether it's an array reference */ + referenceIsArray?: boolean; + /** Whether this is a reverse reference (one-to-many) */ + isReverseReference?: boolean; + /** Foreign key field name for reverse references */ + reverseReferenceForeignKey?: string; +} + +/** Parsed reverse reference declaration from a comment line */ +export interface ReverseReferenceDeclaration { + /** Field name in the current table */ + fieldName: string; + /** Referenced table name */ + tableName: string; + /** Foreign key field name in the referenced table */ + foreignKey: string; + /** Whether it's optional */ + isOptional: boolean; + /** The parsed schema */ + schema: ReverseReferenceSchema; +} diff --git a/src/csv-loader/webpack.ts b/src/csv-loader/webpack.ts index d56cfd7..90d458e 100644 --- a/src/csv-loader/webpack.ts +++ b/src/csv-loader/webpack.ts @@ -1,8 +1,8 @@ -import type { LoaderContext } from '@rspack/core'; -import type { CsvLoaderOptions } from './loader.js'; -import { csvToModule } from './loader.js'; -import * as path from 'path'; -import * as fs from 'fs'; +import type { LoaderContext } from "@rspack/core"; +import * as path from "path"; +import * as fs from "fs"; +import { csvToModule } from "./module-gen"; +import { CsvLoaderOptions } from "./types"; export interface CsvWebpackLoaderOptions extends CsvLoaderOptions { /** Output directory for generated type files (relative to output path) */ @@ -17,17 +17,17 @@ export interface CsvWebpackLoaderOptions extends CsvLoaderOptions { export default function csvLoader( this: LoaderContext, - content: string + content: string, ): string | Buffer { const options = this.getOptions() as CsvWebpackLoaderOptions | undefined; const emitTypes = options?.emitTypes ?? true; - const typesOutputDir = options?.typesOutputDir ?? ''; + const typesOutputDir = options?.typesOutputDir ?? ""; const writeToDisk = options?.writeToDisk ?? false; // Infer resource name from filename - const fileName = path.basename(this.resourcePath, '.csv').split('.')[0]; + const fileName = path.basename(this.resourcePath, ".csv").split(".")[0]; const resourceName = fileName - .replace(/[-_\s]+(.)?/g, (_, char) => char ? char.toUpperCase() : '') + .replace(/[-_\s]+(.)?/g, (_, char) => (char ? char.toUpperCase() : "")) .replace(/^(.)/, (_, char) => char.toUpperCase()); const result = csvToModule(content, { @@ -38,13 +38,13 @@ export default function csvLoader( // Emit type definition file if enabled if (emitTypes && result.dts) { - const context = this.context || ''; + const context = this.context || ""; // Get relative path from context, normalize to forward slashes - let relativePath = this.resourcePath.replace(context, ''); - if (relativePath.startsWith('\\') || relativePath.startsWith('/')) { + let relativePath = this.resourcePath.replace(context, ""); + if (relativePath.startsWith("\\") || relativePath.startsWith("/")) { relativePath = relativePath.substring(1); } - relativePath = relativePath.replace(/\\/g, '/'); + relativePath = relativePath.replace(/\\/g, "/"); // Replace .csv with .csv.d.ts for the output filename const dtsFileName = `${relativePath}.d.ts`; @@ -54,7 +54,11 @@ export default function csvLoader( if (writeToDisk) { // Write directly to disk (useful for dev server) - const absolutePath = path.join(this.context || process.cwd(), typesOutputDir || '', dtsFileName); + const absolutePath = path.join( + this.context || process.cwd(), + typesOutputDir || "", + dtsFileName, + ); fs.mkdirSync(path.dirname(absolutePath), { recursive: true }); fs.writeFileSync(absolutePath, result.dts); } else {