From 53ccac39e643afdc3fb652f16fe817c6108cfe85 Mon Sep 17 00:00:00 2001 From: hypercross Date: Tue, 21 Apr 2026 13:47:16 +0800 Subject: [PATCH] feat(csv-loader): add support for custom type declarations Introduce the ability to define reusable types within CSV files using comment lines with the format `# TypeName := schema`. - Support parsing type declarations from comments or schema cells - Enable recursive expansion of type names within schemas - Integrate declared types into generated TypeScript definitions - Allow columns to reference declared types by name --- src/csv-loader/loader.ts | 283 +++++++++++++++++- .../tests/parseCsv-typeDeclarations.test.ts | 222 ++++++++++++++ src/csv-loader/type-gen.ts | 26 +- src/csv-loader/types.ts | 12 + 4 files changed, 531 insertions(+), 12 deletions(-) create mode 100644 src/csv-loader/tests/parseCsv-typeDeclarations.test.ts diff --git a/src/csv-loader/loader.ts b/src/csv-loader/loader.ts index c5bf19b..cd898de 100644 --- a/src/csv-loader/loader.ts +++ b/src/csv-loader/loader.ts @@ -16,7 +16,9 @@ import type { CsvParseResult, PropertyConfig, ReverseReferenceDeclaration, + TypeDeclaration, } from "./types.js"; +import { ParseError } from "../parser.js"; import { hasNestedReferences, loadReferenceTable, @@ -35,6 +37,204 @@ import { csvToModule } from "./module-gen.js"; import * as fs from "fs"; import * as path from "path"; +/** + * Parse a type declaration from a comment line. + * Format: # TypeName := schema + * Examples: + * # Trigger := 'onPlay' | 'onDraw' | 'onDiscard' + * # Effect := [Trigger, @effect, int] + * Returns null if the line is not a type declaration. + */ +function parseTypeDeclaration( + line: string, + commentChar: string = "#", +): { typeName: string; schemaString: string } | null { + const trimmed = line.trim(); + // Must start with the comment character + if (!trimmed.startsWith(commentChar)) return null; + + const content = trimmed.slice(commentChar.length).trim(); + + // Match pattern: TypeName := schema + const match = content.match(/^([A-Z][a-zA-Z0-9]*)\s*:=\s*(.+)$/); + if (!match) return null; + + const [, typeName, schemaString] = match; + return { typeName, schemaString }; +} + +/** + * Expand a type name to its schema by replacing the type name with its schema inline. + * Returns the schema string with type names expanded, or null if not a type name. + */ +function expandTypeName( + schemaString: string, + declaredTypes: Map, +): string | null { + const trimmed = schemaString.trim(); + if (declaredTypes.has(trimmed)) { + return declaredTypes.get(trimmed)!; + } + return null; +} + +/** + * Recursively expand all type name references in a schema string. + * Handles unions, tuples, arrays, and nested structures. + */ +function expandSchemaString( + schemaString: string, + declaredTypes: Map, +): string { + let result = schemaString; + + // Keep expanding until no more changes (handles recursive dependencies) + let prev = ""; + while (prev !== result) { + prev = result; + result = expandSchemaInString(result, declaredTypes); + } + + return result; +} + +/** + * Single pass of type name expansion in a schema string. + */ +function expandSchemaInString( + schemaString: string, + declaredTypes: Map, +): string { + // Check if the entire string is a type name + const expanded = expandTypeName(schemaString.trim(), declaredTypes); + if (expanded !== null) { + return expanded; + } + + // Handle union types (recursively expand each member) + if (schemaString.includes("|")) { + // Split by | but respect quotes + const parts = splitByToken(schemaString, "|"); + if (parts.length > 1) { + const expandedParts = parts.map((part) => + expandSchemaInString(part.trim(), declaredTypes), + ); + return expandedParts.join(" | "); + } + } + + // Handle tuple/array syntax [el1; el2; ...] or [elements] + // Check if it's a bracketed structure + if (schemaString.startsWith("[") && schemaString.endsWith("]")) { + const inner = schemaString.slice(1, -1); + // Check if it's semicolon-separated (tuple syntax) + if (inner.includes(";")) { + const elements = splitByToken(inner, ";"); + const expandedElements = elements.map((el) => + expandSchemaInString(el.trim(), declaredTypes), + ); + return `[${expandedElements.join("; ")}]`; + } + // Otherwise it's a simple array, expand recursively + return `[${expandSchemaInString(inner, declaredTypes)}]`; + } + + // Check if it's a type name reference (only uppercase start to avoid conflicts with primitives) + const typeNameMatch = schemaString.trim().match(/^[A-Z][a-zA-Z0-9]*$/); + if (typeNameMatch) { + const expanded = expandTypeName(schemaString.trim(), declaredTypes); + if (expanded !== null) { + return expanded; + } + } + + return schemaString; +} + +/** + * Split a string by a token, respecting quoted strings. + */ +function splitByToken(str: string, token: string): string[] { + const result: string[] = []; + let current = ""; + let inQuote: string | null = null; + + for (let i = 0; i < str.length; i++) { + const char = str[i]; + + if (inQuote) { + if (char === inQuote && str[i - 1] !== "\\") { + inQuote = null; + } + current += char; + } else if (char === '"' || char === "'") { + inQuote = char; + current += char; + } else if (char === token && inQuote === null) { + result.push(current); + current = ""; + } else { + current += char; + } + } + + if (current.length > 0 || str.endsWith(token)) { + result.push(current); + } + + return result; +} + +/** + * Resolve type name references within a schema using declared types. + * For example, if "Trigger" is a declared type, references to "Trigger" in + * other schemas will be replaced with the actual Trigger schema definition. + */ +function resolveTypeReferences( + schema: Schema, + declaredTypes: Map, +): Schema { + switch (schema.type) { + case "union": + return { + type: "union", + members: schema.members.map((m) => + resolveTypeReferences(m, declaredTypes), + ), + }; + case "tuple": + return { + type: "tuple", + elements: schema.elements.map((el) => ({ + name: el.name, + schema: resolveTypeReferences(el.schema, declaredTypes), + })), + }; + case "array": + return { + type: "array", + element: resolveTypeReferences(schema.element, declaredTypes), + }; + case "reference": + // Don't resolve references to other tables + return schema; + default: + return schema; + } +} + +/** + * Resolve type name references in a type declaration's schema string. + * Called after all type names are known. + */ +function resolveTypeDeclarationSchema( + schemaString: string, + declaredTypes: Map, +): Schema { + const schema = parseSchema(schemaString.trim()); + return resolveTypeReferences(schema, declaredTypes); +} + /** * Parse a reverse reference declaration from a comment line. * Format: # fieldName := ~tableName(foreignKey) @@ -99,6 +299,8 @@ export function parseCsv( // Pre-strip comment lines from content before passing to csv-parse, // to avoid quote parsing errors in comment lines containing double quotes. const reverseReferences: ReverseReferenceDeclaration[] = []; + // Store raw type declarations (name + schema string) first, resolve after all names are known + const typeDeclarationsRaw: { typeName: string; schemaString: string }[] = []; let filteredContent = content; if (comment) { const lines = content.split(/\r?\n/); @@ -106,14 +308,22 @@ export function parseCsv( for (const line of lines) { const trimmed = line.trim(); if (trimmed.startsWith(comment)) { + // Try to parse as type declaration first + const typeDecl = parseTypeDeclaration(trimmed, comment); + if (typeDecl) { + typeDeclarationsRaw.push(typeDecl); + continue; // Skip type declaration lines + } + // Try to parse as reverse reference const decl = parseReverseReferenceDeclaration(trimmed, comment); if (decl) { reverseReferences.push(decl); + continue; // Skip reverse reference lines } - // Skip comment lines - } else { - nonCommentLines.push(line); + // Regular comment line - strip it (csv-parse can't handle quotes in comments) + continue; } + nonCommentLines.push(line); } filteredContent = nonCommentLines.join("\n"); } @@ -147,11 +357,18 @@ export function parseCsv( const dataRows = filteredRecords.slice(2); - // Also check schema row cells for comment-prefixed reverse reference declarations - // (in case they appear as schema cells rather than separate rows) + // Also check schema row cells for comment-prefixed type declarations + // and reverse reference declarations for (let col = 0; col < schemas.length; col++) { const cell = (schemas[col] ?? "").trim(); if (comment && cell.startsWith(comment)) { + // Try type declaration first + const typeDecl = parseTypeDeclaration(cell, comment); + if (typeDecl) { + typeDeclarationsRaw.push(typeDecl); + continue; + } + // Try reverse reference const decl = parseReverseReferenceDeclaration(cell, comment); if (decl) { reverseReferences.push(decl); @@ -159,18 +376,70 @@ export function parseCsv( } } + // Build a map of declared type names first + const declaredTypeNames = new Set(); + for (const decl of typeDeclarationsRaw) { + declaredTypeNames.add(decl.typeName); + } + + // Build a map of schema strings for expansion (only stores string schemas initially) + const declaredSchemaStrings = new Map(); + for (const decl of typeDeclarationsRaw) { + // If the schema is a string literal union, store it for expansion + declaredSchemaStrings.set(decl.typeName, decl.schemaString); + } + + // Parse type declarations with expansion of type name references + const typeDeclarationsParsed: { name: string; schema: Schema }[] = []; + for (const decl of typeDeclarationsRaw) { + // Expand any type name references before parsing + const expandedSchema = expandSchemaString( + decl.schemaString, + declaredSchemaStrings, + ); + const schema = parseSchema(expandedSchema.trim()); + typeDeclarationsParsed.push({ name: decl.typeName, schema }); + } + + // Build declared types map + const declaredTypes = new Map(); + for (const decl of typeDeclarationsParsed) { + declaredTypes.set(decl.name, decl.schema); + } + + // Now resolve all type references within type declarations (for nested type refs) + const typeDeclarations: TypeDeclaration[] = []; + for (const decl of typeDeclarationsParsed) { + const resolvedSchema = resolveTypeReferences(decl.schema, declaredTypes); + typeDeclarations.push({ name: decl.name, schema: resolvedSchema }); + } + + // Update declaredTypes with resolved schemas for column schema lookup + for (const decl of typeDeclarations) { + declaredTypes.set(decl.name, decl.schema); + } + const resolveReferences = options.resolveReferences ?? true; const propertyConfigs: PropertyConfig[] = headers.map( (header: string, index: number) => { const schemaString = schemas[index]; - const schema = parseSchema(schemaString); + // Check if schema string matches a declared type name + let schema: Schema; + let declaredTypeName: string | undefined; + if (declaredTypes.has(schemaString)) { + schema = declaredTypes.get(schemaString)!; + declaredTypeName = schemaString; + } else { + schema = parseSchema(schemaString); + } const config: PropertyConfig = { name: header, schema, validator: createValidator(schema), parser: (valueString: string) => parseValue(schema, valueString), + declaredTypeName, }; if (schema.type === "reference") { @@ -327,6 +596,7 @@ export function parseCsv( references, referenceFields, reverseReferences, + typeDeclarations, }; if (emitTypes) { @@ -335,6 +605,7 @@ export function parseCsv( propertyConfigs, references, options.currentFilePath, + typeDeclarations, ); } diff --git a/src/csv-loader/tests/parseCsv-typeDeclarations.test.ts b/src/csv-loader/tests/parseCsv-typeDeclarations.test.ts new file mode 100644 index 0000000..c764090 --- /dev/null +++ b/src/csv-loader/tests/parseCsv-typeDeclarations.test.ts @@ -0,0 +1,222 @@ +import { describe, it, expect } from "vitest"; +import { parseCsv } from "../loader"; +import * as path from "path"; +import { fixturesDir } from "../test-utils"; + +describe("parseCsv - type declarations", () => { + it("should parse type declaration from comment line", () => { + const csv = [ + "# Trigger := 'onPlay' | 'onDraw' | 'onDiscard'", + "id,trigger", + "string,Trigger", + "attack,onPlay", + ].join("\n"); + + const result = parseCsv(csv, { emitTypes: false }); + + expect(result.typeDeclarations).toHaveLength(1); + expect(result.typeDeclarations[0]).toMatchObject({ + name: "Trigger", + schema: { + type: "union", + members: [ + { type: "stringLiteral", value: "onPlay" }, + { type: "stringLiteral", value: "onDraw" }, + { type: "stringLiteral", value: "onDiscard" }, + ], + }, + }); + }); + + it("should use declared type as column schema", () => { + const csv = [ + "# Trigger := 'onPlay' | 'onDraw' | 'onDiscard'", + "id,trigger", + "string,Trigger", + "attack,onPlay", + ].join("\n"); + + const result = parseCsv(csv, { emitTypes: false }); + + expect(result.data).toHaveLength(1); + expect(result.data[0]).toEqual({ id: "attack", trigger: "onPlay" }); + }); + + it("should include declared types in type definition", () => { + const csv = [ + "# Trigger := 'onPlay' | 'onDraw' | 'onDiscard'", + "id,trigger", + "string,Trigger", + "attack,onPlay", + ].join("\n"); + + const result = parseCsv(csv, { + emitTypes: true, + resourceName: "card", + currentFilePath: path.join(fixturesDir, "card.csv"), + }); + + expect(result.typeDefinition).toContain("type Trigger ="); + expect(result.typeDefinition).toContain( + '"onPlay" | "onDraw" | "onDiscard"', + ); + }); + + it("should parse multiple type declarations", () => { + const csv = [ + "# Trigger := 'onPlay' | 'onDraw'", + "# Status := 'active' | 'inactive'", + "id,trigger,status", + "string,Trigger,Status", + "attack,onPlay,active", + ].join("\n"); + + const result = parseCsv(csv, { emitTypes: false }); + + expect(result.typeDeclarations).toHaveLength(2); + expect(result.typeDeclarations[0].name).toBe("Trigger"); + expect(result.typeDeclarations[1].name).toBe("Status"); + }); + + it("should include multiple type declarations in type definition", () => { + const csv = [ + "# Trigger := 'onPlay' | 'onDraw'", + "# Status := 'active' | 'inactive'", + "id,trigger,status", + "string,Trigger,Status", + "attack,onPlay,active", + ].join("\n"); + + const result = parseCsv(csv, { + emitTypes: true, + resourceName: "card", + currentFilePath: path.join(fixturesDir, "card.csv"), + }); + + expect(result.typeDefinition).toContain("type Trigger ="); + expect(result.typeDefinition).toContain("type Status ="); + }); + + it("should ignore comment lines that are not type declarations", () => { + const csv = [ + "# This is just a comment", + "# Trigger := 'onPlay' | 'onDraw'", + "id,trigger", + "string,Trigger", + "attack,onPlay", + ].join("\n"); + + const result = parseCsv(csv, { emitTypes: false }); + + expect(result.typeDeclarations).toHaveLength(1); + expect(result.typeDeclarations[0].name).toBe("Trigger"); + }); + + it("should handle type declaration with array schema", () => { + const csv = [ + "# Tags := string[]", + "id,tags", + "string,Tags", + "1,[dev; admin; user]", + ].join("\n"); + + const result = parseCsv(csv, { emitTypes: false }); + + expect(result.typeDeclarations).toHaveLength(1); + expect(result.typeDeclarations[0].name).toBe("Tags"); + expect(result.data[0].tags).toEqual(["dev", "admin", "user"]); + }); + + it("should include type declaration before table type in output", () => { + const csv = [ + "# Status := 'active' | 'inactive'", + "id,status", + "string,Status", + "1,active", + ].join("\n"); + + const result = parseCsv(csv, { + emitTypes: true, + resourceName: "item", + currentFilePath: path.join(fixturesDir, "item.csv"), + }); + + // Type declaration should appear before the table type + const typeDef = result.typeDefinition!; + const statusIndex = typeDef.indexOf("type Status ="); + const itemTableIndex = typeDef.indexOf("type itemTable ="); + expect(statusIndex).toBeLessThan(itemTableIndex); + }); + + it("should work with type declarations alongside reverse references", () => { + const csv = [ + "# Trigger := 'onPlay' | 'onDraw'", + "id,trigger", + "string,Trigger", + "attack,onPlay", + ].join("\n"); + + const result = parseCsv(csv, { emitTypes: false }); + + expect(result.typeDeclarations).toHaveLength(1); + expect(result.reverseReferences).toHaveLength(0); + }); + + it("should resolve type references inside tuple type declarations", () => { + const csv = [ + "# Trigger := 'onPlay' | 'onDraw' | 'onDiscard'", + "# Effect := [Trigger; @effect; int]", + "id,trigger", + "string,Trigger", + "attack,onPlay", + ].join("\n"); + + const result = parseCsv(csv, { emitTypes: false }); + + expect(result.typeDeclarations).toHaveLength(2); + expect(result.typeDeclarations[0].name).toBe("Trigger"); + expect(result.typeDeclarations[1].name).toBe("Effect"); + // Effect should have a tuple schema with Trigger's schema inlined + expect(result.typeDeclarations[1].schema.type).toBe("tuple"); + }); + + it("should include resolved type references in generated type definition", () => { + const csv = [ + "# Trigger := 'onPlay' | 'onDraw' | 'onDiscard'", + "# Effect := [Trigger; @effect; int]", + "id,trigger", + "string,Trigger", + "attack,onPlay", + ].join("\n"); + + const result = parseCsv(csv, { + emitTypes: true, + resourceName: "card", + currentFilePath: path.join(fixturesDir, "card.csv"), + }); + + // Both type declarations should appear + expect(result.typeDefinition).toContain("type Trigger ="); + expect(result.typeDefinition).toContain("type Effect ="); + // Column should use the declared type name, not expanded union + expect(result.typeDefinition).toContain("readonly trigger: Trigger;"); + }); +}); + +describe("parseCsv - type declarations with resolveReferences: false", () => { + it("should populate typeDeclarations even when resolveReferences is false", () => { + const csv = [ + "# Trigger := 'onPlay' | 'onDraw'", + "id,trigger", + "string,Trigger", + "attack,onPlay", + ].join("\n"); + + const result = parseCsv(csv, { + emitTypes: false, + resolveReferences: false, + }); + + expect(result.typeDeclarations).toHaveLength(1); + }); +}); diff --git a/src/csv-loader/type-gen.ts b/src/csv-loader/type-gen.ts index 012a5ed..7630ce2 100644 --- a/src/csv-loader/type-gen.ts +++ b/src/csv-loader/type-gen.ts @@ -1,6 +1,6 @@ import * as path from "path"; import { schemaToTypeString } from "../index.js"; -import type { PropertyConfig } from "./types.js"; +import type { PropertyConfig, TypeDeclaration } from "./types.js"; /** * Generate TypeScript interface for the CSV data @@ -10,6 +10,7 @@ export function generateTypeDefinition( propertyConfigs: PropertyConfig[], references: Set, currentFilePath?: string, + typeDeclarations: TypeDeclaration[] = [], ): string { const typeName = resourceName ? `${resourceName}Table` : "Table"; const currentTableName = currentFilePath @@ -44,11 +45,24 @@ export function generateTypeDefinition( const importSection = imports.length > 0 ? imports.join("\n") + "\n\n" : ""; + // Generate type declarations for user-defined types + const typeDeclarationSection = + typeDeclarations.length > 0 + ? typeDeclarations + .map( + (decl) => + `type ${decl.name} = ${schemaToTypeString(decl.schema, resourceNames)};`, + ) + .join("\n") + "\n\n" + : ""; + const properties = propertyConfigs - .map( - (config) => - ` readonly ${config.name}: ${schemaToTypeString(config.schema, resourceNames)};`, - ) + .map((config) => { + const typeStr = config.declaredTypeName + ? config.declaredTypeName + : schemaToTypeString(config.schema, resourceNames); + return ` readonly ${config.name}: ${typeStr};`; + }) .join("\n"); let exportAlias = ""; @@ -58,7 +72,7 @@ export function generateTypeDefinition( exportAlias = `\nexport type ${singularType} = ${typeName}[number];`; } - return `${importSection}type ${typeName} = readonly { + return `${importSection}${typeDeclarationSection}type ${typeName} = readonly { ${properties} }[]; ${exportAlias} diff --git a/src/csv-loader/types.ts b/src/csv-loader/types.ts index 0d49a9f..ade0468 100644 --- a/src/csv-loader/types.ts +++ b/src/csv-loader/types.ts @@ -57,6 +57,8 @@ export interface CsvParseResult { referenceFields: ReferenceFieldInfo[]; /** Reverse reference declarations parsed from comment lines */ reverseReferences: ReverseReferenceDeclaration[]; + /** Type declarations parsed from comment lines */ + typeDeclarations: TypeDeclaration[]; } export interface PropertyConfig { @@ -74,6 +76,8 @@ export interface PropertyConfig { isReverseReference?: boolean; /** Foreign key field name for reverse references */ reverseReferenceForeignKey?: string; + /** When a column uses a declared type name, this stores that name */ + declaredTypeName?: string; } /** Parsed reverse reference declaration from a comment line */ @@ -89,3 +93,11 @@ export interface ReverseReferenceDeclaration { /** The parsed schema */ schema: ReverseReferenceSchema; } + +/** Parsed type declaration from a comment line */ +export interface TypeDeclaration { + /** Name of the type being defined */ + name: string; + /** The parsed schema for this type */ + schema: Schema; +}