refactor: accessor based imports
This commit is contained in:
parent
392d5f1431
commit
6eba70bb3b
|
|
@ -19,8 +19,8 @@ Single-package TypeScript library with two runtime entry points:
|
||||||
- `src/types.ts` — union type `Schema = Primitive | Tuple | Array | Reference | StringLiteral | Union`
|
- `src/types.ts` — union type `Schema = Primitive | Tuple | Array | Reference | StringLiteral | Union`
|
||||||
|
|
||||||
- **`inline-schema/csv-loader`** (`src/csv-loader/loader.ts`) — CSV loader with `@table` reference resolution
|
- **`inline-schema/csv-loader`** (`src/csv-loader/loader.ts`) — CSV loader with `@table` reference resolution
|
||||||
- `loader.ts` — `parseCsv()` and `csvToModule()`; resolves `@tablename` references by loading referenced CSV files from disk
|
- `loader.ts` — `parseCsv()` (eager resolution) and `csvToModule()` (accessor-based output with lazy resolution); `resolveReferences: false` mode stores IDs instead of resolved objects
|
||||||
- `webpack.ts`, `rollup.ts`, `esbuild.ts` — bundler plugin wrappers around `parseCsv`
|
- `webpack.ts`, `rollup.ts`, `esbuild.ts` — bundler plugin wrappers around `csvToModule`
|
||||||
|
|
||||||
Build produces separate bundles per entry point (see `tsup.config.ts`). The csv-loader entries externalize `csv-parse`, `@rspack/core`, `esbuild`, and `rollup`.
|
Build produces separate bundles per entry point (see `tsup.config.ts`). The csv-loader entries externalize `csv-parse`, `@rspack/core`, `esbuild`, and `rollup`.
|
||||||
|
|
||||||
|
|
@ -29,11 +29,13 @@ Build produces separate bundles per entry point (see `tsup.config.ts`). The csv-
|
||||||
- Schema syntax uses **semicolons** (`;`) as separators, not commas
|
- Schema syntax uses **semicolons** (`;`) as separators, not commas
|
||||||
- Unknown identifiers throw a `ParseError` — only recognized keywords (`string`, `number`, `int`, `float`, `boolean`) and string literals (`"on"`, `'off'`) are valid types
|
- Unknown identifiers throw a `ParseError` — only recognized keywords (`string`, `number`, `int`, `float`, `boolean`) and string literals (`"on"`, `'off'`) are valid types
|
||||||
- `@tablename` / `@tablename[]` are reference schemas resolved at CSV load time
|
- `@tablename` / `@tablename[]` are reference schemas resolved at CSV load time
|
||||||
|
- `csvToModule()` emits accessor functions (`getData()`) for tables with references, and static JSON for tables without; bundler loaders all use `csvToModule`
|
||||||
|
- `parseCsv({ resolveReferences: false })` stores reference IDs instead of resolved objects — used by `csvToModule` to emit import-based lazy resolution
|
||||||
- References can appear nested inside tuples, arrays, and unions; the loader resolves them recursively
|
- References can appear nested inside tuples, arrays, and unions; the loader resolves them recursively
|
||||||
|
|
||||||
## Gotchas
|
## Gotchas
|
||||||
|
|
||||||
- **Circular references** between CSV tables cause stack overflow. The loader detects this via an in-progress loading set and throws `"Circular reference detected"`.
|
- **Circular references** between CSV tables are supported in `csvToModule` output via accessor-based lazy resolution. `parseCsv()` with `resolveReferences: true` (default) still detects and throws on circular references via an in-progress loading set.
|
||||||
- **Run `npm run typecheck` before committing** to catch type errors.
|
- **Run `npm run typecheck` before committing** to catch type errors.
|
||||||
- **Union member ordering matters** — `parseValue` tries union members in order; the first one that parses wins. This affects references in unions (e.g., `@users[] | string` will try `@users[]` first).
|
- **Union member ordering matters** — `parseValue` tries union members in order; the first one that parses wins. This affects references in unions (e.g., `@users[] | string` will try `@users[]` first).
|
||||||
- **csv-parse quote handling** — Double-quoted schema values like `"active" | "inactive"` in CSV rows confuse the csv-parse library. Use single-quoted string literals (`'on' | 'off'`) or unquoted identifiers in the schema row of CSV data when possible.
|
- **csv-parse quote handling** — Double-quoted schema values like `"active" | "inactive"` in CSV rows confuse the csv-parse library. Use single-quoted string literals (`'on' | 'off'`) or unquoted identifiers in the schema row of CSV data when possible.
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||||
import { parseCsv } from './loader';
|
import { parseCsv, csvToModule } from './loader';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
|
|
||||||
|
|
@ -559,4 +559,395 @@ describe('parseCsv - refBaseDir option', () => {
|
||||||
email: 'alice@example.com',
|
email: 'alice@example.com',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('parseCsv - resolveReferences: false', () => {
|
||||||
|
it('should store IDs instead of resolved objects for reference fields', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,customer,items',
|
||||||
|
'string,@users,@parts[]',
|
||||||
|
'1,1,[1; 2]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.data[0].customer).toBe('1');
|
||||||
|
expect(result.data[0].items).toEqual(['1', '2']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should populate referenceFields with metadata', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,customer,items',
|
||||||
|
'string,@users,@parts[]',
|
||||||
|
'1,1,[1; 2]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.referenceFields).toHaveLength(2);
|
||||||
|
expect(result.referenceFields[0]).toEqual({
|
||||||
|
name: 'customer',
|
||||||
|
tableName: 'users',
|
||||||
|
isArray: false,
|
||||||
|
schema: expect.objectContaining({ type: 'reference', tableName: 'users', isArray: false }),
|
||||||
|
});
|
||||||
|
expect(result.referenceFields[1]).toEqual({
|
||||||
|
name: 'items',
|
||||||
|
tableName: 'parts',
|
||||||
|
isArray: true,
|
||||||
|
schema: expect.objectContaining({ type: 'reference', tableName: 'parts', isArray: true }),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not load referenced CSV files', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,customer',
|
||||||
|
'string,@nonexistent',
|
||||||
|
'1,someid',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
expect(() => parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
})).not.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should store IDs for nested references in tuples', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,info',
|
||||||
|
'string,[ref: @users; note: string]',
|
||||||
|
'1,[ref: 1; note: urgent]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect((result.data[0].info as unknown[])[0]).toBe('1');
|
||||||
|
expect((result.data[0].info as unknown[])[1]).toBe('urgent');
|
||||||
|
expect(result.referenceFields).toHaveLength(1);
|
||||||
|
expect(result.referenceFields[0].tableName).toBe('users');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should store IDs for references in unions', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,value',
|
||||||
|
'string,@users | string',
|
||||||
|
'1,1',
|
||||||
|
'2,unknown',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.data[0].value).toBe('1');
|
||||||
|
expect(result.data[1].value).toBe('unknown');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not throw for self-referencing table when resolveReferences is false', () => {
|
||||||
|
const csv = readFixture('self_ref.csv');
|
||||||
|
|
||||||
|
const result = parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
currentFilePath: path.join(fixturesDir, 'self_ref.csv'),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.data).toHaveLength(2);
|
||||||
|
expect(result.data[0].parent).toBe('2');
|
||||||
|
expect(result.data[1].parent).toBe('1');
|
||||||
|
expect(result.referenceFields).toHaveLength(1);
|
||||||
|
expect(result.referenceFields[0].tableName).toBe('self_ref');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not throw for cross-referencing tables when resolveReferences is false', () => {
|
||||||
|
const csv = readFixture('circular_a.csv');
|
||||||
|
|
||||||
|
const result = parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
currentFilePath: path.join(fixturesDir, 'circular_a.csv'),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.data).toHaveLength(1);
|
||||||
|
expect(result.data[0].related).toEqual(['1']);
|
||||||
|
expect(result.referenceFields).toHaveLength(1);
|
||||||
|
expect(result.referenceFields[0].tableName).toBe('circular_b');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should store IDs for nested self-reference in tuple', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,name,parent_info',
|
||||||
|
'string,string,[parent: @self_ref; role: string]',
|
||||||
|
'1,Root,[parent: 2; role: admin]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
currentFilePath: path.join(fixturesDir, 'self_ref.csv'),
|
||||||
|
});
|
||||||
|
|
||||||
|
const parentInfo = result.data[0].parent_info as unknown[];
|
||||||
|
expect(parentInfo[0]).toBe('2');
|
||||||
|
expect(parentInfo[1]).toBe('admin');
|
||||||
|
expect(result.referenceFields).toHaveLength(1);
|
||||||
|
expect(result.referenceFields[0].tableName).toBe('self_ref');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should store IDs for self-reference in union', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,name,ref_or_val',
|
||||||
|
'string,string,@self_ref | string',
|
||||||
|
'1,Root,2',
|
||||||
|
'2,Child,none',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
currentFilePath: path.join(fixturesDir, 'self_ref.csv'),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.data[0].ref_or_val).toBe('2');
|
||||||
|
expect(result.data[1].ref_or_val).toBe('none');
|
||||||
|
expect(result.referenceFields).toHaveLength(1);
|
||||||
|
expect(result.referenceFields[0].tableName).toBe('self_ref');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should store IDs for self-reference array in tuple', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,name,children',
|
||||||
|
'string,string,[@self_ref[]]',
|
||||||
|
'1,Root,[[2]]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = parseCsv(csv, {
|
||||||
|
emitTypes: false,
|
||||||
|
resolveReferences: false,
|
||||||
|
currentFilePath: path.join(fixturesDir, 'self_ref.csv'),
|
||||||
|
});
|
||||||
|
|
||||||
|
const children = result.data[0].children as unknown[];
|
||||||
|
expect(children[0]).toEqual(['2']);
|
||||||
|
expect(result.referenceFields).toHaveLength(1);
|
||||||
|
expect(result.referenceFields[0].tableName).toBe('self_ref');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('csvToModule - accessor-based output', () => {
|
||||||
|
it('should emit static JSON for tables without references', () => {
|
||||||
|
const csv = [
|
||||||
|
'name,age',
|
||||||
|
'string,number',
|
||||||
|
'Alice,30',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false });
|
||||||
|
|
||||||
|
expect(result.js).toContain('export default');
|
||||||
|
expect(result.js).not.toContain('import ');
|
||||||
|
expect(result.js).not.toContain('getData');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should emit accessor function for tables with references', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,customer',
|
||||||
|
'string,@users',
|
||||||
|
'1,1',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _users from './users.csv'");
|
||||||
|
expect(result.js).toContain('export default function getData()');
|
||||||
|
expect(result.js).toContain('_usersLookup');
|
||||||
|
expect(result.js).toContain('_resolved = _raw;');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should emit accessor function for tables with array references', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,items',
|
||||||
|
'string,@parts[]',
|
||||||
|
'1,[1; 2]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _parts from './parts.csv'");
|
||||||
|
expect(result.js).toContain('_partsLookup');
|
||||||
|
expect(result.js).toContain('.map(id =>');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should emit multiple imports for multiple reference tables', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,customer,items',
|
||||||
|
'string,@users,@parts[]',
|
||||||
|
'1,1,[1; 2]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _users from './users.csv'");
|
||||||
|
expect(result.js).toContain("import _parts from './parts.csv'");
|
||||||
|
expect(result.js).toContain('_usersLookup');
|
||||||
|
expect(result.js).toContain('_partsLookup');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should generate function type in dts for tables with references', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,customer',
|
||||||
|
'string,@users',
|
||||||
|
'1,1',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: true, resourceName: 'orders' });
|
||||||
|
|
||||||
|
expect(result.dts).toContain('declare function getData(): ordersTable');
|
||||||
|
expect(result.dts).toContain('export default getData');
|
||||||
|
expect(result.dts).not.toContain('declare const data');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should generate const type in dts for tables without references', () => {
|
||||||
|
const csv = [
|
||||||
|
'name,age',
|
||||||
|
'string,number',
|
||||||
|
'Alice,30',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: true, resourceName: 'people' });
|
||||||
|
|
||||||
|
expect(result.dts).toContain('declare const data: peopleTable');
|
||||||
|
expect(result.dts).toContain('export default data');
|
||||||
|
expect(result.dts).not.toContain('declare function getData');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle nested references in tuples', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,info',
|
||||||
|
'string,[ref: @users; note: string]',
|
||||||
|
'1,[ref: 1; note: urgent]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _users from './users.csv'");
|
||||||
|
expect(result.js).toContain('_usersLookup');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('csvToModule - circular reference support', () => {
|
||||||
|
it('should emit accessor for self-referencing table', () => {
|
||||||
|
const csv = readFixture('self_ref.csv');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false, currentFilePath: path.join(fixturesDir, 'self_ref.csv') });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _self_ref from './self_ref.csv'");
|
||||||
|
expect(result.js).toContain('export default function getData()');
|
||||||
|
expect(result.js).toContain('_self_refLookup');
|
||||||
|
expect(result.js).toContain('_resolved = _raw;');
|
||||||
|
expect(result.js).toContain('parent: _self_refLookup.get(String(row.parent))');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should emit accessor for cross-referencing tables', () => {
|
||||||
|
const csv = readFixture('circular_a.csv');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false, currentFilePath: path.join(fixturesDir, 'circular_a.csv') });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _circular_b from './circular_b.csv'");
|
||||||
|
expect(result.js).toContain('export default function getData()');
|
||||||
|
expect(result.js).toContain('_circular_bLookup');
|
||||||
|
expect(result.js).toContain('related:');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should emit accessor for self-referencing table with nested reference in tuple', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,name,parent_info',
|
||||||
|
'string,string,[parent: @self_ref; role: string]',
|
||||||
|
'1,Root,[parent: 2; role: admin]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false, currentFilePath: path.join(fixturesDir, 'self_ref.csv') });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _self_ref from './self_ref.csv'");
|
||||||
|
expect(result.js).toContain('_self_refLookup');
|
||||||
|
expect(result.js).toContain('export default function getData()');
|
||||||
|
expect(result.js).toContain('parent_info:');
|
||||||
|
expect(result.js).toContain('_self_refLookup.get(String(row.parent_info[0]))');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should emit accessor for self-referencing table with nested reference in union', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,name,ref_or_val',
|
||||||
|
'string,string,@self_ref | string',
|
||||||
|
'1,Root,2',
|
||||||
|
'2,Child,none',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false, currentFilePath: path.join(fixturesDir, 'self_ref.csv') });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _self_ref from './self_ref.csv'");
|
||||||
|
expect(result.js).toContain('_self_refLookup');
|
||||||
|
expect(result.js).toContain('export default function getData()');
|
||||||
|
expect(result.js).toContain('ref_or_val:');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should emit accessor for self-referencing table with nested reference array in tuple', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,name,children',
|
||||||
|
'string,string,[kids: @self_ref[]]',
|
||||||
|
'1,Root,[[2]]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false, currentFilePath: path.join(fixturesDir, 'self_ref.csv') });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _self_ref from './self_ref.csv'");
|
||||||
|
expect(result.js).toContain('_self_refLookup');
|
||||||
|
expect(result.js).toContain('export default function getData()');
|
||||||
|
expect(result.js).toContain('children:');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should generate correct type definition for self-referencing table', () => {
|
||||||
|
const csv = readFixture('self_ref.csv');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: true, resourceName: 'nodes', currentFilePath: path.join(fixturesDir, 'self_ref.csv') });
|
||||||
|
|
||||||
|
expect(result.dts).toContain('declare function getData(): nodesTable');
|
||||||
|
expect(result.dts).toContain('Self_ref');
|
||||||
|
expect(result.dts).toContain("import type { Self_ref } from './self_ref.csv'");
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should emit accessor for cross-referencing tables with array references', () => {
|
||||||
|
const csv = readFixture('circular_a.csv');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false, currentFilePath: path.join(fixturesDir, 'circular_a.csv') });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _circular_b from './circular_b.csv'");
|
||||||
|
expect(result.js).toContain('export default function getData()');
|
||||||
|
expect(result.js).toContain('_circular_bLookup');
|
||||||
|
expect(result.js).toContain('.map(id =>');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should emit accessor for nested cross-reference in tuple', () => {
|
||||||
|
const csv = [
|
||||||
|
'id,name,info',
|
||||||
|
'string,string,[ref: @circular_b; note: string]',
|
||||||
|
'1,A,[ref: 1; note: linked]',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const result = csvToModule(csv, { emitTypes: false, currentFilePath: path.join(fixturesDir, 'circular_a.csv') });
|
||||||
|
|
||||||
|
expect(result.js).toContain("import _circular_b from './circular_b.csv'");
|
||||||
|
expect(result.js).toContain('_circular_bLookup');
|
||||||
|
expect(result.js).toContain('export default function getData()');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
@ -81,6 +81,120 @@ function resolveReferenceId(
|
||||||
return obj;
|
return obj;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseReferenceIds(schema: ReferenceSchema, valueString: string): unknown {
|
||||||
|
const valueParser = new ReferenceValueParser(valueString.trim());
|
||||||
|
const ids = valueParser.parseIds(schema.isArray);
|
||||||
|
if (schema.isArray) {
|
||||||
|
return ids;
|
||||||
|
}
|
||||||
|
return ids[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseValueWithReferenceIds(
|
||||||
|
valueString: string,
|
||||||
|
schema: Schema
|
||||||
|
): unknown {
|
||||||
|
if (!hasNestedReferences(schema)) {
|
||||||
|
return parseValue(schema, valueString);
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (schema.type) {
|
||||||
|
case 'reference':
|
||||||
|
return parseReferenceIds(schema, valueString);
|
||||||
|
case 'tuple': {
|
||||||
|
const parsed = parseValue(schema, valueString) as unknown[];
|
||||||
|
return schema.elements.map((el, i) =>
|
||||||
|
hasNestedReferences(el.schema)
|
||||||
|
? extractNestedReferenceIds(parsed[i], el.schema)
|
||||||
|
: parsed[i]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
case 'array': {
|
||||||
|
const parsed = parseValue(schema, valueString) as unknown[];
|
||||||
|
return parsed.map(item =>
|
||||||
|
hasNestedReferences(schema.element)
|
||||||
|
? extractNestedReferenceIds(item, schema.element)
|
||||||
|
: item
|
||||||
|
);
|
||||||
|
}
|
||||||
|
case 'union': {
|
||||||
|
for (const member of schema.members) {
|
||||||
|
if (hasNestedReferences(member)) {
|
||||||
|
try {
|
||||||
|
const parsed = parseValue(member, valueString);
|
||||||
|
return extractNestedReferenceIds(parsed, member);
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return parseValue(schema, valueString);
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return parseValue(schema, valueString);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractNestedReferenceIds(value: unknown, schema: Schema): unknown {
|
||||||
|
switch (schema.type) {
|
||||||
|
case 'reference':
|
||||||
|
if (schema.isArray) {
|
||||||
|
const ids = Array.isArray(value) ? value : [value];
|
||||||
|
return ids.map(id => String(id));
|
||||||
|
}
|
||||||
|
return String(value);
|
||||||
|
case 'tuple': {
|
||||||
|
if (!Array.isArray(value)) return value;
|
||||||
|
return schema.elements.map((el, i) =>
|
||||||
|
hasNestedReferences(el.schema)
|
||||||
|
? extractNestedReferenceIds(value[i], el.schema)
|
||||||
|
: value[i]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
case 'array': {
|
||||||
|
if (!Array.isArray(value)) return value;
|
||||||
|
return value.map(item =>
|
||||||
|
hasNestedReferences(schema.element)
|
||||||
|
? extractNestedReferenceIds(item, schema.element)
|
||||||
|
: item
|
||||||
|
);
|
||||||
|
}
|
||||||
|
case 'union': {
|
||||||
|
for (const member of schema.members) {
|
||||||
|
if (hasNestedReferences(member)) {
|
||||||
|
try {
|
||||||
|
return extractNestedReferenceIds(value, member);
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectReferenceFields(schema: Schema, name: string): ReferenceFieldInfo[] {
|
||||||
|
const fields: ReferenceFieldInfo[] = [];
|
||||||
|
switch (schema.type) {
|
||||||
|
case 'reference':
|
||||||
|
fields.push({ name, tableName: schema.tableName, isArray: schema.isArray, schema });
|
||||||
|
break;
|
||||||
|
case 'tuple':
|
||||||
|
for (const el of schema.elements) {
|
||||||
|
fields.push(...collectReferenceFields(el.schema, name));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case 'array':
|
||||||
|
fields.push(...collectReferenceFields(schema.element, name));
|
||||||
|
break;
|
||||||
|
case 'union':
|
||||||
|
for (const member of schema.members) {
|
||||||
|
fields.push(...collectReferenceFields(member, name));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
return fields;
|
||||||
|
}
|
||||||
|
|
||||||
function parseValueWithReferences(
|
function parseValueWithReferences(
|
||||||
valueString: string,
|
valueString: string,
|
||||||
schema: Schema,
|
schema: Schema,
|
||||||
|
|
@ -203,6 +317,23 @@ export interface CsvLoaderOptions {
|
||||||
defaultPrimaryKey?: string;
|
defaultPrimaryKey?: string;
|
||||||
/** Current file path (used to resolve relative references) */
|
/** Current file path (used to resolve relative references) */
|
||||||
currentFilePath?: string;
|
currentFilePath?: string;
|
||||||
|
/**
|
||||||
|
* When false, reference fields store parsed IDs instead of resolved objects.
|
||||||
|
* Used by csvToModule to emit accessor-based code with lazy resolution.
|
||||||
|
* Default: true (resolves references eagerly by loading referenced CSV files).
|
||||||
|
*/
|
||||||
|
resolveReferences?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ReferenceFieldInfo {
|
||||||
|
/** Column name in the CSV */
|
||||||
|
name: string;
|
||||||
|
/** Referenced table name */
|
||||||
|
tableName: string;
|
||||||
|
/** Whether it's an array reference */
|
||||||
|
isArray: boolean;
|
||||||
|
/** The schema of this field (for nested references) */
|
||||||
|
schema: Schema;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CsvParseResult {
|
export interface CsvParseResult {
|
||||||
|
|
@ -214,6 +345,8 @@ export interface CsvParseResult {
|
||||||
propertyConfigs: PropertyConfig[];
|
propertyConfigs: PropertyConfig[];
|
||||||
/** Referenced table names */
|
/** Referenced table names */
|
||||||
references: Set<string>;
|
references: Set<string>;
|
||||||
|
/** Reference field metadata (populated when resolveReferences is false) */
|
||||||
|
referenceFields: ReferenceFieldInfo[];
|
||||||
}
|
}
|
||||||
|
|
||||||
interface PropertyConfig {
|
interface PropertyConfig {
|
||||||
|
|
@ -401,7 +534,8 @@ function generateTypeDefinition(
|
||||||
resourceName: string,
|
resourceName: string,
|
||||||
propertyConfigs: PropertyConfig[],
|
propertyConfigs: PropertyConfig[],
|
||||||
references: Set<string>,
|
references: Set<string>,
|
||||||
currentFilePath?: string
|
currentFilePath?: string,
|
||||||
|
hasRefs?: boolean
|
||||||
): string {
|
): string {
|
||||||
const typeName = resourceName ? `${resourceName}Table` : 'Table';
|
const typeName = resourceName ? `${resourceName}Table` : 'Table';
|
||||||
|
|
||||||
|
|
@ -432,17 +566,23 @@ function generateTypeDefinition(
|
||||||
.map((config) => ` readonly ${config.name}: ${schemaToTypeString(config.schema, resourceNames)};`)
|
.map((config) => ` readonly ${config.name}: ${schemaToTypeString(config.schema, resourceNames)};`)
|
||||||
.join('\n');
|
.join('\n');
|
||||||
|
|
||||||
// Generate both the table type and export a singular type alias for references
|
|
||||||
// e.g., for "parts" table, export both "partsTable" and "Parts" (as alias)
|
|
||||||
let exportAlias = '';
|
let exportAlias = '';
|
||||||
if (resourceName) {
|
if (resourceName) {
|
||||||
// Capitalize resource name for the singular type
|
|
||||||
const singularType = resourceName.charAt(0).toUpperCase() + resourceName.slice(1);
|
const singularType = resourceName.charAt(0).toUpperCase() + resourceName.slice(1);
|
||||||
// Remove trailing 's' if it looks like a plural (simple heuristic)
|
|
||||||
// Actually, let's just use the table name capitalized - users can adjust if needed
|
|
||||||
exportAlias = `\nexport type ${singularType} = ${typeName}[number];`;
|
exportAlias = `\nexport type ${singularType} = ${typeName}[number];`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hasRefs) {
|
||||||
|
return `${importSection}type ${typeName} = readonly {
|
||||||
|
${properties}
|
||||||
|
}[];
|
||||||
|
${exportAlias}
|
||||||
|
|
||||||
|
declare function getData(): ${typeName};
|
||||||
|
export default getData;
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
return `${importSection}type ${typeName} = readonly {
|
return `${importSection}type ${typeName} = readonly {
|
||||||
${properties}
|
${properties}
|
||||||
}[];
|
}[];
|
||||||
|
|
@ -499,6 +639,8 @@ export function parseCsv(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const resolveReferences = options.resolveReferences ?? true;
|
||||||
|
|
||||||
const propertyConfigs: PropertyConfig[] = headers.map((header: string, index: number) => {
|
const propertyConfigs: PropertyConfig[] = headers.map((header: string, index: number) => {
|
||||||
const schemaString = schemas[index];
|
const schemaString = schemas[index];
|
||||||
const schema = parseSchema(schemaString);
|
const schema = parseSchema(schemaString);
|
||||||
|
|
@ -514,14 +656,26 @@ export function parseCsv(
|
||||||
config.isReference = true;
|
config.isReference = true;
|
||||||
config.referenceTableName = schema.tableName;
|
config.referenceTableName = schema.tableName;
|
||||||
config.referenceIsArray = schema.isArray;
|
config.referenceIsArray = schema.isArray;
|
||||||
config.parser = (valueString: string) => {
|
if (resolveReferences) {
|
||||||
return parseReferenceValue(schema, valueString, refBaseDir, defaultPrimaryKey, options.currentFilePath);
|
config.parser = (valueString: string) => {
|
||||||
};
|
return parseReferenceValue(schema, valueString, refBaseDir, defaultPrimaryKey, options.currentFilePath);
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
config.parser = (valueString: string) => {
|
||||||
|
return parseReferenceIds(schema, valueString);
|
||||||
|
};
|
||||||
|
}
|
||||||
} else if (hasNestedReferences(schema)) {
|
} else if (hasNestedReferences(schema)) {
|
||||||
config.isReference = true;
|
config.isReference = true;
|
||||||
config.parser = (valueString: string) => {
|
if (resolveReferences) {
|
||||||
return parseValueWithReferences(valueString, schema, refBaseDir, defaultPrimaryKey, options.currentFilePath);
|
config.parser = (valueString: string) => {
|
||||||
};
|
return parseValueWithReferences(valueString, schema, refBaseDir, defaultPrimaryKey, options.currentFilePath);
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
config.parser = (valueString: string) => {
|
||||||
|
return parseValueWithReferenceIds(valueString, schema);
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return config;
|
return config;
|
||||||
|
|
@ -573,10 +727,20 @@ export function parseCsv(
|
||||||
return obj;
|
return obj;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const referenceFields: ReferenceFieldInfo[] = [];
|
||||||
|
if (!resolveReferences) {
|
||||||
|
for (const config of propertyConfigs) {
|
||||||
|
if (hasNestedReferences(config.schema)) {
|
||||||
|
referenceFields.push(...collectReferenceFields(config.schema, config.name));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const result: CsvParseResult = {
|
const result: CsvParseResult = {
|
||||||
data: objects,
|
data: objects,
|
||||||
propertyConfigs,
|
propertyConfigs,
|
||||||
references,
|
references,
|
||||||
|
referenceFields,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (emitTypes) {
|
if (emitTypes) {
|
||||||
|
|
@ -584,29 +748,142 @@ export function parseCsv(
|
||||||
options.resourceName || '',
|
options.resourceName || '',
|
||||||
propertyConfigs,
|
propertyConfigs,
|
||||||
references,
|
references,
|
||||||
options.currentFilePath
|
options.currentFilePath,
|
||||||
|
referenceFields.length > 0
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate runtime reference resolution code for a schema.
|
||||||
|
* Returns a JS expression string that resolves references using lookup maps.
|
||||||
|
*/
|
||||||
|
function generateSchemaResolutionCode(
|
||||||
|
schema: Schema,
|
||||||
|
valueExpr: string,
|
||||||
|
lookupVar: (tableName: string) => string,
|
||||||
|
pkField: string
|
||||||
|
): string {
|
||||||
|
switch (schema.type) {
|
||||||
|
case 'reference': {
|
||||||
|
const lookup = lookupVar(schema.tableName);
|
||||||
|
if (schema.isArray) {
|
||||||
|
return `(Array.isArray(${valueExpr}) ? ${valueExpr}.map(id => ${lookup}.get(String(id))) : ${valueExpr})`;
|
||||||
|
}
|
||||||
|
return `${lookup}.get(String(${valueExpr}))`;
|
||||||
|
}
|
||||||
|
case 'tuple': {
|
||||||
|
const elementResolvers = schema.elements.map((el, i) => {
|
||||||
|
if (hasNestedReferences(el.schema)) {
|
||||||
|
return generateSchemaResolutionCode(el.schema, `${valueExpr}[${i}]`, lookupVar, pkField);
|
||||||
|
}
|
||||||
|
return `${valueExpr}[${i}]`;
|
||||||
|
});
|
||||||
|
return `[${elementResolvers.join(', ')}]`;
|
||||||
|
}
|
||||||
|
case 'array': {
|
||||||
|
if (hasNestedReferences(schema.element)) {
|
||||||
|
const itemResolve = generateSchemaResolutionCode(schema.element, 'item', lookupVar, pkField);
|
||||||
|
return `(${valueExpr}).map(item => ${itemResolve})`;
|
||||||
|
}
|
||||||
|
return valueExpr;
|
||||||
|
}
|
||||||
|
case 'union': {
|
||||||
|
const refMembers = schema.members.filter(m => hasNestedReferences(m));
|
||||||
|
const nonRefMembers = schema.members.filter(m => !hasNestedReferences(m));
|
||||||
|
const parts: string[] = [];
|
||||||
|
for (const member of refMembers) {
|
||||||
|
const resolveCode = generateSchemaResolutionCode(member, valueExpr, lookupVar, pkField);
|
||||||
|
parts.push(resolveCode);
|
||||||
|
}
|
||||||
|
if (nonRefMembers.length > 0) {
|
||||||
|
parts.push(valueExpr);
|
||||||
|
}
|
||||||
|
return parts[0] || valueExpr;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return valueExpr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate JavaScript module code from CSV content.
|
* Generate JavaScript module code from CSV content.
|
||||||
* Returns a string that can be used as a module export.
|
* Emits an accessor function for tables with references (lazy resolution),
|
||||||
*
|
* or static JSON for tables without references.
|
||||||
* @param content - CSV content string
|
|
||||||
* @param options - Parsing options
|
|
||||||
* @returns JavaScript module code string
|
|
||||||
*/
|
*/
|
||||||
export function csvToModule(
|
export function csvToModule(
|
||||||
content: string,
|
content: string,
|
||||||
options: CsvLoaderOptions & { resourceName?: string } = {}
|
options: CsvLoaderOptions & { resourceName?: string } = {}
|
||||||
): { js: string; dts?: string } {
|
): { js: string; dts?: string } {
|
||||||
const result = parseCsv(content, options);
|
const result = parseCsv(content, { ...options, resolveReferences: false });
|
||||||
|
|
||||||
const json = JSON.stringify(result.data, null, 2);
|
const hasRefs = result.referenceFields.length > 0;
|
||||||
const js = `export default ${json};`;
|
const defaultPrimaryKey = options.defaultPrimaryKey ?? 'id';
|
||||||
|
|
||||||
|
if (!hasRefs) {
|
||||||
|
const json = JSON.stringify(result.data, null, 2);
|
||||||
|
return {
|
||||||
|
js: `export default ${json};`,
|
||||||
|
dts: result.typeDefinition,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const imports: string[] = [];
|
||||||
|
const lookupInits: string[] = [];
|
||||||
|
const lookupVarMap = new Map<string, string>();
|
||||||
|
|
||||||
|
const uniqueTables = new Set(result.referenceFields.map(f => f.tableName));
|
||||||
|
uniqueTables.forEach(tableName => {
|
||||||
|
const varName = `_${tableName}`;
|
||||||
|
lookupVarMap.set(tableName, `_${tableName}Lookup`);
|
||||||
|
imports.push(`import ${varName} from './${tableName}.csv';`);
|
||||||
|
lookupInits.push(
|
||||||
|
`const _${tableName}Lookup = new Map(${varName}().map(p => [String(p.${defaultPrimaryKey}), p]));`
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const lookupVar = (tableName: string) => lookupVarMap.get(tableName)!;
|
||||||
|
|
||||||
|
const rowResolvers: string[] = [];
|
||||||
|
for (const config of result.propertyConfigs) {
|
||||||
|
if (hasNestedReferences(config.schema)) {
|
||||||
|
const resolveCode = generateSchemaResolutionCode(
|
||||||
|
config.schema,
|
||||||
|
`row.${config.name}`,
|
||||||
|
lookupVar,
|
||||||
|
defaultPrimaryKey
|
||||||
|
);
|
||||||
|
rowResolvers.push(` ${config.name}: ${resolveCode},`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const rawJson = JSON.stringify(result.data, null, 2);
|
||||||
|
|
||||||
|
let js: string;
|
||||||
|
if (rowResolvers.length > 0) {
|
||||||
|
js = [
|
||||||
|
...imports,
|
||||||
|
'',
|
||||||
|
`const _raw = ${rawJson};`,
|
||||||
|
'',
|
||||||
|
'let _resolved = null;',
|
||||||
|
'',
|
||||||
|
'export default function getData() {',
|
||||||
|
' if (_resolved) return _resolved;',
|
||||||
|
' _resolved = _raw;',
|
||||||
|
...lookupInits.map(l => ` ${l}`),
|
||||||
|
' _resolved = _raw.map(row => ({',
|
||||||
|
' ...row,',
|
||||||
|
...rowResolvers,
|
||||||
|
' }));',
|
||||||
|
' return _resolved;',
|
||||||
|
'}',
|
||||||
|
].join('\n');
|
||||||
|
} else {
|
||||||
|
js = `export default ${rawJson};`;
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
js,
|
js,
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import type { LoaderContext } from '@rspack/core';
|
import type { LoaderContext } from '@rspack/core';
|
||||||
import type { CsvLoaderOptions } from './loader.js';
|
import type { CsvLoaderOptions } from './loader.js';
|
||||||
import { parseCsv } from './loader.js';
|
import { csvToModule } from './loader.js';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
|
|
||||||
|
|
@ -30,14 +30,14 @@ export default function csvLoader(
|
||||||
.replace(/[-_\s]+(.)?/g, (_, char) => char ? char.toUpperCase() : '')
|
.replace(/[-_\s]+(.)?/g, (_, char) => char ? char.toUpperCase() : '')
|
||||||
.replace(/^(.)/, (_, char) => char.toUpperCase());
|
.replace(/^(.)/, (_, char) => char.toUpperCase());
|
||||||
|
|
||||||
const result = parseCsv(content, {
|
const result = csvToModule(content, {
|
||||||
...options,
|
...options,
|
||||||
resourceName,
|
resourceName,
|
||||||
currentFilePath: this.resourcePath,
|
currentFilePath: this.resourcePath,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Emit type definition file if enabled
|
// Emit type definition file if enabled
|
||||||
if (emitTypes && result.typeDefinition) {
|
if (emitTypes && result.dts) {
|
||||||
const context = this.context || '';
|
const context = this.context || '';
|
||||||
// Get relative path from context, normalize to forward slashes
|
// Get relative path from context, normalize to forward slashes
|
||||||
let relativePath = this.resourcePath.replace(context, '');
|
let relativePath = this.resourcePath.replace(context, '');
|
||||||
|
|
@ -56,12 +56,12 @@ export default function csvLoader(
|
||||||
// Write directly to disk (useful for dev server)
|
// Write directly to disk (useful for dev server)
|
||||||
const absolutePath = path.join(this.context || process.cwd(), typesOutputDir || '', dtsFileName);
|
const absolutePath = path.join(this.context || process.cwd(), typesOutputDir || '', dtsFileName);
|
||||||
fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
|
fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
|
||||||
fs.writeFileSync(absolutePath, result.typeDefinition);
|
fs.writeFileSync(absolutePath, result.dts);
|
||||||
} else {
|
} else {
|
||||||
// Emit to in-memory filesystem (for production build)
|
// Emit to in-memory filesystem (for production build)
|
||||||
this.emitFile?.(outputPath, result.typeDefinition);
|
this.emitFile?.(outputPath, result.dts);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return `export default ${JSON.stringify(result.data, null, 2)};`;
|
return result.js;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue