feat(csv-loader): strip comments before parsing CSV
Pre-filter comment lines from the content before passing it to `csv-parse`. This prevents quote parsing errors when comment lines contain double quotes and simplifies the record filtering logic.
This commit is contained in:
+25
-18
@@ -789,34 +789,41 @@ export function parseCsv(
|
||||
const refBaseDir = options.refBaseDir;
|
||||
const defaultPrimaryKey = options.defaultPrimaryKey ?? "id";
|
||||
|
||||
const records = parse(content, {
|
||||
// Pre-strip comment lines from content before passing to csv-parse,
|
||||
// to avoid quote parsing errors in comment lines containing double quotes.
|
||||
const reverseReferences: ReverseReferenceDeclaration[] = [];
|
||||
let filteredContent = content;
|
||||
if (comment) {
|
||||
const lines = content.split(/\r?\n/);
|
||||
const nonCommentLines: string[] = [];
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.startsWith(comment)) {
|
||||
const decl = parseReverseReferenceDeclaration(trimmed, comment);
|
||||
if (decl) {
|
||||
reverseReferences.push(decl);
|
||||
}
|
||||
// Skip comment lines
|
||||
} else {
|
||||
nonCommentLines.push(line);
|
||||
}
|
||||
}
|
||||
filteredContent = nonCommentLines.join("\n");
|
||||
}
|
||||
|
||||
const records = parse(filteredContent, {
|
||||
delimiter,
|
||||
quote,
|
||||
escape,
|
||||
bom,
|
||||
// Don't let csv-parse skip comments; we need to parse them for reverse references.
|
||||
// Comment lines are filtered out manually below using the configured comment character.
|
||||
comment: undefined,
|
||||
trim,
|
||||
skip_empty_lines: true,
|
||||
relax_column_count: true,
|
||||
});
|
||||
|
||||
// Filter out comment lines from all records, collecting reverse reference declarations
|
||||
const reverseReferences: ReverseReferenceDeclaration[] = [];
|
||||
const filteredRecords: string[][] = [];
|
||||
for (const row of records) {
|
||||
const firstCell = (row[0] ?? "").trim();
|
||||
if (comment && firstCell.startsWith(comment)) {
|
||||
const decl = parseReverseReferenceDeclaration(firstCell, comment);
|
||||
if (decl) {
|
||||
reverseReferences.push(decl);
|
||||
}
|
||||
// Skip comment lines (whether or not they're reverse ref declarations)
|
||||
continue;
|
||||
}
|
||||
filteredRecords.push(row);
|
||||
}
|
||||
// Comment lines were already filtered out before parsing
|
||||
const filteredRecords = records;
|
||||
|
||||
if (filteredRecords.length < 2) {
|
||||
throw new Error("CSV must have at least 2 rows: headers and schemas");
|
||||
|
||||
Reference in New Issue
Block a user