refactor(completions): centralize content scanning

Add a browser-safe ContentRegistry owning path and inline content
stores, then derive completions from it. Remove the old block-processor,
directive-scanner, and link-source modules in favor of scanDoc,
deriveCompletions, and injectSparkDirectives, and register an inline
resolver for the frontend file index.
This commit is contained in:
hyper
2026-08-07 12:17:35 +08:00
parent 9e081891df
commit e8aa7165cb
13 changed files with 985 additions and 848 deletions
+689
View File
@@ -0,0 +1,689 @@
/**
* Content registry — the single source of truth for all content in a
* TTRPG Tools project.
*
* Two stores:
* - `pathIndex`: real files on disk, keyed by path (`.md`, `.csv`, `.yarn`, `.svg`)
* - `docContent`: inline content *defined inside* a markdown doc, keyed by
* a stable id and owned by that doc.
*
* Everything structured (completions, declarations, tag modifiers) is a
* *derived* view over this registry — see `deriveCompletions`.
*
* This module is browser-safe (no Node-only imports) so the CLI and the
* frontend share one implementation. The filesystem walk lives in the CLI
* (`buildRegistry` in `commands/serve.ts`).
*/
import Slugger from "github-slugger";
import {
parseDeclareCsv,
type VarDeclaration,
type TagModifier,
} from "./completions/declare-parser.js";
import {
FENCED_BLOCK_RE,
parseBlockAttrs,
resolveBlockAs,
} from "./completions/block-scanner.js";
import type {
CompletionsPayload,
DiceCompletion,
LinkCompletion,
SparkTableCompletion,
} from "./completions/types.js";
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export type ContentKind = "csv" | "text" | "declare";
/** A single piece of inline content defined inside a doc. */
export interface DocContent {
/** Stable id — author-supplied or derived from the body. */
id: string;
kind: ContentKind;
body: string;
/** Origin role that produced this content, for debugging. */
role?: string;
/** Origin `as` value, for debugging. */
as?: string;
}
export interface ContentRegistry {
/** Real files on disk, keyed by path. */
pathIndex: Record<string, string>;
/** Inline content defined inside each doc, keyed by id. */
docContent: Record<string, Record<string, DocContent>>;
}
export const EMPTY_REGISTRY: ContentRegistry = {
pathIndex: {},
docContent: {},
};
// ---------------------------------------------------------------------------
// Id / hash derivation (single source of truth)
// ---------------------------------------------------------------------------
/** Browser-safe content hash — stable across CLI and frontend. */
export function contentHash(body: string): string {
let hash = 0;
for (let i = 0; i < body.length; i++) {
const ch = body.charCodeAt(i);
hash = ((hash << 5) - hash + ch) | 0;
}
return Math.abs(hash).toString(16).slice(0, 8);
}
/**
* Derive a stable content id.
* Author-supplied `id` wins; otherwise `{kind}_{hash}`.
*/
export function deriveContentId(
kind: ContentKind,
body: string,
id?: string,
): string {
if (id) return id;
return `${kind}_${contentHash(body)}`;
}
// ---------------------------------------------------------------------------
// Per-doc scanning
// ---------------------------------------------------------------------------
export interface DocScanResult {
/** Content with blocks processed (stripped or replaced with directives). */
stripped: string;
/** Inline content defined in this doc, keyed by id. */
content: Record<string, DocContent>;
}
/**
* Process a single markdown doc:
* - strips/replaces attributed fenced code blocks based on `as`
* - coerces spark-shaped markdown tables to `:md-table` directives
* - collects inline content (role=file, md-* bodies, spark tables, declare)
* into the doc's content store
*
* Does NOT touch the path index — the caller assembles the registry.
*/
export function scanDoc(content: string, docPath: string): DocScanResult {
const contentStore: Record<string, DocContent> = {};
// ---- Pass 1: attributed fenced code blocks ----
const stripped = content.replace(
FENCED_BLOCK_RE,
(
_match: string,
lang: string,
infoString: string,
body: string,
): string => {
const attrs = parseBlockAttrs(infoString);
attrs.lang = attrs.lang || lang;
const effectiveAs = resolveBlockAs(attrs.role, attrs.as);
if (attrs.role === "declare") {
const id = deriveContentId("declare", body, attrs.id);
contentStore[id] = {
id,
kind: "declare",
body,
role: attrs.role,
as: effectiveAs,
};
}
if (attrs.role === "file") {
const id = deriveContentId("text", body, attrs.id);
contentStore[id] = {
id,
kind: "text",
body,
role: attrs.role,
as: effectiveAs,
};
}
if (effectiveAs === "codeblock") {
return _match;
}
if (effectiveAs === "none") {
return "";
}
if (effectiveAs.startsWith("md-")) {
const id = deriveContentId("csv", body, attrs.id);
contentStore[id] = {
id,
kind: "csv",
body,
role: attrs.role,
as: effectiveAs,
};
const extra = { ...attrs.extra };
const extraStr = Object.keys(extra).length
? `{${Object.entries(extra)
.map(([k, v]) => `${k}=${v}`)
.join(" ")}}`
: "";
return `:${effectiveAs}[./${id}]${extraStr}`;
}
return "";
},
);
// ---- Pass 2: coerce spark-shaped markdown tables to :md-table ----
const rewritten = coerceSparkTables(stripped, contentStore);
return { stripped: rewritten, content: contentStore };
}
// ---------------------------------------------------------------------------
// Spark table coercion
// ---------------------------------------------------------------------------
const DICE_HEADER_RE = /^\d*d\d+$/i;
/** Split a markdown table row into cells. */
function splitTableRow(row: string): string[] {
return row
.replace(/^\|/, "")
.replace(/\|$/, "")
.split("|")
.map((c) => c.trim());
}
/** Escape a cell value for CSV output. */
function escapeCsvCell(cell: string): string {
if (
cell.includes(",") ||
cell.includes("\n") ||
cell.includes('"') ||
cell.includes("#")
) {
return `"${cell.replace(/"/g, '""')}"`;
}
return cell;
}
/** Convert a markdown table (header + separator + rows) to a CSV string. */
function markdownTableToCsv(
headerRow: string,
separatorRow: string,
bodyRows: string[],
): string | null {
const headers = splitTableRow(headerRow);
if (headers.length === 0) return null;
const sepCells = splitTableRow(separatorRow);
if (!sepCells.every((c) => /^:?-{3,}:?$/.test(c))) return null;
if (sepCells.length !== headers.length) return null;
const csvHeader = headers.map(escapeCsvCell).join(",");
const csvRows = bodyRows.map((row) => {
const cells = splitTableRow(row);
while (cells.length < headers.length) cells.push("");
return cells.slice(0, headers.length).map(escapeCsvCell).join(",");
});
return [csvHeader, ...csvRows].join("\n");
}
/**
* Coerce spark-shaped markdown tables (first column header is a dice formula)
* into `:md-table` directives, storing the CSV in the doc's content store and
* injecting `data-spark` for the reveal feature.
*/
function coerceSparkTables(
content: string,
contentStore: Record<string, DocContent>,
): string {
const mdTableRegex = /^(\|.+\|)\n(\|[-: |]+\|)\n((?:\|.+\|\n?)+)/gm;
interface TableMatch {
fullMatch: string;
headerRow: string;
separatorRow: string;
bodyRowsText: string;
index: number;
}
const tableMatches: TableMatch[] = [];
let mdMatch: RegExpExecArray | null;
while ((mdMatch = mdTableRegex.exec(content)) !== null) {
const [, headerRow, separatorRow, bodyRowsText] = mdMatch;
const headers = splitTableRow(headerRow);
if (!DICE_HEADER_RE.test(headers[0])) continue;
const bodyRows = bodyRowsText
.trim()
.split(/\n/)
.filter((r) => r.trim().startsWith("|"));
const csv = markdownTableToCsv(headerRow, separatorRow, bodyRows);
if (!csv) continue;
tableMatches.push({
fullMatch: mdMatch[0],
headerRow,
separatorRow,
bodyRowsText,
index: mdMatch.index,
});
}
let rewritten = content;
for (let i = tableMatches.length - 1; i >= 0; i--) {
const m = tableMatches[i];
const bodyRows = m.bodyRowsText
.trim()
.split(/\n/)
.filter((r) => r.trim().startsWith("|"));
const csv = markdownTableToCsv(m.headerRow, m.separatorRow, bodyRows)!;
const id = deriveContentId("csv", csv);
contentStore[id] = {
id,
kind: "csv",
body: csv,
role: "spark-table",
as: "md-table",
};
const slug = sparkSlug(csv);
const directive = `:md-table[./${id}]{data-spark="${slug}"}`;
rewritten =
rewritten.slice(0, m.index) +
directive +
rewritten.slice(m.index + m.fullMatch.length);
}
return rewritten;
}
// ---------------------------------------------------------------------------
// Directive spark injection (real-file `:md-table` / `:md-card` references)
// ---------------------------------------------------------------------------
/**
* Scan a doc's stripped content for `:md-table[...]` / `:md-card[...]`
* directives that resolve to a spark table, and inject `data-spark` so the
* reveal feature can match them. Idempotent — only injects when missing.
*
* Returns the possibly-rewritten content.
*/
export function injectSparkDirectives(
content: string,
docPath: string,
registry: ContentRegistry,
): string {
const tableDirectiveRegex = /:md-(table|card)\[([^[\]]+)\](?:\{([^}]*)\})?/gi;
let rewritten = content;
let m: RegExpExecArray | null;
while ((m = tableDirectiveRegex.exec(content)) !== null) {
const [, , ref, extraStr] = m;
if (extraStr && extraStr.includes("data-spark=")) continue;
const csv = resolveContent(registry, docPath, ref);
if (!csv) continue;
const slug = sparkSlug(csv);
if (!slug) continue;
const fullMatch = m[0];
const insertPos = fullMatch.indexOf("]") + 1;
const before = fullMatch.slice(0, insertPos);
const after = fullMatch.slice(insertPos);
let replacement: string;
if (after.startsWith("{")) {
replacement = before + after.replace(/^\{/, `{data-spark="${slug}" `);
} else {
replacement = before + `{data-spark="${slug}"}` + after;
}
rewritten =
rewritten.slice(0, m.index) +
replacement +
rewritten.slice(m.index + fullMatch.length);
}
return rewritten;
}
// ---------------------------------------------------------------------------
// Resolution
// ---------------------------------------------------------------------------
/**
* Resolve a content reference from within a doc.
*
* - `ref` is inline CSV → returned as-is.
* - `ref` is an absolute path → path index.
* - `ref` is a relative path → resolved against the doc directory; checks
* the path index first, then the doc's inline content store.
*
* Returns `null` when nothing matches.
*/
export function resolveContent(
registry: ContentRegistry,
docPath: string,
ref: string,
): string | null {
const trimmed = ref.trim();
if (!trimmed) return null;
// Inline CSV body.
if (looksLikeCsv(trimmed)) return trimmed;
if (trimmed.startsWith("/")) {
return registry.pathIndex[trimmed] ?? null;
}
// Inline content in the same doc (e.g. ./{id}).
const docStore = registry.docContent[docPath];
if (docStore) {
const id = trimmed.startsWith("./") ? trimmed.slice(2) : trimmed;
const entry = docStore[id];
if (entry) return entry.body;
}
// Relative path resolved against the doc directory.
const resolved = posixJoin(posixDir(docPath), trimmed);
return registry.pathIndex[resolved] ?? null;
}
/**
* Resolve a *resolved* path (e.g. `/content/csv_abc123`) to inline content
* by searching every doc's content store for a matching id. Used by the
* frontend when a directive ref has already been resolved to a path.
*
* Returns `null` when no inline content matches.
*/
export function resolveInlineByPath(
registry: ContentRegistry,
resolvedPath: string,
): string | null {
const id = resolvedPath.split("/").filter(Boolean).pop() || "";
if (!id) return null;
for (const store of Object.values(registry.docContent)) {
const entry = store[id];
if (entry) return entry.body;
}
return null;
}
/** Naive CSV sniff — matches the frontend `isCSV` heuristic. */
function looksLikeCsv(str: string): boolean {
const trimmed = str.trim();
if (trimmed.startsWith("---\n") || trimmed.startsWith("---\r\n")) return true;
const lines = trimmed.split(/\r?\n/).filter((line) => line.trim() !== "");
if (lines.length < 2) return false;
const separators = [",", "\t", ";", "|"];
const firstLine = lines[0];
for (const sep of separators) {
if (firstLine.includes(sep)) {
const hasInOthers = lines.slice(1).some((line) => line.includes(sep));
if (hasInOthers) return true;
}
}
return false;
}
// ---------------------------------------------------------------------------
// Derived completions
// ---------------------------------------------------------------------------
/**
* Derive the completions payload from the registry.
* Pure function — recompute on any file change.
*
* Dice + spark completions come from scanning each doc's stripped content
* for directives (resolving CSV refs through the registry), so both inline
* and real-file spark tables are covered. Declarations come from the doc
* content store.
*/
export function deriveCompletions(
registry: ContentRegistry,
): CompletionsPayload {
const links = deriveLinks(registry.pathIndex);
const dice: DiceCompletion[] = [];
const sparkTables: SparkTableCompletion[] = [];
const declarations: VarDeclaration[] = [];
const tagModifiers: TagModifier[] = [];
for (const [docPath, content] of Object.entries(registry.pathIndex)) {
if (!docPath.endsWith(".md")) continue;
const found = scanDocDirectives(content, docPath, registry);
dice.push(...found.dice);
sparkTables.push(...found.sparkTables);
}
const blocks = deriveBlocks(registry);
declarations.push(...blocks.declarations);
tagModifiers.push(...blocks.tagModifiers);
return { dice, links, sparkTables, declarations, tagModifiers };
}
/**
* Scan a doc's stripped content for `:md-dice` and `:md-table`/`:md-card`
* directives, resolving CSV refs through the registry.
*/
function scanDocDirectives(
content: string,
docPath: string,
registry: ContentRegistry,
): { dice: DiceCompletion[]; sparkTables: SparkTableCompletion[] } {
const dice = scanDice(content, docPath);
const sparkTables: SparkTableCompletion[] = [];
const tableDirectiveRegex = /:md-(table|card)\[([^[\]]+)\](?:\{([^}]*)\})?/gi;
let m: RegExpExecArray | null;
while ((m = tableDirectiveRegex.exec(content)) !== null) {
const [, , ref, extraStr] = m;
const csv = resolveContent(registry, docPath, ref);
if (!csv) continue;
// csvPath: content id for inline content, resolved path for real files.
const docStore = registry.docContent[docPath];
const id = ref.startsWith("./") ? ref.slice(2) : ref;
const csvPath =
docStore && docStore[id]
? id
: resolveContentPath(registry, docPath, ref);
const attrs = parseDirectiveAttrs(extraStr);
const st = buildSparkTableCompletion(
csv,
docPath,
csvPath,
attrs["remix"] === "true",
);
if (st) sparkTables.push(st);
}
return { dice, sparkTables };
}
/** Resolve a directive ref to a path-index key (for real files). */
function resolveContentPath(
registry: ContentRegistry,
docPath: string,
ref: string,
): string {
const trimmed = ref.trim();
if (trimmed.startsWith("/")) return trimmed;
return posixJoin(posixDir(docPath), trimmed.replace(/^\.\//, ""));
}
/** Parse key=value pairs from a directive extra-attrs string. */
function parseDirectiveAttrs(
extraStr: string | undefined,
): Record<string, string> {
if (!extraStr) return {};
const attrs: Record<string, string> = {};
const re = /(\w+)\s*=\s*("[^"]*"|\S+)/g;
let m: RegExpExecArray | null;
while ((m = re.exec(extraStr)) !== null) {
attrs[m[1]] = m[2].replace(/^"|"$/g, "");
}
return attrs;
}
/** Derive variable declarations + tag modifiers from the registry. */
export function deriveBlocks(registry: ContentRegistry): {
declarations: VarDeclaration[];
tagModifiers: TagModifier[];
} {
const declarations: VarDeclaration[] = [];
const tagModifiers: TagModifier[] = [];
for (const [docPath, store] of Object.entries(registry.docContent)) {
for (const entry of Object.values(store)) {
if (entry.kind !== "declare") continue;
try {
const result = parseDeclareCsv(entry.body, docPath);
declarations.push(...result.variables);
tagModifiers.push(...result.tagModifiers);
} catch (e) {
console.warn(`[content-registry] ${docPath}: ${e}`);
}
}
}
return { declarations, tagModifiers };
}
// ---------------------------------------------------------------------------
// Derivation helpers
// ---------------------------------------------------------------------------
/** Extract headings from all `.md` files as link completions. */
function deriveLinks(pathIndex: Record<string, string>): LinkCompletion[] {
const items: LinkCompletion[] = [];
for (const [filePath, content] of Object.entries(pathIndex)) {
if (!filePath.endsWith(".md")) continue;
const basePath = filePath.replace(/\.md$/, "");
const fileName = fileNameFromPath(basePath);
const slugger = new Slugger();
items.push({ path: basePath, label: fileName, section: null });
const headingRegex = /^(#{1,6})\s+(.+)$/gm;
let match: RegExpExecArray | null;
while ((match = headingRegex.exec(content)) !== null) {
const title = match[2].trim();
const id = slugger.slug(title.toLowerCase());
items.push({
path: basePath,
label: `${fileName} § ${title}`,
section: id,
});
}
}
return items;
}
const DICE_DIRECTIVE_RE = /:md-dice\[([^[\]]+)\]/gi;
function looksLikeDice(raw: string): boolean {
if (raw.length > 80) return false;
return /^\d*d\d+/i.test(raw) || /^[+-]/.test(raw);
}
/** Scan a text body for `:md-dice[...]` directives. */
function scanDice(body: string, source: string): DiceCompletion[] {
const dice: DiceCompletion[] = [];
let m: RegExpExecArray | null;
while ((m = DICE_DIRECTIVE_RE.exec(body)) !== null) {
const raw = m[1].trim();
if (!raw || !looksLikeDice(raw)) continue;
dice.push({ label: raw, notation: raw, source });
}
return dice;
}
/**
* Inspect a CSV body and return its data-column headers if it's a spark
* table (first column header is a dice formula), or null otherwise.
*/
export function inspectSparkTableCsv(csv: string): string[] | null {
const lines = csv.trim().split(/\r?\n/);
if (lines.length < 2) return null;
const headers = lines[0].split(",").map((h) => h.trim());
if (headers.length < 2) return null;
if (!DICE_HEADER_RE.test(headers[0])) return null;
return headers.slice(1);
}
/** Compute the spark slug for a CSV body, or null if it's not a spark table. */
function sparkSlug(csv: string): string | null {
const dataHeaders = inspectSparkTableCsv(csv);
if (!dataHeaders) return null;
const slugger = new Slugger();
return dataHeaders.map((h) => slugger.slug(h.toLowerCase())).join("-");
}
/** Build a SparkTableCompletion from a CSV body. */
export function buildSparkTableCompletion(
csv: string,
docPath: string,
contentId: string,
remix: boolean,
): SparkTableCompletion | null {
const dataHeaders = inspectSparkTableCsv(csv);
if (!dataHeaders) return null;
const notation = csv.trim().split(/\r?\n/)[0].split(",")[0].trim();
const slugger = new Slugger();
const slug = dataHeaders.map((h) => slugger.slug(h.toLowerCase())).join("-");
const basePath = docPath.replace(/\.md$/, "");
const fileName = basePath.split("/").filter(Boolean).pop() || basePath;
const combinedSlug = `${fileName}-${slug}`;
return {
label: `${fileName} § ${slug}`,
notation,
slug: combinedSlug,
filePath: basePath,
docPath,
csvPath: contentId,
headers: dataHeaders,
remix,
};
}
// ---------------------------------------------------------------------------
// Path helpers (browser-safe posix)
// ---------------------------------------------------------------------------
function posixDir(path: string): string {
const idx = path.lastIndexOf("/");
return idx >= 0 ? path.slice(0, idx) : ".";
}
function posixJoin(dir: string, rel: string): string {
if (dir === ".") return rel.startsWith("/") ? rel : `/${rel}`;
const base = dir.replace(/\/+$/, "");
const r = rel.replace(/^\/+/, "");
return `${base}/${r}`;
}
function fileNameFromPath(path: string): string {
const parts = path.split("/").filter(Boolean);
return parts[parts.length - 1] || path;
}