tts-workshop/packages/bgm/src/markdown.ts

/**
 * Extract virtual definition files from markdown code blocks.
 *
 * Each fenced code block is a virtual definition file:
 * - With a `file=` segment in its info string, named relative to the
 *   current markdown file: a yaml block with `file=parts/cargo.yaml`.
 * - Without one, auto-named `./<hash>.yaml` from its content, so every yaml
 *   block is discoverable by the default include pattern (all yaml in the
 *   same and sub folders). Identical blocks dedupe to the same hash.
 *
 * Markdown is tokenized with `marked`; each `code` token is a candidate
 * virtual file.
 */
import * as crypto from 'node:crypto';
import { posix } from 'node:path';
import { marked } from 'marked';
import { BgmError, ROLES, type DefFile, type Role, type RoleMeta } from './types.js';

/** The languages that count as definition files; others are ignored. */
const DEF_LANGS = new Set(['yaml', 'yml', 'json', 'toml']);

/** A single fenced code block. */
export interface Fence {
  /** Line number (1-based) of the opening fence. */
  startLine: number;
  /** Line number (1-based) of the closing fence. */
  endLine: number;
  /** The info string content (e.g. `yaml file=parts/cargo.yaml`). */
  info: string;
  /** The code block's content (without the fences). */
  content: string;
}

/** Result of scanning a markdown file. */
export interface MarkdownResult {
  /** All fenced code blocks found, in order. */
  fences: Fence[];
  /** Virtual def files extracted from the definition-language blocks. */
  files: DefFile[];
}

/**
 * Scan `text` for fenced code blocks.
 *
 * @param text the markdown source
 * @param sourcePath the markdown file's path-style name, for error messages
 *   and for resolving `file=` names relative to the markdown file
 * @returns the fences and the virtual def files derived from them
 */
export function scanMarkdown(text: string, sourcePath: string): MarkdownResult {
  const fences: Fence[] = [];
  const files: DefFile[] = [];

  const tokens = marked.lexer(text);
  for (const token of tokens) {
    if (token.type !== 'code' || token.codeBlockStyle === 'indented') continue;
    const info = token.lang ?? '';
    const startLine = lineOf(text, token.raw);
    const endLine = startLine + token.raw.split(/\r?\n/).length - 1;

    fences.push({ startLine, endLine, info, content: token.text });

    const name = parseInfo(info, token.text);
    if (name) {
      files.push({
        name: posix.join(posix.dirname(sourcePath), name),
        text: token.text,
        source: `${sourcePath}:${startLine}-${endLine}`,
        kind: kindOf(name),
        role: parseRole(info, `${sourcePath}:${startLine}`),
      });
    }
  }

  return { fences, files };
}

/**
 * Parse a fence's info string for a `file=` segment and derive the virtual
 * file name. Blocks without `file=` are auto-named from their content hash.
 */
function parseInfo(info: string, content: string): string | null {
  const fileMatch = /file=(\S+)/.exec(info);
  if (fileMatch) return fileMatch[1]!;

  const lang = info.split(/\s+/)[0];
  if (!lang || !DEF_LANGS.has(lang)) return null;
  return `./${contentHash(content)}.yaml`;
}

/**
 * Parse a fence's info string for a `role=` segment into role metadata.
 * `role=part.cargo` -> `{ role: 'part', type: 'cargo' }`;
 * `role=surface.game#main` -> `{ role: 'surface', type: 'game', id: 'main' }`;
 * `role=package` -> `{ role: 'package' }`. Returns `undefined` when absent.
 */
function parseRole(info: string, source: string): RoleMeta | undefined {
  const match = /role=(\S+)/.exec(info);
  if (!match) return undefined;
  const spec = match[1]!;
  const [role, rest] = spec.split('.');
  if (!role || !ROLES.has(role as Role)) {
    throw new BgmError(`Invalid role "${spec}"`, source);
  }
  if (role === 'package') {
    if (rest) throw new BgmError(`Package role takes no type or id`, source);
    return { role: 'package' };
  }
  const [type, id] = (rest ?? '').split('#');
  return { role: role as Role, type: type || undefined, id: id || undefined };
}

/** Derive the def file type from its name's extension. */
function kindOf(name: string): DefFile['kind'] {
  if (name.endsWith('.json')) return 'json';
  if (name.endsWith('.toml')) return 'toml';
  if (name.endsWith('.md') || name.endsWith('.markdown')) return 'markdown';
  if (name.endsWith('.csv')) return 'csv';
  return 'yaml';
}

/** Hash the block's content, so identical blocks dedupe to the same name. */
function contentHash(content: string): string {
  return crypto.createHash('sha1').update(content).digest('hex').slice(0, 8);
}

/** The 1-based line number where `raw` starts within `text`. */
function lineOf(text: string, raw: string): number {
  const idx = text.indexOf(raw);
  if (idx < 0) return 1;
  return text.slice(0, idx).split(/\r?\n/).length;
}

/**
 * Virtual files gathered from markdown code blocks, keyed by path-style name.
 * Multiple blocks may share a name (e.g. several `file=parts/tokens.yaml`
 * blocks); each is kept as a separate entry. Identical blocks dedupe to the
 * same hash name.
 */
export type VirtualFiles = Map<string, DefFile[]>;

/**
 * Collect virtual def files from a set of markdown sources.
 *
 * @param markdownFiles real markdown files, keyed by their path-style name
 *   relative to the games root, e.g. `harbor/harbor.md`
 * @returns the virtual files, keyed by name
 */
export function collectVirtualFiles(markdownFiles: Map<string, string>): VirtualFiles {
  const files = new Map<string, DefFile[]>();
  for (const [name, text] of markdownFiles) {
    const result = scanMarkdown(text, name);
    for (const file of result.files) {
      const list = files.get(file.name) ?? [];
      list.push(file);
      files.set(file.name, list);
    }
  }
  return files;
}