feat: scaffold workspace with search, fetch, and extract packages

packages/extract/src/download.ts

@@ -0,0 +1,81 @@
+import type { AssetRef } from '@tts/shared';
+
+/** Guess a MIME type from a URL's file extension. */
+export function guessMimeType(url: string): string {
+  const ext = url.split('?')[0]!.split('.').pop()!.toLowerCase();
+  switch (ext) {
+    case 'png':
+      return 'image/png';
+    case 'jpg':
+    case 'jpeg':
+      return 'image/jpeg';
+    case 'gif':
+      return 'image/gif';
+    case 'webp':
+      return 'image/webp';
+    case 'pdf':
+      return 'application/pdf';
+    default:
+      return 'application/octet-stream';
+  }
+}
+
+/** Download a single asset as a `Blob`. */
+export async function downloadAsset(url: string): Promise<Blob> {
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(`Failed to download asset (${res.status}): ${url}`);
+  }
+  return res.blob();
+}
+
+export interface DownloadProgress {
+  done: number;
+  total: number;
+  url: string;
+}
+
+export interface DownloadOptions {
+  concurrency?: number;
+  onProgress?: (p: DownloadProgress) => void;
+}
+
+export interface DownloadedAsset {
+  ref: AssetRef;
+  blob: Blob;
+}
+
+/**
+ * Download multiple assets with limited concurrency and a progress callback.
+ */
+export async function downloadAll(
+  refs: AssetRef[],
+  options: DownloadOptions = {},
+): Promise<DownloadedAsset[]> {
+  const concurrency = options.concurrency ?? 4;
+  const results: DownloadedAsset[] = [];
+  let done = 0;
+
+  const worker = async (ref: AssetRef): Promise<void> => {
+    const blob = await downloadAsset(ref.url);
+    results.push({ ref, blob });
+    done += 1;
+    options.onProgress?.({ done, total: refs.length, url: ref.url });
+  };
+
+  const queue = [...refs];
+  const workers: Promise<void>[] = [];
+  for (let i = 0; i < Math.min(concurrency, queue.length); i++) {
+    workers.push(
+      (async () => {
+        while (queue.length > 0) {
+          const ref = queue.shift()!;
+          await worker(ref);
+        }
+      })(),
+    );
+  }
+
+  await Promise.all(workers);
+  return results;
+}

packages/extract/src/index.ts

@@ -0,0 +1,4 @@
+export * from './traverse.js';
+export * from './objects.js';
+export * from './refs.js';
+export * from './download.js';

packages/extract/src/objects.ts

@@ -0,0 +1,32 @@
+import type { TTSMod, TTSObject } from '@tts/shared';
+import { markParent, traverseMod } from './traverse.js';
+
+/**
+ * Flatten all objects in a save into an array.
+ * Ensures parent links are set before returning.
+ */
+export function flattenObjects(mod: TTSMod): TTSObject[] {
+  for (const state of mod.ObjectStates) {
+    markParent(state);
+  }
+  return [...traverseMod(mod)];
+}
+
+/**
+ * Filter objects by a predicate (name, GUID, type, etc.).
+ */
+export function filterObjects(
+  mod: TTSMod,
+  predicate: (o: TTSObject) => boolean,
+): TTSObject[] {
+  return flattenObjects(mod).filter(predicate);
+}
+
+/**
+ * Find a single object by GUID.
+ */
+export function findObject(mod: TTSMod, guid: string): TTSObject | undefined {
+  return flattenObjects(mod).find((o) => o.GUID === guid);
+}
+
+export { markParent, traverseMod } from './traverse.js';

packages/extract/src/refs.ts

@@ -0,0 +1,60 @@
+import type { AssetRef, TTSMod, TTSObject } from '@tts/shared';
+import { markParent, traverseMod } from './traverse.js';
+
+/**
+ * Extract every external asset reference from a single object.
+ */
+export function extractRefs(o: TTSObject): AssetRef[] {
+  const refs: AssetRef[] = [];
+  const ownerGuid = o.GUID;
+
+  if (o.CustomPDF?.PDFUrl) {
+    refs.push({ kind: 'pdf', url: o.CustomPDF.PDFUrl, ownerGuid });
+  }
+
+  if (o.CustomDeck) {
+    for (const deck of Object.values(o.CustomDeck)) {
+      if (deck.FaceURL) {
+        refs.push({ kind: 'deckFace', url: deck.FaceURL, ownerGuid });
+      }
+      if (deck.BackURL) {
+        refs.push({ kind: 'deckBack', url: deck.BackURL, ownerGuid });
+      }
+    }
+  }
+
+  if (o.CustomImage?.ImageURL) {
+    refs.push({ kind: 'image', url: o.CustomImage.ImageURL, ownerGuid });
+  }
+  if (o.CustomImage?.ImageSecondaryURL) {
+    refs.push({
+      kind: 'imageSecondary',
+      url: o.CustomImage.ImageSecondaryURL,
+      ownerGuid,
+    });
+  }
+
+  return refs;
+}
+
+/**
+ * Collect all asset references across a whole save, deduped by URL.
+ */
+export function collectRefs(mod: TTSMod): AssetRef[] {
+  for (const state of mod.ObjectStates) {
+    markParent(state);
+  }
+  const seen = new Set<string>();
+  const refs: AssetRef[] = [];
+  for (const o of traverseMod(mod)) {
+    for (const ref of extractRefs(o)) {
+      if (!seen.has(ref.url)) {
+        seen.add(ref.url);
+        refs.push(ref);
+      }
+    }
+  }
+  return refs;
+}
+
+export { markParent, traverseMod } from './traverse.js';

packages/extract/src/traverse.ts

@@ -0,0 +1,34 @@
+import type { TTSMod, TTSObject } from '@tts/shared';
+
+/**
+ * Recursively set each object's `Parent` to its container.
+ * Mutates the tree in place.
+ */
+export function markParent(o: TTSObject): void {
+  if (o.ContainedObjects) {
+    for (const each of o.ContainedObjects) {
+      each.Parent = o;
+      markParent(each);
+    }
+  }
+}
+
+/**
+ * Yield every object in a save, descending into Bags and Custom_Model_Bags.
+ * Accepts either a full `TTSMod` or a single `TTSObject`.
+ */
+export function* traverseMod(
+  mod: TTSMod | TTSObject,
+): Iterable<TTSObject> {
+  if ('ObjectStates' in mod) {
+    for (const one of mod.ObjectStates) {
+      yield* traverseMod(one);
+    }
+  } else if (mod.Name === 'Bag' || mod.Name === 'Custom_Model_Bag') {
+    for (const one of mod.ContainedObjects ?? []) {
+      yield* traverseMod(one);
+    }
+  } else {
+    yield mod;
+  }
+}