feat: add core domain types for Photo Gallery SDK including media items, albums, and annotations

2026-07-23 07:39:01 +05:30
parent 7bcd1a2a2d
commit 2613c767a6
107 changed files with 20699 additions and 7 deletions
@@ -0,0 +1,734 @@
+"use client";
+
+/**
+ * The CRM's AIProvider for the Smart Gallery.
+ *
+ * PROVENANCE: a port of the SDK demo's `createDemoAIProvider()` plus its
+ * clip/face/ocr/tensorflow/runpod-yolo providers and `imageEncode.ts` helpers
+ * (advance-photo-gallery-web-sdk/apps/web/src/lib/ai/*), collapsed into one
+ * module and retargeted from `/api/ai/*` to `/api/gallery/ai/*`.
+ *
+ * Capability split:
+ *  - object detection: TensorFlow.js COCO-SSD, fully in-browser (no key), or the
+ *    RunPod YOLO classifier via /api/gallery/ai/classify when
+ *    NEXT_PUBLIC_APG_RUNPOD_DETECT=true (COCO-SSD is the automatic fallback)
+ *  - face detection + recognition: face-api.js in-browser → clustered into People
+ *  - OCR: tesseract.js in-browser → searchable text + the Documents album
+ *  - semantic search: CLIP via transformers.js in-browser
+ *  - background removal: @imgly in-browser WASM, or the RunPod U²-Net endpoint
+ *    when NEXT_PUBLIC_APG_RUNPOD_BG=true (in-browser is the fallback)
+ *  - other generative edits / transcription / denoise / tilt: proxied through the
+ *    server routes so the RunPod key never reaches the browser
+ *
+ * EVERY heavy model is behind `await import(...)` so none of it lands in the
+ * initial bundle, and every capability degrades to []/''/null with a
+ * console.warn rather than throwing — a failed model must never break the
+ * gallery UI.
+ *
+ * The in-browser models fetch weights from public CDNs (jsdelivr, huggingface,
+ * storage.googleapis.com, staticimgly.com). See docs/SMART_GALLERY.md for the
+ * list that would need CSP allow-listing.
+ */
+
+import type { AIProvider, GenerativeEditOp, MediaItem } from "@photo-gallery/sdk";
+
+// Derived from the provider interface so we import only the three public types.
+type DetectedObject = Awaited<ReturnType<NonNullable<AIProvider["detectObjects"]>>>[number];
+type DetectedFace = Awaited<ReturnType<NonNullable<AIProvider["detectFaces"]>>>[number];
+type ImageSource = ImageBitmap | HTMLImageElement;
+
+const API = "/api/gallery/ai";
+
+// ---------------------------------------------------------------------------
+// imageEncode helpers (ported from apps/web/src/lib/ai/imageEncode.ts)
+// ---------------------------------------------------------------------------
+
+export interface EncodedImage {
+  /** base64 JPEG (no data: prefix). */
+  data: string;
+  mimeType: string;
+  /** Actual pixel dims of the encoded image (after downscale). */
+  width: number;
+  height: number;
+}
+
+/** Draw an image to a downscaled canvas and return base64 JPEG + its dims. */
+export function imageToBase64(image: ImageSource, maxDim: number): EncodedImage {
+  const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
+  const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
+  const scale = Math.min(1, maxDim / Math.max(w, h));
+  const cw = Math.max(1, Math.round(w * scale));
+  const ch = Math.max(1, Math.round(h * scale));
+  const canvas = document.createElement("canvas");
+  canvas.width = cw;
+  canvas.height = ch;
+  const ctx = canvas.getContext("2d");
+  if (!ctx) throw new Error("Canvas not supported.");
+  ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
+  const dataUrl = canvas.toDataURL("image/jpeg", 0.9);
+  return { data: dataUrl.split(",")[1] ?? "", mimeType: "image/jpeg", width: cw, height: ch };
+}
+
+/**
+ * Rasterize a mask (ImageData, white = region to regenerate) to a PNG base64
+ * scaled to targetW×targetH so it matches the encoded image exactly — SD 3.5
+ * requires image and mask to be identical pixel sizes.
+ */
+export function maskToBase64(mask: ImageData, targetW: number, targetH: number): string {
+  const tmp = document.createElement("canvas");
+  tmp.width = mask.width;
+  tmp.height = mask.height;
+  const tctx = tmp.getContext("2d");
+  if (!tctx) throw new Error("Canvas not supported.");
+  tctx.putImageData(mask, 0, 0);
+
+  const out = document.createElement("canvas");
+  out.width = targetW;
+  out.height = targetH;
+  const octx = out.getContext("2d");
+  if (!octx) throw new Error("Canvas not supported.");
+  // Nearest-neighbour, not bilinear — keep the mask strictly binary so SD gets
+  // crisp white(regenerate)/black(keep) edges instead of an anti-aliased grey halo.
+  octx.imageSmoothingEnabled = false;
+  octx.drawImage(tmp, 0, 0, targetW, targetH);
+  return out.toDataURL("image/png").split(",")[1] ?? "";
+}
+
+export function base64ToBlob(base64: string, mime: string): Blob {
+  const bin = atob(base64);
+  const bytes = new Uint8Array(bin.length);
+  for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+  return new Blob([bytes], { type: mime });
+}
+
+/**
+ * Pad an image with a neutral border for outpaint and return {imageBase64, maskBase64}
+ * as base64 PNG — the border is WHITE in the mask (regenerate), the original image
+ * area BLACK (keep). Capped at 1280px on the long side.
+ */
+export function padForOutpaint(
+  image: ImageSource,
+  factor: number,
+): { imageBase64: string; maskBase64: string } {
+  const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
+  const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
+  const f = Math.max(1.1, Math.min(2, factor));
+  const maxDim = 1280;
+  let pw = Math.round(w * f);
+  let ph = Math.round(h * f);
+  const scale = Math.min(1, maxDim / Math.max(pw, ph));
+  pw = Math.max(16, Math.round(pw * scale));
+  ph = Math.max(16, Math.round(ph * scale));
+  const iw = Math.max(1, Math.round(w * scale));
+  const ih = Math.max(1, Math.round(h * scale));
+  const ox = Math.floor((pw - iw) / 2);
+  const oy = Math.floor((ph - ih) / 2);
+
+  const imgCanvas = document.createElement("canvas");
+  imgCanvas.width = pw;
+  imgCanvas.height = ph;
+  const ictx = imgCanvas.getContext("2d");
+  if (!ictx) throw new Error("Canvas not supported.");
+  // Fill the new border with a blurred, stretched copy of the photo so the model
+  // has real color/context to continue from — flat gray gives it nothing.
+  ictx.filter = "blur(28px)";
+  ictx.drawImage(image as CanvasImageSource, 0, 0, pw, ph);
+  ictx.filter = "none";
+  ictx.drawImage(image as CanvasImageSource, ox, oy, iw, ih);
+
+  const maskCanvas = document.createElement("canvas");
+  maskCanvas.width = pw;
+  maskCanvas.height = ph;
+  const mctx = maskCanvas.getContext("2d");
+  if (!mctx) throw new Error("Canvas not supported.");
+  mctx.fillStyle = "#ffffff";
+  mctx.fillRect(0, 0, pw, ph);
+  mctx.fillStyle = "#000000";
+  mctx.fillRect(ox, oy, iw, ih);
+
+  return {
+    imageBase64: imgCanvas.toDataURL("image/png").split(",")[1] ?? "",
+    maskBase64: maskCanvas.toDataURL("image/png").split(",")[1] ?? "",
+  };
+}
+
+/** Draw an image to a canvas (downscaled) and return a JPEG Blob. */
+function canvasBlob(image: ImageSource, maxDim: number): Promise<Blob> {
+  const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
+  const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
+  const scale = Math.min(1, maxDim / Math.max(w, h));
+  const cw = Math.max(1, Math.round(w * scale));
+  const ch = Math.max(1, Math.round(h * scale));
+  const canvas = document.createElement("canvas");
+  canvas.width = cw;
+  canvas.height = ch;
+  const ctx = canvas.getContext("2d");
+  if (!ctx) return Promise.reject(new Error("Canvas not supported."));
+  ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
+  return new Promise((resolve, reject) =>
+    canvas.toBlob((b) => (b ? resolve(b) : reject(new Error("toBlob failed"))), "image/jpeg", 0.92),
+  );
+}
+
+function clamp01(n: number): number {
+  return n < 0 ? 0 : n > 1 ? 1 : n;
+}
+
+// ---------------------------------------------------------------------------
+// Object detection — TensorFlow.js COCO-SSD, in-browser
+// ---------------------------------------------------------------------------
+
+interface CocoPrediction {
+  bbox: [number, number, number, number];
+  class: string;
+  score: number;
+}
+interface CocoModel {
+  detect(
+    img: HTMLImageElement | HTMLCanvasElement | HTMLVideoElement,
+    maxNumBoxes?: number,
+    minScore?: number,
+  ): Promise<CocoPrediction[]>;
+}
+
+let cocoPromise: Promise<CocoModel | null> | null = null;
+
+/** Load tfjs + COCO-SSD exactly once; resolves to null if anything fails. */
+function ensureCoco(): Promise<CocoModel | null> {
+  cocoPromise ??= (async () => {
+    try {
+      const tf = await import("@tensorflow/tfjs");
+      try {
+        await tf.setBackend("webgl");
+      } catch {
+        // Fall back to the default backend if WebGL is unavailable.
+      }
+      await tf.ready();
+      const cocoSsd = await import("@tensorflow-models/coco-ssd");
+      return (await cocoSsd.load({ base: "lite_mobilenet_v2" })) as unknown as CocoModel;
+    } catch (err) {
+      console.warn("[gallery-ai] COCO-SSD load failed; object detection disabled.", err);
+      return null;
+    }
+  })();
+  return cocoPromise;
+}
+
+async function detectObjectsInBrowser(
+  item: MediaItem,
+  image: ImageSource,
+): Promise<DetectedObject[]> {
+  const model = await ensureCoco();
+  if (!model) return [];
+  try {
+    const el = image as HTMLImageElement;
+    const w = el.naturalWidth || el.width || item.width || 1;
+    const h = el.naturalHeight || el.height || item.height || 1;
+    const predictions = await model.detect(el, 20, 0.4);
+    return predictions.map((p) => ({
+      label: p.class,
+      confidence: p.score,
+      box: { x: p.bbox[0] / w, y: p.bbox[1] / h, width: p.bbox[2] / w, height: p.bbox[3] / h },
+    })) as DetectedObject[];
+  } catch (err) {
+    console.warn("[gallery-ai] object detection failed.", err);
+    return [];
+  }
+}
+
+/**
+ * Server-side YOLO detection via /api/gallery/ai/classify, with the in-browser
+ * COCO-SSD as the automatic fallback so detection never hard-fails.
+ */
+async function detectObjectsViaRunpod(
+  item: MediaItem,
+  image: ImageSource,
+): Promise<DetectedObject[]> {
+  try {
+    const { data, mimeType, width, height } = imageToBase64(image, 1280);
+    const res = await fetch(`${API}/classify`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ imageBase64: data, mimeType, width, height }),
+    });
+    if (!res.ok) throw new Error(`classify failed (${res.status})`);
+    const { objects } = (await res.json()) as { objects?: DetectedObject[] };
+    if (Array.isArray(objects)) return objects;
+    throw new Error("classify returned no objects");
+  } catch (err) {
+    console.warn("[gallery-ai] RunPod detection failed; falling back to COCO-SSD.", err);
+    return detectObjectsInBrowser(item, image);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Faces — @vladmandic/face-api, in-browser (128-D descriptors → People)
+// ---------------------------------------------------------------------------
+
+const FACE_MODEL_URL = "https://cdn.jsdelivr.net/npm/@vladmandic/face-api@1.7.15/model";
+
+type FaceApi = typeof import("@vladmandic/face-api");
+
+let facePromise: Promise<FaceApi | null> | null = null;
+
+function ensureFaceModels(): Promise<FaceApi | null> {
+  facePromise ??= (async () => {
+    try {
+      const faceapi = await import("@vladmandic/face-api");
+      // The bundled tf re-export is typed narrowly; backend control lives on the
+      // runtime object. Prefer WebGL (no eval; CSP-friendly), fall back gracefully.
+      const tf = faceapi.tf as unknown as {
+        setBackend: (b: string) => Promise<boolean>;
+        ready: () => Promise<void>;
+      };
+      try {
+        await tf.setBackend("webgl");
+      } catch {
+        /* keep default backend */
+      }
+      await tf.ready();
+      await Promise.all([
+        faceapi.nets.tinyFaceDetector.loadFromUri(FACE_MODEL_URL),
+        faceapi.nets.faceLandmark68Net.loadFromUri(FACE_MODEL_URL),
+        faceapi.nets.faceRecognitionNet.loadFromUri(FACE_MODEL_URL),
+      ]);
+      return faceapi;
+    } catch (err) {
+      // Degrade gracefully — People simply stays empty if models can't load.
+      console.warn("[gallery-ai] face model load failed; face clustering disabled.", err);
+      return null;
+    }
+  })();
+  return facePromise;
+}
+
+let faceWarned = false;
+
+async function detectFaces(item: MediaItem, image: ImageSource): Promise<DetectedFace[]> {
+  const faceapi = await ensureFaceModels();
+  if (!faceapi) return [];
+
+  const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width || 1;
+  const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height || 1;
+
+  type TNetInput = Parameters<FaceApi["detectAllFaces"]>[0];
+
+  let results;
+  try {
+    results = await faceapi
+      .detectAllFaces(
+        image as unknown as TNetInput,
+        new faceapi.TinyFaceDetectorOptions({ inputSize: 416, scoreThreshold: 0.5 }),
+      )
+      .withFaceLandmarks()
+      .withFaceDescriptors();
+  } catch (err) {
+    if (!faceWarned) {
+      faceWarned = true;
+      console.warn("[gallery-ai] face detection failed on", item.name, err);
+    }
+    return [];
+  }
+
+  return results.map((r) => {
+    const b = r.detection.box;
+    return {
+      confidence: r.detection.score,
+      box: {
+        x: clamp01(b.x / w),
+        y: clamp01(b.y / h),
+        width: clamp01(b.width / w),
+        height: clamp01(b.height / h),
+      },
+      embedding: Array.from(r.descriptor),
+    };
+  }) as DetectedFace[];
+}
+
+// ---------------------------------------------------------------------------
+// OCR — tesseract.js, in-browser
+// ---------------------------------------------------------------------------
+
+// Must equal the EXACT tesseract.js version in package.json (pinned, no caret)
+// so the worker CDN URL can never drift from the installed main-thread code.
+const TESSERACT_VERSION = "5.1.1";
+const WORKER_PATH = `https://cdn.jsdelivr.net/npm/tesseract.js@${TESSERACT_VERSION}/dist/worker.min.js`;
+const CORE_PATH = "https://cdn.jsdelivr.net/npm/tesseract.js-core@5";
+// jsDelivr's GitHub mirror of naptha/tessdata (same files as projectnaptha.com),
+// so every asset comes from ONE host that a CSP can allow-list.
+const LANG_PATH = "https://cdn.jsdelivr.net/gh/naptha/tessdata@gh-pages/4.0.0";
+
+interface OcrWord {
+  text?: string;
+  confidence?: number;
+}
+interface OcrData {
+  text?: string;
+  confidence?: number;
+  words?: OcrWord[];
+  blocks?: Array<{ paragraphs?: Array<{ lines?: Array<{ words?: OcrWord[] }> }> }> | null;
+}
+
+type TesseractWorker = import("tesseract.js").Worker;
+
+let ocrWorkerPromise: Promise<TesseractWorker | null> | null = null;
+
+function ensureOcrWorker(): Promise<TesseractWorker | null> {
+  ocrWorkerPromise ??= (async () => {
+    try {
+      const { createWorker } = await import("tesseract.js");
+      // v5: createWorker(langs, oem, options) already loads + initializes the
+      // language internally — do NOT call the removed v4 worker.load().
+      return await createWorker("eng", 1, {
+        workerPath: WORKER_PATH,
+        corePath: CORE_PATH,
+        langPath: LANG_PATH,
+      });
+    } catch (err) {
+      console.warn("[gallery-ai] tesseract worker init failed; OCR disabled.", err);
+      return null;
+    }
+  })();
+  return ocrWorkerPromise;
+}
+
+const WORD_CONFIDENCE = 70; // a word tesseract is actually sure about
+const MIN_WORDS = 4; // need several confident words to call it a document
+const MIN_CHARS = 10;
+
+function collectWords(data: OcrData): OcrWord[] {
+  if (Array.isArray(data.words) && data.words.length) return data.words;
+  const out: OcrWord[] = [];
+  for (const b of data.blocks ?? [])
+    for (const p of b.paragraphs ?? [])
+      for (const l of p.lines ?? []) for (const w of l.words ?? []) out.push(w);
+  return out;
+}
+
+/**
+ * Return real text or '' (not a document). tesseract hallucinates low-confidence
+ * gibberish for photos with no text, so we keep only high-confidence, word-shaped
+ * tokens and require several of them.
+ */
+function meaningfulText(data: OcrData): string {
+  const words = collectWords(data);
+  if (words.length > 0) {
+    const good = words.filter(
+      (w) => (w.confidence ?? 0) >= WORD_CONFIDENCE && /[A-Za-z0-9]{2,}/.test(w.text ?? ""),
+    );
+    const text = good
+      .map((w) => (w.text ?? "").trim())
+      .filter(Boolean)
+      .join(" ")
+      .trim();
+    return good.length >= MIN_WORDS && text.length >= MIN_CHARS ? text : "";
+  }
+  // Fallback: overall confidence + count of word-shaped tokens.
+  const raw = (data.text ?? "").trim();
+  const conf = typeof data.confidence === "number" ? data.confidence : 0;
+  const realWords = raw.match(/[A-Za-z]{3,}/g) ?? [];
+  return conf >= 72 && realWords.length >= 6 ? raw : "";
+}
+
+async function ocr(_item: MediaItem, image: ImageSource): Promise<string> {
+  const worker = await ensureOcrWorker();
+  if (!worker) return "";
+  try {
+    // Request the block hierarchy so per-word confidence is available.
+    const { data } = (await worker.recognize(
+      image as unknown as HTMLImageElement,
+      {},
+      { text: true, blocks: true },
+    )) as { data: OcrData };
+    return meaningfulText(data);
+  } catch (err) {
+    console.warn("[gallery-ai] OCR failed.", err);
+    return "";
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Semantic search — CLIP via transformers.js (ONNX-WASM), in-browser
+// ---------------------------------------------------------------------------
+
+const CLIP_MODEL_ID = "Xenova/clip-vit-base-patch16";
+
+type Transformers = typeof import("@huggingface/transformers");
+
+let transformersMod: Transformers | null = null;
+/* eslint-disable @typescript-eslint/no-explicit-any -- transformers.js pipelines are untyped */
+let clipVisionPromise: Promise<{ processor: any; model: any } | null> | null = null;
+let clipTextPromise: Promise<{ tokenizer: any; model: any } | null> | null = null;
+
+async function loadTransformers(): Promise<Transformers> {
+  if (!transformersMod) {
+    transformersMod = await import("@huggingface/transformers");
+    // Remote-only (models from the HF CDN); rely on browser cache between sessions.
+    transformersMod.env.allowLocalModels = false;
+  }
+  return transformersMod;
+}
+
+function ensureClipVision() {
+  clipVisionPromise ??= (async () => {
+    try {
+      const tf = await loadTransformers();
+      const [processor, model] = await Promise.all([
+        tf.AutoProcessor.from_pretrained(CLIP_MODEL_ID),
+        tf.CLIPVisionModelWithProjection.from_pretrained(CLIP_MODEL_ID),
+      ]);
+      return { processor, model };
+    } catch (err) {
+      console.warn("[gallery-ai] CLIP vision load failed; semantic search disabled.", err);
+      return null;
+    }
+  })();
+  return clipVisionPromise;
+}
+
+function ensureClipText() {
+  clipTextPromise ??= (async () => {
+    try {
+      const tf = await loadTransformers();
+      const [tokenizer, model] = await Promise.all([
+        tf.AutoTokenizer.from_pretrained(CLIP_MODEL_ID),
+        tf.CLIPTextModelWithProjection.from_pretrained(CLIP_MODEL_ID),
+      ]);
+      return { tokenizer, model };
+    } catch (err) {
+      console.warn("[gallery-ai] CLIP text load failed; semantic search disabled.", err);
+      return null;
+    }
+  })();
+  return clipTextPromise;
+}
+
+/** Draw an image onto a canvas (downscaled) for the CLIP image processor. */
+function toCanvas(image: ImageSource, maxDim = 384): HTMLCanvasElement {
+  const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
+  const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
+  const scale = Math.min(1, maxDim / Math.max(w, h));
+  const canvas = document.createElement("canvas");
+  canvas.width = Math.max(1, Math.round(w * scale));
+  canvas.height = Math.max(1, Math.round(h * scale));
+  const ctx = canvas.getContext("2d");
+  if (!ctx) throw new Error("Canvas not supported.");
+  ctx.drawImage(image as CanvasImageSource, 0, 0, canvas.width, canvas.height);
+  return canvas;
+}
+
+function tensorToArray(t: any): number[] {
+  const data: Float32Array = t?.data ?? t;
+  return Array.from(data as ArrayLike<number>);
+}
+
+async function embedImage(_item: MediaItem, image: ImageSource): Promise<number[]> {
+  const v = await ensureClipVision();
+  if (!v) return [];
+  try {
+    const tf = await loadTransformers();
+    const canvas = toCanvas(image);
+    const ctx = canvas.getContext("2d");
+    if (!ctx) return [];
+    const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+    const raw = new tf.RawImage(imageData.data, canvas.width, canvas.height, 4).rgb();
+    const inputs = await v.processor(raw);
+    const out = await v.model(inputs);
+    return tensorToArray(out.image_embeds);
+  } catch (err) {
+    console.warn("[gallery-ai] embedImage failed.", err);
+    return [];
+  }
+}
+
+async function embedText(query: string): Promise<number[]> {
+  const t = await ensureClipText();
+  if (!t) return [];
+  try {
+    const inputs = t.tokenizer([query], { padding: true, truncation: true });
+    const out = await t.model(inputs);
+    return tensorToArray(out.text_embeds);
+  } catch (err) {
+    console.warn("[gallery-ai] embedText failed.", err);
+    return [];
+  }
+}
+/* eslint-enable @typescript-eslint/no-explicit-any */
+
+// ---------------------------------------------------------------------------
+// The provider
+// ---------------------------------------------------------------------------
+
+/** `true` only for the literal string "true", matching the SDK demo's semantics. */
+function flag(v: string | undefined): boolean {
+  return v === "true";
+}
+
+export function createCrmAIProvider(): AIProvider {
+  // NEXT_PUBLIC_* are inlined at build time, so these must be read as full
+  // static member expressions — do NOT refactor to dynamic indexing.
+  const useRunpodDetect = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_DETECT);
+  const useRunpodBg = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_BG);
+  const useRunpodTilt = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_TILT);
+
+  return {
+    name: "crm-ai (coco-ssd/yolo + face-api + tesseract + clip + runpod-edit)",
+
+    detectObjects: useRunpodDetect ? detectObjectsViaRunpod : detectObjectsInBrowser,
+    detectFaces,
+    ocr,
+    embedImage,
+    embedText,
+
+    async generativeEdit(item: MediaItem, image: ImageSource, op: GenerativeEditOp) {
+      // Remove Background runs fully in-browser (no key) via @imgly — works even
+      // with no backend. Other ops go through the /api/gallery/ai/edit route.
+      if (op.type === "remove-background") {
+        // Prefer the RunPod U²-Net endpoint when enabled; fall back to in-browser
+        // @imgly if it's off or the request fails, so this always produces a result.
+        if (useRunpodBg) {
+          try {
+            const { data, mimeType } = imageToBase64(image, 1600);
+            const res = await fetch(`${API}/edit`, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: JSON.stringify({
+                imageBase64: data,
+                mimeType,
+                op: { type: "remove-background" },
+                params: {},
+              }),
+            });
+            if (res.ok) {
+              const { imageBase64: out, mimeType: outMime } = (await res.json()) as {
+                imageBase64: string;
+                mimeType?: string;
+              };
+              return base64ToBlob(out, outMime || "image/png");
+            }
+          } catch {
+            /* fall through to the in-browser remover */
+          }
+        }
+        const inputBlob = await canvasBlob(image, 1600);
+        const { removeBackground } = await import("@imgly/background-removal");
+        return removeBackground(inputBlob, { output: { format: "image/png" } });
+      }
+
+      // Outpaint / expand-canvas: pad the image with a neutral border, mark that
+      // border WHITE in the mask, and run it through the same inpaint path as
+      // generative-fill — no extra backend route needed.
+      if (op.type === "outpaint") {
+        const { imageBase64: padded, maskBase64: border } = padForOutpaint(
+          image,
+          typeof op.factor === "number" ? op.factor : 1.5,
+        );
+        const outParams: Record<string, unknown> = {
+          strength: typeof op.strength === "number" ? op.strength : 0.85,
+        };
+        const outRes = await fetch(`${API}/edit`, {
+          method: "POST",
+          headers: { "content-type": "application/json" },
+          body: JSON.stringify({
+            imageBase64: padded,
+            mimeType: "image/png",
+            op: {
+              type: "generative-fill",
+              prompt:
+                op.prompt ||
+                "Extend and continue the scene naturally, matching lighting, colors and perspective.",
+            },
+            maskBase64: border,
+            params: outParams,
+          }),
+        });
+        if (!outRes.ok) {
+          const err = (await outRes.json().catch(() => ({}))) as { error?: string };
+          throw new Error(err.error || `AI request failed (${outRes.status}).`);
+        }
+        const outJson = (await outRes.json()) as { imageBase64: string; mimeType?: string };
+        return base64ToBlob(outJson.imageBase64, outJson.mimeType || "image/png");
+      }
+
+      const { data, mimeType, width, height } = imageToBase64(image, 1280);
+      // Masked ops carry an ImageData mask — rasterize it to a PNG matched to the
+      // (downscaled) image dims, and strip it from the op since ImageData is not
+      // JSON-serializable.
+      const maskBase64 = "mask" in op ? maskToBase64(op.mask, width, height) : undefined;
+      const wireOp: Record<string, unknown> = { type: op.type };
+      if ("prompt" in op && typeof op.prompt === "string") wireOp.prompt = op.prompt;
+      if ("factor" in op && typeof op.factor === "number") wireOp.factor = op.factor;
+
+      // Forward the "edit strength" slider (0..1) so the backend can scale the edit.
+      const params: Record<string, unknown> = {};
+      if ("strength" in op && typeof op.strength === "number") params.strength = op.strength;
+
+      const res = await fetch(`${API}/edit`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ imageBase64: data, mimeType, op: wireOp, maskBase64, params }),
+      });
+      if (!res.ok) {
+        const err = (await res.json().catch(() => ({}))) as { error?: string };
+        throw new Error(err.error || `AI request failed (${res.status}).`);
+      }
+      const { imageBase64, mimeType: outMime } = (await res.json()) as {
+        imageBase64: string;
+        mimeType?: string;
+      };
+      return base64ToBlob(imageBase64, outMime || "image/png");
+    },
+
+    // Voice annotation: record → (optional denoise) → transcribe. Both proxy
+    // through server routes so the RunPod key stays server-side.
+    async transcribeAudio(audioBase64: string) {
+      const res = await fetch(`${API}/transcribe`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ audio: audioBase64 }),
+      });
+      if (!res.ok) {
+        const err = (await res.json().catch(() => ({}))) as { error?: string };
+        throw new Error(err.error || `Transcription failed (${res.status}).`);
+      }
+      const { transcript } = (await res.json()) as { transcript?: string };
+      return (transcript ?? "").trim();
+    },
+
+    async denoiseAudio(audioBase64: string) {
+      const res = await fetch(`${API}/denoise`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ audio: audioBase64 }),
+      });
+      if (!res.ok) {
+        const err = (await res.json().catch(() => ({}))) as { error?: string };
+        throw new Error(err.error || `Denoise failed (${res.status}).`);
+      }
+      const { audio } = (await res.json()) as { audio?: string };
+      return audio ?? audioBase64;
+    },
+
+    // Camera-tilt estimation is opt-in (needs the RunPod tilt endpoint deployed);
+    // gate it so the editor's Auto-straighten button only appears when configured.
+    estimateTilt: useRunpodTilt
+      ? async (_item: MediaItem, image: ImageSource) => {
+          const { data } = imageToBase64(image, 1024);
+          const res = await fetch(`${API}/tilt`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify({ image: data }),
+          });
+          if (!res.ok) {
+            const err = (await res.json().catch(() => ({}))) as { error?: string };
+            throw new Error(err.error || `Tilt estimate failed (${res.status}).`);
+          }
+          return (await res.json()) as {
+            rollDegrees: number;
+            pitchDegrees: number;
+            fovDegrees: number;
+          };
+        }
+      : undefined,
+  };
+}