forked from Goutam/lynkeduppro-crm
735 lines
28 KiB
TypeScript
735 lines
28 KiB
TypeScript
"use client";
|
||
|
||
/**
|
||
* The CRM's AIProvider for the Smart Gallery.
|
||
*
|
||
* PROVENANCE: a port of the SDK demo's `createDemoAIProvider()` plus its
|
||
* clip/face/ocr/tensorflow/runpod-yolo providers and `imageEncode.ts` helpers
|
||
* (advance-photo-gallery-web-sdk/apps/web/src/lib/ai/*), collapsed into one
|
||
* module and retargeted from `/api/ai/*` to `/api/gallery/ai/*`.
|
||
*
|
||
* Capability split:
|
||
* - object detection: TensorFlow.js COCO-SSD, fully in-browser (no key), or the
|
||
* RunPod YOLO classifier via /api/gallery/ai/classify when
|
||
* NEXT_PUBLIC_APG_RUNPOD_DETECT=true (COCO-SSD is the automatic fallback)
|
||
* - face detection + recognition: face-api.js in-browser → clustered into People
|
||
* - OCR: tesseract.js in-browser → searchable text + the Documents album
|
||
* - semantic search: CLIP via transformers.js in-browser
|
||
* - background removal: @imgly in-browser WASM, or the RunPod U²-Net endpoint
|
||
* when NEXT_PUBLIC_APG_RUNPOD_BG=true (in-browser is the fallback)
|
||
* - other generative edits / transcription / denoise / tilt: proxied through the
|
||
* server routes so the RunPod key never reaches the browser
|
||
*
|
||
* EVERY heavy model is behind `await import(...)` so none of it lands in the
|
||
* initial bundle, and every capability degrades to []/''/null with a
|
||
* console.warn rather than throwing — a failed model must never break the
|
||
* gallery UI.
|
||
*
|
||
* The in-browser models fetch weights from public CDNs (jsdelivr, huggingface,
|
||
* storage.googleapis.com, staticimgly.com). See docs/SMART_GALLERY.md for the
|
||
* list that would need CSP allow-listing.
|
||
*/
|
||
|
||
import type { AIProvider, GenerativeEditOp, MediaItem } from "@photo-gallery/sdk";
|
||
|
||
// Derived from the provider interface so we import only the three public types.
|
||
type DetectedObject = Awaited<ReturnType<NonNullable<AIProvider["detectObjects"]>>>[number];
|
||
type DetectedFace = Awaited<ReturnType<NonNullable<AIProvider["detectFaces"]>>>[number];
|
||
type ImageSource = ImageBitmap | HTMLImageElement;
|
||
|
||
const API = "/api/gallery/ai";
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// imageEncode helpers (ported from apps/web/src/lib/ai/imageEncode.ts)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
export interface EncodedImage {
|
||
/** base64 JPEG (no data: prefix). */
|
||
data: string;
|
||
mimeType: string;
|
||
/** Actual pixel dims of the encoded image (after downscale). */
|
||
width: number;
|
||
height: number;
|
||
}
|
||
|
||
/** Draw an image to a downscaled canvas and return base64 JPEG + its dims. */
|
||
export function imageToBase64(image: ImageSource, maxDim: number): EncodedImage {
|
||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||
const scale = Math.min(1, maxDim / Math.max(w, h));
|
||
const cw = Math.max(1, Math.round(w * scale));
|
||
const ch = Math.max(1, Math.round(h * scale));
|
||
const canvas = document.createElement("canvas");
|
||
canvas.width = cw;
|
||
canvas.height = ch;
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) throw new Error("Canvas not supported.");
|
||
ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
|
||
const dataUrl = canvas.toDataURL("image/jpeg", 0.9);
|
||
return { data: dataUrl.split(",")[1] ?? "", mimeType: "image/jpeg", width: cw, height: ch };
|
||
}
|
||
|
||
/**
|
||
* Rasterize a mask (ImageData, white = region to regenerate) to a PNG base64
|
||
* scaled to targetW×targetH so it matches the encoded image exactly — SD 3.5
|
||
* requires image and mask to be identical pixel sizes.
|
||
*/
|
||
export function maskToBase64(mask: ImageData, targetW: number, targetH: number): string {
|
||
const tmp = document.createElement("canvas");
|
||
tmp.width = mask.width;
|
||
tmp.height = mask.height;
|
||
const tctx = tmp.getContext("2d");
|
||
if (!tctx) throw new Error("Canvas not supported.");
|
||
tctx.putImageData(mask, 0, 0);
|
||
|
||
const out = document.createElement("canvas");
|
||
out.width = targetW;
|
||
out.height = targetH;
|
||
const octx = out.getContext("2d");
|
||
if (!octx) throw new Error("Canvas not supported.");
|
||
// Nearest-neighbour, not bilinear — keep the mask strictly binary so SD gets
|
||
// crisp white(regenerate)/black(keep) edges instead of an anti-aliased grey halo.
|
||
octx.imageSmoothingEnabled = false;
|
||
octx.drawImage(tmp, 0, 0, targetW, targetH);
|
||
return out.toDataURL("image/png").split(",")[1] ?? "";
|
||
}
|
||
|
||
export function base64ToBlob(base64: string, mime: string): Blob {
|
||
const bin = atob(base64);
|
||
const bytes = new Uint8Array(bin.length);
|
||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||
return new Blob([bytes], { type: mime });
|
||
}
|
||
|
||
/**
|
||
* Pad an image with a neutral border for outpaint and return {imageBase64, maskBase64}
|
||
* as base64 PNG — the border is WHITE in the mask (regenerate), the original image
|
||
* area BLACK (keep). Capped at 1280px on the long side.
|
||
*/
|
||
export function padForOutpaint(
|
||
image: ImageSource,
|
||
factor: number,
|
||
): { imageBase64: string; maskBase64: string } {
|
||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||
const f = Math.max(1.1, Math.min(2, factor));
|
||
const maxDim = 1280;
|
||
let pw = Math.round(w * f);
|
||
let ph = Math.round(h * f);
|
||
const scale = Math.min(1, maxDim / Math.max(pw, ph));
|
||
pw = Math.max(16, Math.round(pw * scale));
|
||
ph = Math.max(16, Math.round(ph * scale));
|
||
const iw = Math.max(1, Math.round(w * scale));
|
||
const ih = Math.max(1, Math.round(h * scale));
|
||
const ox = Math.floor((pw - iw) / 2);
|
||
const oy = Math.floor((ph - ih) / 2);
|
||
|
||
const imgCanvas = document.createElement("canvas");
|
||
imgCanvas.width = pw;
|
||
imgCanvas.height = ph;
|
||
const ictx = imgCanvas.getContext("2d");
|
||
if (!ictx) throw new Error("Canvas not supported.");
|
||
// Fill the new border with a blurred, stretched copy of the photo so the model
|
||
// has real color/context to continue from — flat gray gives it nothing.
|
||
ictx.filter = "blur(28px)";
|
||
ictx.drawImage(image as CanvasImageSource, 0, 0, pw, ph);
|
||
ictx.filter = "none";
|
||
ictx.drawImage(image as CanvasImageSource, ox, oy, iw, ih);
|
||
|
||
const maskCanvas = document.createElement("canvas");
|
||
maskCanvas.width = pw;
|
||
maskCanvas.height = ph;
|
||
const mctx = maskCanvas.getContext("2d");
|
||
if (!mctx) throw new Error("Canvas not supported.");
|
||
mctx.fillStyle = "#ffffff";
|
||
mctx.fillRect(0, 0, pw, ph);
|
||
mctx.fillStyle = "#000000";
|
||
mctx.fillRect(ox, oy, iw, ih);
|
||
|
||
return {
|
||
imageBase64: imgCanvas.toDataURL("image/png").split(",")[1] ?? "",
|
||
maskBase64: maskCanvas.toDataURL("image/png").split(",")[1] ?? "",
|
||
};
|
||
}
|
||
|
||
/** Draw an image to a canvas (downscaled) and return a JPEG Blob. */
|
||
function canvasBlob(image: ImageSource, maxDim: number): Promise<Blob> {
|
||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||
const scale = Math.min(1, maxDim / Math.max(w, h));
|
||
const cw = Math.max(1, Math.round(w * scale));
|
||
const ch = Math.max(1, Math.round(h * scale));
|
||
const canvas = document.createElement("canvas");
|
||
canvas.width = cw;
|
||
canvas.height = ch;
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) return Promise.reject(new Error("Canvas not supported."));
|
||
ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
|
||
return new Promise((resolve, reject) =>
|
||
canvas.toBlob((b) => (b ? resolve(b) : reject(new Error("toBlob failed"))), "image/jpeg", 0.92),
|
||
);
|
||
}
|
||
|
||
function clamp01(n: number): number {
|
||
return n < 0 ? 0 : n > 1 ? 1 : n;
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Object detection — TensorFlow.js COCO-SSD, in-browser
|
||
// ---------------------------------------------------------------------------
|
||
|
||
interface CocoPrediction {
|
||
bbox: [number, number, number, number];
|
||
class: string;
|
||
score: number;
|
||
}
|
||
interface CocoModel {
|
||
detect(
|
||
img: HTMLImageElement | HTMLCanvasElement | HTMLVideoElement,
|
||
maxNumBoxes?: number,
|
||
minScore?: number,
|
||
): Promise<CocoPrediction[]>;
|
||
}
|
||
|
||
let cocoPromise: Promise<CocoModel | null> | null = null;
|
||
|
||
/** Load tfjs + COCO-SSD exactly once; resolves to null if anything fails. */
|
||
function ensureCoco(): Promise<CocoModel | null> {
|
||
cocoPromise ??= (async () => {
|
||
try {
|
||
const tf = await import("@tensorflow/tfjs");
|
||
try {
|
||
await tf.setBackend("webgl");
|
||
} catch {
|
||
// Fall back to the default backend if WebGL is unavailable.
|
||
}
|
||
await tf.ready();
|
||
const cocoSsd = await import("@tensorflow-models/coco-ssd");
|
||
return (await cocoSsd.load({ base: "lite_mobilenet_v2" })) as unknown as CocoModel;
|
||
} catch (err) {
|
||
console.warn("[gallery-ai] COCO-SSD load failed; object detection disabled.", err);
|
||
return null;
|
||
}
|
||
})();
|
||
return cocoPromise;
|
||
}
|
||
|
||
async function detectObjectsInBrowser(
|
||
item: MediaItem,
|
||
image: ImageSource,
|
||
): Promise<DetectedObject[]> {
|
||
const model = await ensureCoco();
|
||
if (!model) return [];
|
||
try {
|
||
const el = image as HTMLImageElement;
|
||
const w = el.naturalWidth || el.width || item.width || 1;
|
||
const h = el.naturalHeight || el.height || item.height || 1;
|
||
const predictions = await model.detect(el, 20, 0.4);
|
||
return predictions.map((p) => ({
|
||
label: p.class,
|
||
confidence: p.score,
|
||
box: { x: p.bbox[0] / w, y: p.bbox[1] / h, width: p.bbox[2] / w, height: p.bbox[3] / h },
|
||
})) as DetectedObject[];
|
||
} catch (err) {
|
||
console.warn("[gallery-ai] object detection failed.", err);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Server-side YOLO detection via /api/gallery/ai/classify, with the in-browser
|
||
* COCO-SSD as the automatic fallback so detection never hard-fails.
|
||
*/
|
||
async function detectObjectsViaRunpod(
|
||
item: MediaItem,
|
||
image: ImageSource,
|
||
): Promise<DetectedObject[]> {
|
||
try {
|
||
const { data, mimeType, width, height } = imageToBase64(image, 1280);
|
||
const res = await fetch(`${API}/classify`, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify({ imageBase64: data, mimeType, width, height }),
|
||
});
|
||
if (!res.ok) throw new Error(`classify failed (${res.status})`);
|
||
const { objects } = (await res.json()) as { objects?: DetectedObject[] };
|
||
if (Array.isArray(objects)) return objects;
|
||
throw new Error("classify returned no objects");
|
||
} catch (err) {
|
||
console.warn("[gallery-ai] RunPod detection failed; falling back to COCO-SSD.", err);
|
||
return detectObjectsInBrowser(item, image);
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Faces — @vladmandic/face-api, in-browser (128-D descriptors → People)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const FACE_MODEL_URL = "https://cdn.jsdelivr.net/npm/@vladmandic/face-api@1.7.15/model";
|
||
|
||
type FaceApi = typeof import("@vladmandic/face-api");
|
||
|
||
let facePromise: Promise<FaceApi | null> | null = null;
|
||
|
||
function ensureFaceModels(): Promise<FaceApi | null> {
|
||
facePromise ??= (async () => {
|
||
try {
|
||
const faceapi = await import("@vladmandic/face-api");
|
||
// The bundled tf re-export is typed narrowly; backend control lives on the
|
||
// runtime object. Prefer WebGL (no eval; CSP-friendly), fall back gracefully.
|
||
const tf = faceapi.tf as unknown as {
|
||
setBackend: (b: string) => Promise<boolean>;
|
||
ready: () => Promise<void>;
|
||
};
|
||
try {
|
||
await tf.setBackend("webgl");
|
||
} catch {
|
||
/* keep default backend */
|
||
}
|
||
await tf.ready();
|
||
await Promise.all([
|
||
faceapi.nets.tinyFaceDetector.loadFromUri(FACE_MODEL_URL),
|
||
faceapi.nets.faceLandmark68Net.loadFromUri(FACE_MODEL_URL),
|
||
faceapi.nets.faceRecognitionNet.loadFromUri(FACE_MODEL_URL),
|
||
]);
|
||
return faceapi;
|
||
} catch (err) {
|
||
// Degrade gracefully — People simply stays empty if models can't load.
|
||
console.warn("[gallery-ai] face model load failed; face clustering disabled.", err);
|
||
return null;
|
||
}
|
||
})();
|
||
return facePromise;
|
||
}
|
||
|
||
let faceWarned = false;
|
||
|
||
async function detectFaces(item: MediaItem, image: ImageSource): Promise<DetectedFace[]> {
|
||
const faceapi = await ensureFaceModels();
|
||
if (!faceapi) return [];
|
||
|
||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width || 1;
|
||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height || 1;
|
||
|
||
type TNetInput = Parameters<FaceApi["detectAllFaces"]>[0];
|
||
|
||
let results;
|
||
try {
|
||
results = await faceapi
|
||
.detectAllFaces(
|
||
image as unknown as TNetInput,
|
||
new faceapi.TinyFaceDetectorOptions({ inputSize: 416, scoreThreshold: 0.5 }),
|
||
)
|
||
.withFaceLandmarks()
|
||
.withFaceDescriptors();
|
||
} catch (err) {
|
||
if (!faceWarned) {
|
||
faceWarned = true;
|
||
console.warn("[gallery-ai] face detection failed on", item.name, err);
|
||
}
|
||
return [];
|
||
}
|
||
|
||
return results.map((r) => {
|
||
const b = r.detection.box;
|
||
return {
|
||
confidence: r.detection.score,
|
||
box: {
|
||
x: clamp01(b.x / w),
|
||
y: clamp01(b.y / h),
|
||
width: clamp01(b.width / w),
|
||
height: clamp01(b.height / h),
|
||
},
|
||
embedding: Array.from(r.descriptor),
|
||
};
|
||
}) as DetectedFace[];
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// OCR — tesseract.js, in-browser
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// Must equal the EXACT tesseract.js version in package.json (pinned, no caret)
|
||
// so the worker CDN URL can never drift from the installed main-thread code.
|
||
const TESSERACT_VERSION = "5.1.1";
|
||
const WORKER_PATH = `https://cdn.jsdelivr.net/npm/tesseract.js@${TESSERACT_VERSION}/dist/worker.min.js`;
|
||
const CORE_PATH = "https://cdn.jsdelivr.net/npm/tesseract.js-core@5";
|
||
// jsDelivr's GitHub mirror of naptha/tessdata (same files as projectnaptha.com),
|
||
// so every asset comes from ONE host that a CSP can allow-list.
|
||
const LANG_PATH = "https://cdn.jsdelivr.net/gh/naptha/tessdata@gh-pages/4.0.0";
|
||
|
||
interface OcrWord {
|
||
text?: string;
|
||
confidence?: number;
|
||
}
|
||
interface OcrData {
|
||
text?: string;
|
||
confidence?: number;
|
||
words?: OcrWord[];
|
||
blocks?: Array<{ paragraphs?: Array<{ lines?: Array<{ words?: OcrWord[] }> }> }> | null;
|
||
}
|
||
|
||
type TesseractWorker = import("tesseract.js").Worker;
|
||
|
||
let ocrWorkerPromise: Promise<TesseractWorker | null> | null = null;
|
||
|
||
function ensureOcrWorker(): Promise<TesseractWorker | null> {
|
||
ocrWorkerPromise ??= (async () => {
|
||
try {
|
||
const { createWorker } = await import("tesseract.js");
|
||
// v5: createWorker(langs, oem, options) already loads + initializes the
|
||
// language internally — do NOT call the removed v4 worker.load().
|
||
return await createWorker("eng", 1, {
|
||
workerPath: WORKER_PATH,
|
||
corePath: CORE_PATH,
|
||
langPath: LANG_PATH,
|
||
});
|
||
} catch (err) {
|
||
console.warn("[gallery-ai] tesseract worker init failed; OCR disabled.", err);
|
||
return null;
|
||
}
|
||
})();
|
||
return ocrWorkerPromise;
|
||
}
|
||
|
||
const WORD_CONFIDENCE = 70; // a word tesseract is actually sure about
|
||
const MIN_WORDS = 4; // need several confident words to call it a document
|
||
const MIN_CHARS = 10;
|
||
|
||
function collectWords(data: OcrData): OcrWord[] {
|
||
if (Array.isArray(data.words) && data.words.length) return data.words;
|
||
const out: OcrWord[] = [];
|
||
for (const b of data.blocks ?? [])
|
||
for (const p of b.paragraphs ?? [])
|
||
for (const l of p.lines ?? []) for (const w of l.words ?? []) out.push(w);
|
||
return out;
|
||
}
|
||
|
||
/**
|
||
* Return real text or '' (not a document). tesseract hallucinates low-confidence
|
||
* gibberish for photos with no text, so we keep only high-confidence, word-shaped
|
||
* tokens and require several of them.
|
||
*/
|
||
function meaningfulText(data: OcrData): string {
|
||
const words = collectWords(data);
|
||
if (words.length > 0) {
|
||
const good = words.filter(
|
||
(w) => (w.confidence ?? 0) >= WORD_CONFIDENCE && /[A-Za-z0-9]{2,}/.test(w.text ?? ""),
|
||
);
|
||
const text = good
|
||
.map((w) => (w.text ?? "").trim())
|
||
.filter(Boolean)
|
||
.join(" ")
|
||
.trim();
|
||
return good.length >= MIN_WORDS && text.length >= MIN_CHARS ? text : "";
|
||
}
|
||
// Fallback: overall confidence + count of word-shaped tokens.
|
||
const raw = (data.text ?? "").trim();
|
||
const conf = typeof data.confidence === "number" ? data.confidence : 0;
|
||
const realWords = raw.match(/[A-Za-z]{3,}/g) ?? [];
|
||
return conf >= 72 && realWords.length >= 6 ? raw : "";
|
||
}
|
||
|
||
async function ocr(_item: MediaItem, image: ImageSource): Promise<string> {
|
||
const worker = await ensureOcrWorker();
|
||
if (!worker) return "";
|
||
try {
|
||
// Request the block hierarchy so per-word confidence is available.
|
||
const { data } = (await worker.recognize(
|
||
image as unknown as HTMLImageElement,
|
||
{},
|
||
{ text: true, blocks: true },
|
||
)) as { data: OcrData };
|
||
return meaningfulText(data);
|
||
} catch (err) {
|
||
console.warn("[gallery-ai] OCR failed.", err);
|
||
return "";
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Semantic search — CLIP via transformers.js (ONNX-WASM), in-browser
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const CLIP_MODEL_ID = "Xenova/clip-vit-base-patch16";
|
||
|
||
type Transformers = typeof import("@huggingface/transformers");
|
||
|
||
let transformersMod: Transformers | null = null;
|
||
/* eslint-disable @typescript-eslint/no-explicit-any -- transformers.js pipelines are untyped */
|
||
let clipVisionPromise: Promise<{ processor: any; model: any } | null> | null = null;
|
||
let clipTextPromise: Promise<{ tokenizer: any; model: any } | null> | null = null;
|
||
|
||
async function loadTransformers(): Promise<Transformers> {
|
||
if (!transformersMod) {
|
||
transformersMod = await import("@huggingface/transformers");
|
||
// Remote-only (models from the HF CDN); rely on browser cache between sessions.
|
||
transformersMod.env.allowLocalModels = false;
|
||
}
|
||
return transformersMod;
|
||
}
|
||
|
||
function ensureClipVision() {
|
||
clipVisionPromise ??= (async () => {
|
||
try {
|
||
const tf = await loadTransformers();
|
||
const [processor, model] = await Promise.all([
|
||
tf.AutoProcessor.from_pretrained(CLIP_MODEL_ID),
|
||
tf.CLIPVisionModelWithProjection.from_pretrained(CLIP_MODEL_ID),
|
||
]);
|
||
return { processor, model };
|
||
} catch (err) {
|
||
console.warn("[gallery-ai] CLIP vision load failed; semantic search disabled.", err);
|
||
return null;
|
||
}
|
||
})();
|
||
return clipVisionPromise;
|
||
}
|
||
|
||
function ensureClipText() {
|
||
clipTextPromise ??= (async () => {
|
||
try {
|
||
const tf = await loadTransformers();
|
||
const [tokenizer, model] = await Promise.all([
|
||
tf.AutoTokenizer.from_pretrained(CLIP_MODEL_ID),
|
||
tf.CLIPTextModelWithProjection.from_pretrained(CLIP_MODEL_ID),
|
||
]);
|
||
return { tokenizer, model };
|
||
} catch (err) {
|
||
console.warn("[gallery-ai] CLIP text load failed; semantic search disabled.", err);
|
||
return null;
|
||
}
|
||
})();
|
||
return clipTextPromise;
|
||
}
|
||
|
||
/** Draw an image onto a canvas (downscaled) for the CLIP image processor. */
|
||
function toCanvas(image: ImageSource, maxDim = 384): HTMLCanvasElement {
|
||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||
const scale = Math.min(1, maxDim / Math.max(w, h));
|
||
const canvas = document.createElement("canvas");
|
||
canvas.width = Math.max(1, Math.round(w * scale));
|
||
canvas.height = Math.max(1, Math.round(h * scale));
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) throw new Error("Canvas not supported.");
|
||
ctx.drawImage(image as CanvasImageSource, 0, 0, canvas.width, canvas.height);
|
||
return canvas;
|
||
}
|
||
|
||
function tensorToArray(t: any): number[] {
|
||
const data: Float32Array = t?.data ?? t;
|
||
return Array.from(data as ArrayLike<number>);
|
||
}
|
||
|
||
async function embedImage(_item: MediaItem, image: ImageSource): Promise<number[]> {
|
||
const v = await ensureClipVision();
|
||
if (!v) return [];
|
||
try {
|
||
const tf = await loadTransformers();
|
||
const canvas = toCanvas(image);
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) return [];
|
||
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||
const raw = new tf.RawImage(imageData.data, canvas.width, canvas.height, 4).rgb();
|
||
const inputs = await v.processor(raw);
|
||
const out = await v.model(inputs);
|
||
return tensorToArray(out.image_embeds);
|
||
} catch (err) {
|
||
console.warn("[gallery-ai] embedImage failed.", err);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
async function embedText(query: string): Promise<number[]> {
|
||
const t = await ensureClipText();
|
||
if (!t) return [];
|
||
try {
|
||
const inputs = t.tokenizer([query], { padding: true, truncation: true });
|
||
const out = await t.model(inputs);
|
||
return tensorToArray(out.text_embeds);
|
||
} catch (err) {
|
||
console.warn("[gallery-ai] embedText failed.", err);
|
||
return [];
|
||
}
|
||
}
|
||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// The provider
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/** `true` only for the literal string "true", matching the SDK demo's semantics. */
|
||
function flag(v: string | undefined): boolean {
|
||
return v === "true";
|
||
}
|
||
|
||
export function createCrmAIProvider(): AIProvider {
|
||
// NEXT_PUBLIC_* are inlined at build time, so these must be read as full
|
||
// static member expressions — do NOT refactor to dynamic indexing.
|
||
const useRunpodDetect = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_DETECT);
|
||
const useRunpodBg = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_BG);
|
||
const useRunpodTilt = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_TILT);
|
||
|
||
return {
|
||
name: "crm-ai (coco-ssd/yolo + face-api + tesseract + clip + runpod-edit)",
|
||
|
||
detectObjects: useRunpodDetect ? detectObjectsViaRunpod : detectObjectsInBrowser,
|
||
detectFaces,
|
||
ocr,
|
||
embedImage,
|
||
embedText,
|
||
|
||
async generativeEdit(item: MediaItem, image: ImageSource, op: GenerativeEditOp) {
|
||
// Remove Background runs fully in-browser (no key) via @imgly — works even
|
||
// with no backend. Other ops go through the /api/gallery/ai/edit route.
|
||
if (op.type === "remove-background") {
|
||
// Prefer the RunPod U²-Net endpoint when enabled; fall back to in-browser
|
||
// @imgly if it's off or the request fails, so this always produces a result.
|
||
if (useRunpodBg) {
|
||
try {
|
||
const { data, mimeType } = imageToBase64(image, 1600);
|
||
const res = await fetch(`${API}/edit`, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify({
|
||
imageBase64: data,
|
||
mimeType,
|
||
op: { type: "remove-background" },
|
||
params: {},
|
||
}),
|
||
});
|
||
if (res.ok) {
|
||
const { imageBase64: out, mimeType: outMime } = (await res.json()) as {
|
||
imageBase64: string;
|
||
mimeType?: string;
|
||
};
|
||
return base64ToBlob(out, outMime || "image/png");
|
||
}
|
||
} catch {
|
||
/* fall through to the in-browser remover */
|
||
}
|
||
}
|
||
const inputBlob = await canvasBlob(image, 1600);
|
||
const { removeBackground } = await import("@imgly/background-removal");
|
||
return removeBackground(inputBlob, { output: { format: "image/png" } });
|
||
}
|
||
|
||
// Outpaint / expand-canvas: pad the image with a neutral border, mark that
|
||
// border WHITE in the mask, and run it through the same inpaint path as
|
||
// generative-fill — no extra backend route needed.
|
||
if (op.type === "outpaint") {
|
||
const { imageBase64: padded, maskBase64: border } = padForOutpaint(
|
||
image,
|
||
typeof op.factor === "number" ? op.factor : 1.5,
|
||
);
|
||
const outParams: Record<string, unknown> = {
|
||
strength: typeof op.strength === "number" ? op.strength : 0.85,
|
||
};
|
||
const outRes = await fetch(`${API}/edit`, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify({
|
||
imageBase64: padded,
|
||
mimeType: "image/png",
|
||
op: {
|
||
type: "generative-fill",
|
||
prompt:
|
||
op.prompt ||
|
||
"Extend and continue the scene naturally, matching lighting, colors and perspective.",
|
||
},
|
||
maskBase64: border,
|
||
params: outParams,
|
||
}),
|
||
});
|
||
if (!outRes.ok) {
|
||
const err = (await outRes.json().catch(() => ({}))) as { error?: string };
|
||
throw new Error(err.error || `AI request failed (${outRes.status}).`);
|
||
}
|
||
const outJson = (await outRes.json()) as { imageBase64: string; mimeType?: string };
|
||
return base64ToBlob(outJson.imageBase64, outJson.mimeType || "image/png");
|
||
}
|
||
|
||
const { data, mimeType, width, height } = imageToBase64(image, 1280);
|
||
// Masked ops carry an ImageData mask — rasterize it to a PNG matched to the
|
||
// (downscaled) image dims, and strip it from the op since ImageData is not
|
||
// JSON-serializable.
|
||
const maskBase64 = "mask" in op ? maskToBase64(op.mask, width, height) : undefined;
|
||
const wireOp: Record<string, unknown> = { type: op.type };
|
||
if ("prompt" in op && typeof op.prompt === "string") wireOp.prompt = op.prompt;
|
||
if ("factor" in op && typeof op.factor === "number") wireOp.factor = op.factor;
|
||
|
||
// Forward the "edit strength" slider (0..1) so the backend can scale the edit.
|
||
const params: Record<string, unknown> = {};
|
||
if ("strength" in op && typeof op.strength === "number") params.strength = op.strength;
|
||
|
||
const res = await fetch(`${API}/edit`, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify({ imageBase64: data, mimeType, op: wireOp, maskBase64, params }),
|
||
});
|
||
if (!res.ok) {
|
||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||
throw new Error(err.error || `AI request failed (${res.status}).`);
|
||
}
|
||
const { imageBase64, mimeType: outMime } = (await res.json()) as {
|
||
imageBase64: string;
|
||
mimeType?: string;
|
||
};
|
||
return base64ToBlob(imageBase64, outMime || "image/png");
|
||
},
|
||
|
||
// Voice annotation: record → (optional denoise) → transcribe. Both proxy
|
||
// through server routes so the RunPod key stays server-side.
|
||
async transcribeAudio(audioBase64: string) {
|
||
const res = await fetch(`${API}/transcribe`, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify({ audio: audioBase64 }),
|
||
});
|
||
if (!res.ok) {
|
||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||
throw new Error(err.error || `Transcription failed (${res.status}).`);
|
||
}
|
||
const { transcript } = (await res.json()) as { transcript?: string };
|
||
return (transcript ?? "").trim();
|
||
},
|
||
|
||
async denoiseAudio(audioBase64: string) {
|
||
const res = await fetch(`${API}/denoise`, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify({ audio: audioBase64 }),
|
||
});
|
||
if (!res.ok) {
|
||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||
throw new Error(err.error || `Denoise failed (${res.status}).`);
|
||
}
|
||
const { audio } = (await res.json()) as { audio?: string };
|
||
return audio ?? audioBase64;
|
||
},
|
||
|
||
// Camera-tilt estimation is opt-in (needs the RunPod tilt endpoint deployed);
|
||
// gate it so the editor's Auto-straighten button only appears when configured.
|
||
estimateTilt: useRunpodTilt
|
||
? async (_item: MediaItem, image: ImageSource) => {
|
||
const { data } = imageToBase64(image, 1024);
|
||
const res = await fetch(`${API}/tilt`, {
|
||
method: "POST",
|
||
headers: { "content-type": "application/json" },
|
||
body: JSON.stringify({ image: data }),
|
||
});
|
||
if (!res.ok) {
|
||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||
throw new Error(err.error || `Tilt estimate failed (${res.status}).`);
|
||
}
|
||
return (await res.json()) as {
|
||
rollDegrees: number;
|
||
pitchDegrees: number;
|
||
fovDegrees: number;
|
||
};
|
||
}
|
||
: undefined,
|
||
};
|
||
}
|