feat: add core domain types for Photo Gallery SDK including media items, albums, and annotations
This commit is contained in:
@@ -0,0 +1,734 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* The CRM's AIProvider for the Smart Gallery.
|
||||
*
|
||||
* PROVENANCE: a port of the SDK demo's `createDemoAIProvider()` plus its
|
||||
* clip/face/ocr/tensorflow/runpod-yolo providers and `imageEncode.ts` helpers
|
||||
* (advance-photo-gallery-web-sdk/apps/web/src/lib/ai/*), collapsed into one
|
||||
* module and retargeted from `/api/ai/*` to `/api/gallery/ai/*`.
|
||||
*
|
||||
* Capability split:
|
||||
* - object detection: TensorFlow.js COCO-SSD, fully in-browser (no key), or the
|
||||
* RunPod YOLO classifier via /api/gallery/ai/classify when
|
||||
* NEXT_PUBLIC_APG_RUNPOD_DETECT=true (COCO-SSD is the automatic fallback)
|
||||
* - face detection + recognition: face-api.js in-browser → clustered into People
|
||||
* - OCR: tesseract.js in-browser → searchable text + the Documents album
|
||||
* - semantic search: CLIP via transformers.js in-browser
|
||||
* - background removal: @imgly in-browser WASM, or the RunPod U²-Net endpoint
|
||||
* when NEXT_PUBLIC_APG_RUNPOD_BG=true (in-browser is the fallback)
|
||||
* - other generative edits / transcription / denoise / tilt: proxied through the
|
||||
* server routes so the RunPod key never reaches the browser
|
||||
*
|
||||
* EVERY heavy model is behind `await import(...)` so none of it lands in the
|
||||
* initial bundle, and every capability degrades to []/''/null with a
|
||||
* console.warn rather than throwing — a failed model must never break the
|
||||
* gallery UI.
|
||||
*
|
||||
* The in-browser models fetch weights from public CDNs (jsdelivr, huggingface,
|
||||
* storage.googleapis.com, staticimgly.com). See docs/SMART_GALLERY.md for the
|
||||
* list that would need CSP allow-listing.
|
||||
*/
|
||||
|
||||
import type { AIProvider, GenerativeEditOp, MediaItem } from "@photo-gallery/sdk";
|
||||
|
||||
// Derived from the provider interface so we import only the three public types.
|
||||
type DetectedObject = Awaited<ReturnType<NonNullable<AIProvider["detectObjects"]>>>[number];
|
||||
type DetectedFace = Awaited<ReturnType<NonNullable<AIProvider["detectFaces"]>>>[number];
|
||||
type ImageSource = ImageBitmap | HTMLImageElement;
|
||||
|
||||
const API = "/api/gallery/ai";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// imageEncode helpers (ported from apps/web/src/lib/ai/imageEncode.ts)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface EncodedImage {
|
||||
/** base64 JPEG (no data: prefix). */
|
||||
data: string;
|
||||
mimeType: string;
|
||||
/** Actual pixel dims of the encoded image (after downscale). */
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/** Draw an image to a downscaled canvas and return base64 JPEG + its dims. */
|
||||
export function imageToBase64(image: ImageSource, maxDim: number): EncodedImage {
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||||
const scale = Math.min(1, maxDim / Math.max(w, h));
|
||||
const cw = Math.max(1, Math.round(w * scale));
|
||||
const ch = Math.max(1, Math.round(h * scale));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = cw;
|
||||
canvas.height = ch;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) throw new Error("Canvas not supported.");
|
||||
ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
|
||||
const dataUrl = canvas.toDataURL("image/jpeg", 0.9);
|
||||
return { data: dataUrl.split(",")[1] ?? "", mimeType: "image/jpeg", width: cw, height: ch };
|
||||
}
|
||||
|
||||
/**
|
||||
* Rasterize a mask (ImageData, white = region to regenerate) to a PNG base64
|
||||
* scaled to targetW×targetH so it matches the encoded image exactly — SD 3.5
|
||||
* requires image and mask to be identical pixel sizes.
|
||||
*/
|
||||
export function maskToBase64(mask: ImageData, targetW: number, targetH: number): string {
|
||||
const tmp = document.createElement("canvas");
|
||||
tmp.width = mask.width;
|
||||
tmp.height = mask.height;
|
||||
const tctx = tmp.getContext("2d");
|
||||
if (!tctx) throw new Error("Canvas not supported.");
|
||||
tctx.putImageData(mask, 0, 0);
|
||||
|
||||
const out = document.createElement("canvas");
|
||||
out.width = targetW;
|
||||
out.height = targetH;
|
||||
const octx = out.getContext("2d");
|
||||
if (!octx) throw new Error("Canvas not supported.");
|
||||
// Nearest-neighbour, not bilinear — keep the mask strictly binary so SD gets
|
||||
// crisp white(regenerate)/black(keep) edges instead of an anti-aliased grey halo.
|
||||
octx.imageSmoothingEnabled = false;
|
||||
octx.drawImage(tmp, 0, 0, targetW, targetH);
|
||||
return out.toDataURL("image/png").split(",")[1] ?? "";
|
||||
}
|
||||
|
||||
export function base64ToBlob(base64: string, mime: string): Blob {
|
||||
const bin = atob(base64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||
return new Blob([bytes], { type: mime });
|
||||
}
|
||||
|
||||
/**
|
||||
* Pad an image with a neutral border for outpaint and return {imageBase64, maskBase64}
|
||||
* as base64 PNG — the border is WHITE in the mask (regenerate), the original image
|
||||
* area BLACK (keep). Capped at 1280px on the long side.
|
||||
*/
|
||||
export function padForOutpaint(
|
||||
image: ImageSource,
|
||||
factor: number,
|
||||
): { imageBase64: string; maskBase64: string } {
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||||
const f = Math.max(1.1, Math.min(2, factor));
|
||||
const maxDim = 1280;
|
||||
let pw = Math.round(w * f);
|
||||
let ph = Math.round(h * f);
|
||||
const scale = Math.min(1, maxDim / Math.max(pw, ph));
|
||||
pw = Math.max(16, Math.round(pw * scale));
|
||||
ph = Math.max(16, Math.round(ph * scale));
|
||||
const iw = Math.max(1, Math.round(w * scale));
|
||||
const ih = Math.max(1, Math.round(h * scale));
|
||||
const ox = Math.floor((pw - iw) / 2);
|
||||
const oy = Math.floor((ph - ih) / 2);
|
||||
|
||||
const imgCanvas = document.createElement("canvas");
|
||||
imgCanvas.width = pw;
|
||||
imgCanvas.height = ph;
|
||||
const ictx = imgCanvas.getContext("2d");
|
||||
if (!ictx) throw new Error("Canvas not supported.");
|
||||
// Fill the new border with a blurred, stretched copy of the photo so the model
|
||||
// has real color/context to continue from — flat gray gives it nothing.
|
||||
ictx.filter = "blur(28px)";
|
||||
ictx.drawImage(image as CanvasImageSource, 0, 0, pw, ph);
|
||||
ictx.filter = "none";
|
||||
ictx.drawImage(image as CanvasImageSource, ox, oy, iw, ih);
|
||||
|
||||
const maskCanvas = document.createElement("canvas");
|
||||
maskCanvas.width = pw;
|
||||
maskCanvas.height = ph;
|
||||
const mctx = maskCanvas.getContext("2d");
|
||||
if (!mctx) throw new Error("Canvas not supported.");
|
||||
mctx.fillStyle = "#ffffff";
|
||||
mctx.fillRect(0, 0, pw, ph);
|
||||
mctx.fillStyle = "#000000";
|
||||
mctx.fillRect(ox, oy, iw, ih);
|
||||
|
||||
return {
|
||||
imageBase64: imgCanvas.toDataURL("image/png").split(",")[1] ?? "",
|
||||
maskBase64: maskCanvas.toDataURL("image/png").split(",")[1] ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
/** Draw an image to a canvas (downscaled) and return a JPEG Blob. */
|
||||
function canvasBlob(image: ImageSource, maxDim: number): Promise<Blob> {
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||||
const scale = Math.min(1, maxDim / Math.max(w, h));
|
||||
const cw = Math.max(1, Math.round(w * scale));
|
||||
const ch = Math.max(1, Math.round(h * scale));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = cw;
|
||||
canvas.height = ch;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return Promise.reject(new Error("Canvas not supported."));
|
||||
ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
|
||||
return new Promise((resolve, reject) =>
|
||||
canvas.toBlob((b) => (b ? resolve(b) : reject(new Error("toBlob failed"))), "image/jpeg", 0.92),
|
||||
);
|
||||
}
|
||||
|
||||
function clamp01(n: number): number {
|
||||
return n < 0 ? 0 : n > 1 ? 1 : n;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Object detection — TensorFlow.js COCO-SSD, in-browser
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface CocoPrediction {
|
||||
bbox: [number, number, number, number];
|
||||
class: string;
|
||||
score: number;
|
||||
}
|
||||
interface CocoModel {
|
||||
detect(
|
||||
img: HTMLImageElement | HTMLCanvasElement | HTMLVideoElement,
|
||||
maxNumBoxes?: number,
|
||||
minScore?: number,
|
||||
): Promise<CocoPrediction[]>;
|
||||
}
|
||||
|
||||
let cocoPromise: Promise<CocoModel | null> | null = null;
|
||||
|
||||
/** Load tfjs + COCO-SSD exactly once; resolves to null if anything fails. */
|
||||
function ensureCoco(): Promise<CocoModel | null> {
|
||||
cocoPromise ??= (async () => {
|
||||
try {
|
||||
const tf = await import("@tensorflow/tfjs");
|
||||
try {
|
||||
await tf.setBackend("webgl");
|
||||
} catch {
|
||||
// Fall back to the default backend if WebGL is unavailable.
|
||||
}
|
||||
await tf.ready();
|
||||
const cocoSsd = await import("@tensorflow-models/coco-ssd");
|
||||
return (await cocoSsd.load({ base: "lite_mobilenet_v2" })) as unknown as CocoModel;
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] COCO-SSD load failed; object detection disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return cocoPromise;
|
||||
}
|
||||
|
||||
async function detectObjectsInBrowser(
|
||||
item: MediaItem,
|
||||
image: ImageSource,
|
||||
): Promise<DetectedObject[]> {
|
||||
const model = await ensureCoco();
|
||||
if (!model) return [];
|
||||
try {
|
||||
const el = image as HTMLImageElement;
|
||||
const w = el.naturalWidth || el.width || item.width || 1;
|
||||
const h = el.naturalHeight || el.height || item.height || 1;
|
||||
const predictions = await model.detect(el, 20, 0.4);
|
||||
return predictions.map((p) => ({
|
||||
label: p.class,
|
||||
confidence: p.score,
|
||||
box: { x: p.bbox[0] / w, y: p.bbox[1] / h, width: p.bbox[2] / w, height: p.bbox[3] / h },
|
||||
})) as DetectedObject[];
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] object detection failed.", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-side YOLO detection via /api/gallery/ai/classify, with the in-browser
|
||||
* COCO-SSD as the automatic fallback so detection never hard-fails.
|
||||
*/
|
||||
async function detectObjectsViaRunpod(
|
||||
item: MediaItem,
|
||||
image: ImageSource,
|
||||
): Promise<DetectedObject[]> {
|
||||
try {
|
||||
const { data, mimeType, width, height } = imageToBase64(image, 1280);
|
||||
const res = await fetch(`${API}/classify`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ imageBase64: data, mimeType, width, height }),
|
||||
});
|
||||
if (!res.ok) throw new Error(`classify failed (${res.status})`);
|
||||
const { objects } = (await res.json()) as { objects?: DetectedObject[] };
|
||||
if (Array.isArray(objects)) return objects;
|
||||
throw new Error("classify returned no objects");
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] RunPod detection failed; falling back to COCO-SSD.", err);
|
||||
return detectObjectsInBrowser(item, image);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Faces — @vladmandic/face-api, in-browser (128-D descriptors → People)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const FACE_MODEL_URL = "https://cdn.jsdelivr.net/npm/@vladmandic/face-api@1.7.15/model";
|
||||
|
||||
type FaceApi = typeof import("@vladmandic/face-api");
|
||||
|
||||
let facePromise: Promise<FaceApi | null> | null = null;
|
||||
|
||||
function ensureFaceModels(): Promise<FaceApi | null> {
|
||||
facePromise ??= (async () => {
|
||||
try {
|
||||
const faceapi = await import("@vladmandic/face-api");
|
||||
// The bundled tf re-export is typed narrowly; backend control lives on the
|
||||
// runtime object. Prefer WebGL (no eval; CSP-friendly), fall back gracefully.
|
||||
const tf = faceapi.tf as unknown as {
|
||||
setBackend: (b: string) => Promise<boolean>;
|
||||
ready: () => Promise<void>;
|
||||
};
|
||||
try {
|
||||
await tf.setBackend("webgl");
|
||||
} catch {
|
||||
/* keep default backend */
|
||||
}
|
||||
await tf.ready();
|
||||
await Promise.all([
|
||||
faceapi.nets.tinyFaceDetector.loadFromUri(FACE_MODEL_URL),
|
||||
faceapi.nets.faceLandmark68Net.loadFromUri(FACE_MODEL_URL),
|
||||
faceapi.nets.faceRecognitionNet.loadFromUri(FACE_MODEL_URL),
|
||||
]);
|
||||
return faceapi;
|
||||
} catch (err) {
|
||||
// Degrade gracefully — People simply stays empty if models can't load.
|
||||
console.warn("[gallery-ai] face model load failed; face clustering disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return facePromise;
|
||||
}
|
||||
|
||||
let faceWarned = false;
|
||||
|
||||
async function detectFaces(item: MediaItem, image: ImageSource): Promise<DetectedFace[]> {
|
||||
const faceapi = await ensureFaceModels();
|
||||
if (!faceapi) return [];
|
||||
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width || 1;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height || 1;
|
||||
|
||||
type TNetInput = Parameters<FaceApi["detectAllFaces"]>[0];
|
||||
|
||||
let results;
|
||||
try {
|
||||
results = await faceapi
|
||||
.detectAllFaces(
|
||||
image as unknown as TNetInput,
|
||||
new faceapi.TinyFaceDetectorOptions({ inputSize: 416, scoreThreshold: 0.5 }),
|
||||
)
|
||||
.withFaceLandmarks()
|
||||
.withFaceDescriptors();
|
||||
} catch (err) {
|
||||
if (!faceWarned) {
|
||||
faceWarned = true;
|
||||
console.warn("[gallery-ai] face detection failed on", item.name, err);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
return results.map((r) => {
|
||||
const b = r.detection.box;
|
||||
return {
|
||||
confidence: r.detection.score,
|
||||
box: {
|
||||
x: clamp01(b.x / w),
|
||||
y: clamp01(b.y / h),
|
||||
width: clamp01(b.width / w),
|
||||
height: clamp01(b.height / h),
|
||||
},
|
||||
embedding: Array.from(r.descriptor),
|
||||
};
|
||||
}) as DetectedFace[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OCR — tesseract.js, in-browser
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Must equal the EXACT tesseract.js version in package.json (pinned, no caret)
|
||||
// so the worker CDN URL can never drift from the installed main-thread code.
|
||||
const TESSERACT_VERSION = "5.1.1";
|
||||
const WORKER_PATH = `https://cdn.jsdelivr.net/npm/tesseract.js@${TESSERACT_VERSION}/dist/worker.min.js`;
|
||||
const CORE_PATH = "https://cdn.jsdelivr.net/npm/tesseract.js-core@5";
|
||||
// jsDelivr's GitHub mirror of naptha/tessdata (same files as projectnaptha.com),
|
||||
// so every asset comes from ONE host that a CSP can allow-list.
|
||||
const LANG_PATH = "https://cdn.jsdelivr.net/gh/naptha/tessdata@gh-pages/4.0.0";
|
||||
|
||||
interface OcrWord {
|
||||
text?: string;
|
||||
confidence?: number;
|
||||
}
|
||||
interface OcrData {
|
||||
text?: string;
|
||||
confidence?: number;
|
||||
words?: OcrWord[];
|
||||
blocks?: Array<{ paragraphs?: Array<{ lines?: Array<{ words?: OcrWord[] }> }> }> | null;
|
||||
}
|
||||
|
||||
type TesseractWorker = import("tesseract.js").Worker;
|
||||
|
||||
let ocrWorkerPromise: Promise<TesseractWorker | null> | null = null;
|
||||
|
||||
function ensureOcrWorker(): Promise<TesseractWorker | null> {
|
||||
ocrWorkerPromise ??= (async () => {
|
||||
try {
|
||||
const { createWorker } = await import("tesseract.js");
|
||||
// v5: createWorker(langs, oem, options) already loads + initializes the
|
||||
// language internally — do NOT call the removed v4 worker.load().
|
||||
return await createWorker("eng", 1, {
|
||||
workerPath: WORKER_PATH,
|
||||
corePath: CORE_PATH,
|
||||
langPath: LANG_PATH,
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] tesseract worker init failed; OCR disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return ocrWorkerPromise;
|
||||
}
|
||||
|
||||
const WORD_CONFIDENCE = 70; // a word tesseract is actually sure about
|
||||
const MIN_WORDS = 4; // need several confident words to call it a document
|
||||
const MIN_CHARS = 10;
|
||||
|
||||
function collectWords(data: OcrData): OcrWord[] {
|
||||
if (Array.isArray(data.words) && data.words.length) return data.words;
|
||||
const out: OcrWord[] = [];
|
||||
for (const b of data.blocks ?? [])
|
||||
for (const p of b.paragraphs ?? [])
|
||||
for (const l of p.lines ?? []) for (const w of l.words ?? []) out.push(w);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return real text or '' (not a document). tesseract hallucinates low-confidence
|
||||
* gibberish for photos with no text, so we keep only high-confidence, word-shaped
|
||||
* tokens and require several of them.
|
||||
*/
|
||||
function meaningfulText(data: OcrData): string {
|
||||
const words = collectWords(data);
|
||||
if (words.length > 0) {
|
||||
const good = words.filter(
|
||||
(w) => (w.confidence ?? 0) >= WORD_CONFIDENCE && /[A-Za-z0-9]{2,}/.test(w.text ?? ""),
|
||||
);
|
||||
const text = good
|
||||
.map((w) => (w.text ?? "").trim())
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
.trim();
|
||||
return good.length >= MIN_WORDS && text.length >= MIN_CHARS ? text : "";
|
||||
}
|
||||
// Fallback: overall confidence + count of word-shaped tokens.
|
||||
const raw = (data.text ?? "").trim();
|
||||
const conf = typeof data.confidence === "number" ? data.confidence : 0;
|
||||
const realWords = raw.match(/[A-Za-z]{3,}/g) ?? [];
|
||||
return conf >= 72 && realWords.length >= 6 ? raw : "";
|
||||
}
|
||||
|
||||
async function ocr(_item: MediaItem, image: ImageSource): Promise<string> {
|
||||
const worker = await ensureOcrWorker();
|
||||
if (!worker) return "";
|
||||
try {
|
||||
// Request the block hierarchy so per-word confidence is available.
|
||||
const { data } = (await worker.recognize(
|
||||
image as unknown as HTMLImageElement,
|
||||
{},
|
||||
{ text: true, blocks: true },
|
||||
)) as { data: OcrData };
|
||||
return meaningfulText(data);
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] OCR failed.", err);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Semantic search — CLIP via transformers.js (ONNX-WASM), in-browser
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const CLIP_MODEL_ID = "Xenova/clip-vit-base-patch16";
|
||||
|
||||
type Transformers = typeof import("@huggingface/transformers");
|
||||
|
||||
let transformersMod: Transformers | null = null;
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any -- transformers.js pipelines are untyped */
|
||||
let clipVisionPromise: Promise<{ processor: any; model: any } | null> | null = null;
|
||||
let clipTextPromise: Promise<{ tokenizer: any; model: any } | null> | null = null;
|
||||
|
||||
async function loadTransformers(): Promise<Transformers> {
|
||||
if (!transformersMod) {
|
||||
transformersMod = await import("@huggingface/transformers");
|
||||
// Remote-only (models from the HF CDN); rely on browser cache between sessions.
|
||||
transformersMod.env.allowLocalModels = false;
|
||||
}
|
||||
return transformersMod;
|
||||
}
|
||||
|
||||
function ensureClipVision() {
|
||||
clipVisionPromise ??= (async () => {
|
||||
try {
|
||||
const tf = await loadTransformers();
|
||||
const [processor, model] = await Promise.all([
|
||||
tf.AutoProcessor.from_pretrained(CLIP_MODEL_ID),
|
||||
tf.CLIPVisionModelWithProjection.from_pretrained(CLIP_MODEL_ID),
|
||||
]);
|
||||
return { processor, model };
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] CLIP vision load failed; semantic search disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return clipVisionPromise;
|
||||
}
|
||||
|
||||
function ensureClipText() {
|
||||
clipTextPromise ??= (async () => {
|
||||
try {
|
||||
const tf = await loadTransformers();
|
||||
const [tokenizer, model] = await Promise.all([
|
||||
tf.AutoTokenizer.from_pretrained(CLIP_MODEL_ID),
|
||||
tf.CLIPTextModelWithProjection.from_pretrained(CLIP_MODEL_ID),
|
||||
]);
|
||||
return { tokenizer, model };
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] CLIP text load failed; semantic search disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return clipTextPromise;
|
||||
}
|
||||
|
||||
/** Draw an image onto a canvas (downscaled) for the CLIP image processor. */
|
||||
function toCanvas(image: ImageSource, maxDim = 384): HTMLCanvasElement {
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||||
const scale = Math.min(1, maxDim / Math.max(w, h));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(w * scale));
|
||||
canvas.height = Math.max(1, Math.round(h * scale));
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) throw new Error("Canvas not supported.");
|
||||
ctx.drawImage(image as CanvasImageSource, 0, 0, canvas.width, canvas.height);
|
||||
return canvas;
|
||||
}
|
||||
|
||||
function tensorToArray(t: any): number[] {
|
||||
const data: Float32Array = t?.data ?? t;
|
||||
return Array.from(data as ArrayLike<number>);
|
||||
}
|
||||
|
||||
async function embedImage(_item: MediaItem, image: ImageSource): Promise<number[]> {
|
||||
const v = await ensureClipVision();
|
||||
if (!v) return [];
|
||||
try {
|
||||
const tf = await loadTransformers();
|
||||
const canvas = toCanvas(image);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return [];
|
||||
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
const raw = new tf.RawImage(imageData.data, canvas.width, canvas.height, 4).rgb();
|
||||
const inputs = await v.processor(raw);
|
||||
const out = await v.model(inputs);
|
||||
return tensorToArray(out.image_embeds);
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] embedImage failed.", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function embedText(query: string): Promise<number[]> {
|
||||
const t = await ensureClipText();
|
||||
if (!t) return [];
|
||||
try {
|
||||
const inputs = t.tokenizer([query], { padding: true, truncation: true });
|
||||
const out = await t.model(inputs);
|
||||
return tensorToArray(out.text_embeds);
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] embedText failed.", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The provider
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** `true` only for the literal string "true", matching the SDK demo's semantics. */
|
||||
function flag(v: string | undefined): boolean {
|
||||
return v === "true";
|
||||
}
|
||||
|
||||
export function createCrmAIProvider(): AIProvider {
|
||||
// NEXT_PUBLIC_* are inlined at build time, so these must be read as full
|
||||
// static member expressions — do NOT refactor to dynamic indexing.
|
||||
const useRunpodDetect = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_DETECT);
|
||||
const useRunpodBg = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_BG);
|
||||
const useRunpodTilt = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_TILT);
|
||||
|
||||
return {
|
||||
name: "crm-ai (coco-ssd/yolo + face-api + tesseract + clip + runpod-edit)",
|
||||
|
||||
detectObjects: useRunpodDetect ? detectObjectsViaRunpod : detectObjectsInBrowser,
|
||||
detectFaces,
|
||||
ocr,
|
||||
embedImage,
|
||||
embedText,
|
||||
|
||||
async generativeEdit(item: MediaItem, image: ImageSource, op: GenerativeEditOp) {
|
||||
// Remove Background runs fully in-browser (no key) via @imgly — works even
|
||||
// with no backend. Other ops go through the /api/gallery/ai/edit route.
|
||||
if (op.type === "remove-background") {
|
||||
// Prefer the RunPod U²-Net endpoint when enabled; fall back to in-browser
|
||||
// @imgly if it's off or the request fails, so this always produces a result.
|
||||
if (useRunpodBg) {
|
||||
try {
|
||||
const { data, mimeType } = imageToBase64(image, 1600);
|
||||
const res = await fetch(`${API}/edit`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
imageBase64: data,
|
||||
mimeType,
|
||||
op: { type: "remove-background" },
|
||||
params: {},
|
||||
}),
|
||||
});
|
||||
if (res.ok) {
|
||||
const { imageBase64: out, mimeType: outMime } = (await res.json()) as {
|
||||
imageBase64: string;
|
||||
mimeType?: string;
|
||||
};
|
||||
return base64ToBlob(out, outMime || "image/png");
|
||||
}
|
||||
} catch {
|
||||
/* fall through to the in-browser remover */
|
||||
}
|
||||
}
|
||||
const inputBlob = await canvasBlob(image, 1600);
|
||||
const { removeBackground } = await import("@imgly/background-removal");
|
||||
return removeBackground(inputBlob, { output: { format: "image/png" } });
|
||||
}
|
||||
|
||||
// Outpaint / expand-canvas: pad the image with a neutral border, mark that
|
||||
// border WHITE in the mask, and run it through the same inpaint path as
|
||||
// generative-fill — no extra backend route needed.
|
||||
if (op.type === "outpaint") {
|
||||
const { imageBase64: padded, maskBase64: border } = padForOutpaint(
|
||||
image,
|
||||
typeof op.factor === "number" ? op.factor : 1.5,
|
||||
);
|
||||
const outParams: Record<string, unknown> = {
|
||||
strength: typeof op.strength === "number" ? op.strength : 0.85,
|
||||
};
|
||||
const outRes = await fetch(`${API}/edit`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
imageBase64: padded,
|
||||
mimeType: "image/png",
|
||||
op: {
|
||||
type: "generative-fill",
|
||||
prompt:
|
||||
op.prompt ||
|
||||
"Extend and continue the scene naturally, matching lighting, colors and perspective.",
|
||||
},
|
||||
maskBase64: border,
|
||||
params: outParams,
|
||||
}),
|
||||
});
|
||||
if (!outRes.ok) {
|
||||
const err = (await outRes.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `AI request failed (${outRes.status}).`);
|
||||
}
|
||||
const outJson = (await outRes.json()) as { imageBase64: string; mimeType?: string };
|
||||
return base64ToBlob(outJson.imageBase64, outJson.mimeType || "image/png");
|
||||
}
|
||||
|
||||
const { data, mimeType, width, height } = imageToBase64(image, 1280);
|
||||
// Masked ops carry an ImageData mask — rasterize it to a PNG matched to the
|
||||
// (downscaled) image dims, and strip it from the op since ImageData is not
|
||||
// JSON-serializable.
|
||||
const maskBase64 = "mask" in op ? maskToBase64(op.mask, width, height) : undefined;
|
||||
const wireOp: Record<string, unknown> = { type: op.type };
|
||||
if ("prompt" in op && typeof op.prompt === "string") wireOp.prompt = op.prompt;
|
||||
if ("factor" in op && typeof op.factor === "number") wireOp.factor = op.factor;
|
||||
|
||||
// Forward the "edit strength" slider (0..1) so the backend can scale the edit.
|
||||
const params: Record<string, unknown> = {};
|
||||
if ("strength" in op && typeof op.strength === "number") params.strength = op.strength;
|
||||
|
||||
const res = await fetch(`${API}/edit`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ imageBase64: data, mimeType, op: wireOp, maskBase64, params }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `AI request failed (${res.status}).`);
|
||||
}
|
||||
const { imageBase64, mimeType: outMime } = (await res.json()) as {
|
||||
imageBase64: string;
|
||||
mimeType?: string;
|
||||
};
|
||||
return base64ToBlob(imageBase64, outMime || "image/png");
|
||||
},
|
||||
|
||||
// Voice annotation: record → (optional denoise) → transcribe. Both proxy
|
||||
// through server routes so the RunPod key stays server-side.
|
||||
async transcribeAudio(audioBase64: string) {
|
||||
const res = await fetch(`${API}/transcribe`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ audio: audioBase64 }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `Transcription failed (${res.status}).`);
|
||||
}
|
||||
const { transcript } = (await res.json()) as { transcript?: string };
|
||||
return (transcript ?? "").trim();
|
||||
},
|
||||
|
||||
async denoiseAudio(audioBase64: string) {
|
||||
const res = await fetch(`${API}/denoise`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ audio: audioBase64 }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `Denoise failed (${res.status}).`);
|
||||
}
|
||||
const { audio } = (await res.json()) as { audio?: string };
|
||||
return audio ?? audioBase64;
|
||||
},
|
||||
|
||||
// Camera-tilt estimation is opt-in (needs the RunPod tilt endpoint deployed);
|
||||
// gate it so the editor's Auto-straighten button only appears when configured.
|
||||
estimateTilt: useRunpodTilt
|
||||
? async (_item: MediaItem, image: ImageSource) => {
|
||||
const { data } = imageToBase64(image, 1024);
|
||||
const res = await fetch(`${API}/tilt`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ image: data }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `Tilt estimate failed (${res.status}).`);
|
||||
}
|
||||
return (await res.json()) as {
|
||||
rollDegrees: number;
|
||||
pitchDegrees: number;
|
||||
fovDegrees: number;
|
||||
};
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
"use client";
|
||||
|
||||
// Smart Gallery data layer. Serves EITHER a device-local mock (when the Shell isn't configured — the
|
||||
// demo keeps working offline) OR the live be-crm data door (crm.gallery.*), behind one StorageAdapter
|
||||
// so the embedded @photo-gallery/sdk is mode-agnostic. Tenant scoping, comment authorship and byte
|
||||
// authorization are all enforced server-side; this is just glue.
|
||||
//
|
||||
// Live contract (be-crm):
|
||||
// query crm.gallery.state.load {} -> PersistedState
|
||||
// cmd crm.gallery.state.apply StateChanges -> { upserted, removed }
|
||||
// cmd crm.gallery.media.presignUpload { mediaId, mime, sizeBytes, filename? } -> { ref, uploadUrl, method, headers? }
|
||||
// cmd crm.gallery.media.presignDownload { refs: string[] } -> { urls, expiresInSeconds }
|
||||
// query crm.gallery.stats {} -> { items, albums, people, bytes }
|
||||
// query crm.gallery.lock.status {} -> { hasPassword }
|
||||
// cmd crm.gallery.lock.set { password: string | null } -> { hasPassword }
|
||||
// cmd crm.gallery.lock.verify { password: string } -> { ok }
|
||||
//
|
||||
// BYTES NEVER PASS THROUGH be-crm OR THE BFF. `putMedia` mints a short-lived signed PUT URL and the
|
||||
// browser transfers straight to object storage — the same rule media-api.ts follows for attachments.
|
||||
|
||||
import { useMemo } from "react";
|
||||
import { useAppShell, useAuth } from "@abe-kap/appshell-sdk/react";
|
||||
import {
|
||||
createLocalStorageAdapter,
|
||||
type GalleryFeatures,
|
||||
type GalleryUser,
|
||||
type MediaItem,
|
||||
type PersistedState,
|
||||
type StateChanges,
|
||||
type StorageAdapter,
|
||||
type StoredBlob,
|
||||
type ThemeTokens,
|
||||
} from "@photo-gallery/sdk";
|
||||
import { user as demoUser } from "@/components/dashboard/account-data";
|
||||
import { useMyAccess } from "@/lib/access";
|
||||
import { isShellConfigured } from "./appshell";
|
||||
|
||||
const SHELL = isShellConfigured();
|
||||
|
||||
/** Device-local store key for demo mode. Namespaced so it can't collide with the SDK's own demo. */
|
||||
const LOCAL_STORE_KEY = "lup:smart-gallery:v1";
|
||||
|
||||
/** be-crm caps a single presignDownload at 500 refs — chunk anything larger. */
|
||||
const PRESIGN_CHUNK = 500;
|
||||
|
||||
/** Largest single upload the gallery will attempt (matches be-crm's gallery cap). */
|
||||
export const MAX_GALLERY_BYTES = 200 * 1024 * 1024;
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Wire types (be-crm shapes) */
|
||||
/* ======================================================================== */
|
||||
|
||||
interface StateLoadDTO {
|
||||
media: MediaItem[];
|
||||
albums: PersistedState["albums"];
|
||||
people: PersistedState["people"];
|
||||
labelAliases?: Record<string, string>;
|
||||
deletedLabels?: string[];
|
||||
}
|
||||
|
||||
interface PresignUploadDTO {
|
||||
ref: string;
|
||||
uploadUrl: string;
|
||||
method: "PUT";
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
interface PresignDownloadDTO {
|
||||
urls: Record<string, string>;
|
||||
expiresInSeconds: number;
|
||||
}
|
||||
|
||||
/** The subset of the AppShell SDK this module needs — keeps the adapter unit-testable. */
|
||||
interface DataDoor {
|
||||
query<T>(action: string, variables?: Record<string, unknown>): Promise<T>;
|
||||
command<T>(action: string, variables?: Record<string, unknown>): Promise<T>;
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* The live adapter — be-crm data door */
|
||||
/* ======================================================================== */
|
||||
|
||||
function chunk<T>(items: T[], size: number): T[][] {
|
||||
const out: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A StorageAdapter backed by the be-crm data door.
|
||||
*
|
||||
* Metadata (media/albums/people) rides the door as JSON; bytes go direct to object storage via
|
||||
* short-lived signed URLs. `storageRef` is the durable handle we persist — `src` is only ever a
|
||||
* signed URL with a TTL, so it is re-resolved from the refs on every `load()`.
|
||||
*/
|
||||
export function createDataDoorAdapter(sdk: DataDoor): StorageAdapter {
|
||||
/** Resolve durable refs → fresh signed GET URLs, in chunks the door will accept. */
|
||||
async function resolveRefs(refs: string[]): Promise<Record<string, string>> {
|
||||
const unique = [...new Set(refs.filter(Boolean))];
|
||||
if (!unique.length) return {};
|
||||
const urls: Record<string, string> = {};
|
||||
for (const group of chunk(unique, PRESIGN_CHUNK)) {
|
||||
const res = await sdk.command<PresignDownloadDTO>("crm.gallery.media.presignDownload", { refs: group });
|
||||
Object.assign(urls, res.urls ?? {});
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
return {
|
||||
name: "crm-data-door",
|
||||
|
||||
async load(): Promise<PersistedState | null> {
|
||||
const state = await sdk.query<StateLoadDTO>("crm.gallery.state.load", {});
|
||||
const media = state.media ?? [];
|
||||
// Signed URLs expire, so `src` is always rebuilt from `storageRef` at load time. Items with no
|
||||
// ref (e.g. a seeded remote URL) keep whatever `src` they were stored with.
|
||||
const urls = await resolveRefs(media.map((m) => m.storageRef ?? "").filter(Boolean));
|
||||
return {
|
||||
media: media.map((m) => (m.storageRef && urls[m.storageRef] ? { ...m, src: urls[m.storageRef]! } : m)),
|
||||
albums: state.albums ?? [],
|
||||
people: state.people ?? [],
|
||||
labelAliases: state.labelAliases ?? {},
|
||||
deletedLabels: state.deletedLabels ?? [],
|
||||
version: 1,
|
||||
};
|
||||
},
|
||||
|
||||
// Incremental sync is the real path (see applyChanges). `save` only runs if the store ever falls
|
||||
// back to whole-state persistence; express it as one big change set so behaviour is identical.
|
||||
async save(state: PersistedState): Promise<void> {
|
||||
await sdk.command("crm.gallery.state.apply", {
|
||||
upsertMedia: state.media,
|
||||
upsertAlbums: state.albums,
|
||||
upsertPeople: state.people,
|
||||
labelAliases: state.labelAliases ?? {},
|
||||
deletedLabels: state.deletedLabels ?? [],
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Incremental persistence — only the entities that actually changed. This is what makes a shared
|
||||
* tenant library safe for concurrent editors: two people touching different photos never clobber
|
||||
* each other, because neither sends the other's rows.
|
||||
*/
|
||||
async applyChanges(changes: StateChanges): Promise<void> {
|
||||
// StateChanges is a closed interface; the door takes an open variables bag.
|
||||
await sdk.command("crm.gallery.state.apply", { ...changes });
|
||||
},
|
||||
|
||||
/** Presign → direct PUT → resolve a display URL. Bytes never touch be-crm or the BFF. */
|
||||
async putMedia(id: string, blob: Blob, meta: { name: string; mime: string }): Promise<StoredBlob> {
|
||||
if (blob.size > MAX_GALLERY_BYTES) {
|
||||
throw new Error(`File is too large (max ${Math.floor(MAX_GALLERY_BYTES / (1024 * 1024))} MB).`);
|
||||
}
|
||||
const mime = meta.mime || blob.type || "application/octet-stream";
|
||||
const presigned = await sdk.command<PresignUploadDTO>("crm.gallery.media.presignUpload", {
|
||||
mediaId: id,
|
||||
mime,
|
||||
sizeBytes: blob.size,
|
||||
filename: meta.name,
|
||||
});
|
||||
const res = await fetch(presigned.uploadUrl, {
|
||||
method: presigned.method ?? "PUT",
|
||||
headers: { "content-type": mime, ...(presigned.headers ?? {}) },
|
||||
body: blob,
|
||||
});
|
||||
if (!res.ok) throw new Error(`Upload failed (${res.status}).`);
|
||||
const urls = await resolveRefs([presigned.ref]);
|
||||
return { ref: presigned.ref, url: urls[presigned.ref] ?? presigned.ref };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Public hooks */
|
||||
/* ======================================================================== */
|
||||
|
||||
export interface GalleryStorage {
|
||||
/** True when persisting to be-crm; false when running on the device-local demo store. */
|
||||
live: boolean;
|
||||
adapter: StorageAdapter;
|
||||
}
|
||||
|
||||
/**
|
||||
* The storage backend for the embedded gallery. Live against the be-crm data door once the Shell is
|
||||
* configured, otherwise a device-local store so the demo works with no backend at all.
|
||||
*
|
||||
* SHELL is a build-time constant, so this branch is stable across renders (Rules-of-Hooks safe).
|
||||
*/
|
||||
export function useGalleryStorage(): GalleryStorage {
|
||||
const { sdk } = useAppShell();
|
||||
// The adapter identity must be stable — PhotoGallery captures it in a ref on first render.
|
||||
return useMemo<GalleryStorage>(
|
||||
() =>
|
||||
SHELL
|
||||
? { live: true, adapter: createDataDoorAdapter(sdk as unknown as DataDoor) }
|
||||
: { live: false, adapter: createLocalStorageAdapter(LOCAL_STORE_KEY) },
|
||||
[sdk],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The signed-in identity handed to the gallery so comments are attributed to a real person rather
|
||||
* than a free-text name the author can type themselves.
|
||||
*
|
||||
* Falls back to the same static demo user the sidebar and topbar use when the Shell isn't wired, so
|
||||
* the comment module behaves identically in the demo — the CRM never shows an anonymous author.
|
||||
* be-crm re-stamps `authorId` from the PAT on every write regardless, so this value is a display
|
||||
* convenience, never the source of authority.
|
||||
*/
|
||||
export function useGalleryUser(): GalleryUser {
|
||||
const { user, context } = useAuth();
|
||||
return useMemo<GalleryUser>(() => {
|
||||
if (!user) return { id: demoUser.id, name: demoUser.name };
|
||||
const avatarUrl = user.avatarUrl ?? context?.principal?.avatarUrl;
|
||||
return {
|
||||
id: user.id,
|
||||
name: user.displayName || user.email || demoUser.name,
|
||||
...(user.email ? { email: user.email } : {}),
|
||||
...(avatarUrl ? { avatarUrl } : {}),
|
||||
};
|
||||
}, [user, context]);
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Recently Deleted lock */
|
||||
/* ======================================================================== */
|
||||
|
||||
/**
|
||||
* The SDK's `lockProvider` contract, declared here rather than imported so this module keeps
|
||||
* compiling against an SDK build that predates the prop. It is structurally identical to
|
||||
* `PhotoGalleryProps['lockProvider']`.
|
||||
*/
|
||||
export interface GalleryLockProvider {
|
||||
status(): Promise<{ hasPassword: boolean }>;
|
||||
/** `null` clears the password. */
|
||||
set(password: string | null): Promise<void>;
|
||||
verify(password: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A server-backed, per-user lock for the Recently Deleted view.
|
||||
*
|
||||
* Without this the SDK falls back to a device-local `localStorage` hash: the lock exists only on
|
||||
* the browser that set it, so the same account on a second device sees an unlocked trash. Backed
|
||||
* by `crm.gallery.lock.*`, the password becomes an account property — be-crm stores only a
|
||||
* scrypt hash with a per-record random salt, keyed by (tenant, principal), and rate-limits verify.
|
||||
*
|
||||
* Returns `undefined` in demo mode ON PURPOSE: with no backend there is nowhere to put the hash,
|
||||
* and the SDK's own device-local behaviour is the right fallback for a demo.
|
||||
*
|
||||
* SHELL is a build-time constant, so this branch is stable across renders (Rules-of-Hooks safe).
|
||||
*/
|
||||
export function useGalleryLockProvider(): GalleryLockProvider | undefined {
|
||||
const { sdk } = useAppShell();
|
||||
return useMemo<GalleryLockProvider | undefined>(() => {
|
||||
if (!SHELL) return undefined;
|
||||
const door = sdk as unknown as DataDoor;
|
||||
return {
|
||||
status: () => door.query<{ hasPassword: boolean }>("crm.gallery.lock.status", {}),
|
||||
// The SDK's contract returns void; the door's `{ hasPassword }` is redundant after a set.
|
||||
set: async (password) => {
|
||||
await door.command("crm.gallery.lock.set", { password });
|
||||
},
|
||||
// A wrong password is a normal `{ ok: false }`, not an error. A 403 (the verify lockout)
|
||||
// still throws, which is what the SDK's prompt should surface.
|
||||
verify: async (password) => (await door.command<{ ok: boolean }>("crm.gallery.lock.verify", { password })).ok,
|
||||
};
|
||||
}, [sdk]);
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Permission-gated features */
|
||||
/* ======================================================================== */
|
||||
|
||||
/**
|
||||
* The gallery capabilities the CRM gates behind team permissions. Each SDK feature toggle
|
||||
* (and the whole-view `media.view` gate) maps to one permission id from be-crm's Media group.
|
||||
* The SDK still enforces nothing here — hiding a control is UX; be-crm enforces every write via
|
||||
* GALLERY_VISIBILITY + tenant scoping regardless of what the UI shows.
|
||||
*/
|
||||
const MEDIA_PERMISSIONS = [
|
||||
"media.view", "media.upload", "media.capture", "media.edit",
|
||||
"media.delete", "media.export", "media.share", "media.map", "media.ai",
|
||||
] as const;
|
||||
|
||||
/** SDK feature toggle → the permission id that unlocks it. */
|
||||
const FEATURE_PERMISSION: Record<keyof GalleryFeatures, string> = {
|
||||
editor: "media.edit",
|
||||
camera: "media.capture",
|
||||
import: "media.upload",
|
||||
export: "media.export",
|
||||
sharing: "media.share",
|
||||
map: "media.map",
|
||||
ai: "media.ai",
|
||||
};
|
||||
|
||||
export interface ResolvedGalleryFeatures {
|
||||
/** Feature toggles to hand the SDK's `features` prop, resolved from the caller's permissions. */
|
||||
features: GalleryFeatures;
|
||||
/** Whether the whole Smart Gallery view should render at all (`media.view`). */
|
||||
canView: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the gallery's feature flags + view access from the signed-in user's CRM permissions.
|
||||
*
|
||||
* Fallback rule (SAFE default — a control is enabled unless the user is positively known to lack it):
|
||||
* 1. While access is still loading, stay permissive so features don't flash on then vanish.
|
||||
* 2. Superadmins and owners get everything (they hold every permission anyway; this is explicit).
|
||||
* 3. A member who has been assigned AT LEAST ONE `media.*` permission is gated precisely: each
|
||||
* feature is enabled only if its mapped permission is in their list.
|
||||
* 4. A member with ZERO `media.*` permissions assigned is treated as fully enabled. Roles are not
|
||||
* configured with Media perms until an admin opts in (§5 just made them assignable), so gating a
|
||||
* freshly-seeded member down to nothing would cripple the gallery before anyone could grant them
|
||||
* anything. The "has at least one media.* perm" signal is what flips a role from this permissive
|
||||
* default into precise per-feature gating.
|
||||
*
|
||||
* Because be-crm's mock access (demo, no Shell) returns superadmin + all perms, the demo shows
|
||||
* everything via rule 2.
|
||||
*/
|
||||
export function useGalleryFeatures(): ResolvedGalleryFeatures {
|
||||
const access = useMyAccess();
|
||||
return useMemo<ResolvedGalleryFeatures>(() => {
|
||||
const has = (p: string) => access.permissions.includes(p);
|
||||
const privileged = access.isSuperadmin || access.roleSlugs.includes("owner");
|
||||
const hasAnyMedia = MEDIA_PERMISSIONS.some(has);
|
||||
// Permissive whenever we can't (yet) prove the user lacks a permission: loading, privileged, or a
|
||||
// member who has no Media perms assigned at all. Otherwise gate precisely on the mapped id.
|
||||
const allow = (perm: string) => access.loading || privileged || !hasAnyMedia || has(perm);
|
||||
|
||||
const features: GalleryFeatures = {
|
||||
editor: allow(FEATURE_PERMISSION.editor),
|
||||
camera: allow(FEATURE_PERMISSION.camera),
|
||||
ai: allow(FEATURE_PERMISSION.ai),
|
||||
map: allow(FEATURE_PERMISSION.map),
|
||||
import: allow(FEATURE_PERMISSION.import),
|
||||
export: allow(FEATURE_PERMISSION.export),
|
||||
sharing: allow(FEATURE_PERMISSION.sharing),
|
||||
};
|
||||
|
||||
return { features, canView: allow("media.view") };
|
||||
}, [access]);
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Theme bridge */
|
||||
/* ======================================================================== */
|
||||
|
||||
/**
|
||||
* Maps the gallery's design tokens onto the CRM's own CSS variables.
|
||||
*
|
||||
* Every value is a `var(--crm-token)` reference rather than a literal hex, so the gallery inherits the
|
||||
* dashboard's palette through the SAME `[data-theme]` cascade the rest of the app uses — light/dark
|
||||
* switch together, and a future palette change reaches the gallery with no code edit here.
|
||||
*/
|
||||
export const GALLERY_THEME_TOKENS: ThemeTokens = {
|
||||
// Surfaces
|
||||
bgLight: "var(--bg)",
|
||||
bgDark: "var(--bg)",
|
||||
elevatedLight: "var(--panel-3)",
|
||||
elevatedDark: "var(--panel-3)",
|
||||
sidebarBgLight: "var(--sidebar)",
|
||||
sidebarBgDark: "var(--sidebar)",
|
||||
toolbarBgLight: "var(--panel)",
|
||||
toolbarBgDark: "var(--panel)",
|
||||
cardLight: "var(--panel)",
|
||||
cardDark: "var(--panel)",
|
||||
cardHoverLight: "var(--panel-3)",
|
||||
cardHoverDark: "var(--panel-3)",
|
||||
menuBgLight: "var(--panel-3)",
|
||||
menuBgDark: "var(--panel-3)",
|
||||
|
||||
// Text
|
||||
textLight: "var(--text)",
|
||||
textDark: "var(--text)",
|
||||
textSecondaryLight: "var(--muted)",
|
||||
textSecondaryDark: "var(--muted)",
|
||||
textTertiaryLight: "var(--faint)",
|
||||
textTertiaryDark: "var(--faint)",
|
||||
|
||||
// Lines + washes
|
||||
separatorLight: "var(--border)",
|
||||
separatorDark: "var(--border)",
|
||||
separatorStrongLight: "var(--border-2)",
|
||||
separatorStrongDark: "var(--border-2)",
|
||||
hoverLight: "var(--track)",
|
||||
hoverDark: "var(--track)",
|
||||
activeLight: "var(--border-2)",
|
||||
activeDark: "var(--border-2)",
|
||||
sidebarSelectedLight: "var(--track)",
|
||||
sidebarSelectedDark: "var(--track)",
|
||||
glassBorderLight: "var(--border-2)",
|
||||
glassBorderDark: "var(--border-2)",
|
||||
|
||||
// Brand
|
||||
accent: "var(--orange)",
|
||||
accentStrongLight: "var(--orange-2)",
|
||||
accentStrongDark: "var(--orange-2)",
|
||||
accentContrast: "#ffffff",
|
||||
dangerLight: "var(--red)",
|
||||
dangerDark: "var(--red)",
|
||||
tileFav: "var(--red)",
|
||||
segmentedActive: "var(--panel-3)",
|
||||
|
||||
// Overlays + chrome
|
||||
overlayBg: "rgba(2,2,6,0.94)",
|
||||
editorBg: "var(--panel-2)",
|
||||
shadowSm: "0 1px 2px rgba(0,0,0,0.18)",
|
||||
shadowMdLight: "var(--shadow)",
|
||||
shadowMdDark: "var(--shadow)",
|
||||
shadowLgLight: "0 30px 70px -20px rgba(15,23,42,0.25)",
|
||||
shadowLgDark: "0 30px 70px -20px rgba(0,0,0,0.7)",
|
||||
|
||||
fontFamily: "var(--font)",
|
||||
radiusMenu: 14,
|
||||
sidebarRadius: 14,
|
||||
};
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Tiny in-process fixed-window rate limiter for the Smart Gallery AI routes.
|
||||
*
|
||||
* ---------------------------------------------------------------------------
|
||||
* SCOPE AND LIMITATIONS — READ BEFORE RELYING ON THIS
|
||||
* ---------------------------------------------------------------------------
|
||||
* State lives in a plain `Map` in THIS process's memory. That means:
|
||||
*
|
||||
* - PER-INSTANCE, NOT GLOBAL. With N app instances behind a load balancer a
|
||||
* caller gets up to N x the configured budget. On serverless platforms each
|
||||
* cold start begins with an empty map, so the effective limit is weaker
|
||||
* still.
|
||||
* - NOT DURABLE. A restart or redeploy clears every counter.
|
||||
* - FIXED WINDOW, NOT SLIDING. A caller can burst `max` at the very end of one
|
||||
* window and `max` again at the start of the next — up to 2x `max` across a
|
||||
* window boundary. Acceptable here; the goal is to bound runaway cost, not
|
||||
* to meter precisely.
|
||||
*
|
||||
* It exists because the routes it guards spend real money on GPU inference and
|
||||
* shipping them with NO limit at all is worse than shipping an imperfect one.
|
||||
*
|
||||
* REPLACE WITH REDIS (or the platform's rate limiter) BEFORE RUNNING MORE THAN
|
||||
* ONE INSTANCE. The `limit()` signature is deliberately narrow so a Redis-backed
|
||||
* implementation can drop straight in — the only change needed is making it
|
||||
* async at the call sites.
|
||||
*
|
||||
* This is a throttle, not an authorization check. See src/lib/server/session.ts.
|
||||
*/
|
||||
|
||||
interface Window {
|
||||
/** Requests counted so far in the current window. */
|
||||
count: number;
|
||||
/** Epoch ms at which the current window ends. */
|
||||
resetAt: number;
|
||||
}
|
||||
|
||||
const windows = new Map<string, Window>();
|
||||
|
||||
/** Drop expired entries so the map cannot grow without bound. */
|
||||
const SWEEP_INTERVAL_MS = 60_000;
|
||||
let lastSweep = 0;
|
||||
|
||||
function sweep(now: number): void {
|
||||
if (now - lastSweep < SWEEP_INTERVAL_MS) return;
|
||||
lastSweep = now;
|
||||
for (const [key, w] of windows) {
|
||||
if (w.resetAt <= now) windows.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
export interface LimitResult {
|
||||
ok: boolean;
|
||||
/** Seconds until the window resets. Send as `Retry-After` when `ok` is false. */
|
||||
retryAfter: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count one request against `key` and report whether it is allowed.
|
||||
*
|
||||
* @param key Caller identity — use `rateLimitKey()` from ./session.
|
||||
* @param max Requests allowed per window.
|
||||
* @param windowMs Window length in ms.
|
||||
*/
|
||||
export function limit(key: string, max: number, windowMs: number): LimitResult {
|
||||
const now = Date.now();
|
||||
sweep(now);
|
||||
|
||||
const existing = windows.get(key);
|
||||
if (!existing || existing.resetAt <= now) {
|
||||
windows.set(key, { count: 1, resetAt: now + windowMs });
|
||||
return { ok: true, retryAfter: 0 };
|
||||
}
|
||||
|
||||
if (existing.count >= max) {
|
||||
return { ok: false, retryAfter: Math.max(1, Math.ceil((existing.resetAt - now) / 1000)) };
|
||||
}
|
||||
|
||||
existing.count += 1;
|
||||
return { ok: true, retryAfter: 0 };
|
||||
}
|
||||
|
||||
/** Test/maintenance helper — clears all counters. */
|
||||
export function resetAllLimits(): void {
|
||||
windows.clear();
|
||||
lastSweep = 0;
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* base64 / data-URI helpers shared by the RunPod endpoint wrappers.
|
||||
* Different endpoints name their image field differently and some prefix a
|
||||
* `data:` URI — these helpers normalize both.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/base64.ts`.
|
||||
* The `pickOutputImage` normalization is intentionally identical so both apps
|
||||
* tolerate the same set of endpoint response shapes.
|
||||
*
|
||||
* SERVER-ONLY.
|
||||
*/
|
||||
|
||||
/** "data:image/png;base64,XXXX" -> "XXXX" (leaves a bare base64 string untouched). */
|
||||
export function stripDataUri(s: string): string {
|
||||
if (!s.startsWith("data:")) return s;
|
||||
const i = s.indexOf(",");
|
||||
return i === -1 ? s : s.slice(i + 1);
|
||||
}
|
||||
|
||||
/** Wrap a bare base64 string in a data: URI. */
|
||||
export function toDataUri(b64: string, mime: string): string {
|
||||
return `data:${mime};base64,${stripDataUri(b64)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the first image-like base64 string out of a RunPod endpoint's `output`,
|
||||
* regardless of which field name it used. Handles the common shapes:
|
||||
* "iVBOR..." (raw string)
|
||||
* { image: "..." } / { image_png } / { image_base64 }
|
||||
* { images: ["..."] } (array)
|
||||
* { output: { image: "..." } } (nested)
|
||||
*/
|
||||
export function pickOutputImage(output: unknown): string {
|
||||
const s = findImageString(output, false);
|
||||
if (!s) throw new Error("RunPod output did not contain an image.");
|
||||
return stripDataUri(s);
|
||||
}
|
||||
|
||||
/** Keys whose name implies the value IS the image — any non-empty string is accepted. */
|
||||
const IMAGE_KEYS = ["image_png", "image", "image_base64", "images"] as const;
|
||||
/** Generic wrapper keys — a string here must actually look like image data. */
|
||||
const CONTAINER_KEYS = ["output", "result", "data"] as const;
|
||||
|
||||
/** A base64 image payload is long; a status/id string ("success", "job-abc") is short. */
|
||||
function looksLikeImageData(s: string): boolean {
|
||||
if (s.startsWith("data:image/")) return true;
|
||||
return s.length >= 256 && /^[A-Za-z0-9+/=\s]+$/.test(s.slice(0, 256));
|
||||
}
|
||||
|
||||
/**
|
||||
* `strict` is true when we descended through a generic wrapper key (output/result/
|
||||
* data), where a bare string could be a status/id rather than an image — so it must
|
||||
* pass `looksLikeImageData`. Under an explicit image key (or at top level) any
|
||||
* non-empty string is taken as the image.
|
||||
*/
|
||||
function findImageString(value: unknown, strict: boolean, depth = 0): string | undefined {
|
||||
if (depth > 5) return undefined;
|
||||
if (typeof value === "string") {
|
||||
if (!value) return undefined;
|
||||
return !strict || looksLikeImageData(value) ? value : undefined;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
for (const el of value) {
|
||||
const s = findImageString(el, strict, depth + 1);
|
||||
if (s) return s;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
const o = value as Record<string, unknown>;
|
||||
for (const key of IMAGE_KEYS) {
|
||||
if (key in o) {
|
||||
const s = findImageString(o[key], false, depth + 1);
|
||||
if (s) return s;
|
||||
}
|
||||
}
|
||||
for (const key of CONTAINER_KEYS) {
|
||||
if (key in o) {
|
||||
const s = findImageString(o[key], true, depth + 1);
|
||||
if (s) return s;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* Low-level RunPod transport. Every model runs as its own RunPod endpoint; this
|
||||
* handles the common request envelope, bearer auth, and both response modes:
|
||||
*
|
||||
* - `/runsync` (preferred): the job runs synchronously and the body already
|
||||
* contains `output` (or IS the output for a custom handler).
|
||||
* - `/run`: returns `{ id, status }`; we poll `/status/{id}` until the job
|
||||
* reaches a terminal state or the time budget (kept under the 60s serverless
|
||||
* function cap) is exhausted.
|
||||
*
|
||||
* RUNPOD_API_KEY is read here so callers never handle the secret directly, and
|
||||
* it is NEVER echoed into an error message. Upstream error bodies are truncated
|
||||
* to 160 chars before being surfaced.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/client.ts` (same 55s
|
||||
* budget, same polling, same RunpodError.status mapping).
|
||||
*
|
||||
* SERVER-ONLY.
|
||||
*/
|
||||
|
||||
export class RunpodError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status = 502) {
|
||||
super(message);
|
||||
this.name = "RunpodError";
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export interface RunpodCallOpts {
|
||||
/** Human label used in error messages, e.g. "sd-inpaint". */
|
||||
name: string;
|
||||
/** Full endpoint URL from the per-model env var (…/runsync or …/run). */
|
||||
url: string;
|
||||
input: Record<string, unknown>;
|
||||
/** Total budget for the whole call incl. polling. Default 55s (< the 60s cap). */
|
||||
timeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
}
|
||||
|
||||
interface RunpodEnvelope {
|
||||
id?: string;
|
||||
status?: string;
|
||||
output?: unknown;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
export async function runpodCall<TOut = unknown>(opts: RunpodCallOpts): Promise<TOut> {
|
||||
const { name, url, input } = opts;
|
||||
const timeoutMs = opts.timeoutMs ?? 55_000;
|
||||
const pollIntervalMs = opts.pollIntervalMs ?? 1500;
|
||||
|
||||
const key = process.env.RUNPOD_API_KEY;
|
||||
if (!key) throw new RunpodError("RUNPOD_API_KEY is not set.", 500);
|
||||
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const authHeaders = { authorization: `Bearer ${key}` };
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", ...authHeaders },
|
||||
body: JSON.stringify({ input }),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
throw new RunpodError(`Could not reach RunPod (${name}).`, 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
// Truncated on purpose — never surface a full upstream body to the client.
|
||||
const detail = (await res.text().catch(() => "")).slice(0, 160);
|
||||
throw new RunpodError(`RunPod ${name} error (${res.status}). ${detail}`.trim(), 502);
|
||||
}
|
||||
|
||||
const data = (await res.json().catch(() => null)) as RunpodEnvelope | null;
|
||||
if (!data || typeof data !== "object") {
|
||||
throw new RunpodError(`RunPod ${name} returned an invalid response.`, 502);
|
||||
}
|
||||
|
||||
// Terminal failure reported in a job envelope.
|
||||
if (data.status === "FAILED" || data.status === "CANCELLED") {
|
||||
throw new RunpodError(`RunPod ${name} job ${data.status.toLowerCase()}.`, 502);
|
||||
}
|
||||
// No job id → this is not an async envelope; the body itself is the output.
|
||||
// Covers custom /runsync handlers that return their result directly, even when
|
||||
// it carries a `status` field (e.g. "success").
|
||||
if (data.id === undefined) {
|
||||
return (data.output !== undefined ? data.output : data) as TOut;
|
||||
}
|
||||
// Job envelope that already carries a completed/inline output.
|
||||
if (data.output !== undefined && (data.status === undefined || data.status === "COMPLETED")) {
|
||||
return data.output as TOut;
|
||||
}
|
||||
|
||||
// Async: poll /status/{id} until COMPLETED / FAILED / the time budget runs out.
|
||||
// Each wait + fetch is clamped to the remaining budget so the whole call stays
|
||||
// under `timeoutMs` (kept below the 60s function cap).
|
||||
const statusUrl = url.replace(/\/run(sync)?(\/?)$/, `/status/${data.id}`);
|
||||
for (;;) {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining < 500) break; // not enough budget for another round
|
||||
await sleep(Math.min(pollIntervalMs, remaining));
|
||||
const left = deadline - Date.now();
|
||||
if (left <= 0) break;
|
||||
let sres: Response;
|
||||
try {
|
||||
sres = await fetch(statusUrl, {
|
||||
headers: authHeaders,
|
||||
signal: AbortSignal.timeout(Math.min(10_000, Math.max(1000, left))),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
continue; // transient — keep polling until the deadline
|
||||
}
|
||||
if (!sres.ok) continue;
|
||||
const sdata = (await sres.json().catch(() => null)) as RunpodEnvelope | null;
|
||||
if (!sdata) continue;
|
||||
if (sdata.status === "COMPLETED") return sdata.output as TOut;
|
||||
if (sdata.status === "FAILED" || sdata.status === "CANCELLED") {
|
||||
throw new RunpodError(`RunPod ${name} job ${sdata.status.toLowerCase()}.`, 502);
|
||||
}
|
||||
}
|
||||
throw new RunpodError(
|
||||
`RunPod ${name} timed out (raise the endpoint's speed or use /runsync).`,
|
||||
504,
|
||||
);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
/**
|
||||
* Typed wrappers for each RunPod endpoint behind the Smart Gallery.
|
||||
* Every function reads its own `RUNPOD_*_URL` env var, sends the exact `input`
|
||||
* contract the model spec documents, and normalizes the response.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/endpoints.ts` —
|
||||
* identical `normalizeDetections` / `normalizeBox` logic and identical env var
|
||||
* names, so an endpoint deployed for the SDK demo works here unchanged.
|
||||
*
|
||||
* SERVER-ONLY. Missing/invalid URLs throw a RunpodError(500) so an unconfigured
|
||||
* op surfaces as a clear message rather than a crash.
|
||||
*/
|
||||
|
||||
import { stripDataUri, pickOutputImage } from "./base64";
|
||||
import { RunpodError, runpodCall } from "./client";
|
||||
import type {
|
||||
Img2ImgReq,
|
||||
InpaintReq,
|
||||
RunpodDetection,
|
||||
RunpodImageResult,
|
||||
TiltResult,
|
||||
TranscriptResult,
|
||||
} from "./types";
|
||||
|
||||
function envNum(v: string | undefined, fallback: number): number {
|
||||
// Treat a blank/whitespace env var as unset — Number('') is 0 (finite), which
|
||||
// would otherwise send e.g. strength:0 for `RUNPOD_SD_STRENGTH=`.
|
||||
if (v == null || v.trim() === "") return fallback;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : fallback;
|
||||
}
|
||||
|
||||
function endpointUrl(envVar: string): string {
|
||||
const url = process.env[envVar];
|
||||
if (!url || !/^https?:\/\//i.test(url)) {
|
||||
throw new RunpodError(`${envVar} is not set (or is not an http(s) URL).`, 500);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Run an image-in/image-out endpoint and normalize the result to base64 PNG. */
|
||||
async function imageOp(
|
||||
name: string,
|
||||
envVar: string,
|
||||
input: Record<string, unknown>,
|
||||
): Promise<RunpodImageResult> {
|
||||
const output = await runpodCall({ name, url: endpointUrl(envVar), input });
|
||||
return { imageBase64: pickOutputImage(output), mimeType: "image/png" };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Image endpoints
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** #6 Background removal (U²-Net via rembg). model: u2net | u2netp | u2net_human_seg */
|
||||
export function rpRemoveBackground(imageB64: string, model?: string): Promise<RunpodImageResult> {
|
||||
return imageOp("background-removal", "RUNPOD_BG_REMOVE_URL", {
|
||||
task: "remove-bg",
|
||||
image: imageB64,
|
||||
...(model ? { model_name: model } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/** #7 Real-ESRGAN enhance/upscale (RealESRGAN_x4plus). The caller's scale (from the
|
||||
* op / restore pass) is authoritative — it is not overridden by any env default. */
|
||||
export function rpUpscale(
|
||||
imageB64: string,
|
||||
scale: 2 | 4,
|
||||
faceEnhance = false,
|
||||
): Promise<RunpodImageResult> {
|
||||
return imageOp("upscale", "RUNPOD_UPSCALE_URL", {
|
||||
task: "upscale",
|
||||
image: imageB64,
|
||||
scale,
|
||||
face_enhance: faceEnhance,
|
||||
});
|
||||
}
|
||||
|
||||
/** #8 DDColor B&W → colorize. */
|
||||
export function rpColorize(imageB64: string, inputSize?: number): Promise<RunpodImageResult> {
|
||||
return imageOp("colorize", "RUNPOD_COLORIZE_URL", {
|
||||
image: imageB64,
|
||||
...(inputSize ? { input_size: inputSize } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/** #9 SD 3.5 masked inpainting (sky fix / eraser / fill). Also #11 outpaint (pre-padded). */
|
||||
export function rpInpaint(p: InpaintReq): Promise<RunpodImageResult> {
|
||||
const input: Record<string, unknown> = {
|
||||
task: "inpaint",
|
||||
image: p.imageB64,
|
||||
mask: p.maskB64,
|
||||
prompt: p.prompt,
|
||||
strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.8),
|
||||
guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7),
|
||||
num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35),
|
||||
};
|
||||
const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT;
|
||||
if (negative) input.negative_prompt = negative;
|
||||
if (p.seed != null) input.seed = p.seed;
|
||||
return imageOp("sd-inpaint", "RUNPOD_SD_INPAINT_URL", input);
|
||||
}
|
||||
|
||||
/** #10 SD 3.5 general prompt edit (img2img, no mask). */
|
||||
export function rpImg2Img(p: Img2ImgReq): Promise<RunpodImageResult> {
|
||||
const input: Record<string, unknown> = {
|
||||
task: "img2img",
|
||||
image: p.imageB64,
|
||||
prompt: p.prompt,
|
||||
strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.6),
|
||||
guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7),
|
||||
num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35),
|
||||
};
|
||||
const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT;
|
||||
if (negative) input.negative_prompt = negative;
|
||||
if (p.seed != null) input.seed = p.seed;
|
||||
return imageOp("sd-img2img", "RUNPOD_SD_IMG2IMG_URL", input);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// #1 YOLO detection → SDK DetectedObject shape (box as fractions 0..1)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function rpDetect(
|
||||
imageB64: string,
|
||||
width: number,
|
||||
height: number,
|
||||
): Promise<RunpodDetection[]> {
|
||||
const output = await runpodCall<unknown>({
|
||||
name: "yolo-detect",
|
||||
url: endpointUrl("RUNPOD_YOLO_URL"),
|
||||
input: { image: imageB64, task: "detect" },
|
||||
});
|
||||
return normalizeDetections(output, width, height);
|
||||
}
|
||||
|
||||
function extractDetectionArray(output: unknown): unknown[] {
|
||||
if (Array.isArray(output)) return output;
|
||||
if (output && typeof output === "object") {
|
||||
const o = output as Record<string, unknown>;
|
||||
for (const key of ["detections", "predictions", "objects", "results", "boxes"]) {
|
||||
if (Array.isArray(o[key])) return o[key] as unknown[];
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/** First non-empty STRING among the args (numbers ignored — a numeric `class` is an index, not a name). */
|
||||
function firstLabel(...vals: unknown[]): string | undefined {
|
||||
for (const v of vals) {
|
||||
if (typeof v === "string" && v.trim()) return v.trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeDetections(output: unknown, width: number, height: number): RunpodDetection[] {
|
||||
const out: RunpodDetection[] = [];
|
||||
for (const raw of extractDetectionArray(output)) {
|
||||
if (!raw || typeof raw !== "object") continue;
|
||||
const o = raw as Record<string, unknown>;
|
||||
// Prefer a human-readable name (ultralytics tojson puts the string in `name`
|
||||
// and a numeric index in `class`); fall back to class_<id>. Using firstLabel
|
||||
// (not `??`) also means an explicit empty-string label doesn't get kept + dropped.
|
||||
const classId = o.class_id ?? (typeof o.class === "number" ? o.class : undefined);
|
||||
const label = (
|
||||
firstLabel(o.label, o.name, o.class_name, typeof o.class === "string" ? o.class : undefined) ??
|
||||
(classId != null ? `class_${classId}` : "object")
|
||||
).toLowerCase();
|
||||
const confidence = Number(o.confidence ?? o.score ?? o.conf ?? 0) || 0;
|
||||
const box = normalizeBox(o, width, height);
|
||||
if (box && label) out.push({ label, confidence, box });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function num(v: unknown): number | null {
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
function asNum4(v: unknown): [number, number, number, number] | null {
|
||||
if (!Array.isArray(v) || v.length < 4) return null;
|
||||
const a = num(v[0]);
|
||||
const b = num(v[1]);
|
||||
const c = num(v[2]);
|
||||
const d = num(v[3]);
|
||||
return a === null || b === null || c === null || d === null ? null : [a, b, c, d];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a detection box to {x, y, width, height} as fractions 0..1 of the
|
||||
* image, from whatever shape the endpoint emits:
|
||||
* - `xyxy: [x1,y1,x2,y2]` (ultralytics) and generic `box: [...]` → corner form
|
||||
* - `xywh: [...]` and COCO `bbox: [x,y,w,h]` → x/y/width/height form
|
||||
* - object `{x1,y1,x2,y2}` / `{left,top,right,bottom}` (ultralytics tojson) → corners
|
||||
* - object `{x,y,width,height}` → x/y/width/height
|
||||
* Pixel values (any component > 1) are divided by the image dims; already-
|
||||
* normalized fractions pass through.
|
||||
*/
|
||||
function normalizeBox(
|
||||
o: Record<string, unknown>,
|
||||
width: number,
|
||||
height: number,
|
||||
): RunpodDetection["box"] | null {
|
||||
const W = width || 1;
|
||||
const H = height || 1;
|
||||
const clamp01 = (n: number) => Math.max(0, Math.min(1, n));
|
||||
const frac = (x: number, y: number, w: number, h: number): RunpodDetection["box"] => {
|
||||
if (Math.max(Math.abs(x), Math.abs(y), Math.abs(w), Math.abs(h)) > 1) {
|
||||
x /= W;
|
||||
y /= H;
|
||||
w /= W;
|
||||
h /= H;
|
||||
}
|
||||
return { x: clamp01(x), y: clamp01(y), width: clamp01(w), height: clamp01(h) };
|
||||
};
|
||||
const fromXyxy = (x1: number, y1: number, x2: number, y2: number) =>
|
||||
frac(x1, y1, x2 - x1, y2 - y1);
|
||||
|
||||
// 1. Array boxes, interpreted by which key holds them.
|
||||
const xyxyArr = asNum4(o.xyxy);
|
||||
if (xyxyArr) return fromXyxy(xyxyArr[0], xyxyArr[1], xyxyArr[2], xyxyArr[3]);
|
||||
const xywhArr = asNum4(o.xywh) ?? asNum4(o.bbox); // COCO `bbox` is [x,y,w,h]
|
||||
if (xywhArr) return frac(xywhArr[0], xywhArr[1], xywhArr[2], xywhArr[3]);
|
||||
const boxArr = asNum4(o.box); // generic array box → assume corner form
|
||||
if (boxArr) return fromXyxy(boxArr[0], boxArr[1], boxArr[2], boxArr[3]);
|
||||
|
||||
// 2. Object boxes (either nested under `box` or directly on the detection).
|
||||
const src =
|
||||
o.box && typeof o.box === "object" && !Array.isArray(o.box)
|
||||
? (o.box as Record<string, unknown>)
|
||||
: o;
|
||||
const x1 = num(src.x1 ?? src.left);
|
||||
const y1 = num(src.y1 ?? src.top);
|
||||
const x2 = num(src.x2 ?? src.right);
|
||||
const y2 = num(src.y2 ?? src.bottom);
|
||||
if (x1 !== null && y1 !== null && x2 !== null && y2 !== null) return fromXyxy(x1, y1, x2, y2);
|
||||
|
||||
const x = num(src.x);
|
||||
const y = num(src.y);
|
||||
const w = num(src.width);
|
||||
const h = num(src.height);
|
||||
if (x !== null && y !== null && w !== null && h !== null) return frac(x, y, w, h);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audio / calibration endpoints
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** #2 Camera tilt (DeepSingleImageCalibration). */
|
||||
export async function rpTilt(imageB64: string): Promise<TiltResult> {
|
||||
const o = await runpodCall<Record<string, unknown>>({
|
||||
name: "tilt",
|
||||
url: endpointUrl("RUNPOD_TILT_URL"),
|
||||
input: { image: imageB64 },
|
||||
});
|
||||
return {
|
||||
rollDegrees: Number(o.roll_degrees ?? 0) || 0,
|
||||
pitchDegrees: Number(o.pitch_degrees ?? 0) || 0,
|
||||
fovDegrees: Number(o.fov_degrees ?? 0) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
/** #3 Voice-to-text (Parakeet). Audio must be WAV 16kHz mono PCM16. */
|
||||
export async function rpTranscribe(
|
||||
audioB64: string,
|
||||
opts?: { language?: string; timestamps?: boolean; punctuation?: boolean },
|
||||
): Promise<TranscriptResult> {
|
||||
const o = await runpodCall<Record<string, unknown>>({
|
||||
name: "transcribe",
|
||||
url: endpointUrl("RUNPOD_STT_URL"),
|
||||
input: { audio: audioB64, task: "transcribe", ...(opts ?? {}) },
|
||||
});
|
||||
const rawSegments = Array.isArray(o.segments) ? o.segments : [];
|
||||
const segments = rawSegments.map((s) => {
|
||||
const seg = (s ?? {}) as Record<string, unknown>;
|
||||
return {
|
||||
text: String(seg.text ?? ""),
|
||||
startSec: Number(seg.start_sec ?? 0) || 0,
|
||||
endSec: Number(seg.end_sec ?? 0) || 0,
|
||||
};
|
||||
});
|
||||
return {
|
||||
transcript: String(o.transcript ?? ""),
|
||||
segments: segments.length ? segments : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** #12 Audio noise removal (RNNoise). Audio must be WAV 48kHz mono 16-bit PCM. */
|
||||
export async function rpDenoiseAudio(audioB64: string): Promise<{ audioB64: string }> {
|
||||
const o = await runpodCall<Record<string, unknown>>({
|
||||
name: "audio-denoise",
|
||||
url: endpointUrl("RUNPOD_AUDIO_DENOISE_URL"),
|
||||
input: { audio: audioB64, task: "denoise" },
|
||||
});
|
||||
const a = o.audio ?? o.output ?? "";
|
||||
return { audioB64: stripDataUri(String(a)) };
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Request/response types for the RunPod endpoints behind the Smart Gallery.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/types.ts`.
|
||||
* Keep the two in sync — the RunPod endpoints are shared between the standalone
|
||||
* SDK demo and this CRM.
|
||||
*
|
||||
* SERVER-ONLY. Imported by src/app/api/gallery/ai/* route handlers, never by a
|
||||
* client component (which must not see a RunPod URL or key).
|
||||
*/
|
||||
|
||||
/** Normalized image result returned by every image endpoint (raw base64, no data: prefix). */
|
||||
export interface RunpodImageResult {
|
||||
imageBase64: string;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
/** #9 SD 3.5 masked inpainting (also #11 outpaint, pre-padded). white in mask = regenerate. */
|
||||
export interface InpaintReq {
|
||||
imageB64: string;
|
||||
maskB64: string;
|
||||
prompt: string;
|
||||
negativePrompt?: string;
|
||||
strength?: number;
|
||||
guidanceScale?: number;
|
||||
steps?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
/** #10 SD 3.5 general prompt edit (img2img, no mask). */
|
||||
export interface Img2ImgReq {
|
||||
imageB64: string;
|
||||
prompt: string;
|
||||
negativePrompt?: string;
|
||||
strength?: number;
|
||||
guidanceScale?: number;
|
||||
steps?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* #1 YOLO construction-material classifier, normalized to the SDK's
|
||||
* `DetectedObject` shape (box as fractions 0..1 of the image).
|
||||
*/
|
||||
export interface RunpodDetection {
|
||||
label: string;
|
||||
confidence: number;
|
||||
box: { x: number; y: number; width: number; height: number };
|
||||
}
|
||||
|
||||
/** #2 DeepSingleImageCalibration — camera tilt. */
|
||||
export interface TiltResult {
|
||||
rollDegrees: number;
|
||||
pitchDegrees: number;
|
||||
fovDegrees: number;
|
||||
}
|
||||
|
||||
/** #3 Parakeet voice-to-text. */
|
||||
export interface TranscriptSegment {
|
||||
text: string;
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
}
|
||||
export interface TranscriptResult {
|
||||
transcript: string;
|
||||
segments?: TranscriptSegment[];
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Session gate for the Smart Gallery AI routes.
|
||||
*
|
||||
* ---------------------------------------------------------------------------
|
||||
* TRUST MODEL
|
||||
* ---------------------------------------------------------------------------
|
||||
* These routes proxy a paid, rate-limited GPU backend (RunPod) using a secret
|
||||
* held only on this server. An unauthenticated route is therefore not merely an
|
||||
* information-disclosure problem — it is a billable-resource problem: anyone who
|
||||
* can reach the URL can spend the operator's GPU budget, and can use the CRM as
|
||||
* an open relay for arbitrary image/audio processing.
|
||||
*
|
||||
* The upstream SDK demo's routes (apps/web/src/app/api/ai/*) are COMPLETELY
|
||||
* unauthenticated. This module is the fix; every ported route must call it
|
||||
* before doing any work.
|
||||
*
|
||||
* WHO IS TRUSTED
|
||||
* We do not verify a JWT here and we do not hold any signing key. The single
|
||||
* source of truth for "is this caller signed in" is the Shell BFF
|
||||
* (`${BFF_ORIGIN}/api/session/context`), which owns the HttpOnly session cookie.
|
||||
* We forward the caller's raw `cookie` header to it and treat a 200 as proof of
|
||||
* a session. Consequences of that choice, stated explicitly:
|
||||
*
|
||||
* - The BFF is trusted absolutely. If it is compromised or misconfigured to
|
||||
* answer 200 for anonymous callers, these routes are open. BFF_ORIGIN must
|
||||
* therefore only ever point at an origin the operator controls.
|
||||
* - We forward the cookie header verbatim and nothing else. No Authorization
|
||||
* header, no bearer token, and never the RunPod key.
|
||||
* - NOTHING IS CACHED. A cached "yes" would keep a revoked/expired session
|
||||
* alive for the cache lifetime, so every AI request costs one BFF round
|
||||
* trip. That is deliberate: correctness over latency for a spend gate.
|
||||
* - A network failure reaching the BFF returns 503 (fail CLOSED), never 200.
|
||||
* If we cannot prove a session, we do not spend GPU budget.
|
||||
*
|
||||
* DEMO MODE
|
||||
* When the Shell is not configured (`NEXT_PUBLIC_SUPABASE_URL` unset) the CRM
|
||||
* runs on its mock portal and there is no session to check, so we allow the
|
||||
* request and log ONCE at startup-of-first-use. This is the same gate
|
||||
* `isShellConfigured()` uses for mock-vs-real auth elsewhere in the app.
|
||||
* IMPORTANT: never deploy to a public origin with the Shell unconfigured AND a
|
||||
* real RUNPOD_API_KEY present — that combination is an open, billable endpoint.
|
||||
*
|
||||
* WHAT THIS IS NOT
|
||||
* This is authentication only, not authorization. It answers "is there a valid
|
||||
* session", not "may this principal use the gallery". Per-resource policy for
|
||||
* gallery data lives in be-crm behind the `crm.gallery` resource; if these AI
|
||||
* routes ever need the same, check it there rather than re-deriving it here.
|
||||
*/
|
||||
|
||||
export type GallerySession =
|
||||
| { ok: true; principalId?: string }
|
||||
| { ok: false; status: number; error: string };
|
||||
|
||||
/** Mirrors src/lib/appshell.ts — kept local so this stays server-only. */
|
||||
function isShellConfigured(): boolean {
|
||||
return Boolean(process.env.NEXT_PUBLIC_SUPABASE_URL);
|
||||
}
|
||||
|
||||
let demoModeWarned = false;
|
||||
|
||||
/**
|
||||
* Resolve the caller's session. Returns `{ ok: true }` (optionally with the
|
||||
* principal id, used to key rate limits) or a ready-to-return failure with the
|
||||
* status and message the route should emit.
|
||||
*/
|
||||
export async function requireGallerySession(req: Request): Promise<GallerySession> {
|
||||
if (!isShellConfigured()) {
|
||||
if (!demoModeWarned) {
|
||||
demoModeWarned = true;
|
||||
console.warn(
|
||||
"[gallery-ai] Shell is not configured (NEXT_PUBLIC_SUPABASE_URL unset) — " +
|
||||
"AI routes are UNAUTHENTICATED in demo mode. Do not expose this deployment publicly " +
|
||||
"while RUNPOD_API_KEY is set.",
|
||||
);
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
const cookie = req.headers.get("cookie");
|
||||
if (!cookie) return { ok: false, status: 401, error: "Not signed in" };
|
||||
|
||||
const origin = (process.env.BFF_ORIGIN ?? "http://localhost:4000").replace(/\/$/, "");
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`${origin}/api/session/context`, {
|
||||
headers: { cookie, accept: "application/json" },
|
||||
// Never cache an auth decision — see the trust-model note above.
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
} catch {
|
||||
// Fail closed: we could not prove a session, so we do not spend GPU budget.
|
||||
return { ok: false, status: 503, error: "Session service unavailable" };
|
||||
}
|
||||
|
||||
if (res.status === 401) return { ok: false, status: 401, error: "Not signed in" };
|
||||
if (!res.ok) return { ok: false, status: 503, error: "Session service unavailable" };
|
||||
|
||||
// The principal id is best-effort: it only sharpens the rate-limit key, so a
|
||||
// shape we don't recognize degrades to IP-keyed limiting rather than failing.
|
||||
let principalId: string | undefined;
|
||||
try {
|
||||
const data = (await res.json()) as Record<string, unknown> | null;
|
||||
principalId = pickPrincipalId(data);
|
||||
} catch {
|
||||
/* ignore — see above */
|
||||
}
|
||||
|
||||
return principalId ? { ok: true, principalId } : { ok: true };
|
||||
}
|
||||
|
||||
function pickPrincipalId(data: Record<string, unknown> | null): string | undefined {
|
||||
if (!data) return undefined;
|
||||
const direct = data.userId ?? data.principalId ?? data.sub ?? data.id;
|
||||
if (typeof direct === "string" && direct) return direct;
|
||||
const user = data.user;
|
||||
if (user && typeof user === "object") {
|
||||
const u = user as Record<string, unknown>;
|
||||
const nested = u.id ?? u.userId ?? u.sub;
|
||||
if (typeof nested === "string" && nested) return nested;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate-limit key for a request: the authenticated principal when known,
|
||||
* otherwise the first hop of `x-forwarded-for`.
|
||||
*
|
||||
* NOTE the first hop is client-controlled unless a trusted proxy overwrites the
|
||||
* header. It is good enough to throttle honest clients and casual abuse; it is
|
||||
* NOT a security boundary. The session gate above is the security boundary.
|
||||
*/
|
||||
export function rateLimitKey(req: Request, principalId?: string): string {
|
||||
if (principalId) return `u:${principalId}`;
|
||||
const xff = req.headers.get("x-forwarded-for") ?? "";
|
||||
const first = xff.split(",")[0]?.trim();
|
||||
return `ip:${first || "unknown"}`;
|
||||
}
|
||||
Reference in New Issue
Block a user