feat: add core domain types for Photo Gallery SDK including media items, albums, and annotations
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Shared entry gate for every /api/gallery/ai/* route: authenticate, then
|
||||
* throttle. Lives in a `_`-prefixed file so the App Router never treats it as a
|
||||
* route (only `route.ts` defines an endpoint).
|
||||
*
|
||||
* Order matters: we authenticate FIRST so the rate limit can be keyed by
|
||||
* principal rather than by a spoofable `x-forwarded-for` hop wherever possible.
|
||||
* The session check is one cheap BFF round trip; the work it guards is a GPU
|
||||
* call, so paying it before throttling is the right trade.
|
||||
*
|
||||
* SERVER-ONLY.
|
||||
*/
|
||||
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
import { limit } from "@/lib/server/rate-limit";
|
||||
import { rateLimitKey, requireGallerySession } from "@/lib/server/session";
|
||||
|
||||
/** Per-minute budgets, per the Smart Gallery route contract. */
|
||||
export const RATE_LIMITS = {
|
||||
classify: 30,
|
||||
edit: 12,
|
||||
tilt: 30,
|
||||
transcribe: 20,
|
||||
denoise: 20,
|
||||
} as const;
|
||||
|
||||
const WINDOW_MS = 60_000;
|
||||
|
||||
export type GuardResult =
|
||||
| { ok: true; principalId?: string }
|
||||
/** Ready-to-return error response — the route should return it unchanged. */
|
||||
| { ok: false; response: NextResponse };
|
||||
|
||||
/**
|
||||
* @param route Which budget to apply (also namespaces the limiter key so a
|
||||
* caller's `edit` spend does not consume their `classify` budget).
|
||||
*/
|
||||
export async function guard(req: Request, route: keyof typeof RATE_LIMITS): Promise<GuardResult> {
|
||||
const session = await requireGallerySession(req);
|
||||
if (!session.ok) {
|
||||
return {
|
||||
ok: false,
|
||||
response: NextResponse.json({ error: session.error }, { status: session.status }),
|
||||
};
|
||||
}
|
||||
|
||||
const key = `${route}:${rateLimitKey(req, session.principalId)}`;
|
||||
const { ok, retryAfter } = limit(key, RATE_LIMITS[route], WINDOW_MS);
|
||||
if (!ok) {
|
||||
return {
|
||||
ok: false,
|
||||
response: NextResponse.json(
|
||||
{ error: "Too many requests — slow down." },
|
||||
{ status: 429, headers: { "Retry-After": String(retryAfter) } },
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true, principalId: session.principalId };
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import { rpDetect } from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60;
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Object-detection proxy for the RunPod YOLO construction-material classifier (#1).
|
||||
* The key + endpoint URL stay server-side. The client calls this only when
|
||||
* NEXT_PUBLIC_APG_RUNPOD_DETECT is on; otherwise detection runs fully in-browser
|
||||
* (COCO-SSD) with no server round-trip. Returns the SDK's DetectedObject[] shape
|
||||
* (box as 0..1 fractions) so it drops straight into the Objects browser / smart
|
||||
* albums / search.
|
||||
*
|
||||
* POST { imageBase64, width, height } -> { objects: [{ label, confidence, box }] }
|
||||
* Auth: session-gated (see lib/server/session.ts). Rate limit: 30/min.
|
||||
*/
|
||||
|
||||
const MAX_BASE64 = 4_000_000; // ~3 MB decoded — under serverless body limits
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "classify");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
if (!process.env.RUNPOD_API_KEY || !process.env.RUNPOD_YOLO_URL) {
|
||||
return NextResponse.json(
|
||||
{ error: "RunPod detection is not configured (set RUNPOD_API_KEY + RUNPOD_YOLO_URL)." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid request body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const { imageBase64, width, height } = (body ?? {}) as {
|
||||
imageBase64?: unknown;
|
||||
width?: unknown;
|
||||
height?: unknown;
|
||||
};
|
||||
if (
|
||||
typeof imageBase64 !== "string" ||
|
||||
imageBase64.length === 0 ||
|
||||
imageBase64.length > MAX_BASE64
|
||||
) {
|
||||
return NextResponse.json({ error: "Invalid or oversized image." }, { status: 400 });
|
||||
}
|
||||
const w = Number(width);
|
||||
const h = Number(height);
|
||||
|
||||
try {
|
||||
const objects = await rpDetect(
|
||||
imageBase64,
|
||||
Number.isFinite(w) && w > 0 ? w : 1,
|
||||
Number.isFinite(h) && h > 0 ? h : 1,
|
||||
);
|
||||
return NextResponse.json({ objects });
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Detection failed.";
|
||||
const status = err instanceof RunpodError ? err.status : 502;
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import { rpDenoiseAudio } from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60; // cold-start denoise worker can take a while
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Audio noise-removal proxy. Accepts base64 WAV (48 kHz mono PCM16, produced
|
||||
* in-browser) and returns a cleaned base64 WAV. Calls the RunPod audio-denoise
|
||||
* endpoint (RUNPOD_AUDIO_DENOISE_URL) — key stays server-side. Used before
|
||||
* transcription on noisy sites.
|
||||
*
|
||||
* POST { audio } -> { audio }
|
||||
* Auth: session-gated. Rate limit: 20/min.
|
||||
*/
|
||||
|
||||
const MAX_BASE64 = 12_000_000; // ~9 MB decoded WAV
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "denoise");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
let body: { audio?: unknown };
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const audio = typeof body.audio === "string" ? body.audio : "";
|
||||
if (!audio) return NextResponse.json({ error: "Missing audio." }, { status: 400 });
|
||||
if (audio.length > MAX_BASE64) {
|
||||
return NextResponse.json({ error: "Audio too long — keep it under ~30s." }, { status: 413 });
|
||||
}
|
||||
|
||||
try {
|
||||
const { audioB64 } = await rpDenoiseAudio(audio);
|
||||
return NextResponse.json({ audio: audioB64 });
|
||||
} catch (e) {
|
||||
const msg = e instanceof RunpodError ? e.message : e instanceof Error ? e.message : "Denoise failed.";
|
||||
return NextResponse.json({ error: msg }, { status: 502 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import {
|
||||
rpImg2Img,
|
||||
rpInpaint,
|
||||
rpRemoveBackground,
|
||||
rpUpscale,
|
||||
} from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60; // SD / cold-start models can take a while
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Generative image-edit proxy. The BACKEND is pluggable — pick one with env
|
||||
* `AI_EDIT_PROVIDER` (default `auto`):
|
||||
*
|
||||
* - `runpod` → RunPod serverless GPU endpoints (one per model). Maps each
|
||||
* op → endpoint: restore/upscale → Real-ESRGAN (#7), colorize
|
||||
* → img2img (#10), replace-sky / magic-eraser / generative-fill
|
||||
* → SD 3.5 masked inpaint (#9), prompt → SD 3.5 img2img (#10).
|
||||
* Env: RUNPOD_API_KEY + per-model RUNPOD_*_URL. Key stays
|
||||
* server-side.
|
||||
* - `local` → your own Stable Diffusion server (Automatic1111 / Forge /
|
||||
* SD.Next img2img API). Env: LOCAL_SD_URL.
|
||||
* - `huggingface` → Hugging Face Inference API. Env: HF_API_TOKEN, HF_IMAGE_MODEL.
|
||||
* - `gemini` → Google Gemini image model (needs a billed key for image output).
|
||||
* Env: GEMINI_API_KEY, GEMINI_IMAGE_MODEL.
|
||||
* - `auto` → first configured of: runpod → local → huggingface → gemini.
|
||||
*
|
||||
* NOTE: `remove-background` runs in-browser by default (@imgly, no key), so it
|
||||
* usually never reaches here. Object detection uses its own route (./classify).
|
||||
*
|
||||
* POST { imageBase64, mimeType?, op, maskBase64?, params? } -> { imageBase64, mimeType }
|
||||
* Auth: session-gated. Rate limit: 12/min (the most expensive route).
|
||||
*/
|
||||
|
||||
const OP_PROMPTS: Record<string, string> = {
|
||||
restore:
|
||||
"Restore and enhance this photograph: improve sharpness and clarity, correct exposure and white balance, reduce noise and compression artifacts, recover detail. Keep it natural and photorealistic.",
|
||||
colorize: "Colorize this image with natural, realistic, well-balanced colors.",
|
||||
"replace-sky":
|
||||
"Replace the sky with a dramatic, beautiful golden-hour sky with soft clouds. Keep the foreground subject unchanged and the result photorealistic.",
|
||||
};
|
||||
|
||||
const MAX_BASE64 = 4_000_000; // ~3 MB decoded — stays under serverless body limits
|
||||
|
||||
type Provider = "runpod" | "local" | "huggingface" | "gemini" | "none";
|
||||
|
||||
function resolveProvider(): Provider {
|
||||
const explicit = (process.env.AI_EDIT_PROVIDER || "auto").toLowerCase();
|
||||
if (
|
||||
explicit === "runpod" ||
|
||||
explicit === "local" ||
|
||||
explicit === "huggingface" ||
|
||||
explicit === "gemini"
|
||||
)
|
||||
return explicit;
|
||||
if (explicit === "none") return "none";
|
||||
// auto: prefer RunPod GPU endpoints, then a private local server, then HF, then Gemini.
|
||||
// Detect RunPod when the key + ANY image endpoint URL is set (an upscale/colorize-only
|
||||
// deployment is valid — not just the SD ones).
|
||||
if (
|
||||
process.env.RUNPOD_API_KEY &&
|
||||
(process.env.RUNPOD_SD_IMG2IMG_URL ||
|
||||
process.env.RUNPOD_SD_INPAINT_URL ||
|
||||
process.env.RUNPOD_UPSCALE_URL ||
|
||||
process.env.RUNPOD_COLORIZE_URL ||
|
||||
process.env.RUNPOD_BG_REMOVE_URL)
|
||||
)
|
||||
return "runpod";
|
||||
if (process.env.LOCAL_SD_URL) return "local";
|
||||
if (process.env.HF_API_TOKEN) return "huggingface";
|
||||
if (process.env.GEMINI_API_KEY) return "gemini";
|
||||
return "none";
|
||||
}
|
||||
|
||||
/** Ops that only the RunPod (mask/fixed-function) backend can serve. */
|
||||
const RUNPOD_ONLY_OPS = new Set(["upscale", "magic-eraser", "generative-fill"]);
|
||||
|
||||
interface EditResult {
|
||||
imageBase64: string;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "edit");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
const provider = resolveProvider();
|
||||
if (provider === "none") {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
"AI image editing is not configured. Set AI_EDIT_PROVIDER=runpod + RUNPOD_API_KEY + the per-model RUNPOD_*_URL vars (RunPod GPU), or LOCAL_SD_URL (own Stable Diffusion), HF_API_TOKEN (Hugging Face), or GEMINI_API_KEY. Background removal and all analysis still work with no key.",
|
||||
},
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid request body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const { imageBase64, mimeType, op, maskBase64, params } = (body ?? {}) as {
|
||||
imageBase64?: unknown;
|
||||
mimeType?: unknown;
|
||||
op?: { type?: string; prompt?: string; factor?: number };
|
||||
maskBase64?: unknown;
|
||||
params?: unknown;
|
||||
};
|
||||
|
||||
if (typeof imageBase64 !== "string" || imageBase64.length === 0) {
|
||||
return NextResponse.json({ error: "Invalid image." }, { status: 400 });
|
||||
}
|
||||
const hasMask = typeof maskBase64 === "string" && maskBase64.length > 0;
|
||||
// Image + mask share one request body — budget them together against the cap.
|
||||
if (imageBase64.length + (hasMask ? (maskBase64 as string).length : 0) > MAX_BASE64) {
|
||||
return NextResponse.json(
|
||||
{ error: "Image (plus mask) is too large — try a smaller image." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const safeMime =
|
||||
typeof mimeType === "string" && /^image\/(jpeg|png|webp)$/.test(mimeType)
|
||||
? mimeType
|
||||
: "image/jpeg";
|
||||
|
||||
const opType = op?.type ?? "";
|
||||
if (provider !== "runpod" && RUNPOD_ONLY_OPS.has(opType)) {
|
||||
return NextResponse.json(
|
||||
{ error: "This edit needs the RunPod backend (set AI_EDIT_PROVIDER=runpod)." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
// Build the instruction from an allow-listed op (never trust arbitrary server prompts).
|
||||
let instruction = "";
|
||||
if (opType === "prompt" || opType === "generative-fill") {
|
||||
const p = typeof op?.prompt === "string" ? op.prompt.trim() : "";
|
||||
if (!p) return NextResponse.json({ error: "Empty prompt." }, { status: 400 });
|
||||
instruction = p.slice(0, 500);
|
||||
} else if (opType === "replace-sky") {
|
||||
instruction =
|
||||
typeof op?.prompt === "string" && op.prompt.trim()
|
||||
? `Replace the sky with: ${op.prompt.trim().slice(0, 300)}. Keep the foreground unchanged and photorealistic.`
|
||||
: OP_PROMPTS["replace-sky"]!;
|
||||
} else if (opType === "magic-eraser") {
|
||||
instruction =
|
||||
"Fill the selected region with a clean, seamless, plausible background. Photorealistic.";
|
||||
} else if (OP_PROMPTS[opType]) {
|
||||
instruction = OP_PROMPTS[opType]!;
|
||||
} else if (opType !== "upscale" && opType !== "remove-background") {
|
||||
return NextResponse.json({ error: "Unsupported operation." }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
let result: EditResult;
|
||||
if (provider === "runpod")
|
||||
result = await editRunPod(
|
||||
op ?? {},
|
||||
imageBase64,
|
||||
instruction,
|
||||
hasMask ? (maskBase64 as string) : undefined,
|
||||
params,
|
||||
);
|
||||
else if (provider === "local") result = await editLocal(instruction, imageBase64);
|
||||
else if (provider === "huggingface") result = await editHuggingFace(instruction, imageBase64);
|
||||
else result = await editGemini(instruction, imageBase64, safeMime);
|
||||
return NextResponse.json(result);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "AI request failed.";
|
||||
const status = err instanceof AiError || err instanceof RunpodError ? err.status : 502;
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backend: RunPod serverless GPU endpoints (one model per endpoint).
|
||||
// Each op maps to its endpoint; the API key + URLs stay server-side.
|
||||
// ---------------------------------------------------------------------------
|
||||
interface SdParams {
|
||||
negativePrompt?: string;
|
||||
strength?: number;
|
||||
steps?: number;
|
||||
seed?: number;
|
||||
guidanceScale?: number;
|
||||
}
|
||||
|
||||
function sanitizeParams(raw: unknown): SdParams {
|
||||
const p = (raw ?? {}) as Record<string, unknown>;
|
||||
const out: SdParams = {};
|
||||
if (typeof p.negativePrompt === "string" && p.negativePrompt.trim())
|
||||
out.negativePrompt = p.negativePrompt.trim().slice(0, 300);
|
||||
const strength = Number(p.strength);
|
||||
if (Number.isFinite(strength)) out.strength = Math.max(0, Math.min(1, strength));
|
||||
const steps = Number(p.steps);
|
||||
if (Number.isFinite(steps)) out.steps = Math.max(1, Math.min(60, Math.round(steps)));
|
||||
const guidance = Number(p.guidanceScale);
|
||||
if (Number.isFinite(guidance)) out.guidanceScale = Math.max(1, Math.min(20, guidance));
|
||||
const seed = Number(p.seed);
|
||||
if (Number.isFinite(seed)) out.seed = Math.max(0, Math.min(2_147_483_647, Math.round(seed)));
|
||||
return out;
|
||||
}
|
||||
|
||||
async function editRunPod(
|
||||
op: { type?: string; prompt?: string; factor?: number },
|
||||
imageBase64: string,
|
||||
instruction: string,
|
||||
maskBase64: string | undefined,
|
||||
rawParams: unknown,
|
||||
): Promise<EditResult> {
|
||||
const params = sanitizeParams(rawParams);
|
||||
switch (op.type) {
|
||||
case "remove-background":
|
||||
// U²-Net via rembg (#6) — a real endpoint replacing the flaky in-browser remover.
|
||||
return rpRemoveBackground(imageBase64);
|
||||
case "restore":
|
||||
// Real-ESRGAN (#7) with the GFPGAN face pass = "Restore & Enhance".
|
||||
return rpUpscale(imageBase64, 4, true);
|
||||
case "upscale":
|
||||
return rpUpscale(imageBase64, op.factor === 4 ? 4 : 2, false);
|
||||
case "colorize":
|
||||
// The dedicated DDColor endpoint kept hard-crashing (modelscope). Route
|
||||
// colorize through the img2img model as an instruction instead.
|
||||
return rpImg2Img({ imageB64: imageBase64, prompt: instruction, ...params });
|
||||
case "prompt":
|
||||
return rpImg2Img({ imageB64: imageBase64, prompt: instruction, ...params }); // SD 3.5 img2img (#10)
|
||||
case "replace-sky":
|
||||
// True sky replacement is masked inpaint (#9). Without a mask (no in-app sky
|
||||
// segmentation yet) degrade to a low-strength img2img (#10) so the foreground
|
||||
// is mostly preserved.
|
||||
if (maskBase64)
|
||||
return rpInpaint({
|
||||
imageB64: imageBase64,
|
||||
maskB64: maskBase64,
|
||||
prompt: instruction,
|
||||
...params,
|
||||
});
|
||||
return rpImg2Img({
|
||||
imageB64: imageBase64,
|
||||
prompt: instruction,
|
||||
...params,
|
||||
strength: params.strength ?? 0.4,
|
||||
});
|
||||
case "magic-eraser":
|
||||
case "generative-fill":
|
||||
// SD 3.5 masked inpaint (#9) — white in the mask = the region to regenerate.
|
||||
if (!maskBase64) throw new AiError("This edit needs a mask/selection.", 400);
|
||||
return rpInpaint({
|
||||
imageB64: imageBase64,
|
||||
maskB64: maskBase64,
|
||||
prompt: instruction,
|
||||
...params,
|
||||
});
|
||||
default:
|
||||
throw new AiError("Unsupported operation.", 400);
|
||||
}
|
||||
}
|
||||
|
||||
class AiError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status = 502) {
|
||||
super(message);
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backend: local Stable Diffusion (Automatic1111 / Forge / SD.Next img2img API)
|
||||
// ---------------------------------------------------------------------------
|
||||
async function editLocal(instruction: string, imageBase64: string): Promise<EditResult> {
|
||||
const base = process.env.LOCAL_SD_URL;
|
||||
if (!base || !/^https?:\/\//i.test(base)) {
|
||||
throw new AiError("LOCAL_SD_URL is not a valid http(s) URL.", 500);
|
||||
}
|
||||
const url = `${base.replace(/\/$/, "")}/sdapi/v1/img2img`;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
init_images: [imageBase64],
|
||||
prompt: instruction,
|
||||
denoising_strength: Number(process.env.LOCAL_SD_DENOISE ?? 0.55),
|
||||
steps: Number(process.env.LOCAL_SD_STEPS ?? 25),
|
||||
cfg_scale: 7,
|
||||
sampler_name: process.env.LOCAL_SD_SAMPLER || "Euler a",
|
||||
}),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
throw new AiError("Could not reach your local Stable Diffusion server (LOCAL_SD_URL).", 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new AiError(`Local SD server error (${res.status}).`, 502);
|
||||
}
|
||||
const data = (await res.json().catch(() => null)) as { images?: string[] } | null;
|
||||
const out = data?.images?.[0];
|
||||
if (!out) throw new AiError("Local SD server did not return an image.", 502);
|
||||
// A1111 returns raw base64 PNG (no data: prefix).
|
||||
return { imageBase64: out.includes(",") ? out.split(",")[1]! : out, mimeType: "image/png" };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backend: Hugging Face Inference API — instruction image editing.
|
||||
// ---------------------------------------------------------------------------
|
||||
async function editHuggingFace(instruction: string, imageBase64: string): Promise<EditResult> {
|
||||
const token = process.env.HF_API_TOKEN;
|
||||
if (!token) throw new AiError("HF_API_TOKEN is not set.", 500);
|
||||
const model = process.env.HF_IMAGE_MODEL || "timbrooks/instruct-pix2pix";
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`https://api-inference.huggingface.co/models/${model}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
"content-type": "application/json",
|
||||
// Wait for the model to warm up instead of a fast 503.
|
||||
"x-wait-for-model": "true",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
inputs: imageBase64,
|
||||
parameters: { prompt: instruction, guidance_scale: 7, image_guidance_scale: 1.5 },
|
||||
}),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
throw new AiError("Could not reach the Hugging Face Inference API.", 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
// Truncated on purpose — never surface a full upstream body.
|
||||
const detail = (await res.text().catch(() => "")).slice(0, 160);
|
||||
if (res.status === 503) throw new AiError("The model is loading — try again in ~20s.", 503);
|
||||
throw new AiError(`Hugging Face error (${res.status}). ${detail}`, 502);
|
||||
}
|
||||
// Success returns raw image bytes.
|
||||
const outMime = res.headers.get("content-type") || "image/png";
|
||||
if (outMime.startsWith("application/json")) {
|
||||
const j = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new AiError(
|
||||
j?.error ? `Hugging Face: ${j.error}` : "Hugging Face returned no image.",
|
||||
502,
|
||||
);
|
||||
}
|
||||
const buf = await res.arrayBuffer();
|
||||
return { imageBase64: Buffer.from(buf).toString("base64"), mimeType: outMime };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backend: Google Gemini image model (needs a billed key for image output).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function editGemini(
|
||||
instruction: string,
|
||||
imageBase64: string,
|
||||
safeMime: string,
|
||||
): Promise<EditResult> {
|
||||
const apiKey = process.env.GEMINI_API_KEY;
|
||||
if (!apiKey) throw new AiError("GEMINI_API_KEY is not set.", 500);
|
||||
const model = process.env.GEMINI_IMAGE_MODEL || "gemini-2.5-flash-image";
|
||||
const prompt = `Edit this image as follows: ${instruction}. Preserve realism unless explicitly asked otherwise.`;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(
|
||||
`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", "x-goog-api-key": apiKey },
|
||||
body: JSON.stringify({
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{ inlineData: { mimeType: safeMime, data: imageBase64 } },
|
||||
{ text: prompt },
|
||||
],
|
||||
},
|
||||
],
|
||||
generationConfig: { responseModalities: ["IMAGE"] },
|
||||
}),
|
||||
cache: "no-store",
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
throw new AiError("Could not reach the AI service.", 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
// Truncated on purpose — never surface a full upstream body.
|
||||
const detail = (await res.text().catch(() => "")).slice(0, 160);
|
||||
throw new AiError(`AI service error (${res.status}). ${detail}`, 502);
|
||||
}
|
||||
const data = (await res.json().catch(() => null)) as GeminiResponse | null;
|
||||
const parts = data?.candidates?.[0]?.content?.parts ?? [];
|
||||
const imgPart = parts.find((p) => p.inlineData?.data || p.inline_data?.data);
|
||||
const out = imgPart?.inlineData?.data ?? imgPart?.inline_data?.data;
|
||||
if (!out)
|
||||
throw new AiError(
|
||||
"The model did not return an image (the free Gemini tier has no image output — use LOCAL_SD_URL or HF_API_TOKEN instead).",
|
||||
502,
|
||||
);
|
||||
const outMime = imgPart?.inlineData?.mimeType ?? imgPart?.inline_data?.mime_type ?? "image/png";
|
||||
return { imageBase64: out, mimeType: outMime };
|
||||
}
|
||||
|
||||
interface GeminiPart {
|
||||
text?: string;
|
||||
inlineData?: { mimeType?: string; data?: string };
|
||||
inline_data?: { mime_type?: string; data?: string };
|
||||
}
|
||||
interface GeminiResponse {
|
||||
candidates?: Array<{ content?: { parts?: GeminiPart[] } }>;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import { rpTilt } from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60; // cold-start tilt worker can take a while
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Camera-tilt estimation proxy. Accepts a base64 image and returns
|
||||
* {rollDegrees, pitchDegrees, fovDegrees} from the RunPod tilt endpoint
|
||||
* (RUNPOD_TILT_URL) so the editor can auto-straighten.
|
||||
*
|
||||
* POST { image } -> { rollDegrees, pitchDegrees, fovDegrees }
|
||||
* Auth: session-gated. Rate limit: 30/min.
|
||||
*/
|
||||
|
||||
const MAX_BASE64 = 4_000_000; // ~3 MB decoded
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "tilt");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
let body: { image?: unknown };
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const image = typeof body.image === "string" ? body.image : "";
|
||||
if (!image) return NextResponse.json({ error: "Missing image." }, { status: 400 });
|
||||
if (image.length > MAX_BASE64) {
|
||||
return NextResponse.json({ error: "Image too large." }, { status: 413 });
|
||||
}
|
||||
|
||||
try {
|
||||
const tilt = await rpTilt(image);
|
||||
return NextResponse.json(tilt);
|
||||
} catch (e) {
|
||||
const msg =
|
||||
e instanceof RunpodError ? e.message : e instanceof Error ? e.message : "Tilt estimate failed.";
|
||||
return NextResponse.json({ error: msg }, { status: 502 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import { rpTranscribe } from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60; // cold-start STT worker can take a while
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Speech-to-text proxy for voice annotations. Accepts base64 WAV (16 kHz mono
|
||||
* PCM16, produced in-browser) and returns the transcript. Calls the RunPod
|
||||
* voice-to-text endpoint (RUNPOD_STT_URL) — the key stays server-side.
|
||||
*
|
||||
* POST { audio, language? } -> { transcript, segments? }
|
||||
* Auth: session-gated. Rate limit: 20/min.
|
||||
*/
|
||||
|
||||
const MAX_BASE64 = 8_000_000; // ~6 MB decoded WAV — stays under serverless body limits
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "transcribe");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
let body: { audio?: unknown; language?: unknown };
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const audio = typeof body.audio === "string" ? body.audio : "";
|
||||
const language = typeof body.language === "string" ? body.language : undefined;
|
||||
if (!audio) return NextResponse.json({ error: "Missing audio." }, { status: 400 });
|
||||
if (audio.length > MAX_BASE64) {
|
||||
return NextResponse.json({ error: "Audio too long — keep it under ~30s." }, { status: 413 });
|
||||
}
|
||||
|
||||
try {
|
||||
const { transcript, segments } = await rpTranscribe(audio, { language, punctuation: true });
|
||||
return NextResponse.json({ transcript, segments });
|
||||
} catch (e) {
|
||||
const msg =
|
||||
e instanceof RunpodError ? e.message : e instanceof Error ? e.message : "Transcription failed.";
|
||||
return NextResponse.json({ error: msg }, { status: 502 });
|
||||
}
|
||||
}
|
||||
@@ -1140,4 +1140,80 @@
|
||||
.dash-root .ai-bubble { max-width: 86%; }
|
||||
.dash-root .ai-view { height: calc(100vh - 150px); }
|
||||
}
|
||||
|
||||
|
||||
/* =========================================================================
|
||||
Smart Gallery — the embedded @photo-gallery/sdk surface.
|
||||
The SDK is themed entirely through the token map in lib/gallery-api.ts
|
||||
(--apg-* -> this file's own vars), so the rules below only handle the
|
||||
host chrome: sizing, the demo banner, and the load/error placeholders.
|
||||
========================================================================= */
|
||||
|
||||
/* The gallery is the one view that wants the whole viewport: it has its own sidebar, toolbar and
|
||||
scrollers, so any height we leave on the table is wasted chrome. The old big PageHead cost ~90px;
|
||||
a slim header (~40px) + compact banner + tight gaps hand almost all of that back to the shell.
|
||||
96px = the dashboard's top padding + the slim header row; `.gal-shell` (flex:1; min-height:0)
|
||||
consumes whatever is left after the header and the optional demo banner. */
|
||||
.dash-root .gal { display: flex; flex-direction: column; gap: 10px; height: calc(100vh - 96px); min-height: 700px; }
|
||||
|
||||
/* Slim inline header — replaces the tall PageHead. One row, ~40px, so the shell keeps the height. */
|
||||
.dash-root .gal-head { display: flex; align-items: center; gap: 10px; min-height: 36px; flex: 0 0 auto; }
|
||||
.dash-root .gal-head-ic { width: 28px; height: 28px; border-radius: 9px; display: grid; place-items: center; color: #fff; background: var(--grad-brand); box-shadow: var(--glow-orange); flex: 0 0 auto; }
|
||||
.dash-root .gal-head-title { font-size: 16px; font-weight: 700; line-height: 1.1; margin: 0; }
|
||||
.dash-root .gal-head-sub { color: var(--muted); font-size: 12.5px; font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; }
|
||||
@media (max-width: 720px) { .dash-root .gal-head-sub { display: none; } }
|
||||
|
||||
/* `.view` sets `z-index: 1`, which makes it a stacking context and traps the SDK's
|
||||
full-screen overlays (lightbox z1000, editors z1050, camera z1080, modals z1100)
|
||||
underneath the topbar's `z-index: 20`. Opting this one view out of the stacking
|
||||
context lets those overlays cover the whole dashboard, as they must. The view still
|
||||
paints above the ambient `.dash-content::before` glow because it follows it in the DOM. */
|
||||
.dash-root .view.gal { z-index: auto; }
|
||||
|
||||
/* Compact single-line demo banner (~34px). Truncates rather than wrapping so it never steals a
|
||||
second row of height from the shell. */
|
||||
.dash-root .gal-banner { display: flex; align-items: center; gap: 8px; min-height: 34px; padding: 6px 12px; border-radius: 11px; border: 1px solid var(--border); background: color-mix(in srgb, var(--orange) 9%, var(--panel-2)); color: var(--text-2); font-size: 12px; font-weight: 500; flex: 0 0 auto; white-space: nowrap; overflow: hidden; }
|
||||
.dash-root .gal-banner span { overflow: hidden; text-overflow: ellipsis; }
|
||||
.dash-root .gal-banner svg { color: var(--orange); flex: 0 0 auto; }
|
||||
|
||||
/* The gallery's own viewport. `overflow: hidden` keeps the SDK's internal scrollers
|
||||
in charge; its full-screen overlays (lightbox/editor/camera) are position:fixed
|
||||
and deliberately escape this box to cover the whole dashboard. */
|
||||
.dash-root .gal-shell { flex: 1; min-height: 0; position: relative; border-radius: 18px; border: 1px solid var(--border); background: var(--panel-2); overflow: hidden; box-shadow: var(--card-hi), 0 1px 2px rgba(0, 0, 0, 0.18); }
|
||||
.dash-root[data-theme="dark"] .gal-shell { border: 0.5px solid #452b1a; border-radius: 20px; }
|
||||
|
||||
/* The SDK's embedded root fills this box. (--apg-overlay-top is set from the
|
||||
component's `style` prop — the SDK writes an inline default that a stylesheet
|
||||
rule could not override.) */
|
||||
.dash-root .gal-shell .apg { height: 100%; }
|
||||
|
||||
/* ---- Fullscreen (the SDK puts `.apg--fullscreen` on its root: position:fixed; inset:0) ----
|
||||
A position:fixed box is only clipped by an ancestor that is its CONTAINING BLOCK, which
|
||||
`overflow`/`border-radius`/`box-shadow` alone never create — only transform / filter /
|
||||
perspective / backdrop-filter / will-change / contain do. Nothing on the path
|
||||
(.dash-content > .view.gal > .gal-shell) uses any of those: `.view`'s `ds-fade` animates
|
||||
opacity only, and `.view.gal` already drops the `z-index: 1` stacking context. So the
|
||||
fullscreen root does escape today — these rules make that survive an edit above. */
|
||||
|
||||
/* `.dash-root .gal-shell .apg` (0,3,0) would otherwise out-specify the SDK's own sizing; with
|
||||
inset:0 driving the box, height must get out of the way. */
|
||||
.dash-root .gal-shell .apg.apg--fullscreen {
|
||||
height: auto;
|
||||
/* Above the topbar (z-index: 20) and the sidebar, below the SDK's own overlays (1000+). */
|
||||
z-index: 900;
|
||||
}
|
||||
|
||||
/* Belt and braces: if a future rule ever DOES make `.gal-shell` a containing block, an
|
||||
`overflow: hidden` on it would crop the fullscreen root to the embedded box. Drop the clip
|
||||
(and the rounded corner it exists to enforce) for exactly as long as fullscreen is on. */
|
||||
.dash-root .gal-shell:has(.apg--fullscreen) { overflow: visible; }
|
||||
|
||||
.dash-root .gal-placeholder { height: 100%; min-height: 320px; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 12px; text-align: center; padding: 28px; }
|
||||
.dash-root .gal-placeholder-ic { width: 66px; height: 66px; border-radius: 20px; display: grid; place-items: center; color: #fff; background: var(--grad-brand); box-shadow: var(--glow-orange); }
|
||||
.dash-root .gal-placeholder p { color: var(--muted); font-size: 13px; max-width: 420px; }
|
||||
.dash-root .gal-placeholder h3 { font-size: 16px; font-weight: 700; }
|
||||
.dash-root .gal-placeholder-error .gal-placeholder-ic { background: color-mix(in srgb, var(--red) 88%, #000); box-shadow: 0 10px 28px -12px color-mix(in srgb, var(--red) 60%, transparent); }
|
||||
|
||||
@media (max-width: 920px) {
|
||||
/* Narrower chrome: a little less top offset, and a smaller floor so short viewports still work. */
|
||||
.dash-root .gal { height: calc(100vh - 84px); min-height: 560px; }
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import { AiAssistant } from "./ai-assistant";
|
||||
import { TeamManagement } from "./team-management";
|
||||
import { Messenger } from "./messenger";
|
||||
import { Inbox } from "./inbox";
|
||||
import { SmartGallery } from "./smart-gallery";
|
||||
import "../../app/dashboard/dashboard.css";
|
||||
|
||||
export function Dashboard() {
|
||||
@@ -49,6 +50,7 @@ export function Dashboard() {
|
||||
: active === "ai" ? <AiAssistant />
|
||||
: active === "messenger" ? <Messenger />
|
||||
: active === "inbox" ? <Inbox />
|
||||
: active === "gallery" ? <SmartGallery theme={theme} />
|
||||
: active === "team" ? <TeamManagement />
|
||||
: <ComingSoon title={title} icon={item?.icon ?? "dashboard"} onGo={setActive} />}
|
||||
</ToastProvider>
|
||||
|
||||
@@ -34,6 +34,7 @@ export const NAV_GROUPS: NavGroup[] = [
|
||||
items: [
|
||||
{ key: "messenger", label: "Messenger", icon: "send", subtitle: "Chat with your team and clients" },
|
||||
{ key: "inbox", label: "Inbox", icon: "bell", subtitle: "Mentions, messages, alerts and mail — all in one" },
|
||||
{ key: "gallery", label: "Smart Gallery", icon: "gallery", subtitle: "Every photo and video for your jobs — searchable, editable and shareable" },
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -79,7 +80,15 @@ export const NAV_ITEMS: NavItem[] = NAV_GROUPS.flatMap((g) => g.items);
|
||||
// brand-new user with no membership); every other item requires membership, and the
|
||||
// items mapped here additionally require the given permission. Unmapped items are
|
||||
// shown to any member. This is UX only — be-crm still enforces every action.
|
||||
const ALWAYS_VISIBLE = new Set(["dashboard", "profile", "messenger", "inbox"]);
|
||||
//
|
||||
// `gallery` DECISION: it stays in ALWAYS_VISIBLE (nav row always shown to members) rather than
|
||||
// being gated here by `media.view`. The real access gate lives INSIDE the view (SmartGallery reads
|
||||
// `useGalleryFeatures().canView`), which uses the permissive "member with zero media.* perms → all
|
||||
// enabled" fallback. Gating the nav row here would use the stricter sidebar rule (member without the
|
||||
// perm → hidden) and so would hide the gallery from freshly-seeded members before an admin has
|
||||
// configured any Media perms — regressing the demo and the common member. So: always-visible row,
|
||||
// real gating in-view. (be-crm enforces data access regardless of what the nav shows.)
|
||||
const ALWAYS_VISIBLE = new Set(["dashboard", "profile", "messenger", "inbox", "gallery"]);
|
||||
const NAV_PERMISSION: Record<string, string | undefined> = {
|
||||
team: "team.manage",
|
||||
people: "team.manage",
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
"use client";
|
||||
|
||||
// ============================================================
|
||||
// The actual @photo-gallery/sdk mount. Split out of smart-gallery.tsx so it can be
|
||||
// loaded with next/dynamic({ ssr: false }) — the SDK is browser-only (matchMedia,
|
||||
// IndexedDB, Leaflet, canvas) and pulls in heavy optional ML models on demand, so it
|
||||
// must stay out of the dashboard's initial bundle and out of the server render.
|
||||
// ============================================================
|
||||
|
||||
import { useMemo } from "react";
|
||||
import { PhotoGallery, type ViewId } from "@photo-gallery/sdk";
|
||||
import { createCrmAIProvider } from "@/lib/gallery-ai";
|
||||
import {
|
||||
GALLERY_THEME_TOKENS,
|
||||
useGalleryFeatures,
|
||||
useGalleryLockProvider,
|
||||
useGalleryStorage,
|
||||
useGalleryUser,
|
||||
} from "@/lib/gallery-api";
|
||||
|
||||
import "@photo-gallery/sdk/styles.css";
|
||||
import "leaflet/dist/leaflet.css";
|
||||
|
||||
export interface SmartGalleryMountProps {
|
||||
/** The dashboard's current appearance — the gallery must never diverge from the host. */
|
||||
theme: "dark" | "light";
|
||||
}
|
||||
|
||||
export default function SmartGalleryMount({ theme }: SmartGalleryMountProps) {
|
||||
const { adapter } = useGalleryStorage();
|
||||
const currentUser = useGalleryUser();
|
||||
// Server-backed Recently Deleted lock when the Shell is wired; `undefined` in demo mode, which
|
||||
// leaves the SDK on its own device-local lock (see useGalleryLockProvider).
|
||||
const lockProvider = useGalleryLockProvider();
|
||||
// Feature toggles resolved from the caller's CRM permissions (Media group). Superadmins/owners and
|
||||
// the demo see everything; see useGalleryFeatures for the safe permissive fallback.
|
||||
const { features } = useGalleryFeatures();
|
||||
|
||||
// Sidebar rows + Collections sections the CRM never wants to surface. The SDK hides both the row and
|
||||
// the matching Collections section for each id. Screenshots + Documents aren't part of a roofing CRM.
|
||||
const hiddenViews: ViewId[] = ["screenshots", "sys:documents"];
|
||||
|
||||
// One provider per mount. Every model is dynamically imported inside it, so constructing
|
||||
// it is cheap; the weight only arrives when a photo is actually analyzed.
|
||||
const ai = useMemo(() => createCrmAIProvider(), []);
|
||||
|
||||
return (
|
||||
<PhotoGallery
|
||||
embedded
|
||||
adapter={adapter}
|
||||
ai={ai}
|
||||
// The CRM owns light/dark, so the in-gallery Appearance switcher is suppressed and the
|
||||
// theme is driven straight off the dashboard's own toggle.
|
||||
theme={theme}
|
||||
chrome={{ titlebar: false, themeSwitcher: false }}
|
||||
themeTokens={GALLERY_THEME_TOKENS}
|
||||
borderRadius={12}
|
||||
currentUser={currentUser}
|
||||
lockProvider={lockProvider}
|
||||
title="Smart Gallery"
|
||||
// The SDK's floating Info panel defaults to 64px from the top — the height of its
|
||||
// OWN toolbar. Inside the dashboard it has to clear the 84px CRM topbar instead.
|
||||
// `style` is applied after the token vars, so this wins over the SDK's inline default.
|
||||
style={{ ["--apg-overlay-top" as string]: "96px" }}
|
||||
hiddenViews={hiddenViews}
|
||||
features={features}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
"use client";
|
||||
|
||||
// ============================================================
|
||||
// Smart Gallery — the tenant's media library, powered by @photo-gallery/sdk embedded
|
||||
// inside the dashboard shell. The SDK owns the gallery experience (grid, lightbox,
|
||||
// photo + video editors, map, people, versions, comments, AI); this file owns the
|
||||
// LynkedUp chrome around it: page head, demo-mode banner, sizing, and failure
|
||||
// containment so a gallery fault can never take the dashboard down.
|
||||
//
|
||||
// Storage + identity come from `@/lib/gallery-api` (be-crm data door when the Shell is
|
||||
// configured, device-local otherwise). Theming comes from GALLERY_THEME_TOKENS, which
|
||||
// maps the SDK's tokens onto this dashboard's own CSS variables.
|
||||
// ============================================================
|
||||
|
||||
import { Component, type ErrorInfo, type ReactNode } from "react";
|
||||
import dynamic from "next/dynamic";
|
||||
import { Icon } from "./ui";
|
||||
import { useGalleryFeatures, useGalleryStorage } from "@/lib/gallery-api";
|
||||
|
||||
const SmartGalleryMount = dynamic(() => import("./smart-gallery-mount"), {
|
||||
ssr: false,
|
||||
loading: () => <GalleryPlaceholder label="Loading your library…" />,
|
||||
});
|
||||
|
||||
export function SmartGallery({ theme }: { theme: "dark" | "light" }) {
|
||||
const { live } = useGalleryStorage();
|
||||
// Whole-view gate. `media.view` with the same permissive fallback the feature toggles use, so a
|
||||
// superadmin/owner and the demo always pass, and a member is only blocked if they hold some Media
|
||||
// perms but not `media.view`. The sidebar keeps the row visible (see sidebar.tsx §4) — the real
|
||||
// gate is here.
|
||||
const { canView } = useGalleryFeatures();
|
||||
|
||||
return (
|
||||
<div className="view gal">
|
||||
{/* Slim header — the big PageHead ate vertical space the gallery needs. The CRM topbar already
|
||||
shows the "Smart Gallery" title; this compact row (~40px) just adds context and the icon. */}
|
||||
<div className="gal-head">
|
||||
<span className="gal-head-ic">
|
||||
<Icon name="gallery" size={16} />
|
||||
</span>
|
||||
<h1 className="gal-head-title">Smart Gallery</h1>
|
||||
<span className="gal-head-sub">Every photo and video for your jobs — searchable, editable and shareable.</span>
|
||||
</div>
|
||||
|
||||
{!live && (
|
||||
<div className="gal-banner">
|
||||
<Icon name="info" size={14} />
|
||||
<span>Demo mode — stored on this device only. It goes live once the Shell + be-crm are connected.</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{canView ? (
|
||||
<div className="gal-shell">
|
||||
<GalleryBoundary>
|
||||
<SmartGalleryMount theme={theme} />
|
||||
</GalleryBoundary>
|
||||
</div>
|
||||
) : (
|
||||
<div className="gal-shell">
|
||||
<div className="gal-placeholder">
|
||||
<span className="gal-placeholder-ic">
|
||||
<Icon name="lock" size={28} />
|
||||
</span>
|
||||
<h3>You don't have access to the gallery</h3>
|
||||
<p>Ask a workspace admin to grant you the “View Smart Gallery” permission.</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
function GalleryPlaceholder({ label }: { label: string }) {
|
||||
return (
|
||||
<div className="gal-placeholder">
|
||||
<span className="gal-placeholder-ic">
|
||||
<Icon name="gallery" size={30} />
|
||||
</span>
|
||||
<p>{label}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface BoundaryState {
|
||||
error: Error | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The gallery is a large third-party surface with canvas, workers and optional ML models. A render
|
||||
* fault inside it must degrade to a message rather than blanking the whole dashboard, so it gets its
|
||||
* own error boundary. (Error boundaries still require a class component in React 19.)
|
||||
*/
|
||||
class GalleryBoundary extends Component<{ children: ReactNode }, BoundaryState> {
|
||||
state: BoundaryState = { error: null };
|
||||
|
||||
static getDerivedStateFromError(error: Error): BoundaryState {
|
||||
return { error };
|
||||
}
|
||||
|
||||
componentDidCatch(error: Error, info: ErrorInfo): void {
|
||||
console.error("[smart-gallery] render failed", error, info.componentStack);
|
||||
}
|
||||
|
||||
render(): ReactNode {
|
||||
const { error } = this.state;
|
||||
if (!error) return this.props.children;
|
||||
return (
|
||||
<div className="gal-placeholder gal-placeholder-error">
|
||||
<span className="gal-placeholder-ic">
|
||||
<Icon name="alert" size={30} />
|
||||
</span>
|
||||
<h3>The gallery could not be displayed</h3>
|
||||
<p>{error.message || "An unexpected error occurred."}</p>
|
||||
<button className="ds-btn v-outline s-sm" onClick={() => this.setState({ error: null })}>
|
||||
<Icon name="refresh" size={14} /> Try again
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,8 @@ import {
|
||||
LayoutDashboard, Building2, FolderKanban, UserPlus, BadgeCheck, Filter,
|
||||
Truck, CloudLightning, Map as MapIcon, PenTool, Calculator, CalendarDays,
|
||||
Trophy, ListChecks, Users, Settings, Sparkles, MoreHorizontal,
|
||||
UsersRound, type LucideIcon,
|
||||
UsersRound, Image as ImageIcon, Images, File, FolderOpen, Download,
|
||||
Video, Play, LayoutGrid, ZoomIn, type LucideIcon,
|
||||
} from "lucide-react";
|
||||
|
||||
/* ---------------------------------------------------------- */
|
||||
@@ -50,6 +51,10 @@ const ICONS: Record<string, LucideIcon> = {
|
||||
estimates: Calculator, schedule: CalendarDays, leaderboard: Trophy,
|
||||
subtasks: ListChecks, people: Users, settings: Settings, ai: Sparkles,
|
||||
team: UsersRound, dots: MoreHorizontal,
|
||||
// media / gallery (also used by messenger.tsx, which already asks for image/file)
|
||||
image: ImageIcon, gallery: Images, file: File, folder: FolderOpen,
|
||||
download: Download, video: Video, play: Play, grid: LayoutGrid,
|
||||
filter: Filter, zoom: ZoomIn, sparkle: Sparkles, "map-pin": MapPin,
|
||||
};
|
||||
|
||||
export function Icon({ name, size = 18, className, strokeWidth = 2 }: { name: string; size?: number; className?: string; strokeWidth?: number }) {
|
||||
|
||||
@@ -0,0 +1,734 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* The CRM's AIProvider for the Smart Gallery.
|
||||
*
|
||||
* PROVENANCE: a port of the SDK demo's `createDemoAIProvider()` plus its
|
||||
* clip/face/ocr/tensorflow/runpod-yolo providers and `imageEncode.ts` helpers
|
||||
* (advance-photo-gallery-web-sdk/apps/web/src/lib/ai/*), collapsed into one
|
||||
* module and retargeted from `/api/ai/*` to `/api/gallery/ai/*`.
|
||||
*
|
||||
* Capability split:
|
||||
* - object detection: TensorFlow.js COCO-SSD, fully in-browser (no key), or the
|
||||
* RunPod YOLO classifier via /api/gallery/ai/classify when
|
||||
* NEXT_PUBLIC_APG_RUNPOD_DETECT=true (COCO-SSD is the automatic fallback)
|
||||
* - face detection + recognition: face-api.js in-browser → clustered into People
|
||||
* - OCR: tesseract.js in-browser → searchable text + the Documents album
|
||||
* - semantic search: CLIP via transformers.js in-browser
|
||||
* - background removal: @imgly in-browser WASM, or the RunPod U²-Net endpoint
|
||||
* when NEXT_PUBLIC_APG_RUNPOD_BG=true (in-browser is the fallback)
|
||||
* - other generative edits / transcription / denoise / tilt: proxied through the
|
||||
* server routes so the RunPod key never reaches the browser
|
||||
*
|
||||
* EVERY heavy model is behind `await import(...)` so none of it lands in the
|
||||
* initial bundle, and every capability degrades to []/''/null with a
|
||||
* console.warn rather than throwing — a failed model must never break the
|
||||
* gallery UI.
|
||||
*
|
||||
* The in-browser models fetch weights from public CDNs (jsdelivr, huggingface,
|
||||
* storage.googleapis.com, staticimgly.com). See docs/SMART_GALLERY.md for the
|
||||
* list that would need CSP allow-listing.
|
||||
*/
|
||||
|
||||
import type { AIProvider, GenerativeEditOp, MediaItem } from "@photo-gallery/sdk";
|
||||
|
||||
// Derived from the provider interface so we import only the three public types.
|
||||
type DetectedObject = Awaited<ReturnType<NonNullable<AIProvider["detectObjects"]>>>[number];
|
||||
type DetectedFace = Awaited<ReturnType<NonNullable<AIProvider["detectFaces"]>>>[number];
|
||||
type ImageSource = ImageBitmap | HTMLImageElement;
|
||||
|
||||
const API = "/api/gallery/ai";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// imageEncode helpers (ported from apps/web/src/lib/ai/imageEncode.ts)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface EncodedImage {
|
||||
/** base64 JPEG (no data: prefix). */
|
||||
data: string;
|
||||
mimeType: string;
|
||||
/** Actual pixel dims of the encoded image (after downscale). */
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/** Draw an image to a downscaled canvas and return base64 JPEG + its dims. */
|
||||
export function imageToBase64(image: ImageSource, maxDim: number): EncodedImage {
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||||
const scale = Math.min(1, maxDim / Math.max(w, h));
|
||||
const cw = Math.max(1, Math.round(w * scale));
|
||||
const ch = Math.max(1, Math.round(h * scale));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = cw;
|
||||
canvas.height = ch;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) throw new Error("Canvas not supported.");
|
||||
ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
|
||||
const dataUrl = canvas.toDataURL("image/jpeg", 0.9);
|
||||
return { data: dataUrl.split(",")[1] ?? "", mimeType: "image/jpeg", width: cw, height: ch };
|
||||
}
|
||||
|
||||
/**
|
||||
* Rasterize a mask (ImageData, white = region to regenerate) to a PNG base64
|
||||
* scaled to targetW×targetH so it matches the encoded image exactly — SD 3.5
|
||||
* requires image and mask to be identical pixel sizes.
|
||||
*/
|
||||
export function maskToBase64(mask: ImageData, targetW: number, targetH: number): string {
|
||||
const tmp = document.createElement("canvas");
|
||||
tmp.width = mask.width;
|
||||
tmp.height = mask.height;
|
||||
const tctx = tmp.getContext("2d");
|
||||
if (!tctx) throw new Error("Canvas not supported.");
|
||||
tctx.putImageData(mask, 0, 0);
|
||||
|
||||
const out = document.createElement("canvas");
|
||||
out.width = targetW;
|
||||
out.height = targetH;
|
||||
const octx = out.getContext("2d");
|
||||
if (!octx) throw new Error("Canvas not supported.");
|
||||
// Nearest-neighbour, not bilinear — keep the mask strictly binary so SD gets
|
||||
// crisp white(regenerate)/black(keep) edges instead of an anti-aliased grey halo.
|
||||
octx.imageSmoothingEnabled = false;
|
||||
octx.drawImage(tmp, 0, 0, targetW, targetH);
|
||||
return out.toDataURL("image/png").split(",")[1] ?? "";
|
||||
}
|
||||
|
||||
export function base64ToBlob(base64: string, mime: string): Blob {
|
||||
const bin = atob(base64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||
return new Blob([bytes], { type: mime });
|
||||
}
|
||||
|
||||
/**
|
||||
* Pad an image with a neutral border for outpaint and return {imageBase64, maskBase64}
|
||||
* as base64 PNG — the border is WHITE in the mask (regenerate), the original image
|
||||
* area BLACK (keep). Capped at 1280px on the long side.
|
||||
*/
|
||||
export function padForOutpaint(
|
||||
image: ImageSource,
|
||||
factor: number,
|
||||
): { imageBase64: string; maskBase64: string } {
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||||
const f = Math.max(1.1, Math.min(2, factor));
|
||||
const maxDim = 1280;
|
||||
let pw = Math.round(w * f);
|
||||
let ph = Math.round(h * f);
|
||||
const scale = Math.min(1, maxDim / Math.max(pw, ph));
|
||||
pw = Math.max(16, Math.round(pw * scale));
|
||||
ph = Math.max(16, Math.round(ph * scale));
|
||||
const iw = Math.max(1, Math.round(w * scale));
|
||||
const ih = Math.max(1, Math.round(h * scale));
|
||||
const ox = Math.floor((pw - iw) / 2);
|
||||
const oy = Math.floor((ph - ih) / 2);
|
||||
|
||||
const imgCanvas = document.createElement("canvas");
|
||||
imgCanvas.width = pw;
|
||||
imgCanvas.height = ph;
|
||||
const ictx = imgCanvas.getContext("2d");
|
||||
if (!ictx) throw new Error("Canvas not supported.");
|
||||
// Fill the new border with a blurred, stretched copy of the photo so the model
|
||||
// has real color/context to continue from — flat gray gives it nothing.
|
||||
ictx.filter = "blur(28px)";
|
||||
ictx.drawImage(image as CanvasImageSource, 0, 0, pw, ph);
|
||||
ictx.filter = "none";
|
||||
ictx.drawImage(image as CanvasImageSource, ox, oy, iw, ih);
|
||||
|
||||
const maskCanvas = document.createElement("canvas");
|
||||
maskCanvas.width = pw;
|
||||
maskCanvas.height = ph;
|
||||
const mctx = maskCanvas.getContext("2d");
|
||||
if (!mctx) throw new Error("Canvas not supported.");
|
||||
mctx.fillStyle = "#ffffff";
|
||||
mctx.fillRect(0, 0, pw, ph);
|
||||
mctx.fillStyle = "#000000";
|
||||
mctx.fillRect(ox, oy, iw, ih);
|
||||
|
||||
return {
|
||||
imageBase64: imgCanvas.toDataURL("image/png").split(",")[1] ?? "",
|
||||
maskBase64: maskCanvas.toDataURL("image/png").split(",")[1] ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
/** Draw an image to a canvas (downscaled) and return a JPEG Blob. */
|
||||
function canvasBlob(image: ImageSource, maxDim: number): Promise<Blob> {
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||||
const scale = Math.min(1, maxDim / Math.max(w, h));
|
||||
const cw = Math.max(1, Math.round(w * scale));
|
||||
const ch = Math.max(1, Math.round(h * scale));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = cw;
|
||||
canvas.height = ch;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return Promise.reject(new Error("Canvas not supported."));
|
||||
ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
|
||||
return new Promise((resolve, reject) =>
|
||||
canvas.toBlob((b) => (b ? resolve(b) : reject(new Error("toBlob failed"))), "image/jpeg", 0.92),
|
||||
);
|
||||
}
|
||||
|
||||
function clamp01(n: number): number {
|
||||
return n < 0 ? 0 : n > 1 ? 1 : n;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Object detection — TensorFlow.js COCO-SSD, in-browser
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface CocoPrediction {
|
||||
bbox: [number, number, number, number];
|
||||
class: string;
|
||||
score: number;
|
||||
}
|
||||
interface CocoModel {
|
||||
detect(
|
||||
img: HTMLImageElement | HTMLCanvasElement | HTMLVideoElement,
|
||||
maxNumBoxes?: number,
|
||||
minScore?: number,
|
||||
): Promise<CocoPrediction[]>;
|
||||
}
|
||||
|
||||
let cocoPromise: Promise<CocoModel | null> | null = null;
|
||||
|
||||
/** Load tfjs + COCO-SSD exactly once; resolves to null if anything fails. */
|
||||
function ensureCoco(): Promise<CocoModel | null> {
|
||||
cocoPromise ??= (async () => {
|
||||
try {
|
||||
const tf = await import("@tensorflow/tfjs");
|
||||
try {
|
||||
await tf.setBackend("webgl");
|
||||
} catch {
|
||||
// Fall back to the default backend if WebGL is unavailable.
|
||||
}
|
||||
await tf.ready();
|
||||
const cocoSsd = await import("@tensorflow-models/coco-ssd");
|
||||
return (await cocoSsd.load({ base: "lite_mobilenet_v2" })) as unknown as CocoModel;
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] COCO-SSD load failed; object detection disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return cocoPromise;
|
||||
}
|
||||
|
||||
async function detectObjectsInBrowser(
|
||||
item: MediaItem,
|
||||
image: ImageSource,
|
||||
): Promise<DetectedObject[]> {
|
||||
const model = await ensureCoco();
|
||||
if (!model) return [];
|
||||
try {
|
||||
const el = image as HTMLImageElement;
|
||||
const w = el.naturalWidth || el.width || item.width || 1;
|
||||
const h = el.naturalHeight || el.height || item.height || 1;
|
||||
const predictions = await model.detect(el, 20, 0.4);
|
||||
return predictions.map((p) => ({
|
||||
label: p.class,
|
||||
confidence: p.score,
|
||||
box: { x: p.bbox[0] / w, y: p.bbox[1] / h, width: p.bbox[2] / w, height: p.bbox[3] / h },
|
||||
})) as DetectedObject[];
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] object detection failed.", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-side YOLO detection via /api/gallery/ai/classify, with the in-browser
|
||||
* COCO-SSD as the automatic fallback so detection never hard-fails.
|
||||
*/
|
||||
async function detectObjectsViaRunpod(
|
||||
item: MediaItem,
|
||||
image: ImageSource,
|
||||
): Promise<DetectedObject[]> {
|
||||
try {
|
||||
const { data, mimeType, width, height } = imageToBase64(image, 1280);
|
||||
const res = await fetch(`${API}/classify`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ imageBase64: data, mimeType, width, height }),
|
||||
});
|
||||
if (!res.ok) throw new Error(`classify failed (${res.status})`);
|
||||
const { objects } = (await res.json()) as { objects?: DetectedObject[] };
|
||||
if (Array.isArray(objects)) return objects;
|
||||
throw new Error("classify returned no objects");
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] RunPod detection failed; falling back to COCO-SSD.", err);
|
||||
return detectObjectsInBrowser(item, image);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Faces — @vladmandic/face-api, in-browser (128-D descriptors → People)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const FACE_MODEL_URL = "https://cdn.jsdelivr.net/npm/@vladmandic/face-api@1.7.15/model";
|
||||
|
||||
type FaceApi = typeof import("@vladmandic/face-api");
|
||||
|
||||
let facePromise: Promise<FaceApi | null> | null = null;
|
||||
|
||||
function ensureFaceModels(): Promise<FaceApi | null> {
|
||||
facePromise ??= (async () => {
|
||||
try {
|
||||
const faceapi = await import("@vladmandic/face-api");
|
||||
// The bundled tf re-export is typed narrowly; backend control lives on the
|
||||
// runtime object. Prefer WebGL (no eval; CSP-friendly), fall back gracefully.
|
||||
const tf = faceapi.tf as unknown as {
|
||||
setBackend: (b: string) => Promise<boolean>;
|
||||
ready: () => Promise<void>;
|
||||
};
|
||||
try {
|
||||
await tf.setBackend("webgl");
|
||||
} catch {
|
||||
/* keep default backend */
|
||||
}
|
||||
await tf.ready();
|
||||
await Promise.all([
|
||||
faceapi.nets.tinyFaceDetector.loadFromUri(FACE_MODEL_URL),
|
||||
faceapi.nets.faceLandmark68Net.loadFromUri(FACE_MODEL_URL),
|
||||
faceapi.nets.faceRecognitionNet.loadFromUri(FACE_MODEL_URL),
|
||||
]);
|
||||
return faceapi;
|
||||
} catch (err) {
|
||||
// Degrade gracefully — People simply stays empty if models can't load.
|
||||
console.warn("[gallery-ai] face model load failed; face clustering disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return facePromise;
|
||||
}
|
||||
|
||||
let faceWarned = false;
|
||||
|
||||
async function detectFaces(item: MediaItem, image: ImageSource): Promise<DetectedFace[]> {
|
||||
const faceapi = await ensureFaceModels();
|
||||
if (!faceapi) return [];
|
||||
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width || 1;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height || 1;
|
||||
|
||||
type TNetInput = Parameters<FaceApi["detectAllFaces"]>[0];
|
||||
|
||||
let results;
|
||||
try {
|
||||
results = await faceapi
|
||||
.detectAllFaces(
|
||||
image as unknown as TNetInput,
|
||||
new faceapi.TinyFaceDetectorOptions({ inputSize: 416, scoreThreshold: 0.5 }),
|
||||
)
|
||||
.withFaceLandmarks()
|
||||
.withFaceDescriptors();
|
||||
} catch (err) {
|
||||
if (!faceWarned) {
|
||||
faceWarned = true;
|
||||
console.warn("[gallery-ai] face detection failed on", item.name, err);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
return results.map((r) => {
|
||||
const b = r.detection.box;
|
||||
return {
|
||||
confidence: r.detection.score,
|
||||
box: {
|
||||
x: clamp01(b.x / w),
|
||||
y: clamp01(b.y / h),
|
||||
width: clamp01(b.width / w),
|
||||
height: clamp01(b.height / h),
|
||||
},
|
||||
embedding: Array.from(r.descriptor),
|
||||
};
|
||||
}) as DetectedFace[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OCR — tesseract.js, in-browser
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Must equal the EXACT tesseract.js version in package.json (pinned, no caret)
|
||||
// so the worker CDN URL can never drift from the installed main-thread code.
|
||||
const TESSERACT_VERSION = "5.1.1";
|
||||
const WORKER_PATH = `https://cdn.jsdelivr.net/npm/tesseract.js@${TESSERACT_VERSION}/dist/worker.min.js`;
|
||||
const CORE_PATH = "https://cdn.jsdelivr.net/npm/tesseract.js-core@5";
|
||||
// jsDelivr's GitHub mirror of naptha/tessdata (same files as projectnaptha.com),
|
||||
// so every asset comes from ONE host that a CSP can allow-list.
|
||||
const LANG_PATH = "https://cdn.jsdelivr.net/gh/naptha/tessdata@gh-pages/4.0.0";
|
||||
|
||||
interface OcrWord {
|
||||
text?: string;
|
||||
confidence?: number;
|
||||
}
|
||||
interface OcrData {
|
||||
text?: string;
|
||||
confidence?: number;
|
||||
words?: OcrWord[];
|
||||
blocks?: Array<{ paragraphs?: Array<{ lines?: Array<{ words?: OcrWord[] }> }> }> | null;
|
||||
}
|
||||
|
||||
type TesseractWorker = import("tesseract.js").Worker;
|
||||
|
||||
let ocrWorkerPromise: Promise<TesseractWorker | null> | null = null;
|
||||
|
||||
function ensureOcrWorker(): Promise<TesseractWorker | null> {
|
||||
ocrWorkerPromise ??= (async () => {
|
||||
try {
|
||||
const { createWorker } = await import("tesseract.js");
|
||||
// v5: createWorker(langs, oem, options) already loads + initializes the
|
||||
// language internally — do NOT call the removed v4 worker.load().
|
||||
return await createWorker("eng", 1, {
|
||||
workerPath: WORKER_PATH,
|
||||
corePath: CORE_PATH,
|
||||
langPath: LANG_PATH,
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] tesseract worker init failed; OCR disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return ocrWorkerPromise;
|
||||
}
|
||||
|
||||
const WORD_CONFIDENCE = 70; // a word tesseract is actually sure about
|
||||
const MIN_WORDS = 4; // need several confident words to call it a document
|
||||
const MIN_CHARS = 10;
|
||||
|
||||
function collectWords(data: OcrData): OcrWord[] {
|
||||
if (Array.isArray(data.words) && data.words.length) return data.words;
|
||||
const out: OcrWord[] = [];
|
||||
for (const b of data.blocks ?? [])
|
||||
for (const p of b.paragraphs ?? [])
|
||||
for (const l of p.lines ?? []) for (const w of l.words ?? []) out.push(w);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return real text or '' (not a document). tesseract hallucinates low-confidence
|
||||
* gibberish for photos with no text, so we keep only high-confidence, word-shaped
|
||||
* tokens and require several of them.
|
||||
*/
|
||||
function meaningfulText(data: OcrData): string {
|
||||
const words = collectWords(data);
|
||||
if (words.length > 0) {
|
||||
const good = words.filter(
|
||||
(w) => (w.confidence ?? 0) >= WORD_CONFIDENCE && /[A-Za-z0-9]{2,}/.test(w.text ?? ""),
|
||||
);
|
||||
const text = good
|
||||
.map((w) => (w.text ?? "").trim())
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
.trim();
|
||||
return good.length >= MIN_WORDS && text.length >= MIN_CHARS ? text : "";
|
||||
}
|
||||
// Fallback: overall confidence + count of word-shaped tokens.
|
||||
const raw = (data.text ?? "").trim();
|
||||
const conf = typeof data.confidence === "number" ? data.confidence : 0;
|
||||
const realWords = raw.match(/[A-Za-z]{3,}/g) ?? [];
|
||||
return conf >= 72 && realWords.length >= 6 ? raw : "";
|
||||
}
|
||||
|
||||
async function ocr(_item: MediaItem, image: ImageSource): Promise<string> {
|
||||
const worker = await ensureOcrWorker();
|
||||
if (!worker) return "";
|
||||
try {
|
||||
// Request the block hierarchy so per-word confidence is available.
|
||||
const { data } = (await worker.recognize(
|
||||
image as unknown as HTMLImageElement,
|
||||
{},
|
||||
{ text: true, blocks: true },
|
||||
)) as { data: OcrData };
|
||||
return meaningfulText(data);
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] OCR failed.", err);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Semantic search — CLIP via transformers.js (ONNX-WASM), in-browser
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const CLIP_MODEL_ID = "Xenova/clip-vit-base-patch16";
|
||||
|
||||
type Transformers = typeof import("@huggingface/transformers");
|
||||
|
||||
let transformersMod: Transformers | null = null;
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any -- transformers.js pipelines are untyped */
|
||||
let clipVisionPromise: Promise<{ processor: any; model: any } | null> | null = null;
|
||||
let clipTextPromise: Promise<{ tokenizer: any; model: any } | null> | null = null;
|
||||
|
||||
async function loadTransformers(): Promise<Transformers> {
|
||||
if (!transformersMod) {
|
||||
transformersMod = await import("@huggingface/transformers");
|
||||
// Remote-only (models from the HF CDN); rely on browser cache between sessions.
|
||||
transformersMod.env.allowLocalModels = false;
|
||||
}
|
||||
return transformersMod;
|
||||
}
|
||||
|
||||
function ensureClipVision() {
|
||||
clipVisionPromise ??= (async () => {
|
||||
try {
|
||||
const tf = await loadTransformers();
|
||||
const [processor, model] = await Promise.all([
|
||||
tf.AutoProcessor.from_pretrained(CLIP_MODEL_ID),
|
||||
tf.CLIPVisionModelWithProjection.from_pretrained(CLIP_MODEL_ID),
|
||||
]);
|
||||
return { processor, model };
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] CLIP vision load failed; semantic search disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return clipVisionPromise;
|
||||
}
|
||||
|
||||
function ensureClipText() {
|
||||
clipTextPromise ??= (async () => {
|
||||
try {
|
||||
const tf = await loadTransformers();
|
||||
const [tokenizer, model] = await Promise.all([
|
||||
tf.AutoTokenizer.from_pretrained(CLIP_MODEL_ID),
|
||||
tf.CLIPTextModelWithProjection.from_pretrained(CLIP_MODEL_ID),
|
||||
]);
|
||||
return { tokenizer, model };
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] CLIP text load failed; semantic search disabled.", err);
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
return clipTextPromise;
|
||||
}
|
||||
|
||||
/** Draw an image onto a canvas (downscaled) for the CLIP image processor. */
|
||||
function toCanvas(image: ImageSource, maxDim = 384): HTMLCanvasElement {
|
||||
const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
|
||||
const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
|
||||
const scale = Math.min(1, maxDim / Math.max(w, h));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(w * scale));
|
||||
canvas.height = Math.max(1, Math.round(h * scale));
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) throw new Error("Canvas not supported.");
|
||||
ctx.drawImage(image as CanvasImageSource, 0, 0, canvas.width, canvas.height);
|
||||
return canvas;
|
||||
}
|
||||
|
||||
function tensorToArray(t: any): number[] {
|
||||
const data: Float32Array = t?.data ?? t;
|
||||
return Array.from(data as ArrayLike<number>);
|
||||
}
|
||||
|
||||
async function embedImage(_item: MediaItem, image: ImageSource): Promise<number[]> {
|
||||
const v = await ensureClipVision();
|
||||
if (!v) return [];
|
||||
try {
|
||||
const tf = await loadTransformers();
|
||||
const canvas = toCanvas(image);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return [];
|
||||
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
const raw = new tf.RawImage(imageData.data, canvas.width, canvas.height, 4).rgb();
|
||||
const inputs = await v.processor(raw);
|
||||
const out = await v.model(inputs);
|
||||
return tensorToArray(out.image_embeds);
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] embedImage failed.", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function embedText(query: string): Promise<number[]> {
|
||||
const t = await ensureClipText();
|
||||
if (!t) return [];
|
||||
try {
|
||||
const inputs = t.tokenizer([query], { padding: true, truncation: true });
|
||||
const out = await t.model(inputs);
|
||||
return tensorToArray(out.text_embeds);
|
||||
} catch (err) {
|
||||
console.warn("[gallery-ai] embedText failed.", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The provider
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** `true` only for the literal string "true", matching the SDK demo's semantics. */
|
||||
function flag(v: string | undefined): boolean {
|
||||
return v === "true";
|
||||
}
|
||||
|
||||
export function createCrmAIProvider(): AIProvider {
|
||||
// NEXT_PUBLIC_* are inlined at build time, so these must be read as full
|
||||
// static member expressions — do NOT refactor to dynamic indexing.
|
||||
const useRunpodDetect = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_DETECT);
|
||||
const useRunpodBg = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_BG);
|
||||
const useRunpodTilt = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_TILT);
|
||||
|
||||
return {
|
||||
name: "crm-ai (coco-ssd/yolo + face-api + tesseract + clip + runpod-edit)",
|
||||
|
||||
detectObjects: useRunpodDetect ? detectObjectsViaRunpod : detectObjectsInBrowser,
|
||||
detectFaces,
|
||||
ocr,
|
||||
embedImage,
|
||||
embedText,
|
||||
|
||||
async generativeEdit(item: MediaItem, image: ImageSource, op: GenerativeEditOp) {
|
||||
// Remove Background runs fully in-browser (no key) via @imgly — works even
|
||||
// with no backend. Other ops go through the /api/gallery/ai/edit route.
|
||||
if (op.type === "remove-background") {
|
||||
// Prefer the RunPod U²-Net endpoint when enabled; fall back to in-browser
|
||||
// @imgly if it's off or the request fails, so this always produces a result.
|
||||
if (useRunpodBg) {
|
||||
try {
|
||||
const { data, mimeType } = imageToBase64(image, 1600);
|
||||
const res = await fetch(`${API}/edit`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
imageBase64: data,
|
||||
mimeType,
|
||||
op: { type: "remove-background" },
|
||||
params: {},
|
||||
}),
|
||||
});
|
||||
if (res.ok) {
|
||||
const { imageBase64: out, mimeType: outMime } = (await res.json()) as {
|
||||
imageBase64: string;
|
||||
mimeType?: string;
|
||||
};
|
||||
return base64ToBlob(out, outMime || "image/png");
|
||||
}
|
||||
} catch {
|
||||
/* fall through to the in-browser remover */
|
||||
}
|
||||
}
|
||||
const inputBlob = await canvasBlob(image, 1600);
|
||||
const { removeBackground } = await import("@imgly/background-removal");
|
||||
return removeBackground(inputBlob, { output: { format: "image/png" } });
|
||||
}
|
||||
|
||||
// Outpaint / expand-canvas: pad the image with a neutral border, mark that
|
||||
// border WHITE in the mask, and run it through the same inpaint path as
|
||||
// generative-fill — no extra backend route needed.
|
||||
if (op.type === "outpaint") {
|
||||
const { imageBase64: padded, maskBase64: border } = padForOutpaint(
|
||||
image,
|
||||
typeof op.factor === "number" ? op.factor : 1.5,
|
||||
);
|
||||
const outParams: Record<string, unknown> = {
|
||||
strength: typeof op.strength === "number" ? op.strength : 0.85,
|
||||
};
|
||||
const outRes = await fetch(`${API}/edit`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
imageBase64: padded,
|
||||
mimeType: "image/png",
|
||||
op: {
|
||||
type: "generative-fill",
|
||||
prompt:
|
||||
op.prompt ||
|
||||
"Extend and continue the scene naturally, matching lighting, colors and perspective.",
|
||||
},
|
||||
maskBase64: border,
|
||||
params: outParams,
|
||||
}),
|
||||
});
|
||||
if (!outRes.ok) {
|
||||
const err = (await outRes.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `AI request failed (${outRes.status}).`);
|
||||
}
|
||||
const outJson = (await outRes.json()) as { imageBase64: string; mimeType?: string };
|
||||
return base64ToBlob(outJson.imageBase64, outJson.mimeType || "image/png");
|
||||
}
|
||||
|
||||
const { data, mimeType, width, height } = imageToBase64(image, 1280);
|
||||
// Masked ops carry an ImageData mask — rasterize it to a PNG matched to the
|
||||
// (downscaled) image dims, and strip it from the op since ImageData is not
|
||||
// JSON-serializable.
|
||||
const maskBase64 = "mask" in op ? maskToBase64(op.mask, width, height) : undefined;
|
||||
const wireOp: Record<string, unknown> = { type: op.type };
|
||||
if ("prompt" in op && typeof op.prompt === "string") wireOp.prompt = op.prompt;
|
||||
if ("factor" in op && typeof op.factor === "number") wireOp.factor = op.factor;
|
||||
|
||||
// Forward the "edit strength" slider (0..1) so the backend can scale the edit.
|
||||
const params: Record<string, unknown> = {};
|
||||
if ("strength" in op && typeof op.strength === "number") params.strength = op.strength;
|
||||
|
||||
const res = await fetch(`${API}/edit`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ imageBase64: data, mimeType, op: wireOp, maskBase64, params }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `AI request failed (${res.status}).`);
|
||||
}
|
||||
const { imageBase64, mimeType: outMime } = (await res.json()) as {
|
||||
imageBase64: string;
|
||||
mimeType?: string;
|
||||
};
|
||||
return base64ToBlob(imageBase64, outMime || "image/png");
|
||||
},
|
||||
|
||||
// Voice annotation: record → (optional denoise) → transcribe. Both proxy
|
||||
// through server routes so the RunPod key stays server-side.
|
||||
async transcribeAudio(audioBase64: string) {
|
||||
const res = await fetch(`${API}/transcribe`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ audio: audioBase64 }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `Transcription failed (${res.status}).`);
|
||||
}
|
||||
const { transcript } = (await res.json()) as { transcript?: string };
|
||||
return (transcript ?? "").trim();
|
||||
},
|
||||
|
||||
async denoiseAudio(audioBase64: string) {
|
||||
const res = await fetch(`${API}/denoise`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ audio: audioBase64 }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `Denoise failed (${res.status}).`);
|
||||
}
|
||||
const { audio } = (await res.json()) as { audio?: string };
|
||||
return audio ?? audioBase64;
|
||||
},
|
||||
|
||||
// Camera-tilt estimation is opt-in (needs the RunPod tilt endpoint deployed);
|
||||
// gate it so the editor's Auto-straighten button only appears when configured.
|
||||
estimateTilt: useRunpodTilt
|
||||
? async (_item: MediaItem, image: ImageSource) => {
|
||||
const { data } = imageToBase64(image, 1024);
|
||||
const res = await fetch(`${API}/tilt`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ image: data }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({}))) as { error?: string };
|
||||
throw new Error(err.error || `Tilt estimate failed (${res.status}).`);
|
||||
}
|
||||
return (await res.json()) as {
|
||||
rollDegrees: number;
|
||||
pitchDegrees: number;
|
||||
fovDegrees: number;
|
||||
};
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
"use client";
|
||||
|
||||
// Smart Gallery data layer. Serves EITHER a device-local mock (when the Shell isn't configured — the
|
||||
// demo keeps working offline) OR the live be-crm data door (crm.gallery.*), behind one StorageAdapter
|
||||
// so the embedded @photo-gallery/sdk is mode-agnostic. Tenant scoping, comment authorship and byte
|
||||
// authorization are all enforced server-side; this is just glue.
|
||||
//
|
||||
// Live contract (be-crm):
|
||||
// query crm.gallery.state.load {} -> PersistedState
|
||||
// cmd crm.gallery.state.apply StateChanges -> { upserted, removed }
|
||||
// cmd crm.gallery.media.presignUpload { mediaId, mime, sizeBytes, filename? } -> { ref, uploadUrl, method, headers? }
|
||||
// cmd crm.gallery.media.presignDownload { refs: string[] } -> { urls, expiresInSeconds }
|
||||
// query crm.gallery.stats {} -> { items, albums, people, bytes }
|
||||
// query crm.gallery.lock.status {} -> { hasPassword }
|
||||
// cmd crm.gallery.lock.set { password: string | null } -> { hasPassword }
|
||||
// cmd crm.gallery.lock.verify { password: string } -> { ok }
|
||||
//
|
||||
// BYTES NEVER PASS THROUGH be-crm OR THE BFF. `putMedia` mints a short-lived signed PUT URL and the
|
||||
// browser transfers straight to object storage — the same rule media-api.ts follows for attachments.
|
||||
|
||||
import { useMemo } from "react";
|
||||
import { useAppShell, useAuth } from "@abe-kap/appshell-sdk/react";
|
||||
import {
|
||||
createLocalStorageAdapter,
|
||||
type GalleryFeatures,
|
||||
type GalleryUser,
|
||||
type MediaItem,
|
||||
type PersistedState,
|
||||
type StateChanges,
|
||||
type StorageAdapter,
|
||||
type StoredBlob,
|
||||
type ThemeTokens,
|
||||
} from "@photo-gallery/sdk";
|
||||
import { user as demoUser } from "@/components/dashboard/account-data";
|
||||
import { useMyAccess } from "@/lib/access";
|
||||
import { isShellConfigured } from "./appshell";
|
||||
|
||||
const SHELL = isShellConfigured();
|
||||
|
||||
/** Device-local store key for demo mode. Namespaced so it can't collide with the SDK's own demo. */
|
||||
const LOCAL_STORE_KEY = "lup:smart-gallery:v1";
|
||||
|
||||
/** be-crm caps a single presignDownload at 500 refs — chunk anything larger. */
|
||||
const PRESIGN_CHUNK = 500;
|
||||
|
||||
/** Largest single upload the gallery will attempt (matches be-crm's gallery cap). */
|
||||
export const MAX_GALLERY_BYTES = 200 * 1024 * 1024;
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Wire types (be-crm shapes) */
|
||||
/* ======================================================================== */
|
||||
|
||||
interface StateLoadDTO {
|
||||
media: MediaItem[];
|
||||
albums: PersistedState["albums"];
|
||||
people: PersistedState["people"];
|
||||
labelAliases?: Record<string, string>;
|
||||
deletedLabels?: string[];
|
||||
}
|
||||
|
||||
interface PresignUploadDTO {
|
||||
ref: string;
|
||||
uploadUrl: string;
|
||||
method: "PUT";
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
interface PresignDownloadDTO {
|
||||
urls: Record<string, string>;
|
||||
expiresInSeconds: number;
|
||||
}
|
||||
|
||||
/** The subset of the AppShell SDK this module needs — keeps the adapter unit-testable. */
|
||||
interface DataDoor {
|
||||
query<T>(action: string, variables?: Record<string, unknown>): Promise<T>;
|
||||
command<T>(action: string, variables?: Record<string, unknown>): Promise<T>;
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* The live adapter — be-crm data door */
|
||||
/* ======================================================================== */
|
||||
|
||||
function chunk<T>(items: T[], size: number): T[][] {
|
||||
const out: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A StorageAdapter backed by the be-crm data door.
|
||||
*
|
||||
* Metadata (media/albums/people) rides the door as JSON; bytes go direct to object storage via
|
||||
* short-lived signed URLs. `storageRef` is the durable handle we persist — `src` is only ever a
|
||||
* signed URL with a TTL, so it is re-resolved from the refs on every `load()`.
|
||||
*/
|
||||
export function createDataDoorAdapter(sdk: DataDoor): StorageAdapter {
|
||||
/** Resolve durable refs → fresh signed GET URLs, in chunks the door will accept. */
|
||||
async function resolveRefs(refs: string[]): Promise<Record<string, string>> {
|
||||
const unique = [...new Set(refs.filter(Boolean))];
|
||||
if (!unique.length) return {};
|
||||
const urls: Record<string, string> = {};
|
||||
for (const group of chunk(unique, PRESIGN_CHUNK)) {
|
||||
const res = await sdk.command<PresignDownloadDTO>("crm.gallery.media.presignDownload", { refs: group });
|
||||
Object.assign(urls, res.urls ?? {});
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
return {
|
||||
name: "crm-data-door",
|
||||
|
||||
async load(): Promise<PersistedState | null> {
|
||||
const state = await sdk.query<StateLoadDTO>("crm.gallery.state.load", {});
|
||||
const media = state.media ?? [];
|
||||
// Signed URLs expire, so `src` is always rebuilt from `storageRef` at load time. Items with no
|
||||
// ref (e.g. a seeded remote URL) keep whatever `src` they were stored with.
|
||||
const urls = await resolveRefs(media.map((m) => m.storageRef ?? "").filter(Boolean));
|
||||
return {
|
||||
media: media.map((m) => (m.storageRef && urls[m.storageRef] ? { ...m, src: urls[m.storageRef]! } : m)),
|
||||
albums: state.albums ?? [],
|
||||
people: state.people ?? [],
|
||||
labelAliases: state.labelAliases ?? {},
|
||||
deletedLabels: state.deletedLabels ?? [],
|
||||
version: 1,
|
||||
};
|
||||
},
|
||||
|
||||
// Incremental sync is the real path (see applyChanges). `save` only runs if the store ever falls
|
||||
// back to whole-state persistence; express it as one big change set so behaviour is identical.
|
||||
async save(state: PersistedState): Promise<void> {
|
||||
await sdk.command("crm.gallery.state.apply", {
|
||||
upsertMedia: state.media,
|
||||
upsertAlbums: state.albums,
|
||||
upsertPeople: state.people,
|
||||
labelAliases: state.labelAliases ?? {},
|
||||
deletedLabels: state.deletedLabels ?? [],
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Incremental persistence — only the entities that actually changed. This is what makes a shared
|
||||
* tenant library safe for concurrent editors: two people touching different photos never clobber
|
||||
* each other, because neither sends the other's rows.
|
||||
*/
|
||||
async applyChanges(changes: StateChanges): Promise<void> {
|
||||
// StateChanges is a closed interface; the door takes an open variables bag.
|
||||
await sdk.command("crm.gallery.state.apply", { ...changes });
|
||||
},
|
||||
|
||||
/** Presign → direct PUT → resolve a display URL. Bytes never touch be-crm or the BFF. */
|
||||
async putMedia(id: string, blob: Blob, meta: { name: string; mime: string }): Promise<StoredBlob> {
|
||||
if (blob.size > MAX_GALLERY_BYTES) {
|
||||
throw new Error(`File is too large (max ${Math.floor(MAX_GALLERY_BYTES / (1024 * 1024))} MB).`);
|
||||
}
|
||||
const mime = meta.mime || blob.type || "application/octet-stream";
|
||||
const presigned = await sdk.command<PresignUploadDTO>("crm.gallery.media.presignUpload", {
|
||||
mediaId: id,
|
||||
mime,
|
||||
sizeBytes: blob.size,
|
||||
filename: meta.name,
|
||||
});
|
||||
const res = await fetch(presigned.uploadUrl, {
|
||||
method: presigned.method ?? "PUT",
|
||||
headers: { "content-type": mime, ...(presigned.headers ?? {}) },
|
||||
body: blob,
|
||||
});
|
||||
if (!res.ok) throw new Error(`Upload failed (${res.status}).`);
|
||||
const urls = await resolveRefs([presigned.ref]);
|
||||
return { ref: presigned.ref, url: urls[presigned.ref] ?? presigned.ref };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Public hooks */
|
||||
/* ======================================================================== */
|
||||
|
||||
export interface GalleryStorage {
|
||||
/** True when persisting to be-crm; false when running on the device-local demo store. */
|
||||
live: boolean;
|
||||
adapter: StorageAdapter;
|
||||
}
|
||||
|
||||
/**
|
||||
* The storage backend for the embedded gallery. Live against the be-crm data door once the Shell is
|
||||
* configured, otherwise a device-local store so the demo works with no backend at all.
|
||||
*
|
||||
* SHELL is a build-time constant, so this branch is stable across renders (Rules-of-Hooks safe).
|
||||
*/
|
||||
export function useGalleryStorage(): GalleryStorage {
|
||||
const { sdk } = useAppShell();
|
||||
// The adapter identity must be stable — PhotoGallery captures it in a ref on first render.
|
||||
return useMemo<GalleryStorage>(
|
||||
() =>
|
||||
SHELL
|
||||
? { live: true, adapter: createDataDoorAdapter(sdk as unknown as DataDoor) }
|
||||
: { live: false, adapter: createLocalStorageAdapter(LOCAL_STORE_KEY) },
|
||||
[sdk],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The signed-in identity handed to the gallery so comments are attributed to a real person rather
|
||||
* than a free-text name the author can type themselves.
|
||||
*
|
||||
* Falls back to the same static demo user the sidebar and topbar use when the Shell isn't wired, so
|
||||
* the comment module behaves identically in the demo — the CRM never shows an anonymous author.
|
||||
* be-crm re-stamps `authorId` from the PAT on every write regardless, so this value is a display
|
||||
* convenience, never the source of authority.
|
||||
*/
|
||||
export function useGalleryUser(): GalleryUser {
|
||||
const { user, context } = useAuth();
|
||||
return useMemo<GalleryUser>(() => {
|
||||
if (!user) return { id: demoUser.id, name: demoUser.name };
|
||||
const avatarUrl = user.avatarUrl ?? context?.principal?.avatarUrl;
|
||||
return {
|
||||
id: user.id,
|
||||
name: user.displayName || user.email || demoUser.name,
|
||||
...(user.email ? { email: user.email } : {}),
|
||||
...(avatarUrl ? { avatarUrl } : {}),
|
||||
};
|
||||
}, [user, context]);
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Recently Deleted lock */
|
||||
/* ======================================================================== */
|
||||
|
||||
/**
|
||||
* The SDK's `lockProvider` contract, declared here rather than imported so this module keeps
|
||||
* compiling against an SDK build that predates the prop. It is structurally identical to
|
||||
* `PhotoGalleryProps['lockProvider']`.
|
||||
*/
|
||||
export interface GalleryLockProvider {
|
||||
status(): Promise<{ hasPassword: boolean }>;
|
||||
/** `null` clears the password. */
|
||||
set(password: string | null): Promise<void>;
|
||||
verify(password: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A server-backed, per-user lock for the Recently Deleted view.
|
||||
*
|
||||
* Without this the SDK falls back to a device-local `localStorage` hash: the lock exists only on
|
||||
* the browser that set it, so the same account on a second device sees an unlocked trash. Backed
|
||||
* by `crm.gallery.lock.*`, the password becomes an account property — be-crm stores only a
|
||||
* scrypt hash with a per-record random salt, keyed by (tenant, principal), and rate-limits verify.
|
||||
*
|
||||
* Returns `undefined` in demo mode ON PURPOSE: with no backend there is nowhere to put the hash,
|
||||
* and the SDK's own device-local behaviour is the right fallback for a demo.
|
||||
*
|
||||
* SHELL is a build-time constant, so this branch is stable across renders (Rules-of-Hooks safe).
|
||||
*/
|
||||
export function useGalleryLockProvider(): GalleryLockProvider | undefined {
|
||||
const { sdk } = useAppShell();
|
||||
return useMemo<GalleryLockProvider | undefined>(() => {
|
||||
if (!SHELL) return undefined;
|
||||
const door = sdk as unknown as DataDoor;
|
||||
return {
|
||||
status: () => door.query<{ hasPassword: boolean }>("crm.gallery.lock.status", {}),
|
||||
// The SDK's contract returns void; the door's `{ hasPassword }` is redundant after a set.
|
||||
set: async (password) => {
|
||||
await door.command("crm.gallery.lock.set", { password });
|
||||
},
|
||||
// A wrong password is a normal `{ ok: false }`, not an error. A 403 (the verify lockout)
|
||||
// still throws, which is what the SDK's prompt should surface.
|
||||
verify: async (password) => (await door.command<{ ok: boolean }>("crm.gallery.lock.verify", { password })).ok,
|
||||
};
|
||||
}, [sdk]);
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Permission-gated features */
|
||||
/* ======================================================================== */
|
||||
|
||||
/**
|
||||
* The gallery capabilities the CRM gates behind team permissions. Each SDK feature toggle
|
||||
* (and the whole-view `media.view` gate) maps to one permission id from be-crm's Media group.
|
||||
* The SDK still enforces nothing here — hiding a control is UX; be-crm enforces every write via
|
||||
* GALLERY_VISIBILITY + tenant scoping regardless of what the UI shows.
|
||||
*/
|
||||
const MEDIA_PERMISSIONS = [
|
||||
"media.view", "media.upload", "media.capture", "media.edit",
|
||||
"media.delete", "media.export", "media.share", "media.map", "media.ai",
|
||||
] as const;
|
||||
|
||||
/** SDK feature toggle → the permission id that unlocks it. */
|
||||
const FEATURE_PERMISSION: Record<keyof GalleryFeatures, string> = {
|
||||
editor: "media.edit",
|
||||
camera: "media.capture",
|
||||
import: "media.upload",
|
||||
export: "media.export",
|
||||
sharing: "media.share",
|
||||
map: "media.map",
|
||||
ai: "media.ai",
|
||||
};
|
||||
|
||||
export interface ResolvedGalleryFeatures {
|
||||
/** Feature toggles to hand the SDK's `features` prop, resolved from the caller's permissions. */
|
||||
features: GalleryFeatures;
|
||||
/** Whether the whole Smart Gallery view should render at all (`media.view`). */
|
||||
canView: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the gallery's feature flags + view access from the signed-in user's CRM permissions.
|
||||
*
|
||||
* Fallback rule (SAFE default — a control is enabled unless the user is positively known to lack it):
|
||||
* 1. While access is still loading, stay permissive so features don't flash on then vanish.
|
||||
* 2. Superadmins and owners get everything (they hold every permission anyway; this is explicit).
|
||||
* 3. A member who has been assigned AT LEAST ONE `media.*` permission is gated precisely: each
|
||||
* feature is enabled only if its mapped permission is in their list.
|
||||
* 4. A member with ZERO `media.*` permissions assigned is treated as fully enabled. Roles are not
|
||||
* configured with Media perms until an admin opts in (§5 just made them assignable), so gating a
|
||||
* freshly-seeded member down to nothing would cripple the gallery before anyone could grant them
|
||||
* anything. The "has at least one media.* perm" signal is what flips a role from this permissive
|
||||
* default into precise per-feature gating.
|
||||
*
|
||||
* Because be-crm's mock access (demo, no Shell) returns superadmin + all perms, the demo shows
|
||||
* everything via rule 2.
|
||||
*/
|
||||
export function useGalleryFeatures(): ResolvedGalleryFeatures {
|
||||
const access = useMyAccess();
|
||||
return useMemo<ResolvedGalleryFeatures>(() => {
|
||||
const has = (p: string) => access.permissions.includes(p);
|
||||
const privileged = access.isSuperadmin || access.roleSlugs.includes("owner");
|
||||
const hasAnyMedia = MEDIA_PERMISSIONS.some(has);
|
||||
// Permissive whenever we can't (yet) prove the user lacks a permission: loading, privileged, or a
|
||||
// member who has no Media perms assigned at all. Otherwise gate precisely on the mapped id.
|
||||
const allow = (perm: string) => access.loading || privileged || !hasAnyMedia || has(perm);
|
||||
|
||||
const features: GalleryFeatures = {
|
||||
editor: allow(FEATURE_PERMISSION.editor),
|
||||
camera: allow(FEATURE_PERMISSION.camera),
|
||||
ai: allow(FEATURE_PERMISSION.ai),
|
||||
map: allow(FEATURE_PERMISSION.map),
|
||||
import: allow(FEATURE_PERMISSION.import),
|
||||
export: allow(FEATURE_PERMISSION.export),
|
||||
sharing: allow(FEATURE_PERMISSION.sharing),
|
||||
};
|
||||
|
||||
return { features, canView: allow("media.view") };
|
||||
}, [access]);
|
||||
}
|
||||
|
||||
/* ======================================================================== */
|
||||
/* Theme bridge */
|
||||
/* ======================================================================== */
|
||||
|
||||
/**
|
||||
* Maps the gallery's design tokens onto the CRM's own CSS variables.
|
||||
*
|
||||
* Every value is a `var(--crm-token)` reference rather than a literal hex, so the gallery inherits the
|
||||
* dashboard's palette through the SAME `[data-theme]` cascade the rest of the app uses — light/dark
|
||||
* switch together, and a future palette change reaches the gallery with no code edit here.
|
||||
*/
|
||||
export const GALLERY_THEME_TOKENS: ThemeTokens = {
|
||||
// Surfaces
|
||||
bgLight: "var(--bg)",
|
||||
bgDark: "var(--bg)",
|
||||
elevatedLight: "var(--panel-3)",
|
||||
elevatedDark: "var(--panel-3)",
|
||||
sidebarBgLight: "var(--sidebar)",
|
||||
sidebarBgDark: "var(--sidebar)",
|
||||
toolbarBgLight: "var(--panel)",
|
||||
toolbarBgDark: "var(--panel)",
|
||||
cardLight: "var(--panel)",
|
||||
cardDark: "var(--panel)",
|
||||
cardHoverLight: "var(--panel-3)",
|
||||
cardHoverDark: "var(--panel-3)",
|
||||
menuBgLight: "var(--panel-3)",
|
||||
menuBgDark: "var(--panel-3)",
|
||||
|
||||
// Text
|
||||
textLight: "var(--text)",
|
||||
textDark: "var(--text)",
|
||||
textSecondaryLight: "var(--muted)",
|
||||
textSecondaryDark: "var(--muted)",
|
||||
textTertiaryLight: "var(--faint)",
|
||||
textTertiaryDark: "var(--faint)",
|
||||
|
||||
// Lines + washes
|
||||
separatorLight: "var(--border)",
|
||||
separatorDark: "var(--border)",
|
||||
separatorStrongLight: "var(--border-2)",
|
||||
separatorStrongDark: "var(--border-2)",
|
||||
hoverLight: "var(--track)",
|
||||
hoverDark: "var(--track)",
|
||||
activeLight: "var(--border-2)",
|
||||
activeDark: "var(--border-2)",
|
||||
sidebarSelectedLight: "var(--track)",
|
||||
sidebarSelectedDark: "var(--track)",
|
||||
glassBorderLight: "var(--border-2)",
|
||||
glassBorderDark: "var(--border-2)",
|
||||
|
||||
// Brand
|
||||
accent: "var(--orange)",
|
||||
accentStrongLight: "var(--orange-2)",
|
||||
accentStrongDark: "var(--orange-2)",
|
||||
accentContrast: "#ffffff",
|
||||
dangerLight: "var(--red)",
|
||||
dangerDark: "var(--red)",
|
||||
tileFav: "var(--red)",
|
||||
segmentedActive: "var(--panel-3)",
|
||||
|
||||
// Overlays + chrome
|
||||
overlayBg: "rgba(2,2,6,0.94)",
|
||||
editorBg: "var(--panel-2)",
|
||||
shadowSm: "0 1px 2px rgba(0,0,0,0.18)",
|
||||
shadowMdLight: "var(--shadow)",
|
||||
shadowMdDark: "var(--shadow)",
|
||||
shadowLgLight: "0 30px 70px -20px rgba(15,23,42,0.25)",
|
||||
shadowLgDark: "0 30px 70px -20px rgba(0,0,0,0.7)",
|
||||
|
||||
fontFamily: "var(--font)",
|
||||
radiusMenu: 14,
|
||||
sidebarRadius: 14,
|
||||
};
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Tiny in-process fixed-window rate limiter for the Smart Gallery AI routes.
|
||||
*
|
||||
* ---------------------------------------------------------------------------
|
||||
* SCOPE AND LIMITATIONS — READ BEFORE RELYING ON THIS
|
||||
* ---------------------------------------------------------------------------
|
||||
* State lives in a plain `Map` in THIS process's memory. That means:
|
||||
*
|
||||
* - PER-INSTANCE, NOT GLOBAL. With N app instances behind a load balancer a
|
||||
* caller gets up to N x the configured budget. On serverless platforms each
|
||||
* cold start begins with an empty map, so the effective limit is weaker
|
||||
* still.
|
||||
* - NOT DURABLE. A restart or redeploy clears every counter.
|
||||
* - FIXED WINDOW, NOT SLIDING. A caller can burst `max` at the very end of one
|
||||
* window and `max` again at the start of the next — up to 2x `max` across a
|
||||
* window boundary. Acceptable here; the goal is to bound runaway cost, not
|
||||
* to meter precisely.
|
||||
*
|
||||
* It exists because the routes it guards spend real money on GPU inference and
|
||||
* shipping them with NO limit at all is worse than shipping an imperfect one.
|
||||
*
|
||||
* REPLACE WITH REDIS (or the platform's rate limiter) BEFORE RUNNING MORE THAN
|
||||
* ONE INSTANCE. The `limit()` signature is deliberately narrow so a Redis-backed
|
||||
* implementation can drop straight in — the only change needed is making it
|
||||
* async at the call sites.
|
||||
*
|
||||
* This is a throttle, not an authorization check. See src/lib/server/session.ts.
|
||||
*/
|
||||
|
||||
interface Window {
|
||||
/** Requests counted so far in the current window. */
|
||||
count: number;
|
||||
/** Epoch ms at which the current window ends. */
|
||||
resetAt: number;
|
||||
}
|
||||
|
||||
const windows = new Map<string, Window>();
|
||||
|
||||
/** Drop expired entries so the map cannot grow without bound. */
|
||||
const SWEEP_INTERVAL_MS = 60_000;
|
||||
let lastSweep = 0;
|
||||
|
||||
function sweep(now: number): void {
|
||||
if (now - lastSweep < SWEEP_INTERVAL_MS) return;
|
||||
lastSweep = now;
|
||||
for (const [key, w] of windows) {
|
||||
if (w.resetAt <= now) windows.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
export interface LimitResult {
|
||||
ok: boolean;
|
||||
/** Seconds until the window resets. Send as `Retry-After` when `ok` is false. */
|
||||
retryAfter: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count one request against `key` and report whether it is allowed.
|
||||
*
|
||||
* @param key Caller identity — use `rateLimitKey()` from ./session.
|
||||
* @param max Requests allowed per window.
|
||||
* @param windowMs Window length in ms.
|
||||
*/
|
||||
export function limit(key: string, max: number, windowMs: number): LimitResult {
|
||||
const now = Date.now();
|
||||
sweep(now);
|
||||
|
||||
const existing = windows.get(key);
|
||||
if (!existing || existing.resetAt <= now) {
|
||||
windows.set(key, { count: 1, resetAt: now + windowMs });
|
||||
return { ok: true, retryAfter: 0 };
|
||||
}
|
||||
|
||||
if (existing.count >= max) {
|
||||
return { ok: false, retryAfter: Math.max(1, Math.ceil((existing.resetAt - now) / 1000)) };
|
||||
}
|
||||
|
||||
existing.count += 1;
|
||||
return { ok: true, retryAfter: 0 };
|
||||
}
|
||||
|
||||
/** Test/maintenance helper — clears all counters. */
|
||||
export function resetAllLimits(): void {
|
||||
windows.clear();
|
||||
lastSweep = 0;
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* base64 / data-URI helpers shared by the RunPod endpoint wrappers.
|
||||
* Different endpoints name their image field differently and some prefix a
|
||||
* `data:` URI — these helpers normalize both.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/base64.ts`.
|
||||
* The `pickOutputImage` normalization is intentionally identical so both apps
|
||||
* tolerate the same set of endpoint response shapes.
|
||||
*
|
||||
* SERVER-ONLY.
|
||||
*/
|
||||
|
||||
/** "data:image/png;base64,XXXX" -> "XXXX" (leaves a bare base64 string untouched). */
|
||||
export function stripDataUri(s: string): string {
|
||||
if (!s.startsWith("data:")) return s;
|
||||
const i = s.indexOf(",");
|
||||
return i === -1 ? s : s.slice(i + 1);
|
||||
}
|
||||
|
||||
/** Wrap a bare base64 string in a data: URI. */
|
||||
export function toDataUri(b64: string, mime: string): string {
|
||||
return `data:${mime};base64,${stripDataUri(b64)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the first image-like base64 string out of a RunPod endpoint's `output`,
|
||||
* regardless of which field name it used. Handles the common shapes:
|
||||
* "iVBOR..." (raw string)
|
||||
* { image: "..." } / { image_png } / { image_base64 }
|
||||
* { images: ["..."] } (array)
|
||||
* { output: { image: "..." } } (nested)
|
||||
*/
|
||||
export function pickOutputImage(output: unknown): string {
|
||||
const s = findImageString(output, false);
|
||||
if (!s) throw new Error("RunPod output did not contain an image.");
|
||||
return stripDataUri(s);
|
||||
}
|
||||
|
||||
/** Keys whose name implies the value IS the image — any non-empty string is accepted. */
|
||||
const IMAGE_KEYS = ["image_png", "image", "image_base64", "images"] as const;
|
||||
/** Generic wrapper keys — a string here must actually look like image data. */
|
||||
const CONTAINER_KEYS = ["output", "result", "data"] as const;
|
||||
|
||||
/** A base64 image payload is long; a status/id string ("success", "job-abc") is short. */
|
||||
function looksLikeImageData(s: string): boolean {
|
||||
if (s.startsWith("data:image/")) return true;
|
||||
return s.length >= 256 && /^[A-Za-z0-9+/=\s]+$/.test(s.slice(0, 256));
|
||||
}
|
||||
|
||||
/**
|
||||
* `strict` is true when we descended through a generic wrapper key (output/result/
|
||||
* data), where a bare string could be a status/id rather than an image — so it must
|
||||
* pass `looksLikeImageData`. Under an explicit image key (or at top level) any
|
||||
* non-empty string is taken as the image.
|
||||
*/
|
||||
function findImageString(value: unknown, strict: boolean, depth = 0): string | undefined {
|
||||
if (depth > 5) return undefined;
|
||||
if (typeof value === "string") {
|
||||
if (!value) return undefined;
|
||||
return !strict || looksLikeImageData(value) ? value : undefined;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
for (const el of value) {
|
||||
const s = findImageString(el, strict, depth + 1);
|
||||
if (s) return s;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
const o = value as Record<string, unknown>;
|
||||
for (const key of IMAGE_KEYS) {
|
||||
if (key in o) {
|
||||
const s = findImageString(o[key], false, depth + 1);
|
||||
if (s) return s;
|
||||
}
|
||||
}
|
||||
for (const key of CONTAINER_KEYS) {
|
||||
if (key in o) {
|
||||
const s = findImageString(o[key], true, depth + 1);
|
||||
if (s) return s;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* Low-level RunPod transport. Every model runs as its own RunPod endpoint; this
|
||||
* handles the common request envelope, bearer auth, and both response modes:
|
||||
*
|
||||
* - `/runsync` (preferred): the job runs synchronously and the body already
|
||||
* contains `output` (or IS the output for a custom handler).
|
||||
* - `/run`: returns `{ id, status }`; we poll `/status/{id}` until the job
|
||||
* reaches a terminal state or the time budget (kept under the 60s serverless
|
||||
* function cap) is exhausted.
|
||||
*
|
||||
* RUNPOD_API_KEY is read here so callers never handle the secret directly, and
|
||||
* it is NEVER echoed into an error message. Upstream error bodies are truncated
|
||||
* to 160 chars before being surfaced.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/client.ts` (same 55s
|
||||
* budget, same polling, same RunpodError.status mapping).
|
||||
*
|
||||
* SERVER-ONLY.
|
||||
*/
|
||||
|
||||
export class RunpodError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status = 502) {
|
||||
super(message);
|
||||
this.name = "RunpodError";
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export interface RunpodCallOpts {
|
||||
/** Human label used in error messages, e.g. "sd-inpaint". */
|
||||
name: string;
|
||||
/** Full endpoint URL from the per-model env var (…/runsync or …/run). */
|
||||
url: string;
|
||||
input: Record<string, unknown>;
|
||||
/** Total budget for the whole call incl. polling. Default 55s (< the 60s cap). */
|
||||
timeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
}
|
||||
|
||||
interface RunpodEnvelope {
|
||||
id?: string;
|
||||
status?: string;
|
||||
output?: unknown;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
export async function runpodCall<TOut = unknown>(opts: RunpodCallOpts): Promise<TOut> {
|
||||
const { name, url, input } = opts;
|
||||
const timeoutMs = opts.timeoutMs ?? 55_000;
|
||||
const pollIntervalMs = opts.pollIntervalMs ?? 1500;
|
||||
|
||||
const key = process.env.RUNPOD_API_KEY;
|
||||
if (!key) throw new RunpodError("RUNPOD_API_KEY is not set.", 500);
|
||||
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const authHeaders = { authorization: `Bearer ${key}` };
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", ...authHeaders },
|
||||
body: JSON.stringify({ input }),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
throw new RunpodError(`Could not reach RunPod (${name}).`, 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
// Truncated on purpose — never surface a full upstream body to the client.
|
||||
const detail = (await res.text().catch(() => "")).slice(0, 160);
|
||||
throw new RunpodError(`RunPod ${name} error (${res.status}). ${detail}`.trim(), 502);
|
||||
}
|
||||
|
||||
const data = (await res.json().catch(() => null)) as RunpodEnvelope | null;
|
||||
if (!data || typeof data !== "object") {
|
||||
throw new RunpodError(`RunPod ${name} returned an invalid response.`, 502);
|
||||
}
|
||||
|
||||
// Terminal failure reported in a job envelope.
|
||||
if (data.status === "FAILED" || data.status === "CANCELLED") {
|
||||
throw new RunpodError(`RunPod ${name} job ${data.status.toLowerCase()}.`, 502);
|
||||
}
|
||||
// No job id → this is not an async envelope; the body itself is the output.
|
||||
// Covers custom /runsync handlers that return their result directly, even when
|
||||
// it carries a `status` field (e.g. "success").
|
||||
if (data.id === undefined) {
|
||||
return (data.output !== undefined ? data.output : data) as TOut;
|
||||
}
|
||||
// Job envelope that already carries a completed/inline output.
|
||||
if (data.output !== undefined && (data.status === undefined || data.status === "COMPLETED")) {
|
||||
return data.output as TOut;
|
||||
}
|
||||
|
||||
// Async: poll /status/{id} until COMPLETED / FAILED / the time budget runs out.
|
||||
// Each wait + fetch is clamped to the remaining budget so the whole call stays
|
||||
// under `timeoutMs` (kept below the 60s function cap).
|
||||
const statusUrl = url.replace(/\/run(sync)?(\/?)$/, `/status/${data.id}`);
|
||||
for (;;) {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining < 500) break; // not enough budget for another round
|
||||
await sleep(Math.min(pollIntervalMs, remaining));
|
||||
const left = deadline - Date.now();
|
||||
if (left <= 0) break;
|
||||
let sres: Response;
|
||||
try {
|
||||
sres = await fetch(statusUrl, {
|
||||
headers: authHeaders,
|
||||
signal: AbortSignal.timeout(Math.min(10_000, Math.max(1000, left))),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
continue; // transient — keep polling until the deadline
|
||||
}
|
||||
if (!sres.ok) continue;
|
||||
const sdata = (await sres.json().catch(() => null)) as RunpodEnvelope | null;
|
||||
if (!sdata) continue;
|
||||
if (sdata.status === "COMPLETED") return sdata.output as TOut;
|
||||
if (sdata.status === "FAILED" || sdata.status === "CANCELLED") {
|
||||
throw new RunpodError(`RunPod ${name} job ${sdata.status.toLowerCase()}.`, 502);
|
||||
}
|
||||
}
|
||||
throw new RunpodError(
|
||||
`RunPod ${name} timed out (raise the endpoint's speed or use /runsync).`,
|
||||
504,
|
||||
);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
/**
|
||||
* Typed wrappers for each RunPod endpoint behind the Smart Gallery.
|
||||
* Every function reads its own `RUNPOD_*_URL` env var, sends the exact `input`
|
||||
* contract the model spec documents, and normalizes the response.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/endpoints.ts` —
|
||||
* identical `normalizeDetections` / `normalizeBox` logic and identical env var
|
||||
* names, so an endpoint deployed for the SDK demo works here unchanged.
|
||||
*
|
||||
* SERVER-ONLY. Missing/invalid URLs throw a RunpodError(500) so an unconfigured
|
||||
* op surfaces as a clear message rather than a crash.
|
||||
*/
|
||||
|
||||
import { stripDataUri, pickOutputImage } from "./base64";
|
||||
import { RunpodError, runpodCall } from "./client";
|
||||
import type {
|
||||
Img2ImgReq,
|
||||
InpaintReq,
|
||||
RunpodDetection,
|
||||
RunpodImageResult,
|
||||
TiltResult,
|
||||
TranscriptResult,
|
||||
} from "./types";
|
||||
|
||||
function envNum(v: string | undefined, fallback: number): number {
|
||||
// Treat a blank/whitespace env var as unset — Number('') is 0 (finite), which
|
||||
// would otherwise send e.g. strength:0 for `RUNPOD_SD_STRENGTH=`.
|
||||
if (v == null || v.trim() === "") return fallback;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : fallback;
|
||||
}
|
||||
|
||||
function endpointUrl(envVar: string): string {
|
||||
const url = process.env[envVar];
|
||||
if (!url || !/^https?:\/\//i.test(url)) {
|
||||
throw new RunpodError(`${envVar} is not set (or is not an http(s) URL).`, 500);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Run an image-in/image-out endpoint and normalize the result to base64 PNG. */
|
||||
async function imageOp(
|
||||
name: string,
|
||||
envVar: string,
|
||||
input: Record<string, unknown>,
|
||||
): Promise<RunpodImageResult> {
|
||||
const output = await runpodCall({ name, url: endpointUrl(envVar), input });
|
||||
return { imageBase64: pickOutputImage(output), mimeType: "image/png" };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Image endpoints
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** #6 Background removal (U²-Net via rembg). model: u2net | u2netp | u2net_human_seg */
|
||||
export function rpRemoveBackground(imageB64: string, model?: string): Promise<RunpodImageResult> {
|
||||
return imageOp("background-removal", "RUNPOD_BG_REMOVE_URL", {
|
||||
task: "remove-bg",
|
||||
image: imageB64,
|
||||
...(model ? { model_name: model } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/** #7 Real-ESRGAN enhance/upscale (RealESRGAN_x4plus). The caller's scale (from the
|
||||
* op / restore pass) is authoritative — it is not overridden by any env default. */
|
||||
export function rpUpscale(
|
||||
imageB64: string,
|
||||
scale: 2 | 4,
|
||||
faceEnhance = false,
|
||||
): Promise<RunpodImageResult> {
|
||||
return imageOp("upscale", "RUNPOD_UPSCALE_URL", {
|
||||
task: "upscale",
|
||||
image: imageB64,
|
||||
scale,
|
||||
face_enhance: faceEnhance,
|
||||
});
|
||||
}
|
||||
|
||||
/** #8 DDColor B&W → colorize. */
|
||||
export function rpColorize(imageB64: string, inputSize?: number): Promise<RunpodImageResult> {
|
||||
return imageOp("colorize", "RUNPOD_COLORIZE_URL", {
|
||||
image: imageB64,
|
||||
...(inputSize ? { input_size: inputSize } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/** #9 SD 3.5 masked inpainting (sky fix / eraser / fill). Also #11 outpaint (pre-padded). */
|
||||
export function rpInpaint(p: InpaintReq): Promise<RunpodImageResult> {
|
||||
const input: Record<string, unknown> = {
|
||||
task: "inpaint",
|
||||
image: p.imageB64,
|
||||
mask: p.maskB64,
|
||||
prompt: p.prompt,
|
||||
strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.8),
|
||||
guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7),
|
||||
num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35),
|
||||
};
|
||||
const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT;
|
||||
if (negative) input.negative_prompt = negative;
|
||||
if (p.seed != null) input.seed = p.seed;
|
||||
return imageOp("sd-inpaint", "RUNPOD_SD_INPAINT_URL", input);
|
||||
}
|
||||
|
||||
/** #10 SD 3.5 general prompt edit (img2img, no mask). */
|
||||
export function rpImg2Img(p: Img2ImgReq): Promise<RunpodImageResult> {
|
||||
const input: Record<string, unknown> = {
|
||||
task: "img2img",
|
||||
image: p.imageB64,
|
||||
prompt: p.prompt,
|
||||
strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.6),
|
||||
guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7),
|
||||
num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35),
|
||||
};
|
||||
const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT;
|
||||
if (negative) input.negative_prompt = negative;
|
||||
if (p.seed != null) input.seed = p.seed;
|
||||
return imageOp("sd-img2img", "RUNPOD_SD_IMG2IMG_URL", input);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// #1 YOLO detection → SDK DetectedObject shape (box as fractions 0..1)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function rpDetect(
|
||||
imageB64: string,
|
||||
width: number,
|
||||
height: number,
|
||||
): Promise<RunpodDetection[]> {
|
||||
const output = await runpodCall<unknown>({
|
||||
name: "yolo-detect",
|
||||
url: endpointUrl("RUNPOD_YOLO_URL"),
|
||||
input: { image: imageB64, task: "detect" },
|
||||
});
|
||||
return normalizeDetections(output, width, height);
|
||||
}
|
||||
|
||||
function extractDetectionArray(output: unknown): unknown[] {
|
||||
if (Array.isArray(output)) return output;
|
||||
if (output && typeof output === "object") {
|
||||
const o = output as Record<string, unknown>;
|
||||
for (const key of ["detections", "predictions", "objects", "results", "boxes"]) {
|
||||
if (Array.isArray(o[key])) return o[key] as unknown[];
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/** First non-empty STRING among the args (numbers ignored — a numeric `class` is an index, not a name). */
|
||||
function firstLabel(...vals: unknown[]): string | undefined {
|
||||
for (const v of vals) {
|
||||
if (typeof v === "string" && v.trim()) return v.trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeDetections(output: unknown, width: number, height: number): RunpodDetection[] {
|
||||
const out: RunpodDetection[] = [];
|
||||
for (const raw of extractDetectionArray(output)) {
|
||||
if (!raw || typeof raw !== "object") continue;
|
||||
const o = raw as Record<string, unknown>;
|
||||
// Prefer a human-readable name (ultralytics tojson puts the string in `name`
|
||||
// and a numeric index in `class`); fall back to class_<id>. Using firstLabel
|
||||
// (not `??`) also means an explicit empty-string label doesn't get kept + dropped.
|
||||
const classId = o.class_id ?? (typeof o.class === "number" ? o.class : undefined);
|
||||
const label = (
|
||||
firstLabel(o.label, o.name, o.class_name, typeof o.class === "string" ? o.class : undefined) ??
|
||||
(classId != null ? `class_${classId}` : "object")
|
||||
).toLowerCase();
|
||||
const confidence = Number(o.confidence ?? o.score ?? o.conf ?? 0) || 0;
|
||||
const box = normalizeBox(o, width, height);
|
||||
if (box && label) out.push({ label, confidence, box });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function num(v: unknown): number | null {
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
function asNum4(v: unknown): [number, number, number, number] | null {
|
||||
if (!Array.isArray(v) || v.length < 4) return null;
|
||||
const a = num(v[0]);
|
||||
const b = num(v[1]);
|
||||
const c = num(v[2]);
|
||||
const d = num(v[3]);
|
||||
return a === null || b === null || c === null || d === null ? null : [a, b, c, d];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a detection box to {x, y, width, height} as fractions 0..1 of the
|
||||
* image, from whatever shape the endpoint emits:
|
||||
* - `xyxy: [x1,y1,x2,y2]` (ultralytics) and generic `box: [...]` → corner form
|
||||
* - `xywh: [...]` and COCO `bbox: [x,y,w,h]` → x/y/width/height form
|
||||
* - object `{x1,y1,x2,y2}` / `{left,top,right,bottom}` (ultralytics tojson) → corners
|
||||
* - object `{x,y,width,height}` → x/y/width/height
|
||||
* Pixel values (any component > 1) are divided by the image dims; already-
|
||||
* normalized fractions pass through.
|
||||
*/
|
||||
function normalizeBox(
|
||||
o: Record<string, unknown>,
|
||||
width: number,
|
||||
height: number,
|
||||
): RunpodDetection["box"] | null {
|
||||
const W = width || 1;
|
||||
const H = height || 1;
|
||||
const clamp01 = (n: number) => Math.max(0, Math.min(1, n));
|
||||
const frac = (x: number, y: number, w: number, h: number): RunpodDetection["box"] => {
|
||||
if (Math.max(Math.abs(x), Math.abs(y), Math.abs(w), Math.abs(h)) > 1) {
|
||||
x /= W;
|
||||
y /= H;
|
||||
w /= W;
|
||||
h /= H;
|
||||
}
|
||||
return { x: clamp01(x), y: clamp01(y), width: clamp01(w), height: clamp01(h) };
|
||||
};
|
||||
const fromXyxy = (x1: number, y1: number, x2: number, y2: number) =>
|
||||
frac(x1, y1, x2 - x1, y2 - y1);
|
||||
|
||||
// 1. Array boxes, interpreted by which key holds them.
|
||||
const xyxyArr = asNum4(o.xyxy);
|
||||
if (xyxyArr) return fromXyxy(xyxyArr[0], xyxyArr[1], xyxyArr[2], xyxyArr[3]);
|
||||
const xywhArr = asNum4(o.xywh) ?? asNum4(o.bbox); // COCO `bbox` is [x,y,w,h]
|
||||
if (xywhArr) return frac(xywhArr[0], xywhArr[1], xywhArr[2], xywhArr[3]);
|
||||
const boxArr = asNum4(o.box); // generic array box → assume corner form
|
||||
if (boxArr) return fromXyxy(boxArr[0], boxArr[1], boxArr[2], boxArr[3]);
|
||||
|
||||
// 2. Object boxes (either nested under `box` or directly on the detection).
|
||||
const src =
|
||||
o.box && typeof o.box === "object" && !Array.isArray(o.box)
|
||||
? (o.box as Record<string, unknown>)
|
||||
: o;
|
||||
const x1 = num(src.x1 ?? src.left);
|
||||
const y1 = num(src.y1 ?? src.top);
|
||||
const x2 = num(src.x2 ?? src.right);
|
||||
const y2 = num(src.y2 ?? src.bottom);
|
||||
if (x1 !== null && y1 !== null && x2 !== null && y2 !== null) return fromXyxy(x1, y1, x2, y2);
|
||||
|
||||
const x = num(src.x);
|
||||
const y = num(src.y);
|
||||
const w = num(src.width);
|
||||
const h = num(src.height);
|
||||
if (x !== null && y !== null && w !== null && h !== null) return frac(x, y, w, h);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audio / calibration endpoints
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** #2 Camera tilt (DeepSingleImageCalibration). */
|
||||
export async function rpTilt(imageB64: string): Promise<TiltResult> {
|
||||
const o = await runpodCall<Record<string, unknown>>({
|
||||
name: "tilt",
|
||||
url: endpointUrl("RUNPOD_TILT_URL"),
|
||||
input: { image: imageB64 },
|
||||
});
|
||||
return {
|
||||
rollDegrees: Number(o.roll_degrees ?? 0) || 0,
|
||||
pitchDegrees: Number(o.pitch_degrees ?? 0) || 0,
|
||||
fovDegrees: Number(o.fov_degrees ?? 0) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
/** #3 Voice-to-text (Parakeet). Audio must be WAV 16kHz mono PCM16. */
|
||||
export async function rpTranscribe(
|
||||
audioB64: string,
|
||||
opts?: { language?: string; timestamps?: boolean; punctuation?: boolean },
|
||||
): Promise<TranscriptResult> {
|
||||
const o = await runpodCall<Record<string, unknown>>({
|
||||
name: "transcribe",
|
||||
url: endpointUrl("RUNPOD_STT_URL"),
|
||||
input: { audio: audioB64, task: "transcribe", ...(opts ?? {}) },
|
||||
});
|
||||
const rawSegments = Array.isArray(o.segments) ? o.segments : [];
|
||||
const segments = rawSegments.map((s) => {
|
||||
const seg = (s ?? {}) as Record<string, unknown>;
|
||||
return {
|
||||
text: String(seg.text ?? ""),
|
||||
startSec: Number(seg.start_sec ?? 0) || 0,
|
||||
endSec: Number(seg.end_sec ?? 0) || 0,
|
||||
};
|
||||
});
|
||||
return {
|
||||
transcript: String(o.transcript ?? ""),
|
||||
segments: segments.length ? segments : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** #12 Audio noise removal (RNNoise). Audio must be WAV 48kHz mono 16-bit PCM. */
|
||||
export async function rpDenoiseAudio(audioB64: string): Promise<{ audioB64: string }> {
|
||||
const o = await runpodCall<Record<string, unknown>>({
|
||||
name: "audio-denoise",
|
||||
url: endpointUrl("RUNPOD_AUDIO_DENOISE_URL"),
|
||||
input: { audio: audioB64, task: "denoise" },
|
||||
});
|
||||
const a = o.audio ?? o.output ?? "";
|
||||
return { audioB64: stripDataUri(String(a)) };
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Request/response types for the RunPod endpoints behind the Smart Gallery.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/types.ts`.
|
||||
* Keep the two in sync — the RunPod endpoints are shared between the standalone
|
||||
* SDK demo and this CRM.
|
||||
*
|
||||
* SERVER-ONLY. Imported by src/app/api/gallery/ai/* route handlers, never by a
|
||||
* client component (which must not see a RunPod URL or key).
|
||||
*/
|
||||
|
||||
/** Normalized image result returned by every image endpoint (raw base64, no data: prefix). */
|
||||
export interface RunpodImageResult {
|
||||
imageBase64: string;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
/** #9 SD 3.5 masked inpainting (also #11 outpaint, pre-padded). white in mask = regenerate. */
|
||||
export interface InpaintReq {
|
||||
imageB64: string;
|
||||
maskB64: string;
|
||||
prompt: string;
|
||||
negativePrompt?: string;
|
||||
strength?: number;
|
||||
guidanceScale?: number;
|
||||
steps?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
/** #10 SD 3.5 general prompt edit (img2img, no mask). */
|
||||
export interface Img2ImgReq {
|
||||
imageB64: string;
|
||||
prompt: string;
|
||||
negativePrompt?: string;
|
||||
strength?: number;
|
||||
guidanceScale?: number;
|
||||
steps?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* #1 YOLO construction-material classifier, normalized to the SDK's
|
||||
* `DetectedObject` shape (box as fractions 0..1 of the image).
|
||||
*/
|
||||
export interface RunpodDetection {
|
||||
label: string;
|
||||
confidence: number;
|
||||
box: { x: number; y: number; width: number; height: number };
|
||||
}
|
||||
|
||||
/** #2 DeepSingleImageCalibration — camera tilt. */
|
||||
export interface TiltResult {
|
||||
rollDegrees: number;
|
||||
pitchDegrees: number;
|
||||
fovDegrees: number;
|
||||
}
|
||||
|
||||
/** #3 Parakeet voice-to-text. */
|
||||
export interface TranscriptSegment {
|
||||
text: string;
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
}
|
||||
export interface TranscriptResult {
|
||||
transcript: string;
|
||||
segments?: TranscriptSegment[];
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Session gate for the Smart Gallery AI routes.
|
||||
*
|
||||
* ---------------------------------------------------------------------------
|
||||
* TRUST MODEL
|
||||
* ---------------------------------------------------------------------------
|
||||
* These routes proxy a paid, rate-limited GPU backend (RunPod) using a secret
|
||||
* held only on this server. An unauthenticated route is therefore not merely an
|
||||
* information-disclosure problem — it is a billable-resource problem: anyone who
|
||||
* can reach the URL can spend the operator's GPU budget, and can use the CRM as
|
||||
* an open relay for arbitrary image/audio processing.
|
||||
*
|
||||
* The upstream SDK demo's routes (apps/web/src/app/api/ai/*) are COMPLETELY
|
||||
* unauthenticated. This module is the fix; every ported route must call it
|
||||
* before doing any work.
|
||||
*
|
||||
* WHO IS TRUSTED
|
||||
* We do not verify a JWT here and we do not hold any signing key. The single
|
||||
* source of truth for "is this caller signed in" is the Shell BFF
|
||||
* (`${BFF_ORIGIN}/api/session/context`), which owns the HttpOnly session cookie.
|
||||
* We forward the caller's raw `cookie` header to it and treat a 200 as proof of
|
||||
* a session. Consequences of that choice, stated explicitly:
|
||||
*
|
||||
* - The BFF is trusted absolutely. If it is compromised or misconfigured to
|
||||
* answer 200 for anonymous callers, these routes are open. BFF_ORIGIN must
|
||||
* therefore only ever point at an origin the operator controls.
|
||||
* - We forward the cookie header verbatim and nothing else. No Authorization
|
||||
* header, no bearer token, and never the RunPod key.
|
||||
* - NOTHING IS CACHED. A cached "yes" would keep a revoked/expired session
|
||||
* alive for the cache lifetime, so every AI request costs one BFF round
|
||||
* trip. That is deliberate: correctness over latency for a spend gate.
|
||||
* - A network failure reaching the BFF returns 503 (fail CLOSED), never 200.
|
||||
* If we cannot prove a session, we do not spend GPU budget.
|
||||
*
|
||||
* DEMO MODE
|
||||
* When the Shell is not configured (`NEXT_PUBLIC_SUPABASE_URL` unset) the CRM
|
||||
* runs on its mock portal and there is no session to check, so we allow the
|
||||
* request and log ONCE at startup-of-first-use. This is the same gate
|
||||
* `isShellConfigured()` uses for mock-vs-real auth elsewhere in the app.
|
||||
* IMPORTANT: never deploy to a public origin with the Shell unconfigured AND a
|
||||
* real RUNPOD_API_KEY present — that combination is an open, billable endpoint.
|
||||
*
|
||||
* WHAT THIS IS NOT
|
||||
* This is authentication only, not authorization. It answers "is there a valid
|
||||
* session", not "may this principal use the gallery". Per-resource policy for
|
||||
* gallery data lives in be-crm behind the `crm.gallery` resource; if these AI
|
||||
* routes ever need the same, check it there rather than re-deriving it here.
|
||||
*/
|
||||
|
||||
export type GallerySession =
|
||||
| { ok: true; principalId?: string }
|
||||
| { ok: false; status: number; error: string };
|
||||
|
||||
/** Mirrors src/lib/appshell.ts — kept local so this stays server-only. */
|
||||
function isShellConfigured(): boolean {
|
||||
return Boolean(process.env.NEXT_PUBLIC_SUPABASE_URL);
|
||||
}
|
||||
|
||||
let demoModeWarned = false;
|
||||
|
||||
/**
|
||||
* Resolve the caller's session. Returns `{ ok: true }` (optionally with the
|
||||
* principal id, used to key rate limits) or a ready-to-return failure with the
|
||||
* status and message the route should emit.
|
||||
*/
|
||||
export async function requireGallerySession(req: Request): Promise<GallerySession> {
|
||||
if (!isShellConfigured()) {
|
||||
if (!demoModeWarned) {
|
||||
demoModeWarned = true;
|
||||
console.warn(
|
||||
"[gallery-ai] Shell is not configured (NEXT_PUBLIC_SUPABASE_URL unset) — " +
|
||||
"AI routes are UNAUTHENTICATED in demo mode. Do not expose this deployment publicly " +
|
||||
"while RUNPOD_API_KEY is set.",
|
||||
);
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
const cookie = req.headers.get("cookie");
|
||||
if (!cookie) return { ok: false, status: 401, error: "Not signed in" };
|
||||
|
||||
const origin = (process.env.BFF_ORIGIN ?? "http://localhost:4000").replace(/\/$/, "");
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`${origin}/api/session/context`, {
|
||||
headers: { cookie, accept: "application/json" },
|
||||
// Never cache an auth decision — see the trust-model note above.
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
} catch {
|
||||
// Fail closed: we could not prove a session, so we do not spend GPU budget.
|
||||
return { ok: false, status: 503, error: "Session service unavailable" };
|
||||
}
|
||||
|
||||
if (res.status === 401) return { ok: false, status: 401, error: "Not signed in" };
|
||||
if (!res.ok) return { ok: false, status: 503, error: "Session service unavailable" };
|
||||
|
||||
// The principal id is best-effort: it only sharpens the rate-limit key, so a
|
||||
// shape we don't recognize degrades to IP-keyed limiting rather than failing.
|
||||
let principalId: string | undefined;
|
||||
try {
|
||||
const data = (await res.json()) as Record<string, unknown> | null;
|
||||
principalId = pickPrincipalId(data);
|
||||
} catch {
|
||||
/* ignore — see above */
|
||||
}
|
||||
|
||||
return principalId ? { ok: true, principalId } : { ok: true };
|
||||
}
|
||||
|
||||
function pickPrincipalId(data: Record<string, unknown> | null): string | undefined {
|
||||
if (!data) return undefined;
|
||||
const direct = data.userId ?? data.principalId ?? data.sub ?? data.id;
|
||||
if (typeof direct === "string" && direct) return direct;
|
||||
const user = data.user;
|
||||
if (user && typeof user === "object") {
|
||||
const u = user as Record<string, unknown>;
|
||||
const nested = u.id ?? u.userId ?? u.sub;
|
||||
if (typeof nested === "string" && nested) return nested;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate-limit key for a request: the authenticated principal when known,
|
||||
* otherwise the first hop of `x-forwarded-for`.
|
||||
*
|
||||
* NOTE the first hop is client-controlled unless a trusted proxy overwrites the
|
||||
* header. It is good enough to throttle honest clients and casual abuse; it is
|
||||
* NOT a security boundary. The session gate above is the security boundary.
|
||||
*/
|
||||
export function rateLimitKey(req: Request, principalId?: string): string {
|
||||
if (principalId) return `u:${principalId}`;
|
||||
const xff = req.headers.get("x-forwarded-for") ?? "";
|
||||
const first = xff.split(",")[0]?.trim();
|
||||
return `ip:${first || "unknown"}`;
|
||||
}
|
||||
Reference in New Issue
Block a user