feat: add core domain types for Photo Gallery SDK including media items, albums, and annotations
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Shared entry gate for every /api/gallery/ai/* route: authenticate, then
|
||||
* throttle. Lives in a `_`-prefixed file so the App Router never treats it as a
|
||||
* route (only `route.ts` defines an endpoint).
|
||||
*
|
||||
* Order matters: we authenticate FIRST so the rate limit can be keyed by
|
||||
* principal rather than by a spoofable `x-forwarded-for` hop wherever possible.
|
||||
* The session check is one cheap BFF round trip; the work it guards is a GPU
|
||||
* call, so paying it before throttling is the right trade.
|
||||
*
|
||||
* SERVER-ONLY.
|
||||
*/
|
||||
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
import { limit } from "@/lib/server/rate-limit";
|
||||
import { rateLimitKey, requireGallerySession } from "@/lib/server/session";
|
||||
|
||||
/** Per-minute budgets, per the Smart Gallery route contract. */
|
||||
export const RATE_LIMITS = {
|
||||
classify: 30,
|
||||
edit: 12,
|
||||
tilt: 30,
|
||||
transcribe: 20,
|
||||
denoise: 20,
|
||||
} as const;
|
||||
|
||||
const WINDOW_MS = 60_000;
|
||||
|
||||
export type GuardResult =
|
||||
| { ok: true; principalId?: string }
|
||||
/** Ready-to-return error response — the route should return it unchanged. */
|
||||
| { ok: false; response: NextResponse };
|
||||
|
||||
/**
|
||||
* @param route Which budget to apply (also namespaces the limiter key so a
|
||||
* caller's `edit` spend does not consume their `classify` budget).
|
||||
*/
|
||||
export async function guard(req: Request, route: keyof typeof RATE_LIMITS): Promise<GuardResult> {
|
||||
const session = await requireGallerySession(req);
|
||||
if (!session.ok) {
|
||||
return {
|
||||
ok: false,
|
||||
response: NextResponse.json({ error: session.error }, { status: session.status }),
|
||||
};
|
||||
}
|
||||
|
||||
const key = `${route}:${rateLimitKey(req, session.principalId)}`;
|
||||
const { ok, retryAfter } = limit(key, RATE_LIMITS[route], WINDOW_MS);
|
||||
if (!ok) {
|
||||
return {
|
||||
ok: false,
|
||||
response: NextResponse.json(
|
||||
{ error: "Too many requests — slow down." },
|
||||
{ status: 429, headers: { "Retry-After": String(retryAfter) } },
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true, principalId: session.principalId };
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import { rpDetect } from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60;
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Object-detection proxy for the RunPod YOLO construction-material classifier (#1).
|
||||
* The key + endpoint URL stay server-side. The client calls this only when
|
||||
* NEXT_PUBLIC_APG_RUNPOD_DETECT is on; otherwise detection runs fully in-browser
|
||||
* (COCO-SSD) with no server round-trip. Returns the SDK's DetectedObject[] shape
|
||||
* (box as 0..1 fractions) so it drops straight into the Objects browser / smart
|
||||
* albums / search.
|
||||
*
|
||||
* POST { imageBase64, width, height } -> { objects: [{ label, confidence, box }] }
|
||||
* Auth: session-gated (see lib/server/session.ts). Rate limit: 30/min.
|
||||
*/
|
||||
|
||||
const MAX_BASE64 = 4_000_000; // ~3 MB decoded — under serverless body limits
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "classify");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
if (!process.env.RUNPOD_API_KEY || !process.env.RUNPOD_YOLO_URL) {
|
||||
return NextResponse.json(
|
||||
{ error: "RunPod detection is not configured (set RUNPOD_API_KEY + RUNPOD_YOLO_URL)." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid request body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const { imageBase64, width, height } = (body ?? {}) as {
|
||||
imageBase64?: unknown;
|
||||
width?: unknown;
|
||||
height?: unknown;
|
||||
};
|
||||
if (
|
||||
typeof imageBase64 !== "string" ||
|
||||
imageBase64.length === 0 ||
|
||||
imageBase64.length > MAX_BASE64
|
||||
) {
|
||||
return NextResponse.json({ error: "Invalid or oversized image." }, { status: 400 });
|
||||
}
|
||||
const w = Number(width);
|
||||
const h = Number(height);
|
||||
|
||||
try {
|
||||
const objects = await rpDetect(
|
||||
imageBase64,
|
||||
Number.isFinite(w) && w > 0 ? w : 1,
|
||||
Number.isFinite(h) && h > 0 ? h : 1,
|
||||
);
|
||||
return NextResponse.json({ objects });
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Detection failed.";
|
||||
const status = err instanceof RunpodError ? err.status : 502;
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import { rpDenoiseAudio } from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60; // cold-start denoise worker can take a while
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Audio noise-removal proxy. Accepts base64 WAV (48 kHz mono PCM16, produced
|
||||
* in-browser) and returns a cleaned base64 WAV. Calls the RunPod audio-denoise
|
||||
* endpoint (RUNPOD_AUDIO_DENOISE_URL) — key stays server-side. Used before
|
||||
* transcription on noisy sites.
|
||||
*
|
||||
* POST { audio } -> { audio }
|
||||
* Auth: session-gated. Rate limit: 20/min.
|
||||
*/
|
||||
|
||||
const MAX_BASE64 = 12_000_000; // ~9 MB decoded WAV
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "denoise");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
let body: { audio?: unknown };
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const audio = typeof body.audio === "string" ? body.audio : "";
|
||||
if (!audio) return NextResponse.json({ error: "Missing audio." }, { status: 400 });
|
||||
if (audio.length > MAX_BASE64) {
|
||||
return NextResponse.json({ error: "Audio too long — keep it under ~30s." }, { status: 413 });
|
||||
}
|
||||
|
||||
try {
|
||||
const { audioB64 } = await rpDenoiseAudio(audio);
|
||||
return NextResponse.json({ audio: audioB64 });
|
||||
} catch (e) {
|
||||
const msg = e instanceof RunpodError ? e.message : e instanceof Error ? e.message : "Denoise failed.";
|
||||
return NextResponse.json({ error: msg }, { status: 502 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import {
|
||||
rpImg2Img,
|
||||
rpInpaint,
|
||||
rpRemoveBackground,
|
||||
rpUpscale,
|
||||
} from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60; // SD / cold-start models can take a while
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Generative image-edit proxy. The BACKEND is pluggable — pick one with env
|
||||
* `AI_EDIT_PROVIDER` (default `auto`):
|
||||
*
|
||||
* - `runpod` → RunPod serverless GPU endpoints (one per model). Maps each
|
||||
* op → endpoint: restore/upscale → Real-ESRGAN (#7), colorize
|
||||
* → img2img (#10), replace-sky / magic-eraser / generative-fill
|
||||
* → SD 3.5 masked inpaint (#9), prompt → SD 3.5 img2img (#10).
|
||||
* Env: RUNPOD_API_KEY + per-model RUNPOD_*_URL. Key stays
|
||||
* server-side.
|
||||
* - `local` → your own Stable Diffusion server (Automatic1111 / Forge /
|
||||
* SD.Next img2img API). Env: LOCAL_SD_URL.
|
||||
* - `huggingface` → Hugging Face Inference API. Env: HF_API_TOKEN, HF_IMAGE_MODEL.
|
||||
* - `gemini` → Google Gemini image model (needs a billed key for image output).
|
||||
* Env: GEMINI_API_KEY, GEMINI_IMAGE_MODEL.
|
||||
* - `auto` → first configured of: runpod → local → huggingface → gemini.
|
||||
*
|
||||
* NOTE: `remove-background` runs in-browser by default (@imgly, no key), so it
|
||||
* usually never reaches here. Object detection uses its own route (./classify).
|
||||
*
|
||||
* POST { imageBase64, mimeType?, op, maskBase64?, params? } -> { imageBase64, mimeType }
|
||||
* Auth: session-gated. Rate limit: 12/min (the most expensive route).
|
||||
*/
|
||||
|
||||
const OP_PROMPTS: Record<string, string> = {
|
||||
restore:
|
||||
"Restore and enhance this photograph: improve sharpness and clarity, correct exposure and white balance, reduce noise and compression artifacts, recover detail. Keep it natural and photorealistic.",
|
||||
colorize: "Colorize this image with natural, realistic, well-balanced colors.",
|
||||
"replace-sky":
|
||||
"Replace the sky with a dramatic, beautiful golden-hour sky with soft clouds. Keep the foreground subject unchanged and the result photorealistic.",
|
||||
};
|
||||
|
||||
const MAX_BASE64 = 4_000_000; // ~3 MB decoded — stays under serverless body limits
|
||||
|
||||
type Provider = "runpod" | "local" | "huggingface" | "gemini" | "none";
|
||||
|
||||
function resolveProvider(): Provider {
|
||||
const explicit = (process.env.AI_EDIT_PROVIDER || "auto").toLowerCase();
|
||||
if (
|
||||
explicit === "runpod" ||
|
||||
explicit === "local" ||
|
||||
explicit === "huggingface" ||
|
||||
explicit === "gemini"
|
||||
)
|
||||
return explicit;
|
||||
if (explicit === "none") return "none";
|
||||
// auto: prefer RunPod GPU endpoints, then a private local server, then HF, then Gemini.
|
||||
// Detect RunPod when the key + ANY image endpoint URL is set (an upscale/colorize-only
|
||||
// deployment is valid — not just the SD ones).
|
||||
if (
|
||||
process.env.RUNPOD_API_KEY &&
|
||||
(process.env.RUNPOD_SD_IMG2IMG_URL ||
|
||||
process.env.RUNPOD_SD_INPAINT_URL ||
|
||||
process.env.RUNPOD_UPSCALE_URL ||
|
||||
process.env.RUNPOD_COLORIZE_URL ||
|
||||
process.env.RUNPOD_BG_REMOVE_URL)
|
||||
)
|
||||
return "runpod";
|
||||
if (process.env.LOCAL_SD_URL) return "local";
|
||||
if (process.env.HF_API_TOKEN) return "huggingface";
|
||||
if (process.env.GEMINI_API_KEY) return "gemini";
|
||||
return "none";
|
||||
}
|
||||
|
||||
/** Ops that only the RunPod (mask/fixed-function) backend can serve. */
|
||||
const RUNPOD_ONLY_OPS = new Set(["upscale", "magic-eraser", "generative-fill"]);
|
||||
|
||||
interface EditResult {
|
||||
imageBase64: string;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "edit");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
const provider = resolveProvider();
|
||||
if (provider === "none") {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
"AI image editing is not configured. Set AI_EDIT_PROVIDER=runpod + RUNPOD_API_KEY + the per-model RUNPOD_*_URL vars (RunPod GPU), or LOCAL_SD_URL (own Stable Diffusion), HF_API_TOKEN (Hugging Face), or GEMINI_API_KEY. Background removal and all analysis still work with no key.",
|
||||
},
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid request body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const { imageBase64, mimeType, op, maskBase64, params } = (body ?? {}) as {
|
||||
imageBase64?: unknown;
|
||||
mimeType?: unknown;
|
||||
op?: { type?: string; prompt?: string; factor?: number };
|
||||
maskBase64?: unknown;
|
||||
params?: unknown;
|
||||
};
|
||||
|
||||
if (typeof imageBase64 !== "string" || imageBase64.length === 0) {
|
||||
return NextResponse.json({ error: "Invalid image." }, { status: 400 });
|
||||
}
|
||||
const hasMask = typeof maskBase64 === "string" && maskBase64.length > 0;
|
||||
// Image + mask share one request body — budget them together against the cap.
|
||||
if (imageBase64.length + (hasMask ? (maskBase64 as string).length : 0) > MAX_BASE64) {
|
||||
return NextResponse.json(
|
||||
{ error: "Image (plus mask) is too large — try a smaller image." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const safeMime =
|
||||
typeof mimeType === "string" && /^image\/(jpeg|png|webp)$/.test(mimeType)
|
||||
? mimeType
|
||||
: "image/jpeg";
|
||||
|
||||
const opType = op?.type ?? "";
|
||||
if (provider !== "runpod" && RUNPOD_ONLY_OPS.has(opType)) {
|
||||
return NextResponse.json(
|
||||
{ error: "This edit needs the RunPod backend (set AI_EDIT_PROVIDER=runpod)." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
// Build the instruction from an allow-listed op (never trust arbitrary server prompts).
|
||||
let instruction = "";
|
||||
if (opType === "prompt" || opType === "generative-fill") {
|
||||
const p = typeof op?.prompt === "string" ? op.prompt.trim() : "";
|
||||
if (!p) return NextResponse.json({ error: "Empty prompt." }, { status: 400 });
|
||||
instruction = p.slice(0, 500);
|
||||
} else if (opType === "replace-sky") {
|
||||
instruction =
|
||||
typeof op?.prompt === "string" && op.prompt.trim()
|
||||
? `Replace the sky with: ${op.prompt.trim().slice(0, 300)}. Keep the foreground unchanged and photorealistic.`
|
||||
: OP_PROMPTS["replace-sky"]!;
|
||||
} else if (opType === "magic-eraser") {
|
||||
instruction =
|
||||
"Fill the selected region with a clean, seamless, plausible background. Photorealistic.";
|
||||
} else if (OP_PROMPTS[opType]) {
|
||||
instruction = OP_PROMPTS[opType]!;
|
||||
} else if (opType !== "upscale" && opType !== "remove-background") {
|
||||
return NextResponse.json({ error: "Unsupported operation." }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
let result: EditResult;
|
||||
if (provider === "runpod")
|
||||
result = await editRunPod(
|
||||
op ?? {},
|
||||
imageBase64,
|
||||
instruction,
|
||||
hasMask ? (maskBase64 as string) : undefined,
|
||||
params,
|
||||
);
|
||||
else if (provider === "local") result = await editLocal(instruction, imageBase64);
|
||||
else if (provider === "huggingface") result = await editHuggingFace(instruction, imageBase64);
|
||||
else result = await editGemini(instruction, imageBase64, safeMime);
|
||||
return NextResponse.json(result);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "AI request failed.";
|
||||
const status = err instanceof AiError || err instanceof RunpodError ? err.status : 502;
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backend: RunPod serverless GPU endpoints (one model per endpoint).
|
||||
// Each op maps to its endpoint; the API key + URLs stay server-side.
|
||||
// ---------------------------------------------------------------------------
|
||||
interface SdParams {
|
||||
negativePrompt?: string;
|
||||
strength?: number;
|
||||
steps?: number;
|
||||
seed?: number;
|
||||
guidanceScale?: number;
|
||||
}
|
||||
|
||||
function sanitizeParams(raw: unknown): SdParams {
|
||||
const p = (raw ?? {}) as Record<string, unknown>;
|
||||
const out: SdParams = {};
|
||||
if (typeof p.negativePrompt === "string" && p.negativePrompt.trim())
|
||||
out.negativePrompt = p.negativePrompt.trim().slice(0, 300);
|
||||
const strength = Number(p.strength);
|
||||
if (Number.isFinite(strength)) out.strength = Math.max(0, Math.min(1, strength));
|
||||
const steps = Number(p.steps);
|
||||
if (Number.isFinite(steps)) out.steps = Math.max(1, Math.min(60, Math.round(steps)));
|
||||
const guidance = Number(p.guidanceScale);
|
||||
if (Number.isFinite(guidance)) out.guidanceScale = Math.max(1, Math.min(20, guidance));
|
||||
const seed = Number(p.seed);
|
||||
if (Number.isFinite(seed)) out.seed = Math.max(0, Math.min(2_147_483_647, Math.round(seed)));
|
||||
return out;
|
||||
}
|
||||
|
||||
async function editRunPod(
|
||||
op: { type?: string; prompt?: string; factor?: number },
|
||||
imageBase64: string,
|
||||
instruction: string,
|
||||
maskBase64: string | undefined,
|
||||
rawParams: unknown,
|
||||
): Promise<EditResult> {
|
||||
const params = sanitizeParams(rawParams);
|
||||
switch (op.type) {
|
||||
case "remove-background":
|
||||
// U²-Net via rembg (#6) — a real endpoint replacing the flaky in-browser remover.
|
||||
return rpRemoveBackground(imageBase64);
|
||||
case "restore":
|
||||
// Real-ESRGAN (#7) with the GFPGAN face pass = "Restore & Enhance".
|
||||
return rpUpscale(imageBase64, 4, true);
|
||||
case "upscale":
|
||||
return rpUpscale(imageBase64, op.factor === 4 ? 4 : 2, false);
|
||||
case "colorize":
|
||||
// The dedicated DDColor endpoint kept hard-crashing (modelscope). Route
|
||||
// colorize through the img2img model as an instruction instead.
|
||||
return rpImg2Img({ imageB64: imageBase64, prompt: instruction, ...params });
|
||||
case "prompt":
|
||||
return rpImg2Img({ imageB64: imageBase64, prompt: instruction, ...params }); // SD 3.5 img2img (#10)
|
||||
case "replace-sky":
|
||||
// True sky replacement is masked inpaint (#9). Without a mask (no in-app sky
|
||||
// segmentation yet) degrade to a low-strength img2img (#10) so the foreground
|
||||
// is mostly preserved.
|
||||
if (maskBase64)
|
||||
return rpInpaint({
|
||||
imageB64: imageBase64,
|
||||
maskB64: maskBase64,
|
||||
prompt: instruction,
|
||||
...params,
|
||||
});
|
||||
return rpImg2Img({
|
||||
imageB64: imageBase64,
|
||||
prompt: instruction,
|
||||
...params,
|
||||
strength: params.strength ?? 0.4,
|
||||
});
|
||||
case "magic-eraser":
|
||||
case "generative-fill":
|
||||
// SD 3.5 masked inpaint (#9) — white in the mask = the region to regenerate.
|
||||
if (!maskBase64) throw new AiError("This edit needs a mask/selection.", 400);
|
||||
return rpInpaint({
|
||||
imageB64: imageBase64,
|
||||
maskB64: maskBase64,
|
||||
prompt: instruction,
|
||||
...params,
|
||||
});
|
||||
default:
|
||||
throw new AiError("Unsupported operation.", 400);
|
||||
}
|
||||
}
|
||||
|
||||
class AiError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status = 502) {
|
||||
super(message);
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backend: local Stable Diffusion (Automatic1111 / Forge / SD.Next img2img API)
|
||||
// ---------------------------------------------------------------------------
|
||||
async function editLocal(instruction: string, imageBase64: string): Promise<EditResult> {
|
||||
const base = process.env.LOCAL_SD_URL;
|
||||
if (!base || !/^https?:\/\//i.test(base)) {
|
||||
throw new AiError("LOCAL_SD_URL is not a valid http(s) URL.", 500);
|
||||
}
|
||||
const url = `${base.replace(/\/$/, "")}/sdapi/v1/img2img`;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
init_images: [imageBase64],
|
||||
prompt: instruction,
|
||||
denoising_strength: Number(process.env.LOCAL_SD_DENOISE ?? 0.55),
|
||||
steps: Number(process.env.LOCAL_SD_STEPS ?? 25),
|
||||
cfg_scale: 7,
|
||||
sampler_name: process.env.LOCAL_SD_SAMPLER || "Euler a",
|
||||
}),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
throw new AiError("Could not reach your local Stable Diffusion server (LOCAL_SD_URL).", 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new AiError(`Local SD server error (${res.status}).`, 502);
|
||||
}
|
||||
const data = (await res.json().catch(() => null)) as { images?: string[] } | null;
|
||||
const out = data?.images?.[0];
|
||||
if (!out) throw new AiError("Local SD server did not return an image.", 502);
|
||||
// A1111 returns raw base64 PNG (no data: prefix).
|
||||
return { imageBase64: out.includes(",") ? out.split(",")[1]! : out, mimeType: "image/png" };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backend: Hugging Face Inference API — instruction image editing.
|
||||
// ---------------------------------------------------------------------------
|
||||
async function editHuggingFace(instruction: string, imageBase64: string): Promise<EditResult> {
|
||||
const token = process.env.HF_API_TOKEN;
|
||||
if (!token) throw new AiError("HF_API_TOKEN is not set.", 500);
|
||||
const model = process.env.HF_IMAGE_MODEL || "timbrooks/instruct-pix2pix";
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`https://api-inference.huggingface.co/models/${model}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
"content-type": "application/json",
|
||||
// Wait for the model to warm up instead of a fast 503.
|
||||
"x-wait-for-model": "true",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
inputs: imageBase64,
|
||||
parameters: { prompt: instruction, guidance_scale: 7, image_guidance_scale: 1.5 },
|
||||
}),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
throw new AiError("Could not reach the Hugging Face Inference API.", 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
// Truncated on purpose — never surface a full upstream body.
|
||||
const detail = (await res.text().catch(() => "")).slice(0, 160);
|
||||
if (res.status === 503) throw new AiError("The model is loading — try again in ~20s.", 503);
|
||||
throw new AiError(`Hugging Face error (${res.status}). ${detail}`, 502);
|
||||
}
|
||||
// Success returns raw image bytes.
|
||||
const outMime = res.headers.get("content-type") || "image/png";
|
||||
if (outMime.startsWith("application/json")) {
|
||||
const j = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new AiError(
|
||||
j?.error ? `Hugging Face: ${j.error}` : "Hugging Face returned no image.",
|
||||
502,
|
||||
);
|
||||
}
|
||||
const buf = await res.arrayBuffer();
|
||||
return { imageBase64: Buffer.from(buf).toString("base64"), mimeType: outMime };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backend: Google Gemini image model (needs a billed key for image output).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function editGemini(
|
||||
instruction: string,
|
||||
imageBase64: string,
|
||||
safeMime: string,
|
||||
): Promise<EditResult> {
|
||||
const apiKey = process.env.GEMINI_API_KEY;
|
||||
if (!apiKey) throw new AiError("GEMINI_API_KEY is not set.", 500);
|
||||
const model = process.env.GEMINI_IMAGE_MODEL || "gemini-2.5-flash-image";
|
||||
const prompt = `Edit this image as follows: ${instruction}. Preserve realism unless explicitly asked otherwise.`;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(
|
||||
`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", "x-goog-api-key": apiKey },
|
||||
body: JSON.stringify({
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{ inlineData: { mimeType: safeMime, data: imageBase64 } },
|
||||
{ text: prompt },
|
||||
],
|
||||
},
|
||||
],
|
||||
generationConfig: { responseModalities: ["IMAGE"] },
|
||||
}),
|
||||
cache: "no-store",
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
throw new AiError("Could not reach the AI service.", 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
// Truncated on purpose — never surface a full upstream body.
|
||||
const detail = (await res.text().catch(() => "")).slice(0, 160);
|
||||
throw new AiError(`AI service error (${res.status}). ${detail}`, 502);
|
||||
}
|
||||
const data = (await res.json().catch(() => null)) as GeminiResponse | null;
|
||||
const parts = data?.candidates?.[0]?.content?.parts ?? [];
|
||||
const imgPart = parts.find((p) => p.inlineData?.data || p.inline_data?.data);
|
||||
const out = imgPart?.inlineData?.data ?? imgPart?.inline_data?.data;
|
||||
if (!out)
|
||||
throw new AiError(
|
||||
"The model did not return an image (the free Gemini tier has no image output — use LOCAL_SD_URL or HF_API_TOKEN instead).",
|
||||
502,
|
||||
);
|
||||
const outMime = imgPart?.inlineData?.mimeType ?? imgPart?.inline_data?.mime_type ?? "image/png";
|
||||
return { imageBase64: out, mimeType: outMime };
|
||||
}
|
||||
|
||||
interface GeminiPart {
|
||||
text?: string;
|
||||
inlineData?: { mimeType?: string; data?: string };
|
||||
inline_data?: { mime_type?: string; data?: string };
|
||||
}
|
||||
interface GeminiResponse {
|
||||
candidates?: Array<{ content?: { parts?: GeminiPart[] } }>;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import { rpTilt } from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60; // cold-start tilt worker can take a while
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Camera-tilt estimation proxy. Accepts a base64 image and returns
|
||||
* {rollDegrees, pitchDegrees, fovDegrees} from the RunPod tilt endpoint
|
||||
* (RUNPOD_TILT_URL) so the editor can auto-straighten.
|
||||
*
|
||||
* POST { image } -> { rollDegrees, pitchDegrees, fovDegrees }
|
||||
* Auth: session-gated. Rate limit: 30/min.
|
||||
*/
|
||||
|
||||
const MAX_BASE64 = 4_000_000; // ~3 MB decoded
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "tilt");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
let body: { image?: unknown };
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const image = typeof body.image === "string" ? body.image : "";
|
||||
if (!image) return NextResponse.json({ error: "Missing image." }, { status: 400 });
|
||||
if (image.length > MAX_BASE64) {
|
||||
return NextResponse.json({ error: "Image too large." }, { status: 413 });
|
||||
}
|
||||
|
||||
try {
|
||||
const tilt = await rpTilt(image);
|
||||
return NextResponse.json(tilt);
|
||||
} catch (e) {
|
||||
const msg =
|
||||
e instanceof RunpodError ? e.message : e instanceof Error ? e.message : "Tilt estimate failed.";
|
||||
return NextResponse.json({ error: msg }, { status: 502 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { RunpodError } from "@/lib/server/runpod/client";
|
||||
import { rpTranscribe } from "@/lib/server/runpod/endpoints";
|
||||
|
||||
import { guard } from "../_guard";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60; // cold-start STT worker can take a while
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/**
|
||||
* Speech-to-text proxy for voice annotations. Accepts base64 WAV (16 kHz mono
|
||||
* PCM16, produced in-browser) and returns the transcript. Calls the RunPod
|
||||
* voice-to-text endpoint (RUNPOD_STT_URL) — the key stays server-side.
|
||||
*
|
||||
* POST { audio, language? } -> { transcript, segments? }
|
||||
* Auth: session-gated. Rate limit: 20/min.
|
||||
*/
|
||||
|
||||
const MAX_BASE64 = 8_000_000; // ~6 MB decoded WAV — stays under serverless body limits
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const gate = await guard(req, "transcribe");
|
||||
if (!gate.ok) return gate.response;
|
||||
|
||||
let body: { audio?: unknown; language?: unknown };
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 });
|
||||
}
|
||||
|
||||
const audio = typeof body.audio === "string" ? body.audio : "";
|
||||
const language = typeof body.language === "string" ? body.language : undefined;
|
||||
if (!audio) return NextResponse.json({ error: "Missing audio." }, { status: 400 });
|
||||
if (audio.length > MAX_BASE64) {
|
||||
return NextResponse.json({ error: "Audio too long — keep it under ~30s." }, { status: 413 });
|
||||
}
|
||||
|
||||
try {
|
||||
const { transcript, segments } = await rpTranscribe(audio, { language, punctuation: true });
|
||||
return NextResponse.json({ transcript, segments });
|
||||
} catch (e) {
|
||||
const msg =
|
||||
e instanceof RunpodError ? e.message : e instanceof Error ? e.message : "Transcription failed.";
|
||||
return NextResponse.json({ error: msg }, { status: 502 });
|
||||
}
|
||||
}
|
||||
@@ -1140,4 +1140,80 @@
|
||||
.dash-root .ai-bubble { max-width: 86%; }
|
||||
.dash-root .ai-view { height: calc(100vh - 150px); }
|
||||
}
|
||||
|
||||
|
||||
/* =========================================================================
|
||||
Smart Gallery — the embedded @photo-gallery/sdk surface.
|
||||
The SDK is themed entirely through the token map in lib/gallery-api.ts
|
||||
(--apg-* -> this file's own vars), so the rules below only handle the
|
||||
host chrome: sizing, the demo banner, and the load/error placeholders.
|
||||
========================================================================= */
|
||||
|
||||
/* The gallery is the one view that wants the whole viewport: it has its own sidebar, toolbar and
|
||||
scrollers, so any height we leave on the table is wasted chrome. The old big PageHead cost ~90px;
|
||||
a slim header (~40px) + compact banner + tight gaps hand almost all of that back to the shell.
|
||||
96px = the dashboard's top padding + the slim header row; `.gal-shell` (flex:1; min-height:0)
|
||||
consumes whatever is left after the header and the optional demo banner. */
|
||||
.dash-root .gal { display: flex; flex-direction: column; gap: 10px; height: calc(100vh - 96px); min-height: 700px; }
|
||||
|
||||
/* Slim inline header — replaces the tall PageHead. One row, ~40px, so the shell keeps the height. */
|
||||
.dash-root .gal-head { display: flex; align-items: center; gap: 10px; min-height: 36px; flex: 0 0 auto; }
|
||||
.dash-root .gal-head-ic { width: 28px; height: 28px; border-radius: 9px; display: grid; place-items: center; color: #fff; background: var(--grad-brand); box-shadow: var(--glow-orange); flex: 0 0 auto; }
|
||||
.dash-root .gal-head-title { font-size: 16px; font-weight: 700; line-height: 1.1; margin: 0; }
|
||||
.dash-root .gal-head-sub { color: var(--muted); font-size: 12.5px; font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; }
|
||||
@media (max-width: 720px) { .dash-root .gal-head-sub { display: none; } }
|
||||
|
||||
/* `.view` sets `z-index: 1`, which makes it a stacking context and traps the SDK's
|
||||
full-screen overlays (lightbox z1000, editors z1050, camera z1080, modals z1100)
|
||||
underneath the topbar's `z-index: 20`. Opting this one view out of the stacking
|
||||
context lets those overlays cover the whole dashboard, as they must. The view still
|
||||
paints above the ambient `.dash-content::before` glow because it follows it in the DOM. */
|
||||
.dash-root .view.gal { z-index: auto; }
|
||||
|
||||
/* Compact single-line demo banner (~34px). Truncates rather than wrapping so it never steals a
|
||||
second row of height from the shell. */
|
||||
.dash-root .gal-banner { display: flex; align-items: center; gap: 8px; min-height: 34px; padding: 6px 12px; border-radius: 11px; border: 1px solid var(--border); background: color-mix(in srgb, var(--orange) 9%, var(--panel-2)); color: var(--text-2); font-size: 12px; font-weight: 500; flex: 0 0 auto; white-space: nowrap; overflow: hidden; }
|
||||
.dash-root .gal-banner span { overflow: hidden; text-overflow: ellipsis; }
|
||||
.dash-root .gal-banner svg { color: var(--orange); flex: 0 0 auto; }
|
||||
|
||||
/* The gallery's own viewport. `overflow: hidden` keeps the SDK's internal scrollers
|
||||
in charge; its full-screen overlays (lightbox/editor/camera) are position:fixed
|
||||
and deliberately escape this box to cover the whole dashboard. */
|
||||
.dash-root .gal-shell { flex: 1; min-height: 0; position: relative; border-radius: 18px; border: 1px solid var(--border); background: var(--panel-2); overflow: hidden; box-shadow: var(--card-hi), 0 1px 2px rgba(0, 0, 0, 0.18); }
|
||||
.dash-root[data-theme="dark"] .gal-shell { border: 0.5px solid #452b1a; border-radius: 20px; }
|
||||
|
||||
/* The SDK's embedded root fills this box. (--apg-overlay-top is set from the
|
||||
component's `style` prop — the SDK writes an inline default that a stylesheet
|
||||
rule could not override.) */
|
||||
.dash-root .gal-shell .apg { height: 100%; }
|
||||
|
||||
/* ---- Fullscreen (the SDK puts `.apg--fullscreen` on its root: position:fixed; inset:0) ----
|
||||
A position:fixed box is only clipped by an ancestor that is its CONTAINING BLOCK, which
|
||||
`overflow`/`border-radius`/`box-shadow` alone never create — only transform / filter /
|
||||
perspective / backdrop-filter / will-change / contain do. Nothing on the path
|
||||
(.dash-content > .view.gal > .gal-shell) uses any of those: `.view`'s `ds-fade` animates
|
||||
opacity only, and `.view.gal` already drops the `z-index: 1` stacking context. So the
|
||||
fullscreen root does escape today — these rules make that survive an edit above. */
|
||||
|
||||
/* `.dash-root .gal-shell .apg` (0,3,0) would otherwise out-specify the SDK's own sizing; with
|
||||
inset:0 driving the box, height must get out of the way. */
|
||||
.dash-root .gal-shell .apg.apg--fullscreen {
|
||||
height: auto;
|
||||
/* Above the topbar (z-index: 20) and the sidebar, below the SDK's own overlays (1000+). */
|
||||
z-index: 900;
|
||||
}
|
||||
|
||||
/* Belt and braces: if a future rule ever DOES make `.gal-shell` a containing block, an
|
||||
`overflow: hidden` on it would crop the fullscreen root to the embedded box. Drop the clip
|
||||
(and the rounded corner it exists to enforce) for exactly as long as fullscreen is on. */
|
||||
.dash-root .gal-shell:has(.apg--fullscreen) { overflow: visible; }
|
||||
|
||||
.dash-root .gal-placeholder { height: 100%; min-height: 320px; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 12px; text-align: center; padding: 28px; }
|
||||
.dash-root .gal-placeholder-ic { width: 66px; height: 66px; border-radius: 20px; display: grid; place-items: center; color: #fff; background: var(--grad-brand); box-shadow: var(--glow-orange); }
|
||||
.dash-root .gal-placeholder p { color: var(--muted); font-size: 13px; max-width: 420px; }
|
||||
.dash-root .gal-placeholder h3 { font-size: 16px; font-weight: 700; }
|
||||
.dash-root .gal-placeholder-error .gal-placeholder-ic { background: color-mix(in srgb, var(--red) 88%, #000); box-shadow: 0 10px 28px -12px color-mix(in srgb, var(--red) 60%, transparent); }
|
||||
|
||||
@media (max-width: 920px) {
|
||||
/* Narrower chrome: a little less top offset, and a smaller floor so short viewports still work. */
|
||||
.dash-root .gal { height: calc(100vh - 84px); min-height: 560px; }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user