328 lines
12 KiB
TypeScript
328 lines
12 KiB
TypeScript
'use server';
|
|
|
|
import { GoogleGenerativeAI, SchemaType, HarmCategory, HarmBlockThreshold } from '@google/generative-ai';
|
|
import { BottleMetadataSchema, BottleMetadata } from '@/types/whisky';
|
|
import { createClient } from '@/lib/supabase/server';
|
|
import { trackApiUsage } from '@/services/track-api-usage';
|
|
import { checkCreditBalance, deductCredits } from '@/services/credit-service';
|
|
import { getAIProvider, getOpenRouterClient, OPENROUTER_VISION_MODEL, OPENROUTER_PROVIDER_PREFERENCES } from '@/lib/openrouter';
|
|
import { normalizeWhiskyData } from '@/lib/distillery-matcher';
|
|
|
|
// Schema for Gemini Vision extraction
|
|
const visionSchema = {
|
|
description: "Whisky bottle label metadata extracted from image",
|
|
type: SchemaType.OBJECT as const,
|
|
properties: {
|
|
name: { type: SchemaType.STRING, description: "Full whisky name", nullable: false },
|
|
distillery: { type: SchemaType.STRING, description: "Distillery name", nullable: true },
|
|
bottler: { type: SchemaType.STRING, description: "Independent bottler if applicable", nullable: true },
|
|
category: { type: SchemaType.STRING, description: "Whisky category (Single Malt, Blended, Bourbon, etc.)", nullable: true },
|
|
abv: { type: SchemaType.NUMBER, description: "Alcohol by volume percentage", nullable: true },
|
|
age: { type: SchemaType.NUMBER, description: "Age statement in years", nullable: true },
|
|
vintage: { type: SchemaType.STRING, description: "Vintage/distillation year", nullable: true },
|
|
cask_type: { type: SchemaType.STRING, description: "Cask type (Sherry, Bourbon, Port, etc.)", nullable: true },
|
|
distilled_at: { type: SchemaType.STRING, description: "Distillation date", nullable: true },
|
|
bottled_at: { type: SchemaType.STRING, description: "Bottling date", nullable: true },
|
|
batch_info: { type: SchemaType.STRING, description: "Batch or cask number", nullable: true },
|
|
is_whisky: { type: SchemaType.BOOLEAN, description: "Whether this is a whisky product", nullable: false },
|
|
confidence: { type: SchemaType.NUMBER, description: "Confidence score 0-1", nullable: false },
|
|
},
|
|
required: ["name", "is_whisky", "confidence"],
|
|
};
|
|
|
|
export interface GeminiVisionResult {
|
|
success: boolean;
|
|
data?: BottleMetadata;
|
|
error?: string;
|
|
provider?: 'gemini' | 'openrouter';
|
|
perf?: {
|
|
apiCall: number;
|
|
total: number;
|
|
};
|
|
}
|
|
|
|
const VISION_PROMPT = `Analyze this whisky bottle label image and extract all visible metadata.
|
|
Look carefully for:
|
|
- Brand/Distillery name
|
|
- Bottle name or expression
|
|
- Age statement (e.g., "12 Years Old")
|
|
- ABV/Alcohol percentage
|
|
- Vintage year (if shown)
|
|
- Cask type (e.g., Sherry, Bourbon cask)
|
|
- Bottler name (if independent bottling)
|
|
- Category (Single Malt, Blended Malt, Bourbon, etc.)
|
|
|
|
Be precise and only include information you can clearly read from the label.
|
|
If you cannot read something clearly, leave it null.
|
|
|
|
Respond ONLY with valid JSON in this format:
|
|
{
|
|
"name": "Full whisky name",
|
|
"distillery": "Distillery name or null",
|
|
"bottler": "Bottler name or null",
|
|
"category": "Whisky category or null",
|
|
"abv": 46.0,
|
|
"age": 12,
|
|
"vintage": "2010 or null",
|
|
"cask_type": "Cask type or null",
|
|
"distilled_at": "Date or null",
|
|
"bottled_at": "Date or null",
|
|
"batch_info": "Batch info or null",
|
|
"is_whisky": true,
|
|
"confidence": 0.85
|
|
}`;
|
|
|
|
/**
|
|
* Sleep helper for retry delays
|
|
*/
|
|
function sleep(ms: number): Promise<void> {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
/**
|
|
* Analyze whisky label with OpenRouter (Gemma 3 27B)
|
|
* Includes retry logic for 429 rate limit errors
|
|
*/
|
|
async function analyzeWithOpenRouter(base64Data: string, mimeType: string): Promise<{ data: any; apiTime: number; responseText: string }> {
|
|
const client = getOpenRouterClient();
|
|
const startApi = performance.now();
|
|
const maxRetries = 3;
|
|
let lastError: any = null;
|
|
|
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
try {
|
|
const response = await client.chat.completions.create({
|
|
model: OPENROUTER_VISION_MODEL,
|
|
messages: [
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'image_url',
|
|
image_url: {
|
|
url: `data:${mimeType};base64,${base64Data}`,
|
|
},
|
|
},
|
|
{
|
|
type: 'text',
|
|
text: VISION_PROMPT,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
temperature: 0.1,
|
|
max_tokens: 1024,
|
|
// @ts-ignore - OpenRouter-specific field
|
|
provider: OPENROUTER_PROVIDER_PREFERENCES,
|
|
});
|
|
|
|
const endApi = performance.now();
|
|
const content = response.choices[0]?.message?.content || '{}';
|
|
|
|
// Extract JSON from response (may have markdown code blocks)
|
|
let jsonStr = content;
|
|
const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
if (jsonMatch) {
|
|
jsonStr = jsonMatch[1].trim();
|
|
}
|
|
|
|
return {
|
|
data: JSON.parse(jsonStr),
|
|
apiTime: endApi - startApi,
|
|
responseText: content
|
|
};
|
|
|
|
} catch (error: any) {
|
|
lastError = error;
|
|
const status = error?.status || error?.response?.status;
|
|
|
|
// Only retry on 429 (rate limit) or 503 (service unavailable)
|
|
if (status === 429 || status === 503) {
|
|
const delay = Math.pow(2, attempt) * 1000; // 2s, 4s, 8s
|
|
console.log(`[OpenRouter] Rate limited (${status}), retry ${attempt}/${maxRetries} in ${delay}ms...`);
|
|
await sleep(delay);
|
|
continue;
|
|
}
|
|
|
|
// Other errors - don't retry
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
// All retries exhausted
|
|
throw lastError || new Error('OpenRouter request failed after retries');
|
|
}
|
|
|
|
/**
|
|
* Analyze whisky label with Gemini
|
|
*/
|
|
async function analyzeWithGemini(base64Data: string, mimeType: string): Promise<{ data: any; apiTime: number; responseText: string }> {
|
|
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!);
|
|
const model = genAI.getGenerativeModel({
|
|
model: 'gemini-2.5-flash',
|
|
generationConfig: {
|
|
responseMimeType: "application/json",
|
|
responseSchema: visionSchema as any,
|
|
temperature: 0.1,
|
|
},
|
|
safetySettings: [
|
|
{ category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_NONE },
|
|
{ category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_NONE },
|
|
{ category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.BLOCK_NONE },
|
|
{ category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: HarmBlockThreshold.BLOCK_NONE },
|
|
] as any,
|
|
});
|
|
|
|
const startApi = performance.now();
|
|
const result = await model.generateContent([
|
|
{ inlineData: { data: base64Data, mimeType } },
|
|
{ text: VISION_PROMPT },
|
|
]);
|
|
const endApi = performance.now();
|
|
|
|
const responseText = result.response.text();
|
|
return {
|
|
data: JSON.parse(responseText),
|
|
apiTime: endApi - startApi,
|
|
responseText: responseText
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Analyze a whisky bottle label image using configured AI provider
|
|
*
|
|
* Provider is controlled by AI_PROVIDER env variable:
|
|
* - "openrouter" (default) - Uses OpenRouter with Gemma 3 27B
|
|
* - "gemini" - Uses Google Gemini 2.5 Flash
|
|
*
|
|
* @param imageBase64 - Base64 encoded image (with data URL prefix)
|
|
* @returns GeminiVisionResult with extracted metadata
|
|
*/
|
|
export async function analyzeLabelWithGemini(imageBase64: string): Promise<GeminiVisionResult> {
|
|
const startTotal = performance.now();
|
|
const provider = getAIProvider();
|
|
|
|
// Check API key based on provider
|
|
if (provider === 'gemini' && !process.env.GEMINI_API_KEY) {
|
|
return { success: false, error: 'GEMINI_API_KEY is not configured.' };
|
|
}
|
|
if (provider === 'openrouter' && !process.env.OPENROUTER_API_KEY) {
|
|
return { success: false, error: 'OPENROUTER_API_KEY is not configured.' };
|
|
}
|
|
|
|
if (!imageBase64 || imageBase64.length < 100) {
|
|
return { success: false, error: 'Invalid image data provided.' };
|
|
}
|
|
|
|
try {
|
|
// Auth check
|
|
const supabase = await createClient();
|
|
const { data: { user } } = await supabase.auth.getUser();
|
|
|
|
if (!user) {
|
|
return { success: false, error: 'Not authorized.' };
|
|
}
|
|
|
|
// Credit check (use same type for both providers)
|
|
const creditCheck = await checkCreditBalance(user.id, 'gemini_ai');
|
|
if (!creditCheck.allowed) {
|
|
return {
|
|
success: false,
|
|
error: `Insufficient credits. Required: ${creditCheck.cost}, Available: ${creditCheck.balance}.`
|
|
};
|
|
}
|
|
|
|
// Extract base64 data (remove data URL prefix if present)
|
|
let base64Data = imageBase64;
|
|
let mimeType = 'image/webp';
|
|
|
|
if (imageBase64.startsWith('data:')) {
|
|
const matches = imageBase64.match(/^data:([^;]+);base64,(.+)$/);
|
|
if (matches) {
|
|
mimeType = matches[1];
|
|
base64Data = matches[2];
|
|
}
|
|
}
|
|
|
|
// Call appropriate provider
|
|
console.log(`[Vision] Using provider: ${provider}`);
|
|
let result: { data: any; apiTime: number; responseText: string };
|
|
|
|
if (provider === 'openrouter') {
|
|
result = await analyzeWithOpenRouter(base64Data, mimeType);
|
|
} else {
|
|
result = await analyzeWithGemini(base64Data, mimeType);
|
|
}
|
|
|
|
// Validate with Zod schema
|
|
const validatedData = BottleMetadataSchema.parse(result.data);
|
|
|
|
// ========================================
|
|
// NORMALIZE DISTILLERY NAME
|
|
// ========================================
|
|
console.log(`[Vision] 🔍 RAW FROM AI: name="${validatedData.name}", distillery="${validatedData.distillery}"`);
|
|
|
|
const normalized = normalizeWhiskyData(
|
|
validatedData.name || '',
|
|
validatedData.distillery || ''
|
|
);
|
|
|
|
console.log(`[Vision] ✅ AFTER FUSE: name="${normalized.name}", distillery="${normalized.distillery}", matched=${normalized.distilleryMatched}`);
|
|
const finalData = {
|
|
...validatedData,
|
|
name: normalized.name || validatedData.name,
|
|
distillery: normalized.distillery || validatedData.distillery,
|
|
};
|
|
|
|
console.log(`[Vision] Normalized: distillery="${normalized.distillery}", name="${normalized.name}"`);
|
|
|
|
// Track usage and deduct credits
|
|
await trackApiUsage({
|
|
userId: user.id,
|
|
apiType: 'gemini_ai', // Keep same type for tracking
|
|
endpoint: `analyzeLabelWith${provider === 'openrouter' ? 'OpenRouter' : 'Gemini'}`,
|
|
success: true,
|
|
provider: provider,
|
|
model: provider === 'openrouter' ? OPENROUTER_VISION_MODEL : 'gemini-2.5-flash',
|
|
responseText: result.responseText
|
|
});
|
|
await deductCredits(user.id, 'gemini_ai', `Vision label analysis (${provider})`);
|
|
|
|
return {
|
|
success: true,
|
|
data: finalData,
|
|
provider,
|
|
perf: {
|
|
apiCall: result.apiTime,
|
|
total: performance.now() - startTotal,
|
|
}
|
|
};
|
|
|
|
} catch (error: any) {
|
|
console.error(`[Vision] Analysis failed (${provider}):`, error);
|
|
|
|
// Try to track the failure
|
|
try {
|
|
const supabase = await createClient();
|
|
const { data: { user } } = await supabase.auth.getUser();
|
|
if (user) {
|
|
await trackApiUsage({
|
|
userId: user.id,
|
|
apiType: 'gemini_ai',
|
|
endpoint: `analyzeLabelWith${provider === 'openrouter' ? 'OpenRouter' : 'Gemini'}`,
|
|
success: false,
|
|
errorMessage: error.message
|
|
});
|
|
}
|
|
} catch (trackError) {
|
|
console.warn('[Vision] Failed to track error:', trackError);
|
|
}
|
|
|
|
return {
|
|
success: false,
|
|
error: error.message || 'Vision analysis failed.',
|
|
provider,
|
|
};
|
|
}
|
|
}
|