'use server'; import { GoogleGenerativeAI, SchemaType, HarmCategory, HarmBlockThreshold } from '@google/generative-ai'; import { BottleMetadataSchema, BottleMetadata } from '@/types/whisky'; import { createClient } from '@/lib/supabase/server'; import { trackApiUsage } from '@/services/track-api-usage'; import { checkCreditBalance, deductCredits } from '@/services/credit-service'; import { getAIProvider, getOpenRouterClient, OPENROUTER_VISION_MODEL, OPENROUTER_PROVIDER_PREFERENCES } from '@/lib/openrouter'; import { normalizeWhiskyData } from '@/lib/distillery-matcher'; // Schema for Gemini Vision extraction const visionSchema = { description: "Whisky bottle label metadata extracted from image", type: SchemaType.OBJECT as const, properties: { name: { type: SchemaType.STRING, description: "Full whisky name", nullable: false }, distillery: { type: SchemaType.STRING, description: "Distillery name", nullable: true }, bottler: { type: SchemaType.STRING, description: "Independent bottler if applicable", nullable: true }, category: { type: SchemaType.STRING, description: "Whisky category (Single Malt, Blended, Bourbon, etc.)", nullable: true }, abv: { type: SchemaType.NUMBER, description: "Alcohol by volume percentage", nullable: true }, age: { type: SchemaType.NUMBER, description: "Age statement in years", nullable: true }, vintage: { type: SchemaType.STRING, description: "Vintage/distillation year", nullable: true }, cask_type: { type: SchemaType.STRING, description: "Cask type (Sherry, Bourbon, Port, etc.)", nullable: true }, distilled_at: { type: SchemaType.STRING, description: "Distillation date", nullable: true }, bottled_at: { type: SchemaType.STRING, description: "Bottling date", nullable: true }, batch_info: { type: SchemaType.STRING, description: "Batch or cask number", nullable: true }, is_whisky: { type: SchemaType.BOOLEAN, description: "Whether this is a whisky product", nullable: false }, confidence: { type: SchemaType.NUMBER, description: "Confidence score 0-1", nullable: false }, }, required: ["name", "is_whisky", "confidence"], }; export interface GeminiVisionResult { success: boolean; data?: BottleMetadata; error?: string; provider?: 'gemini' | 'openrouter'; perf?: { apiCall: number; total: number; }; } const VISION_PROMPT = `Analyze this whisky bottle label image and extract all visible metadata. Look carefully for: - Brand/Distillery name - Bottle name or expression - Age statement (e.g., "12 Years Old") - ABV/Alcohol percentage - Vintage year (if shown) - Cask type (e.g., Sherry, Bourbon cask) - Bottler name (if independent bottling) - Category (Single Malt, Blended Malt, Bourbon, etc.) Be precise and only include information you can clearly read from the label. If you cannot read something clearly, leave it null. Respond ONLY with valid JSON in this format: { "name": "Full whisky name", "distillery": "Distillery name or null", "bottler": "Bottler name or null", "category": "Whisky category or null", "abv": 46.0, "age": 12, "vintage": "2010 or null", "cask_type": "Cask type or null", "distilled_at": "Date or null", "bottled_at": "Date or null", "batch_info": "Batch info or null", "is_whisky": true, "confidence": 0.85 }`; /** * Sleep helper for retry delays */ function sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } /** * Analyze whisky label with OpenRouter (Gemma 3 27B) * Includes retry logic for 429 rate limit errors */ async function analyzeWithOpenRouter(base64Data: string, mimeType: string): Promise<{ data: any; apiTime: number; responseText: string }> { const client = getOpenRouterClient(); const startApi = performance.now(); const maxRetries = 3; let lastError: any = null; for (let attempt = 1; attempt <= maxRetries; attempt++) { try { const response = await client.chat.completions.create({ model: OPENROUTER_VISION_MODEL, messages: [ { role: 'user', content: [ { type: 'image_url', image_url: { url: `data:${mimeType};base64,${base64Data}`, }, }, { type: 'text', text: VISION_PROMPT, }, ], }, ], temperature: 0.1, max_tokens: 1024, // @ts-ignore - OpenRouter-specific field provider: OPENROUTER_PROVIDER_PREFERENCES, }); const endApi = performance.now(); const content = response.choices[0]?.message?.content || '{}'; // Extract JSON from response (may have markdown code blocks) let jsonStr = content; const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/); if (jsonMatch) { jsonStr = jsonMatch[1].trim(); } return { data: JSON.parse(jsonStr), apiTime: endApi - startApi, responseText: content }; } catch (error: any) { lastError = error; const status = error?.status || error?.response?.status; // Only retry on 429 (rate limit) or 503 (service unavailable) if (status === 429 || status === 503) { const delay = Math.pow(2, attempt) * 1000; // 2s, 4s, 8s console.log(`[OpenRouter] Rate limited (${status}), retry ${attempt}/${maxRetries} in ${delay}ms...`); await sleep(delay); continue; } // Other errors - don't retry throw error; } } // All retries exhausted throw lastError || new Error('OpenRouter request failed after retries'); } /** * Analyze whisky label with Gemini */ async function analyzeWithGemini(base64Data: string, mimeType: string): Promise<{ data: any; apiTime: number; responseText: string }> { const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!); const model = genAI.getGenerativeModel({ model: 'gemini-2.5-flash', generationConfig: { responseMimeType: "application/json", responseSchema: visionSchema as any, temperature: 0.1, }, safetySettings: [ { category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_NONE }, { category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_NONE }, { category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.BLOCK_NONE }, { category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: HarmBlockThreshold.BLOCK_NONE }, ] as any, }); const startApi = performance.now(); const result = await model.generateContent([ { inlineData: { data: base64Data, mimeType } }, { text: VISION_PROMPT }, ]); const endApi = performance.now(); const responseText = result.response.text(); return { data: JSON.parse(responseText), apiTime: endApi - startApi, responseText: responseText }; } /** * Analyze a whisky bottle label image using configured AI provider * * Provider is controlled by AI_PROVIDER env variable: * - "openrouter" (default) - Uses OpenRouter with Gemma 3 27B * - "gemini" - Uses Google Gemini 2.5 Flash * * @param imageBase64 - Base64 encoded image (with data URL prefix) * @returns GeminiVisionResult with extracted metadata */ export async function analyzeLabelWithGemini(imageBase64: string): Promise { const startTotal = performance.now(); const provider = getAIProvider(); // Check API key based on provider if (provider === 'gemini' && !process.env.GEMINI_API_KEY) { return { success: false, error: 'GEMINI_API_KEY is not configured.' }; } if (provider === 'openrouter' && !process.env.OPENROUTER_API_KEY) { return { success: false, error: 'OPENROUTER_API_KEY is not configured.' }; } if (!imageBase64 || imageBase64.length < 100) { return { success: false, error: 'Invalid image data provided.' }; } try { // Auth check const supabase = await createClient(); const { data: { user } } = await supabase.auth.getUser(); if (!user) { return { success: false, error: 'Not authorized.' }; } // Credit check (use same type for both providers) const creditCheck = await checkCreditBalance(user.id, 'gemini_ai'); if (!creditCheck.allowed) { return { success: false, error: `Insufficient credits. Required: ${creditCheck.cost}, Available: ${creditCheck.balance}.` }; } // Extract base64 data (remove data URL prefix if present) let base64Data = imageBase64; let mimeType = 'image/webp'; if (imageBase64.startsWith('data:')) { const matches = imageBase64.match(/^data:([^;]+);base64,(.+)$/); if (matches) { mimeType = matches[1]; base64Data = matches[2]; } } // Call appropriate provider console.log(`[Vision] Using provider: ${provider}`); let result: { data: any; apiTime: number; responseText: string }; if (provider === 'openrouter') { result = await analyzeWithOpenRouter(base64Data, mimeType); } else { result = await analyzeWithGemini(base64Data, mimeType); } // Validate with Zod schema const validatedData = BottleMetadataSchema.parse(result.data); // ======================================== // NORMALIZE DISTILLERY NAME // ======================================== console.log(`[Vision] 🔍 RAW FROM AI: name="${validatedData.name}", distillery="${validatedData.distillery}"`); const normalized = normalizeWhiskyData( validatedData.name || '', validatedData.distillery || '' ); console.log(`[Vision] ✅ AFTER FUSE: name="${normalized.name}", distillery="${normalized.distillery}", matched=${normalized.distilleryMatched}`); const finalData = { ...validatedData, name: normalized.name || validatedData.name, distillery: normalized.distillery || validatedData.distillery, }; console.log(`[Vision] Normalized: distillery="${normalized.distillery}", name="${normalized.name}"`); // Track usage and deduct credits await trackApiUsage({ userId: user.id, apiType: 'gemini_ai', // Keep same type for tracking endpoint: `analyzeLabelWith${provider === 'openrouter' ? 'OpenRouter' : 'Gemini'}`, success: true, provider: provider, model: provider === 'openrouter' ? OPENROUTER_VISION_MODEL : 'gemini-2.5-flash', responseText: result.responseText }); await deductCredits(user.id, 'gemini_ai', `Vision label analysis (${provider})`); return { success: true, data: finalData, provider, perf: { apiCall: result.apiTime, total: performance.now() - startTotal, } }; } catch (error: any) { console.error(`[Vision] Analysis failed (${provider}):`, error); // Try to track the failure try { const supabase = await createClient(); const { data: { user } } = await supabase.auth.getUser(); if (user) { await trackApiUsage({ userId: user.id, apiType: 'gemini_ai', endpoint: `analyzeLabelWith${provider === 'openrouter' ? 'OpenRouter' : 'Gemini'}`, success: false, errorMessage: error.message }); } } catch (trackError) { console.warn('[Vision] Failed to track error:', trackError); } return { success: false, error: error.message || 'Vision analysis failed.', provider, }; } }