From 7d06ba7a57928ce1c3af67e5807ef2c4de7955d2 Mon Sep 17 00:00:00 2001 From: robin Date: Mon, 22 Dec 2025 10:36:26 +0100 Subject: [PATCH] feat: robust FormData handling for AI services to improve N100 performance and stability --- src/components/CameraCapture.tsx | 8 +- src/components/ScanAndTasteFlow.tsx | 10 +- src/services/analyze-bottle-mistral.ts | 78 ++++++++++----- src/services/analyze-bottle.ts | 125 ++++++++++++++----------- src/services/magic-scan.ts | 36 +++++-- 5 files changed, 169 insertions(+), 88 deletions(-) diff --git a/src/components/CameraCapture.tsx b/src/components/CameraCapture.tsx index bef0101..d936beb 100644 --- a/src/components/CameraCapture.tsx +++ b/src/components/CameraCapture.tsx @@ -5,7 +5,6 @@ import { Camera, Upload, CheckCircle2, AlertCircle, X, Search, ExternalLink, Arr import { createClient } from '@/lib/supabase/client'; import { useRouter, useSearchParams } from 'next/navigation'; -import { analyzeBottle } from '@/services/analyze-bottle'; import { saveBottle } from '@/services/save-bottle'; import { BottleMetadata } from '@/types/whisky'; import { db } from '@/lib/db'; @@ -158,8 +157,13 @@ export default function CameraCapture({ onImageCaptured, onAnalysisComplete, onS return; } + const formData = new FormData(); + formData.append('file', processed.file); + formData.append('provider', aiProvider); + formData.append('locale', locale); + const startAi = performance.now(); - const response = await magicScan(compressedBase64, aiProvider, locale); + const response = await magicScan(formData); const endAi = performance.now(); const startPrep = performance.now(); diff --git a/src/components/ScanAndTasteFlow.tsx b/src/components/ScanAndTasteFlow.tsx index 9432d59..7a3a017 100644 --- a/src/components/ScanAndTasteFlow.tsx +++ b/src/components/ScanAndTasteFlow.tsx @@ -87,11 +87,15 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile }: ScanAnd const endComp = performance.now(); setProcessedImage(processed); - const cleanBase64 = processed.base64.split(',')[1] || processed.base64; - console.log('[ScanFlow] Calling magicScan service with compressed images (WebP)...'); + console.log('[ScanFlow] Calling magicScan service with FormData (optimized WebP)...'); + + const formData = new FormData(); + formData.append('file', processed.file); + formData.append('provider', 'gemini'); + formData.append('locale', locale); const startAi = performance.now(); - const result = await magicScan(cleanBase64, 'gemini', locale); + const result = await magicScan(formData); const endAi = performance.now(); const startPrep = performance.now(); diff --git a/src/services/analyze-bottle-mistral.ts b/src/services/analyze-bottle-mistral.ts index c4c2c5f..a4c4397 100644 --- a/src/services/analyze-bottle-mistral.ts +++ b/src/services/analyze-bottle-mistral.ts @@ -8,13 +8,33 @@ import { createHash } from 'crypto'; import { trackApiUsage } from './track-api-usage'; import { checkCreditBalance, deductCredits } from './credit-service'; -export async function analyzeBottleMistral(base64Image: string, tags?: string[], locale: string = 'de'): Promise { +// WICHTIG: Wir akzeptieren jetzt FormData statt Strings +export async function analyzeBottleMistral(input: any): Promise { if (!process.env.MISTRAL_API_KEY) { return { success: false, error: 'MISTRAL_API_KEY is not configured.' }; } let supabase; try { + // Helper to get value from either FormData or POJO + const getValue = (obj: any, key: string): any => { + if (obj && typeof obj.get === 'function') return obj.get(key); + if (obj && typeof obj[key] !== 'undefined') return obj[key]; + return null; + }; + + // 1. Daten extrahieren + const file = getValue(input, 'file') as File; + const tagsString = getValue(input, 'tags') as string; + const locale = getValue(input, 'locale') || 'de'; + + if (!file) { + return { success: false, error: 'Kein Bild empfangen.' }; + } + + const tags = tagsString ? (typeof tagsString === 'string' ? JSON.parse(tagsString) : tagsString) : []; + + // 2. Auth & Credits supabase = await createClient(); const { data: { session } } = await supabase.auth.getSession(); if (!session || !session.user) { @@ -22,18 +42,22 @@ export async function analyzeBottleMistral(base64Image: string, tags?: string[], } const userId = session.user.id; - const creditCheck = await checkCreditBalance(userId, 'gemini_ai'); if (!creditCheck.allowed) { return { success: false, - error: `Nicht genügend Credits. Du benötigst ${creditCheck.cost} Credits, hast aber nur ${creditCheck.balance}.` + error: `Nicht genügend Credits. Benötigt: ${creditCheck.cost}, Verfügbar: ${creditCheck.balance}.` }; } - const base64Data = base64Image.split(',')[1] || base64Image; - const imageHash = createHash('sha256').update(base64Data).digest('hex'); + // 3. Datei in Buffer umwandeln + const arrayBuffer = await file.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + // 4. Hash für Cache erstellen + const imageHash = createHash('sha256').update(buffer).digest('hex'); + + // Cache Check const { data: cachedResult } = await supabase .from('vision_cache') .select('result') @@ -47,10 +71,13 @@ export async function analyzeBottleMistral(base64Image: string, tags?: string[], }; } + // 5. Für Mistral vorbereiten const client = new Mistral({ apiKey: process.env.MISTRAL_API_KEY }); - const dataUrl = `data:image/webp;base64,${base64Data}`; + const base64Data = buffer.toString('base64'); + const mimeType = file.type || 'image/webp'; + const dataUrl = `data:${mimeType};base64,${base64Data}`; - const prompt = getSystemPrompt(tags ? tags.join(', ') : 'Keine Tags verfügbar', locale); + const prompt = getSystemPrompt(tags.length > 0 ? tags.join(', ') : 'Keine Tags verfügbar', locale); const chatResponse = await client.chat.complete({ model: 'mistral-large-latest', @@ -70,7 +97,15 @@ export async function analyzeBottleMistral(base64Image: string, tags?: string[], const rawContent = chatResponse.choices?.[0].message.content; if (!rawContent) throw new Error("Keine Antwort von Mistral"); - const jsonData = JSON.parse(rawContent as string); + let jsonData; + try { + jsonData = JSON.parse(rawContent as string); + } catch (e) { + const cleanedText = (rawContent as string).replace(/```json/g, '').replace(/```/g, ''); + jsonData = JSON.parse(cleanedText); + } + + if (Array.isArray(jsonData)) jsonData = jsonData[0]; // Extract search_string before validation const searchString = jsonData.search_string; @@ -90,7 +125,7 @@ export async function analyzeBottleMistral(base64Image: string, tags?: string[], // Track usage await trackApiUsage({ userId: userId, - apiType: 'gemini_ai', // Keep as generic 'gemini_ai' for now or update schema later + apiType: 'gemini_ai', endpoint: 'mistral/mistral-large', success: true }); @@ -112,22 +147,17 @@ export async function analyzeBottleMistral(base64Image: string, tags?: string[], } catch (error) { console.error('Mistral Analysis Error:', error); - // Track failed API call - try { - if (supabase) { - const { data: { session } } = await supabase.auth.getSession(); - if (session?.user) { - await trackApiUsage({ - userId: session.user.id, - apiType: 'gemini_ai', - endpoint: 'mistral/mistral-large', - success: false, - errorMessage: error instanceof Error ? error.message : 'Unknown error' - }); - } + if (supabase) { + const { data: { session } } = await supabase.auth.getSession(); + if (session?.user) { + await trackApiUsage({ + userId: session.user.id, + apiType: 'gemini_ai', + endpoint: 'mistral/mistral-large', + success: false, + errorMessage: error instanceof Error ? error.message : 'Unknown error' + }); } - } catch (trackError) { - console.error('Failed to track error:', trackError); } return { diff --git a/src/services/analyze-bottle.ts b/src/services/analyze-bottle.ts index 9ad7da8..c30fcd1 100644 --- a/src/services/analyze-bottle.ts +++ b/src/services/analyze-bottle.ts @@ -8,36 +8,60 @@ import { createHash } from 'crypto'; import { trackApiUsage } from './track-api-usage'; import { checkCreditBalance, deductCredits } from './credit-service'; -export async function analyzeBottle(base64Image: string, tags?: string[], locale: string = 'de'): Promise { +// WICHTIG: Wir akzeptieren jetzt FormData statt Strings +export async function analyzeBottle(input: any): Promise { if (!process.env.GEMINI_API_KEY) { return { success: false, error: 'GEMINI_API_KEY is not configured.' }; } - let supabase; // Declare supabase outside try block for error tracking access + let supabase; try { - // Initialize Supabase client inside the try block - supabase = await createClient(); - console.log('[analyzeBottle] Initialized Supabase client'); + // Helper to get value from either FormData or POJO + const getValue = (obj: any, key: string): any => { + if (obj && typeof obj.get === 'function') return obj.get(key); + if (obj && typeof obj[key] !== 'undefined') return obj[key]; + return null; + }; - // ... (auth and credit check remain same) ... + // 1. Daten extrahieren (leichtgewichtig für den N100) + const file = getValue(input, 'file') as File; + const tagsString = getValue(input, 'tags') as string; + const locale = getValue(input, 'locale') || 'de'; + + if (!file) { + return { success: false, error: 'Kein Bild empfangen.' }; + } + + // Tags müssen manuell geparst werden, da FormData alles flach macht + const tags = tagsString ? (typeof tagsString === 'string' ? JSON.parse(tagsString) : tagsString) : []; + + // 2. Auth & Credits (bleibt gleich) + supabase = await createClient(); const { data: { session } } = await supabase.auth.getSession(); + if (!session || !session.user) { return { success: false, error: 'Nicht autorisiert oder Session abgelaufen.' }; } const userId = session.user.id; - const creditCheck = await checkCreditBalance(userId, 'gemini_ai'); + if (!creditCheck.allowed) { return { success: false, - error: `Nicht genügend Credits. Du benötigst ${creditCheck.cost} Credits, hast aber nur ${creditCheck.balance}.` + error: `Nicht genügend Credits. Benötigt: ${creditCheck.cost}, Verfügbar: ${creditCheck.balance}.` }; } - const base64Data = base64Image.split(',')[1] || base64Image; - const imageHash = createHash('sha256').update(base64Data).digest('hex'); + // 3. Datei in Buffer umwandeln (Schneller als String-Manipulation) + // Der N100 mag ArrayBuffer lieber als riesige Base64 Strings im JSON + const arrayBuffer = await file.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + // 4. Hash für Cache erstellen (direkt vom Buffer -> sehr schnell) + const imageHash = createHash('sha256').update(buffer).digest('hex'); + + // Cache Check const { data: cachedResult } = await supabase .from('vision_cache') .select('result') @@ -45,42 +69,50 @@ export async function analyzeBottle(base64Image: string, tags?: string[], locale .maybeSingle(); if (cachedResult) { - return { - success: true, - data: cachedResult.result as any, - }; + return { success: true, data: cachedResult.result as any }; } - const instruction = getSystemPrompt(tags ? tags.join(', ') : 'No tags available', locale); + // 5. Für Gemini vorbereiten + // Wir müssen es hier zwar zu Base64 machen, aber Node.js (C++) macht das + // extrem effizient. Das Problem vorher war der JSON Parser von Next.js. + const base64Data = buffer.toString('base64'); + const mimeType = file.type || 'image/webp'; // Fallback + const instruction = getSystemPrompt(tags.length > 0 ? tags.join(', ') : 'No tags available', locale); + + // API Call const result = await geminiModel.generateContent([ { inlineData: { data: base64Data, - mimeType: 'image/webp', + mimeType: mimeType, }, }, { text: instruction }, ]); const responseText = result.response.text(); - let jsonData = JSON.parse(responseText); - if (Array.isArray(jsonData)) { - jsonData = jsonData[0]; + // JSON Parsing der ANTWORT (das ist klein, das schafft der N100 locker) + let jsonData; + try { + jsonData = JSON.parse(responseText); + } catch (e) { + // Fallback falls Gemini Markdown ```json Blöcke schickt + const cleanedText = responseText.replace(/```json/g, '').replace(/```/g, ''); + jsonData = JSON.parse(cleanedText); } - // Extract search_string if present + if (Array.isArray(jsonData)) jsonData = jsonData[0]; + const searchString = jsonData.search_string; delete jsonData.search_string; - if (!jsonData) { - throw new Error('Keine Daten in der KI-Antwort gefunden.'); - } + if (!jsonData) throw new Error('Keine Daten in der KI-Antwort gefunden.'); const validatedData = BottleMetadataSchema.parse(jsonData); - // Track successful API call + // 6. Tracking & Credits (bleibt gleich) await trackApiUsage({ userId: userId, apiType: 'gemini_ai', @@ -88,48 +120,35 @@ export async function analyzeBottle(base64Image: string, tags?: string[], locale success: true }); - // Deduct credits after successful API call - const creditDeduction = await deductCredits(userId, 'gemini_ai', 'Bottle analysis'); - if (!creditDeduction.success) { - console.error('Failed to deduct credits:', creditDeduction.error); - // Don't fail the analysis if credit deduction fails - } + await deductCredits(userId, 'gemini_ai', 'Bottle analysis'); - // 4. Store in Cache + // Cache speichern const { error: storeError } = await supabase .from('vision_cache') .insert({ hash: imageHash, result: validatedData }); - if (storeError) { - console.warn(`[AI Cache] Storage failed: ${storeError.message}`); - } else { - console.log(`[AI Cache] Stored new result for hash: ${imageHash}`); - } + if (storeError) console.warn(`[AI Cache] Storage failed: ${storeError.message}`); return { success: true, data: validatedData, search_string: searchString } as any; + } catch (error) { console.error('Gemini Analysis Error:', error); - - // Track failed API call - try { - if (supabase) { - const { data: { session } } = await supabase.auth.getSession(); - if (session?.user) { - await trackApiUsage({ - userId: session.user.id, - apiType: 'gemini_ai', - endpoint: 'generateContent', - success: false, - errorMessage: error instanceof Error ? error.message : 'Unknown error' - }); - } + // Error Tracking Logic (bleibt gleich) + if (supabase) { + const { data: { session } } = await supabase.auth.getSession(); + if (session?.user) { + await trackApiUsage({ + userId: session.user.id, + apiType: 'gemini_ai', + endpoint: 'generateContent', + success: false, + errorMessage: error instanceof Error ? error.message : 'Unknown error' + }); } - } catch (trackError) { - console.error('Failed to track error:', trackError); } return { @@ -137,4 +156,4 @@ export async function analyzeBottle(base64Image: string, tags?: string[], locale error: error instanceof Error ? error.message : 'An unknown error occurred during analysis.', }; } -} +} \ No newline at end of file diff --git a/src/services/magic-scan.ts b/src/services/magic-scan.ts index c55ddf2..5036f05 100644 --- a/src/services/magic-scan.ts +++ b/src/services/magic-scan.ts @@ -8,9 +8,20 @@ import { supabase } from '@/lib/supabase'; import { supabaseAdmin } from '@/lib/supabase-admin'; import { AnalysisResponse, BottleMetadata } from '@/types/whisky'; -export async function magicScan(base64Image: string, provider: 'gemini' | 'mistral' = 'gemini', locale: string = 'de'): Promise { +export async function magicScan(input: any): Promise { try { - console.log('[magicScan] Starting scan process...'); + // Helper to get value from either FormData or POJO + const getValue = (obj: any, key: string): string | null => { + if (obj && typeof obj.get === 'function') return obj.get(key) as string | null; + if (obj && typeof obj[key] !== 'undefined') return String(obj[key]); + return null; + }; + + const provider = getValue(input, 'provider') || 'gemini'; + const locale = getValue(input, 'locale') || 'de'; + + console.log(`[magicScan] Start (Provider: ${provider}, Locale: ${locale})`); + if (!supabase) { throw new Error('Supabase client is not initialized. Check environment variables.'); } @@ -20,12 +31,26 @@ export async function magicScan(base64Image: string, provider: 'gemini' | 'mistr const tagNames = systemTags.map(t => t.name); console.log(`[magicScan] Fetched ${tagNames.length} system tags`); + // Prepare context for sub-services (handle if input is not FormData) + let context = input; + if (!(input instanceof FormData)) { + console.log('[magicScan] Converting POJO to FormData for sub-services...'); + context = new FormData(); + // Copy keys if possible + if (input && typeof input === 'object') { + Object.keys(input).forEach(key => context.append(key, input[key])); + } + } + + // Add tags to context + context.set('tags', JSON.stringify(tagNames)); + // 1. AI Analysis let aiResponse: any; if (provider === 'mistral') { - aiResponse = await analyzeBottleMistral(base64Image, tagNames, locale); + aiResponse = await analyzeBottleMistral(context); } else { - aiResponse = await analyzeBottle(base64Image, tagNames, locale); + aiResponse = await analyzeBottle(context); } if (!aiResponse.success || !aiResponse.data) { @@ -40,7 +65,6 @@ export async function magicScan(base64Image: string, provider: 'gemini' | 'mistr } // 2. DB Cache Check (global_products) - // We use the regular supabase client for reading const { data: cacheHit } = await supabase .from('global_products') .select('wb_id') @@ -72,7 +96,7 @@ export async function magicScan(base64Image: string, provider: 'gemini' | 'mistr .from('global_products') .insert({ wb_id: braveResult.id, - full_name: searchString, // We save the search string as the name for future hits + full_name: searchString, }); if (saveError) {