From 68ac857091f6e30c40155586dabb97158e52b3de Mon Sep 17 00:00:00 2001 From: robin Date: Mon, 5 Jan 2026 23:33:10 +0100 Subject: [PATCH] feat: restore scan quality, implement standardized naming, and add cask_type integration --- .eslintrc.json | 15 ++ package.json | 1 + pnpm-lock.yaml | 23 ++ src/app/actions/gemini-vision.ts | 327 --------------------------- src/app/actions/scan-label.ts | 308 -------------------------- src/app/actions/scanner.ts | 330 ++++++++++++++++++++++++++++ src/components/BottleDetails.tsx | 5 + src/components/CameraCapture.tsx | 2 +- src/components/EditBottleForm.tsx | 35 ++- src/components/TastingEditor.tsx | 39 +++- src/components/UploadQueue.tsx | 2 +- src/hooks/useScanner.ts | 4 +- src/i18n/de.ts | 15 ++ src/lib/ai-prompts.ts | 43 +++- src/services/analyze-bottle.ts | 254 --------------------- src/services/bulk-scan.ts | 101 +-------- src/services/discover-whiskybase.ts | 10 +- src/services/magic-scan.ts | 4 +- src/services/save-bottle.ts | 1 + src/services/update-bottle.ts | 1 + src/types/whisky.ts | 25 ++- src/utils/formatWhiskyName.ts | 51 +++++ 22 files changed, 576 insertions(+), 1020 deletions(-) create mode 100644 .eslintrc.json delete mode 100644 src/app/actions/gemini-vision.ts delete mode 100644 src/app/actions/scan-label.ts create mode 100644 src/app/actions/scanner.ts delete mode 100644 src/services/analyze-bottle.ts create mode 100644 src/utils/formatWhiskyName.ts diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..28b8938 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,15 @@ +{ + "extends": [ + "next/core-web-vitals" + ], + "plugins": [ + "security" + ], + "rules": { + "security/detect-object-injection": "warn", + "security/detect-unsafe-regex": "warn", + "security/detect-eval-with-expression": "error", + "security/detect-non-literal-fs-filename": "warn", + "security/detect-child-process": "warn" + } +} \ No newline at end of file diff --git a/package.json b/package.json index 07f8c19..69803e2 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "autoprefixer": "^10.0.1", "eslint": "^8", "eslint-config-next": "16.1.0", + "eslint-plugin-security": "^2.1.1", "jsdom": "^27.3.0", "postcss": "^8", "tailwindcss": "^3.3.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 41f6423..0f900c6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -123,6 +123,9 @@ importers: eslint-config-next: specifier: 16.1.0 version: 16.1.0(@typescript-eslint/parser@8.50.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3) + eslint-plugin-security: + specifier: ^2.1.1 + version: 2.1.1 jsdom: specifier: ^27.3.0 version: 27.3.0 @@ -1729,6 +1732,9 @@ packages: peerDependencies: eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 || ^9.7 + eslint-plugin-security@2.1.1: + resolution: {integrity: sha512-7cspIGj7WTfR3EhaILzAPcfCo5R9FbeWvbgsPYWivSurTBKW88VQxtP3c4aWMG9Hz/GfJlJVdXEJ3c8LqS+u2w==} + eslint-scope@7.2.2: resolution: {integrity: sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -2629,6 +2635,10 @@ packages: regenerator-runtime@0.13.11: resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==} + regexp-tree@0.1.27: + resolution: {integrity: sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==} + hasBin: true + regexp.prototype.flags@1.5.4: resolution: {integrity: sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==} engines: {node: '>= 0.4'} @@ -2685,6 +2695,9 @@ packages: resolution: {integrity: sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==} engines: {node: '>= 0.4'} + safe-regex@2.1.1: + resolution: {integrity: sha512-rx+x8AMzKb5Q5lQ95Zoi6ZbJqwCLkqi3XuJXp5P3rT8OEc6sZCJG5AE5dU3lsgRr/F4Bs31jSlVN+j5KrsGu9A==} + safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} @@ -4779,6 +4792,10 @@ snapshots: string.prototype.matchall: 4.0.12 string.prototype.repeat: 1.0.0 + eslint-plugin-security@2.1.1: + dependencies: + safe-regex: 2.1.1 + eslint-scope@7.2.2: dependencies: esrecurse: 4.3.0 @@ -5678,6 +5695,8 @@ snapshots: regenerator-runtime@0.13.11: {} + regexp-tree@0.1.27: {} + regexp.prototype.flags@1.5.4: dependencies: call-bind: 1.0.8 @@ -5764,6 +5783,10 @@ snapshots: es-errors: 1.3.0 is-regex: 1.2.1 + safe-regex@2.1.1: + dependencies: + regexp-tree: 0.1.27 + safer-buffer@2.1.2: {} saxes@6.0.0: diff --git a/src/app/actions/gemini-vision.ts b/src/app/actions/gemini-vision.ts deleted file mode 100644 index d151b8c..0000000 --- a/src/app/actions/gemini-vision.ts +++ /dev/null @@ -1,327 +0,0 @@ -'use server'; - -import { GoogleGenerativeAI, SchemaType, HarmCategory, HarmBlockThreshold } from '@google/generative-ai'; -import { BottleMetadataSchema, BottleMetadata } from '@/types/whisky'; -import { createClient } from '@/lib/supabase/server'; -import { trackApiUsage } from '@/services/track-api-usage'; -import { checkCreditBalance, deductCredits } from '@/services/credit-service'; -import { getAIProvider, getOpenRouterClient, OPENROUTER_VISION_MODEL, OPENROUTER_PROVIDER_PREFERENCES } from '@/lib/openrouter'; -import { normalizeWhiskyData } from '@/lib/distillery-matcher'; - -// Schema for Gemini Vision extraction -const visionSchema = { - description: "Whisky bottle label metadata extracted from image", - type: SchemaType.OBJECT as const, - properties: { - name: { type: SchemaType.STRING, description: "Full whisky name", nullable: false }, - distillery: { type: SchemaType.STRING, description: "Distillery name", nullable: true }, - bottler: { type: SchemaType.STRING, description: "Independent bottler if applicable", nullable: true }, - category: { type: SchemaType.STRING, description: "Whisky category (Single Malt, Blended, Bourbon, etc.)", nullable: true }, - abv: { type: SchemaType.NUMBER, description: "Alcohol by volume percentage", nullable: true }, - age: { type: SchemaType.NUMBER, description: "Age statement in years", nullable: true }, - vintage: { type: SchemaType.STRING, description: "Vintage/distillation year", nullable: true }, - cask_type: { type: SchemaType.STRING, description: "Cask type (Sherry, Bourbon, Port, etc.)", nullable: true }, - distilled_at: { type: SchemaType.STRING, description: "Distillation date", nullable: true }, - bottled_at: { type: SchemaType.STRING, description: "Bottling date", nullable: true }, - batch_info: { type: SchemaType.STRING, description: "Batch or cask number", nullable: true }, - is_whisky: { type: SchemaType.BOOLEAN, description: "Whether this is a whisky product", nullable: false }, - confidence: { type: SchemaType.NUMBER, description: "Confidence score 0-1", nullable: false }, - }, - required: ["name", "is_whisky", "confidence"], -}; - -export interface GeminiVisionResult { - success: boolean; - data?: BottleMetadata; - error?: string; - provider?: 'gemini' | 'openrouter'; - perf?: { - apiCall: number; - total: number; - }; -} - -const VISION_PROMPT = `Analyze this whisky bottle label image and extract all visible metadata. -Look carefully for: -- Brand/Distillery name -- Bottle name or expression -- Age statement (e.g., "12 Years Old") -- ABV/Alcohol percentage -- Vintage year (if shown) -- Cask type (e.g., Sherry, Bourbon cask) -- Bottler name (if independent bottling) -- Category (Single Malt, Blended Malt, Bourbon, etc.) - -Be precise and only include information you can clearly read from the label. -If you cannot read something clearly, leave it null. - -Respond ONLY with valid JSON in this format: -{ - "name": "Full whisky name", - "distillery": "Distillery name or null", - "bottler": "Bottler name or null", - "category": "Whisky category or null", - "abv": 46.0, - "age": 12, - "vintage": "2010 or null", - "cask_type": "Cask type or null", - "distilled_at": "Date or null", - "bottled_at": "Date or null", - "batch_info": "Batch info or null", - "is_whisky": true, - "confidence": 0.85 -}`; - -/** - * Sleep helper for retry delays - */ -function sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -/** - * Analyze whisky label with OpenRouter (Gemma 3 27B) - * Includes retry logic for 429 rate limit errors - */ -async function analyzeWithOpenRouter(base64Data: string, mimeType: string): Promise<{ data: any; apiTime: number; responseText: string }> { - const client = getOpenRouterClient(); - const startApi = performance.now(); - const maxRetries = 3; - let lastError: any = null; - - for (let attempt = 1; attempt <= maxRetries; attempt++) { - try { - const response = await client.chat.completions.create({ - model: OPENROUTER_VISION_MODEL, - messages: [ - { - role: 'user', - content: [ - { - type: 'image_url', - image_url: { - url: `data:${mimeType};base64,${base64Data}`, - }, - }, - { - type: 'text', - text: VISION_PROMPT, - }, - ], - }, - ], - temperature: 0.1, - max_tokens: 1024, - // @ts-ignore - OpenRouter-specific field - provider: OPENROUTER_PROVIDER_PREFERENCES, - }); - - const endApi = performance.now(); - const content = response.choices[0]?.message?.content || '{}'; - - // Extract JSON from response (may have markdown code blocks) - let jsonStr = content; - const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/); - if (jsonMatch) { - jsonStr = jsonMatch[1].trim(); - } - - return { - data: JSON.parse(jsonStr), - apiTime: endApi - startApi, - responseText: content - }; - - } catch (error: any) { - lastError = error; - const status = error?.status || error?.response?.status; - - // Only retry on 429 (rate limit) or 503 (service unavailable) - if (status === 429 || status === 503) { - const delay = Math.pow(2, attempt) * 1000; // 2s, 4s, 8s - console.log(`[OpenRouter] Rate limited (${status}), retry ${attempt}/${maxRetries} in ${delay}ms...`); - await sleep(delay); - continue; - } - - // Other errors - don't retry - throw error; - } - } - - // All retries exhausted - throw lastError || new Error('OpenRouter request failed after retries'); -} - -/** - * Analyze whisky label with Gemini - */ -async function analyzeWithGemini(base64Data: string, mimeType: string): Promise<{ data: any; apiTime: number; responseText: string }> { - const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!); - const model = genAI.getGenerativeModel({ - model: 'gemini-2.5-flash', - generationConfig: { - responseMimeType: "application/json", - responseSchema: visionSchema as any, - temperature: 0.1, - }, - safetySettings: [ - { category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_NONE }, - { category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_NONE }, - { category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.BLOCK_NONE }, - { category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: HarmBlockThreshold.BLOCK_NONE }, - ] as any, - }); - - const startApi = performance.now(); - const result = await model.generateContent([ - { inlineData: { data: base64Data, mimeType } }, - { text: VISION_PROMPT }, - ]); - const endApi = performance.now(); - - const responseText = result.response.text(); - return { - data: JSON.parse(responseText), - apiTime: endApi - startApi, - responseText: responseText - }; -} - -/** - * Analyze a whisky bottle label image using configured AI provider - * - * Provider is controlled by AI_PROVIDER env variable: - * - "openrouter" (default) - Uses OpenRouter with Gemma 3 27B - * - "gemini" - Uses Google Gemini 2.5 Flash - * - * @param imageBase64 - Base64 encoded image (with data URL prefix) - * @returns GeminiVisionResult with extracted metadata - */ -export async function analyzeLabelWithGemini(imageBase64: string): Promise { - const startTotal = performance.now(); - const provider = getAIProvider(); - - // Check API key based on provider - if (provider === 'gemini' && !process.env.GEMINI_API_KEY) { - return { success: false, error: 'GEMINI_API_KEY is not configured.' }; - } - if (provider === 'openrouter' && !process.env.OPENROUTER_API_KEY) { - return { success: false, error: 'OPENROUTER_API_KEY is not configured.' }; - } - - if (!imageBase64 || imageBase64.length < 100) { - return { success: false, error: 'Invalid image data provided.' }; - } - - try { - // Auth check - const supabase = await createClient(); - const { data: { user } } = await supabase.auth.getUser(); - - if (!user) { - return { success: false, error: 'Not authorized.' }; - } - - // Credit check (use same type for both providers) - const creditCheck = await checkCreditBalance(user.id, 'gemini_ai'); - if (!creditCheck.allowed) { - return { - success: false, - error: `Insufficient credits. Required: ${creditCheck.cost}, Available: ${creditCheck.balance}.` - }; - } - - // Extract base64 data (remove data URL prefix if present) - let base64Data = imageBase64; - let mimeType = 'image/webp'; - - if (imageBase64.startsWith('data:')) { - const matches = imageBase64.match(/^data:([^;]+);base64,(.+)$/); - if (matches) { - mimeType = matches[1]; - base64Data = matches[2]; - } - } - - // Call appropriate provider - console.log(`[Vision] Using provider: ${provider}`); - let result: { data: any; apiTime: number; responseText: string }; - - if (provider === 'openrouter') { - result = await analyzeWithOpenRouter(base64Data, mimeType); - } else { - result = await analyzeWithGemini(base64Data, mimeType); - } - - // Validate with Zod schema - const validatedData = BottleMetadataSchema.parse(result.data); - - // ======================================== - // NORMALIZE DISTILLERY NAME - // ======================================== - console.log(`[Vision] 🔍 RAW FROM AI: name="${validatedData.name}", distillery="${validatedData.distillery}"`); - - const normalized = normalizeWhiskyData( - validatedData.name || '', - validatedData.distillery || '' - ); - - console.log(`[Vision] ✅ AFTER FUSE: name="${normalized.name}", distillery="${normalized.distillery}", matched=${normalized.distilleryMatched}`); - const finalData = { - ...validatedData, - name: normalized.name || validatedData.name, - distillery: normalized.distillery || validatedData.distillery, - }; - - console.log(`[Vision] Normalized: distillery="${normalized.distillery}", name="${normalized.name}"`); - - // Track usage and deduct credits - await trackApiUsage({ - userId: user.id, - apiType: 'gemini_ai', // Keep same type for tracking - endpoint: `analyzeLabelWith${provider === 'openrouter' ? 'OpenRouter' : 'Gemini'}`, - success: true, - provider: provider, - model: provider === 'openrouter' ? OPENROUTER_VISION_MODEL : 'gemini-2.5-flash', - responseText: result.responseText - }); - await deductCredits(user.id, 'gemini_ai', `Vision label analysis (${provider})`); - - return { - success: true, - data: finalData, - provider, - perf: { - apiCall: result.apiTime, - total: performance.now() - startTotal, - } - }; - - } catch (error: any) { - console.error(`[Vision] Analysis failed (${provider}):`, error); - - // Try to track the failure - try { - const supabase = await createClient(); - const { data: { user } } = await supabase.auth.getUser(); - if (user) { - await trackApiUsage({ - userId: user.id, - apiType: 'gemini_ai', - endpoint: `analyzeLabelWith${provider === 'openrouter' ? 'OpenRouter' : 'Gemini'}`, - success: false, - errorMessage: error.message - }); - } - } catch (trackError) { - console.warn('[Vision] Failed to track error:', trackError); - } - - return { - success: false, - error: error.message || 'Vision analysis failed.', - provider, - }; - } -} diff --git a/src/app/actions/scan-label.ts b/src/app/actions/scan-label.ts deleted file mode 100644 index 7611e35..0000000 --- a/src/app/actions/scan-label.ts +++ /dev/null @@ -1,308 +0,0 @@ -'use server'; - -import { GoogleGenerativeAI, SchemaType, HarmCategory, HarmBlockThreshold } from '@google/generative-ai'; -import { BottleMetadataSchema, AnalysisResponse } from '@/types/whisky'; -import { createClient } from '@/lib/supabase/server'; -import { createHash } from 'crypto'; -import { trackApiUsage } from '@/services/track-api-usage'; -import { checkCreditBalance, deductCredits } from '@/services/credit-service'; -import { getAIProvider, getOpenRouterClient, OPENROUTER_PROVIDER_PREFERENCES } from '@/lib/openrouter'; -import sharp from 'sharp'; - -// Native Schema Definition for Gemini API -const metadataSchema = { - description: "Technical metadata extracted from whisky label", - type: SchemaType.OBJECT as const, - properties: { - name: { type: SchemaType.STRING, description: "Full whisky name including vintage/age", nullable: false }, - distillery: { type: SchemaType.STRING, description: "Distillery name", nullable: true }, - bottler: { type: SchemaType.STRING, description: "Independent bottler if applicable", nullable: true }, - category: { type: SchemaType.STRING, description: "Whisky category (e.g., Single Malt, Blended)", nullable: true }, - abv: { type: SchemaType.NUMBER, description: "Alcohol by volume percentage", nullable: true }, - age: { type: SchemaType.NUMBER, description: "Age statement in years", nullable: true }, - vintage: { type: SchemaType.STRING, description: "Vintage year", nullable: true }, - distilled_at: { type: SchemaType.STRING, description: "Distillation date", nullable: true }, - bottled_at: { type: SchemaType.STRING, description: "Bottling date", nullable: true }, - batch_info: { type: SchemaType.STRING, description: "Batch or cask information", nullable: true }, - bottleCode: { type: SchemaType.STRING, description: "Bottle code or serial number", nullable: true }, - is_whisky: { type: SchemaType.BOOLEAN, description: "Whether this is a whisky product", nullable: false }, - confidence: { type: SchemaType.NUMBER, description: "Confidence score 0-1", nullable: false }, - }, - required: ["name", "is_whisky", "confidence"], -}; - -const SCAN_PROMPT = `Extract whisky label metadata. Return JSON with: -- name: Full product name -- distillery: Distillery name -- bottler: Independent bottler if applicable -- category: e.g. "Single Malt", "Bourbon" -- abv: Alcohol percentage -- age: Age statement in years -- vintage: Vintage year -- distilled_at: Distillation date -- bottled_at: Bottling date -- batch_info: Batch or cask info -- is_whisky: boolean -- confidence: 0-1`; - -export async function scanLabel(input: any): Promise { - const provider = getAIProvider(); - - // Check API key based on provider - if (provider === 'gemini' && !process.env.GEMINI_API_KEY) { - return { success: false, error: 'GEMINI_API_KEY is not configured.' }; - } - if (provider === 'openrouter' && !process.env.OPENROUTER_API_KEY) { - return { success: false, error: 'OPENROUTER_API_KEY is not configured.' }; - } - - let supabase; - try { - const getValue = (obj: any, key: string): any => { - if (obj && typeof obj.get === 'function') return obj.get(key); - if (obj && typeof obj[key] !== 'undefined') return obj[key]; - return null; - }; - - const file = getValue(input, 'file') as File; - if (!file) { - return { success: false, error: 'Kein Bild empfangen.' }; - } - - supabase = await createClient(); - const { data: { user } } = await supabase.auth.getUser(); - - if (!user) { - return { success: false, error: 'Nicht autorisiert.' }; - } - - const userId = user.id; - const creditCheck = await checkCreditBalance(userId, 'gemini_ai'); - - if (!creditCheck.allowed) { - return { - success: false, - error: `Nicht genügend Credits. Benötigt: ${creditCheck.cost}, Verfügbar: ${creditCheck.balance}.` - }; - } - - const perfTotal = performance.now(); - - // Step 1: Image Preparation - const startImagePrep = performance.now(); - const arrayBuffer = await file.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - const imageHash = createHash('sha256').update(buffer).digest('hex'); - const endImagePrep = performance.now(); - - // Step 2: Cache Check - const startCacheCheck = performance.now(); - const { data: cachedResult } = await supabase - .from('vision_cache') - .select('result') - .eq('hash', imageHash) - .maybeSingle(); - const endCacheCheck = performance.now(); - - if (cachedResult) { - return { - success: true, - data: cachedResult.result as any, - perf: { - imagePrep: endImagePrep - startImagePrep, - cacheCheck: endCacheCheck - startCacheCheck, - apiCall: 0, - parsing: 0, - validation: 0, - dbOps: 0, - uploadSize: buffer.length, - total: performance.now() - perfTotal, - cacheHit: true - } - }; - } - - // Step 3: AI-Specific Image Optimization - const startOptimization = performance.now(); - const optimizedBuffer = await sharp(buffer) - .resize(1024, 1024, { fit: 'inside', withoutEnlargement: true }) - .grayscale() - .normalize() - .toBuffer(); - const endOptimization = performance.now(); - - // Step 4: Base64 Encoding - const startEncoding = performance.now(); - const base64Data = optimizedBuffer.toString('base64'); - const mimeType = 'image/webp'; - const uploadSize = optimizedBuffer.length; - const endEncoding = performance.now(); - - // Step 5: AI Analysis - const startAiTotal = performance.now(); - let jsonData; - let validatedData; - - try { - console.log(`[ScanLabel] Using provider: ${provider}`); - - if (provider === 'openrouter') { - // OpenRouter path - const client = getOpenRouterClient(); - const startApi = performance.now(); - - const response = await client.chat.completions.create({ - model: 'google/gemma-3-27b-it', - messages: [{ - role: 'user', - content: [ - { type: 'image_url', image_url: { url: `data:${mimeType};base64,${base64Data}` } }, - { type: 'text', text: SCAN_PROMPT + '\n\nRespond ONLY with valid JSON.' }, - ], - }], - temperature: 0.1, - max_tokens: 1024, - // @ts-ignore - provider: OPENROUTER_PROVIDER_PREFERENCES, - }); - - const endApi = performance.now(); - const content = response.choices[0]?.message?.content || '{}'; - - const startParse = performance.now(); - let jsonStr = content; - const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/); - if (jsonMatch) jsonStr = jsonMatch[1].trim(); - jsonData = JSON.parse(jsonStr); - const endParse = performance.now(); - - const startValidation = performance.now(); - validatedData = BottleMetadataSchema.parse(jsonData); - const endValidation = performance.now(); - - await supabase.from('vision_cache').insert({ hash: imageHash, result: validatedData }); - - await trackApiUsage({ - userId: userId, - apiType: 'gemini_ai', - endpoint: 'scanLabel_openrouter', - success: true, - provider: 'openrouter', - model: 'google/gemma-3-27b-it', - responseText: content - }); - await deductCredits(userId, 'gemini_ai', 'Bottle OCR scan (OpenRouter)'); - - return { - success: true, - data: validatedData, - perf: { - imagePrep: endImagePrep - startImagePrep, - optimization: endOptimization - startOptimization, - cacheCheck: endCacheCheck - startCacheCheck, - encoding: endEncoding - startEncoding, - apiCall: endApi - startApi, - parsing: endParse - startParse, - validation: endValidation - startValidation, - total: performance.now() - perfTotal, - cacheHit: false - }, - raw: jsonData - } as any; - - } else { - // Gemini path - const startModelInit = performance.now(); - const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!); - const model = genAI.getGenerativeModel({ - model: 'gemini-2.5-flash', - generationConfig: { - responseMimeType: "application/json", - responseSchema: metadataSchema as any, - temperature: 0.1, - }, - safetySettings: [ - { category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_NONE }, - { category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_NONE }, - { category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.BLOCK_NONE }, - { category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: HarmBlockThreshold.BLOCK_NONE }, - ] as any, - }); - const endModelInit = performance.now(); - - const startApi = performance.now(); - const result = await model.generateContent([ - { inlineData: { data: base64Data, mimeType: mimeType } }, - { text: 'Extract whisky label metadata.' }, - ]); - const endApi = performance.now(); - - const startParse = performance.now(); - jsonData = JSON.parse(result.response.text()); - const endParse = performance.now(); - - const startValidation = performance.now(); - validatedData = BottleMetadataSchema.parse(jsonData); - const endValidation = performance.now(); - - await supabase.from('vision_cache').insert({ hash: imageHash, result: validatedData }); - - await trackApiUsage({ - userId: userId, - apiType: 'gemini_ai', - endpoint: 'scanLabel_gemini', - success: true, - provider: 'google', - model: 'gemini-2.5-flash', - responseText: result.response.text() - }); - await deductCredits(userId, 'gemini_ai', 'Bottle OCR scan (Gemini)'); - - return { - success: true, - data: validatedData, - perf: { - imagePrep: endImagePrep - startImagePrep, - optimization: endOptimization - startOptimization, - cacheCheck: endCacheCheck - startCacheCheck, - encoding: endEncoding - startEncoding, - modelInit: endModelInit - startModelInit, - apiCall: endApi - startApi, - parsing: endParse - startParse, - validation: endValidation - startValidation, - total: performance.now() - perfTotal, - cacheHit: false - }, - raw: jsonData - } as any; - } - - } catch (aiError: any) { - console.warn(`[ScanLabel] ${provider} failed:`, aiError.message); - - await trackApiUsage({ - userId: userId, - apiType: 'gemini_ai', - endpoint: `scanLabel_${provider}`, - success: false, - errorMessage: aiError.message, - provider: provider, - model: provider === 'openrouter' ? 'google/gemma-3-27b-it' : 'gemini-2.5-flash' - }); - - return { - success: false, - isAiError: true, - error: aiError.message, - imageHash: imageHash - } as any; - } - - } catch (error) { - console.error('Scan Label Global Error:', error); - return { - success: false, - error: error instanceof Error ? error.message : 'Fehler bei der Label-Analyse.', - }; - } -} diff --git a/src/app/actions/scanner.ts b/src/app/actions/scanner.ts new file mode 100644 index 0000000..40cb55f --- /dev/null +++ b/src/app/actions/scanner.ts @@ -0,0 +1,330 @@ +'use server'; + +import { GoogleGenerativeAI, SchemaType, HarmCategory, HarmBlockThreshold } from '@google/generative-ai'; +import { BottleMetadataSchema, BottleMetadata } from '@/types/whisky'; +import { createClient } from '@/lib/supabase/server'; +import { trackApiUsage } from '@/services/track-api-usage'; +import { checkCreditBalance, deductCredits } from '@/services/credit-service'; +import { getAIProvider, getOpenRouterClient, OPENROUTER_VISION_MODEL, OPENROUTER_PROVIDER_PREFERENCES } from '@/lib/openrouter'; +import { normalizeWhiskyData } from '@/lib/distillery-matcher'; +import { formatWhiskyName } from '@/utils/formatWhiskyName'; +import { createHash } from 'crypto'; +import sharp from 'sharp'; + +// Schema for AI extraction +const visionSchema = { + description: "Whisky bottle label metadata extracted from image", + type: SchemaType.OBJECT as const, + properties: { + name: { type: SchemaType.STRING, description: "Full whisky name (constructed)", nullable: false }, + distillery: { type: SchemaType.STRING, description: "Distillery name", nullable: true }, + bottler: { type: SchemaType.STRING, description: "Independent bottler if applicable", nullable: true }, + series: { type: SchemaType.STRING, description: "Whisky series or collection (e.g. Cadenhead's Natural Strength)", nullable: true }, + category: { type: SchemaType.STRING, description: "Whisky category (Single Malt, Blended, Bourbon, etc.)", nullable: true }, + abv: { type: SchemaType.NUMBER, description: "Alcohol by volume percentage", nullable: true }, + age: { type: SchemaType.NUMBER, description: "Age statement in years", nullable: true }, + vintage: { type: SchemaType.STRING, description: "Vintage/distillation year", nullable: true }, + cask_type: { type: SchemaType.STRING, description: "Cask type (Sherry, Bourbon, Port, etc.)", nullable: true }, + distilled_at: { type: SchemaType.STRING, description: "Distillation date", nullable: true }, + bottled_at: { type: SchemaType.STRING, description: "Bottling date", nullable: true }, + batch_info: { type: SchemaType.STRING, description: "Batch or cask number", nullable: true }, + is_whisky: { type: SchemaType.BOOLEAN, description: "Whether this is a whisky product", nullable: false }, + confidence: { type: SchemaType.NUMBER, description: "Confidence score 0-1", nullable: false }, + }, + required: ["name", "is_whisky", "confidence"], +}; + +const VISION_PROMPT = `ROLE: Senior Whisky Database Curator. + +OBJECTIVE: Extract metadata from the bottle image. +CRITICAL: You must CONSTRUCT the product name from its parts, not just read lines. + +INPUT IMAGE ANALYSIS (Mental Steps): +1. FIND DISTILLERY: Look for the most prominent brand name (e.g. "Annandale"). +2. FIND AGE: Scan the center for "Aged X Years" or large numbers. (In this image: Look for a large "10" in the center). +3. FIND VINTAGE: Look for small script/cursive text like "Distilled 2011". +4. FIND CASK TYPE: Look for specific maturation terms like 'Sherry', 'Bourbon', 'Port', 'Oloroso', 'PX', 'Madeira', 'Rum', 'Hogshead', 'Butt', 'Barrel', or 'Finish'. Extract ONLY this phrase into this field (e.g., 'Oloroso Cask Matured') cask_type. Do not leave null if these words are visible. +5. FIND SERIES: Look at the top logo (e.g. "Cadenhead's Natural Strength"). + +COMPOSITION RULES (How to fill the 'name' field): +- DO NOT just write "Single Malt". +- Format: "[Age/Vintage] [Series] [Cask Info]" +- Example: "10 Year Old Cadenhead's Natural Strength Oloroso Matured" + +OUTPUT SCHEMA (Strict JSON): +{ + "name": "string (The constructed full name based on rules above)", + "distillery": "string", + "bottler": "string", + "series": "string (e.g. Natural Strength)", + "abv": number, + "age": numberOrNull, + "vintage": "stringOrNull", + "cask_type": "stringOrNull", + "distilled_at": "stringOrNull", + "bottled_at": "stringOrNull", + "batch_info": "stringOrNull", + "is_whisky": true, + "confidence": number +}`; + +const GEMINI_MODEL = 'gemini-2.5-flash'; + +export interface ScannerResult { + success: boolean; + data?: BottleMetadata; + error?: string; + provider?: 'gemini' | 'openrouter'; + perf?: { + imagePrep?: number; + apiCall: number; + total: number; + cacheHit?: boolean; + apiDuration?: number; + parsing?: number; + parseDuration?: number; + uploadSize?: number; + cacheCheck?: number; + encoding?: number; + modelInit?: number; + validation?: number; + dbOps?: number; + }; + raw?: any; + search_string?: string; +} + +/** + * Compatibility wrapper for older call sites that use an object/FormData input. + */ +export async function scanLabel(input: any): Promise { + const getValue = (obj: any, key: string): any => { + if (obj && typeof obj.get === 'function') return obj.get(key); + if (obj && typeof obj[key] !== 'undefined') return obj[key]; + return null; + }; + + const file = getValue(input, 'file'); + if (file && file instanceof File) { + // Handle file input (e.g. from FormData) + const arrayBuffer = await file.arrayBuffer(); + const base64 = Buffer.from(arrayBuffer).toString('base64'); + return analyzeBottleLabel(`data:${file.type};base64,${base64}`); + } + + if (typeof input === 'string') { + return analyzeBottleLabel(input); + } + + return { success: false, error: 'Ungültiges Eingabeformat.' }; +} + +/** + * Unified action for analyzing a whisky bottle label. + * Replaces redundant gemini-vision.ts, scan-label.ts, and analyze-bottle.ts. + */ +export async function analyzeBottleLabel(imageBase64: string): Promise { + const startTotal = performance.now(); + const provider = getAIProvider(); + + if (!imageBase64 || imageBase64.length < 100) { + return { success: false, error: 'Ungültige Bilddaten.' }; + } + + try { + // 1. Auth & Credit Check + const supabase = await createClient(); + const { data: { user } } = await supabase.auth.getUser(); + if (!user) return { success: false, error: 'Nicht autorisiert.' }; + + const creditCheck = await checkCreditBalance(user.id, 'gemini_ai'); + if (!creditCheck.allowed) { + return { + success: false, + error: `Nicht genügend Credits. Benötigt: ${creditCheck.cost}, Verfügbar: ${creditCheck.balance}.` + }; + } + + // 2. Extract base64 and hash for caching + let base64Data = imageBase64; + let mimeType = 'image/webp'; + if (imageBase64.startsWith('data:')) { + const matches = imageBase64.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + mimeType = matches[1]; + base64Data = matches[2]; + } + } + + const buffer = Buffer.from(base64Data, 'base64'); + const imageHash = createHash('sha256').update(buffer).digest('hex'); + + // 3. Cache Check + const { data: cachedResult } = await supabase + .from('vision_cache') + .select('result') + .eq('hash', imageHash) + .maybeSingle(); + + if (cachedResult) { + console.log(`[Scanner] Cache HIT for hash: ${imageHash.slice(0, 8)}...`); + return { + success: true, + data: cachedResult.result as BottleMetadata, + perf: { + apiCall: 0, + total: performance.now() - startTotal, + cacheHit: true + } + }; + } + + // 4. AI Analysis with retry logic for OpenRouter + console.log(`[Scanner] Using provider: ${provider}`); + let aiResult: { data: any; apiTime: number; responseText: string }; + + if (provider === 'openrouter') { + const client = getOpenRouterClient(); + const startApi = performance.now(); + const maxRetries = 3; + let lastError: any = null; + let response: any = null; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + response = await client.chat.completions.create({ + model: OPENROUTER_VISION_MODEL, + messages: [ + { + role: 'user', + content: [ + { type: 'image_url', image_url: { url: `data:${mimeType};base64,${base64Data}` } }, + { type: 'text', text: VISION_PROMPT }, + ], + }, + ], + temperature: 0.1, + max_tokens: 1024, + // @ts-ignore + provider: OPENROUTER_PROVIDER_PREFERENCES, + }); + break; // Success! + } catch (err: any) { + lastError = err; + if (err.status === 429 && attempt < maxRetries) { + const delay = Math.pow(2, attempt) * 1000; + console.warn(`[Scanner] Rate limited (429). Retrying in ${delay}ms...`); + await new Promise(resolve => setTimeout(resolve, delay)); + continue; + } + throw err; + } + } + + if (!response) throw lastError || new Error('OpenRouter response failed after retries'); + + const content = response.choices[0]?.message?.content || '{}'; + let jsonStr = content; + const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/) || content.match(/\{[\s\S]*\}/); + if (jsonMatch) { + jsonStr = jsonMatch[jsonMatch.length - 1].trim(); + } + aiResult = { + data: JSON.parse(jsonStr), + apiTime: performance.now() - startApi, + responseText: content + }; + } else { + const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!); + const model = genAI.getGenerativeModel({ + model: GEMINI_MODEL, + generationConfig: { + responseMimeType: "application/json", + responseSchema: visionSchema as any, + temperature: 0.1, + }, + safetySettings: [ + { category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_NONE }, + { category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_NONE }, + { category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.BLOCK_NONE }, + { category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: HarmBlockThreshold.BLOCK_NONE }, + ] as any, + }); + const startApi = performance.now(); + const result = await model.generateContent([ + { inlineData: { data: base64Data, mimeType } }, + { text: VISION_PROMPT }, + ]); + const responseText = result.response.text(); + aiResult = { + data: JSON.parse(responseText), + apiTime: performance.now() - startApi, + responseText: responseText + }; + } + + // 6. Name Composition & Normalization + // Use standardized helper to construct the perfect name + console.log(`[Uncleaned Data]: ${JSON.stringify(aiResult.data)}`); + + const d = aiResult.data; + const constructedName = formatWhiskyName({ + distillery: d.distillery || '', + bottler: d.bottler, + series: d.series, + age: d.age, + vintage: d.vintage, + cask_type: d.cask_type + }) || d.name; + + // Validation & Normalization + const validatedData = BottleMetadataSchema.parse({ + ...d, + name: constructedName + }); + + const normalized = normalizeWhiskyData( + validatedData.name || '', + validatedData.distillery || '' + ); + + const finalData = { + ...validatedData, + name: normalized.name || validatedData.name, + distillery: normalized.distillery || validatedData.distillery, + }; + + // 7. Success Tracking & Caching + await trackApiUsage({ + userId: user.id, + apiType: 'gemini_ai', + endpoint: `analyzeBottleLabel_${provider}`, + success: true, + provider, + model: provider === 'openrouter' ? OPENROUTER_VISION_MODEL : GEMINI_MODEL, + responseText: aiResult.responseText + }); + await deductCredits(user.id, 'gemini_ai', `Scanner analysis (${provider})`); + + await supabase.from('vision_cache').insert({ hash: imageHash, result: finalData }); + + return { + success: true, + data: finalData, + provider, + perf: { + apiCall: aiResult.apiTime, + apiDuration: aiResult.apiTime, // Compatibility + total: performance.now() - startTotal, + cacheHit: false + } + }; + + } catch (error: any) { + console.error(`[Scanner] Analysis failed:`, error); + return { + success: false, + error: error.message || 'Vision analysis failed.', + }; + } +} diff --git a/src/components/BottleDetails.tsx b/src/components/BottleDetails.tsx index 3a760c0..0d45163 100644 --- a/src/components/BottleDetails.tsx +++ b/src/components/BottleDetails.tsx @@ -173,6 +173,11 @@ export default function BottleDetails({ bottleId, sessionId, userId }: BottleDet } highlight={!bottle.abv} /> } /> } /> + {(bottle as any).cask_type && ( +
+ } /> +
+ )} {/* Status & Last Dram Row */} diff --git a/src/components/CameraCapture.tsx b/src/components/CameraCapture.tsx index 07c7ea5..93b2d15 100644 --- a/src/components/CameraCapture.tsx +++ b/src/components/CameraCapture.tsx @@ -16,7 +16,7 @@ import Link from 'next/link'; import { useI18n } from '@/i18n/I18nContext'; import { useSession } from '@/context/SessionContext'; import { shortenCategory } from '@/lib/format'; -import { scanLabel } from '@/app/actions/scan-label'; +import { scanLabel } from '@/app/actions/scanner'; import { enrichData } from '@/app/actions/enrich-data'; import { processImageForAI } from '@/utils/image-processing'; diff --git a/src/components/EditBottleForm.tsx b/src/components/EditBottleForm.tsx index 572ff75..d8f4be3 100644 --- a/src/components/EditBottleForm.tsx +++ b/src/components/EditBottleForm.tsx @@ -19,6 +19,7 @@ interface EditBottleFormProps { distilled_at?: string | null; bottled_at?: string | null; batch_info?: string | null; + cask_type?: string | null; }; onComplete?: () => void; } @@ -42,6 +43,7 @@ export default function EditBottleForm({ bottle, onComplete }: EditBottleFormPro distilled_at: bottle.distilled_at || '', bottled_at: bottle.bottled_at || '', batch_info: bottle.batch_info || '', + cask_type: bottle.cask_type || '', }); const handleDiscover = async () => { @@ -87,6 +89,7 @@ export default function EditBottleForm({ bottle, onComplete }: EditBottleFormPro distilled_at: formData.distilled_at || undefined, bottled_at: formData.bottled_at || undefined, batch_info: formData.batch_info || undefined, + cask_type: formData.cask_type || undefined, }); if (response.success) { @@ -251,16 +254,28 @@ export default function EditBottleForm({ bottle, onComplete }: EditBottleFormPro - {/* Batch Info */} -
- - setFormData({ ...formData, batch_info: e.target.value })} - className="w-full px-5 py-4 bg-black/40 border border-white/5 rounded-2xl outline-none focus:ring-2 focus:ring-orange-600/50 text-zinc-100 text-sm font-bold transition-all placeholder:text-zinc-700" - /> + {/* Batch and Cask */} +
+
+ + setFormData({ ...formData, batch_info: e.target.value })} + className="w-full px-5 py-4 bg-black/40 border border-white/5 rounded-2xl outline-none focus:ring-2 focus:ring-orange-600/50 text-zinc-100 text-sm font-bold transition-all placeholder:text-zinc-700" + /> +
+
+ + setFormData({ ...formData, cask_type: e.target.value })} + className="w-full px-5 py-4 bg-black/40 border border-white/5 rounded-2xl outline-none focus:ring-2 focus:ring-orange-600/50 text-zinc-100 text-sm font-bold transition-all placeholder:text-zinc-700" + /> +
diff --git a/src/components/TastingEditor.tsx b/src/components/TastingEditor.tsx index 5873dd3..e1f428d 100644 --- a/src/components/TastingEditor.tsx +++ b/src/components/TastingEditor.tsx @@ -48,6 +48,7 @@ export default function TastingEditor({ bottleMetadata, image, onSave, onOpenSes const [bottleCategory, setBottleCategory] = useState(bottleMetadata.category || 'Whisky'); const [bottleVintage, setBottleVintage] = useState(bottleMetadata.vintage || ''); + const [bottleCaskType, setBottleCaskType] = useState(bottleMetadata.cask_type || ''); const [bottleBottler, setBottleBottler] = useState(bottleMetadata.bottler || ''); const [bottleBatchInfo, setBottleBatchInfo] = useState(bottleMetadata.batch_info || ''); const [bottleCode, setBottleCode] = useState(bottleMetadata.bottleCode || ''); @@ -106,6 +107,7 @@ export default function TastingEditor({ bottleMetadata, image, onSave, onOpenSes if (bottleMetadata.age) setBottleAge(bottleMetadata.age.toString()); if (bottleMetadata.category) setBottleCategory(bottleMetadata.category); if (bottleMetadata.vintage) setBottleVintage(bottleMetadata.vintage); + if (bottleMetadata.cask_type) setBottleCaskType(bottleMetadata.cask_type); if (bottleMetadata.bottler) setBottleBottler(bottleMetadata.bottler); if (bottleMetadata.batch_info) setBottleBatchInfo(bottleMetadata.batch_info); if (bottleMetadata.distilled_at) setBottleDistilledAt(bottleMetadata.distilled_at); @@ -118,6 +120,7 @@ export default function TastingEditor({ bottleMetadata, image, onSave, onOpenSes if (!bottleAge && bottleMetadata.age) setBottleAge(bottleMetadata.age.toString()); if ((!bottleCategory || bottleCategory === 'Whisky') && bottleMetadata.category) setBottleCategory(bottleMetadata.category); if (!bottleVintage && bottleMetadata.vintage) setBottleVintage(bottleMetadata.vintage); + if (!bottleCaskType && bottleMetadata.cask_type) setBottleCaskType(bottleMetadata.cask_type); if (!bottleBottler && bottleMetadata.bottler) setBottleBottler(bottleMetadata.bottler); if (!bottleBatchInfo && bottleMetadata.batch_info) setBottleBatchInfo(bottleMetadata.batch_info); if (!bottleDistilledAt && bottleMetadata.distilled_at) setBottleDistilledAt(bottleMetadata.distilled_at); @@ -182,17 +185,23 @@ export default function TastingEditor({ bottleMetadata, image, onSave, onOpenSes // Automatic Whiskybase discovery when details are expanded useEffect(() => { const searchWhiskybase = async () => { - if (showBottleDetails && !whiskybaseId && !whiskybaseDiscovery && !isDiscoveringWb) { + if (showBottleDetails && (bottleName || bottleMetadata.name) && !whiskybaseId && !whiskybaseDiscovery && !isDiscoveringWb) { setIsDiscoveringWb(true); try { + const searchName = bottleName || bottleMetadata.name; + if (!searchName) { + setIsDiscoveringWb(false); + return; + } + const result = await discoverWhiskybaseId({ - name: bottleMetadata.name || '', - distillery: bottleMetadata.distillery ?? undefined, - abv: bottleMetadata.abv ?? undefined, - age: bottleMetadata.age ?? undefined, - batch_info: bottleMetadata.batch_info ?? undefined, - distilled_at: bottleMetadata.distilled_at ?? undefined, - bottled_at: bottleMetadata.bottled_at ?? undefined, + name: searchName, + distillery: bottleDistillery || undefined, + abv: bottleAbv ? parseFloat(bottleAbv) : undefined, + age: bottleAge ? parseInt(bottleAge) : undefined, + batch_info: bottleBatchInfo || undefined, + distilled_at: bottleDistilledAt || undefined, + bottled_at: bottleBottledAt || undefined, }); if (result.success && result.id) { @@ -238,6 +247,7 @@ export default function TastingEditor({ bottleMetadata, image, onSave, onOpenSes age: bottleAge ? parseInt(bottleAge) : bottleMetadata.age, category: bottleCategory || bottleMetadata.category, vintage: bottleVintage || null, + cask_type: bottleCaskType || null, bottler: bottleBottler || null, batch_info: bottleBatchInfo || null, bottleCode: bottleCode || null, @@ -388,6 +398,19 @@ export default function TastingEditor({ bottleMetadata, image, onSave, onOpenSes className="w-full bg-zinc-950 border border-zinc-800 rounded-lg px-3 py-2 text-sm text-zinc-100 placeholder-zinc-700 focus:outline-none focus:border-orange-600 transition-colors" /> + {/* Cask Type */} +
+ + setBottleCaskType(e.target.value)} + placeholder="e.g. Oloroso Sherry Cask" + className="w-full bg-zinc-950 border border-zinc-800 rounded-lg px-3 py-2 text-sm text-zinc-100 placeholder-zinc-700 focus:outline-none focus:border-orange-600 transition-colors" + /> +
{/* Vintage */}