- bulk-scan.ts: Now uses OpenRouter for batch analysis - scan-label.ts: Now uses OpenRouter with Gemini fallback - analyze-bottle.ts: Now uses OpenRouter with Gemini fallback - All AI calls now respect AI_PROVIDER env variable - Uses Nebius/FP8 provider preferences consistently - Unified logging: [FunctionName] Using provider: openrouter/gemini
301 lines
12 KiB
TypeScript
301 lines
12 KiB
TypeScript
'use server';
|
|
|
|
import { GoogleGenerativeAI, SchemaType, HarmCategory, HarmBlockThreshold } from '@google/generative-ai';
|
|
import { BottleMetadataSchema, AnalysisResponse } from '@/types/whisky';
|
|
import { createClient } from '@/lib/supabase/server';
|
|
import { createHash } from 'crypto';
|
|
import { trackApiUsage } from '@/services/track-api-usage';
|
|
import { checkCreditBalance, deductCredits } from '@/services/credit-service';
|
|
import { getAIProvider, getOpenRouterClient, OPENROUTER_PROVIDER_PREFERENCES } from '@/lib/openrouter';
|
|
import sharp from 'sharp';
|
|
|
|
// Native Schema Definition for Gemini API
|
|
const metadataSchema = {
|
|
description: "Technical metadata extracted from whisky label",
|
|
type: SchemaType.OBJECT as const,
|
|
properties: {
|
|
name: { type: SchemaType.STRING, description: "Full whisky name including vintage/age", nullable: false },
|
|
distillery: { type: SchemaType.STRING, description: "Distillery name", nullable: true },
|
|
bottler: { type: SchemaType.STRING, description: "Independent bottler if applicable", nullable: true },
|
|
category: { type: SchemaType.STRING, description: "Whisky category (e.g., Single Malt, Blended)", nullable: true },
|
|
abv: { type: SchemaType.NUMBER, description: "Alcohol by volume percentage", nullable: true },
|
|
age: { type: SchemaType.NUMBER, description: "Age statement in years", nullable: true },
|
|
vintage: { type: SchemaType.STRING, description: "Vintage year", nullable: true },
|
|
distilled_at: { type: SchemaType.STRING, description: "Distillation date", nullable: true },
|
|
bottled_at: { type: SchemaType.STRING, description: "Bottling date", nullable: true },
|
|
batch_info: { type: SchemaType.STRING, description: "Batch or cask information", nullable: true },
|
|
bottleCode: { type: SchemaType.STRING, description: "Bottle code or serial number", nullable: true },
|
|
is_whisky: { type: SchemaType.BOOLEAN, description: "Whether this is a whisky product", nullable: false },
|
|
confidence: { type: SchemaType.NUMBER, description: "Confidence score 0-1", nullable: false },
|
|
},
|
|
required: ["name", "is_whisky", "confidence"],
|
|
};
|
|
|
|
const SCAN_PROMPT = `Extract whisky label metadata. Return JSON with:
|
|
- name: Full product name
|
|
- distillery: Distillery name
|
|
- bottler: Independent bottler if applicable
|
|
- category: e.g. "Single Malt", "Bourbon"
|
|
- abv: Alcohol percentage
|
|
- age: Age statement in years
|
|
- vintage: Vintage year
|
|
- distilled_at: Distillation date
|
|
- bottled_at: Bottling date
|
|
- batch_info: Batch or cask info
|
|
- is_whisky: boolean
|
|
- confidence: 0-1`;
|
|
|
|
export async function scanLabel(input: any): Promise<AnalysisResponse> {
|
|
const provider = getAIProvider();
|
|
|
|
// Check API key based on provider
|
|
if (provider === 'gemini' && !process.env.GEMINI_API_KEY) {
|
|
return { success: false, error: 'GEMINI_API_KEY is not configured.' };
|
|
}
|
|
if (provider === 'openrouter' && !process.env.OPENROUTER_API_KEY) {
|
|
return { success: false, error: 'OPENROUTER_API_KEY is not configured.' };
|
|
}
|
|
|
|
let supabase;
|
|
try {
|
|
const getValue = (obj: any, key: string): any => {
|
|
if (obj && typeof obj.get === 'function') return obj.get(key);
|
|
if (obj && typeof obj[key] !== 'undefined') return obj[key];
|
|
return null;
|
|
};
|
|
|
|
const file = getValue(input, 'file') as File;
|
|
if (!file) {
|
|
return { success: false, error: 'Kein Bild empfangen.' };
|
|
}
|
|
|
|
supabase = await createClient();
|
|
const { data: { user } } = await supabase.auth.getUser();
|
|
|
|
if (!user) {
|
|
return { success: false, error: 'Nicht autorisiert.' };
|
|
}
|
|
|
|
const userId = user.id;
|
|
const creditCheck = await checkCreditBalance(userId, 'gemini_ai');
|
|
|
|
if (!creditCheck.allowed) {
|
|
return {
|
|
success: false,
|
|
error: `Nicht genügend Credits. Benötigt: ${creditCheck.cost}, Verfügbar: ${creditCheck.balance}.`
|
|
};
|
|
}
|
|
|
|
const perfTotal = performance.now();
|
|
|
|
// Step 1: Image Preparation
|
|
const startImagePrep = performance.now();
|
|
const arrayBuffer = await file.arrayBuffer();
|
|
const buffer = Buffer.from(arrayBuffer);
|
|
const imageHash = createHash('sha256').update(buffer).digest('hex');
|
|
const endImagePrep = performance.now();
|
|
|
|
// Step 2: Cache Check
|
|
const startCacheCheck = performance.now();
|
|
const { data: cachedResult } = await supabase
|
|
.from('vision_cache')
|
|
.select('result')
|
|
.eq('hash', imageHash)
|
|
.maybeSingle();
|
|
const endCacheCheck = performance.now();
|
|
|
|
if (cachedResult) {
|
|
return {
|
|
success: true,
|
|
data: cachedResult.result as any,
|
|
perf: {
|
|
imagePrep: endImagePrep - startImagePrep,
|
|
cacheCheck: endCacheCheck - startCacheCheck,
|
|
apiCall: 0,
|
|
parsing: 0,
|
|
validation: 0,
|
|
dbOps: 0,
|
|
uploadSize: buffer.length,
|
|
total: performance.now() - perfTotal,
|
|
cacheHit: true
|
|
}
|
|
};
|
|
}
|
|
|
|
// Step 3: AI-Specific Image Optimization
|
|
const startOptimization = performance.now();
|
|
const optimizedBuffer = await sharp(buffer)
|
|
.resize(1024, 1024, { fit: 'inside', withoutEnlargement: true })
|
|
.grayscale()
|
|
.normalize()
|
|
.toBuffer();
|
|
const endOptimization = performance.now();
|
|
|
|
// Step 4: Base64 Encoding
|
|
const startEncoding = performance.now();
|
|
const base64Data = optimizedBuffer.toString('base64');
|
|
const mimeType = 'image/webp';
|
|
const uploadSize = optimizedBuffer.length;
|
|
const endEncoding = performance.now();
|
|
|
|
// Step 5: AI Analysis
|
|
const startAiTotal = performance.now();
|
|
let jsonData;
|
|
let validatedData;
|
|
|
|
try {
|
|
console.log(`[ScanLabel] Using provider: ${provider}`);
|
|
|
|
if (provider === 'openrouter') {
|
|
// OpenRouter path
|
|
const client = getOpenRouterClient();
|
|
const startApi = performance.now();
|
|
|
|
const response = await client.chat.completions.create({
|
|
model: 'google/gemma-3-27b-it',
|
|
messages: [{
|
|
role: 'user',
|
|
content: [
|
|
{ type: 'image_url', image_url: { url: `data:${mimeType};base64,${base64Data}` } },
|
|
{ type: 'text', text: SCAN_PROMPT + '\n\nRespond ONLY with valid JSON.' },
|
|
],
|
|
}],
|
|
temperature: 0.1,
|
|
max_tokens: 1024,
|
|
// @ts-ignore
|
|
provider: OPENROUTER_PROVIDER_PREFERENCES,
|
|
});
|
|
|
|
const endApi = performance.now();
|
|
const content = response.choices[0]?.message?.content || '{}';
|
|
|
|
const startParse = performance.now();
|
|
let jsonStr = content;
|
|
const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
if (jsonMatch) jsonStr = jsonMatch[1].trim();
|
|
jsonData = JSON.parse(jsonStr);
|
|
const endParse = performance.now();
|
|
|
|
const startValidation = performance.now();
|
|
validatedData = BottleMetadataSchema.parse(jsonData);
|
|
const endValidation = performance.now();
|
|
|
|
await supabase.from('vision_cache').insert({ hash: imageHash, result: validatedData });
|
|
|
|
await trackApiUsage({
|
|
userId: userId,
|
|
apiType: 'gemini_ai',
|
|
endpoint: 'scanLabel_openrouter',
|
|
success: true
|
|
});
|
|
await deductCredits(userId, 'gemini_ai', 'Bottle OCR scan (OpenRouter)');
|
|
|
|
return {
|
|
success: true,
|
|
data: validatedData,
|
|
perf: {
|
|
imagePrep: endImagePrep - startImagePrep,
|
|
optimization: endOptimization - startOptimization,
|
|
cacheCheck: endCacheCheck - startCacheCheck,
|
|
encoding: endEncoding - startEncoding,
|
|
apiCall: endApi - startApi,
|
|
parsing: endParse - startParse,
|
|
validation: endValidation - startValidation,
|
|
total: performance.now() - perfTotal,
|
|
cacheHit: false
|
|
},
|
|
raw: jsonData
|
|
} as any;
|
|
|
|
} else {
|
|
// Gemini path
|
|
const startModelInit = performance.now();
|
|
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!);
|
|
const model = genAI.getGenerativeModel({
|
|
model: 'gemini-2.5-flash',
|
|
generationConfig: {
|
|
responseMimeType: "application/json",
|
|
responseSchema: metadataSchema as any,
|
|
temperature: 0.1,
|
|
},
|
|
safetySettings: [
|
|
{ category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_NONE },
|
|
{ category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_NONE },
|
|
{ category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.BLOCK_NONE },
|
|
{ category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: HarmBlockThreshold.BLOCK_NONE },
|
|
] as any,
|
|
});
|
|
const endModelInit = performance.now();
|
|
|
|
const startApi = performance.now();
|
|
const result = await model.generateContent([
|
|
{ inlineData: { data: base64Data, mimeType: mimeType } },
|
|
{ text: 'Extract whisky label metadata.' },
|
|
]);
|
|
const endApi = performance.now();
|
|
|
|
const startParse = performance.now();
|
|
jsonData = JSON.parse(result.response.text());
|
|
const endParse = performance.now();
|
|
|
|
const startValidation = performance.now();
|
|
validatedData = BottleMetadataSchema.parse(jsonData);
|
|
const endValidation = performance.now();
|
|
|
|
await supabase.from('vision_cache').insert({ hash: imageHash, result: validatedData });
|
|
|
|
await trackApiUsage({
|
|
userId: userId,
|
|
apiType: 'gemini_ai',
|
|
endpoint: 'scanLabel_gemini',
|
|
success: true
|
|
});
|
|
await deductCredits(userId, 'gemini_ai', 'Bottle OCR scan (Gemini)');
|
|
|
|
return {
|
|
success: true,
|
|
data: validatedData,
|
|
perf: {
|
|
imagePrep: endImagePrep - startImagePrep,
|
|
optimization: endOptimization - startOptimization,
|
|
cacheCheck: endCacheCheck - startCacheCheck,
|
|
encoding: endEncoding - startEncoding,
|
|
modelInit: endModelInit - startModelInit,
|
|
apiCall: endApi - startApi,
|
|
parsing: endParse - startParse,
|
|
validation: endValidation - startValidation,
|
|
total: performance.now() - perfTotal,
|
|
cacheHit: false
|
|
},
|
|
raw: jsonData
|
|
} as any;
|
|
}
|
|
|
|
} catch (aiError: any) {
|
|
console.warn(`[ScanLabel] ${provider} failed:`, aiError.message);
|
|
|
|
await trackApiUsage({
|
|
userId: userId,
|
|
apiType: 'gemini_ai',
|
|
endpoint: `scanLabel_${provider}`,
|
|
success: false,
|
|
errorMessage: aiError.message
|
|
});
|
|
|
|
return {
|
|
success: false,
|
|
isAiError: true,
|
|
error: aiError.message,
|
|
imageHash: imageHash
|
|
} as any;
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('Scan Label Global Error:', error);
|
|
return {
|
|
success: false,
|
|
error: error instanceof Error ? error.message : 'Fehler bei der Label-Analyse.',
|
|
};
|
|
}
|
|
}
|