feat: improved local OCR with Strip & Match distillery detection
- Added comprehensive distillery database (200+ entries) - Implemented Strip & Match heuristic for fuzzy matching - Added contextual age detection from distillery lines - Added whitespace normalization for OCR text - Disabled local name extraction (too noisy, let Gemini handle it) - Fixed confidence scale normalization in TastingEditor (0-1 vs 0-100) - Improved extractName filter (60% letters required) - Relaxed Fuse.js thresholds for partial matches
This commit is contained in:
177
src/app/actions/gemini-vision.ts
Normal file
177
src/app/actions/gemini-vision.ts
Normal file
@@ -0,0 +1,177 @@
|
||||
'use server';
|
||||
|
||||
import { GoogleGenerativeAI, SchemaType, HarmCategory, HarmBlockThreshold } from '@google/generative-ai';
|
||||
import { BottleMetadataSchema, BottleMetadata } from '@/types/whisky';
|
||||
import { createClient } from '@/lib/supabase/server';
|
||||
import { trackApiUsage } from '@/services/track-api-usage';
|
||||
import { checkCreditBalance, deductCredits } from '@/services/credit-service';
|
||||
|
||||
// Schema for Gemini Vision extraction
|
||||
const visionSchema = {
|
||||
description: "Whisky bottle label metadata extracted from image",
|
||||
type: SchemaType.OBJECT as const,
|
||||
properties: {
|
||||
name: { type: SchemaType.STRING, description: "Full whisky name", nullable: false },
|
||||
distillery: { type: SchemaType.STRING, description: "Distillery name", nullable: true },
|
||||
bottler: { type: SchemaType.STRING, description: "Independent bottler if applicable", nullable: true },
|
||||
category: { type: SchemaType.STRING, description: "Whisky category (Single Malt, Blended, Bourbon, etc.)", nullable: true },
|
||||
abv: { type: SchemaType.NUMBER, description: "Alcohol by volume percentage", nullable: true },
|
||||
age: { type: SchemaType.NUMBER, description: "Age statement in years", nullable: true },
|
||||
vintage: { type: SchemaType.STRING, description: "Vintage/distillation year", nullable: true },
|
||||
cask_type: { type: SchemaType.STRING, description: "Cask type (Sherry, Bourbon, Port, etc.)", nullable: true },
|
||||
distilled_at: { type: SchemaType.STRING, description: "Distillation date", nullable: true },
|
||||
bottled_at: { type: SchemaType.STRING, description: "Bottling date", nullable: true },
|
||||
batch_info: { type: SchemaType.STRING, description: "Batch or cask number", nullable: true },
|
||||
is_whisky: { type: SchemaType.BOOLEAN, description: "Whether this is a whisky product", nullable: false },
|
||||
confidence: { type: SchemaType.NUMBER, description: "Confidence score 0-1", nullable: false },
|
||||
},
|
||||
required: ["name", "is_whisky", "confidence"],
|
||||
};
|
||||
|
||||
export interface GeminiVisionResult {
|
||||
success: boolean;
|
||||
data?: BottleMetadata;
|
||||
error?: string;
|
||||
perf?: {
|
||||
apiCall: number;
|
||||
total: number;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a whisky bottle label image using Gemini Vision
|
||||
*
|
||||
* @param imageBase64 - Base64 encoded image (with data URL prefix)
|
||||
* @returns GeminiVisionResult with extracted metadata
|
||||
*/
|
||||
export async function analyzeLabelWithGemini(imageBase64: string): Promise<GeminiVisionResult> {
|
||||
const startTotal = performance.now();
|
||||
|
||||
if (!process.env.GEMINI_API_KEY) {
|
||||
return { success: false, error: 'GEMINI_API_KEY is not configured.' };
|
||||
}
|
||||
|
||||
if (!imageBase64 || imageBase64.length < 100) {
|
||||
return { success: false, error: 'Invalid image data provided.' };
|
||||
}
|
||||
|
||||
try {
|
||||
// Auth check
|
||||
const supabase = await createClient();
|
||||
const { data: { user } } = await supabase.auth.getUser();
|
||||
|
||||
if (!user) {
|
||||
return { success: false, error: 'Not authorized.' };
|
||||
}
|
||||
|
||||
// Credit check
|
||||
const creditCheck = await checkCreditBalance(user.id, 'gemini_ai');
|
||||
if (!creditCheck.allowed) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Insufficient credits. Required: ${creditCheck.cost}, Available: ${creditCheck.balance}.`
|
||||
};
|
||||
}
|
||||
|
||||
// Extract base64 data (remove data URL prefix if present)
|
||||
let base64Data = imageBase64;
|
||||
let mimeType = 'image/webp';
|
||||
|
||||
if (imageBase64.startsWith('data:')) {
|
||||
const matches = imageBase64.match(/^data:([^;]+);base64,(.+)$/);
|
||||
if (matches) {
|
||||
mimeType = matches[1];
|
||||
base64Data = matches[2];
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize Gemini
|
||||
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY);
|
||||
const model = genAI.getGenerativeModel({
|
||||
model: 'gemini-2.5-flash',
|
||||
generationConfig: {
|
||||
responseMimeType: "application/json",
|
||||
responseSchema: visionSchema as any,
|
||||
temperature: 0.1,
|
||||
},
|
||||
safetySettings: [
|
||||
{ category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_NONE },
|
||||
{ category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_NONE },
|
||||
{ category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.BLOCK_NONE },
|
||||
{ category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: HarmBlockThreshold.BLOCK_NONE },
|
||||
] as any,
|
||||
});
|
||||
|
||||
// Vision prompt
|
||||
const prompt = `Analyze this whisky bottle label image and extract all visible metadata.
|
||||
Look carefully for:
|
||||
- Brand/Distillery name
|
||||
- Bottle name or expression
|
||||
- Age statement (e.g., "12 Years Old")
|
||||
- ABV/Alcohol percentage
|
||||
- Vintage year (if shown)
|
||||
- Cask type (e.g., Sherry, Bourbon cask)
|
||||
- Bottler name (if independent bottling)
|
||||
- Category (Single Malt, Blended Malt, Bourbon, etc.)
|
||||
|
||||
Be precise and only include information you can clearly read from the label.
|
||||
If you cannot read something clearly, leave it null.`;
|
||||
|
||||
// API call with timing
|
||||
const startApi = performance.now();
|
||||
const result = await model.generateContent([
|
||||
{ inlineData: { data: base64Data, mimeType } },
|
||||
{ text: prompt },
|
||||
]);
|
||||
const endApi = performance.now();
|
||||
|
||||
// Parse response
|
||||
const jsonData = JSON.parse(result.response.text());
|
||||
|
||||
// Validate with Zod schema
|
||||
const validatedData = BottleMetadataSchema.parse(jsonData);
|
||||
|
||||
// Track usage and deduct credits
|
||||
await trackApiUsage({
|
||||
userId: user.id,
|
||||
apiType: 'gemini_ai',
|
||||
endpoint: 'analyzeLabelWithGemini',
|
||||
success: true
|
||||
});
|
||||
await deductCredits(user.id, 'gemini_ai', 'Vision label analysis');
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: validatedData,
|
||||
perf: {
|
||||
apiCall: endApi - startApi,
|
||||
total: performance.now() - startTotal,
|
||||
}
|
||||
};
|
||||
|
||||
} catch (error: any) {
|
||||
console.error('[GeminiVision] Analysis failed:', error);
|
||||
|
||||
// Try to track the failure
|
||||
try {
|
||||
const supabase = await createClient();
|
||||
const { data: { user } } = await supabase.auth.getUser();
|
||||
if (user) {
|
||||
await trackApiUsage({
|
||||
userId: user.id,
|
||||
apiType: 'gemini_ai',
|
||||
endpoint: 'analyzeLabelWithGemini',
|
||||
success: false,
|
||||
errorMessage: error.message
|
||||
});
|
||||
}
|
||||
} catch (trackError) {
|
||||
console.warn('[GeminiVision] Failed to track error:', trackError);
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: error.message || 'Vision analysis failed.'
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,19 @@
|
||||
'use client';
|
||||
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import React, { useState, useEffect, useCallback } from 'react';
|
||||
import { motion, AnimatePresence } from 'framer-motion';
|
||||
import { X, Loader2, Sparkles, AlertCircle, Clock } from 'lucide-react';
|
||||
import { X, Loader2, Sparkles, AlertCircle, Clock, Eye, Cloud, Cpu } from 'lucide-react';
|
||||
import TastingEditor from './TastingEditor';
|
||||
import SessionBottomSheet from './SessionBottomSheet';
|
||||
import ResultCard from './ResultCard';
|
||||
import { useSession } from '@/context/SessionContext';
|
||||
import { scanLabel } from '@/app/actions/scan-label';
|
||||
import { enrichData } from '@/app/actions/enrich-data';
|
||||
|
||||
import { saveBottle } from '@/services/save-bottle';
|
||||
import { saveTasting } from '@/services/save-tasting';
|
||||
import { BottleMetadata } from '@/types/whisky';
|
||||
import { useI18n } from '@/i18n/I18nContext';
|
||||
import { createClient } from '@/lib/supabase/client';
|
||||
import { processImageForAI, ProcessedImage } from '@/utils/image-processing';
|
||||
import { generateDummyMetadata } from '@/utils/generate-dummy-metadata';
|
||||
import { useScanner, ScanStatus } from '@/hooks/useScanner';
|
||||
import { db } from '@/lib/db';
|
||||
|
||||
type FlowState = 'IDLE' | 'SCANNING' | 'EDITOR' | 'RESULT' | 'ERROR';
|
||||
@@ -32,7 +29,6 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
const [state, setState] = useState<FlowState>('IDLE');
|
||||
const [isSessionsOpen, setIsSessionsOpen] = useState(false);
|
||||
const { activeSession } = useSession();
|
||||
const [processedImage, setProcessedImage] = useState<ProcessedImage | null>(null);
|
||||
const [tastingData, setTastingData] = useState<any>(null);
|
||||
const [bottleMetadata, setBottleMetadata] = useState<BottleMetadata | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
@@ -40,24 +36,38 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
const { locale } = useI18n();
|
||||
const supabase = createClient();
|
||||
const [isAdmin, setIsAdmin] = useState(false);
|
||||
const [isOffline, setIsOffline] = useState(!navigator.onLine);
|
||||
const [perfMetrics, setPerfMetrics] = useState<{
|
||||
comp: number;
|
||||
aiTotal: number;
|
||||
aiApi: number;
|
||||
aiParse: number;
|
||||
uploadSize: number;
|
||||
prep: number;
|
||||
// Detailed metrics
|
||||
imagePrep?: number;
|
||||
cacheCheck?: number;
|
||||
encoding?: number;
|
||||
modelInit?: number;
|
||||
validation?: number;
|
||||
dbOps?: number;
|
||||
total?: number;
|
||||
cacheHit?: boolean;
|
||||
} | null>(null);
|
||||
const [isOffline, setIsOffline] = useState(typeof navigator !== 'undefined' ? !navigator.onLine : false);
|
||||
const [isEnriching, setIsEnriching] = useState(false);
|
||||
const [aiFallbackActive, setAiFallbackActive] = useState(false);
|
||||
|
||||
// Use the new hybrid scanner hook
|
||||
const scanner = useScanner({
|
||||
locale,
|
||||
onLocalComplete: (localResult) => {
|
||||
console.log('[ScanFlow] Local OCR complete, updating preview:', localResult);
|
||||
// Immediately update bottleMetadata with local results for optimistic UI
|
||||
setBottleMetadata(prev => ({
|
||||
...prev,
|
||||
name: localResult.name || prev?.name || null,
|
||||
distillery: localResult.distillery || prev?.distillery || null,
|
||||
abv: localResult.abv ?? prev?.abv ?? null,
|
||||
age: localResult.age ?? prev?.age ?? null,
|
||||
vintage: localResult.vintage || prev?.vintage || null,
|
||||
is_whisky: true,
|
||||
confidence: 50,
|
||||
} as BottleMetadata));
|
||||
},
|
||||
onCloudComplete: (cloudResult) => {
|
||||
console.log('[ScanFlow] Cloud vision complete:', cloudResult);
|
||||
// Update with cloud results (this is the "truth")
|
||||
setBottleMetadata(cloudResult);
|
||||
|
||||
// Trigger background enrichment if we have name and distillery
|
||||
if (cloudResult.name && cloudResult.distillery) {
|
||||
runEnrichment(cloudResult.name, cloudResult.distillery);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Admin Check
|
||||
useEffect(() => {
|
||||
@@ -81,8 +91,31 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
checkAdmin();
|
||||
}, [supabase]);
|
||||
|
||||
const [aiFallbackActive, setAiFallbackActive] = useState(false);
|
||||
const [isEnriching, setIsEnriching] = useState(false);
|
||||
// Background enrichment function
|
||||
const runEnrichment = useCallback(async (name: string, distillery: string) => {
|
||||
setIsEnriching(true);
|
||||
console.log('[ScanFlow] Starting background enrichment for:', name);
|
||||
try {
|
||||
const enrichResult = await enrichData(name, distillery, undefined, locale);
|
||||
if (enrichResult.success && enrichResult.data) {
|
||||
console.log('[ScanFlow] Enrichment data received:', enrichResult.data);
|
||||
setBottleMetadata(prev => {
|
||||
if (!prev) return prev;
|
||||
return {
|
||||
...prev,
|
||||
suggested_tags: enrichResult.data.suggested_tags,
|
||||
suggested_custom_tags: enrichResult.data.suggested_custom_tags,
|
||||
};
|
||||
});
|
||||
} else {
|
||||
console.warn('[ScanFlow] Enrichment unsuccessful:', enrichResult.error);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[ScanFlow] Enrichment failed:', err);
|
||||
} finally {
|
||||
setIsEnriching(false);
|
||||
}
|
||||
}, [locale]);
|
||||
|
||||
// Trigger scan when open and image provided
|
||||
useEffect(() => {
|
||||
@@ -93,10 +126,10 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
setState('IDLE');
|
||||
setTastingData(null);
|
||||
setBottleMetadata(null);
|
||||
setProcessedImage(null);
|
||||
setError(null);
|
||||
setIsSaving(false);
|
||||
setAiFallbackActive(false);
|
||||
scanner.reset();
|
||||
}
|
||||
}, [isOpen, imageFile]);
|
||||
|
||||
@@ -114,146 +147,68 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Sync scanner status to UI state
|
||||
useEffect(() => {
|
||||
const scannerStatus = scanner.status;
|
||||
|
||||
if (scannerStatus === 'idle') {
|
||||
// Don't change state on idle
|
||||
} else if (scannerStatus === 'compressing' || scannerStatus === 'analyzing_local') {
|
||||
setState('SCANNING');
|
||||
} else if (scannerStatus === 'analyzing_cloud') {
|
||||
// Show EDITOR immediately when we have local results - don't wait for cloud
|
||||
if (scanner.localResult || scanner.mergedResult) {
|
||||
setState('EDITOR');
|
||||
} else {
|
||||
setState('SCANNING');
|
||||
}
|
||||
} else if (scannerStatus === 'complete' || scannerStatus === 'queued') {
|
||||
// Use merged result from scanner
|
||||
if (scanner.mergedResult) {
|
||||
setBottleMetadata(scanner.mergedResult);
|
||||
}
|
||||
setState('EDITOR');
|
||||
|
||||
// If this was a queued offline scan, mark as fallback
|
||||
if (scannerStatus === 'queued') {
|
||||
setAiFallbackActive(true);
|
||||
setIsOffline(true);
|
||||
}
|
||||
} else if (scannerStatus === 'error') {
|
||||
if (scanner.mergedResult) {
|
||||
// We have partial results, show editor anyway
|
||||
setBottleMetadata(scanner.mergedResult);
|
||||
setState('EDITOR');
|
||||
setAiFallbackActive(true);
|
||||
} else {
|
||||
setError(scanner.error || 'Scan failed');
|
||||
setState('ERROR');
|
||||
}
|
||||
}
|
||||
}, [scanner.status, scanner.mergedResult, scanner.localResult, scanner.error]);
|
||||
|
||||
const handleScan = async (file: File) => {
|
||||
setState('SCANNING');
|
||||
setError(null);
|
||||
setPerfMetrics(null);
|
||||
|
||||
try {
|
||||
const startComp = performance.now();
|
||||
const processed = await processImageForAI(file);
|
||||
const endComp = performance.now();
|
||||
setProcessedImage(processed);
|
||||
|
||||
// OFFLINE: Skip AI scan, use dummy metadata
|
||||
if (isOffline) {
|
||||
const dummyMetadata = generateDummyMetadata(file);
|
||||
setBottleMetadata(dummyMetadata);
|
||||
setState('EDITOR');
|
||||
|
||||
if (isAdmin) {
|
||||
setPerfMetrics({
|
||||
comp: endComp - startComp,
|
||||
aiTotal: 0,
|
||||
aiApi: 0,
|
||||
aiParse: 0,
|
||||
uploadSize: processed.file.size,
|
||||
prep: 0,
|
||||
cacheCheck: 0,
|
||||
cacheHit: false
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// ONLINE: Normal AI scan
|
||||
const formData = new FormData();
|
||||
formData.append('file', processed.file);
|
||||
|
||||
const startAi = performance.now();
|
||||
const result = await scanLabel(formData);
|
||||
const endAi = performance.now();
|
||||
|
||||
const startPrep = performance.now();
|
||||
if (result.success && result.data) {
|
||||
setBottleMetadata(result.data);
|
||||
|
||||
const endPrep = performance.now();
|
||||
if (isAdmin && result.perf) {
|
||||
setPerfMetrics({
|
||||
comp: endComp - startComp,
|
||||
aiTotal: endAi - startAi,
|
||||
aiApi: result.perf.apiCall || result.perf.apiDuration || 0,
|
||||
aiParse: result.perf.parsing || result.perf.parseDuration || 0,
|
||||
uploadSize: result.perf.uploadSize || 0,
|
||||
prep: endPrep - startPrep,
|
||||
imagePrep: result.perf.imagePrep,
|
||||
cacheCheck: result.perf.cacheCheck,
|
||||
encoding: result.perf.encoding,
|
||||
modelInit: result.perf.modelInit,
|
||||
validation: result.perf.validation,
|
||||
dbOps: result.perf.dbOps,
|
||||
total: result.perf.total,
|
||||
cacheHit: result.perf.cacheHit
|
||||
});
|
||||
}
|
||||
|
||||
setState('EDITOR');
|
||||
|
||||
// Step 2: Background Enrichment
|
||||
if (result.data.name && result.data.distillery) {
|
||||
setIsEnriching(true);
|
||||
console.log('[ScanFlow] Starting background enrichment for:', result.data.name);
|
||||
enrichData(result.data.name, result.data.distillery, undefined, locale)
|
||||
.then(enrichResult => {
|
||||
if (enrichResult.success && enrichResult.data) {
|
||||
console.log('[ScanFlow] Enrichment data received:', enrichResult.data);
|
||||
setBottleMetadata(prev => {
|
||||
if (!prev) return prev;
|
||||
const updated = {
|
||||
...prev,
|
||||
suggested_tags: enrichResult.data.suggested_tags,
|
||||
suggested_custom_tags: enrichResult.data.suggested_custom_tags,
|
||||
search_string: enrichResult.data.search_string
|
||||
};
|
||||
console.log('[ScanFlow] State updated with enriched metadata');
|
||||
return updated;
|
||||
});
|
||||
} else {
|
||||
console.warn('[ScanFlow] Enrichment result unsuccessful:', enrichResult.error);
|
||||
}
|
||||
})
|
||||
.catch(err => console.error('[ScanFlow] Enrichment failed:', err))
|
||||
.finally(() => setIsEnriching(false));
|
||||
}
|
||||
} else if (result.isAiError) {
|
||||
console.warn('[ScanFlow] AI Analysis failed, falling back to offline mode');
|
||||
setIsOffline(true);
|
||||
setAiFallbackActive(true);
|
||||
const dummyMetadata = generateDummyMetadata(file);
|
||||
setBottleMetadata(dummyMetadata);
|
||||
setState('EDITOR');
|
||||
return;
|
||||
} else {
|
||||
throw new Error(result.error || 'Fehler bei der Analyse.');
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error('[ScanFlow] handleScan error:', err);
|
||||
|
||||
// Check if this is a network error (offline)
|
||||
if (err.message?.includes('Failed to fetch') || err.message?.includes('NetworkError') || err.message?.includes('ERR_INTERNET_DISCONNECTED')) {
|
||||
console.log('[ScanFlow] Network error detected - switching to offline mode');
|
||||
setIsOffline(true);
|
||||
|
||||
// Use dummy metadata for offline scan
|
||||
const dummyMetadata = generateDummyMetadata(file);
|
||||
setBottleMetadata(dummyMetadata);
|
||||
setState('EDITOR');
|
||||
return;
|
||||
}
|
||||
|
||||
// Other errors
|
||||
setError(err.message);
|
||||
setState('ERROR');
|
||||
}
|
||||
scanner.handleScan(file);
|
||||
};
|
||||
|
||||
const handleSaveTasting = async (formData: any) => {
|
||||
if (!bottleMetadata || !processedImage) return;
|
||||
if (!bottleMetadata || !scanner.processedImage) return;
|
||||
|
||||
setIsSaving(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// OFFLINE: Save to IndexedDB queue (skip auth check)
|
||||
// OFFLINE: Save to IndexedDB queue
|
||||
if (isOffline) {
|
||||
console.log('[ScanFlow] Offline mode - queuing for upload');
|
||||
const tempId = `temp_${Date.now()}`;
|
||||
const bottleDataToSave = formData.bottleMetadata || bottleMetadata;
|
||||
|
||||
// Check for existing pending scan with same image to prevent duplicates
|
||||
// Check for existing pending scan with same image
|
||||
const existingScan = await db.pending_scans
|
||||
.filter(s => s.imageBase64 === processedImage.base64)
|
||||
.filter(s => s.imageBase64 === scanner.processedImage!.base64)
|
||||
.first();
|
||||
|
||||
let currentTempId = tempId;
|
||||
@@ -262,13 +217,11 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
console.log('[ScanFlow] Existing pending scan found, reusing temp_id:', existingScan.temp_id);
|
||||
currentTempId = existingScan.temp_id;
|
||||
} else {
|
||||
// Save pending scan with metadata
|
||||
await db.pending_scans.add({
|
||||
temp_id: tempId,
|
||||
imageBase64: processedImage.base64,
|
||||
imageBase64: scanner.processedImage.base64,
|
||||
timestamp: Date.now(),
|
||||
locale,
|
||||
// Store bottle metadata in a custom field
|
||||
metadata: bottleDataToSave as any
|
||||
});
|
||||
}
|
||||
@@ -302,69 +255,25 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
if (!authUser) throw new Error('Nicht autorisiert');
|
||||
user = authUser;
|
||||
} catch (authError: any) {
|
||||
// If auth fails due to network, treat as offline
|
||||
if (authError.message?.includes('Failed to fetch') || authError.message?.includes('NetworkError')) {
|
||||
console.log('[ScanFlow] Auth failed due to network - switching to offline mode');
|
||||
setIsOffline(true);
|
||||
|
||||
// Save to queue instead
|
||||
const tempId = `temp_${Date.now()}`;
|
||||
const bottleDataToSave = formData.bottleMetadata || bottleMetadata;
|
||||
|
||||
// Check for existing pending scan with same image to prevent duplicates
|
||||
const existingScan = await db.pending_scans
|
||||
.filter(s => s.imageBase64 === processedImage.base64)
|
||||
.first();
|
||||
|
||||
let currentTempId = tempId;
|
||||
|
||||
if (existingScan) {
|
||||
console.log('[ScanFlow] Existing pending scan found, reusing temp_id:', existingScan.temp_id);
|
||||
currentTempId = existingScan.temp_id;
|
||||
} else {
|
||||
await db.pending_scans.add({
|
||||
temp_id: tempId,
|
||||
imageBase64: processedImage.base64,
|
||||
timestamp: Date.now(),
|
||||
locale,
|
||||
metadata: bottleDataToSave as any
|
||||
});
|
||||
}
|
||||
|
||||
await db.pending_tastings.add({
|
||||
pending_bottle_id: currentTempId,
|
||||
data: {
|
||||
session_id: activeSession?.id,
|
||||
rating: formData.rating,
|
||||
nose_notes: formData.nose_notes,
|
||||
palate_notes: formData.palate_notes,
|
||||
finish_notes: formData.finish_notes,
|
||||
is_sample: formData.is_sample,
|
||||
buddy_ids: formData.buddy_ids,
|
||||
tag_ids: formData.tag_ids,
|
||||
},
|
||||
tasted_at: new Date().toISOString()
|
||||
});
|
||||
|
||||
setTastingData(formData);
|
||||
setState('RESULT');
|
||||
setIsSaving(false);
|
||||
return;
|
||||
// Retry save in offline mode
|
||||
return handleSaveTasting(formData);
|
||||
}
|
||||
// Other auth errors
|
||||
throw authError;
|
||||
}
|
||||
|
||||
// 1. Save Bottle - Use edited metadata if provided
|
||||
// Save Bottle
|
||||
const bottleDataToSave = formData.bottleMetadata || bottleMetadata;
|
||||
const bottleResult = await saveBottle(bottleDataToSave, processedImage.base64, user.id);
|
||||
const bottleResult = await saveBottle(bottleDataToSave, scanner.processedImage.base64, user.id);
|
||||
if (!bottleResult.success || !bottleResult.data) {
|
||||
throw new Error(bottleResult.error || 'Fehler beim Speichern der Flasche');
|
||||
}
|
||||
|
||||
const bottleId = bottleResult.data.id;
|
||||
|
||||
// 2. Save Tasting
|
||||
// Save Tasting
|
||||
const tastingNote = {
|
||||
...formData,
|
||||
bottle_id: bottleId,
|
||||
@@ -378,7 +287,6 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
setTastingData(tastingNote);
|
||||
setState('RESULT');
|
||||
|
||||
// Trigger bottle list refresh in parent
|
||||
if (onBottleSaved) {
|
||||
onBottleSaved(bottleId);
|
||||
}
|
||||
@@ -406,6 +314,22 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
}
|
||||
};
|
||||
|
||||
// Map scanner status to display text
|
||||
const getScanStatusDisplay = (status: ScanStatus): { text: string; icon: React.ReactNode } => {
|
||||
switch (status) {
|
||||
case 'compressing':
|
||||
return { text: 'Bild optimieren...', icon: <Loader2 size={12} className="animate-spin" /> };
|
||||
case 'analyzing_local':
|
||||
return { text: 'Lokale OCR-Analyse...', icon: <Cpu size={12} /> };
|
||||
case 'analyzing_cloud':
|
||||
return { text: 'KI-Vision-Analyse...', icon: <Cloud size={12} /> };
|
||||
default:
|
||||
return { text: 'Analysiere Etikett...', icon: <Loader2 size={12} className="animate-spin" /> };
|
||||
}
|
||||
};
|
||||
|
||||
const statusDisplay = getScanStatusDisplay(scanner.status);
|
||||
|
||||
return (
|
||||
<AnimatePresence>
|
||||
{isOpen && (
|
||||
@@ -424,11 +348,6 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
</button>
|
||||
|
||||
<div className="flex-1 w-full h-full flex flex-col relative min-h-0">
|
||||
{/*
|
||||
Robust state check:
|
||||
If we are IDLE but have an image, we are essentially SCANNING (or about to be).
|
||||
If we have no image, we shouldn't really be here, but show error just in case.
|
||||
*/}
|
||||
{(state === 'SCANNING' || (state === 'IDLE' && imageFile)) && (
|
||||
<div className="flex-1 flex flex-col items-center justify-center">
|
||||
<motion.div
|
||||
@@ -449,43 +368,39 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
<div className="text-center space-y-2">
|
||||
<h2 className="text-2xl font-bold text-zinc-50 uppercase tracking-tight">Analysiere Etikett...</h2>
|
||||
<p className="text-orange-500 font-bold uppercase tracking-widest text-[10px] flex items-center justify-center gap-2">
|
||||
<Sparkles size={12} /> KI-gestütztes Scanning
|
||||
{statusDisplay.icon}
|
||||
{statusDisplay.text}
|
||||
</p>
|
||||
{/* Show scan stage indicators */}
|
||||
<div className="flex items-center justify-center gap-2 mt-4">
|
||||
<div className={`w-2 h-2 rounded-full transition-colors ${['compressing', 'analyzing_local', 'analyzing_cloud', 'complete'].includes(scanner.status)
|
||||
? 'bg-orange-500' : 'bg-zinc-700'
|
||||
}`} />
|
||||
<div className={`w-2 h-2 rounded-full transition-colors ${['analyzing_local', 'analyzing_cloud', 'complete'].includes(scanner.status)
|
||||
? 'bg-orange-500' : 'bg-zinc-700'
|
||||
}`} />
|
||||
<div className={`w-2 h-2 rounded-full transition-colors ${['analyzing_cloud', 'complete'].includes(scanner.status)
|
||||
? 'bg-orange-500' : 'bg-zinc-700'
|
||||
}`} />
|
||||
</div>
|
||||
</div>
|
||||
</motion.div>
|
||||
|
||||
{isAdmin && perfMetrics && (
|
||||
{/* Admin perf metrics */}
|
||||
{isAdmin && scanner.perf && (
|
||||
<div className="mt-8 p-6 bg-zinc-950/80 backdrop-blur-xl rounded-3xl border border-orange-500/20 text-[10px] font-mono text-zinc-400 animate-in fade-in slide-in-from-bottom-4 shadow-2xl">
|
||||
<div className="grid grid-cols-3 gap-6 text-center">
|
||||
<div>
|
||||
<p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">Client</p>
|
||||
<p className="text-orange-500 font-bold">{perfMetrics.comp.toFixed(0)}ms</p>
|
||||
<p className="text-[8px] opacity-40 mt-1">{(perfMetrics.uploadSize / 1024).toFixed(0)} KB</p>
|
||||
<p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">Compress</p>
|
||||
<p className="text-orange-500 font-bold">{scanner.perf.compression.toFixed(0)}ms</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">AI Engine</p>
|
||||
{perfMetrics.cacheHit ? (
|
||||
<div className="flex flex-col items-center">
|
||||
<p className="text-green-500 font-bold tracking-tighter">CACHE HIT</p>
|
||||
<p className="text-[7px] opacity-40 mt-1">DB RESULTS</p>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<p className="text-orange-500 font-bold">{perfMetrics.aiTotal.toFixed(0)}ms</p>
|
||||
<div className="flex flex-col gap-0.5 mt-1 text-[7px] opacity-60">
|
||||
{perfMetrics.imagePrep !== undefined && <span>Prep: {perfMetrics.imagePrep.toFixed(0)}ms</span>}
|
||||
{perfMetrics.encoding !== undefined && <span>Encode: {perfMetrics.encoding.toFixed(0)}ms</span>}
|
||||
{perfMetrics.modelInit !== undefined && <span>Init: {perfMetrics.modelInit.toFixed(0)}ms</span>}
|
||||
<span className="text-orange-400">API: {perfMetrics.aiApi.toFixed(0)}ms</span>
|
||||
{perfMetrics.validation !== undefined && <span>Valid: {perfMetrics.validation.toFixed(0)}ms</span>}
|
||||
{perfMetrics.dbOps !== undefined && <span>DB: {perfMetrics.dbOps.toFixed(0)}ms</span>}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">Local OCR</p>
|
||||
<p className="text-orange-500 font-bold">{scanner.perf.localOcr.toFixed(0)}ms</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">App Logic</p>
|
||||
<p className="text-orange-500 font-bold">{perfMetrics.prep.toFixed(0)}ms</p>
|
||||
<p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">Cloud Vision</p>
|
||||
<p className="text-orange-500 font-bold">{scanner.perf.cloudVision.toFixed(0)}ms</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -525,6 +440,7 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
exit={{ y: -50, opacity: 0 }}
|
||||
className="flex-1 w-full h-full flex flex-col min-h-0"
|
||||
>
|
||||
{/* Status banners */}
|
||||
{isOffline && (
|
||||
<div className="bg-orange-500/10 border-b border-orange-500/20 p-4">
|
||||
<div className="max-w-2xl mx-auto flex items-center gap-3">
|
||||
@@ -537,9 +453,26 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Local preview indicator */}
|
||||
{scanner.status === 'analyzing_cloud' && scanner.localResult && (
|
||||
<div className="bg-blue-500/10 border-b border-blue-500/20 p-3">
|
||||
<div className="max-w-2xl mx-auto flex items-center gap-3">
|
||||
<Eye size={14} className="text-blue-500" />
|
||||
<p className="text-xs font-bold text-blue-500 uppercase tracking-wider flex items-center gap-2">
|
||||
Lokale Vorschau
|
||||
<span className="flex items-center gap-1 text-zinc-400 font-normal normal-case">
|
||||
<Loader2 size={10} className="animate-spin" />
|
||||
KI-Vision verfeinert Ergebnisse...
|
||||
</span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<TastingEditor
|
||||
bottleMetadata={bottleMetadata}
|
||||
image={processedImage?.base64 || null}
|
||||
image={scanner.processedImage?.base64 || null}
|
||||
onSave={handleSaveTasting}
|
||||
onOpenSessions={() => setIsSessionsOpen(true)}
|
||||
activeSessionName={activeSession?.name}
|
||||
@@ -547,38 +480,27 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
isEnriching={isEnriching}
|
||||
defaultExpanded={true}
|
||||
/>
|
||||
{isAdmin && perfMetrics && (
|
||||
<div className="absolute top-24 left-6 right-6 z-50 p-3 bg-zinc-950/80 backdrop-blur-md rounded-2xl border border-orange-500/20 text-[9px] font-mono text-white/90 shadow-xl overflow-x-auto">
|
||||
|
||||
{/* Admin perf overlay - positioned at bottom */}
|
||||
{isAdmin && scanner.perf && (
|
||||
<div className="fixed bottom-24 left-6 right-6 z-50 p-3 bg-zinc-950/80 backdrop-blur-md rounded-2xl border border-orange-500/20 text-[9px] font-mono text-white/90 shadow-xl overflow-x-auto">
|
||||
<div className="flex items-center justify-between gap-4 whitespace-nowrap">
|
||||
<div className="flex items-center gap-2">
|
||||
<Clock size={10} className="text-orange-500" />
|
||||
<span className="text-zinc-500">CLIENT:</span>
|
||||
<span className="text-orange-500 font-bold">{perfMetrics.comp.toFixed(0)}ms</span>
|
||||
<span className="text-zinc-600">({(perfMetrics.uploadSize / 1024).toFixed(0)}KB)</span>
|
||||
<span className="text-zinc-500">COMPRESS:</span>
|
||||
<span className="text-orange-500 font-bold">{scanner.perf.compression.toFixed(0)}ms</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-zinc-500">AI:</span>
|
||||
{perfMetrics.cacheHit ? (
|
||||
<span className="text-green-500 font-bold tracking-tight">CACHE HIT</span>
|
||||
) : (
|
||||
<>
|
||||
<span className="text-orange-500 font-bold">{perfMetrics.aiTotal.toFixed(0)}ms</span>
|
||||
<span className="text-zinc-600 ml-1 text-[10px]">
|
||||
(
|
||||
{perfMetrics.imagePrep !== undefined && `Prep:${perfMetrics.imagePrep.toFixed(0)} `}
|
||||
{perfMetrics.encoding !== undefined && `Enc:${perfMetrics.encoding.toFixed(0)} `}
|
||||
{perfMetrics.modelInit !== undefined && `Init:${perfMetrics.modelInit.toFixed(0)} `}
|
||||
<span className="text-orange-400 font-bold">API:{perfMetrics.aiApi.toFixed(0)}</span>
|
||||
{perfMetrics.validation !== undefined && ` Val:${perfMetrics.validation.toFixed(0)}`}
|
||||
{perfMetrics.dbOps !== undefined && ` DB:${perfMetrics.dbOps.toFixed(0)}`}
|
||||
)
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
<span className="text-zinc-500">LOCAL:</span>
|
||||
<span className="text-blue-500 font-bold">{scanner.perf.localOcr.toFixed(0)}ms</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-zinc-500">APP:</span>
|
||||
<span className="text-orange-500 font-bold">{perfMetrics.prep.toFixed(0)}ms</span>
|
||||
<span className="text-zinc-500">CLOUD:</span>
|
||||
<span className="text-green-500 font-bold">{scanner.perf.cloudVision.toFixed(0)}ms</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-zinc-500">TOTAL:</span>
|
||||
<span className="text-orange-500 font-bold">{scanner.perf.total.toFixed(0)}ms</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -613,7 +535,7 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
|
||||
balance: tastingData.balance || 85,
|
||||
}}
|
||||
bottleName={bottleMetadata.name || 'Unknown Whisky'}
|
||||
image={processedImage?.base64 || null}
|
||||
image={scanner.processedImage?.base64 || null}
|
||||
onShare={handleShare}
|
||||
/>
|
||||
</motion.div>
|
||||
|
||||
@@ -76,6 +76,57 @@ export default function TastingEditor({ bottleMetadata, image, onSave, onOpenSes
|
||||
const suggestedTags = bottleMetadata.suggested_tags || [];
|
||||
const suggestedCustomTags = bottleMetadata.suggested_custom_tags || [];
|
||||
|
||||
// Track last seen confidence to detect cloud vs local updates
|
||||
const lastConfidenceRef = React.useRef<number>(0);
|
||||
|
||||
// Sync bottleMetadata prop changes to internal state (for live Gemini updates)
|
||||
// Cloud data (confidence >= 0.6 OR >= 60) overrides local OCR (confidence ~50 or ~0.5)
|
||||
useEffect(() => {
|
||||
// Normalize confidence to 0-100 scale (Gemini returns 0-1, local returns 0-100)
|
||||
const rawConfidence = bottleMetadata.confidence ?? 0;
|
||||
const newConfidence = rawConfidence <= 1 ? rawConfidence * 100 : rawConfidence;
|
||||
const isCloudUpdate = newConfidence >= 60;
|
||||
const isUpgrade = newConfidence > lastConfidenceRef.current;
|
||||
|
||||
console.log('[TastingEditor] Syncing bottleMetadata update:', {
|
||||
rawConfidence,
|
||||
normalizedConfidence: newConfidence,
|
||||
isCloudUpdate,
|
||||
isUpgrade,
|
||||
previousConfidence: lastConfidenceRef.current,
|
||||
});
|
||||
|
||||
// For cloud updates (higher confidence), allow overwriting local OCR values
|
||||
// For local updates, only fill empty fields
|
||||
if (isCloudUpdate || isUpgrade) {
|
||||
// Cloud vision: update all fields that have new data
|
||||
if (bottleMetadata.name) setBottleName(bottleMetadata.name);
|
||||
if (bottleMetadata.distillery) setBottleDistillery(bottleMetadata.distillery);
|
||||
if (bottleMetadata.abv) setBottleAbv(bottleMetadata.abv.toString());
|
||||
if (bottleMetadata.age) setBottleAge(bottleMetadata.age.toString());
|
||||
if (bottleMetadata.category) setBottleCategory(bottleMetadata.category);
|
||||
if (bottleMetadata.vintage) setBottleVintage(bottleMetadata.vintage);
|
||||
if (bottleMetadata.bottler) setBottleBottler(bottleMetadata.bottler);
|
||||
if (bottleMetadata.batch_info) setBottleBatchInfo(bottleMetadata.batch_info);
|
||||
if (bottleMetadata.distilled_at) setBottleDistilledAt(bottleMetadata.distilled_at);
|
||||
if (bottleMetadata.bottled_at) setBottleBottledAt(bottleMetadata.bottled_at);
|
||||
} else {
|
||||
// Local OCR or initial: only update empty fields
|
||||
if (!bottleName && bottleMetadata.name) setBottleName(bottleMetadata.name);
|
||||
if (!bottleDistillery && bottleMetadata.distillery) setBottleDistillery(bottleMetadata.distillery);
|
||||
if (!bottleAbv && bottleMetadata.abv) setBottleAbv(bottleMetadata.abv.toString());
|
||||
if (!bottleAge && bottleMetadata.age) setBottleAge(bottleMetadata.age.toString());
|
||||
if ((!bottleCategory || bottleCategory === 'Whisky') && bottleMetadata.category) setBottleCategory(bottleMetadata.category);
|
||||
if (!bottleVintage && bottleMetadata.vintage) setBottleVintage(bottleMetadata.vintage);
|
||||
if (!bottleBottler && bottleMetadata.bottler) setBottleBottler(bottleMetadata.bottler);
|
||||
if (!bottleBatchInfo && bottleMetadata.batch_info) setBottleBatchInfo(bottleMetadata.batch_info);
|
||||
if (!bottleDistilledAt && bottleMetadata.distilled_at) setBottleDistilledAt(bottleMetadata.distilled_at);
|
||||
if (!bottleBottledAt && bottleMetadata.bottled_at) setBottleBottledAt(bottleMetadata.bottled_at);
|
||||
}
|
||||
|
||||
lastConfidenceRef.current = newConfidence;
|
||||
}, [bottleMetadata]);
|
||||
|
||||
// Session-based pre-fill and Palette Checker
|
||||
useEffect(() => {
|
||||
const fetchSessionData = async () => {
|
||||
|
||||
1014
src/data/distilleries.json
Normal file
1014
src/data/distilleries.json
Normal file
File diff suppressed because it is too large
Load Diff
342
src/hooks/useScanner.ts
Normal file
342
src/hooks/useScanner.ts
Normal file
@@ -0,0 +1,342 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useCallback, useRef } from 'react';
|
||||
import { BottleMetadata } from '@/types/whisky';
|
||||
import { processImageForAI, ProcessedImage } from '@/utils/image-processing';
|
||||
import { analyzeLocalOcr, LocalOcrResult, terminateOcrWorker } from '@/lib/ocr/local-engine';
|
||||
import { isTesseractReady, isOnline } from '@/lib/ocr/scanner-utils';
|
||||
import { analyzeLabelWithGemini } from '@/app/actions/gemini-vision';
|
||||
import { generateDummyMetadata } from '@/utils/generate-dummy-metadata';
|
||||
import { db } from '@/lib/db';
|
||||
|
||||
export type ScanStatus =
|
||||
| 'idle'
|
||||
| 'compressing'
|
||||
| 'analyzing_local'
|
||||
| 'analyzing_cloud'
|
||||
| 'complete'
|
||||
| 'queued'
|
||||
| 'error';
|
||||
|
||||
export interface ScanResult {
|
||||
status: ScanStatus;
|
||||
localResult: Partial<BottleMetadata> | null;
|
||||
cloudResult: BottleMetadata | null;
|
||||
mergedResult: BottleMetadata | null;
|
||||
processedImage: ProcessedImage | null;
|
||||
error: string | null;
|
||||
perf: {
|
||||
compression: number;
|
||||
localOcr: number;
|
||||
cloudVision: number;
|
||||
total: number;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface UseScannerOptions {
|
||||
locale?: string;
|
||||
onLocalComplete?: (result: Partial<BottleMetadata>) => void;
|
||||
onCloudComplete?: (result: BottleMetadata) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* React hook for the hybrid Local OCR + Cloud Vision scanning flow
|
||||
*
|
||||
* Flow:
|
||||
* 1. Compress image (browser-side)
|
||||
* 2. Run local OCR (tesseract.js) → immediate preview
|
||||
* 3. Run cloud vision (Gemini) → refined results
|
||||
* 4. Merge results (cloud overrides local, except user-edited fields)
|
||||
*/
|
||||
export function useScanner(options: UseScannerOptions = {}) {
|
||||
const { locale = 'en', onLocalComplete, onCloudComplete } = options;
|
||||
|
||||
const [result, setResult] = useState<ScanResult>({
|
||||
status: 'idle',
|
||||
localResult: null,
|
||||
cloudResult: null,
|
||||
mergedResult: null,
|
||||
processedImage: null,
|
||||
error: null,
|
||||
perf: null,
|
||||
});
|
||||
|
||||
// Track which fields the user has manually edited
|
||||
const dirtyFieldsRef = useRef<Set<string>>(new Set());
|
||||
|
||||
/**
|
||||
* Mark a field as user-edited (won't be overwritten by cloud results)
|
||||
*/
|
||||
const markFieldDirty = useCallback((field: string) => {
|
||||
dirtyFieldsRef.current.add(field);
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Clear all dirty field markers
|
||||
*/
|
||||
const clearDirtyFields = useCallback(() => {
|
||||
dirtyFieldsRef.current.clear();
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Merge local and cloud results, respecting user edits
|
||||
*/
|
||||
const mergeResults = useCallback((
|
||||
local: Partial<BottleMetadata> | null,
|
||||
cloud: BottleMetadata | null,
|
||||
dirtyFields: Set<string>
|
||||
): BottleMetadata | null => {
|
||||
if (!cloud && !local) return null;
|
||||
if (!cloud) {
|
||||
return {
|
||||
name: local?.name || null,
|
||||
distillery: local?.distillery || null,
|
||||
abv: local?.abv || null,
|
||||
age: local?.age || null,
|
||||
vintage: local?.vintage || null,
|
||||
is_whisky: true,
|
||||
confidence: 50,
|
||||
} as BottleMetadata;
|
||||
}
|
||||
if (!local) return cloud;
|
||||
|
||||
// Start with cloud result as base
|
||||
const merged = { ...cloud };
|
||||
|
||||
// For each field, keep local value if user edited it
|
||||
for (const field of dirtyFields) {
|
||||
if (field in local && (local as any)[field] !== undefined) {
|
||||
(merged as any)[field] = (local as any)[field];
|
||||
}
|
||||
}
|
||||
|
||||
return merged;
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Main scan handler
|
||||
*/
|
||||
const handleScan = useCallback(async (file: File) => {
|
||||
const perfStart = performance.now();
|
||||
let perfCompression = 0;
|
||||
let perfLocalOcr = 0;
|
||||
let perfCloudVision = 0;
|
||||
|
||||
// Reset state
|
||||
clearDirtyFields();
|
||||
setResult({
|
||||
status: 'compressing',
|
||||
localResult: null,
|
||||
cloudResult: null,
|
||||
mergedResult: null,
|
||||
processedImage: null,
|
||||
error: null,
|
||||
perf: null,
|
||||
});
|
||||
|
||||
try {
|
||||
// Step 1: Compress image
|
||||
const compressStart = performance.now();
|
||||
const processedImage = await processImageForAI(file);
|
||||
perfCompression = performance.now() - compressStart;
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'analyzing_local',
|
||||
processedImage,
|
||||
}));
|
||||
|
||||
// Step 2: Check if we're offline or tesseract isn't ready
|
||||
const online = isOnline();
|
||||
const tesseractReady = await isTesseractReady();
|
||||
|
||||
// If offline and tesseract not ready, queue immediately
|
||||
if (!online && !tesseractReady) {
|
||||
console.log('[useScanner] Offline + no tesseract cache → queuing');
|
||||
const dummyMetadata = generateDummyMetadata(file);
|
||||
|
||||
await db.pending_scans.add({
|
||||
temp_id: `temp_${Date.now()}`,
|
||||
imageBase64: processedImage.base64,
|
||||
timestamp: Date.now(),
|
||||
locale,
|
||||
metadata: dummyMetadata as any,
|
||||
});
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'queued',
|
||||
mergedResult: dummyMetadata,
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: 0,
|
||||
cloudVision: 0,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 3: Run local OCR (testing new line-by-line matching)
|
||||
let localResult: Partial<BottleMetadata> | null = null;
|
||||
|
||||
try {
|
||||
const localStart = performance.now();
|
||||
console.log('[useScanner] Running local OCR...');
|
||||
const ocrResult = await analyzeLocalOcr(processedImage.file, 10000);
|
||||
perfLocalOcr = performance.now() - localStart;
|
||||
|
||||
if (ocrResult.rawText && ocrResult.rawText.length > 10) {
|
||||
localResult = {
|
||||
name: ocrResult.name || undefined,
|
||||
distillery: ocrResult.distillery || undefined,
|
||||
abv: ocrResult.abv || undefined,
|
||||
age: ocrResult.age || undefined,
|
||||
vintage: ocrResult.vintage || undefined,
|
||||
};
|
||||
|
||||
// Update state with local results
|
||||
const localMerged = mergeResults(localResult, null, dirtyFieldsRef.current);
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
localResult,
|
||||
mergedResult: localMerged,
|
||||
}));
|
||||
|
||||
onLocalComplete?.(localResult);
|
||||
console.log('[useScanner] Local OCR complete:', localResult);
|
||||
}
|
||||
} catch (ocrError) {
|
||||
console.warn('[useScanner] Local OCR failed:', ocrError);
|
||||
}
|
||||
|
||||
// Step 4: If offline, use local results only
|
||||
if (!online) {
|
||||
console.log('[useScanner] Offline → using local results only');
|
||||
const offlineMerged = mergeResults(localResult, null, dirtyFieldsRef.current);
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'complete',
|
||||
mergedResult: offlineMerged || generateDummyMetadata(file),
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: perfLocalOcr,
|
||||
cloudVision: 0,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 5: Run cloud vision analysis
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'analyzing_cloud',
|
||||
}));
|
||||
|
||||
const cloudStart = performance.now();
|
||||
const cloudResponse = await analyzeLabelWithGemini(processedImage.base64);
|
||||
perfCloudVision = performance.now() - cloudStart;
|
||||
|
||||
if (cloudResponse.success && cloudResponse.data) {
|
||||
const cloudResult = cloudResponse.data;
|
||||
const finalMerged = mergeResults(localResult, cloudResult, dirtyFieldsRef.current);
|
||||
|
||||
onCloudComplete?.(cloudResult);
|
||||
console.log('[useScanner] Cloud vision complete:', cloudResult);
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'complete',
|
||||
cloudResult,
|
||||
mergedResult: finalMerged,
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: perfLocalOcr,
|
||||
cloudVision: perfCloudVision,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
} else {
|
||||
// Cloud failed, fall back to local results
|
||||
console.warn('[useScanner] Cloud vision failed:', cloudResponse.error);
|
||||
const fallbackMerged = mergeResults(localResult, null, dirtyFieldsRef.current);
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'complete',
|
||||
error: cloudResponse.error || null,
|
||||
mergedResult: fallbackMerged || generateDummyMetadata(file),
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: perfLocalOcr,
|
||||
cloudVision: perfCloudVision,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
} catch (error: any) {
|
||||
console.error('[useScanner] Scan failed:', error);
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'error',
|
||||
error: error.message || 'Scan failed',
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: perfLocalOcr,
|
||||
cloudVision: perfCloudVision,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
}
|
||||
}, [locale, mergeResults, clearDirtyFields, onLocalComplete, onCloudComplete]);
|
||||
|
||||
/**
|
||||
* Reset scanner state
|
||||
*/
|
||||
const reset = useCallback(() => {
|
||||
clearDirtyFields();
|
||||
setResult({
|
||||
status: 'idle',
|
||||
localResult: null,
|
||||
cloudResult: null,
|
||||
mergedResult: null,
|
||||
processedImage: null,
|
||||
error: null,
|
||||
perf: null,
|
||||
});
|
||||
}, [clearDirtyFields]);
|
||||
|
||||
/**
|
||||
* Update the merged result (for user edits)
|
||||
*/
|
||||
const updateMergedResult = useCallback((updates: Partial<BottleMetadata>) => {
|
||||
// Mark updated fields as dirty
|
||||
Object.keys(updates).forEach(key => {
|
||||
dirtyFieldsRef.current.add(key);
|
||||
});
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
mergedResult: prev.mergedResult ? { ...prev.mergedResult, ...updates } : null,
|
||||
}));
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Cleanup (terminate OCR worker)
|
||||
*/
|
||||
const cleanup = useCallback(() => {
|
||||
terminateOcrWorker();
|
||||
}, []);
|
||||
|
||||
return {
|
||||
...result,
|
||||
handleScan,
|
||||
reset,
|
||||
markFieldDirty,
|
||||
updateMergedResult,
|
||||
cleanup,
|
||||
};
|
||||
}
|
||||
313
src/lib/ocr/local-engine.ts
Normal file
313
src/lib/ocr/local-engine.ts
Normal file
@@ -0,0 +1,313 @@
|
||||
/**
|
||||
* Local OCR Engine
|
||||
* Client-side OCR using Tesseract.js with Fuse.js fuzzy matching
|
||||
*
|
||||
* Optimized for whisky label scanning with:
|
||||
* - Image preprocessing (grayscale, binarization, center crop)
|
||||
* - PSM 11 (Sparse text mode)
|
||||
* - Character whitelisting
|
||||
* - Bag-of-words fuzzy matching
|
||||
*/
|
||||
|
||||
import Tesseract from 'tesseract.js';
|
||||
import Fuse from 'fuse.js';
|
||||
import { extractNumbers, ExtractedNumbers, preprocessImageForOCR } from './scanner-utils';
|
||||
import distilleries from '@/data/distilleries.json';
|
||||
|
||||
export interface LocalOcrResult {
|
||||
distillery: string | null;
|
||||
distilleryRegion: string | null;
|
||||
name: string | null;
|
||||
age: number | null;
|
||||
abv: number | null;
|
||||
vintage: string | null;
|
||||
rawText: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
// Fuse.js configuration for fuzzy matching distillery names
|
||||
// Balanced matching to catch partial OCR errors while avoiding false positives
|
||||
const fuseOptions = {
|
||||
keys: ['name'],
|
||||
threshold: 0.35, // 0 = exact match, 0.35 = allow some fuzziness
|
||||
distance: 50, // Characters between matched chars
|
||||
includeScore: true,
|
||||
minMatchCharLength: 4, // Minimum chars to match
|
||||
};
|
||||
|
||||
const distilleryFuse = new Fuse(distilleries, fuseOptions);
|
||||
|
||||
// Tesseract worker singleton (reused across scans)
|
||||
let tesseractWorker: Tesseract.Worker | null = null;
|
||||
|
||||
// Character whitelist for whisky labels (no special symbols that cause noise)
|
||||
const CHAR_WHITELIST = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789%.,:\'"-/ ';
|
||||
|
||||
/**
|
||||
* Initialize or get the Tesseract worker
|
||||
* Uses local files from /public/tessdata for offline capability
|
||||
*/
|
||||
async function getWorker(): Promise<Tesseract.Worker> {
|
||||
if (tesseractWorker) {
|
||||
return tesseractWorker;
|
||||
}
|
||||
|
||||
console.log('[LocalOCR] Initializing Tesseract worker with local files...');
|
||||
|
||||
// Use local files from /public/tessdata
|
||||
tesseractWorker = await Tesseract.createWorker('eng', Tesseract.OEM.LSTM_ONLY, {
|
||||
corePath: '/tessdata/',
|
||||
langPath: '/tessdata/',
|
||||
logger: (m) => {
|
||||
if (m.status === 'recognizing text') {
|
||||
console.log(`[LocalOCR] Progress: ${Math.round(m.progress * 100)}%`);
|
||||
} else {
|
||||
console.log(`[LocalOCR] ${m.status}`);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Configure Tesseract for whisky label OCR
|
||||
await tesseractWorker.setParameters({
|
||||
tessedit_pageseg_mode: Tesseract.PSM.SINGLE_BLOCK, // PSM 6 - treat as single block of text
|
||||
tessedit_char_whitelist: CHAR_WHITELIST,
|
||||
preserve_interword_spaces: '1', // Keep word spacing
|
||||
});
|
||||
|
||||
console.log('[LocalOCR] Tesseract worker ready (PSM: SINGLE_BLOCK, Whitelist enabled)');
|
||||
return tesseractWorker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run OCR on an image and extract whisky metadata
|
||||
*
|
||||
* @param imageSource - File, Blob, or base64 string of the image
|
||||
* @param timeoutMs - Maximum time to wait for OCR (default 10000ms)
|
||||
* @returns LocalOcrResult with extracted metadata
|
||||
*/
|
||||
export async function analyzeLocalOcr(
|
||||
imageSource: File | Blob | string,
|
||||
timeoutMs: number = 10000
|
||||
): Promise<LocalOcrResult> {
|
||||
const result: LocalOcrResult = {
|
||||
distillery: null,
|
||||
distilleryRegion: null,
|
||||
name: null,
|
||||
age: null,
|
||||
abv: null,
|
||||
vintage: null,
|
||||
rawText: '',
|
||||
confidence: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
// Step 1: Preprocess the image for better OCR
|
||||
let processedImage: string;
|
||||
if (typeof imageSource === 'string') {
|
||||
// Already a data URL, use as-is (can't preprocess string)
|
||||
processedImage = imageSource;
|
||||
console.log('[LocalOCR] Using raw image (string input)');
|
||||
} else {
|
||||
// Preprocess File/Blob: grayscale + sharpen + contrast boost
|
||||
console.log('[LocalOCR] Preprocessing image...');
|
||||
processedImage = await preprocessImageForOCR(imageSource);
|
||||
// Uses defaults: 5% edge crop, 1200px height, sharpen=true, 1.3x contrast
|
||||
}
|
||||
|
||||
// Create a timeout promise
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
setTimeout(() => reject(new Error('OCR timeout')), timeoutMs);
|
||||
});
|
||||
|
||||
// Race OCR against timeout
|
||||
const worker = await getWorker();
|
||||
const ocrResult = await Promise.race([
|
||||
worker.recognize(processedImage),
|
||||
timeoutPromise,
|
||||
]);
|
||||
|
||||
result.rawText = ocrResult.data.text;
|
||||
result.confidence = ocrResult.data.confidence / 100; // Normalize to 0-1
|
||||
|
||||
// Extract numbers using regex (this works reliably)
|
||||
const numbers = extractNumbers(result.rawText);
|
||||
result.abv = numbers.abv;
|
||||
result.age = numbers.age;
|
||||
result.vintage = numbers.vintage;
|
||||
|
||||
// NOTE: Distillery fuzzy matching disabled - causes too many false positives
|
||||
// with noisy OCR text. Let Gemini Vision handle distillery identification.
|
||||
// const distilleryMatch = findDistillery(result.rawText);
|
||||
// if (distilleryMatch) {
|
||||
// result.distillery = distilleryMatch.name;
|
||||
// result.distilleryRegion = distilleryMatch.region;
|
||||
// }
|
||||
|
||||
// Fuzzy match distillery (new algorithm with sanity checks)
|
||||
const distilleryMatch = findDistillery(result.rawText);
|
||||
if (distilleryMatch) {
|
||||
result.distillery = distilleryMatch.name;
|
||||
result.distilleryRegion = distilleryMatch.region;
|
||||
|
||||
// Use contextual age if regex extraction failed
|
||||
if (!result.age && distilleryMatch.contextualAge) {
|
||||
result.age = distilleryMatch.contextualAge;
|
||||
console.log(`[LocalOCR] Using contextual age: ${result.age}`);
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: Name extraction disabled - Tesseract too noisy for full bottle names
|
||||
// Let Gemini Vision handle the name field
|
||||
// result.name = extractName(result.rawText, result.distillery);
|
||||
result.name = null;
|
||||
|
||||
// Detailed logging for debugging
|
||||
const cleanedText = result.rawText
|
||||
.split('\n')
|
||||
.map(line => line.trim())
|
||||
.filter(line => line.length > 0)
|
||||
.join(' | ');
|
||||
|
||||
console.log('[LocalOCR] ========== OCR RESULTS ==========');
|
||||
console.log('[LocalOCR] Raw Text:\n', result.rawText);
|
||||
console.log('[LocalOCR] Cleaned Text:', cleanedText);
|
||||
console.log('[LocalOCR] Confidence:', (result.confidence * 100).toFixed(1) + '%');
|
||||
console.log('[LocalOCR] Extracted Data:', {
|
||||
distillery: result.distillery,
|
||||
distilleryRegion: result.distilleryRegion,
|
||||
name: result.name,
|
||||
age: result.age,
|
||||
abv: result.abv,
|
||||
vintage: result.vintage,
|
||||
});
|
||||
console.log('[LocalOCR] ===================================');
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
console.warn('[LocalOCR] Analysis failed:', error);
|
||||
return result; // Return partial/empty result
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a distillery name in OCR text using fuzzy matching
|
||||
*
|
||||
* Strategy:
|
||||
* 1. Normalize whitespace (fix Tesseract's formatting gaps)
|
||||
* 2. Split OCR text into lines, filter garbage
|
||||
* 3. "Strip & Match": Remove numbers before Fuse matching (helps with "N NEVIS 27")
|
||||
* 4. Sanity check: match length must be reasonable
|
||||
* 5. Contextual age: if distillery found, look for age in original line
|
||||
*/
|
||||
function findDistillery(text: string): { name: string; region: string; contextualAge?: number } | null {
|
||||
// Split into lines, normalize whitespace, and filter garbage
|
||||
const lines = text
|
||||
.split('\n')
|
||||
.map(line => line.trim().replace(/\s+/g, ' ')) // Normalize whitespace
|
||||
.filter(line => {
|
||||
// Minimum 4 characters
|
||||
if (line.length < 4) return false;
|
||||
// Must have at least 40% letters (lowered to allow lines with numbers)
|
||||
const letters = line.replace(/[^a-zA-Z]/g, '');
|
||||
return letters.length >= line.length * 0.4;
|
||||
});
|
||||
|
||||
console.log('[LocalOCR] Lines for distillery matching:', lines.length);
|
||||
|
||||
// Try to match each line
|
||||
for (const originalLine of lines) {
|
||||
// STRIP & MATCH: Remove numbers for cleaner Fuse matching
|
||||
// "Bad N NEVIS 27" → "Bad N NEVIS "
|
||||
const textOnlyLine = originalLine.replace(/[0-9]/g, '').replace(/\s+/g, ' ').trim();
|
||||
|
||||
if (textOnlyLine.length < 4) continue;
|
||||
|
||||
const results = distilleryFuse.search(textOnlyLine);
|
||||
|
||||
if (results.length > 0 && results[0].score !== undefined && results[0].score < 0.4) {
|
||||
const match = results[0].item;
|
||||
const matchScore = results[0].score;
|
||||
|
||||
// SANITY CHECK: The text-only part should be similar length to distillery name
|
||||
// Max 60% deviation allowed (relaxed for partial matches)
|
||||
const lengthRatio = textOnlyLine.length / match.name.length;
|
||||
const lengthDeviation = Math.abs(1 - lengthRatio);
|
||||
|
||||
if (lengthDeviation > 0.6) {
|
||||
console.log(`[LocalOCR] Match rejected (length): "${textOnlyLine}" → ${match.name} (ratio: ${lengthRatio.toFixed(2)}, deviation: ${(lengthDeviation * 100).toFixed(0)}%)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// CONTEXTUAL AGE DETECTION: Look for 2-digit number (3-60) in ORIGINAL line
|
||||
let contextualAge: number | undefined;
|
||||
const ageMatch = originalLine.match(/\b(\d{1,2})\b/);
|
||||
if (ageMatch) {
|
||||
const potentialAge = parseInt(ageMatch[1], 10);
|
||||
if (potentialAge >= 3 && potentialAge <= 60) {
|
||||
contextualAge = potentialAge;
|
||||
console.log(`[LocalOCR] Contextual age detected: ${potentialAge} years`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[LocalOCR] Distillery match: "${textOnlyLine}" → ${match.name} (score: ${matchScore.toFixed(3)}, original: "${originalLine}")`);
|
||||
return {
|
||||
name: match.name,
|
||||
region: match.region,
|
||||
contextualAge,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a potential bottle name from OCR text
|
||||
*/
|
||||
function extractName(text: string, distillery: string | null): string | null {
|
||||
const lines = text
|
||||
.split('\n')
|
||||
.map(l => l.trim())
|
||||
.filter(line => {
|
||||
// Minimum 5 characters
|
||||
if (line.length < 5) return false;
|
||||
// Must have at least 60% letters (filter out garbage like "ee" or "4 . .")
|
||||
const letters = line.replace(/[^a-zA-Z]/g, '');
|
||||
if (letters.length < line.length * 0.6) return false;
|
||||
// Skip lines that are just punctuation/numbers
|
||||
if (/^[\d\s.,\-'"]+$/.test(line)) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
// Skip lines that are just the distillery name
|
||||
const candidates = lines.filter(line => {
|
||||
if (distillery && line.toLowerCase().includes(distillery.toLowerCase())) {
|
||||
// Only skip if the line IS the distillery name (not contains more)
|
||||
return line.length > distillery.length + 5;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
// Return the first substantial line (likely the bottle name)
|
||||
for (const line of candidates) {
|
||||
// Skip lines that look like numbers only
|
||||
if (/^\d+[\s%]+/.test(line)) continue;
|
||||
// Skip lines that are just common whisky words
|
||||
if (/^(single|malt|scotch|whisky|whiskey|aged|years?|proof|edition|distilled|distillery)$/i.test(line)) continue;
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminate the Tesseract worker (call on cleanup)
|
||||
*/
|
||||
export async function terminateOcrWorker(): Promise<void> {
|
||||
if (tesseractWorker) {
|
||||
await tesseractWorker.terminate();
|
||||
tesseractWorker = null;
|
||||
}
|
||||
}
|
||||
312
src/lib/ocr/scanner-utils.ts
Normal file
312
src/lib/ocr/scanner-utils.ts
Normal file
@@ -0,0 +1,312 @@
|
||||
/**
|
||||
* Scanner Utilities
|
||||
* Cache checking and helper functions for client-side OCR
|
||||
*/
|
||||
|
||||
/**
|
||||
* Check if Tesseract.js is ready to run
|
||||
* When online, tesseract will auto-download from CDN, so return true
|
||||
* When offline, check if files are cached
|
||||
* @returns Promise<boolean> - true if OCR can run
|
||||
*/
|
||||
export async function isTesseractReady(): Promise<boolean> {
|
||||
if (typeof window === 'undefined') {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If online, tesseract.js will auto-download what it needs
|
||||
if (navigator.onLine) {
|
||||
console.log('[Scanner] Online - tesseract will use CDN');
|
||||
return true;
|
||||
}
|
||||
|
||||
// If offline, check cache
|
||||
if (!('caches' in window)) {
|
||||
console.log('[Scanner] Offline + no cache API - tesseract not ready');
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
// Check for the core files in cache (matching actual file names in /public/tessdata)
|
||||
const wasmMatch = await window.caches.match('/tessdata/tesseract-core-simd.wasm');
|
||||
const langMatch = await window.caches.match('/tessdata/eng.traineddata');
|
||||
|
||||
const ready = !!(wasmMatch && langMatch);
|
||||
console.log('[Scanner] Offline cache check:', { wasmMatch: !!wasmMatch, langMatch: !!langMatch, ready });
|
||||
return ready;
|
||||
} catch (error) {
|
||||
console.warn('[Scanner] Cache check failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract numeric values from OCR text using regex patterns
|
||||
*/
|
||||
export interface ExtractedNumbers {
|
||||
abv: number | null;
|
||||
age: number | null;
|
||||
vintage: string | null;
|
||||
}
|
||||
|
||||
export function extractNumbers(text: string): ExtractedNumbers {
|
||||
const result: ExtractedNumbers = {
|
||||
abv: null,
|
||||
age: null,
|
||||
vintage: null
|
||||
};
|
||||
|
||||
if (!text) return result;
|
||||
|
||||
// Normalize text: lowercase, clean up common OCR mistakes
|
||||
const normalizedText = text
|
||||
.replace(/[oO]/g, '0') // Common OCR mistake: O -> 0
|
||||
.replace(/[lI]/g, '1') // Common OCR mistake: l/I -> 1
|
||||
.toLowerCase();
|
||||
|
||||
// ABV patterns: "43%", "43.5%", "43,5 %", "ABV 43", "vol. 43"
|
||||
const abvPatterns = [
|
||||
/(\d{2}[.,]\d{1,2})\s*%/, // 43.5% or 43,5 %
|
||||
/(\d{2})\s*%/, // 43%
|
||||
/abv[:\s]*(\d{2}[.,]?\d{0,2})/i, // ABV: 43 or ABV 43.5
|
||||
/vol[.\s]*(\d{2}[.,]?\d{0,2})/i, // vol. 43
|
||||
/(\d{2}[.,]\d{1,2})\s*vol/i, // 43.5 vol
|
||||
];
|
||||
|
||||
for (const pattern of abvPatterns) {
|
||||
const match = normalizedText.match(pattern);
|
||||
if (match) {
|
||||
const value = parseFloat(match[1].replace(',', '.'));
|
||||
if (value >= 35 && value <= 75) { // Reasonable whisky ABV range
|
||||
result.abv = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Age patterns: "12 years", "12 year old", "12 YO", "aged 12"
|
||||
const agePatterns = [
|
||||
/(\d{1,2})\s*(?:years?|yrs?|y\.?o\.?|jahre?)/i,
|
||||
/aged\s*(\d{1,2})/i,
|
||||
/(\d{1,2})\s*year\s*old/i,
|
||||
];
|
||||
|
||||
for (const pattern of agePatterns) {
|
||||
const match = text.match(pattern);
|
||||
if (match) {
|
||||
const value = parseInt(match[1], 10);
|
||||
if (value >= 3 && value <= 60) { // Reasonable whisky age range
|
||||
result.age = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Vintage patterns: "1990", "Vintage 1990", "Distilled 1990"
|
||||
const vintagePatterns = [
|
||||
/(?:vintage|distilled|dist\.?)\s*(19\d{2}|20[0-2]\d)/i,
|
||||
/\b(19[789]\d|20[0-2]\d)\b/, // Years 1970-2029
|
||||
];
|
||||
|
||||
for (const pattern of vintagePatterns) {
|
||||
const match = text.match(pattern);
|
||||
if (match) {
|
||||
const year = parseInt(match[1], 10);
|
||||
const currentYear = new Date().getFullYear();
|
||||
if (year >= 1970 && year <= currentYear) {
|
||||
result.vintage = match[1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an image blob to base64 string
|
||||
*/
|
||||
export function imageToBase64(blob: Blob): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
if (typeof reader.result === 'string') {
|
||||
resolve(reader.result);
|
||||
} else {
|
||||
reject(new Error('Failed to convert image to base64'));
|
||||
}
|
||||
};
|
||||
reader.onerror = reject;
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the browser is online
|
||||
*/
|
||||
export function isOnline(): boolean {
|
||||
return typeof navigator !== 'undefined' && navigator.onLine;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for image preprocessing
|
||||
*/
|
||||
export interface PreprocessOptions {
|
||||
/** Crop left/right edges (0-0.25) to remove bottle curves. Default: 0.05 */
|
||||
edgeCrop?: number;
|
||||
/** Target height for resizing. Default: 1200 */
|
||||
targetHeight?: number;
|
||||
/** Apply binarization (hard black/white). Default: false */
|
||||
binarize?: boolean;
|
||||
/** Contrast boost factor (1.0 = no change). Default: 1.3 */
|
||||
contrastBoost?: number;
|
||||
/** Apply sharpening. Default: true */
|
||||
sharpen?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Preprocess an image for better OCR results
|
||||
*
|
||||
* Applies:
|
||||
* 1. Center crop (removes curved bottle edges)
|
||||
* 2. Resize to optimal OCR size
|
||||
* 3. Grayscale conversion
|
||||
* 4. Sharpening (helps with blurry text)
|
||||
* 5. Contrast enhancement
|
||||
* 6. Optional binarization
|
||||
*
|
||||
* @param imageSource - File, Blob, or HTMLImageElement
|
||||
* @param options - Preprocessing options
|
||||
* @returns Promise<string> - Preprocessed image as data URL
|
||||
*/
|
||||
export async function preprocessImageForOCR(
|
||||
imageSource: File | Blob | HTMLImageElement,
|
||||
options: PreprocessOptions = {}
|
||||
): Promise<string> {
|
||||
const {
|
||||
edgeCrop = 0.05, // Remove 5% from each edge (minimal)
|
||||
targetHeight = 1200, // High resolution
|
||||
binarize = false, // Don't binarize by default
|
||||
contrastBoost = 1.3, // 30% contrast boost
|
||||
sharpen = false, // Disabled - creates noise on photos
|
||||
} = options;
|
||||
|
||||
// Load image into an HTMLImageElement if not already
|
||||
let img: HTMLImageElement;
|
||||
|
||||
if (imageSource instanceof HTMLImageElement) {
|
||||
img = imageSource;
|
||||
} else {
|
||||
img = await loadImageFromBlob(imageSource as Blob);
|
||||
}
|
||||
|
||||
// Create canvas
|
||||
const canvas = document.createElement('canvas');
|
||||
const ctx = canvas.getContext('2d')!;
|
||||
|
||||
// Calculate crop dimensions (remove edges to focus on center)
|
||||
const cropX = Math.floor(img.width * edgeCrop);
|
||||
const cropWidth = img.width - (cropX * 2);
|
||||
const cropHeight = img.height;
|
||||
|
||||
// Calculate resize dimensions (maintain aspect ratio)
|
||||
const scale = targetHeight / cropHeight;
|
||||
const newWidth = Math.floor(cropWidth * scale);
|
||||
const newHeight = targetHeight;
|
||||
|
||||
canvas.width = newWidth;
|
||||
canvas.height = newHeight;
|
||||
|
||||
// Draw cropped & resized image
|
||||
ctx.drawImage(
|
||||
img,
|
||||
cropX, 0, cropWidth, cropHeight, // Source: center crop
|
||||
0, 0, newWidth, newHeight // Destination: full canvas
|
||||
);
|
||||
|
||||
// Get pixel data for processing
|
||||
const imageData = ctx.getImageData(0, 0, newWidth, newHeight);
|
||||
const data = imageData.data;
|
||||
|
||||
// First pass: Convert to grayscale
|
||||
for (let i = 0; i < data.length; i += 4) {
|
||||
const r = data[i];
|
||||
const g = data[i + 1];
|
||||
const b = data[i + 2];
|
||||
const gray = 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
||||
data[i] = data[i + 1] = data[i + 2] = gray;
|
||||
}
|
||||
|
||||
// Apply sharpening using a 3x3 kernel
|
||||
if (sharpen) {
|
||||
const tempData = new Uint8ClampedArray(data);
|
||||
// Sharpen kernel: enhances edges
|
||||
// [ 0, -1, 0]
|
||||
// [-1, 5, -1]
|
||||
// [ 0, -1, 0]
|
||||
const kernel = [0, -1, 0, -1, 5, -1, 0, -1, 0];
|
||||
|
||||
for (let y = 1; y < newHeight - 1; y++) {
|
||||
for (let x = 1; x < newWidth - 1; x++) {
|
||||
let sum = 0;
|
||||
for (let ky = -1; ky <= 1; ky++) {
|
||||
for (let kx = -1; kx <= 1; kx++) {
|
||||
const idx = ((y + ky) * newWidth + (x + kx)) * 4;
|
||||
const ki = (ky + 1) * 3 + (kx + 1);
|
||||
sum += tempData[idx] * kernel[ki];
|
||||
}
|
||||
}
|
||||
const idx = (y * newWidth + x) * 4;
|
||||
const clamped = Math.min(255, Math.max(0, sum));
|
||||
data[idx] = data[idx + 1] = data[idx + 2] = clamped;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass: Apply contrast enhancement
|
||||
for (let i = 0; i < data.length; i += 4) {
|
||||
let gray = data[i];
|
||||
gray = ((gray - 128) * contrastBoost) + 128;
|
||||
gray = Math.min(255, Math.max(0, gray));
|
||||
|
||||
if (binarize) {
|
||||
gray = gray >= 128 ? 255 : 0;
|
||||
}
|
||||
|
||||
data[i] = data[i + 1] = data[i + 2] = gray;
|
||||
}
|
||||
|
||||
// Put processed data back
|
||||
ctx.putImageData(imageData, 0, 0);
|
||||
|
||||
console.log('[PreprocessOCR] Image preprocessed:', {
|
||||
original: `${img.width}x${img.height}`,
|
||||
cropped: `${cropWidth}x${cropHeight} (${(edgeCrop * 100).toFixed(0)}% edge crop)`,
|
||||
final: `${newWidth}x${newHeight}`,
|
||||
sharpen,
|
||||
contrastBoost,
|
||||
mode: binarize ? 'binarized' : 'grayscale',
|
||||
});
|
||||
|
||||
return canvas.toDataURL('image/png');
|
||||
}
|
||||
|
||||
/**
|
||||
* Load an image from a Blob/File into an HTMLImageElement
|
||||
*/
|
||||
function loadImageFromBlob(blob: Blob): Promise<HTMLImageElement> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
const url = URL.createObjectURL(blob);
|
||||
|
||||
img.onload = () => {
|
||||
URL.revokeObjectURL(url);
|
||||
resolve(img);
|
||||
};
|
||||
img.onerror = () => {
|
||||
URL.revokeObjectURL(url);
|
||||
reject(new Error('Failed to load image'));
|
||||
};
|
||||
img.src = url;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user