feat: improved local OCR with Strip & Match distillery detection
- Added comprehensive distillery database (200+ entries) - Implemented Strip & Match heuristic for fuzzy matching - Added contextual age detection from distillery lines - Added whitespace normalization for OCR text - Disabled local name extraction (too noisy, let Gemini handle it) - Fixed confidence scale normalization in TastingEditor (0-1 vs 0-100) - Improved extractName filter (60% letters required) - Relaxed Fuse.js thresholds for partial matches
This commit is contained in:
342
src/hooks/useScanner.ts
Normal file
342
src/hooks/useScanner.ts
Normal file
@@ -0,0 +1,342 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useCallback, useRef } from 'react';
|
||||
import { BottleMetadata } from '@/types/whisky';
|
||||
import { processImageForAI, ProcessedImage } from '@/utils/image-processing';
|
||||
import { analyzeLocalOcr, LocalOcrResult, terminateOcrWorker } from '@/lib/ocr/local-engine';
|
||||
import { isTesseractReady, isOnline } from '@/lib/ocr/scanner-utils';
|
||||
import { analyzeLabelWithGemini } from '@/app/actions/gemini-vision';
|
||||
import { generateDummyMetadata } from '@/utils/generate-dummy-metadata';
|
||||
import { db } from '@/lib/db';
|
||||
|
||||
export type ScanStatus =
|
||||
| 'idle'
|
||||
| 'compressing'
|
||||
| 'analyzing_local'
|
||||
| 'analyzing_cloud'
|
||||
| 'complete'
|
||||
| 'queued'
|
||||
| 'error';
|
||||
|
||||
export interface ScanResult {
|
||||
status: ScanStatus;
|
||||
localResult: Partial<BottleMetadata> | null;
|
||||
cloudResult: BottleMetadata | null;
|
||||
mergedResult: BottleMetadata | null;
|
||||
processedImage: ProcessedImage | null;
|
||||
error: string | null;
|
||||
perf: {
|
||||
compression: number;
|
||||
localOcr: number;
|
||||
cloudVision: number;
|
||||
total: number;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface UseScannerOptions {
|
||||
locale?: string;
|
||||
onLocalComplete?: (result: Partial<BottleMetadata>) => void;
|
||||
onCloudComplete?: (result: BottleMetadata) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* React hook for the hybrid Local OCR + Cloud Vision scanning flow
|
||||
*
|
||||
* Flow:
|
||||
* 1. Compress image (browser-side)
|
||||
* 2. Run local OCR (tesseract.js) → immediate preview
|
||||
* 3. Run cloud vision (Gemini) → refined results
|
||||
* 4. Merge results (cloud overrides local, except user-edited fields)
|
||||
*/
|
||||
export function useScanner(options: UseScannerOptions = {}) {
|
||||
const { locale = 'en', onLocalComplete, onCloudComplete } = options;
|
||||
|
||||
const [result, setResult] = useState<ScanResult>({
|
||||
status: 'idle',
|
||||
localResult: null,
|
||||
cloudResult: null,
|
||||
mergedResult: null,
|
||||
processedImage: null,
|
||||
error: null,
|
||||
perf: null,
|
||||
});
|
||||
|
||||
// Track which fields the user has manually edited
|
||||
const dirtyFieldsRef = useRef<Set<string>>(new Set());
|
||||
|
||||
/**
|
||||
* Mark a field as user-edited (won't be overwritten by cloud results)
|
||||
*/
|
||||
const markFieldDirty = useCallback((field: string) => {
|
||||
dirtyFieldsRef.current.add(field);
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Clear all dirty field markers
|
||||
*/
|
||||
const clearDirtyFields = useCallback(() => {
|
||||
dirtyFieldsRef.current.clear();
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Merge local and cloud results, respecting user edits
|
||||
*/
|
||||
const mergeResults = useCallback((
|
||||
local: Partial<BottleMetadata> | null,
|
||||
cloud: BottleMetadata | null,
|
||||
dirtyFields: Set<string>
|
||||
): BottleMetadata | null => {
|
||||
if (!cloud && !local) return null;
|
||||
if (!cloud) {
|
||||
return {
|
||||
name: local?.name || null,
|
||||
distillery: local?.distillery || null,
|
||||
abv: local?.abv || null,
|
||||
age: local?.age || null,
|
||||
vintage: local?.vintage || null,
|
||||
is_whisky: true,
|
||||
confidence: 50,
|
||||
} as BottleMetadata;
|
||||
}
|
||||
if (!local) return cloud;
|
||||
|
||||
// Start with cloud result as base
|
||||
const merged = { ...cloud };
|
||||
|
||||
// For each field, keep local value if user edited it
|
||||
for (const field of dirtyFields) {
|
||||
if (field in local && (local as any)[field] !== undefined) {
|
||||
(merged as any)[field] = (local as any)[field];
|
||||
}
|
||||
}
|
||||
|
||||
return merged;
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Main scan handler
|
||||
*/
|
||||
const handleScan = useCallback(async (file: File) => {
|
||||
const perfStart = performance.now();
|
||||
let perfCompression = 0;
|
||||
let perfLocalOcr = 0;
|
||||
let perfCloudVision = 0;
|
||||
|
||||
// Reset state
|
||||
clearDirtyFields();
|
||||
setResult({
|
||||
status: 'compressing',
|
||||
localResult: null,
|
||||
cloudResult: null,
|
||||
mergedResult: null,
|
||||
processedImage: null,
|
||||
error: null,
|
||||
perf: null,
|
||||
});
|
||||
|
||||
try {
|
||||
// Step 1: Compress image
|
||||
const compressStart = performance.now();
|
||||
const processedImage = await processImageForAI(file);
|
||||
perfCompression = performance.now() - compressStart;
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'analyzing_local',
|
||||
processedImage,
|
||||
}));
|
||||
|
||||
// Step 2: Check if we're offline or tesseract isn't ready
|
||||
const online = isOnline();
|
||||
const tesseractReady = await isTesseractReady();
|
||||
|
||||
// If offline and tesseract not ready, queue immediately
|
||||
if (!online && !tesseractReady) {
|
||||
console.log('[useScanner] Offline + no tesseract cache → queuing');
|
||||
const dummyMetadata = generateDummyMetadata(file);
|
||||
|
||||
await db.pending_scans.add({
|
||||
temp_id: `temp_${Date.now()}`,
|
||||
imageBase64: processedImage.base64,
|
||||
timestamp: Date.now(),
|
||||
locale,
|
||||
metadata: dummyMetadata as any,
|
||||
});
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'queued',
|
||||
mergedResult: dummyMetadata,
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: 0,
|
||||
cloudVision: 0,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 3: Run local OCR (testing new line-by-line matching)
|
||||
let localResult: Partial<BottleMetadata> | null = null;
|
||||
|
||||
try {
|
||||
const localStart = performance.now();
|
||||
console.log('[useScanner] Running local OCR...');
|
||||
const ocrResult = await analyzeLocalOcr(processedImage.file, 10000);
|
||||
perfLocalOcr = performance.now() - localStart;
|
||||
|
||||
if (ocrResult.rawText && ocrResult.rawText.length > 10) {
|
||||
localResult = {
|
||||
name: ocrResult.name || undefined,
|
||||
distillery: ocrResult.distillery || undefined,
|
||||
abv: ocrResult.abv || undefined,
|
||||
age: ocrResult.age || undefined,
|
||||
vintage: ocrResult.vintage || undefined,
|
||||
};
|
||||
|
||||
// Update state with local results
|
||||
const localMerged = mergeResults(localResult, null, dirtyFieldsRef.current);
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
localResult,
|
||||
mergedResult: localMerged,
|
||||
}));
|
||||
|
||||
onLocalComplete?.(localResult);
|
||||
console.log('[useScanner] Local OCR complete:', localResult);
|
||||
}
|
||||
} catch (ocrError) {
|
||||
console.warn('[useScanner] Local OCR failed:', ocrError);
|
||||
}
|
||||
|
||||
// Step 4: If offline, use local results only
|
||||
if (!online) {
|
||||
console.log('[useScanner] Offline → using local results only');
|
||||
const offlineMerged = mergeResults(localResult, null, dirtyFieldsRef.current);
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'complete',
|
||||
mergedResult: offlineMerged || generateDummyMetadata(file),
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: perfLocalOcr,
|
||||
cloudVision: 0,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 5: Run cloud vision analysis
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'analyzing_cloud',
|
||||
}));
|
||||
|
||||
const cloudStart = performance.now();
|
||||
const cloudResponse = await analyzeLabelWithGemini(processedImage.base64);
|
||||
perfCloudVision = performance.now() - cloudStart;
|
||||
|
||||
if (cloudResponse.success && cloudResponse.data) {
|
||||
const cloudResult = cloudResponse.data;
|
||||
const finalMerged = mergeResults(localResult, cloudResult, dirtyFieldsRef.current);
|
||||
|
||||
onCloudComplete?.(cloudResult);
|
||||
console.log('[useScanner] Cloud vision complete:', cloudResult);
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'complete',
|
||||
cloudResult,
|
||||
mergedResult: finalMerged,
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: perfLocalOcr,
|
||||
cloudVision: perfCloudVision,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
} else {
|
||||
// Cloud failed, fall back to local results
|
||||
console.warn('[useScanner] Cloud vision failed:', cloudResponse.error);
|
||||
const fallbackMerged = mergeResults(localResult, null, dirtyFieldsRef.current);
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'complete',
|
||||
error: cloudResponse.error || null,
|
||||
mergedResult: fallbackMerged || generateDummyMetadata(file),
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: perfLocalOcr,
|
||||
cloudVision: perfCloudVision,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
} catch (error: any) {
|
||||
console.error('[useScanner] Scan failed:', error);
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
status: 'error',
|
||||
error: error.message || 'Scan failed',
|
||||
perf: {
|
||||
compression: perfCompression,
|
||||
localOcr: perfLocalOcr,
|
||||
cloudVision: perfCloudVision,
|
||||
total: performance.now() - perfStart,
|
||||
},
|
||||
}));
|
||||
}
|
||||
}, [locale, mergeResults, clearDirtyFields, onLocalComplete, onCloudComplete]);
|
||||
|
||||
/**
|
||||
* Reset scanner state
|
||||
*/
|
||||
const reset = useCallback(() => {
|
||||
clearDirtyFields();
|
||||
setResult({
|
||||
status: 'idle',
|
||||
localResult: null,
|
||||
cloudResult: null,
|
||||
mergedResult: null,
|
||||
processedImage: null,
|
||||
error: null,
|
||||
perf: null,
|
||||
});
|
||||
}, [clearDirtyFields]);
|
||||
|
||||
/**
|
||||
* Update the merged result (for user edits)
|
||||
*/
|
||||
const updateMergedResult = useCallback((updates: Partial<BottleMetadata>) => {
|
||||
// Mark updated fields as dirty
|
||||
Object.keys(updates).forEach(key => {
|
||||
dirtyFieldsRef.current.add(key);
|
||||
});
|
||||
|
||||
setResult(prev => ({
|
||||
...prev,
|
||||
mergedResult: prev.mergedResult ? { ...prev.mergedResult, ...updates } : null,
|
||||
}));
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Cleanup (terminate OCR worker)
|
||||
*/
|
||||
const cleanup = useCallback(() => {
|
||||
terminateOcrWorker();
|
||||
}, []);
|
||||
|
||||
return {
|
||||
...result,
|
||||
handleScan,
|
||||
reset,
|
||||
markFieldDirty,
|
||||
updateMergedResult,
|
||||
cleanup,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user