feat: improved local OCR with Strip & Match distillery detection

- Added comprehensive distillery database (200+ entries)
- Implemented Strip & Match heuristic for fuzzy matching
- Added contextual age detection from distillery lines
- Added whitespace normalization for OCR text
- Disabled local name extraction (too noisy, let Gemini handle it)
- Fixed confidence scale normalization in TastingEditor (0-1 vs 0-100)
- Improved extractName filter (60% letters required)
- Relaxed Fuse.js thresholds for partial matches
This commit is contained in:
2025-12-25 13:14:08 +01:00
parent a1a91795d1
commit afe9197776
17 changed files with 3642 additions and 262 deletions

View File

@@ -24,6 +24,7 @@
"dexie": "^4.2.1", "dexie": "^4.2.1",
"dexie-react-hooks": "^4.2.0", "dexie-react-hooks": "^4.2.0",
"framer-motion": "^12.23.26", "framer-motion": "^12.23.26",
"fuse.js": "^7.1.0",
"heic2any": "^0.0.4", "heic2any": "^0.0.4",
"lucide-react": "^0.468.0", "lucide-react": "^0.468.0",
"next": "16.1.0", "next": "16.1.0",
@@ -32,6 +33,7 @@
"react-dom": "^19.2.0", "react-dom": "^19.2.0",
"recharts": "^3.6.0", "recharts": "^3.6.0",
"sharp": "^0.34.5", "sharp": "^0.34.5",
"tesseract.js": "^7.0.0",
"uuid": "^13.0.0", "uuid": "^13.0.0",
"zod": "^3.23.8", "zod": "^3.23.8",
"zod-to-json-schema": "^3.25.0" "zod-to-json-schema": "^3.25.0"

101
pnpm-lock.yaml generated
View File

@@ -44,6 +44,9 @@ importers:
framer-motion: framer-motion:
specifier: ^12.23.26 specifier: ^12.23.26
version: 12.23.26(react-dom@19.2.3(react@19.2.3))(react@19.2.3) version: 12.23.26(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
fuse.js:
specifier: ^7.1.0
version: 7.1.0
heic2any: heic2any:
specifier: ^0.0.4 specifier: ^0.0.4
version: 0.0.4 version: 0.0.4
@@ -68,6 +71,9 @@ importers:
sharp: sharp:
specifier: ^0.34.5 specifier: ^0.34.5
version: 0.34.5 version: 0.34.5
tesseract.js:
specifier: ^7.0.0
version: 7.0.0
uuid: uuid:
specifier: ^13.0.0 specifier: ^13.0.0
version: 13.0.0 version: 13.0.0
@@ -1333,6 +1339,9 @@ packages:
resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
engines: {node: '>=8'} engines: {node: '>=8'}
bmp-js@0.1.0:
resolution: {integrity: sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==}
brace-expansion@1.1.12: brace-expansion@1.1.12:
resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==}
@@ -1858,6 +1867,10 @@ packages:
functions-have-names@1.2.3: functions-have-names@1.2.3:
resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==}
fuse.js@7.1.0:
resolution: {integrity: sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ==}
engines: {node: '>=10'}
generator-function@2.0.1: generator-function@2.0.1:
resolution: {integrity: sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==} resolution: {integrity: sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==}
engines: {node: '>= 0.4'} engines: {node: '>= 0.4'}
@@ -1968,6 +1981,9 @@ packages:
resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==}
engines: {node: '>=0.10.0'} engines: {node: '>=0.10.0'}
idb-keyval@6.2.2:
resolution: {integrity: sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==}
ignore@5.3.2: ignore@5.3.2:
resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==}
engines: {node: '>= 4'} engines: {node: '>= 4'}
@@ -2111,6 +2127,9 @@ packages:
resolution: {integrity: sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==} resolution: {integrity: sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==}
engines: {node: '>= 0.4'} engines: {node: '>= 0.4'}
is-url@1.2.4:
resolution: {integrity: sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==}
is-weakmap@2.0.2: is-weakmap@2.0.2:
resolution: {integrity: sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w==} resolution: {integrity: sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w==}
engines: {node: '>= 0.4'} engines: {node: '>= 0.4'}
@@ -2309,6 +2328,15 @@ packages:
sass: sass:
optional: true optional: true
node-fetch@2.7.0:
resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==}
engines: {node: 4.x || >=6.0.0}
peerDependencies:
encoding: ^0.1.0
peerDependenciesMeta:
encoding:
optional: true
node-releases@2.0.27: node-releases@2.0.27:
resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==} resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==}
@@ -2370,6 +2398,10 @@ packages:
zod: zod:
optional: true optional: true
opencollective-postinstall@2.0.3:
resolution: {integrity: sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==}
hasBin: true
optionator@0.9.4: optionator@0.9.4:
resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==}
engines: {node: '>= 0.8.0'} engines: {node: '>= 0.8.0'}
@@ -2575,6 +2607,9 @@ packages:
resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==}
engines: {node: '>= 0.4'} engines: {node: '>= 0.4'}
regenerator-runtime@0.13.11:
resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==}
regexp.prototype.flags@1.5.4: regexp.prototype.flags@1.5.4:
resolution: {integrity: sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==} resolution: {integrity: sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==}
engines: {node: '>= 0.4'} engines: {node: '>= 0.4'}
@@ -2783,6 +2818,12 @@ packages:
engines: {node: '>=14.0.0'} engines: {node: '>=14.0.0'}
hasBin: true hasBin: true
tesseract.js-core@7.0.0:
resolution: {integrity: sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==}
tesseract.js@7.0.0:
resolution: {integrity: sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==}
text-table@0.2.0: text-table@0.2.0:
resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==}
@@ -2826,6 +2867,9 @@ packages:
resolution: {integrity: sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==} resolution: {integrity: sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==}
engines: {node: '>=16'} engines: {node: '>=16'}
tr46@0.0.3:
resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}
tr46@6.0.0: tr46@6.0.0:
resolution: {integrity: sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==} resolution: {integrity: sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==}
engines: {node: '>=20'} engines: {node: '>=20'}
@@ -2996,6 +3040,12 @@ packages:
resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==}
engines: {node: '>=18'} engines: {node: '>=18'}
wasm-feature-detect@1.8.0:
resolution: {integrity: sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==}
webidl-conversions@3.0.1:
resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
webidl-conversions@8.0.0: webidl-conversions@8.0.0:
resolution: {integrity: sha512-n4W4YFyz5JzOfQeA8oN7dUYpR+MBP3PIUsn2jLjWXwK5ASUzt0Jc/A5sAUZoCYFJRGF0FBKJ+1JjN43rNdsQzA==} resolution: {integrity: sha512-n4W4YFyz5JzOfQeA8oN7dUYpR+MBP3PIUsn2jLjWXwK5ASUzt0Jc/A5sAUZoCYFJRGF0FBKJ+1JjN43rNdsQzA==}
engines: {node: '>=20'} engines: {node: '>=20'}
@@ -3012,6 +3062,9 @@ packages:
resolution: {integrity: sha512-2ytDk0kiEj/yu90JOAp44PVPUkO9+jVhyf+SybKlRHSDlvOOZhdPIrr7xTH64l4WixO2cP+wQIcgujkGBPPz6g==} resolution: {integrity: sha512-2ytDk0kiEj/yu90JOAp44PVPUkO9+jVhyf+SybKlRHSDlvOOZhdPIrr7xTH64l4WixO2cP+wQIcgujkGBPPz6g==}
engines: {node: '>=20'} engines: {node: '>=20'}
whatwg-url@5.0.0:
resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==}
which-boxed-primitive@1.1.1: which-boxed-primitive@1.1.1:
resolution: {integrity: sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==} resolution: {integrity: sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==}
engines: {node: '>= 0.4'} engines: {node: '>= 0.4'}
@@ -3071,6 +3124,9 @@ packages:
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
engines: {node: '>=10'} engines: {node: '>=10'}
zlibjs@0.3.1:
resolution: {integrity: sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==}
zod-to-json-schema@3.25.0: zod-to-json-schema@3.25.0:
resolution: {integrity: sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==} resolution: {integrity: sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==}
peerDependencies: peerDependencies:
@@ -4190,6 +4246,8 @@ snapshots:
binary-extensions@2.3.0: {} binary-extensions@2.3.0: {}
bmp-js@0.1.0: {}
brace-expansion@1.1.12: brace-expansion@1.1.12:
dependencies: dependencies:
balanced-match: 1.0.2 balanced-match: 1.0.2
@@ -4869,6 +4927,8 @@ snapshots:
functions-have-names@1.2.3: {} functions-have-names@1.2.3: {}
fuse.js@7.1.0: {}
generator-function@2.0.1: {} generator-function@2.0.1: {}
gensync@1.0.0-beta.2: {} gensync@1.0.0-beta.2: {}
@@ -4987,6 +5047,8 @@ snapshots:
dependencies: dependencies:
safer-buffer: 2.1.2 safer-buffer: 2.1.2
idb-keyval@6.2.2: {}
ignore@5.3.2: {} ignore@5.3.2: {}
ignore@7.0.5: {} ignore@7.0.5: {}
@@ -5128,6 +5190,8 @@ snapshots:
dependencies: dependencies:
which-typed-array: 1.1.19 which-typed-array: 1.1.19
is-url@1.2.4: {}
is-weakmap@2.0.2: {} is-weakmap@2.0.2: {}
is-weakref@1.1.1: is-weakref@1.1.1:
@@ -5324,6 +5388,10 @@ snapshots:
- '@babel/core' - '@babel/core'
- babel-plugin-macros - babel-plugin-macros
node-fetch@2.7.0:
dependencies:
whatwg-url: 5.0.0
node-releases@2.0.27: {} node-releases@2.0.27: {}
normalize-path@3.0.0: {} normalize-path@3.0.0: {}
@@ -5383,6 +5451,8 @@ snapshots:
ws: 8.18.3 ws: 8.18.3
zod: 3.25.76 zod: 3.25.76
opencollective-postinstall@2.0.3: {}
optionator@0.9.4: optionator@0.9.4:
dependencies: dependencies:
deep-is: 0.1.4 deep-is: 0.1.4
@@ -5577,6 +5647,8 @@ snapshots:
get-proto: 1.0.1 get-proto: 1.0.1
which-builtin-type: 1.2.1 which-builtin-type: 1.2.1
regenerator-runtime@0.13.11: {}
regexp.prototype.flags@1.5.4: regexp.prototype.flags@1.5.4:
dependencies: dependencies:
call-bind: 1.0.8 call-bind: 1.0.8
@@ -5892,6 +5964,22 @@ snapshots:
- tsx - tsx
- yaml - yaml
tesseract.js-core@7.0.0: {}
tesseract.js@7.0.0:
dependencies:
bmp-js: 0.1.0
idb-keyval: 6.2.2
is-url: 1.2.4
node-fetch: 2.7.0
opencollective-postinstall: 2.0.3
regenerator-runtime: 0.13.11
tesseract.js-core: 7.0.0
wasm-feature-detect: 1.8.0
zlibjs: 0.3.1
transitivePeerDependencies:
- encoding
text-table@0.2.0: {} text-table@0.2.0: {}
thenify-all@1.6.0: thenify-all@1.6.0:
@@ -5929,6 +6017,8 @@ snapshots:
dependencies: dependencies:
tldts: 7.0.19 tldts: 7.0.19
tr46@0.0.3: {}
tr46@6.0.0: tr46@6.0.0:
dependencies: dependencies:
punycode: 2.3.1 punycode: 2.3.1
@@ -6126,6 +6216,10 @@ snapshots:
dependencies: dependencies:
xml-name-validator: 5.0.0 xml-name-validator: 5.0.0
wasm-feature-detect@1.8.0: {}
webidl-conversions@3.0.1: {}
webidl-conversions@8.0.0: {} webidl-conversions@8.0.0: {}
whatwg-encoding@3.1.1: whatwg-encoding@3.1.1:
@@ -6139,6 +6233,11 @@ snapshots:
tr46: 6.0.0 tr46: 6.0.0
webidl-conversions: 8.0.0 webidl-conversions: 8.0.0
whatwg-url@5.0.0:
dependencies:
tr46: 0.0.3
webidl-conversions: 3.0.1
which-boxed-primitive@1.1.1: which-boxed-primitive@1.1.1:
dependencies: dependencies:
is-bigint: 1.1.0 is-bigint: 1.1.0
@@ -6203,6 +6302,8 @@ snapshots:
yocto-queue@0.1.0: {} yocto-queue@0.1.0: {}
zlibjs@0.3.1: {}
zod-to-json-schema@3.25.0(zod@3.25.76): zod-to-json-schema@3.25.0(zod@3.25.76):
dependencies: dependencies:
zod: 3.25.76 zod: 3.25.76

Binary file not shown.

Binary file not shown.

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,177 @@
'use server';
import { GoogleGenerativeAI, SchemaType, HarmCategory, HarmBlockThreshold } from '@google/generative-ai';
import { BottleMetadataSchema, BottleMetadata } from '@/types/whisky';
import { createClient } from '@/lib/supabase/server';
import { trackApiUsage } from '@/services/track-api-usage';
import { checkCreditBalance, deductCredits } from '@/services/credit-service';
// Schema for Gemini Vision extraction
const visionSchema = {
description: "Whisky bottle label metadata extracted from image",
type: SchemaType.OBJECT as const,
properties: {
name: { type: SchemaType.STRING, description: "Full whisky name", nullable: false },
distillery: { type: SchemaType.STRING, description: "Distillery name", nullable: true },
bottler: { type: SchemaType.STRING, description: "Independent bottler if applicable", nullable: true },
category: { type: SchemaType.STRING, description: "Whisky category (Single Malt, Blended, Bourbon, etc.)", nullable: true },
abv: { type: SchemaType.NUMBER, description: "Alcohol by volume percentage", nullable: true },
age: { type: SchemaType.NUMBER, description: "Age statement in years", nullable: true },
vintage: { type: SchemaType.STRING, description: "Vintage/distillation year", nullable: true },
cask_type: { type: SchemaType.STRING, description: "Cask type (Sherry, Bourbon, Port, etc.)", nullable: true },
distilled_at: { type: SchemaType.STRING, description: "Distillation date", nullable: true },
bottled_at: { type: SchemaType.STRING, description: "Bottling date", nullable: true },
batch_info: { type: SchemaType.STRING, description: "Batch or cask number", nullable: true },
is_whisky: { type: SchemaType.BOOLEAN, description: "Whether this is a whisky product", nullable: false },
confidence: { type: SchemaType.NUMBER, description: "Confidence score 0-1", nullable: false },
},
required: ["name", "is_whisky", "confidence"],
};
export interface GeminiVisionResult {
success: boolean;
data?: BottleMetadata;
error?: string;
perf?: {
apiCall: number;
total: number;
};
}
/**
* Analyze a whisky bottle label image using Gemini Vision
*
* @param imageBase64 - Base64 encoded image (with data URL prefix)
* @returns GeminiVisionResult with extracted metadata
*/
export async function analyzeLabelWithGemini(imageBase64: string): Promise<GeminiVisionResult> {
const startTotal = performance.now();
if (!process.env.GEMINI_API_KEY) {
return { success: false, error: 'GEMINI_API_KEY is not configured.' };
}
if (!imageBase64 || imageBase64.length < 100) {
return { success: false, error: 'Invalid image data provided.' };
}
try {
// Auth check
const supabase = await createClient();
const { data: { user } } = await supabase.auth.getUser();
if (!user) {
return { success: false, error: 'Not authorized.' };
}
// Credit check
const creditCheck = await checkCreditBalance(user.id, 'gemini_ai');
if (!creditCheck.allowed) {
return {
success: false,
error: `Insufficient credits. Required: ${creditCheck.cost}, Available: ${creditCheck.balance}.`
};
}
// Extract base64 data (remove data URL prefix if present)
let base64Data = imageBase64;
let mimeType = 'image/webp';
if (imageBase64.startsWith('data:')) {
const matches = imageBase64.match(/^data:([^;]+);base64,(.+)$/);
if (matches) {
mimeType = matches[1];
base64Data = matches[2];
}
}
// Initialize Gemini
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY);
const model = genAI.getGenerativeModel({
model: 'gemini-2.5-flash',
generationConfig: {
responseMimeType: "application/json",
responseSchema: visionSchema as any,
temperature: 0.1,
},
safetySettings: [
{ category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_NONE },
{ category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_NONE },
{ category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.BLOCK_NONE },
{ category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: HarmBlockThreshold.BLOCK_NONE },
] as any,
});
// Vision prompt
const prompt = `Analyze this whisky bottle label image and extract all visible metadata.
Look carefully for:
- Brand/Distillery name
- Bottle name or expression
- Age statement (e.g., "12 Years Old")
- ABV/Alcohol percentage
- Vintage year (if shown)
- Cask type (e.g., Sherry, Bourbon cask)
- Bottler name (if independent bottling)
- Category (Single Malt, Blended Malt, Bourbon, etc.)
Be precise and only include information you can clearly read from the label.
If you cannot read something clearly, leave it null.`;
// API call with timing
const startApi = performance.now();
const result = await model.generateContent([
{ inlineData: { data: base64Data, mimeType } },
{ text: prompt },
]);
const endApi = performance.now();
// Parse response
const jsonData = JSON.parse(result.response.text());
// Validate with Zod schema
const validatedData = BottleMetadataSchema.parse(jsonData);
// Track usage and deduct credits
await trackApiUsage({
userId: user.id,
apiType: 'gemini_ai',
endpoint: 'analyzeLabelWithGemini',
success: true
});
await deductCredits(user.id, 'gemini_ai', 'Vision label analysis');
return {
success: true,
data: validatedData,
perf: {
apiCall: endApi - startApi,
total: performance.now() - startTotal,
}
};
} catch (error: any) {
console.error('[GeminiVision] Analysis failed:', error);
// Try to track the failure
try {
const supabase = await createClient();
const { data: { user } } = await supabase.auth.getUser();
if (user) {
await trackApiUsage({
userId: user.id,
apiType: 'gemini_ai',
endpoint: 'analyzeLabelWithGemini',
success: false,
errorMessage: error.message
});
}
} catch (trackError) {
console.warn('[GeminiVision] Failed to track error:', trackError);
}
return {
success: false,
error: error.message || 'Vision analysis failed.'
};
}
}

View File

@@ -1,22 +1,19 @@
'use client'; 'use client';
import React, { useState, useEffect } from 'react'; import React, { useState, useEffect, useCallback } from 'react';
import { motion, AnimatePresence } from 'framer-motion'; import { motion, AnimatePresence } from 'framer-motion';
import { X, Loader2, Sparkles, AlertCircle, Clock } from 'lucide-react'; import { X, Loader2, Sparkles, AlertCircle, Clock, Eye, Cloud, Cpu } from 'lucide-react';
import TastingEditor from './TastingEditor'; import TastingEditor from './TastingEditor';
import SessionBottomSheet from './SessionBottomSheet'; import SessionBottomSheet from './SessionBottomSheet';
import ResultCard from './ResultCard'; import ResultCard from './ResultCard';
import { useSession } from '@/context/SessionContext'; import { useSession } from '@/context/SessionContext';
import { scanLabel } from '@/app/actions/scan-label';
import { enrichData } from '@/app/actions/enrich-data'; import { enrichData } from '@/app/actions/enrich-data';
import { saveBottle } from '@/services/save-bottle'; import { saveBottle } from '@/services/save-bottle';
import { saveTasting } from '@/services/save-tasting'; import { saveTasting } from '@/services/save-tasting';
import { BottleMetadata } from '@/types/whisky'; import { BottleMetadata } from '@/types/whisky';
import { useI18n } from '@/i18n/I18nContext'; import { useI18n } from '@/i18n/I18nContext';
import { createClient } from '@/lib/supabase/client'; import { createClient } from '@/lib/supabase/client';
import { processImageForAI, ProcessedImage } from '@/utils/image-processing'; import { useScanner, ScanStatus } from '@/hooks/useScanner';
import { generateDummyMetadata } from '@/utils/generate-dummy-metadata';
import { db } from '@/lib/db'; import { db } from '@/lib/db';
type FlowState = 'IDLE' | 'SCANNING' | 'EDITOR' | 'RESULT' | 'ERROR'; type FlowState = 'IDLE' | 'SCANNING' | 'EDITOR' | 'RESULT' | 'ERROR';
@@ -32,7 +29,6 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
const [state, setState] = useState<FlowState>('IDLE'); const [state, setState] = useState<FlowState>('IDLE');
const [isSessionsOpen, setIsSessionsOpen] = useState(false); const [isSessionsOpen, setIsSessionsOpen] = useState(false);
const { activeSession } = useSession(); const { activeSession } = useSession();
const [processedImage, setProcessedImage] = useState<ProcessedImage | null>(null);
const [tastingData, setTastingData] = useState<any>(null); const [tastingData, setTastingData] = useState<any>(null);
const [bottleMetadata, setBottleMetadata] = useState<BottleMetadata | null>(null); const [bottleMetadata, setBottleMetadata] = useState<BottleMetadata | null>(null);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
@@ -40,24 +36,38 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
const { locale } = useI18n(); const { locale } = useI18n();
const supabase = createClient(); const supabase = createClient();
const [isAdmin, setIsAdmin] = useState(false); const [isAdmin, setIsAdmin] = useState(false);
const [isOffline, setIsOffline] = useState(!navigator.onLine); const [isOffline, setIsOffline] = useState(typeof navigator !== 'undefined' ? !navigator.onLine : false);
const [perfMetrics, setPerfMetrics] = useState<{ const [isEnriching, setIsEnriching] = useState(false);
comp: number; const [aiFallbackActive, setAiFallbackActive] = useState(false);
aiTotal: number;
aiApi: number; // Use the new hybrid scanner hook
aiParse: number; const scanner = useScanner({
uploadSize: number; locale,
prep: number; onLocalComplete: (localResult) => {
// Detailed metrics console.log('[ScanFlow] Local OCR complete, updating preview:', localResult);
imagePrep?: number; // Immediately update bottleMetadata with local results for optimistic UI
cacheCheck?: number; setBottleMetadata(prev => ({
encoding?: number; ...prev,
modelInit?: number; name: localResult.name || prev?.name || null,
validation?: number; distillery: localResult.distillery || prev?.distillery || null,
dbOps?: number; abv: localResult.abv ?? prev?.abv ?? null,
total?: number; age: localResult.age ?? prev?.age ?? null,
cacheHit?: boolean; vintage: localResult.vintage || prev?.vintage || null,
} | null>(null); is_whisky: true,
confidence: 50,
} as BottleMetadata));
},
onCloudComplete: (cloudResult) => {
console.log('[ScanFlow] Cloud vision complete:', cloudResult);
// Update with cloud results (this is the "truth")
setBottleMetadata(cloudResult);
// Trigger background enrichment if we have name and distillery
if (cloudResult.name && cloudResult.distillery) {
runEnrichment(cloudResult.name, cloudResult.distillery);
}
},
});
// Admin Check // Admin Check
useEffect(() => { useEffect(() => {
@@ -81,8 +91,31 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
checkAdmin(); checkAdmin();
}, [supabase]); }, [supabase]);
const [aiFallbackActive, setAiFallbackActive] = useState(false); // Background enrichment function
const [isEnriching, setIsEnriching] = useState(false); const runEnrichment = useCallback(async (name: string, distillery: string) => {
setIsEnriching(true);
console.log('[ScanFlow] Starting background enrichment for:', name);
try {
const enrichResult = await enrichData(name, distillery, undefined, locale);
if (enrichResult.success && enrichResult.data) {
console.log('[ScanFlow] Enrichment data received:', enrichResult.data);
setBottleMetadata(prev => {
if (!prev) return prev;
return {
...prev,
suggested_tags: enrichResult.data.suggested_tags,
suggested_custom_tags: enrichResult.data.suggested_custom_tags,
};
});
} else {
console.warn('[ScanFlow] Enrichment unsuccessful:', enrichResult.error);
}
} catch (err) {
console.error('[ScanFlow] Enrichment failed:', err);
} finally {
setIsEnriching(false);
}
}, [locale]);
// Trigger scan when open and image provided // Trigger scan when open and image provided
useEffect(() => { useEffect(() => {
@@ -93,10 +126,10 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
setState('IDLE'); setState('IDLE');
setTastingData(null); setTastingData(null);
setBottleMetadata(null); setBottleMetadata(null);
setProcessedImage(null);
setError(null); setError(null);
setIsSaving(false); setIsSaving(false);
setAiFallbackActive(false); setAiFallbackActive(false);
scanner.reset();
} }
}, [isOpen, imageFile]); }, [isOpen, imageFile]);
@@ -114,146 +147,68 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
}; };
}, []); }, []);
// Sync scanner status to UI state
useEffect(() => {
const scannerStatus = scanner.status;
if (scannerStatus === 'idle') {
// Don't change state on idle
} else if (scannerStatus === 'compressing' || scannerStatus === 'analyzing_local') {
setState('SCANNING');
} else if (scannerStatus === 'analyzing_cloud') {
// Show EDITOR immediately when we have local results - don't wait for cloud
if (scanner.localResult || scanner.mergedResult) {
setState('EDITOR');
} else {
setState('SCANNING');
}
} else if (scannerStatus === 'complete' || scannerStatus === 'queued') {
// Use merged result from scanner
if (scanner.mergedResult) {
setBottleMetadata(scanner.mergedResult);
}
setState('EDITOR');
// If this was a queued offline scan, mark as fallback
if (scannerStatus === 'queued') {
setAiFallbackActive(true);
setIsOffline(true);
}
} else if (scannerStatus === 'error') {
if (scanner.mergedResult) {
// We have partial results, show editor anyway
setBottleMetadata(scanner.mergedResult);
setState('EDITOR');
setAiFallbackActive(true);
} else {
setError(scanner.error || 'Scan failed');
setState('ERROR');
}
}
}, [scanner.status, scanner.mergedResult, scanner.localResult, scanner.error]);
const handleScan = async (file: File) => { const handleScan = async (file: File) => {
setState('SCANNING'); setState('SCANNING');
setError(null); setError(null);
setPerfMetrics(null); scanner.handleScan(file);
try {
const startComp = performance.now();
const processed = await processImageForAI(file);
const endComp = performance.now();
setProcessedImage(processed);
// OFFLINE: Skip AI scan, use dummy metadata
if (isOffline) {
const dummyMetadata = generateDummyMetadata(file);
setBottleMetadata(dummyMetadata);
setState('EDITOR');
if (isAdmin) {
setPerfMetrics({
comp: endComp - startComp,
aiTotal: 0,
aiApi: 0,
aiParse: 0,
uploadSize: processed.file.size,
prep: 0,
cacheCheck: 0,
cacheHit: false
});
}
return;
}
// ONLINE: Normal AI scan
const formData = new FormData();
formData.append('file', processed.file);
const startAi = performance.now();
const result = await scanLabel(formData);
const endAi = performance.now();
const startPrep = performance.now();
if (result.success && result.data) {
setBottleMetadata(result.data);
const endPrep = performance.now();
if (isAdmin && result.perf) {
setPerfMetrics({
comp: endComp - startComp,
aiTotal: endAi - startAi,
aiApi: result.perf.apiCall || result.perf.apiDuration || 0,
aiParse: result.perf.parsing || result.perf.parseDuration || 0,
uploadSize: result.perf.uploadSize || 0,
prep: endPrep - startPrep,
imagePrep: result.perf.imagePrep,
cacheCheck: result.perf.cacheCheck,
encoding: result.perf.encoding,
modelInit: result.perf.modelInit,
validation: result.perf.validation,
dbOps: result.perf.dbOps,
total: result.perf.total,
cacheHit: result.perf.cacheHit
});
}
setState('EDITOR');
// Step 2: Background Enrichment
if (result.data.name && result.data.distillery) {
setIsEnriching(true);
console.log('[ScanFlow] Starting background enrichment for:', result.data.name);
enrichData(result.data.name, result.data.distillery, undefined, locale)
.then(enrichResult => {
if (enrichResult.success && enrichResult.data) {
console.log('[ScanFlow] Enrichment data received:', enrichResult.data);
setBottleMetadata(prev => {
if (!prev) return prev;
const updated = {
...prev,
suggested_tags: enrichResult.data.suggested_tags,
suggested_custom_tags: enrichResult.data.suggested_custom_tags,
search_string: enrichResult.data.search_string
};
console.log('[ScanFlow] State updated with enriched metadata');
return updated;
});
} else {
console.warn('[ScanFlow] Enrichment result unsuccessful:', enrichResult.error);
}
})
.catch(err => console.error('[ScanFlow] Enrichment failed:', err))
.finally(() => setIsEnriching(false));
}
} else if (result.isAiError) {
console.warn('[ScanFlow] AI Analysis failed, falling back to offline mode');
setIsOffline(true);
setAiFallbackActive(true);
const dummyMetadata = generateDummyMetadata(file);
setBottleMetadata(dummyMetadata);
setState('EDITOR');
return;
} else {
throw new Error(result.error || 'Fehler bei der Analyse.');
}
} catch (err: any) {
console.error('[ScanFlow] handleScan error:', err);
// Check if this is a network error (offline)
if (err.message?.includes('Failed to fetch') || err.message?.includes('NetworkError') || err.message?.includes('ERR_INTERNET_DISCONNECTED')) {
console.log('[ScanFlow] Network error detected - switching to offline mode');
setIsOffline(true);
// Use dummy metadata for offline scan
const dummyMetadata = generateDummyMetadata(file);
setBottleMetadata(dummyMetadata);
setState('EDITOR');
return;
}
// Other errors
setError(err.message);
setState('ERROR');
}
}; };
const handleSaveTasting = async (formData: any) => { const handleSaveTasting = async (formData: any) => {
if (!bottleMetadata || !processedImage) return; if (!bottleMetadata || !scanner.processedImage) return;
setIsSaving(true); setIsSaving(true);
setError(null); setError(null);
try { try {
// OFFLINE: Save to IndexedDB queue (skip auth check) // OFFLINE: Save to IndexedDB queue
if (isOffline) { if (isOffline) {
console.log('[ScanFlow] Offline mode - queuing for upload'); console.log('[ScanFlow] Offline mode - queuing for upload');
const tempId = `temp_${Date.now()}`; const tempId = `temp_${Date.now()}`;
const bottleDataToSave = formData.bottleMetadata || bottleMetadata; const bottleDataToSave = formData.bottleMetadata || bottleMetadata;
// Check for existing pending scan with same image to prevent duplicates // Check for existing pending scan with same image
const existingScan = await db.pending_scans const existingScan = await db.pending_scans
.filter(s => s.imageBase64 === processedImage.base64) .filter(s => s.imageBase64 === scanner.processedImage!.base64)
.first(); .first();
let currentTempId = tempId; let currentTempId = tempId;
@@ -262,13 +217,11 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
console.log('[ScanFlow] Existing pending scan found, reusing temp_id:', existingScan.temp_id); console.log('[ScanFlow] Existing pending scan found, reusing temp_id:', existingScan.temp_id);
currentTempId = existingScan.temp_id; currentTempId = existingScan.temp_id;
} else { } else {
// Save pending scan with metadata
await db.pending_scans.add({ await db.pending_scans.add({
temp_id: tempId, temp_id: tempId,
imageBase64: processedImage.base64, imageBase64: scanner.processedImage.base64,
timestamp: Date.now(), timestamp: Date.now(),
locale, locale,
// Store bottle metadata in a custom field
metadata: bottleDataToSave as any metadata: bottleDataToSave as any
}); });
} }
@@ -302,69 +255,25 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
if (!authUser) throw new Error('Nicht autorisiert'); if (!authUser) throw new Error('Nicht autorisiert');
user = authUser; user = authUser;
} catch (authError: any) { } catch (authError: any) {
// If auth fails due to network, treat as offline
if (authError.message?.includes('Failed to fetch') || authError.message?.includes('NetworkError')) { if (authError.message?.includes('Failed to fetch') || authError.message?.includes('NetworkError')) {
console.log('[ScanFlow] Auth failed due to network - switching to offline mode'); console.log('[ScanFlow] Auth failed due to network - switching to offline mode');
setIsOffline(true); setIsOffline(true);
// Retry save in offline mode
// Save to queue instead return handleSaveTasting(formData);
const tempId = `temp_${Date.now()}`;
const bottleDataToSave = formData.bottleMetadata || bottleMetadata;
// Check for existing pending scan with same image to prevent duplicates
const existingScan = await db.pending_scans
.filter(s => s.imageBase64 === processedImage.base64)
.first();
let currentTempId = tempId;
if (existingScan) {
console.log('[ScanFlow] Existing pending scan found, reusing temp_id:', existingScan.temp_id);
currentTempId = existingScan.temp_id;
} else {
await db.pending_scans.add({
temp_id: tempId,
imageBase64: processedImage.base64,
timestamp: Date.now(),
locale,
metadata: bottleDataToSave as any
});
}
await db.pending_tastings.add({
pending_bottle_id: currentTempId,
data: {
session_id: activeSession?.id,
rating: formData.rating,
nose_notes: formData.nose_notes,
palate_notes: formData.palate_notes,
finish_notes: formData.finish_notes,
is_sample: formData.is_sample,
buddy_ids: formData.buddy_ids,
tag_ids: formData.tag_ids,
},
tasted_at: new Date().toISOString()
});
setTastingData(formData);
setState('RESULT');
setIsSaving(false);
return;
} }
// Other auth errors
throw authError; throw authError;
} }
// 1. Save Bottle - Use edited metadata if provided // Save Bottle
const bottleDataToSave = formData.bottleMetadata || bottleMetadata; const bottleDataToSave = formData.bottleMetadata || bottleMetadata;
const bottleResult = await saveBottle(bottleDataToSave, processedImage.base64, user.id); const bottleResult = await saveBottle(bottleDataToSave, scanner.processedImage.base64, user.id);
if (!bottleResult.success || !bottleResult.data) { if (!bottleResult.success || !bottleResult.data) {
throw new Error(bottleResult.error || 'Fehler beim Speichern der Flasche'); throw new Error(bottleResult.error || 'Fehler beim Speichern der Flasche');
} }
const bottleId = bottleResult.data.id; const bottleId = bottleResult.data.id;
// 2. Save Tasting // Save Tasting
const tastingNote = { const tastingNote = {
...formData, ...formData,
bottle_id: bottleId, bottle_id: bottleId,
@@ -378,7 +287,6 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
setTastingData(tastingNote); setTastingData(tastingNote);
setState('RESULT'); setState('RESULT');
// Trigger bottle list refresh in parent
if (onBottleSaved) { if (onBottleSaved) {
onBottleSaved(bottleId); onBottleSaved(bottleId);
} }
@@ -406,6 +314,22 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
} }
}; };
// Map scanner status to display text
const getScanStatusDisplay = (status: ScanStatus): { text: string; icon: React.ReactNode } => {
switch (status) {
case 'compressing':
return { text: 'Bild optimieren...', icon: <Loader2 size={12} className="animate-spin" /> };
case 'analyzing_local':
return { text: 'Lokale OCR-Analyse...', icon: <Cpu size={12} /> };
case 'analyzing_cloud':
return { text: 'KI-Vision-Analyse...', icon: <Cloud size={12} /> };
default:
return { text: 'Analysiere Etikett...', icon: <Loader2 size={12} className="animate-spin" /> };
}
};
const statusDisplay = getScanStatusDisplay(scanner.status);
return ( return (
<AnimatePresence> <AnimatePresence>
{isOpen && ( {isOpen && (
@@ -424,11 +348,6 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
</button> </button>
<div className="flex-1 w-full h-full flex flex-col relative min-h-0"> <div className="flex-1 w-full h-full flex flex-col relative min-h-0">
{/*
Robust state check:
If we are IDLE but have an image, we are essentially SCANNING (or about to be).
If we have no image, we shouldn't really be here, but show error just in case.
*/}
{(state === 'SCANNING' || (state === 'IDLE' && imageFile)) && ( {(state === 'SCANNING' || (state === 'IDLE' && imageFile)) && (
<div className="flex-1 flex flex-col items-center justify-center"> <div className="flex-1 flex flex-col items-center justify-center">
<motion.div <motion.div
@@ -449,43 +368,39 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
<div className="text-center space-y-2"> <div className="text-center space-y-2">
<h2 className="text-2xl font-bold text-zinc-50 uppercase tracking-tight">Analysiere Etikett...</h2> <h2 className="text-2xl font-bold text-zinc-50 uppercase tracking-tight">Analysiere Etikett...</h2>
<p className="text-orange-500 font-bold uppercase tracking-widest text-[10px] flex items-center justify-center gap-2"> <p className="text-orange-500 font-bold uppercase tracking-widest text-[10px] flex items-center justify-center gap-2">
<Sparkles size={12} /> KI-gestütztes Scanning {statusDisplay.icon}
{statusDisplay.text}
</p> </p>
{/* Show scan stage indicators */}
<div className="flex items-center justify-center gap-2 mt-4">
<div className={`w-2 h-2 rounded-full transition-colors ${['compressing', 'analyzing_local', 'analyzing_cloud', 'complete'].includes(scanner.status)
? 'bg-orange-500' : 'bg-zinc-700'
}`} />
<div className={`w-2 h-2 rounded-full transition-colors ${['analyzing_local', 'analyzing_cloud', 'complete'].includes(scanner.status)
? 'bg-orange-500' : 'bg-zinc-700'
}`} />
<div className={`w-2 h-2 rounded-full transition-colors ${['analyzing_cloud', 'complete'].includes(scanner.status)
? 'bg-orange-500' : 'bg-zinc-700'
}`} />
</div>
</div> </div>
</motion.div> </motion.div>
{isAdmin && perfMetrics && ( {/* Admin perf metrics */}
{isAdmin && scanner.perf && (
<div className="mt-8 p-6 bg-zinc-950/80 backdrop-blur-xl rounded-3xl border border-orange-500/20 text-[10px] font-mono text-zinc-400 animate-in fade-in slide-in-from-bottom-4 shadow-2xl"> <div className="mt-8 p-6 bg-zinc-950/80 backdrop-blur-xl rounded-3xl border border-orange-500/20 text-[10px] font-mono text-zinc-400 animate-in fade-in slide-in-from-bottom-4 shadow-2xl">
<div className="grid grid-cols-3 gap-6 text-center"> <div className="grid grid-cols-3 gap-6 text-center">
<div> <div>
<p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">Client</p> <p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">Compress</p>
<p className="text-orange-500 font-bold">{perfMetrics.comp.toFixed(0)}ms</p> <p className="text-orange-500 font-bold">{scanner.perf.compression.toFixed(0)}ms</p>
<p className="text-[8px] opacity-40 mt-1">{(perfMetrics.uploadSize / 1024).toFixed(0)} KB</p>
</div> </div>
<div> <div>
<p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">AI Engine</p> <p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">Local OCR</p>
{perfMetrics.cacheHit ? ( <p className="text-orange-500 font-bold">{scanner.perf.localOcr.toFixed(0)}ms</p>
<div className="flex flex-col items-center">
<p className="text-green-500 font-bold tracking-tighter">CACHE HIT</p>
<p className="text-[7px] opacity-40 mt-1">DB RESULTS</p>
</div>
) : (
<>
<p className="text-orange-500 font-bold">{perfMetrics.aiTotal.toFixed(0)}ms</p>
<div className="flex flex-col gap-0.5 mt-1 text-[7px] opacity-60">
{perfMetrics.imagePrep !== undefined && <span>Prep: {perfMetrics.imagePrep.toFixed(0)}ms</span>}
{perfMetrics.encoding !== undefined && <span>Encode: {perfMetrics.encoding.toFixed(0)}ms</span>}
{perfMetrics.modelInit !== undefined && <span>Init: {perfMetrics.modelInit.toFixed(0)}ms</span>}
<span className="text-orange-400">API: {perfMetrics.aiApi.toFixed(0)}ms</span>
{perfMetrics.validation !== undefined && <span>Valid: {perfMetrics.validation.toFixed(0)}ms</span>}
{perfMetrics.dbOps !== undefined && <span>DB: {perfMetrics.dbOps.toFixed(0)}ms</span>}
</div>
</>
)}
</div> </div>
<div> <div>
<p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">App Logic</p> <p className="text-zinc-500 mb-2 uppercase tracking-widest text-[8px]">Cloud Vision</p>
<p className="text-orange-500 font-bold">{perfMetrics.prep.toFixed(0)}ms</p> <p className="text-orange-500 font-bold">{scanner.perf.cloudVision.toFixed(0)}ms</p>
</div> </div>
</div> </div>
</div> </div>
@@ -525,6 +440,7 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
exit={{ y: -50, opacity: 0 }} exit={{ y: -50, opacity: 0 }}
className="flex-1 w-full h-full flex flex-col min-h-0" className="flex-1 w-full h-full flex flex-col min-h-0"
> >
{/* Status banners */}
{isOffline && ( {isOffline && (
<div className="bg-orange-500/10 border-b border-orange-500/20 p-4"> <div className="bg-orange-500/10 border-b border-orange-500/20 p-4">
<div className="max-w-2xl mx-auto flex items-center gap-3"> <div className="max-w-2xl mx-auto flex items-center gap-3">
@@ -537,9 +453,26 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
</div> </div>
</div> </div>
)} )}
{/* Local preview indicator */}
{scanner.status === 'analyzing_cloud' && scanner.localResult && (
<div className="bg-blue-500/10 border-b border-blue-500/20 p-3">
<div className="max-w-2xl mx-auto flex items-center gap-3">
<Eye size={14} className="text-blue-500" />
<p className="text-xs font-bold text-blue-500 uppercase tracking-wider flex items-center gap-2">
Lokale Vorschau
<span className="flex items-center gap-1 text-zinc-400 font-normal normal-case">
<Loader2 size={10} className="animate-spin" />
KI-Vision verfeinert Ergebnisse...
</span>
</p>
</div>
</div>
)}
<TastingEditor <TastingEditor
bottleMetadata={bottleMetadata} bottleMetadata={bottleMetadata}
image={processedImage?.base64 || null} image={scanner.processedImage?.base64 || null}
onSave={handleSaveTasting} onSave={handleSaveTasting}
onOpenSessions={() => setIsSessionsOpen(true)} onOpenSessions={() => setIsSessionsOpen(true)}
activeSessionName={activeSession?.name} activeSessionName={activeSession?.name}
@@ -547,38 +480,27 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
isEnriching={isEnriching} isEnriching={isEnriching}
defaultExpanded={true} defaultExpanded={true}
/> />
{isAdmin && perfMetrics && (
<div className="absolute top-24 left-6 right-6 z-50 p-3 bg-zinc-950/80 backdrop-blur-md rounded-2xl border border-orange-500/20 text-[9px] font-mono text-white/90 shadow-xl overflow-x-auto"> {/* Admin perf overlay - positioned at bottom */}
{isAdmin && scanner.perf && (
<div className="fixed bottom-24 left-6 right-6 z-50 p-3 bg-zinc-950/80 backdrop-blur-md rounded-2xl border border-orange-500/20 text-[9px] font-mono text-white/90 shadow-xl overflow-x-auto">
<div className="flex items-center justify-between gap-4 whitespace-nowrap"> <div className="flex items-center justify-between gap-4 whitespace-nowrap">
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<Clock size={10} className="text-orange-500" /> <Clock size={10} className="text-orange-500" />
<span className="text-zinc-500">CLIENT:</span> <span className="text-zinc-500">COMPRESS:</span>
<span className="text-orange-500 font-bold">{perfMetrics.comp.toFixed(0)}ms</span> <span className="text-orange-500 font-bold">{scanner.perf.compression.toFixed(0)}ms</span>
<span className="text-zinc-600">({(perfMetrics.uploadSize / 1024).toFixed(0)}KB)</span>
</div> </div>
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<span className="text-zinc-500">AI:</span> <span className="text-zinc-500">LOCAL:</span>
{perfMetrics.cacheHit ? ( <span className="text-blue-500 font-bold">{scanner.perf.localOcr.toFixed(0)}ms</span>
<span className="text-green-500 font-bold tracking-tight">CACHE HIT</span>
) : (
<>
<span className="text-orange-500 font-bold">{perfMetrics.aiTotal.toFixed(0)}ms</span>
<span className="text-zinc-600 ml-1 text-[10px]">
(
{perfMetrics.imagePrep !== undefined && `Prep:${perfMetrics.imagePrep.toFixed(0)} `}
{perfMetrics.encoding !== undefined && `Enc:${perfMetrics.encoding.toFixed(0)} `}
{perfMetrics.modelInit !== undefined && `Init:${perfMetrics.modelInit.toFixed(0)} `}
<span className="text-orange-400 font-bold">API:{perfMetrics.aiApi.toFixed(0)}</span>
{perfMetrics.validation !== undefined && ` Val:${perfMetrics.validation.toFixed(0)}`}
{perfMetrics.dbOps !== undefined && ` DB:${perfMetrics.dbOps.toFixed(0)}`}
)
</span>
</>
)}
</div> </div>
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<span className="text-zinc-500">APP:</span> <span className="text-zinc-500">CLOUD:</span>
<span className="text-orange-500 font-bold">{perfMetrics.prep.toFixed(0)}ms</span> <span className="text-green-500 font-bold">{scanner.perf.cloudVision.toFixed(0)}ms</span>
</div>
<div className="flex items-center gap-2">
<span className="text-zinc-500">TOTAL:</span>
<span className="text-orange-500 font-bold">{scanner.perf.total.toFixed(0)}ms</span>
</div> </div>
</div> </div>
</div> </div>
@@ -613,7 +535,7 @@ export default function ScanAndTasteFlow({ isOpen, onClose, imageFile, onBottleS
balance: tastingData.balance || 85, balance: tastingData.balance || 85,
}} }}
bottleName={bottleMetadata.name || 'Unknown Whisky'} bottleName={bottleMetadata.name || 'Unknown Whisky'}
image={processedImage?.base64 || null} image={scanner.processedImage?.base64 || null}
onShare={handleShare} onShare={handleShare}
/> />
</motion.div> </motion.div>

View File

@@ -76,6 +76,57 @@ export default function TastingEditor({ bottleMetadata, image, onSave, onOpenSes
const suggestedTags = bottleMetadata.suggested_tags || []; const suggestedTags = bottleMetadata.suggested_tags || [];
const suggestedCustomTags = bottleMetadata.suggested_custom_tags || []; const suggestedCustomTags = bottleMetadata.suggested_custom_tags || [];
// Track last seen confidence to detect cloud vs local updates
const lastConfidenceRef = React.useRef<number>(0);
// Sync bottleMetadata prop changes to internal state (for live Gemini updates)
// Cloud data (confidence >= 0.6 OR >= 60) overrides local OCR (confidence ~50 or ~0.5)
useEffect(() => {
// Normalize confidence to 0-100 scale (Gemini returns 0-1, local returns 0-100)
const rawConfidence = bottleMetadata.confidence ?? 0;
const newConfidence = rawConfidence <= 1 ? rawConfidence * 100 : rawConfidence;
const isCloudUpdate = newConfidence >= 60;
const isUpgrade = newConfidence > lastConfidenceRef.current;
console.log('[TastingEditor] Syncing bottleMetadata update:', {
rawConfidence,
normalizedConfidence: newConfidence,
isCloudUpdate,
isUpgrade,
previousConfidence: lastConfidenceRef.current,
});
// For cloud updates (higher confidence), allow overwriting local OCR values
// For local updates, only fill empty fields
if (isCloudUpdate || isUpgrade) {
// Cloud vision: update all fields that have new data
if (bottleMetadata.name) setBottleName(bottleMetadata.name);
if (bottleMetadata.distillery) setBottleDistillery(bottleMetadata.distillery);
if (bottleMetadata.abv) setBottleAbv(bottleMetadata.abv.toString());
if (bottleMetadata.age) setBottleAge(bottleMetadata.age.toString());
if (bottleMetadata.category) setBottleCategory(bottleMetadata.category);
if (bottleMetadata.vintage) setBottleVintage(bottleMetadata.vintage);
if (bottleMetadata.bottler) setBottleBottler(bottleMetadata.bottler);
if (bottleMetadata.batch_info) setBottleBatchInfo(bottleMetadata.batch_info);
if (bottleMetadata.distilled_at) setBottleDistilledAt(bottleMetadata.distilled_at);
if (bottleMetadata.bottled_at) setBottleBottledAt(bottleMetadata.bottled_at);
} else {
// Local OCR or initial: only update empty fields
if (!bottleName && bottleMetadata.name) setBottleName(bottleMetadata.name);
if (!bottleDistillery && bottleMetadata.distillery) setBottleDistillery(bottleMetadata.distillery);
if (!bottleAbv && bottleMetadata.abv) setBottleAbv(bottleMetadata.abv.toString());
if (!bottleAge && bottleMetadata.age) setBottleAge(bottleMetadata.age.toString());
if ((!bottleCategory || bottleCategory === 'Whisky') && bottleMetadata.category) setBottleCategory(bottleMetadata.category);
if (!bottleVintage && bottleMetadata.vintage) setBottleVintage(bottleMetadata.vintage);
if (!bottleBottler && bottleMetadata.bottler) setBottleBottler(bottleMetadata.bottler);
if (!bottleBatchInfo && bottleMetadata.batch_info) setBottleBatchInfo(bottleMetadata.batch_info);
if (!bottleDistilledAt && bottleMetadata.distilled_at) setBottleDistilledAt(bottleMetadata.distilled_at);
if (!bottleBottledAt && bottleMetadata.bottled_at) setBottleBottledAt(bottleMetadata.bottled_at);
}
lastConfidenceRef.current = newConfidence;
}, [bottleMetadata]);
// Session-based pre-fill and Palette Checker // Session-based pre-fill and Palette Checker
useEffect(() => { useEffect(() => {
const fetchSessionData = async () => { const fetchSessionData = async () => {

1014
src/data/distilleries.json Normal file

File diff suppressed because it is too large Load Diff

342
src/hooks/useScanner.ts Normal file
View File

@@ -0,0 +1,342 @@
'use client';
import { useState, useCallback, useRef } from 'react';
import { BottleMetadata } from '@/types/whisky';
import { processImageForAI, ProcessedImage } from '@/utils/image-processing';
import { analyzeLocalOcr, LocalOcrResult, terminateOcrWorker } from '@/lib/ocr/local-engine';
import { isTesseractReady, isOnline } from '@/lib/ocr/scanner-utils';
import { analyzeLabelWithGemini } from '@/app/actions/gemini-vision';
import { generateDummyMetadata } from '@/utils/generate-dummy-metadata';
import { db } from '@/lib/db';
export type ScanStatus =
| 'idle'
| 'compressing'
| 'analyzing_local'
| 'analyzing_cloud'
| 'complete'
| 'queued'
| 'error';
export interface ScanResult {
status: ScanStatus;
localResult: Partial<BottleMetadata> | null;
cloudResult: BottleMetadata | null;
mergedResult: BottleMetadata | null;
processedImage: ProcessedImage | null;
error: string | null;
perf: {
compression: number;
localOcr: number;
cloudVision: number;
total: number;
} | null;
}
export interface UseScannerOptions {
locale?: string;
onLocalComplete?: (result: Partial<BottleMetadata>) => void;
onCloudComplete?: (result: BottleMetadata) => void;
}
/**
* React hook for the hybrid Local OCR + Cloud Vision scanning flow
*
* Flow:
* 1. Compress image (browser-side)
* 2. Run local OCR (tesseract.js) → immediate preview
* 3. Run cloud vision (Gemini) → refined results
* 4. Merge results (cloud overrides local, except user-edited fields)
*/
export function useScanner(options: UseScannerOptions = {}) {
const { locale = 'en', onLocalComplete, onCloudComplete } = options;
const [result, setResult] = useState<ScanResult>({
status: 'idle',
localResult: null,
cloudResult: null,
mergedResult: null,
processedImage: null,
error: null,
perf: null,
});
// Track which fields the user has manually edited
const dirtyFieldsRef = useRef<Set<string>>(new Set());
/**
* Mark a field as user-edited (won't be overwritten by cloud results)
*/
const markFieldDirty = useCallback((field: string) => {
dirtyFieldsRef.current.add(field);
}, []);
/**
* Clear all dirty field markers
*/
const clearDirtyFields = useCallback(() => {
dirtyFieldsRef.current.clear();
}, []);
/**
* Merge local and cloud results, respecting user edits
*/
const mergeResults = useCallback((
local: Partial<BottleMetadata> | null,
cloud: BottleMetadata | null,
dirtyFields: Set<string>
): BottleMetadata | null => {
if (!cloud && !local) return null;
if (!cloud) {
return {
name: local?.name || null,
distillery: local?.distillery || null,
abv: local?.abv || null,
age: local?.age || null,
vintage: local?.vintage || null,
is_whisky: true,
confidence: 50,
} as BottleMetadata;
}
if (!local) return cloud;
// Start with cloud result as base
const merged = { ...cloud };
// For each field, keep local value if user edited it
for (const field of dirtyFields) {
if (field in local && (local as any)[field] !== undefined) {
(merged as any)[field] = (local as any)[field];
}
}
return merged;
}, []);
/**
* Main scan handler
*/
const handleScan = useCallback(async (file: File) => {
const perfStart = performance.now();
let perfCompression = 0;
let perfLocalOcr = 0;
let perfCloudVision = 0;
// Reset state
clearDirtyFields();
setResult({
status: 'compressing',
localResult: null,
cloudResult: null,
mergedResult: null,
processedImage: null,
error: null,
perf: null,
});
try {
// Step 1: Compress image
const compressStart = performance.now();
const processedImage = await processImageForAI(file);
perfCompression = performance.now() - compressStart;
setResult(prev => ({
...prev,
status: 'analyzing_local',
processedImage,
}));
// Step 2: Check if we're offline or tesseract isn't ready
const online = isOnline();
const tesseractReady = await isTesseractReady();
// If offline and tesseract not ready, queue immediately
if (!online && !tesseractReady) {
console.log('[useScanner] Offline + no tesseract cache → queuing');
const dummyMetadata = generateDummyMetadata(file);
await db.pending_scans.add({
temp_id: `temp_${Date.now()}`,
imageBase64: processedImage.base64,
timestamp: Date.now(),
locale,
metadata: dummyMetadata as any,
});
setResult(prev => ({
...prev,
status: 'queued',
mergedResult: dummyMetadata,
perf: {
compression: perfCompression,
localOcr: 0,
cloudVision: 0,
total: performance.now() - perfStart,
},
}));
return;
}
// Step 3: Run local OCR (testing new line-by-line matching)
let localResult: Partial<BottleMetadata> | null = null;
try {
const localStart = performance.now();
console.log('[useScanner] Running local OCR...');
const ocrResult = await analyzeLocalOcr(processedImage.file, 10000);
perfLocalOcr = performance.now() - localStart;
if (ocrResult.rawText && ocrResult.rawText.length > 10) {
localResult = {
name: ocrResult.name || undefined,
distillery: ocrResult.distillery || undefined,
abv: ocrResult.abv || undefined,
age: ocrResult.age || undefined,
vintage: ocrResult.vintage || undefined,
};
// Update state with local results
const localMerged = mergeResults(localResult, null, dirtyFieldsRef.current);
setResult(prev => ({
...prev,
localResult,
mergedResult: localMerged,
}));
onLocalComplete?.(localResult);
console.log('[useScanner] Local OCR complete:', localResult);
}
} catch (ocrError) {
console.warn('[useScanner] Local OCR failed:', ocrError);
}
// Step 4: If offline, use local results only
if (!online) {
console.log('[useScanner] Offline → using local results only');
const offlineMerged = mergeResults(localResult, null, dirtyFieldsRef.current);
setResult(prev => ({
...prev,
status: 'complete',
mergedResult: offlineMerged || generateDummyMetadata(file),
perf: {
compression: perfCompression,
localOcr: perfLocalOcr,
cloudVision: 0,
total: performance.now() - perfStart,
},
}));
return;
}
// Step 5: Run cloud vision analysis
setResult(prev => ({
...prev,
status: 'analyzing_cloud',
}));
const cloudStart = performance.now();
const cloudResponse = await analyzeLabelWithGemini(processedImage.base64);
perfCloudVision = performance.now() - cloudStart;
if (cloudResponse.success && cloudResponse.data) {
const cloudResult = cloudResponse.data;
const finalMerged = mergeResults(localResult, cloudResult, dirtyFieldsRef.current);
onCloudComplete?.(cloudResult);
console.log('[useScanner] Cloud vision complete:', cloudResult);
setResult(prev => ({
...prev,
status: 'complete',
cloudResult,
mergedResult: finalMerged,
perf: {
compression: perfCompression,
localOcr: perfLocalOcr,
cloudVision: perfCloudVision,
total: performance.now() - perfStart,
},
}));
} else {
// Cloud failed, fall back to local results
console.warn('[useScanner] Cloud vision failed:', cloudResponse.error);
const fallbackMerged = mergeResults(localResult, null, dirtyFieldsRef.current);
setResult(prev => ({
...prev,
status: 'complete',
error: cloudResponse.error || null,
mergedResult: fallbackMerged || generateDummyMetadata(file),
perf: {
compression: perfCompression,
localOcr: perfLocalOcr,
cloudVision: perfCloudVision,
total: performance.now() - perfStart,
},
}));
}
} catch (error: any) {
console.error('[useScanner] Scan failed:', error);
setResult(prev => ({
...prev,
status: 'error',
error: error.message || 'Scan failed',
perf: {
compression: perfCompression,
localOcr: perfLocalOcr,
cloudVision: perfCloudVision,
total: performance.now() - perfStart,
},
}));
}
}, [locale, mergeResults, clearDirtyFields, onLocalComplete, onCloudComplete]);
/**
* Reset scanner state
*/
const reset = useCallback(() => {
clearDirtyFields();
setResult({
status: 'idle',
localResult: null,
cloudResult: null,
mergedResult: null,
processedImage: null,
error: null,
perf: null,
});
}, [clearDirtyFields]);
/**
* Update the merged result (for user edits)
*/
const updateMergedResult = useCallback((updates: Partial<BottleMetadata>) => {
// Mark updated fields as dirty
Object.keys(updates).forEach(key => {
dirtyFieldsRef.current.add(key);
});
setResult(prev => ({
...prev,
mergedResult: prev.mergedResult ? { ...prev.mergedResult, ...updates } : null,
}));
}, []);
/**
* Cleanup (terminate OCR worker)
*/
const cleanup = useCallback(() => {
terminateOcrWorker();
}, []);
return {
...result,
handleScan,
reset,
markFieldDirty,
updateMergedResult,
cleanup,
};
}

313
src/lib/ocr/local-engine.ts Normal file
View File

@@ -0,0 +1,313 @@
/**
* Local OCR Engine
* Client-side OCR using Tesseract.js with Fuse.js fuzzy matching
*
* Optimized for whisky label scanning with:
* - Image preprocessing (grayscale, binarization, center crop)
* - PSM 11 (Sparse text mode)
* - Character whitelisting
* - Bag-of-words fuzzy matching
*/
import Tesseract from 'tesseract.js';
import Fuse from 'fuse.js';
import { extractNumbers, ExtractedNumbers, preprocessImageForOCR } from './scanner-utils';
import distilleries from '@/data/distilleries.json';
export interface LocalOcrResult {
distillery: string | null;
distilleryRegion: string | null;
name: string | null;
age: number | null;
abv: number | null;
vintage: string | null;
rawText: string;
confidence: number;
}
// Fuse.js configuration for fuzzy matching distillery names
// Balanced matching to catch partial OCR errors while avoiding false positives
const fuseOptions = {
keys: ['name'],
threshold: 0.35, // 0 = exact match, 0.35 = allow some fuzziness
distance: 50, // Characters between matched chars
includeScore: true,
minMatchCharLength: 4, // Minimum chars to match
};
const distilleryFuse = new Fuse(distilleries, fuseOptions);
// Tesseract worker singleton (reused across scans)
let tesseractWorker: Tesseract.Worker | null = null;
// Character whitelist for whisky labels (no special symbols that cause noise)
const CHAR_WHITELIST = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789%.,:\'"-/ ';
/**
* Initialize or get the Tesseract worker
* Uses local files from /public/tessdata for offline capability
*/
async function getWorker(): Promise<Tesseract.Worker> {
if (tesseractWorker) {
return tesseractWorker;
}
console.log('[LocalOCR] Initializing Tesseract worker with local files...');
// Use local files from /public/tessdata
tesseractWorker = await Tesseract.createWorker('eng', Tesseract.OEM.LSTM_ONLY, {
corePath: '/tessdata/',
langPath: '/tessdata/',
logger: (m) => {
if (m.status === 'recognizing text') {
console.log(`[LocalOCR] Progress: ${Math.round(m.progress * 100)}%`);
} else {
console.log(`[LocalOCR] ${m.status}`);
}
},
});
// Configure Tesseract for whisky label OCR
await tesseractWorker.setParameters({
tessedit_pageseg_mode: Tesseract.PSM.SINGLE_BLOCK, // PSM 6 - treat as single block of text
tessedit_char_whitelist: CHAR_WHITELIST,
preserve_interword_spaces: '1', // Keep word spacing
});
console.log('[LocalOCR] Tesseract worker ready (PSM: SINGLE_BLOCK, Whitelist enabled)');
return tesseractWorker;
}
/**
* Run OCR on an image and extract whisky metadata
*
* @param imageSource - File, Blob, or base64 string of the image
* @param timeoutMs - Maximum time to wait for OCR (default 10000ms)
* @returns LocalOcrResult with extracted metadata
*/
export async function analyzeLocalOcr(
imageSource: File | Blob | string,
timeoutMs: number = 10000
): Promise<LocalOcrResult> {
const result: LocalOcrResult = {
distillery: null,
distilleryRegion: null,
name: null,
age: null,
abv: null,
vintage: null,
rawText: '',
confidence: 0,
};
try {
// Step 1: Preprocess the image for better OCR
let processedImage: string;
if (typeof imageSource === 'string') {
// Already a data URL, use as-is (can't preprocess string)
processedImage = imageSource;
console.log('[LocalOCR] Using raw image (string input)');
} else {
// Preprocess File/Blob: grayscale + sharpen + contrast boost
console.log('[LocalOCR] Preprocessing image...');
processedImage = await preprocessImageForOCR(imageSource);
// Uses defaults: 5% edge crop, 1200px height, sharpen=true, 1.3x contrast
}
// Create a timeout promise
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error('OCR timeout')), timeoutMs);
});
// Race OCR against timeout
const worker = await getWorker();
const ocrResult = await Promise.race([
worker.recognize(processedImage),
timeoutPromise,
]);
result.rawText = ocrResult.data.text;
result.confidence = ocrResult.data.confidence / 100; // Normalize to 0-1
// Extract numbers using regex (this works reliably)
const numbers = extractNumbers(result.rawText);
result.abv = numbers.abv;
result.age = numbers.age;
result.vintage = numbers.vintage;
// NOTE: Distillery fuzzy matching disabled - causes too many false positives
// with noisy OCR text. Let Gemini Vision handle distillery identification.
// const distilleryMatch = findDistillery(result.rawText);
// if (distilleryMatch) {
// result.distillery = distilleryMatch.name;
// result.distilleryRegion = distilleryMatch.region;
// }
// Fuzzy match distillery (new algorithm with sanity checks)
const distilleryMatch = findDistillery(result.rawText);
if (distilleryMatch) {
result.distillery = distilleryMatch.name;
result.distilleryRegion = distilleryMatch.region;
// Use contextual age if regex extraction failed
if (!result.age && distilleryMatch.contextualAge) {
result.age = distilleryMatch.contextualAge;
console.log(`[LocalOCR] Using contextual age: ${result.age}`);
}
}
// NOTE: Name extraction disabled - Tesseract too noisy for full bottle names
// Let Gemini Vision handle the name field
// result.name = extractName(result.rawText, result.distillery);
result.name = null;
// Detailed logging for debugging
const cleanedText = result.rawText
.split('\n')
.map(line => line.trim())
.filter(line => line.length > 0)
.join(' | ');
console.log('[LocalOCR] ========== OCR RESULTS ==========');
console.log('[LocalOCR] Raw Text:\n', result.rawText);
console.log('[LocalOCR] Cleaned Text:', cleanedText);
console.log('[LocalOCR] Confidence:', (result.confidence * 100).toFixed(1) + '%');
console.log('[LocalOCR] Extracted Data:', {
distillery: result.distillery,
distilleryRegion: result.distilleryRegion,
name: result.name,
age: result.age,
abv: result.abv,
vintage: result.vintage,
});
console.log('[LocalOCR] ===================================');
return result;
} catch (error) {
console.warn('[LocalOCR] Analysis failed:', error);
return result; // Return partial/empty result
}
}
/**
* Find a distillery name in OCR text using fuzzy matching
*
* Strategy:
* 1. Normalize whitespace (fix Tesseract's formatting gaps)
* 2. Split OCR text into lines, filter garbage
* 3. "Strip & Match": Remove numbers before Fuse matching (helps with "N NEVIS 27")
* 4. Sanity check: match length must be reasonable
* 5. Contextual age: if distillery found, look for age in original line
*/
function findDistillery(text: string): { name: string; region: string; contextualAge?: number } | null {
// Split into lines, normalize whitespace, and filter garbage
const lines = text
.split('\n')
.map(line => line.trim().replace(/\s+/g, ' ')) // Normalize whitespace
.filter(line => {
// Minimum 4 characters
if (line.length < 4) return false;
// Must have at least 40% letters (lowered to allow lines with numbers)
const letters = line.replace(/[^a-zA-Z]/g, '');
return letters.length >= line.length * 0.4;
});
console.log('[LocalOCR] Lines for distillery matching:', lines.length);
// Try to match each line
for (const originalLine of lines) {
// STRIP & MATCH: Remove numbers for cleaner Fuse matching
// "Bad N NEVIS 27" → "Bad N NEVIS "
const textOnlyLine = originalLine.replace(/[0-9]/g, '').replace(/\s+/g, ' ').trim();
if (textOnlyLine.length < 4) continue;
const results = distilleryFuse.search(textOnlyLine);
if (results.length > 0 && results[0].score !== undefined && results[0].score < 0.4) {
const match = results[0].item;
const matchScore = results[0].score;
// SANITY CHECK: The text-only part should be similar length to distillery name
// Max 60% deviation allowed (relaxed for partial matches)
const lengthRatio = textOnlyLine.length / match.name.length;
const lengthDeviation = Math.abs(1 - lengthRatio);
if (lengthDeviation > 0.6) {
console.log(`[LocalOCR] Match rejected (length): "${textOnlyLine}" → ${match.name} (ratio: ${lengthRatio.toFixed(2)}, deviation: ${(lengthDeviation * 100).toFixed(0)}%)`);
continue;
}
// CONTEXTUAL AGE DETECTION: Look for 2-digit number (3-60) in ORIGINAL line
let contextualAge: number | undefined;
const ageMatch = originalLine.match(/\b(\d{1,2})\b/);
if (ageMatch) {
const potentialAge = parseInt(ageMatch[1], 10);
if (potentialAge >= 3 && potentialAge <= 60) {
contextualAge = potentialAge;
console.log(`[LocalOCR] Contextual age detected: ${potentialAge} years`);
}
}
console.log(`[LocalOCR] Distillery match: "${textOnlyLine}" → ${match.name} (score: ${matchScore.toFixed(3)}, original: "${originalLine}")`);
return {
name: match.name,
region: match.region,
contextualAge,
};
}
}
return null;
}
/**
* Extract a potential bottle name from OCR text
*/
function extractName(text: string, distillery: string | null): string | null {
const lines = text
.split('\n')
.map(l => l.trim())
.filter(line => {
// Minimum 5 characters
if (line.length < 5) return false;
// Must have at least 60% letters (filter out garbage like "ee" or "4 . .")
const letters = line.replace(/[^a-zA-Z]/g, '');
if (letters.length < line.length * 0.6) return false;
// Skip lines that are just punctuation/numbers
if (/^[\d\s.,\-'"]+$/.test(line)) return false;
return true;
});
// Skip lines that are just the distillery name
const candidates = lines.filter(line => {
if (distillery && line.toLowerCase().includes(distillery.toLowerCase())) {
// Only skip if the line IS the distillery name (not contains more)
return line.length > distillery.length + 5;
}
return true;
});
// Return the first substantial line (likely the bottle name)
for (const line of candidates) {
// Skip lines that look like numbers only
if (/^\d+[\s%]+/.test(line)) continue;
// Skip lines that are just common whisky words
if (/^(single|malt|scotch|whisky|whiskey|aged|years?|proof|edition|distilled|distillery)$/i.test(line)) continue;
return line;
}
return null;
}
/**
* Terminate the Tesseract worker (call on cleanup)
*/
export async function terminateOcrWorker(): Promise<void> {
if (tesseractWorker) {
await tesseractWorker.terminate();
tesseractWorker = null;
}
}

View File

@@ -0,0 +1,312 @@
/**
* Scanner Utilities
* Cache checking and helper functions for client-side OCR
*/
/**
* Check if Tesseract.js is ready to run
* When online, tesseract will auto-download from CDN, so return true
* When offline, check if files are cached
* @returns Promise<boolean> - true if OCR can run
*/
export async function isTesseractReady(): Promise<boolean> {
if (typeof window === 'undefined') {
return false;
}
// If online, tesseract.js will auto-download what it needs
if (navigator.onLine) {
console.log('[Scanner] Online - tesseract will use CDN');
return true;
}
// If offline, check cache
if (!('caches' in window)) {
console.log('[Scanner] Offline + no cache API - tesseract not ready');
return false;
}
try {
// Check for the core files in cache (matching actual file names in /public/tessdata)
const wasmMatch = await window.caches.match('/tessdata/tesseract-core-simd.wasm');
const langMatch = await window.caches.match('/tessdata/eng.traineddata');
const ready = !!(wasmMatch && langMatch);
console.log('[Scanner] Offline cache check:', { wasmMatch: !!wasmMatch, langMatch: !!langMatch, ready });
return ready;
} catch (error) {
console.warn('[Scanner] Cache check failed:', error);
return false;
}
}
/**
* Extract numeric values from OCR text using regex patterns
*/
export interface ExtractedNumbers {
abv: number | null;
age: number | null;
vintage: string | null;
}
export function extractNumbers(text: string): ExtractedNumbers {
const result: ExtractedNumbers = {
abv: null,
age: null,
vintage: null
};
if (!text) return result;
// Normalize text: lowercase, clean up common OCR mistakes
const normalizedText = text
.replace(/[oO]/g, '0') // Common OCR mistake: O -> 0
.replace(/[lI]/g, '1') // Common OCR mistake: l/I -> 1
.toLowerCase();
// ABV patterns: "43%", "43.5%", "43,5 %", "ABV 43", "vol. 43"
const abvPatterns = [
/(\d{2}[.,]\d{1,2})\s*%/, // 43.5% or 43,5 %
/(\d{2})\s*%/, // 43%
/abv[:\s]*(\d{2}[.,]?\d{0,2})/i, // ABV: 43 or ABV 43.5
/vol[.\s]*(\d{2}[.,]?\d{0,2})/i, // vol. 43
/(\d{2}[.,]\d{1,2})\s*vol/i, // 43.5 vol
];
for (const pattern of abvPatterns) {
const match = normalizedText.match(pattern);
if (match) {
const value = parseFloat(match[1].replace(',', '.'));
if (value >= 35 && value <= 75) { // Reasonable whisky ABV range
result.abv = value;
break;
}
}
}
// Age patterns: "12 years", "12 year old", "12 YO", "aged 12"
const agePatterns = [
/(\d{1,2})\s*(?:years?|yrs?|y\.?o\.?|jahre?)/i,
/aged\s*(\d{1,2})/i,
/(\d{1,2})\s*year\s*old/i,
];
for (const pattern of agePatterns) {
const match = text.match(pattern);
if (match) {
const value = parseInt(match[1], 10);
if (value >= 3 && value <= 60) { // Reasonable whisky age range
result.age = value;
break;
}
}
}
// Vintage patterns: "1990", "Vintage 1990", "Distilled 1990"
const vintagePatterns = [
/(?:vintage|distilled|dist\.?)\s*(19\d{2}|20[0-2]\d)/i,
/\b(19[789]\d|20[0-2]\d)\b/, // Years 1970-2029
];
for (const pattern of vintagePatterns) {
const match = text.match(pattern);
if (match) {
const year = parseInt(match[1], 10);
const currentYear = new Date().getFullYear();
if (year >= 1970 && year <= currentYear) {
result.vintage = match[1];
break;
}
}
}
return result;
}
/**
* Convert an image blob to base64 string
*/
export function imageToBase64(blob: Blob): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
if (typeof reader.result === 'string') {
resolve(reader.result);
} else {
reject(new Error('Failed to convert image to base64'));
}
};
reader.onerror = reject;
reader.readAsDataURL(blob);
});
}
/**
* Check if the browser is online
*/
export function isOnline(): boolean {
return typeof navigator !== 'undefined' && navigator.onLine;
}
/**
* Options for image preprocessing
*/
export interface PreprocessOptions {
/** Crop left/right edges (0-0.25) to remove bottle curves. Default: 0.05 */
edgeCrop?: number;
/** Target height for resizing. Default: 1200 */
targetHeight?: number;
/** Apply binarization (hard black/white). Default: false */
binarize?: boolean;
/** Contrast boost factor (1.0 = no change). Default: 1.3 */
contrastBoost?: number;
/** Apply sharpening. Default: true */
sharpen?: boolean;
}
/**
* Preprocess an image for better OCR results
*
* Applies:
* 1. Center crop (removes curved bottle edges)
* 2. Resize to optimal OCR size
* 3. Grayscale conversion
* 4. Sharpening (helps with blurry text)
* 5. Contrast enhancement
* 6. Optional binarization
*
* @param imageSource - File, Blob, or HTMLImageElement
* @param options - Preprocessing options
* @returns Promise<string> - Preprocessed image as data URL
*/
export async function preprocessImageForOCR(
imageSource: File | Blob | HTMLImageElement,
options: PreprocessOptions = {}
): Promise<string> {
const {
edgeCrop = 0.05, // Remove 5% from each edge (minimal)
targetHeight = 1200, // High resolution
binarize = false, // Don't binarize by default
contrastBoost = 1.3, // 30% contrast boost
sharpen = false, // Disabled - creates noise on photos
} = options;
// Load image into an HTMLImageElement if not already
let img: HTMLImageElement;
if (imageSource instanceof HTMLImageElement) {
img = imageSource;
} else {
img = await loadImageFromBlob(imageSource as Blob);
}
// Create canvas
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d')!;
// Calculate crop dimensions (remove edges to focus on center)
const cropX = Math.floor(img.width * edgeCrop);
const cropWidth = img.width - (cropX * 2);
const cropHeight = img.height;
// Calculate resize dimensions (maintain aspect ratio)
const scale = targetHeight / cropHeight;
const newWidth = Math.floor(cropWidth * scale);
const newHeight = targetHeight;
canvas.width = newWidth;
canvas.height = newHeight;
// Draw cropped & resized image
ctx.drawImage(
img,
cropX, 0, cropWidth, cropHeight, // Source: center crop
0, 0, newWidth, newHeight // Destination: full canvas
);
// Get pixel data for processing
const imageData = ctx.getImageData(0, 0, newWidth, newHeight);
const data = imageData.data;
// First pass: Convert to grayscale
for (let i = 0; i < data.length; i += 4) {
const r = data[i];
const g = data[i + 1];
const b = data[i + 2];
const gray = 0.2126 * r + 0.7152 * g + 0.0722 * b;
data[i] = data[i + 1] = data[i + 2] = gray;
}
// Apply sharpening using a 3x3 kernel
if (sharpen) {
const tempData = new Uint8ClampedArray(data);
// Sharpen kernel: enhances edges
// [ 0, -1, 0]
// [-1, 5, -1]
// [ 0, -1, 0]
const kernel = [0, -1, 0, -1, 5, -1, 0, -1, 0];
for (let y = 1; y < newHeight - 1; y++) {
for (let x = 1; x < newWidth - 1; x++) {
let sum = 0;
for (let ky = -1; ky <= 1; ky++) {
for (let kx = -1; kx <= 1; kx++) {
const idx = ((y + ky) * newWidth + (x + kx)) * 4;
const ki = (ky + 1) * 3 + (kx + 1);
sum += tempData[idx] * kernel[ki];
}
}
const idx = (y * newWidth + x) * 4;
const clamped = Math.min(255, Math.max(0, sum));
data[idx] = data[idx + 1] = data[idx + 2] = clamped;
}
}
}
// Second pass: Apply contrast enhancement
for (let i = 0; i < data.length; i += 4) {
let gray = data[i];
gray = ((gray - 128) * contrastBoost) + 128;
gray = Math.min(255, Math.max(0, gray));
if (binarize) {
gray = gray >= 128 ? 255 : 0;
}
data[i] = data[i + 1] = data[i + 2] = gray;
}
// Put processed data back
ctx.putImageData(imageData, 0, 0);
console.log('[PreprocessOCR] Image preprocessed:', {
original: `${img.width}x${img.height}`,
cropped: `${cropWidth}x${cropHeight} (${(edgeCrop * 100).toFixed(0)}% edge crop)`,
final: `${newWidth}x${newHeight}`,
sharpen,
contrastBoost,
mode: binarize ? 'binarized' : 'grayscale',
});
return canvas.toDataURL('image/png');
}
/**
* Load an image from a Blob/File into an HTMLImageElement
*/
function loadImageFromBlob(blob: Blob): Promise<HTMLImageElement> {
return new Promise((resolve, reject) => {
const img = new Image();
const url = URL.createObjectURL(blob);
img.onload = () => {
URL.revokeObjectURL(url);
resolve(img);
};
img.onerror = () => {
URL.revokeObjectURL(url);
reject(new Error('Failed to load image'));
};
img.src = url;
});
}