log artifact detection

libs/gi-art-scanner/src/lib/processImg.ts

@@ -43,7 +43,7 @@ import {
   parseSubstats,
 } from './parse'
 
-import { computeCropBoxes } from './processImgML'
+import { processEntryML } from './processImgML'
 
 export type Processed = {
   fileName: string
@@ -74,7 +74,7 @@ export async function processEntry(
   const artifactCardCanvas = imageDataToCanvas(artifactCardImageData)
 
   console.log('processImg.ts: ', { imageURL, imageData })
-  await computeCropBoxes(imageData, debugImgs)
+  await processEntryML(imageData, textsFromImage, debugImgs)
 
   const titleHistogram = findTitle(artifactCardImageData)
   const [titleTop, titleBot] = titleHistogram
@@ -348,7 +348,7 @@ function verticallyCropArtifactCard(
   return cropped
 }
 
-function parseRarity(
+export function parseRarity(
   headerData: ImageData,
   debugImgs?: Record<string, string>
 ) {

libs/gi-art-scanner/src/lib/processImgML.ts

@@ -1,20 +1,25 @@
 import * as ort from 'onnxruntime-web'
+import { lockColor } from './consts'
 import {
-  bandPass,
   crop,
-  darkerColor,
-  drawHistogram,
-  drawline,
-  fileToURL,
-  findHistogramRange,
-  histogramAnalysis,
-  histogramContAnalysis,
   imageDataToCanvas,
-  lighterColor,
-  urlToImageData,
   resize,
   drawBox,
+  invert,
+  histogramAnalysis,
+  darkerColor,
+  lighterColor,
 } from '@genshin-optimizer/img-util'
+import { PSM } from 'tesseract.js'
+import { parseRarity } from './processImg'
+import {
+  parseMainStatKeys,
+  parseMainStatValues,
+  parseSetKeys,
+  parseSlotKeys,
+  parseSubstats,
+} from './parse'
+import { findBestArtifact } from './findBestArtifact'
 
 type Box = {
   x: number
@@ -58,7 +63,8 @@ function padBox(box: Box, pad: number): Box {
     h: box.h * (1 + pad),
   }
 }
-function box2CropOption(box: Box) {
+function box2CropOption(box: Box, pad?: number) {
+  if (pad) box = padBox(box, pad)
   return {
     x1: box.x,
     y1: box.y,
@@ -131,8 +137,12 @@ async function doInference(
   }
 }
 
-export async function computeCropBoxes(
+export async function processEntryML(
   imageDataRaw: ImageData,
+  textsFromImage: (
+    imageData: ImageData,
+    options?: object | undefined
+  ) => Promise<string[]>,
   debugImgs?: Record<string, string>
 ) {
   // const session = await ort.InferenceSession.create('https://github.com/tooflesswulf/genshin-scanner/raw/main/onnx/simplenet.onnx')
@@ -144,7 +154,7 @@ export async function computeCropBoxes(
   const mlBoxes = await doInference(
     imageDataRaw,
     session,
-    box2CropOption(padBox(mlBoxes0.bbox, 0.2)),
+    box2CropOption(mlBoxes0.bbox, 0.2),
     debugImgs
   )
 
@@ -164,5 +174,50 @@ export async function computeCropBoxes(
     debugImgs['MLBoxes'] = canvas.toDataURL()
   }
 
+  const rawCanvas = imageDataToCanvas(imageDataRaw)
+  const titleCrop = crop(rawCanvas, box2CropOption(mlBoxes.title, 0.1))
+  const titleText = textsFromImage(titleCrop)
+
+  const slotCrop = crop(rawCanvas, box2CropOption(mlBoxes.slot, 0.1))
+  const slotText = textsFromImage(slotCrop)
+
+  const levelCrop = crop(rawCanvas, box2CropOption(mlBoxes.level, 0.1))
+  const levelText = textsFromImage(invert(levelCrop))
+
+  const mainstatCrop = crop(rawCanvas, box2CropOption(mlBoxes.mainstat, 0.1))
+  const mainstatText = textsFromImage(mainstatCrop, {
+    tessedit_pageseg_mode: PSM.SPARSE_TEXT,
+  })
+
+  const substatCrop = crop(rawCanvas, box2CropOption(mlBoxes.substats, 0.1))
+  const substatText = textsFromImage(substatCrop)
+
+  const setCrop = crop(rawCanvas, box2CropOption(mlBoxes.set, 0.1))
+  const setText = textsFromImage(setCrop)
+
+  const lockCrop = crop(rawCanvas, box2CropOption(mlBoxes.lock, 0.1))
+  const lockHisto = histogramAnalysis(
+    lockCrop,
+    darkerColor(lockColor),
+    lighterColor(lockColor)
+  )
+  const locked = lockHisto.filter((v) => v > 5).length > 5
+
+  const rarityCrop = crop(rawCanvas, box2CropOption(mlBoxes.rarity, 0.1))
+  const rarity = parseRarity(rarityCrop, debugImgs)
+
+  const [artifact, texts] = findBestArtifact(
+    new Set([rarity]),
+    parseSetKeys(await setText),
+    parseSlotKeys(await slotText),
+    parseSubstats(await substatText),
+    parseMainStatKeys(await mainstatText),
+    parseMainStatValues(await mainstatText),
+    '',
+    locked
+  )
+
+  console.log('DETECTION: ', { artifact, texts })
+
   return [0, 0, 1, 1]
 }

libs/img-util/src/imageData.ts

@@ -92,6 +92,21 @@ export function resize(
   const resized = new ImageData(dataBuffer, width, height)
   return resized
 }
+export function invert(imageData: ImageData) {
+  const width = imageData.width,
+    height = imageData.height
+
+  const invDataBuffer = new Uint8ClampedArray(width * height * 4)
+  for (let i = 0; i < width * height * 4; i++) {
+    if (i % 4 == 3) {
+      invDataBuffer[i] = imageData.data[i]
+      continue
+    }
+    invDataBuffer[i] = 255 - imageData.data[i]
+  }
+
+  return new ImageData(invDataBuffer, width, height)
+}
 
 export const fileToURL = (file: File): Promise<string> =>
   new Promise((resolve) => {