Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Scanner ml #1402

Draft
wants to merge 10 commits into
base: master
Choose a base branch
from
Draft
Prev Previous commit
Next Next commit
log artifact detection
tooflesswulf committed Dec 31, 2023
commit 72aea01ef9587f64a45d18545a0690aeb06b5f3a
6 changes: 3 additions & 3 deletions libs/gi-art-scanner/src/lib/processImg.ts
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ import {
parseSubstats,
} from './parse'

import { computeCropBoxes } from './processImgML'
import { processEntryML } from './processImgML'

export type Processed = {
fileName: string
@@ -74,7 +74,7 @@ export async function processEntry(
const artifactCardCanvas = imageDataToCanvas(artifactCardImageData)

console.log('processImg.ts: ', { imageURL, imageData })
await computeCropBoxes(imageData, debugImgs)
await processEntryML(imageData, textsFromImage, debugImgs)

const titleHistogram = findTitle(artifactCardImageData)
const [titleTop, titleBot] = titleHistogram
@@ -348,7 +348,7 @@ function verticallyCropArtifactCard(
return cropped
}

function parseRarity(
export function parseRarity(
headerData: ImageData,
debugImgs?: Record<string, string>
) {
81 changes: 68 additions & 13 deletions libs/gi-art-scanner/src/lib/processImgML.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,25 @@
import * as ort from 'onnxruntime-web'
import { lockColor } from './consts'
import {
bandPass,
crop,
darkerColor,
drawHistogram,
drawline,
fileToURL,
findHistogramRange,
histogramAnalysis,
histogramContAnalysis,
imageDataToCanvas,
lighterColor,
urlToImageData,
resize,
drawBox,
invert,
histogramAnalysis,
darkerColor,
lighterColor,
} from '@genshin-optimizer/img-util'
import { PSM } from 'tesseract.js'
import { parseRarity } from './processImg'
import {
parseMainStatKeys,
parseMainStatValues,
parseSetKeys,
parseSlotKeys,
parseSubstats,
} from './parse'
import { findBestArtifact } from './findBestArtifact'

type Box = {
x: number
@@ -58,7 +63,8 @@ function padBox(box: Box, pad: number): Box {
h: box.h * (1 + pad),
}
}
function box2CropOption(box: Box) {
function box2CropOption(box: Box, pad?: number) {
if (pad) box = padBox(box, pad)
return {
x1: box.x,
y1: box.y,
@@ -131,8 +137,12 @@ async function doInference(
}
}

export async function computeCropBoxes(
export async function processEntryML(
imageDataRaw: ImageData,
textsFromImage: (
imageData: ImageData,
options?: object | undefined
) => Promise<string[]>,
debugImgs?: Record<string, string>
) {
// const session = await ort.InferenceSession.create('https://github.com/tooflesswulf/genshin-scanner/raw/main/onnx/simplenet.onnx')
@@ -144,7 +154,7 @@ export async function computeCropBoxes(
const mlBoxes = await doInference(
imageDataRaw,
session,
box2CropOption(padBox(mlBoxes0.bbox, 0.2)),
box2CropOption(mlBoxes0.bbox, 0.2),
debugImgs
)

@@ -164,5 +174,50 @@ export async function computeCropBoxes(
debugImgs['MLBoxes'] = canvas.toDataURL()
}

const rawCanvas = imageDataToCanvas(imageDataRaw)
const titleCrop = crop(rawCanvas, box2CropOption(mlBoxes.title, 0.1))
const titleText = textsFromImage(titleCrop)

const slotCrop = crop(rawCanvas, box2CropOption(mlBoxes.slot, 0.1))
const slotText = textsFromImage(slotCrop)

const levelCrop = crop(rawCanvas, box2CropOption(mlBoxes.level, 0.1))
const levelText = textsFromImage(invert(levelCrop))

const mainstatCrop = crop(rawCanvas, box2CropOption(mlBoxes.mainstat, 0.1))
const mainstatText = textsFromImage(mainstatCrop, {
tessedit_pageseg_mode: PSM.SPARSE_TEXT,
})

const substatCrop = crop(rawCanvas, box2CropOption(mlBoxes.substats, 0.1))
const substatText = textsFromImage(substatCrop)

const setCrop = crop(rawCanvas, box2CropOption(mlBoxes.set, 0.1))
const setText = textsFromImage(setCrop)

const lockCrop = crop(rawCanvas, box2CropOption(mlBoxes.lock, 0.1))
const lockHisto = histogramAnalysis(
lockCrop,
darkerColor(lockColor),
lighterColor(lockColor)
)
const locked = lockHisto.filter((v) => v > 5).length > 5

const rarityCrop = crop(rawCanvas, box2CropOption(mlBoxes.rarity, 0.1))
const rarity = parseRarity(rarityCrop, debugImgs)

const [artifact, texts] = findBestArtifact(
new Set([rarity]),
parseSetKeys(await setText),
parseSlotKeys(await slotText),
parseSubstats(await substatText),
parseMainStatKeys(await mainstatText),
parseMainStatValues(await mainstatText),
'',
locked
)

console.log('DETECTION: ', { artifact, texts })

return [0, 0, 1, 1]
}
15 changes: 15 additions & 0 deletions libs/img-util/src/imageData.ts
Original file line number Diff line number Diff line change
@@ -92,6 +92,21 @@ export function resize(
const resized = new ImageData(dataBuffer, width, height)
return resized
}
export function invert(imageData: ImageData) {
const width = imageData.width,
height = imageData.height

const invDataBuffer = new Uint8ClampedArray(width * height * 4)
for (let i = 0; i < width * height * 4; i++) {
if (i % 4 == 3) {
invDataBuffer[i] = imageData.data[i]
continue
}
invDataBuffer[i] = 255 - imageData.data[i]
}

return new ImageData(invDataBuffer, width, height)
}

export const fileToURL = (file: File): Promise<string> =>
new Promise((resolve) => {