test

Author: pashidlos

src/compare/libs/vlm/README.md

@@ -57,7 +57,6 @@ Optional custom prompt (replaces default system prompt):
 | Model | Size | Speed | Accuracy | Best For |
 |-------|------|-------|----------|----------|
 | `llava:7b` | 4.7GB | ⚡⚡ | ⭐⭐⭐ | **Recommended** - best balance (minimal) |
-| `qwen3-vl:8b` | ~8GB | ⚡⚡ | ⭐⭐⭐ | Minimal model option |
 | `gemma3:latest` | ~ | ⚡⚡ | ⭐⭐⭐ | Minimal model option |
 | `llava:13b` | 8GB | ⚡ | ⭐⭐⭐⭐ | Best accuracy |
 | `moondream` | 1.7GB | ⚡⚡⚡ | ⭐⭐ | Fast, may hallucinate |

src/compare/libs/vlm/vlm.service.spec.ts

@@ -6,15 +6,14 @@ import { NO_BASELINE_RESULT } from '../consts';
 import { DEFAULT_CONFIG, VlmService } from './vlm.service';
 import { OllamaService } from './ollama.service';
 
-const initService = async ({ getImageMock = jest.fn(), saveImageMock = jest.fn(), ollamaGenerateMock = jest.fn() }) => {
+const initService = async ({ getImageMock = jest.fn(), ollamaGenerateMock = jest.fn() }) => {
   const module: TestingModule = await Test.createTestingModule({
     providers: [
       VlmService,
       {
         provide: StaticService,
         useValue: {
           getImage: getImageMock,
-          saveImage: saveImageMock,
         },
       },
       {
@@ -65,12 +64,11 @@ describe('VlmService', () => {
 
   it('should return unresolved when VLM returns identical=false in JSON', async () => {
     const getImageMock = jest.fn().mockReturnValue(image);
-    const saveImageMock = jest.fn().mockResolvedValue('diff.png');
     const ollamaGenerateMock = jest.fn().mockResolvedValue({
       response: '{"identical": false, "description": "Button text changed from Submit to Send."}',
       done: true,
     });
-    const service = await initService({ getImageMock, saveImageMock, ollamaGenerateMock });
+    const service = await initService({ getImageMock, ollamaGenerateMock });
 
     const result = await service.getDiff(
       { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: true },
@@ -79,9 +77,9 @@ describe('VlmService', () => {
 
     expect(result.status).toBe(TestStatus.unresolved);
     expect(result.vlmDescription).toBe('Button text changed from Submit to Send.');
-    expect(result.diffName).toBe('diff.png');
-    expect(result.pixelMisMatchCount).toBeDefined();
-    expect(result.diffPercent).toBeDefined();
+    expect(result.diffName).toBeNull();
+    expect(result.pixelMisMatchCount).toBe(0);
+    expect(result.diffPercent).toBe(0);
   });
 
   it('should handle invalid JSON response as error', async () => {

src/compare/libs/vlm/vlm.service.ts

@@ -1,10 +1,8 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { TestStatus } from '@prisma/client';
-import Pixelmatch from 'pixelmatch';
-import { PNG } from 'pngjs';
 import { StaticService } from '../../../static/static.service';
 import { DiffResult } from '../../../test-runs/diffResult';
-import { parseConfig, pngToBase64, scaleImageToSize } from '../../utils';
+import { parseConfig, pngToBase64 } from '../../utils';
 import { NO_BASELINE_RESULT } from '../consts';
 import { ImageComparator } from '../image-comparator.interface';
 import { ImageCompareInput } from '../ImageCompareInput';
@@ -23,11 +21,21 @@ CHECK for differences:
 IGNORE rendering artifacts: anti-aliasing, shadows, 1-2px shifts.`;
 
 // Internal constant - not exposed to user config to ensure consistent JSON output
-const JSON_FORMAT_INSTRUCTION = `
-Respond with JSON: {"identical": true/false, "description": "explanation"}
-- Set "identical": true if screenshots match or have only ignorable artifacts
-- Set "identical": false if meaningful differences exist
-- Always provide a brief description`;
+const JSON_FORMAT_INSTRUCTION = `CRITICAL: You must respond with ONLY valid JSON in this exact format:
+{"identical": Boolean, "description": String}
+
+**JSON Schema Reference:**
+The JSON object MUST conform to the following schema:
+{
+  "identical": <boolean>,
+  "description": <string>
+}
+
+**Requirements:**
+1.  **"identical":** Must be a standard boolean (\`true\` or \`false\`).
+2.  **"description":** Must be a detailed string explaining the reasoning.
+    * If identical is \`true\`, the description should be "Screenshots are functionally identical based on all comparison criteria."
+    * If identical is \`false\`, the description must clearly and concisely list the differences found (e.g., "The user count changed from 12 to 15, and the 'New User' button is missing."). Escape any internal double quotes with \\".`;
 
 export const DEFAULT_CONFIG: VlmConfig = {
   model: 'llava:7b',
@@ -75,10 +83,9 @@ export class VlmService implements ImageComparator {
         result.diffName = null;
       } else {
         result.status = TestStatus.unresolved;
-        const pixelDiff = this.calculatePixelDiff(baseline, image);
-        result.pixelMisMatchCount = pixelDiff.pixelMisMatchCount;
-        result.diffPercent = pixelDiff.diffPercent;
-        result.diffName = data.saveDiffAsFile ? await this.saveDiffImage(baseline, image) : null;
+        result.pixelMisMatchCount = 0;
+        result.diffPercent = 0;
+        result.diffName = null;
       }
     } catch (error) {
       this.logger.error(`VLM comparison failed: ${error.message}`, error.stack);
@@ -130,38 +137,4 @@ export class VlmService implements ImageComparator {
       description: parsed.description || 'No description provided',
     };
   }
-
-  private calculatePixelDiff(baseline: PNG, image: PNG): { pixelMisMatchCount: number; diffPercent: number } {
-    const maxWidth = Math.max(baseline.width, image.width);
-    const maxHeight = Math.max(baseline.height, image.height);
-    const scaledBaseline = scaleImageToSize(baseline, maxWidth, maxHeight);
-    const scaledImage = scaleImageToSize(image, maxWidth, maxHeight);
-
-    const diff = new PNG({ width: maxWidth, height: maxHeight });
-    const pixelMisMatchCount = Pixelmatch(scaledBaseline.data, scaledImage.data, diff.data, maxWidth, maxHeight, {
-      threshold: 0.1,
-      includeAA: true,
-    });
-
-    const diffPercent = Number(((pixelMisMatchCount * 100) / (maxWidth * maxHeight)).toFixed(2));
-    this.logger.debug(`Pixelmatch: ${pixelMisMatchCount} pixels (${diffPercent}%)`);
-
-    return { pixelMisMatchCount, diffPercent };
-  }
-
-  private async saveDiffImage(baseline: PNG, image: PNG): Promise<string> {
-    const maxWidth = Math.max(baseline.width, image.width);
-    const maxHeight = Math.max(baseline.height, image.height);
-    const scaledBaseline = scaleImageToSize(baseline, maxWidth, maxHeight);
-    const scaledImage = scaleImageToSize(image, maxWidth, maxHeight);
-
-    const diff = new PNG({ width: maxWidth, height: maxHeight });
-    Pixelmatch(scaledBaseline.data, scaledImage.data, diff.data, maxWidth, maxHeight, {
-      threshold: 0.1,
-      includeAA: true,
-    });
-
-    const diffBuffer = PNG.sync.write(diff);
-    return this.staticService.saveImage('diff', diffBuffer);
-  }
 }

src/test-runs/dto/testRun.dto.spec.ts

@@ -0,0 +1,39 @@
+import { TestRun } from '@prisma/client';
+import { generateTestRun } from '../../_data_';
+import { TestRunDto } from './testRun.dto';
+
+describe('TestRunDto', () => {
+  it('should map all fields correctly including vlmDescription', () => {
+    const testRun: TestRun = generateTestRun({
+      vlmDescription: 'VLM analysis result',
+    });
+
+    const result = new TestRunDto(testRun);
+
+    expect(result).toMatchObject({
+      id: testRun.id,
+      buildId: testRun.buildId,
+      imageName: testRun.imageName,
+      diffName: testRun.diffName,
+      diffPercent: testRun.diffPercent,
+      diffTollerancePercent: testRun.diffTollerancePercent,
+      status: testRun.status,
+      testVariationId: testRun.testVariationId,
+      name: testRun.name,
+      baselineName: testRun.baselineName,
+      os: testRun.os,
+      browser: testRun.browser,
+      viewport: testRun.viewport,
+      device: testRun.device,
+      customTags: testRun.customTags,
+      ignoreAreas: testRun.ignoreAreas,
+      tempIgnoreAreas: testRun.tempIgnoreAreas,
+      comment: testRun.comment,
+      branchName: testRun.branchName,
+      baselineBranchName: testRun.baselineBranchName,
+      merge: testRun.merge,
+      vlmDescription: testRun.vlmDescription,
+    });
+  });
+});
+

src/test-runs/dto/testRun.dto.ts

@@ -44,6 +44,8 @@ export class TestRunDto {
   baselineBranchName: string;
   @ApiProperty()
   merge: boolean;
+  @ApiPropertyOptional()
+  vlmDescription?: string;
 
   constructor(testRun: TestRun) {
     this.id = testRun.id;
@@ -67,5 +69,6 @@ export class TestRunDto {
     this.branchName = testRun.branchName;
     this.baselineBranchName = testRun.baselineBranchName;
     this.merge = testRun.merge;
+    this.vlmDescription = testRun.vlmDescription;
   }
 }

src/test-runs/test-runs.service.spec.ts

@@ -345,6 +345,7 @@ describe('TestRunsService', () => {
           diffName: null,
           pixelMisMatchCount: null,
           diffPercent: null,
+          vlmDescription: null,
         },
       });
       expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun);
@@ -357,6 +358,7 @@ describe('TestRunsService', () => {
         pixelMisMatchCount: 11,
         diffPercent: 22,
         isSameDimension: true,
+        vlmDescription: 'VLM detected significant color differences in the header section',
       };
       const id = 'some id';
       const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun);
@@ -375,6 +377,7 @@ describe('TestRunsService', () => {
           diffName: diff.diffName,
           pixelMisMatchCount: diff.pixelMisMatchCount,
           diffPercent: diff.diffPercent,
+          vlmDescription: diff.vlmDescription,
         },
       });
       expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun);
@@ -383,7 +386,9 @@ describe('TestRunsService', () => {
 
   it('findMany', async () => {
     const buildId = 'some id';
-    const testRun: TestRun = generateTestRun();
+    const testRun: TestRun = generateTestRun({
+      vlmDescription: 'VLM analysis completed',
+    });
     const testRunFindManyMock = jest.fn().mockResolvedValueOnce([testRun]);
     service = await initService({
       testRunFindManyMock,

src/test-runs/test-runs.service.ts

@@ -173,7 +173,7 @@ export class TestRunsService {
           pixelMisMatchCount: diffResult && diffResult.pixelMisMatchCount,
           diffPercent: diffResult && diffResult.diffPercent,
           status: diffResult ? diffResult.status : TestStatus.new,
-          vlmDescription: diffResult?.vlmDescription,
+          vlmDescription: diffResult && diffResult?.vlmDescription,
         },
       })
       .then((testRun) => {