Stale scan data handling

apps/scan-engine/src/scan-engine.consumer.spec.ts

@@ -11,6 +11,19 @@ import { CoreScannerService } from 'libs/core-scanner/src';
 
 import { ScanEngineConsumer } from './scan-engine.consumer';
 import { QueueService } from '@app/queue';
+import { ScanStatus } from 'entities/scan-status';
+
+// URLs in test inputs use the production format: bare domain without protocol
+// (e.g. '18f.gov', not 'https://18f.gov'). This mirrors how CoreInputDto.url
+// is populated by the ingest pipeline.
+const defaultInput: CoreInputDto = {
+  websiteId: 1,
+  url: '18f.gov',
+  filter: false,
+  pageviews: 1,
+  visits: 1,
+  scanId: '123',
+};
 
 describe('ScanEngineConsumer', () => {
   let consumer: ScanEngineConsumer;
@@ -22,11 +35,7 @@ describe('ScanEngineConsumer', () => {
 
   beforeEach(async () => {
     mockCoreResultService = mock<CoreResultService>();
-    mockCoreScanner = mock<Scanner<CoreInputDto, CoreResult>>({
-      scan: async (input) => {
-        return { id: input.websiteId } as CoreResult;
-      },
-    });
+    mockCoreScanner = mock<Scanner<CoreInputDto, CoreResult>>();
     mockCoreJob = mock<Job<CoreInputDto>>();
     mockQueueService = mock<QueueService>();
     module = await Test.createTestingModule({
@@ -55,18 +64,11 @@ describe('ScanEngineConsumer', () => {
   });
 
   it('should call the CoreResultService', async () => {
-    const input: CoreInputDto = {
-      websiteId: 1,
-      url: 'https://18f.gov',
-      filter: false,
-      pageviews: 1,
-      visits: 1,
-      scanId: '123',
-    };
-
+    const input = defaultInput;
     mockCoreJob.data = input;
+    const coreResultFromPages = { id: input.websiteId } as CoreResult;
+    (mockCoreScanner.scan as any) = jest.fn().mockResolvedValue(coreResultFromPages);
 
-    const coreResultFromPages = await mockCoreScanner.scan(input);
     await consumer.processCore(mockCoreJob);
     expect(
       mockCoreResultService.createFromCoreResultPages,
@@ -80,4 +82,58 @@ describe('ScanEngineConsumer', () => {
       input.url,
     );
   });
+
+  it('should call writeFailedResult when coreScanner.scan throws a permanent DNS error', async () => {
+    const input = defaultInput;
+    mockCoreJob.data = input;
+    const dnsError = new Error('net::ERR_NAME_NOT_RESOLVED');
+    (mockCoreScanner.scan as any) = jest.fn().mockRejectedValue(dnsError);
+
+    await consumer.processCore(mockCoreJob);
+
+    expect(mockCoreResultService.writeFailedResult).toHaveBeenCalledWith(
+      input.websiteId,
+      ScanStatus.DNSResolutionError,
+      expect.anything(),
+      input.filter,
+      input.pageviews,
+      input.visits,
+      input.url,
+    );
+    expect(mockCoreResultService.createFromCoreResultPages).not.toHaveBeenCalled();
+  });
+
+  it('should call writeFailedResult when coreScanner.scan throws a permanent SSL error', async () => {
+    const input: CoreInputDto = { ...defaultInput, url: 'bad-ssl.example.gov' };
+    mockCoreJob.data = input;
+    const sslError = new Error('net::ERR_CERT_COMMON_NAME_INVALID');
+    (mockCoreScanner.scan as any) = jest.fn().mockRejectedValue(sslError);
+
+    await consumer.processCore(mockCoreJob);
+
+    expect(mockCoreResultService.writeFailedResult).toHaveBeenCalledWith(
+      input.websiteId,
+      ScanStatus.InvalidSSLCert,
+      expect.anything(),
+      input.filter,
+      input.pageviews,
+      input.visits,
+      input.url,
+    );
+    expect(mockCoreResultService.createFromCoreResultPages).not.toHaveBeenCalled();
+  });
+
+  it('should rethrow non-permanent errors for Bull to retry', async () => {
+    const input: CoreInputDto = {
+      ...defaultInput,
+      url: 'blackhole.webpagetest.org',
+    };
+    mockCoreJob.data = input;
+    const timeoutError = new Error('Timeout');
+    (mockCoreScanner.scan as any) = jest.fn().mockRejectedValue(timeoutError);
+
+    await expect(consumer.processCore(mockCoreJob)).rejects.toThrow('Timeout');
+    expect(mockCoreResultService.writeFailedResult).not.toHaveBeenCalled();
+    expect(mockCoreResultService.createFromCoreResultPages).not.toHaveBeenCalled();
+  });
 });

apps/scan-engine/src/scan-engine.consumer.ts

@@ -8,13 +8,16 @@ import {
   Process,
   Processor,
 } from '@nestjs/bull';
-import { Logger } from '@nestjs/common';
+// use as NestLogger to avoid confusion with `Logger` from Pino that is used
+// in `entities/scan-status`
+import { Logger as NestLogger } from '@nestjs/common';
 import { Job } from 'bull';
 
 import { CoreScannerService } from '@app/core-scanner';
 import { CoreInputDto } from '@app/core-scanner/core.input.dto';
 import { CoreResultService } from '@app/database/core-results/core-result.service';
 import { QueueService } from '@app/queue';
+import { isPermanentFailure, parseBrowserError } from 'entities/scan-status';
 
 /**
  * ScanEngineConsumer is a consumer of the Scanner message queue.
@@ -35,7 +38,7 @@ import { QueueService } from '@app/queue';
  */
 @Processor(SCANNER_QUEUE_NAME)
 export class ScanEngineConsumer {
-  private logger = new Logger(ScanEngineConsumer.name);
+  private logger = new NestLogger(ScanEngineConsumer.name);
 
   constructor(
     private coreResultService: CoreResultService,
@@ -81,18 +84,29 @@ export class ScanEngineConsumer {
       const err = e as Error;
       this.logger.error(err.message, err.stack);
 
-      // Don't retry errors that are permanent
-      const errorMsg = err.message || '';
-      const isPermanent =
-        errorMsg.includes('ERR_NAME_NOT_RESOLVED') ||
-        errorMsg.includes('ENOTFOUND') ||
-        errorMsg.includes('ERR_CERT_') ||
-        errorMsg.includes('ERR_SSL_UNRECOGNIZED_NAME_ALERT') ||
-        errorMsg.includes('unable to verify the first certificate') ||
-        errorMsg.includes('ERR_SSL_VERSION_OR_CIPHER_MISMATCH');
-
-      if (isPermanent) {
+      // Classify once — parseBrowserError is the single source of truth for
+      // error → ScanStatus mapping. isPermanentFailure determines retry policy.
+      const failureStatus = parseBrowserError(err);
+
+      if (isPermanentFailure(failureStatus)) {
         this.logger.warn(`Permanent failure for ${job.data.url}, not retrying`);
+
+        await this.coreResultService.writeFailedResult(
+          job.data.websiteId,
+          failureStatus,
+          this.logger,
+          job.data.filter,
+          job.data.pageviews,
+          job.data.visits,
+          job.data.url,
+        );
+
+        this.logger.log({
+          msg: `wrote failure result for ${job.data.url}`,
+          status: failureStatus,
+          job,
+        });
+
         return;
       }