diff --git a/apps/scan-engine/src/scan-engine.consumer.spec.ts b/apps/scan-engine/src/scan-engine.consumer.spec.ts index 7a051848..009f20b4 100644 --- a/apps/scan-engine/src/scan-engine.consumer.spec.ts +++ b/apps/scan-engine/src/scan-engine.consumer.spec.ts @@ -11,6 +11,19 @@ import { CoreScannerService } from 'libs/core-scanner/src'; import { ScanEngineConsumer } from './scan-engine.consumer'; import { QueueService } from '@app/queue'; +import { ScanStatus } from 'entities/scan-status'; + +// URLs in test inputs use the production format: bare domain without protocol +// (e.g. '18f.gov', not 'https://18f.gov'). This mirrors how CoreInputDto.url +// is populated by the ingest pipeline. +const defaultInput: CoreInputDto = { + websiteId: 1, + url: '18f.gov', + filter: false, + pageviews: 1, + visits: 1, + scanId: '123', +}; describe('ScanEngineConsumer', () => { let consumer: ScanEngineConsumer; @@ -22,11 +35,7 @@ describe('ScanEngineConsumer', () => { beforeEach(async () => { mockCoreResultService = mock(); - mockCoreScanner = mock>({ - scan: async (input) => { - return { id: input.websiteId } as CoreResult; - }, - }); + mockCoreScanner = mock>(); mockCoreJob = mock>(); mockQueueService = mock(); module = await Test.createTestingModule({ @@ -55,18 +64,11 @@ describe('ScanEngineConsumer', () => { }); it('should call the CoreResultService', async () => { - const input: CoreInputDto = { - websiteId: 1, - url: 'https://18f.gov', - filter: false, - pageviews: 1, - visits: 1, - scanId: '123', - }; - + const input = defaultInput; mockCoreJob.data = input; + const coreResultFromPages = { id: input.websiteId } as CoreResult; + (mockCoreScanner.scan as any) = jest.fn().mockResolvedValue(coreResultFromPages); - const coreResultFromPages = await mockCoreScanner.scan(input); await consumer.processCore(mockCoreJob); expect( mockCoreResultService.createFromCoreResultPages, @@ -80,4 +82,58 @@ describe('ScanEngineConsumer', () => { input.url, ); }); + + it('should call writeFailedResult when coreScanner.scan throws a permanent DNS error', async () => { + const input = defaultInput; + mockCoreJob.data = input; + const dnsError = new Error('net::ERR_NAME_NOT_RESOLVED'); + (mockCoreScanner.scan as any) = jest.fn().mockRejectedValue(dnsError); + + await consumer.processCore(mockCoreJob); + + expect(mockCoreResultService.writeFailedResult).toHaveBeenCalledWith( + input.websiteId, + ScanStatus.DNSResolutionError, + expect.anything(), + input.filter, + input.pageviews, + input.visits, + input.url, + ); + expect(mockCoreResultService.createFromCoreResultPages).not.toHaveBeenCalled(); + }); + + it('should call writeFailedResult when coreScanner.scan throws a permanent SSL error', async () => { + const input: CoreInputDto = { ...defaultInput, url: 'bad-ssl.example.gov' }; + mockCoreJob.data = input; + const sslError = new Error('net::ERR_CERT_COMMON_NAME_INVALID'); + (mockCoreScanner.scan as any) = jest.fn().mockRejectedValue(sslError); + + await consumer.processCore(mockCoreJob); + + expect(mockCoreResultService.writeFailedResult).toHaveBeenCalledWith( + input.websiteId, + ScanStatus.InvalidSSLCert, + expect.anything(), + input.filter, + input.pageviews, + input.visits, + input.url, + ); + expect(mockCoreResultService.createFromCoreResultPages).not.toHaveBeenCalled(); + }); + + it('should rethrow non-permanent errors for Bull to retry', async () => { + const input: CoreInputDto = { + ...defaultInput, + url: 'blackhole.webpagetest.org', + }; + mockCoreJob.data = input; + const timeoutError = new Error('Timeout'); + (mockCoreScanner.scan as any) = jest.fn().mockRejectedValue(timeoutError); + + await expect(consumer.processCore(mockCoreJob)).rejects.toThrow('Timeout'); + expect(mockCoreResultService.writeFailedResult).not.toHaveBeenCalled(); + expect(mockCoreResultService.createFromCoreResultPages).not.toHaveBeenCalled(); + }); }); diff --git a/apps/scan-engine/src/scan-engine.consumer.ts b/apps/scan-engine/src/scan-engine.consumer.ts index e04dd351..87adc6f8 100644 --- a/apps/scan-engine/src/scan-engine.consumer.ts +++ b/apps/scan-engine/src/scan-engine.consumer.ts @@ -8,13 +8,16 @@ import { Process, Processor, } from '@nestjs/bull'; -import { Logger } from '@nestjs/common'; +// use as NestLogger to avoid confusion with `Logger` from Pino that is used +// in `entities/scan-status` +import { Logger as NestLogger } from '@nestjs/common'; import { Job } from 'bull'; import { CoreScannerService } from '@app/core-scanner'; import { CoreInputDto } from '@app/core-scanner/core.input.dto'; import { CoreResultService } from '@app/database/core-results/core-result.service'; import { QueueService } from '@app/queue'; +import { isPermanentFailure, parseBrowserError } from 'entities/scan-status'; /** * ScanEngineConsumer is a consumer of the Scanner message queue. @@ -35,7 +38,7 @@ import { QueueService } from '@app/queue'; */ @Processor(SCANNER_QUEUE_NAME) export class ScanEngineConsumer { - private logger = new Logger(ScanEngineConsumer.name); + private logger = new NestLogger(ScanEngineConsumer.name); constructor( private coreResultService: CoreResultService, @@ -81,18 +84,29 @@ export class ScanEngineConsumer { const err = e as Error; this.logger.error(err.message, err.stack); - // Don't retry errors that are permanent - const errorMsg = err.message || ''; - const isPermanent = - errorMsg.includes('ERR_NAME_NOT_RESOLVED') || - errorMsg.includes('ENOTFOUND') || - errorMsg.includes('ERR_CERT_') || - errorMsg.includes('ERR_SSL_UNRECOGNIZED_NAME_ALERT') || - errorMsg.includes('unable to verify the first certificate') || - errorMsg.includes('ERR_SSL_VERSION_OR_CIPHER_MISMATCH'); - - if (isPermanent) { + // Classify once — parseBrowserError is the single source of truth for + // error → ScanStatus mapping. isPermanentFailure determines retry policy. + const failureStatus = parseBrowserError(err); + + if (isPermanentFailure(failureStatus)) { this.logger.warn(`Permanent failure for ${job.data.url}, not retrying`); + + await this.coreResultService.writeFailedResult( + job.data.websiteId, + failureStatus, + this.logger, + job.data.filter, + job.data.pageviews, + job.data.visits, + job.data.url, + ); + + this.logger.log({ + msg: `wrote failure result for ${job.data.url}`, + status: failureStatus, + job, + }); + return; } diff --git a/entities/scan-status.ts b/entities/scan-status.ts index eacfaf96..1949575b 100644 --- a/entities/scan-status.ts +++ b/entities/scan-status.ts @@ -28,16 +28,48 @@ export enum ScanStatus { export type AnySuccessfulStatus = ScanStatus.Completed; export type AnyFailureStatus = Exclude; +/** + * The set of failure statuses that represent permanent, non-retryable errors + * (e.g. DNS resolution failures, invalid SSL certificates). Bull should not + * retry jobs that result in one of these statuses. + */ +export const PERMANENT_FAILURE_STATUSES = new Set([ + ScanStatus.DNSResolutionError, + ScanStatus.InvalidSSLCert, + ScanStatus.SslVersionCipherMismatch, +]); + +/** + * Returns true when the given failure status represents a permanent error that + * should not be retried by the job queue. + */ +export const isPermanentFailure = (status: AnyFailureStatus): boolean => + PERMANENT_FAILURE_STATUSES.has(status); + +/** + * Classifies a browser (Puppeteer/Chrome) error into a ScanStatus. + * + * @param err - The error thrown during scanning. + * @param logger - Optional pino logger. When omitted, classification still + * works but no log entries are produced. This allows callers that hold a + * non-pino logger (e.g. NestJS Logger) to classify errors without needing + * to construct a fake pino-compatible object. + */ export const parseBrowserError = ( err: Error, - logger: Logger, + logger?: Logger, ): AnyFailureStatus => { - const childLogger = logger.child({ - function: 'parseBrowserError', - errorName: err.name, - errorStack: err.stack, - errorMessage: err.message, - }); + const childLogger = logger + ? logger.child({ + function: 'parseBrowserError', + errorName: err.name, + errorStack: err.stack, + errorMessage: err.message, + }) + : null; + + const warn = (meta: object, msg: string) => childLogger?.warn(meta, msg); + if ( (err.name && err.name === 'TimeoutError') || (err.message && @@ -45,7 +77,7 @@ export const parseBrowserError = ( err.message.startsWith('connect ETIMEDOUT') || err.message.startsWith('net::ERR_TIMED_OUT'))) ) { - childLogger.warn( + warn( { timeoutError: true, errorReturn: ScanStatus.Timeout }, `Timeout: ${err.message}`, ); @@ -57,7 +89,7 @@ export const parseBrowserError = ( err.message.startsWith('net::ERR_NAME_NOT_RESOLVED') || err.message.startsWith('getaddrinfo ENOTFOUND') ) { - childLogger.warn( + warn( { dnsResolutionError: true, errorReturn: ScanStatus.DNSResolutionError, @@ -74,7 +106,7 @@ export const parseBrowserError = ( err.message.startsWith('net::ERR_SSL_UNRECOGNIZED_NAME_ALERT') || err.message.startsWith('unable to verify the first certificate') ) { - childLogger.warn( + warn( { invalidSSLCertError: true, errorReturn: ScanStatus.InvalidSSLCert }, `Invalid SSL cert: ${err.message}`, ); @@ -85,7 +117,7 @@ export const parseBrowserError = ( err.message.startsWith('net::ERR_CONNECTION_REFUSED') || err.message.startsWith('connect ECONNREFUSED') ) { - childLogger.warn( + warn( { connectionRefusedError: true, errorReturn: ScanStatus.ConnectionRefused, @@ -99,15 +131,18 @@ export const parseBrowserError = ( err.message.startsWith('net::ERR_CONNECTION_RESET') || err.message.startsWith('read ECONNRESET') ) { - childLogger.warn( - { connectionResetError: true, errorReturn: ScanStatus.ConnectionReset }, + warn( + { + connectionResetError: true, + errorReturn: ScanStatus.ConnectionReset, + }, `Connection reset: ${err.message}`, ); return ScanStatus.ConnectionReset; } if (err.message.startsWith('net::ERR_CONNECTION_CLOSED')) { - childLogger.warn( + warn( { connectionClosedError: true, errorReturn: ScanStatus.ConnectionClosed, @@ -118,7 +153,7 @@ export const parseBrowserError = ( } if (err.message.startsWith('net::ERR_ADDRESS_UNREACHABLE')) { - childLogger.warn( + warn( { addressUnreachableError: true, errorReturn: ScanStatus.AddressUnreachable, @@ -129,7 +164,7 @@ export const parseBrowserError = ( } if (err.message.startsWith('net::ERR_SSL_VERSION_OR_CIPHER_MISMATCH')) { - childLogger.warn( + warn( { sslVersionCipherMismatchError: true, errorReturn: ScanStatus.SslVersionCipherMismatch, @@ -140,7 +175,7 @@ export const parseBrowserError = ( } if (err.message.startsWith('net::ERR_EMPTY_RESPONSE')) { - childLogger.warn( + warn( { emptyResponseError: true, errorReturn: ScanStatus.EmptyResponse }, `Empty response: ${err.message}`, ); @@ -148,7 +183,7 @@ export const parseBrowserError = ( } if (err.message.startsWith('net::ERR_HTTP2_PROTOCOL_ERROR')) { - childLogger.warn( + warn( { http2ErrorError: true, errorReturn: ScanStatus.Http2Error }, `HTTP2 error: ${err.message}`, ); @@ -156,7 +191,7 @@ export const parseBrowserError = ( } if (err.message.startsWith('net::ERR_TOO_MANY_REDIRECTS')) { - childLogger.warn( + warn( { tooManyRedirectsError: true, errorReturn: ScanStatus.TooManyRedirects, @@ -167,7 +202,7 @@ export const parseBrowserError = ( } if (err.message.startsWith('Execution context was destroyed')) { - childLogger.warn( + warn( { executionContextDestroyedError: true, errorReturn: ScanStatus.ExecutionContextDestroyed, @@ -178,15 +213,18 @@ export const parseBrowserError = ( } if (err.message.startsWith('Page/Frame is not ready')) { - childLogger.warn( - { pageFrameNotReady: true, errorReturn: ScanStatus.PageFrameNotReady }, + warn( + { + pageFrameNotReady: true, + errorReturn: ScanStatus.PageFrameNotReady, + }, `Page/Frame is not ready: ${err.message}`, ); return ScanStatus.PageFrameNotReady; } if (err.message.startsWith('Evaluation failed')) { - childLogger.warn( + warn( { evaluationFailedError: true, errorReturn: ScanStatus.EvaluationFailed, @@ -197,15 +235,18 @@ export const parseBrowserError = ( } if (err.message.startsWith('net::ERR_INVALID_RESPONSE')) { - childLogger.warn( - { invalidResponseError: true, errorReturn: ScanStatus.InvalidResponse }, + warn( + { + invalidResponseError: true, + errorReturn: ScanStatus.InvalidResponse, + }, `Invalid response: ${err.message}`, ); return ScanStatus.InvalidResponse; } if (err.message.startsWith('net::ERR_INVALID_AUTH_CREDENTIALS')) { - childLogger.warn( + warn( { invalidAuthCredentialsError: true, errorReturn: ScanStatus.InvalidAuthCredentials, @@ -216,7 +257,7 @@ export const parseBrowserError = ( } if (err.message.startsWith('net::ERR_SSL_PROTOCOL_ERROR')) { - childLogger.warn( + warn( { sslProtocolErrorError: true, errorReturn: ScanStatus.SslProtocolError, @@ -227,7 +268,7 @@ export const parseBrowserError = ( } if (err.message.startsWith('net::ERR_ABORTED')) { - childLogger.warn( + warn( { abortedError: true, errorReturn: ScanStatus.Aborted }, `Aborted: ${err.message}`, ); @@ -235,14 +276,15 @@ export const parseBrowserError = ( } if (err.toString() === 'Processing timed out') { - childLogger.warn( + warn( { timeoutError: true, errorReturn: ScanStatus.Timeout }, `Timeout: ${err.message}`, ); return ScanStatus.Timeout; } } - childLogger.warn( + + warn( { unknownError: true, errorReturn: ScanStatus.UnknownError }, `Unknown error: ${err.message}`, ); diff --git a/libs/database/src/core-results/core-result.service.spec.ts b/libs/database/src/core-results/core-result.service.spec.ts index 7522c1d5..636c8f1c 100644 --- a/libs/database/src/core-results/core-result.service.spec.ts +++ b/libs/database/src/core-results/core-result.service.spec.ts @@ -487,4 +487,81 @@ describe('CoreResultService', () => { expect.objectContaining({ filter: true }), ); }); + + it('should write a failure result with all statuses set and all data nulled', async () => { + const websiteId = 1; + const failureStatus = ScanStatus.InvalidSSLCert; + const websiteUrl = '18f.gov'; + const logger = mock(); + + mockRepository.findOne.calledWith().mockResolvedValue(null); + + await service.writeFailedResult( + websiteId, + failureStatus, + logger, + false, + 100, + 50, + websiteUrl, + ); + + expect(mockRepository.insert).toHaveBeenCalled(); + expect(mockRepository.insert).toHaveBeenCalledWith( + expect.objectContaining({ + primaryScanStatus: failureStatus, + notFoundScanStatus: failureStatus, + robotsTxtScanStatus: failureStatus, + sitemapXmlScanStatus: failureStatus, + dnsScanStatus: failureStatus, + accessibilityScanStatus: failureStatus, + performanceScanStatus: failureStatus, + securityScanStatus: failureStatus, + wwwScanStatus: failureStatus, + finalUrl: null, + pageTitle: null, + accessibilityResults: null, + wwwFinalUrl: null, + targetUrl404Test: null, + dnsHostname: null, + filter: false, + pageviews: 100, + visits: 50, + initialUrl: 'https://18f.gov', + }), + ); + expect(mockWebsiteService.setFilter).not.toHaveBeenCalled(); + }); + + it('should update existing result when writeFailedResult is called for existing website', async () => { + const websiteId = 1; + const failureStatus = ScanStatus.DNSResolutionError; + const websiteUrl = '18f.gov'; + const logger = mock(); + + const existingResult = new CoreResult(); + existingResult.id = 123; + mockRepository.findOne.calledWith().mockResolvedValue(existingResult); + + await service.writeFailedResult( + websiteId, + failureStatus, + logger, + false, + 100, + 50, + websiteUrl, + ); + + expect(mockRepository.update).toHaveBeenCalledWith( + 123, + expect.objectContaining({ + primaryScanStatus: failureStatus, + notFoundScanStatus: failureStatus, + finalUrl: null, + pageTitle: null, + }), + ); + expect(mockRepository.insert).not.toHaveBeenCalled(); + }); }); diff --git a/libs/database/src/core-results/core-result.service.ts b/libs/database/src/core-results/core-result.service.ts index 189fc55c..40b3b497 100644 --- a/libs/database/src/core-results/core-result.service.ts +++ b/libs/database/src/core-results/core-result.service.ts @@ -6,6 +6,7 @@ import { Website } from 'entities/website.entity'; import { ScanStatus } from 'entities/scan-status'; import { CoreResultPages } from 'entities/core-result.entity'; import { WebsiteService } from '@app/database/websites/websites.service'; +import { getBaseDomain, getHttpsUrl } from '@app/core-scanner/util'; @Injectable() export class CoreResultService { @@ -49,14 +50,6 @@ export class CoreResultService { this.updateSecurityScanResults(coreResult, pages, logger); this.updateWwwScanResults(coreResult, pages, logger); - const exists = await this.coreResultRepository.findOne({ - where: { - website: { - id: coreResult.website.id, - }, - }, - }); - if ( coreResult.finalUrlMIMEType && CoreResult.filteredMediaTypes.includes(coreResult.finalUrlMIMEType) @@ -65,11 +58,7 @@ export class CoreResultService { await this.websiteService.setFilter(websiteId, true); } - if (exists) { - await this.coreResultRepository.update(exists.id, coreResult); - } else { - await this.coreResultRepository.insert(coreResult); - } + await this.upsertCoreResult(coreResult); } async findOne(id: number): Promise { @@ -232,70 +221,74 @@ export class CoreResultService { page: 'primary', }); - coreResult.dapDetected = null; - coreResult.dapParameters = null; - coreResult.dapVersion = null; - coreResult.gaTagIds = null; - coreResult.mainElementFinalUrl = null; - coreResult.ogArticleModifiedFinalUrl = null; - coreResult.ogArticlePublishedFinalUrl = null; - coreResult.ogDescriptionFinalUrl = null; - coreResult.ogTitleFinalUrl = null; - coreResult.canonicalLink = null; - coreResult.pageTitle = null; - coreResult.metaDescriptionContent = null; - coreResult.metaKeywordsContent = null; - coreResult.ogImageContent = null; - coreResult.ogTypeContent = null; - coreResult.ogUrlContent = null; - coreResult.htmlLangContent = null; - coreResult.hrefLangContent = null; - coreResult.dcDateContent = null; - coreResult.dcDateCreatedContent = null; - coreResult.dctermsCreatedContent = null; - coreResult.revisedContent = null; - coreResult.lastModifiedContent = null; - coreResult.dateContent = null; - coreResult.thirdPartyServiceCount = null; - coreResult.thirdPartyServiceDomains = null; - coreResult.thirdPartyServiceUrls = null; - coreResult.cookieDomains = null; - coreResult.finalUrl = null; - coreResult.finalUrlBaseDomain = null; - coreResult.finalUrlWebsite = null; - coreResult.finalUrlTopLevelDomain = null; - coreResult.finalUrlIsLive = null; - coreResult.finalUrlMIMEType = null; - coreResult.finalUrlStatusCode = null; - coreResult.finalSiteName = null; - coreResult.finalUrlPageHash = null; - coreResult.targetUrlRedirects = null; - coreResult.usaClasses = null; - coreResult.usaClassesUsed = null; - coreResult.uswdsString = null; - coreResult.uswdsInlineCss = null; - coreResult.uswdsUsFlag = null; - coreResult.uswdsUsFlagInCss = null; - coreResult.uswdsStringInCss = null; - coreResult.uswdsPublicSansFont = null; - coreResult.uswdsSemanticVersion = null; - coreResult.uswdsVersion = null; - coreResult.uswdsCount = null; - coreResult.heresHowYouKnowBanner = null; - coreResult.loginDetected = null; - coreResult.loginProvider = null; - coreResult.cms = null; - coreResult.hyperlinkDomains; - coreResult.requiredLinksUrl = null; - coreResult.requiredLinksText = null; - coreResult.feedbackLinksText = null; - coreResult.searchDetected = null; - coreResult.searchgov = null; - coreResult.viewportMetaTag = null; - coreResult.tooling = null; + this.clearPrimaryScanResults(coreResult); } } + private clearPrimaryScanResults(coreResult: CoreResult) { + coreResult.dapDetected = null; + coreResult.dapParameters = null; + coreResult.dapVersion = null; + coreResult.gaTagIds = null; + coreResult.mainElementFinalUrl = null; + coreResult.ogArticleModifiedFinalUrl = null; + coreResult.ogArticlePublishedFinalUrl = null; + coreResult.ogDescriptionFinalUrl = null; + coreResult.ogTitleFinalUrl = null; + coreResult.canonicalLink = null; + coreResult.pageTitle = null; + coreResult.metaDescriptionContent = null; + coreResult.metaKeywordsContent = null; + coreResult.ogImageContent = null; + coreResult.ogTypeContent = null; + coreResult.ogUrlContent = null; + coreResult.htmlLangContent = null; + coreResult.hrefLangContent = null; + coreResult.dcDateContent = null; + coreResult.dcDateCreatedContent = null; + coreResult.dctermsCreatedContent = null; + coreResult.revisedContent = null; + coreResult.lastModifiedContent = null; + coreResult.dateContent = null; + coreResult.thirdPartyServiceCount = null; + coreResult.thirdPartyServiceDomains = null; + coreResult.thirdPartyServiceUrls = null; + coreResult.cookieDomains = null; + coreResult.finalUrl = null; + coreResult.finalUrlBaseDomain = null; + coreResult.finalUrlWebsite = null; + coreResult.finalUrlTopLevelDomain = null; + coreResult.finalUrlIsLive = null; + coreResult.finalUrlMIMEType = null; + coreResult.finalUrlStatusCode = null; + coreResult.finalSiteName = null; + coreResult.finalUrlPageHash = null; + coreResult.targetUrlRedirects = null; + coreResult.usaClasses = null; + coreResult.usaClassesUsed = null; + coreResult.uswdsString = null; + coreResult.uswdsInlineCss = null; + coreResult.uswdsUsFlag = null; + coreResult.uswdsUsFlagInCss = null; + coreResult.uswdsStringInCss = null; + coreResult.uswdsPublicSansFont = null; + coreResult.uswdsSemanticVersion = null; + coreResult.uswdsVersion = null; + coreResult.uswdsCount = null; + coreResult.heresHowYouKnowBanner = null; + coreResult.loginDetected = null; + coreResult.loginProvider = null; + coreResult.cms = null; + coreResult.hyperlinkDomains = null; + coreResult.requiredLinksUrl = null; + coreResult.requiredLinksText = null; + coreResult.feedbackLinksText = null; + coreResult.searchDetected = null; + coreResult.searchgov = null; + coreResult.viewportMetaTag = null; + coreResult.tooling = null; + } + private updateNotFoundScanResults( coreResult: CoreResult, pages: CoreResultPages, @@ -312,10 +305,14 @@ export class CoreResultService { page: 'notFound', }); - coreResult.targetUrl404Test = null; + this.clearNotFoundScanResults(coreResult); } } + private clearNotFoundScanResults(coreResult: CoreResult) { + coreResult.targetUrl404Test = null; + } + private updateRobotsTxtScanResults( coreResult: CoreResult, pages: CoreResultPages, @@ -340,16 +337,20 @@ export class CoreResultService { page: 'robotsTxt', }); - coreResult.robotsTxtFinalUrlSize = null; - coreResult.robotsTxtCrawlDelay = null; - coreResult.robotsTxtSitemapLocations = null; - coreResult.robotsTxtFinalUrl = null; - coreResult.robotsTxtFinalUrlMimeType = null; - coreResult.robotsTxtStatusCode = null; - coreResult.robotsTxtDetected = null; + this.clearRobotsTxtScanResults(coreResult); } } + private clearRobotsTxtScanResults(coreResult: CoreResult) { + coreResult.robotsTxtFinalUrlSize = null; + coreResult.robotsTxtCrawlDelay = null; + coreResult.robotsTxtSitemapLocations = null; + coreResult.robotsTxtFinalUrl = null; + coreResult.robotsTxtFinalUrlMimeType = null; + coreResult.robotsTxtStatusCode = null; + coreResult.robotsTxtDetected = null; + } + private updateSitemapXmlScanResults( coreResult: CoreResult, pages: CoreResultPages, @@ -376,18 +377,22 @@ export class CoreResultService { page: 'sitemap.xml', }); - coreResult.sitemapXmlFinalUrlFilesize = null; - coreResult.sitemapXmlCount = null; - coreResult.sitemapXmlPdfCount = null; - coreResult.sitemapXmlFinalUrl = null; - coreResult.sitemapXmlFinalUrlMimeType = null; - coreResult.sitemapXmlStatusCode = null; - coreResult.sitemapXmlDetected = null; - coreResult.sitemapXmlLastMod = null; - coreResult.sitemapXmlPageHash = null; + this.clearSitemapXmlScanResults(coreResult); } } + private clearSitemapXmlScanResults(coreResult: CoreResult) { + coreResult.sitemapXmlFinalUrlFilesize = null; + coreResult.sitemapXmlCount = null; + coreResult.sitemapXmlPdfCount = null; + coreResult.sitemapXmlFinalUrl = null; + coreResult.sitemapXmlFinalUrlMimeType = null; + coreResult.sitemapXmlStatusCode = null; + coreResult.sitemapXmlDetected = null; + coreResult.sitemapXmlLastMod = null; + coreResult.sitemapXmlPageHash = null; + } + private updateDnsScanResults( coreResult: CoreResult, pages: CoreResultPages, @@ -404,11 +409,15 @@ export class CoreResultService { page: 'dns', }); - coreResult.dnsIpv6 = null; - coreResult.dnsHostname = null; + this.clearDnsScanResults(coreResult); } } + private clearDnsScanResults(coreResult: CoreResult) { + coreResult.dnsIpv6 = null; + coreResult.dnsHostname = null; + } + private updateAccessibilityScanResults( coreResult: CoreResult, pages: CoreResultPages, @@ -427,11 +436,15 @@ export class CoreResultService { page: 'accessibility', }); - coreResult.accessibilityResults = null; - coreResult.accessibilityResultsList = null; + this.clearAccessibilityScanResults(coreResult); } } + private clearAccessibilityScanResults(coreResult: CoreResult) { + coreResult.accessibilityResults = null; + coreResult.accessibilityResultsList = null; + } + private updatePerformanceScanResults( coreResult: CoreResult, pages: CoreResultPages, @@ -451,11 +464,15 @@ export class CoreResultService { page: 'performance', }); - coreResult.largestContentfulPaint = null; - coreResult.cumulativeLayoutShift = null; + this.clearPerformanceScanResults(coreResult); } } + private clearPerformanceScanResults(coreResult: CoreResult) { + coreResult.largestContentfulPaint = null; + coreResult.cumulativeLayoutShift = null; + } + private updateSecurityScanResults( coreResult: CoreResult, pages: CoreResultPages, @@ -473,11 +490,15 @@ export class CoreResultService { page: 'security', }); - coreResult.httpsEnforced = null; - coreResult.hsts = null; + this.clearSecurityScanResults(coreResult); } } + private clearSecurityScanResults(coreResult: CoreResult) { + coreResult.httpsEnforced = null; + coreResult.hsts = null; + } + private updateWwwScanResults( coreResult: CoreResult, pages: CoreResultPages, @@ -491,20 +512,82 @@ export class CoreResultService { coreResult.wwwTitle = pages.www.result.wwwScan.wwwTitle; coreResult.wwwSame = pages.www.result.wwwScan.wwwSame; } else if (pages.www.status === ScanStatus.NotApplicable) { - coreResult.wwwFinalUrl = null; - coreResult.wwwStatusCode = null; - coreResult.wwwTitle = null; - coreResult.wwwSame = null; + this.clearWwwScanResults(coreResult); } else { logger.error({ msg: pages.www.error, page: 'www', }); - coreResult.wwwFinalUrl = null; - coreResult.wwwStatusCode = null; - coreResult.wwwTitle = null; - coreResult.wwwSame = null; + this.clearWwwScanResults(coreResult); + } + } + + private clearWwwScanResults(coreResult: CoreResult) { + coreResult.wwwFinalUrl = null; + coreResult.wwwStatusCode = null; + coreResult.wwwTitle = null; + coreResult.wwwSame = null; + } + + async writeFailedResult( + websiteId: number, + status: ScanStatus, + logger: Logger, + filter: boolean, + pageviews: number, + visits: number, + websiteUrl: string, + ) { + const coreResult = new CoreResult(); + const website = new Website(); + website.id = websiteId; + coreResult.website = website; + coreResult.targetUrlBaseDomain = getBaseDomain(getHttpsUrl(websiteUrl)); + coreResult.filter = filter; + coreResult.pageviews = pageviews; + coreResult.visits = visits; + coreResult.initialUrl = getHttpsUrl(websiteUrl); + + // Set all scan status columns to the failure status + coreResult.primaryScanStatus = status; + coreResult.notFoundScanStatus = status; + coreResult.robotsTxtScanStatus = status; + coreResult.sitemapXmlScanStatus = status; + coreResult.dnsScanStatus = status; + coreResult.accessibilityScanStatus = status; + coreResult.performanceScanStatus = status; + coreResult.securityScanStatus = status; + coreResult.wwwScanStatus = status; + + // Clear all data columns + this.clearPrimaryScanResults(coreResult); + this.clearNotFoundScanResults(coreResult); + this.clearRobotsTxtScanResults(coreResult); + this.clearSitemapXmlScanResults(coreResult); + this.clearDnsScanResults(coreResult); + this.clearAccessibilityScanResults(coreResult); + this.clearPerformanceScanResults(coreResult); + this.clearSecurityScanResults(coreResult); + this.clearWwwScanResults(coreResult); + + // Upsert: update if exists, insert otherwise + await this.upsertCoreResult(coreResult); + } + + private async upsertCoreResult(coreResult: CoreResult) { + const exists = await this.coreResultRepository.findOne({ + where: { + website: { + id: coreResult.website.id, + }, + }, + }); + + if (exists) { + await this.coreResultRepository.update(exists.id, coreResult); + } else { + await this.coreResultRepository.insert(coreResult); } } }