From 72225a004b2636c0e031715b68f5ea0558934e54 Mon Sep 17 00:00:00 2001 From: Michael Reimsbach Date: Thu, 13 Nov 2025 17:02:05 +0100 Subject: [PATCH 1/7] wip --- internal/database/mariadb/entity.go | 4 +- internal/database/mariadb/issue_test.go | 90 ----- ...dd_mv_count_vulnerability_component.up.sql | 4 + ...ters_replace_enum_rows_with_columns.up.sql | 29 +- .../database/mariadb/mv_vulnerabilities.go | 264 ++++---------- .../mariadb/mv_vulnerabilities_test.go | 331 ++++++++++++++++++ internal/database/mariadb/test/fixture.go | 36 +- 7 files changed, 445 insertions(+), 313 deletions(-) create mode 100644 internal/database/mariadb/mv_vulnerabilities_test.go diff --git a/internal/database/mariadb/entity.go b/internal/database/mariadb/entity.go index 87af68be..7fd3bb3d 100644 --- a/internal/database/mariadb/entity.go +++ b/internal/database/mariadb/entity.go @@ -1191,7 +1191,7 @@ func (rr *RemediationRow) FromRemediation(r *entity.Remediation) { rr.Description = sql.NullString{String: r.Description, Valid: true} rr.Type = sql.NullString{String: r.Type.String(), Valid: true} rr.Component = sql.NullString{String: r.Component, Valid: true} - rr.ComponentId = sql.NullInt64{Int64: r.ComponentId, Valid: true} + rr.ComponentId = sql.NullInt64{Int64: r.ComponentId, Valid: r.ComponentId != -1} rr.Service = sql.NullString{String: r.Service, Valid: true} rr.ServiceId = sql.NullInt64{Int64: r.ServiceId, Valid: true} rr.Issue = sql.NullString{String: r.Issue, Valid: true} @@ -1199,7 +1199,7 @@ func (rr *RemediationRow) FromRemediation(r *entity.Remediation) { rr.RemediationDate = sql.NullTime{Time: r.RemediationDate, Valid: true} rr.ExpirationDate = sql.NullTime{Time: r.ExpirationDate, Valid: true} rr.RemediatedBy = sql.NullString{String: r.RemediatedBy, Valid: true} - rr.RemediatedById = sql.NullInt64{Int64: r.RemediatedById, Valid: true} + rr.RemediatedById = sql.NullInt64{Int64: r.RemediatedById, Valid: r.RemediatedById != -1} rr.CreatedAt = sql.NullTime{Time: r.CreatedAt, Valid: true} rr.CreatedBy = sql.NullInt64{Int64: r.CreatedBy, Valid: true} rr.DeletedAt = sql.NullTime{Time: r.DeletedAt, Valid: true} diff --git a/internal/database/mariadb/issue_test.go b/internal/database/mariadb/issue_test.go index 6c219d93..f8c412a0 100644 --- a/internal/database/mariadb/issue_test.go +++ b/internal/database/mariadb/issue_test.go @@ -1180,93 +1180,3 @@ var _ = Describe("Ordering Issues", Label("IssueOrder"), func() { }) }) - -var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { - var db *mariadb.SqlDatabase - var seeder *test.DatabaseSeeder - var seedCollection *test.SeedCollection - - var testIssueSeverityCount = func(filter *entity.IssueFilter, counts entity.IssueSeverityCounts) { - issueSeverityCounts, err := db.CountIssueRatings(filter) - - By("throwing no error", func() { - Expect(err).To(BeNil()) - }) - - By("returning the correct counts", func() { - Expect(issueSeverityCounts.Critical).To(BeEquivalentTo(counts.Critical)) - Expect(issueSeverityCounts.High).To(BeEquivalentTo(counts.High)) - Expect(issueSeverityCounts.Medium).To(BeEquivalentTo(counts.Medium)) - Expect(issueSeverityCounts.Low).To(BeEquivalentTo(counts.Low)) - Expect(issueSeverityCounts.None).To(BeEquivalentTo(counts.None)) - Expect(issueSeverityCounts.Total).To(BeEquivalentTo(counts.Total)) - }) - } - - BeforeEach(func() { - var err error - db = dbm.NewTestSchema() - seeder, err = test.NewDatabaseSeeder(dbm.DbConfig()) - Expect(err).To(BeNil(), "Database Seeder Setup should work") - seedCollection, err = seeder.SeedForIssueCounts() - Expect(err).To(BeNil()) - err = seeder.RefreshCountIssueRatings() - Expect(err).To(BeNil()) - }) - AfterEach(func() { - dbm.TestTearDown(db) - }) - - It("returns the correct count for all issues", func() { - severityCounts, err := test.LoadIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_severity.json")) - Expect(err).To(BeNil()) - testIssueSeverityCount(nil, severityCounts) - }) - It("returns the correct count for component version issues", func() { - severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) - Expect(err).To(BeNil()) - - for _, cvi := range seedCollection.ComponentVersionIssueRows { - cvId := cvi.ComponentVersionId.Int64 - filter := &entity.IssueFilter{ - ComponentVersionId: []*int64{&cvId}, - } - - strId := fmt.Sprintf("%d", cvId) - - testIssueSeverityCount(filter, severityCounts[strId]) - } - - }) - It("returns the correct count for services", func() { - severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) - Expect(err).To(BeNil()) - - for _, service := range seedCollection.ServiceRows { - serviceId := service.Id.Int64 - - filter := &entity.IssueFilter{ - ServiceId: []*int64{&serviceId}, - } - - strId := fmt.Sprintf("%d", serviceId) - - testIssueSeverityCount(filter, severityCounts[strId]) - } - }) - It("returns the correct count for supportgroup", func() { - severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) - Expect(err).To(BeNil()) - - for _, sg := range seedCollection.SupportGroupRows { - filter := &entity.IssueFilter{ - SupportGroupCCRN: []*string{&sg.CCRN.String}, - } - - strId := fmt.Sprintf("%d", sg.Id.Int64) - - testIssueSeverityCount(filter, severityCounts[strId]) - } - }) - -}) diff --git a/internal/database/mariadb/migrations/20250827115156_add_mv_count_vulnerability_component.up.sql b/internal/database/mariadb/migrations/20250827115156_add_mv_count_vulnerability_component.up.sql index de8c452e..3400528f 100644 --- a/internal/database/mariadb/migrations/20250827115156_add_mv_count_vulnerability_component.up.sql +++ b/internal/database/mariadb/migrations/20250827115156_add_mv_count_vulnerability_component.up.sql @@ -54,6 +54,7 @@ BEGIN INNER JOIN ComponentVersion AS CV ON CV.componentversion_id = CI.componentinstance_component_version_id INNER JOIN IssueVariant AS IV ON IV.issuevariant_issue_id = IM.issuematch_issue_id INNER JOIN Issue AS I ON I.issue_id = IV.issuevariant_issue_id + LEFT JOIN Remediation R ON CI.componentinstance_service_id = R.remediation_service_id AND I.issue_id = R.remediation_issue_id AND R.remediation_deleted_at IS NULL WHERE IM.issuematch_status = 'new' AND I.issue_type = 'Vulnerability' @@ -63,6 +64,7 @@ BEGIN AND I.issue_deleted_at IS NULL AND CI.componentinstance_deleted_at IS NULL AND CV.componentversion_deleted_at IS NULL + AND (R.remediation_id IS NULL OR R.remediation_expiration_date < CURDATE()) GROUP BY CI.componentinstance_service_id, CV.componentversion_component_id; @@ -112,6 +114,7 @@ BEGIN ON IV.issuevariant_issue_id = IM.issuematch_issue_id INNER JOIN Issue AS I ON I.issue_id = IV.issuevariant_issue_id + LEFT JOIN Remediation R ON CI.componentinstance_service_id = R.remediation_service_id AND I.issue_id = R.remediation_issue_id AND R.remediation_deleted_at IS NULL WHERE IM.issuematch_status = 'new' AND I.issue_type = 'Vulnerability' @@ -120,6 +123,7 @@ BEGIN AND I.issue_deleted_at IS NULL AND CI.componentinstance_deleted_at IS NULL AND CV.componentversion_deleted_at IS NULL + AND (R.remediation_id IS NULL OR R.remediation_expiration_date < CURDATE()) GROUP BY CI.componentinstance_service_id; diff --git a/internal/database/mariadb/migrations/20251031095712_mv_counters_replace_enum_rows_with_columns.up.sql b/internal/database/mariadb/migrations/20251031095712_mv_counters_replace_enum_rows_with_columns.up.sql index b2821889..7b5a1f79 100644 --- a/internal/database/mariadb/migrations/20251031095712_mv_counters_replace_enum_rows_with_columns.up.sql +++ b/internal/database/mariadb/migrations/20251031095712_mv_counters_replace_enum_rows_with_columns.up.sql @@ -94,7 +94,9 @@ BEGIN LEFT JOIN Service S ON S.service_id = CI.componentinstance_service_id LEFT JOIN SupportGroupService SGS ON SGS.supportgroupservice_service_id = CI.componentinstance_service_id LEFT JOIN SupportGroup SG ON SGS.supportgroupservice_support_group_id = SG.supportgroup_id + LEFT JOIN Remediation R ON S.service_id = R.remediation_service_id AND I.issue_id = R.remediation_issue_id AND R.remediation_deleted_at IS NULL WHERE I.issue_deleted_at IS NULL + AND (R.remediation_id IS NULL OR R.remediation_expiration_date < CURDATE()) GROUP BY SG.supportgroup_ccrn; END; @@ -143,7 +145,9 @@ BEGIN LEFT JOIN ComponentInstance CI ON CI.componentinstance_id = IM.issuematch_component_instance_id LEFT JOIN ComponentVersion CV ON CI.componentinstance_component_version_id = CV.componentversion_id LEFT JOIN Service S ON S.service_id = CI.componentinstance_service_id - WHERE I.issue_deleted_at IS NULL; + LEFT JOIN Remediation R ON S.service_id = R.remediation_service_id AND I.issue_id = R.remediation_issue_id AND R.remediation_deleted_at IS NULL + WHERE I.issue_deleted_at IS NULL + AND (R.remediation_id IS NULL OR R.remediation_expiration_date < CURDATE()); END; -- @@ -204,7 +208,11 @@ BEGIN LEFT JOIN ComponentInstance CI ON CI.componentinstance_id = IM.issuematch_component_instance_id LEFT JOIN SupportGroupService SGS ON SGS.supportgroupservice_service_id = CI.componentinstance_service_id LEFT JOIN SupportGroup SG ON SGS.supportgroupservice_support_group_id = SG.supportgroup_id + LEFT JOIN Remediation R ON SGS.supportgroupservice_service_id = R.remediation_service_id AND I.issue_id = R.remediation_issue_id AND R.remediation_deleted_at IS NULL WHERE I.issue_deleted_at IS NULL + AND CI.componentinstance_deleted_at IS NULL + -- Count only non-remediated or with expired remediation + AND (R.remediation_id IS NULL OR R.remediation_expiration_date < CURDATE()) GROUP BY SG.supportgroup_ccrn; END; @@ -225,6 +233,7 @@ SET critical_count = CASE WHEN issue_value = 'Critical' THEN issue_count ELSE 0 none_count = CASE WHEN issue_value = 'None' THEN issue_count ELSE 0 END; ALTER TABLE mvCountIssueRatingsComponentVersion +ADD COLUMN service_id INT DEFAULT NULL, DROP COLUMN issue_value, DROP COLUMN issue_count, ADD COLUMN issue_count INT GENERATED ALWAYS AS ( @@ -237,6 +246,7 @@ BEGIN TRUNCATE TABLE mvCountIssueRatingsComponentVersion; INSERT INTO mvCountIssueRatingsComponentVersion ( component_version_id, + service_id, critical_count, high_count, medium_count, @@ -245,15 +255,19 @@ BEGIN ) SELECT CVI.componentversionissue_component_version_id AS component_version_id, + CI.componentinstance_service_id AS service_id, COUNT(DISTINCT CASE WHEN IV.issuevariant_rating = 'Critical' THEN CONCAT(CVI.componentversionissue_component_version_id, ',', CVI.componentversionissue_issue_id) END) AS critical_count, COUNT(DISTINCT CASE WHEN IV.issuevariant_rating = 'High' THEN CONCAT(CVI.componentversionissue_component_version_id, ',', CVI.componentversionissue_issue_id) END) AS high_count, COUNT(DISTINCT CASE WHEN IV.issuevariant_rating = 'Medium' THEN CONCAT(CVI.componentversionissue_component_version_id, ',', CVI.componentversionissue_issue_id) END) AS medium_count, COUNT(DISTINCT CASE WHEN IV.issuevariant_rating = 'Low' THEN CONCAT(CVI.componentversionissue_component_version_id, ',', CVI.componentversionissue_issue_id) END) AS low_count, COUNT(DISTINCT CASE WHEN IV.issuevariant_rating = 'None' THEN CONCAT(CVI.componentversionissue_component_version_id, ',', CVI.componentversionissue_issue_id) END) AS none_count - FROM Issue I - LEFT JOIN IssueVariant IV ON IV.issuevariant_issue_id = I.issue_id - LEFT JOIN ComponentVersionIssue CVI ON I.issue_id = CVI.componentversionissue_issue_id - WHERE I.issue_deleted_at IS NULL + FROM ComponentVersionIssue CVI + LEFT JOIN IssueVariant IV ON IV.issuevariant_issue_id = CVI.componentversionissue_issue_id + INNER JOIN ComponentInstance CI ON CVI.componentversionissue_component_version_id = CI.componentinstance_component_version_id + LEFT JOIN Remediation R ON CI.componentinstance_service_id = R.remediation_service_id AND CVI.componentversionissue_issue_id = R.remediation_issue_id AND R.remediation_deleted_at IS NULL + WHERE + -- Count only non-remediated or with expired remediation + (R.remediation_id IS NULL OR R.remediation_expiration_date < CURDATE()) GROUP BY CVI.componentversionissue_component_version_id; END; @@ -302,8 +316,11 @@ BEGIN FROM Issue I LEFT JOIN IssueVariant IV ON IV.issuevariant_issue_id = I.issue_id LEFT JOIN IssueMatch IM ON I.issue_id = IM.issuematch_issue_id - LEFT JOIN ComponentInstance CI ON CI.componentinstance_id = IM.issuematch_component_instance_id + LEFT JOIN ComponentInstance CI ON CI.componentinstance_id = IM.issuematch_component_instance_id AND CI.componentinstance_deleted_at IS NULL + LEFT JOIN Remediation R ON CI.componentinstance_service_id = R.remediation_service_id AND I.issue_id = R.remediation_issue_id AND R.remediation_deleted_at IS NULL WHERE I.issue_deleted_at IS NULL + -- Count only non-remediated or with expired remediation + AND (R.remediation_id IS NULL OR R.remediation_expiration_date < CURDATE()) GROUP BY CI.componentinstance_service_id; END; diff --git a/internal/database/mariadb/mv_vulnerabilities.go b/internal/database/mariadb/mv_vulnerabilities.go index 9ea4409f..1009f851 100644 --- a/internal/database/mariadb/mv_vulnerabilities.go +++ b/internal/database/mariadb/mv_vulnerabilities.go @@ -10,172 +10,77 @@ import ( "github.com/sirupsen/logrus" ) -func getMvCountIssueRatingsJoin(filter *entity.IssueFilter) string { +func getCountTable(filter *entity.IssueFilter) string { if filter.AllServices && filter.Unique { - // Conunt unique issues. AllServices filter is set, so we count issues that are matched to a service - // COUNT(distinct IV.issuevariant_issue_id) - // LEFT JOIN mvCountIssueRatingsUniqueService CIR ON IV.issuevariant_rating = CIR.issue_value - return ` - LEFT JOIN ( - SELECT 'Critical' AS issue_value, critical_count AS issue_count FROM mvCountIssueRatingsUniqueService - UNION ALL - SELECT 'High', high_count FROM mvCountIssueRatingsUniqueService - UNION ALL - SELECT 'Medium', medium_count FROM mvCountIssueRatingsUniqueService - UNION ALL - SELECT 'Low', low_count FROM mvCountIssueRatingsUniqueService - UNION ALL - SELECT 'None', none_count FROM mvCountIssueRatingsUniqueService - ) CIR ON IV.issuevariant_rating = CIR.issue_value - ` + // Total count of unique issues + return "mvCountIssueRatingsUniqueService" } else if filter.AllServices { - // Count issues that appear in multiple services and in multiple component versions per service - //COUNT(distinct CONCAT(CI.componentinstance_component_version_id, ',', I.issue_id, ',', S.service_id)) - // LEFT JOIN mvCountIssueRatingsService CIR ON SG.supportgroup_ccrn = CIR.supportgroup_ccrn - // AND IV.issuevariant_rating = CIR.issue_value if len(filter.SupportGroupCCRN) > 0 { - return ` - LEFT JOIN ( - SELECT supportgroup_ccrn, critical_count AS issue_count, 'Critical' AS issue_value FROM mvCountIssueRatingsService - UNION ALL - SELECT supportgroup_ccrn, high_count, 'High' FROM mvCountIssueRatingsService - UNION ALL - SELECT supportgroup_ccrn, medium_count, 'Medium' FROM mvCountIssueRatingsService - UNION ALL - SELECT supportgroup_ccrn, low_count, 'Low' FROM mvCountIssueRatingsService - UNION ALL - SELECT supportgroup_ccrn, none_count, 'None' FROM mvCountIssueRatingsService - ) CIR - ON SG.supportgroup_ccrn = CIR.supportgroup_ccrn - AND IV.issuevariant_rating = CIR.issue_value - ` + // total count of issues in support group across all services + // service list view total count with support group filter + return "mvCountIssueRatingsService" } else { - // call/branch can be replaced with (something to consider): - // SELECT issue_value, issue_count - // FROM mvCountIssueRatingsServiceWithoutSupportGroup - // ORDER BY issue_value ASC; - // LEFT JOIN mvCountIssueRatingsServiceWithoutSupportGroup CIR ON IV.issuevariant_rating = CIR.issue_value - return ` - LEFT JOIN ( - SELECT critical_count AS issue_count, 'Critical' AS issue_value FROM mvCountIssueRatingsServiceWithoutSupportGroup - UNION ALL - SELECT high_count, 'High' FROM mvCountIssueRatingsServiceWithoutSupportGroup - UNION ALL - SELECT medium_count, 'Medium' FROM mvCountIssueRatingsServiceWithoutSupportGroup - UNION ALL - SELECT low_count, 'Low' FROM mvCountIssueRatingsServiceWithoutSupportGroup - UNION ALL - SELECT none_count, 'None' FROM mvCountIssueRatingsServiceWithoutSupportGroup - ) CIR - ON IV.issuevariant_rating = CIR.issue_value - ` + // total count of issues in all services (across all support groups) + // service list view total count without support group filter + return "mvCountIssueRatingsServiceWithoutSupportGroup" } } else if len(filter.SupportGroupCCRN) > 0 { - // Count issues that appear in multiple support groups - // COUNT(distinct CONCAT(CI.componentinstance_component_version_id, ',', I.issue_id, ',', SGS.supportgroupservice_service_id, ',', SG.supportgroup_id)) - // LEFT JOIN mvCountIssueRatingsSupportGroup CIR ON SG.supportgroup_ccrn = CIR.supportgroup_ccrn - // AND IV.issuevariant_rating = CIR.issue_value - return ` - LEFT JOIN ( - SELECT supportgroup_ccrn, critical_count AS issue_count, 'Critical' AS issue_value FROM mvCountIssueRatingsSupportGroup - UNION ALL - SELECT supportgroup_ccrn, high_count, 'High' FROM mvCountIssueRatingsSupportGroup - UNION ALL - SELECT supportgroup_ccrn, medium_count, 'Medium' FROM mvCountIssueRatingsSupportGroup - UNION ALL - SELECT supportgroup_ccrn, low_count, 'Low' FROM mvCountIssueRatingsSupportGroup - UNION ALL - SELECT supportgroup_ccrn, none_count, 'None' FROM mvCountIssueRatingsSupportGroup - ) CIR - ON SG.supportgroup_ccrn = CIR.supportgroup_ccrn - AND IV.issuevariant_rating = CIR.issue_value - ` + // Count issues in a support group + return "mvCountIssueRatingsSupportGroup" } else if len(filter.ComponentVersionId) > 0 { - // Count issues that appear in multiple component versions - // COUNT(DISTINCT CONCAT(CVI.componentversionissue_component_version_id, ',', CVI.componentversionissue_issue_id)) " - // LEFT JOIN mvCountIssueRatingsComponentVersion CIR ON CVI.componentversionissue_component_version_id = CIR.component_version_id - // AND IV.issuevariant_rating = CIR.issue_value - return ` - LEFT JOIN ( - SELECT component_version_id, critical_count AS issue_count, 'Critical' AS issue_value FROM mvCountIssueRatingsComponentVersion - UNION ALL - SELECT component_version_id, high_count, 'High' FROM mvCountIssueRatingsComponentVersion - UNION ALL - SELECT component_version_id, medium_count, 'Medium' FROM mvCountIssueRatingsComponentVersion - UNION ALL - SELECT component_version_id, low_count, 'Low' FROM mvCountIssueRatingsComponentVersion - UNION ALL - SELECT component_version_id, none_count, 'None' FROM mvCountIssueRatingsComponentVersion - ) CIR - ON CVI.componentversionissue_component_version_id = CIR.component_version_id - AND IV.issuevariant_rating = CIR.issue_value - ` + // Count issues in a component version of a *service* + return "mvCountIssueRatingsComponentVersion" } else if len(filter.ServiceCCRN) > 0 || len(filter.ServiceId) > 0 { - // COUNT(distinct CONCAT(CI.componentinstance_component_version_id, ',', I.issue_id)) - // LEFT JOIN mvCountIssueRatingsServiceId CIR ON CI.componentinstance_service_id = CIR.service_id - // AND IV.issuevariant_rating = CIR.issue_value - return ` - LEFT JOIN ( - SELECT service_id, critical_count AS issue_count, 'Critical' AS issue_value FROM mvCountIssueRatingsServiceId - UNION ALL - SELECT service_id, high_count, 'High' FROM mvCountIssueRatingsServiceId - UNION ALL - SELECT service_id, medium_count, 'Medium' FROM mvCountIssueRatingsServiceId - UNION ALL - SELECT service_id, low_count, 'Low' FROM mvCountIssueRatingsServiceId - UNION ALL - SELECT service_id, none_count, 'None' FROM mvCountIssueRatingsServiceId - ) CIR - ON CI.componentinstance_service_id = CIR.service_id - AND IV.issuevariant_rating = CIR.issue_value - ` + // Count issues that appear in single service + return "mvCountIssueRatingsServiceId" } else { - // COUNT(distinct IV.issuevariant_issue_id) - // LEFT JOIN mvCountIssueRatingsOther CIR ON IV.issuevariant_rating = CIR.issue_value - return ` - LEFT JOIN ( - SELECT critical_count AS issue_count, 'Critical' AS issue_value FROM mvCountIssueRatingsOther - UNION ALL - SELECT high_count, 'High' FROM mvCountIssueRatingsOther - UNION ALL - SELECT medium_count, 'Medium' FROM mvCountIssueRatingsOther - UNION ALL - SELECT low_count, 'Low' FROM mvCountIssueRatingsOther - UNION ALL - SELECT none_count, 'None' FROM mvCountIssueRatingsOther - ) CIR - ON IV.issuevariant_rating = CIR.issue_value - ` + // Total count of issues + return "mvCountIssueRatingsOther" } } -func getIssueJoinsWithMvCountIssueRatingsJoin(filter *entity.IssueFilter, order []entity.Order) string { - joins := getIssueJoins(filter, order) - joins = fmt.Sprintf("%s\n%s", joins, getMvCountIssueRatingsJoin(filter)) - return joins -} +func (s *SqlDatabase) CountIssueRatings(filter *entity.IssueFilter) (*entity.IssueSeverityCounts, error) { + l := logrus.WithFields(logrus.Fields{ + "event": "database.CountIssueRatings", + }) + var fl []string + var filterParameters []any -func getIssueQueryWithMvCountIssueRatingsJoin(baseQuery string, order []entity.Order, filter *entity.IssueFilter) string { - issueColumns := getIssueColumns(order) - defaultOrder := GetDefaultOrder(order, entity.IssueId, entity.OrderDirectionAsc) - joins := getIssueJoinsWithMvCountIssueRatingsJoin(filter, order) - whereClause := getIssueFilterWhereClause(filter) - orderStr := CreateOrderString(defaultOrder) - return fmt.Sprintf(baseQuery, issueColumns, joins, whereClause, orderStr) -} + filter = s.ensureIssueFilter(filter) -func (s *SqlDatabase) buildIssueStatementWithMvCountIssueRatingsJoin(baseQuery string, filter *entity.IssueFilter, order []entity.Order, l *logrus.Entry) (Stmt, []interface{}, error) { - ifilter := s.ensureIssueFilter(filter) - l.WithFields(logrus.Fields{"filter": ifilter}) + baseQuery := ` + SELECT CIR.critical_count, CIR.high_count, CIR.medium_count, CIR.low_count, CIR.none_count FROM %s AS CIR + ` - cursorFields, err := DecodeCursor(ifilter.PaginatedX.After) - if err != nil { - return nil, nil, err + tableName := getCountTable(filter) + + query := fmt.Sprintf(baseQuery, tableName) + + if len(filter.ServiceId) > 0 { + filterParameters = buildQueryParameters(filterParameters, filter.ServiceId) + fl = append(fl, buildFilterQuery(filter.ServiceId, "CIR.service_id = ?", OP_OR)) + } + + if len(filter.ServiceCCRN) > 0 { + filterParameters = buildQueryParameters(filterParameters, filter.ServiceCCRN) + fl = append(fl, buildFilterQuery(filter.ServiceCCRN, "CIR.service_ccrn = ?", OP_OR)) + } + + if len(filter.ComponentVersionId) > 0 { + filterParameters = buildQueryParameters(filterParameters, filter.ComponentVersionId) + fl = append(fl, buildFilterQuery(filter.ComponentVersionId, "CIR.component_version_id = ?", OP_OR)) } - query := getIssueQueryWithMvCountIssueRatingsJoin(baseQuery, order, ifilter) + if len(filter.SupportGroupCCRN) > 0 { + filterParameters = buildQueryParameters(filterParameters, filter.SupportGroupCCRN) + fl = append(fl, buildFilterQuery(filter.SupportGroupCCRN, "CIR.supportgroup_ccrn = ?", OP_OR)) + } + + filterStr := combineFilterQueries(fl, OP_AND) + if filterStr != "" { + query = fmt.Sprintf("%s WHERE %s", query, filterStr) + } - //construct prepared statement and if where clause does exist add parameters stmt, err := s.db.Preparex(query) if err != nil { msg := ERROR_MSG_PREPARED_STMT @@ -185,38 +90,7 @@ func (s *SqlDatabase) buildIssueStatementWithMvCountIssueRatingsJoin(baseQuery s "query": query, "stmt": stmt, }).Error(msg) - return nil, nil, fmt.Errorf("%s", msg) - } - - //adding parameters - filterParameters := s.buildIssueFilterParameters(ifilter, cursorFields) - - return stmt, filterParameters, nil -} - -func (s *SqlDatabase) CountIssueRatings(filter *entity.IssueFilter) (*entity.IssueSeverityCounts, error) { - l := logrus.WithFields(logrus.Fields{ - "event": "database.CountIssueRatings", - }) - - filter = s.ensureIssueFilter(filter) - - baseQuery := ` - SELECT IV.issuevariant_rating AS issue_value, CIR.issue_count AS issue_count FROM %s Issue I - %s - %s - %s - GROUP BY IV.issuevariant_rating ORDER BY %s - ` - - if len(filter.IssueRepositoryId) == 0 { - baseQuery = fmt.Sprintf(baseQuery, "%s", "LEFT JOIN IssueVariant IV ON IV.issuevariant_issue_id = I.issue_id", "%s", "%s", "%s") - } - - stmt, filterParameters, err := s.buildIssueStatementWithMvCountIssueRatingsJoin(baseQuery, filter, []entity.Order{}, l) - - if err != nil { - return nil, err + return nil, fmt.Errorf("%s", msg) } defer stmt.Close() @@ -225,8 +99,8 @@ func (s *SqlDatabase) CountIssueRatings(filter *entity.IssueFilter) (*entity.Iss stmt, filterParameters, l, - func(l []entity.IssueCount, e IssueCountRow) []entity.IssueCount { - return append(l, e.AsIssueCount()) + func(l []entity.IssueSeverityCounts, e RatingCount) []entity.IssueSeverityCounts { + return append(l, e.AsIssueSeverityCounts()) }, ) @@ -234,21 +108,15 @@ func (s *SqlDatabase) CountIssueRatings(filter *entity.IssueFilter) (*entity.Iss return nil, err } - var issueSeverityCounts entity.IssueSeverityCounts - for _, count := range counts { - switch count.Value { - case entity.SeverityValuesCritical.String(): - issueSeverityCounts.Critical = count.Count - case entity.SeverityValuesHigh.String(): - issueSeverityCounts.High = count.Count - case entity.SeverityValuesMedium.String(): - issueSeverityCounts.Medium = count.Count - case entity.SeverityValuesLow.String(): - issueSeverityCounts.Low = count.Count - case entity.SeverityValuesNone.String(): - issueSeverityCounts.None = count.Count - } - issueSeverityCounts.Total += count.Count + if len(counts) == 0 { + return &entity.IssueSeverityCounts{ + Critical: 0, + High: 0, + Medium: 0, + Low: 0, + None: 0, + }, nil } - return &issueSeverityCounts, nil + + return &counts[0], err } diff --git a/internal/database/mariadb/mv_vulnerabilities_test.go b/internal/database/mariadb/mv_vulnerabilities_test.go new file mode 100644 index 00000000..ab6da906 --- /dev/null +++ b/internal/database/mariadb/mv_vulnerabilities_test.go @@ -0,0 +1,331 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and Greenhouse contributors +// SPDX-License-Identifier: Apache-2.0 + +package mariadb_test + +import ( + "database/sql" + "fmt" + "time" + + "github.com/cloudoperators/heureka/internal/database/mariadb" + "github.com/cloudoperators/heureka/internal/database/mariadb/common" + "github.com/cloudoperators/heureka/internal/database/mariadb/test" + "github.com/cloudoperators/heureka/internal/entity" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { + var db *mariadb.SqlDatabase + var seeder *test.DatabaseSeeder + var seedCollection *test.SeedCollection + + var testIssueSeverityCount = func(filter *entity.IssueFilter, counts entity.IssueSeverityCounts) { + issueSeverityCounts, err := db.CountIssueRatings(filter) + + By("throwing no error", func() { + Expect(err).To(BeNil()) + }) + + By("returning the correct counts", func() { + Expect(issueSeverityCounts.Critical).To(BeEquivalentTo(counts.Critical)) + Expect(issueSeverityCounts.High).To(BeEquivalentTo(counts.High)) + Expect(issueSeverityCounts.Medium).To(BeEquivalentTo(counts.Medium)) + Expect(issueSeverityCounts.Low).To(BeEquivalentTo(counts.Low)) + Expect(issueSeverityCounts.None).To(BeEquivalentTo(counts.None)) + Expect(issueSeverityCounts.Total).To(BeEquivalentTo(counts.Total)) + }) + } + + var testServices = func(counts map[string]entity.IssueSeverityCounts) { + for _, service := range seedCollection.ServiceRows { + serviceId := service.Id.Int64 + + filter := &entity.IssueFilter{ + ServiceId: []*int64{&serviceId}, + } + + strId := fmt.Sprintf("%d", serviceId) + + testIssueSeverityCount(filter, counts[strId]) + } + } + + var testComponentVersions = func(counts map[string]entity.IssueSeverityCounts) { + for _, cvi := range seedCollection.ComponentVersionIssueRows { + cvId := cvi.ComponentVersionId.Int64 + filter := &entity.IssueFilter{ + ComponentVersionId: []*int64{&cvId}, + } + + strId := fmt.Sprintf("%d", cvId) + + testIssueSeverityCount(filter, counts[strId]) + } + } + + var testSupportGroups = func(counts map[string]entity.IssueSeverityCounts) { + for _, sg := range seedCollection.SupportGroupRows { + filter := &entity.IssueFilter{ + SupportGroupCCRN: []*string{&sg.CCRN.String}, + } + + strId := fmt.Sprintf("%d", sg.Id.Int64) + + testIssueSeverityCount(filter, counts[strId]) + } + } + + var insertRemediation = func(serviceRow *mariadb.BaseServiceRow, componentRow *mariadb.ComponentRow, expirationDate time.Time) *entity.Remediation { + remediation := test.NewFakeRemediation() + if serviceRow != nil { + remediation.ServiceId = serviceRow.Id + remediation.Service = serviceRow.CCRN + } + if componentRow != nil { + remediation.ComponentId = componentRow.Id + remediation.Component = componentRow.CCRN + } + remediation.IssueId = seedCollection.IssueRows[0].Id + remediation.Issue = seedCollection.IssueRows[0].PrimaryName + remediation.ExpirationDate = sql.NullTime{Time: expirationDate, Valid: true} + r := remediation.AsRemediation() + newRemediation, err := db.CreateRemediation(&r) + Expect(err).To(BeNil()) + return newRemediation + } + + BeforeEach(func() { + var err error + db = dbm.NewTestSchema() + seeder, err = test.NewDatabaseSeeder(dbm.DbConfig()) + Expect(err).To(BeNil(), "Database Seeder Setup should work") + seedCollection, err = seeder.SeedForIssueCounts() + Expect(err).To(BeNil()) + err = seeder.RefreshCountIssueRatings() + Expect(err).To(BeNil()) + }) + AfterEach(func() { + dbm.TestTearDown(db) + }) + + When("there are no remediations", Label("NoRemediations"), func() { + It("returns the correct count for all issues", func() { + severityCounts, err := test.LoadIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_severity.json")) + Expect(err).To(BeNil()) + testIssueSeverityCount(nil, severityCounts) + }) + It("returns the correct count for component version issues", func() { + severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) + Expect(err).To(BeNil()) + + testComponentVersions(severityCounts) + }) + It("returns the correct count for services", func() { + severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) + Expect(err).To(BeNil()) + + testServices(severityCounts) + }) + It("returns the correct count for supportgroup", func() { + severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) + Expect(err).To(BeNil()) + + testSupportGroups(severityCounts) + }) + }) + When("there is an expired remediation for a service", Label("WithRemediations"), func() { + BeforeEach(func() { + expirationDate := time.Now().Add(-10 * 24 * time.Hour) + insertRemediation(&seedCollection.ServiceRows[0], nil, expirationDate) + seeder.RefreshCountIssueRatings() + + }) + It("returns the correct count for component version issues", func() { + severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) + Expect(err).To(BeNil()) + + testComponentVersions(severityCounts) + }) + It("returns the correct count for services", func() { + seeder.RefreshCountIssueRatings() + severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) + Expect(err).To(BeNil()) + + testServices(severityCounts) + }) + It("returns the correct count for supportgroup", func() { + severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) + Expect(err).To(BeNil()) + + testSupportGroups(severityCounts) + }) + }) + When("there is an active remediation for a service", Label("WithRemediations"), func() { + var serviceCounts entity.IssueSeverityCounts + var serviceId string + var remediation *entity.Remediation + BeforeEach(func() { + expirationDate := time.Now().Add(10 * 24 * time.Hour) + remediation = insertRemediation(&seedCollection.ServiceRows[0], nil, expirationDate) + seeder.RefreshCountIssueRatings() + // remediation for previously critical issue + serviceCounts = entity.IssueSeverityCounts{ + Critical: 0, + High: 0, + Medium: 0, + Low: 1, + None: 0, + Total: 1, + } + serviceId = fmt.Sprintf("%d", remediation.ServiceId) + }) + It("returns the correct count for component version issues", func() { + severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) + severityCounts["1"] = entity.IssueSeverityCounts{} + Expect(err).To(BeNil()) + + testComponentVersions(severityCounts) + }) + It("return the correct count for component version used in two services", func() { + severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) + Expect(err).To(BeNil()) + + cv := seedCollection.ComponentVersionRows[0] + // create new component instance with component version that has been remediated in another service + ci := test.NewFakeComponentInstance() + ci.ComponentVersionId = seedCollection.ComponentVersionRows[0].Id + ci.ServiceId = seedCollection.ServiceRows[3].Id + componentInstance := ci.AsComponentInstance() + newCi, err := db.CreateComponentInstance(&componentInstance) + Expect(err).To(BeNil()) + seeder.RefreshCountIssueRatings() + + counts, err := db.CountIssueRatings(&entity.IssueFilter{ + ComponentVersionId: []*int64{&cv.Id.Int64}, + ServiceId: []*int64{&remediation.ServiceId}, + }) + + Expect(err).To(BeNil()) + + Expect(counts.Critical).To(BeEquivalentTo(0)) + Expect(counts.High).To(BeEquivalentTo(0)) + Expect(counts.Medium).To(BeEquivalentTo(0)) + Expect(counts.Low).To(BeEquivalentTo(0)) + Expect(counts.None).To(BeEquivalentTo(0)) + Expect(counts.Total).To(BeEquivalentTo(0)) + + countsEmpty, err := db.CountIssueRatings(&entity.IssueFilter{ + ComponentVersionId: []*int64{&cv.Id.Int64}, + ServiceId: []*int64{&newCi.ServiceId}, + }) + Expect(err).To(BeNil()) + + cvId := fmt.Sprintf("%d", cv.Id.Int64) + Expect(countsEmpty.Critical).To(BeEquivalentTo(severityCounts[cvId].Critical)) + Expect(countsEmpty.High).To(BeEquivalentTo(severityCounts[cvId].High)) + Expect(countsEmpty.Medium).To(BeEquivalentTo(severityCounts[cvId].Medium)) + Expect(countsEmpty.Low).To(BeEquivalentTo(severityCounts[cvId].Low)) + Expect(countsEmpty.None).To(BeEquivalentTo(severityCounts[cvId].None)) + Expect(countsEmpty.Total).To(BeEquivalentTo(severityCounts[cvId].Total)) + }) + It("returns the correct count for services", func() { + seeder.RefreshCountIssueRatings() + severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) + Expect(err).To(BeNil()) + severityCounts[serviceId] = serviceCounts + + testServices(severityCounts) + }) + It("returns the correct count for supportgroup", func() { + severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) + severityCounts["1"] = entity.IssueSeverityCounts{ + Critical: 1, + Medium: 1, + Low: 2, + None: 1, + Total: 5, + } + Expect(err).To(BeNil()) + + testSupportGroups(severityCounts) + }) + }) + When("there is a deleted remediation for a service", Label("WithRemediations"), func() { + BeforeEach(func() { + expirationDate := time.Now().Add(10 * 24 * time.Hour) + createdRemediation := insertRemediation(&seedCollection.ServiceRows[0], nil, expirationDate) + err := db.DeleteRemediation(createdRemediation.Id, common.SystemUserId) + Expect(err).To(BeNil()) + err = seeder.RefreshCountIssueRatings() + Expect(err).To(BeNil()) + }) + It("returns the correct count for component version issues", func() { + severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) + Expect(err).To(BeNil()) + + testComponentVersions(severityCounts) + }) + It("returns the correct count for services", func() { + severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) + Expect(err).To(BeNil()) + + testServices(severityCounts) + }) + It("returns the correct count for supportgroup", func() { + severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) + Expect(err).To(BeNil()) + + testSupportGroups(severityCounts) + }) + }) + When("there is an active remediation for a component in a service", Label("WithRemediations"), func() { + var serviceCounts entity.IssueSeverityCounts + var serviceId string + BeforeEach(func() { + expirationDate := time.Now().Add(10 * 24 * time.Hour) + r := insertRemediation(&seedCollection.ServiceRows[0], &seedCollection.ComponentRows[0], expirationDate) + seeder.RefreshCountIssueRatings() + // remediation for previously critical issue + serviceCounts = entity.IssueSeverityCounts{ + Critical: 0, + High: 0, + Medium: 0, + Low: 1, + None: 0, + Total: 1, + } + serviceId = fmt.Sprintf("%d", r.ServiceId) + }) + It("returns the correct count for component version issues", func() { + severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) + severityCounts["1"] = entity.IssueSeverityCounts{} + Expect(err).To(BeNil()) + + testComponentVersions(severityCounts) + }) + It("returns the correct count for services", func() { + seeder.RefreshCountIssueRatings() + severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) + Expect(err).To(BeNil()) + severityCounts[serviceId] = serviceCounts + + testServices(severityCounts) + }) + It("returns the correct count for supportgroup", func() { + severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) + severityCounts["1"] = entity.IssueSeverityCounts{ + Critical: 1, + Medium: 1, + Low: 2, + None: 1, + Total: 5, + } + Expect(err).To(BeNil()) + + testSupportGroups(severityCounts) + }) + }) + +}) diff --git a/internal/database/mariadb/test/fixture.go b/internal/database/mariadb/test/fixture.go index 1c785788..36160413 100644 --- a/internal/database/mariadb/test/fixture.go +++ b/internal/database/mariadb/test/fixture.go @@ -474,7 +474,7 @@ func (s *DatabaseSeeder) SeedForIssueCounts() (*SeedCollection, error) { issueRepositories := s.SeedIssueRepositories() supportGroups := s.SeedSupportGroups(2) issues := s.SeedIssues(10) - components := s.SeedComponents(1) + components := s.SeedComponents(2) componentVersions := s.SeedComponentVersions(10, components) services := s.SeedServices(5) issueVariants, err := LoadIssueVariants(GetTestDataPath("../testdata/component_version_order/issue_variant.json")) @@ -969,22 +969,13 @@ func (s *DatabaseSeeder) SeedRemediations(num int, services []mariadb.BaseServic issue := issues[rand.Intn(len(issues))] component := components[rand.Intn(len(components))] // does not check if relation exists - r := mariadb.RemediationRow{ - Description: sql.NullString{String: gofakeit.Sentence(10), Valid: true}, - RemediationDate: sql.NullTime{Time: time.Now().AddDate(0, 0, rand.Intn(30)), Valid: true}, - ExpirationDate: sql.NullTime{Time: time.Now().AddDate(0, 1, rand.Intn(30)), Valid: true}, - Type: sql.NullString{String: entity.AllRemediationTypes[rand.Intn(len(entity.AllRemediationTypes))], Valid: true}, - ServiceId: service.Id, - Service: service.CCRN, - ComponentId: component.Id, - Component: component.CCRN, - IssueId: issue.Id, - Issue: issue.PrimaryName, - RemediatedBy: sql.NullString{String: gofakeit.Name(), Valid: true}, - RemediatedById: sql.NullInt64{Int64: e2e_common.SystemUserId, Valid: true}, - CreatedBy: sql.NullInt64{Int64: e2e_common.SystemUserId, Valid: true}, - UpdatedBy: sql.NullInt64{Int64: e2e_common.SystemUserId, Valid: true}, - } + r := NewFakeRemediation() + r.ComponentId = component.Id + r.Component = component.CCRN + r.ServiceId = service.Id + r.Service = service.CCRN + r.IssueId = issue.Id + r.Issue = issue.PrimaryName id, err := s.InsertFakeRemediation(r) r.Id = sql.NullInt64{Int64: id, Valid: true} if err != nil { @@ -1676,6 +1667,17 @@ func NewFakeIssueMatchChange() mariadb.IssueMatchChangeRow { } } +func NewFakeRemediation() mariadb.RemediationRow { + return mariadb.RemediationRow{ + Description: sql.NullString{String: gofakeit.Sentence(10), Valid: true}, + RemediationDate: sql.NullTime{Time: gofakeit.Date(), Valid: true}, + ExpirationDate: sql.NullTime{Time: gofakeit.Date(), Valid: true}, + Type: sql.NullString{String: "false_positive", Valid: true}, + CreatedBy: sql.NullInt64{Int64: e2e_common.SystemUserId, Valid: true}, + UpdatedBy: sql.NullInt64{Int64: e2e_common.SystemUserId, Valid: true}, + } +} + func (s *DatabaseSeeder) SeedRealSupportGroups() map[string]mariadb.SupportGroupRow { supportGroups := map[string]mariadb.SupportGroupRow{} From 7a76722546b340f6749db0693851bec8c97d53c4 Mon Sep 17 00:00:00 2001 From: Michael Reimsbach Date: Mon, 8 Dec 2025 14:45:19 +0100 Subject: [PATCH 2/7] feat(remediation): add tests !changeexistingmigration --- ...ters_replace_enum_rows_with_columns.up.sql | 1 - .../mariadb/mv_vulnerabilities_test.go | 207 +++++++++++++----- 2 files changed, 149 insertions(+), 59 deletions(-) diff --git a/internal/database/mariadb/migrations/20251031095712_mv_counters_replace_enum_rows_with_columns.up.sql b/internal/database/mariadb/migrations/20251031095712_mv_counters_replace_enum_rows_with_columns.up.sql index 7b5a1f79..41dd072d 100644 --- a/internal/database/mariadb/migrations/20251031095712_mv_counters_replace_enum_rows_with_columns.up.sql +++ b/internal/database/mariadb/migrations/20251031095712_mv_counters_replace_enum_rows_with_columns.up.sql @@ -90,7 +90,6 @@ BEGIN LEFT JOIN IssueVariant IV ON IV.issuevariant_issue_id = I.issue_id RIGHT JOIN IssueMatch IM ON I.issue_id = IM.issuematch_issue_id LEFT JOIN ComponentInstance CI ON CI.componentinstance_id = IM.issuematch_component_instance_id - LEFT JOIN ComponentVersion CV ON CI.componentinstance_component_version_id = CV.componentversion_id LEFT JOIN Service S ON S.service_id = CI.componentinstance_service_id LEFT JOIN SupportGroupService SGS ON SGS.supportgroupservice_service_id = CI.componentinstance_service_id LEFT JOIN SupportGroup SG ON SGS.supportgroupservice_support_group_id = SG.supportgroup_id diff --git a/internal/database/mariadb/mv_vulnerabilities_test.go b/internal/database/mariadb/mv_vulnerabilities_test.go index ab6da906..21d259a5 100644 --- a/internal/database/mariadb/mv_vulnerabilities_test.go +++ b/internal/database/mariadb/mv_vulnerabilities_test.go @@ -77,6 +77,27 @@ var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { } } + var testServicesTotalCountWithSupportGroup = func(counts map[string]entity.IssueSeverityCounts) { + for _, sg := range seedCollection.SupportGroupRows { + filter := &entity.IssueFilter{ + SupportGroupCCRN: []*string{&sg.CCRN.String}, + AllServices: true, + } + + strId := fmt.Sprintf("%d", sg.Id.Int64) + + testIssueSeverityCount(filter, counts[strId]) + } + } + + var testServicesTotalCount = func(counts entity.IssueSeverityCounts) { + filter := &entity.IssueFilter{ + AllServices: true, + } + + testIssueSeverityCount(filter, counts) + } + var insertRemediation = func(serviceRow *mariadb.BaseServiceRow, componentRow *mariadb.ComponentRow, expirationDate time.Time) *entity.Remediation { remediation := test.NewFakeRemediation() if serviceRow != nil { @@ -87,7 +108,7 @@ var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { remediation.ComponentId = componentRow.Id remediation.Component = componentRow.CCRN } - remediation.IssueId = seedCollection.IssueRows[0].Id + remediation.IssueId = sql.NullInt64{Int64: 1, Valid: true} remediation.Issue = seedCollection.IssueRows[0].PrimaryName remediation.ExpirationDate = sql.NullTime{Time: expirationDate, Valid: true} r := remediation.AsRemediation() @@ -96,26 +117,7 @@ var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { return newRemediation } - BeforeEach(func() { - var err error - db = dbm.NewTestSchema() - seeder, err = test.NewDatabaseSeeder(dbm.DbConfig()) - Expect(err).To(BeNil(), "Database Seeder Setup should work") - seedCollection, err = seeder.SeedForIssueCounts() - Expect(err).To(BeNil()) - err = seeder.RefreshCountIssueRatings() - Expect(err).To(BeNil()) - }) - AfterEach(func() { - dbm.TestTearDown(db) - }) - - When("there are no remediations", Label("NoRemediations"), func() { - It("returns the correct count for all issues", func() { - severityCounts, err := test.LoadIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_severity.json")) - Expect(err).To(BeNil()) - testIssueSeverityCount(nil, severityCounts) - }) + var testNoActiveRemediation = func() { It("returns the correct count for component version issues", func() { severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) Expect(err).To(BeNil()) @@ -134,33 +136,67 @@ var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { testSupportGroups(severityCounts) }) + It("return the total count for all services with support group filter", func() { + severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) + Expect(err).To(BeNil()) + + testServicesTotalCountWithSupportGroup(severityCounts) + }) + It("return the total count for all services without support group filter", func() { + severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) + Expect(err).To(BeNil()) + + totalCount := entity.IssueSeverityCounts{} + for _, count := range severityCounts { + totalCount.Critical += count.Critical + totalCount.High += count.High + totalCount.Medium += count.Medium + totalCount.Low += count.Low + totalCount.None += count.None + totalCount.Total += count.Total + } + + testServicesTotalCount(totalCount) + }) + + } + + BeforeEach(func() { + var err error + db = dbm.NewTestSchema() + seeder, err = test.NewDatabaseSeeder(dbm.DbConfig()) + Expect(err).To(BeNil(), "Database Seeder Setup should work") + seedCollection, err = seeder.SeedForIssueCounts() + Expect(err).To(BeNil()) + err = seeder.RefreshCountIssueRatings() + Expect(err).To(BeNil()) }) + AfterEach(func() { + dbm.TestTearDown(db) + }) + + When("there are no remediations", Label("NoRemediations"), func() { + testNoActiveRemediation() + }) + When("there is an expired remediation for a service", Label("WithRemediations"), func() { BeforeEach(func() { expirationDate := time.Now().Add(-10 * 24 * time.Hour) insertRemediation(&seedCollection.ServiceRows[0], nil, expirationDate) seeder.RefreshCountIssueRatings() - }) - It("returns the correct count for component version issues", func() { - severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) - Expect(err).To(BeNil()) - - testComponentVersions(severityCounts) - }) - It("returns the correct count for services", func() { - seeder.RefreshCountIssueRatings() - severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) + testNoActiveRemediation() + }) + When("there is a deleted remediation for a service", Label("WithRemediations"), func() { + BeforeEach(func() { + expirationDate := time.Now().Add(10 * 24 * time.Hour) + createdRemediation := insertRemediation(&seedCollection.ServiceRows[0], nil, expirationDate) + err := db.DeleteRemediation(createdRemediation.Id, common.SystemUserId) Expect(err).To(BeNil()) - - testServices(severityCounts) - }) - It("returns the correct count for supportgroup", func() { - severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) + err = seeder.RefreshCountIssueRatings() Expect(err).To(BeNil()) - - testSupportGroups(severityCounts) }) + testNoActiveRemediation() }) When("there is an active remediation for a service", Label("WithRemediations"), func() { var serviceCounts entity.IssueSeverityCounts @@ -251,33 +287,37 @@ var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { testSupportGroups(severityCounts) }) - }) - When("there is a deleted remediation for a service", Label("WithRemediations"), func() { - BeforeEach(func() { - expirationDate := time.Now().Add(10 * 24 * time.Hour) - createdRemediation := insertRemediation(&seedCollection.ServiceRows[0], nil, expirationDate) - err := db.DeleteRemediation(createdRemediation.Id, common.SystemUserId) - Expect(err).To(BeNil()) - err = seeder.RefreshCountIssueRatings() - Expect(err).To(BeNil()) - }) - It("returns the correct count for component version issues", func() { - severityCounts, err := test.LoadComponentVersionIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_component_version.json")) + It("return the total count for all services with support group filter", func() { + severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) Expect(err).To(BeNil()) + severityCounts["1"] = entity.IssueSeverityCounts{ + Critical: 1, + Medium: 1, + Low: 2, + None: 1, + Total: 5, + } - testComponentVersions(severityCounts) + testServicesTotalCountWithSupportGroup(severityCounts) }) - It("returns the correct count for services", func() { + It("return the total count for all services without support group filter", func() { severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) Expect(err).To(BeNil()) - testServices(severityCounts) - }) - It("returns the correct count for supportgroup", func() { - severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) - Expect(err).To(BeNil()) + totalCount := entity.IssueSeverityCounts{} + for _, count := range severityCounts { + totalCount.Critical += count.Critical + totalCount.High += count.High + totalCount.Medium += count.Medium + totalCount.Low += count.Low + totalCount.None += count.None + totalCount.Total += count.Total + } + // remediation for one critical issue + totalCount.Critical -= 1 + totalCount.Total -= 1 - testSupportGroups(severityCounts) + testServicesTotalCount(totalCount) }) }) When("there is an active remediation for a component in a service", Label("WithRemediations"), func() { @@ -326,6 +366,57 @@ var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { testSupportGroups(severityCounts) }) + It("return the total count for all services with support group filter", func() { + severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) + Expect(err).To(BeNil()) + severityCounts["1"] = entity.IssueSeverityCounts{ + Critical: 1, + Medium: 1, + Low: 2, + None: 1, + Total: 5, + } + + testServicesTotalCountWithSupportGroup(severityCounts) + }) + It("return the total count for all services without support group filter", func() { + severityCounts, err := test.LoadServiceIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_service.json")) + Expect(err).To(BeNil()) + + totalCount := entity.IssueSeverityCounts{} + for _, count := range severityCounts { + totalCount.Critical += count.Critical + totalCount.High += count.High + totalCount.Medium += count.Medium + totalCount.Low += count.Low + totalCount.None += count.None + totalCount.Total += count.Total + } + // remediation for one critical issue + totalCount.Critical -= 1 + totalCount.Total -= 1 + + testServicesTotalCount(totalCount) + }) + }) + When("there is an active remediation for a vulnerability only in one service", Label("WithRemediations"), func() { + BeforeEach(func() { + expirationDate := time.Now().Add(10 * 24 * time.Hour) + r := insertRemediation(&seedCollection.ServiceRows[0], &seedCollection.ComponentRows[0], expirationDate) + im := test.NewFakeIssueMatch() + im.ComponentInstanceId = sql.NullInt64{Int64: 3, Valid: true} + im.IssueId = sql.NullInt64{Int64: r.IssueId, Valid: true} + im.UserId = sql.NullInt64{Int64: common.SystemUserId, Valid: true} + _, err := seeder.InsertFakeIssueMatch(im) + Expect(err).To(BeNil()) + seeder.RefreshCountIssueRatings() + }) + It("returns the total count for all services with support group filter", func() { + severityCounts, err := test.LoadSupportGroupIssueCounts(test.GetTestDataPath("../mariadb/testdata/issue_counts/issue_counts_per_support_group.json")) + Expect(err).To(BeNil()) + + testServicesTotalCountWithSupportGroup(severityCounts) + }) }) }) From 15f0d625aabe213a7a5b595cc28306f43e46f35e Mon Sep 17 00:00:00 2001 From: Michael Reimsbach Date: Mon, 8 Dec 2025 16:05:01 +0100 Subject: [PATCH 3/7] fix(fixture): user common.SystemUserId --- internal/database/mariadb/test/fixture.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/database/mariadb/test/fixture.go b/internal/database/mariadb/test/fixture.go index d261efd6..236a15be 100644 --- a/internal/database/mariadb/test/fixture.go +++ b/internal/database/mariadb/test/fixture.go @@ -1680,8 +1680,8 @@ func NewFakeRemediation() mariadb.RemediationRow { RemediationDate: sql.NullTime{Time: gofakeit.Date(), Valid: true}, ExpirationDate: sql.NullTime{Time: gofakeit.Date(), Valid: true}, Type: sql.NullString{String: "false_positive", Valid: true}, - CreatedBy: sql.NullInt64{Int64: e2e_common.SystemUserId, Valid: true}, - UpdatedBy: sql.NullInt64{Int64: e2e_common.SystemUserId, Valid: true}, + CreatedBy: sql.NullInt64{Int64: util.SystemUserId, Valid: true}, + UpdatedBy: sql.NullInt64{Int64: util.SystemUserId, Valid: true}, } } From c6746263444d811b9f3fc2a1de75ac9da1ac907d Mon Sep 17 00:00:00 2001 From: Michael Reimsbach Date: Mon, 8 Dec 2025 16:14:37 +0100 Subject: [PATCH 4/7] fix: import SystemUserId --- internal/database/mariadb/mv_vulnerabilities_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/database/mariadb/mv_vulnerabilities_test.go b/internal/database/mariadb/mv_vulnerabilities_test.go index 21d259a5..d805a876 100644 --- a/internal/database/mariadb/mv_vulnerabilities_test.go +++ b/internal/database/mariadb/mv_vulnerabilities_test.go @@ -9,9 +9,9 @@ import ( "time" "github.com/cloudoperators/heureka/internal/database/mariadb" - "github.com/cloudoperators/heureka/internal/database/mariadb/common" "github.com/cloudoperators/heureka/internal/database/mariadb/test" "github.com/cloudoperators/heureka/internal/entity" + "github.com/cloudoperators/heureka/internal/util" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -191,7 +191,7 @@ var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { BeforeEach(func() { expirationDate := time.Now().Add(10 * 24 * time.Hour) createdRemediation := insertRemediation(&seedCollection.ServiceRows[0], nil, expirationDate) - err := db.DeleteRemediation(createdRemediation.Id, common.SystemUserId) + err := db.DeleteRemediation(createdRemediation.Id, util.SystemUserId) Expect(err).To(BeNil()) err = seeder.RefreshCountIssueRatings() Expect(err).To(BeNil()) @@ -406,7 +406,7 @@ var _ = Describe("Counting Issues by Severity", Label("IssueCounts"), func() { im := test.NewFakeIssueMatch() im.ComponentInstanceId = sql.NullInt64{Int64: 3, Valid: true} im.IssueId = sql.NullInt64{Int64: r.IssueId, Valid: true} - im.UserId = sql.NullInt64{Int64: common.SystemUserId, Valid: true} + im.UserId = sql.NullInt64{Int64: util.SystemUserId, Valid: true} _, err := seeder.InsertFakeIssueMatch(im) Expect(err).To(BeNil()) seeder.RefreshCountIssueRatings() From 24ee376b8d4952d2a592defe6f80a561d8beaa83 Mon Sep 17 00:00:00 2001 From: Michael Reimsbach Date: Tue, 9 Dec 2025 11:14:38 +0100 Subject: [PATCH 5/7] fix: add missing exist check --- .../20250827115156_add_mv_count_vulnerability_component.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/database/mariadb/migrations/20250827115156_add_mv_count_vulnerability_component.up.sql b/internal/database/mariadb/migrations/20250827115156_add_mv_count_vulnerability_component.up.sql index 2d8b8dfa..fe8f466d 100644 --- a/internal/database/mariadb/migrations/20250827115156_add_mv_count_vulnerability_component.up.sql +++ b/internal/database/mariadb/migrations/20250827115156_add_mv_count_vulnerability_component.up.sql @@ -31,7 +31,7 @@ CREATE TABLE IF NOT EXISTS mvAllComponentsByServiceVulnerabilityCounts ( CREATE PROCEDURE refresh_mvSingleComponentByServiceVulnerabilityCounts_proc() BEGIN -- Step 1: Create a temporary table - CREATE TABLE mvSingleComponentByServiceVulnerabilityCounts_tmp LIKE mvSingleComponentByServiceVulnerabilityCounts; + CREATE TABLE IF NOT EXISTS mvSingleComponentByServiceVulnerabilityCounts_tmp LIKE mvSingleComponentByServiceVulnerabilityCounts; DELETE FROM mvSingleComponentByServiceVulnerabilityCounts_tmp; -- Step 2: Populate it in a single query From 1e7be5bbbe7e58055e27cacab59c03a4eaed53db Mon Sep 17 00:00:00 2001 From: Michal Krzyz Date: Tue, 9 Dec 2025 13:04:41 +0100 Subject: [PATCH 6/7] fix the problem with remediation tests On-behalf-of: SAP Michal Krzyz Signed-off-by: Michal Krzyz --- internal/e2e/remediation_query_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/e2e/remediation_query_test.go b/internal/e2e/remediation_query_test.go index 1fc81748..954fe268 100644 --- a/internal/e2e/remediation_query_test.go +++ b/internal/e2e/remediation_query_test.go @@ -82,15 +82,13 @@ var _ = Describe("Getting Remediations via API", Label("e2e", "Remediations"), f "after": ""}) Expect(respData.Remediations.TotalCount).To(Equal(len(seedCollection.RemediationRows))) Expect(len(respData.Remediations.Edges)).To(Equal(5)) - }) - It("- returns the expected PageInfo", func() { + //- returns the expected PageInfo Expect(*respData.Remediations.PageInfo.HasNextPage).To(BeTrue(), "hasNextPage is set") Expect(*respData.Remediations.PageInfo.HasPreviousPage).To(BeFalse(), "hasPreviousPage is set") Expect(respData.Remediations.PageInfo.NextPageAfter).ToNot(BeNil(), "nextPageAfter is set") Expect(len(respData.Remediations.PageInfo.Pages)).To(Equal(2), "Correct amount of pages") Expect(*respData.Remediations.PageInfo.PageNumber).To(Equal(1), "Correct page number") - }) - It("- returns the expected content", func() { + //- returns the expected content for _, remediation := range respData.Remediations.Edges { Expect(remediation.Node.ID).ToNot(BeNil(), "remediation has ID set") Expect(remediation.Node.Description).ToNot(BeNil(), "remediation has description set") @@ -106,7 +104,6 @@ var _ = Describe("Getting Remediations via API", Label("e2e", "Remediations"), f Expect(remediationFound).To(BeTrue(), "remediation exists in seeded data") } }) - }) }) }) From ca7a5d2778fcb1f3bb8694c2f0682aef5ab8f34d Mon Sep 17 00:00:00 2001 From: Michael Reimsbach Date: Tue, 9 Dec 2025 16:43:13 +0100 Subject: [PATCH 7/7] chore: refactor --- .github/workflows/tests.yaml | 2 +- internal/database/mariadb/entity.go | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 525fc8a5..c2ba3354 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -42,4 +42,4 @@ jobs: DB_ADDRESS: localhost DB_PORT: 3306 DB_SCHEMA: internal/database/mariadb/init/schema.sql - run: ginkgo --trace --progress -r -randomize-all -randomize-suites + run: ginkgo --trace -r -randomize-all -randomize-suites diff --git a/internal/database/mariadb/entity.go b/internal/database/mariadb/entity.go index 7fd3bb3d..ebbe2c3d 100644 --- a/internal/database/mariadb/entity.go +++ b/internal/database/mariadb/entity.go @@ -60,6 +60,10 @@ func GetUserTypeValue(v sql.NullInt64) entity.UserType { } } +func IsValidId(id int64) bool { + return id > 0 +} + // RowComposite is a composite type that contains all the row types for the database // This is used to unmarshal the database rows into the corresponding entity types in a dynamical manner type RowComposite struct { @@ -1191,7 +1195,7 @@ func (rr *RemediationRow) FromRemediation(r *entity.Remediation) { rr.Description = sql.NullString{String: r.Description, Valid: true} rr.Type = sql.NullString{String: r.Type.String(), Valid: true} rr.Component = sql.NullString{String: r.Component, Valid: true} - rr.ComponentId = sql.NullInt64{Int64: r.ComponentId, Valid: r.ComponentId != -1} + rr.ComponentId = sql.NullInt64{Int64: r.ComponentId, Valid: IsValidId(r.ComponentId)} rr.Service = sql.NullString{String: r.Service, Valid: true} rr.ServiceId = sql.NullInt64{Int64: r.ServiceId, Valid: true} rr.Issue = sql.NullString{String: r.Issue, Valid: true} @@ -1199,7 +1203,7 @@ func (rr *RemediationRow) FromRemediation(r *entity.Remediation) { rr.RemediationDate = sql.NullTime{Time: r.RemediationDate, Valid: true} rr.ExpirationDate = sql.NullTime{Time: r.ExpirationDate, Valid: true} rr.RemediatedBy = sql.NullString{String: r.RemediatedBy, Valid: true} - rr.RemediatedById = sql.NullInt64{Int64: r.RemediatedById, Valid: r.RemediatedById != -1} + rr.RemediatedById = sql.NullInt64{Int64: r.RemediatedById, Valid: IsValidId(r.RemediatedById)} rr.CreatedAt = sql.NullTime{Time: r.CreatedAt, Valid: true} rr.CreatedBy = sql.NullInt64{Int64: r.CreatedBy, Valid: true} rr.DeletedAt = sql.NullTime{Time: r.DeletedAt, Valid: true}