feat: Handle moved features during data ingestion (#1769)

This change introduces logic to handle features that have been marked as "moved" to a new feature key.

During data ingestion for WPT and Chromium histograms, if a feature has been moved, this change ensures that the metrics are associated with the new, redirected feature key. This allows for future ingestions to be correctly attributed while upstream sources (like WPT test annotations or Chromium mojom files) are being updated.

These checks are implemented at different layers depending on the data source:

- For WPT, the migration happens in the workflow layer before scoring. This allows the scores to be calculated with the corrected feature keys.
- For Chromium histograms, the migration happens in the spanner adapter layer. This is because the full list of feature keys is not available until this layer, where it is fetched from the database.

The implementation also introduces a check to prevent conflicts where both the old and new feature keys are present in the source data. When this conflict occurs, an error is logged with the message "conflict migrating feature key", which will be used for alerting in GCP.

We can also add alerting for these messages too:
- "migrating feature key for histogram"
- "migrating feature key for test"

These will let us know to update the upstream sources

Author: jcscottiii

lib/gcpspanner/spanneradapters/chromium_historgram_enum_consumer.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"github.com/GoogleChrome/webstatus.dev/lib/gcpspanner"
+	"github.com/GoogleChrome/webstatus.dev/lib/gen/jsonschema/web_platform_dx__web_features"
 	"github.com/GoogleChrome/webstatus.dev/lib/metricdatatypes"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
@@ -45,18 +46,45 @@ type ChromiumHistogramEnumsClient interface {
 	UpsertWebFeatureChromiumHistogramEnumValue(context.Context, gcpspanner.WebFeatureChromiumHistogramEnumValue) error
 	GetIDFromFeatureKey(context.Context, *gcpspanner.FeatureIDFilter) (*string, error)
 	FetchAllFeatureKeys(context.Context) ([]string, error)
+	GetAllMovedWebFeatures(ctx context.Context) ([]gcpspanner.MovedWebFeature, error)
 }
 
 // Used by GCP Log-based metrics to extract the data about mismatch mappings.
 const logMissingFeatureIDMetricMsg = "unable to find feature ID. skipping mapping"
 
+var ErrConflictMigratingFeatureKey = errors.New("conflict migrating feature key")
+
+func (c *ChromiumHistogramEnumConsumer) GetAllMovedWebFeatures(
+	ctx context.Context) (map[string]web_platform_dx__web_features.FeatureMovedData, error) {
+	movedFeatures, err := c.client.GetAllMovedWebFeatures(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return convertGCPSpannerMovedFeaturesToMap(movedFeatures), nil
+}
+
 func (c *ChromiumHistogramEnumConsumer) SaveHistogramEnums(
 	ctx context.Context, data metricdatatypes.HistogramMapping) error {
 	featureKeys, err := c.client.FetchAllFeatureKeys(ctx)
 	if err != nil {
 		return errors.Join(ErrFailedToGetFeatureKeys, err)
 	}
 	enumToFeatureKeyMap := createEnumToFeatureKeyMap(featureKeys)
+
+	histogramsToEnumMap, histogramsToAllFeatureKeySet := buildHistogramMaps(data, enumToFeatureKeyMap)
+
+	// Get the moved features
+	movedFeatures, err := c.GetAllMovedWebFeatures(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = migrateMovedFeatures(ctx, histogramsToEnumMap, histogramsToAllFeatureKeySet, movedFeatures)
+	if err != nil {
+		return err
+	}
+
 	// Create mapping of anticipated enums to feature keys
 	for histogram, enums := range data {
 		enumID, err := c.client.UpsertChromiumHistogramEnum(ctx, gcpspanner.ChromiumHistogramEnum{
@@ -75,30 +103,27 @@ func (c *ChromiumHistogramEnumConsumer) SaveHistogramEnums(
 				return errors.Join(ErrFailedToStoreEnumValue, err)
 			}
 
-			featureKey, found := enumToFeatureKeyMap[enum.Label]
-			if !found {
-				slog.WarnContext(ctx,
-					logMissingFeatureIDMetricMsg,
-					"label", enum.Label)
-
+			featureKey := histogramsToEnumMap[histogram][enum.Value]
+			if featureKey == nil {
 				continue
 			}
 
 			featureID, err := c.client.GetIDFromFeatureKey(
-				ctx, gcpspanner.NewFeatureKeyFilter(featureKey))
+				ctx, gcpspanner.NewFeatureKeyFilter(*featureKey))
 			if err != nil {
 				slog.WarnContext(ctx,
 					logMissingFeatureIDMetricMsg,
 					"error", err,
-					"featureKey", featureKey,
+					"featureKey", *featureKey,
 					"label", enum.Label)
 
 				continue
 			}
-			err = c.client.UpsertWebFeatureChromiumHistogramEnumValue(ctx, gcpspanner.WebFeatureChromiumHistogramEnumValue{
-				WebFeatureID:                 *featureID,
-				ChromiumHistogramEnumValueID: *enumValueID,
-			})
+			err = c.client.UpsertWebFeatureChromiumHistogramEnumValue(ctx,
+				gcpspanner.WebFeatureChromiumHistogramEnumValue{
+					WebFeatureID:                 *featureID,
+					ChromiumHistogramEnumValueID: *enumValueID,
+				})
 			if err != nil {
 				return errors.Join(ErrFailedToStoreEnumValueWebFeatureMapping, err)
 			}
@@ -108,6 +133,69 @@ func (c *ChromiumHistogramEnumConsumer) SaveHistogramEnums(
 	return nil
 }
 
+func buildHistogramMaps(
+	data metricdatatypes.HistogramMapping,
+	enumToFeatureKeyMap map[string]string,
+) (map[metricdatatypes.HistogramName]map[int64]*string,
+	map[metricdatatypes.HistogramName]map[string]metricdatatypes.HistogramEnumValue) {
+	histogramsToEnumMap := map[metricdatatypes.HistogramName]map[int64]*string{}
+	histogramsToAllFeatureKeySet := make(
+		map[metricdatatypes.HistogramName]map[string]metricdatatypes.HistogramEnumValue)
+
+	for histogram, enums := range data {
+		enumIDToFeatureKeyMap := make(map[int64]*string, len(enums))
+		allFeatureKeySet := make(map[string]metricdatatypes.HistogramEnumValue, len(enums))
+		for _, enum := range enums {
+			featureKey, found := enumToFeatureKeyMap[enum.Label]
+			if !found {
+				enumIDToFeatureKeyMap[enum.Value] = nil
+
+				continue
+			}
+			enumIDToFeatureKeyMap[enum.Value] = &featureKey
+			allFeatureKeySet[featureKey] = enum
+		}
+		histogramsToEnumMap[histogram] = enumIDToFeatureKeyMap
+		histogramsToAllFeatureKeySet[histogram] = allFeatureKeySet
+	}
+
+	return histogramsToEnumMap, histogramsToAllFeatureKeySet
+}
+
+func migrateMovedFeatures(
+	ctx context.Context,
+	histogramsToEnumMap map[metricdatatypes.HistogramName]map[int64]*string,
+	histogramsToAllFeatureKeySet map[metricdatatypes.HistogramName]map[string]metricdatatypes.HistogramEnumValue,
+	movedFeatures map[string]web_platform_dx__web_features.FeatureMovedData,
+) error {
+	for histogram, allFeaturesKeySet := range histogramsToAllFeatureKeySet {
+		for featureKey, featureEnum := range allFeaturesKeySet {
+			if movedFeatureData, found := movedFeatures[featureKey]; found {
+				if _, exists := allFeaturesKeySet[movedFeatureData.RedirectTarget]; exists {
+					// This new key already exists somewhere in the data.
+					// That means upstream has done a partial migration to the new feature key.
+					// Instead of assuming, we should error out and Chromium mojom file needs to be updated.
+					slog.ErrorContext(ctx, "conflict migrating feature key. upstream currently using both keys",
+						"histogram", histogram,
+						"old_key", featureKey,
+						"new_key", movedFeatureData.RedirectTarget,
+					)
+
+					return ErrConflictMigratingFeatureKey
+				}
+				slog.WarnContext(ctx, "migrating feature key for histogram",
+					"histogram", histogram,
+					"old_key", featureKey,
+					"new_key", movedFeatureData.RedirectTarget,
+				)
+				histogramsToEnumMap[histogram][featureEnum.Value] = &movedFeatureData.RedirectTarget
+			}
+		}
+	}
+
+	return nil
+}
+
 var (
 	// ErrFailedToStoreEnum indicates the storage layer failed to store chromium enum.
 	ErrFailedToStoreEnum = errors.New("failed to store chromium enum")