Canonicalize targets (#108)

This canonicalize all the labels in all attributes found that are label related using the output of `bazel mod dump_repo_mapping ""` for each commit.

This addresses #105

It doesn't have any integration tests yet because it would be much better to leverage #104 which is still in the works but it was tested against our internal monorepo and the reproducer in #105 

Label like attributes (string defined but has nodep = True) are special and handled as an exception as if they were labels, and thus also converted. (See [this thread](https://bazelbuild.slack.com/archives/CDCMRLS23/p1742821059464199))

Author: darkrift

pkg/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
         "bazel.go",
         "configurations.go",
         "hash_cache.go",
+        "normalizer.go",
         "target_determinator.go",
         "targets_list.go",
         "walker.go",
@@ -31,6 +32,7 @@ go_test(
     name = "pkg_test",
     srcs = [
         "hash_cache_test.go",
+        "normalizer_test.go",
         "target_determinator_test.go",
     ],
     data = ["//testdata/HelloWorld:all_srcs"],

pkg/hash_cache.go

@@ -25,14 +25,19 @@ import (
 )
 
 // NewTargetHashCache creates a TargetHashCache which uses context for metadata lookups.
-func NewTargetHashCache(context map[gazelle_label.Label]map[Configuration]*analysis.ConfiguredTarget, bazelRelease string) *TargetHashCache {
+func NewTargetHashCache(
+	context map[gazelle_label.Label]map[Configuration]*analysis.ConfiguredTarget,
+	normalizer *Normalizer,
+	bazelRelease string,
+) *TargetHashCache {
 	bazelVersionSupportsConfiguredRuleInputs := isConfiguredRuleInputsSupported(bazelRelease)
 
 	return &TargetHashCache{
 		context: context,
 		fileHashCache: &fileHashCache{
 			cache: make(map[string]*cacheEntry),
 		},
+		normalizer:                               normalizer,
 		bazelRelease:                             bazelRelease,
 		bazelVersionSupportsConfiguredRuleInputs: bazelVersionSupportsConfiguredRuleInputs,
 		cache:                                    make(map[gazelle_label.Label]map[Configuration]*cacheEntry),
@@ -63,6 +68,8 @@ type TargetHashCache struct {
 	bazelRelease                             string
 	bazelVersionSupportsConfiguredRuleInputs bool
 
+	normalizer *Normalizer
+
 	frozen bool
 
 	cacheLock sync.Mutex
@@ -138,6 +145,10 @@ func (thc *TargetHashCache) Freeze() {
 	thc.frozen = true
 }
 
+func (thc *TargetHashCache) ParseCanonicalLabel(label string) (gazelle_label.Label, error) {
+	return thc.normalizer.ParseCanonicalLabel(label)
+}
+
 // Difference represents a difference of a target between two commits.
 // All fields except Category are optional.
 type Difference struct {
@@ -264,14 +275,16 @@ func WalkDiffs(before *TargetHashCache, after *TargetHashCache, labelAndConfigur
 				Before:   string(attributeBeforeJson),
 			})
 		} else {
-			if !equivalentAttributes(attributeBefore, attributeAfter) {
+			normalizedBeforeAttribute := before.AttributeForSerialization(attributeBefore)
+			normalizedAfterAttribute := after.AttributeForSerialization(attributeAfter)
+			if !equivalentAttributes(normalizedBeforeAttribute, normalizedAfterAttribute) {
 				if attributeName == "$rule_implementation_hash" {
 					differences = append(differences, Difference{
 						Category: "RuleImplementedChanged",
 					})
 				} else {
-					attributeBeforeJson, _ := protojson.Marshal(AttributeForSerialization(attributeBefore))
-					attributeAfterJson, _ := protojson.Marshal(AttributeForSerialization(attributeAfter))
+					attributeBeforeJson, _ := protojson.Marshal(normalizedBeforeAttribute)
+					attributeAfterJson, _ := protojson.Marshal(normalizedAfterAttribute)
 					differences = append(differences, Difference{
 						Category: "AttributeChanged",
 						Key:      attributeName,
@@ -285,7 +298,7 @@ func WalkDiffs(before *TargetHashCache, after *TargetHashCache, labelAndConfigur
 	sortedAttributeNamesAfter := sortKeys(attributesAfter)
 	for _, attributeName := range sortedAttributeNamesAfter {
 		if _, ok := attributesBefore[attributeName]; !ok {
-			attributeAfterJson, _ := protojson.Marshal(AttributeForSerialization(attributesAfter[attributeName]))
+			attributeAfterJson, _ := protojson.Marshal(after.AttributeForSerialization(attributesAfter[attributeName]))
 			differences = append(differences, Difference{
 				Category: "AttributeAdded",
 				Key:      attributeName,
@@ -308,7 +321,8 @@ func WalkDiffs(before *TargetHashCache, after *TargetHashCache, labelAndConfigur
 
 	for _, ruleInputLabelAndConfigurations := range ruleInputLabelsAndConfigurationsAfter {
 		ruleInputLabel := ruleInputLabelAndConfigurations.Label
-		if _, ok := ruleInputLabelsToConfigurationsBefore[ruleInputLabel]; !ok {
+		knownConfigurationsBefore, ok := ruleInputLabelsToConfigurationsBefore[ruleInputLabel]
+		if !ok {
 			differences = append(differences, Difference{
 				Category: "RuleInputAdded",
 				Key:      ruleInputLabel.String(),
@@ -318,7 +332,6 @@ func WalkDiffs(before *TargetHashCache, after *TargetHashCache, labelAndConfigur
 			// query information, so we could filter away e.g. host changes when we only have a target dep.
 			// Unfortunately, Bazel doesn't currently expose this.
 			// See https://github.com/bazelbuild/bazel/issues/14610#issuecomment-1024460141
-			knownConfigurationsBefore := ruleInputLabelsToConfigurationsBefore[ruleInputLabel]
 			knownConfigurationsAfter := ruleInputLabelsToConfigurationsAfter[ruleInputLabel]
 
 			for _, knownConfigurationAfter := range knownConfigurationsAfter.SortedSlice() {
@@ -369,6 +382,34 @@ func WalkDiffs(before *TargetHashCache, after *TargetHashCache, labelAndConfigur
 	return differences, nil
 }
 
+// AttributeForSerialization redacts details about an attribute which don't affect the output of
+// building them, and returns equivalent canonical attribute metadata.
+// In particular it redacts:
+//   - Whether an attribute was explicitly specified (because the effective value is all that
+//     matters).
+//   - Any attribute named `generator_location`, because these point to absolute paths for
+//     built-in `cc_toolchain_suite` targets such as `@local_config_cc//:toolchain`.
+func (thc *TargetHashCache) AttributeForSerialization(rawAttr *build.Attribute) *build.Attribute {
+	normalized := *rawAttr
+	normalized.ExplicitlySpecified = nil
+
+	// Redact generator_location, which typically contains absolute paths but has no bearing on the
+	// functioning of a rule.
+	// This is also done in Bazel's internal target hash computation. See:
+	// https://github.com/bazelbuild/bazel/blob/6971b016f1e258e3bb567a0f9fe7a88ad565d8f2/src/main/java/com/google/devtools/build/lib/query2/query/output/SyntheticAttributeHashCalculator.java#L78-L81
+	if normalized.Name != nil {
+		if *normalized.Name == "generator_location" {
+			normalized.StringValue = nil
+		}
+	}
+
+	return thc.normalizer.NormalizeAttribute(&normalized)
+}
+
+func equivalentAttributes(left, right *build.Attribute) bool {
+	return proto.Equal(left, right)
+}
+
 func indexByLabel(labelsAndConfigurations []LabelAndConfigurations) map[gazelle_label.Label]*ss.SortedSet[Configuration] {
 	m := make(map[gazelle_label.Label]*ss.SortedSet[Configuration], len(labelsAndConfigurations))
 	for _, labelAndConfigurations := range labelsAndConfigurations {
@@ -443,7 +484,7 @@ func hashTarget(thc *TargetHashCache, labelAndConfiguration LabelAndConfiguratio
 		return hashRule(thc, target.Rule, configuredTarget.Configuration)
 	case build.Target_GENERATED_FILE:
 		hasher := sha256.New()
-		generatingLabel, err := ParseCanonicalLabel(*target.GeneratedFile.GeneratingRule)
+		generatingLabel, err := thc.ParseCanonicalLabel(*target.GeneratedFile.GeneratingRule)
 		if err != nil {
 			return nil, fmt.Errorf("failed to parse generated file generating rule label %s: %w", *target.GeneratedFile.GeneratingRule, err)
 		}
@@ -472,15 +513,19 @@ func hashRule(thc *TargetHashCache, rule *build.Rule, configuration *analysis.Co
 	hasher.Write([]byte(rule.GetRuleClass()))
 	hasher.Write([]byte(rule.GetSkylarkEnvironmentHashCode()))
 	hasher.Write([]byte(configuration.GetChecksum()))
+
 	// TODO: Consider using `$internal_attr_hash` from https://github.com/bazelbuild/bazel/blob/6971b016f1e258e3bb567a0f9fe7a88ad565d8f2/src/main/java/com/google/devtools/build/lib/query2/query/output/SyntheticAttributeHashCalculator.java
 	// rather than hashing attributes ourselves.
 	// On the plus side, this builds in some heuristics from Bazel (e.g. ignoring `generator_location`).
 	// On the down side, it would even further decouple our "hashing" and "diffing" procedures.
 	for _, attr := range rule.GetAttribute() {
-		protoBytes, err := proto.Marshal(AttributeForSerialization(attr))
+		normalizedAttribute := thc.AttributeForSerialization(attr)
+
+		protoBytes, err := proto.Marshal(normalizedAttribute)
 		if err != nil {
 			return nil, err
 		}
+
 		hasher.Write(protoBytes)
 	}
 
@@ -512,7 +557,7 @@ func getConfiguredRuleInputs(thc *TargetHashCache, rule *build.Rule, ownConfigur
 	labelsAndConfigurations := make([]LabelAndConfigurations, 0)
 	if thc.bazelVersionSupportsConfiguredRuleInputs {
 		for _, configuredRuleInput := range rule.ConfiguredRuleInput {
-			ruleInputLabel, err := ParseCanonicalLabel(configuredRuleInput.GetLabel())
+			ruleInputLabel, err := thc.ParseCanonicalLabel(configuredRuleInput.GetLabel())
 			if err != nil {
 				return nil, fmt.Errorf("failed to parse configuredRuleInput label %s: %w", configuredRuleInput.GetLabel(), err)
 			}
@@ -534,7 +579,7 @@ func getConfiguredRuleInputs(thc *TargetHashCache, rule *build.Rule, ownConfigur
 		}
 	} else {
 		for _, ruleInputLabelString := range rule.RuleInput {
-			ruleInputLabel, err := ParseCanonicalLabel(ruleInputLabelString)
+			ruleInputLabel, err := thc.ParseCanonicalLabel(ruleInputLabelString)
 			if err != nil {
 				return nil, fmt.Errorf("failed to parse ruleInput label %s: %w", ruleInputLabelString, err)
 			}

pkg/hash_cache_test.go

@@ -380,19 +380,21 @@ func layoutProject(t *testing.T) (string, *analysis.CqueryResult) {
 }
 
 func parseResult(t *testing.T, result *analysis.CqueryResult, bazelRelease string) *TargetHashCache {
-	cqueryResult, err := ParseCqueryResult(result)
+	n := Normalizer{}
+	cqueryResult, err := ParseCqueryResult(result, &n)
 	if err != nil {
 		t.Fatalf("Failed to parse cquery result: %v", err)
 	}
-	return NewTargetHashCache(cqueryResult, bazelRelease)
+	return NewTargetHashCache(cqueryResult, &n, bazelRelease)
 }
 
 func areHashesEqual(left, right []byte) bool {
 	return reflect.DeepEqual(left, right)
 }
 
 func mustParseLabel(s string) label.Label {
-	l, err := ParseCanonicalLabel(s)
+	n := Normalizer{}
+	l, err := n.ParseCanonicalLabel(s)
 	if err != nil {
 		panic(err)
 	}

pkg/normalizer.go

@@ -0,0 +1,101 @@
+package pkg
+
+import (
+	"github.com/bazel-contrib/target-determinator/third_party/protobuf/bazel/build"
+	"github.com/bazelbuild/bazel-gazelle/label"
+)
+
+// Normalizer is a struct that contains a mapping of non-canonical repository names to canonical repository names.
+type Normalizer struct {
+	Mapping map[string]string
+}
+
+// ParseCanonicalLabel parses a label from a string, and removes sources of inconsequential difference which would make comparing two labels fail.
+// In particular, it treats @// the same as //
+// If the label is not canonical, it will attempt to map the repository to its canonical form coming from `bazel mod dump_repo_mapping ""`.
+func (n *Normalizer) ParseCanonicalLabel(s string) (label.Label, error) {
+	l, err := label.Parse(s)
+	if err != nil {
+		return l, err
+	}
+
+	if !l.Canonical && l.Repo != "" {
+		mappedValue, ok := n.Mapping[l.Repo]
+		if ok && l.Repo != mappedValue {
+			l.Repo = mappedValue
+			l.Canonical = true
+		}
+	}
+
+	if l.Repo == "@" {
+		l.Repo = ""
+	}
+
+	return l, nil
+}
+
+func (n *Normalizer) NormalizeAttribute(attr *build.Attribute) *build.Attribute {
+	attrType := attr.GetType()
+
+	// An attribute with a nodep property can also hold labels
+	// It should be handled as an exception, see https://bazelbuild.slack.com/archives/CDCMRLS23/p1742821059464199
+	isNoDepAttribute := attrType == build.Attribute_STRING && attr.Nodep != nil && *attr.Nodep
+
+	if attrType == build.Attribute_OUTPUT || attrType == build.Attribute_LABEL || isNoDepAttribute {
+		keyLabel, parseErr := n.ParseCanonicalLabel(attr.GetStringValue())
+
+		if parseErr == nil {
+			value := keyLabel.String()
+			attr.StringValue = &value
+		}
+	}
+
+	isNoDepListAttribute := attrType == build.Attribute_STRING_LIST && attr.Nodep != nil && *attr.Nodep
+
+	if attrType == build.Attribute_OUTPUT_LIST || attrType == build.Attribute_LABEL_LIST || isNoDepListAttribute {
+		for idx, dep := range attr.GetStringListValue() {
+			keyLabel, parseErr := n.ParseCanonicalLabel(dep)
+
+			if parseErr == nil {
+				attr.StringListValue[idx] = keyLabel.String()
+			}
+		}
+	}
+
+	if attrType == build.Attribute_LABEL_DICT_UNARY {
+		for idx, dep := range attr.GetLabelDictUnaryValue() {
+			keyLabel, parseErr := n.ParseCanonicalLabel(*dep.Value)
+
+			if parseErr == nil {
+				newValue := keyLabel.String()
+				attr.GetLabelDictUnaryValue()[idx].Value = &newValue
+			}
+		}
+	}
+
+	if attrType == build.Attribute_LABEL_LIST_DICT {
+		for idx, dep := range attr.GetLabelListDictValue() {
+			for key, value := range dep.Value {
+				l, parseErr := n.ParseCanonicalLabel(value)
+
+				if parseErr == nil {
+					attr.GetLabelListDictValue()[idx].Value[key] = l.String()
+				}
+			}
+		}
+	}
+
+	if attrType == build.Attribute_LABEL_KEYED_STRING_DICT {
+		for idx, dep := range attr.GetLabelKeyedStringDictValue() {
+			keyLabel, parseErr := n.ParseCanonicalLabel(*dep.Key)
+
+			if parseErr == nil {
+				newKey := keyLabel.String()
+				attr.GetLabelKeyedStringDictValue()[idx].Key = &newKey
+			}
+
+		}
+	}
+
+	return attr
+}