Create the gradle/gradle-test-versions.yml file (#287)

Create the gradle/gradle-test-versions.yml file

Author: kelvinou01

README.md

@@ -52,6 +52,7 @@ or in some "Versions" class that has a listing of all the dependencies the test
 final class TestPluginVersions {
     static final String CONJURE_JAVA = "com.palantir.conjure.java:conjure-java:5.7.1";
     static final String CONJURE = "com.palantir.conjure:conjure:4.10.1";
+}
 ```
 
 Once these tests are written, the versions of the plugins are often not updated, even when the project under test keeps its dependencies of those plugins up-to-date.  This can cause tests to fail when the plugin is updated not because the plugin is bad, but because there is an incompatibility in the old versions of plugins used in the integration tests.  This can often happen with Gradle version bumps.
@@ -106,7 +107,53 @@ dependencies {
 # Resolution of Gradle versions to test against
 Similarly, tests may hardcode versions of Gradle that they need to stay compatible with. These versions also get stale and PRs start failing for the inverse reason of the above - the code in the plugin or a dependency of it is updated and is no longer compatible with an old version of Gradle. For example, attempting to update jackson libraries from `2.15.0` -> `2.17.0` would fail if a test tried to run on Gradle versiosn < `7.6.4` (when compatibility with jackson `2.17.0` was fixed).
 
-The `GradleTestVersions` class can provide up-to-date versions of Gradle to test against.  
+## Setting Gradle test versions
+
+There are two ways to set which Gradle versions to test against:
+* setting the versions in the `gradle/gradle-test-versions.yml` file
+* setting `gradleVersions` in the the `gradleTestUtils` extension (this is deprecated)
+  When using `@GradlePluginTests`, the versions of Gradle used will be the union of the versions above.
+
+If you haven't set either, tests will fall back to using the Gradle version of the test driver.
+
+### The `gradle/gradle-test-versions.yml` file
+
+An example file:
+
+```yaml
+major-versions:
+  8: 8.14.3
+  9: 9.2.0-rc1
+extra-versions:
+- 8.8
+- 8.14.2
+```
+The motivation for having a separate config file is to more easily allow an automated process of bumping the versions of Gradle we test against, whilst also allowing users to pick out specific versions to test against.
+Concretely, an automated process would bump versions in the `major-versions` blob, whilst leaving `extra-versions` alone.
+When support for a Gradle major version is phased out, it can be just removed from this file in one fell swoop.
+
+> [!NOTE]
+> This file is global, and affects every project that is using this plugin.
+
+### [Deprecated] Setting `gradleVersions` in the `gradleTestUtils` extension
+You can set Gradle versions to test by setting `gradleVersions` property on the `gradleTestUtils` extension. For example, to test against Gradle versions `7.6.4` and `8.8`, you would do:
+```groovy
+gradleTestUtils {
+  gradleVersions = ['7.6.4', '8.8']
+}
+```
+
+> [!WARNING]
+> This is deprecated because it's merges together the intent of a user wanting a _specific_ version and also the intent of testing against the latest versions of Gradle. How does an automated process know if the user wanted to test this specific version which also happens to be the latest version?## Accessing Gradle test versions
+
+### Java tests with `@GradlePluginTests`
+
+Tests annotated with `@GradlePluginTests` automatically run with all the versions specified in the config file. 
+
+### [Deprecated] Groovy tests with spock e.g. `IntegrationSpec`
+
+Access Gradle test versions via the `GradleTestVersions#gradleVersionForTests` method
+
 ```groovy
 import nebula.test.IntegrationSpec
 import com.palantir.gradle.plugintesting.GradleTestVersions
@@ -125,13 +172,6 @@ class HelloWorldSpec extends IntegrationSpec {
     }
 }
 ```
-The plugin sets default versions to test against, but these can be overridden using the `gradleTestUtils` extension.  e.g.
-
-```groovy
-gradleTestUtils {
-    gradleVersions = ['7.6.4', '8.8']
-}
-```
 
 # Gradle Plugin Testing Framework
 

gradle-plugin-testing/build.gradle

@@ -6,10 +6,18 @@ apply plugin: 'com.palantir.external-publish-gradle-plugin'
 
 dependencies {
     implementation project(':plugin-testing-core')
+    implementation 'com.fasterxml.jackson.core:jackson-databind'
+    implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
     implementation 'com.palantir.baseline:gradle-baseline-java'
     implementation 'com.palantir.suppressible-error-prone:gradle-suppressible-error-prone'
+    implementation 'one.util:streamex'
+    implementation 'com.google.guava:guava'
+
+    annotationProcessor 'org.immutables:value'
 
     compileOnly 'com.palantir.gradle.consistentversions:gradle-consistent-versions'
+    compileOnlyApi 'org.immutables:value::annotations'
+
 
     testImplementation project(':gradle-plugin-testing-junit')
     testImplementation 'com.netflix.nebula:nebula-test'

gradle-plugin-testing/src/main/java/com/palantir/gradle/plugintesting/GradleTestVersionsConfig.java

@@ -0,0 +1,76 @@
+/*
+ * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.gradle.plugintesting;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.google.common.collect.ImmutableSortedSet;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import one.util.streamex.EntryStream;
+import one.util.streamex.StreamEx;
+import org.immutables.value.Value;
+
+@Value.Immutable
+@JsonSerialize(as = ImmutableGradleTestVersionsConfig.class)
+@JsonDeserialize(as = ImmutableGradleTestVersionsConfig.class)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public interface GradleTestVersionsConfig {
+    @JsonProperty("major-versions")
+    @Value.NaturalOrder
+    SortedMap<Integer, String> majorVersions();
+
+    @JsonProperty("extra-versions")
+    @Value.NaturalOrder
+    SortedSet<String> extraVersions();
+
+    final class Builder extends ImmutableGradleTestVersionsConfig.Builder {}
+
+    static Builder builder() {
+        return new Builder();
+    }
+
+    default GradleTestVersionsConfig withoutMajorVersion(int majorVersion) {
+        SortedMap<Integer, String> newMajorVersions = EntryStream.of(majorVersions())
+                .filterKeys(maj -> maj != majorVersion)
+                .toSortedMap();
+        SortedSet<String> newExtraVersions = StreamEx.of(extraVersions())
+                .filter(version -> getMajorVersion(version) != majorVersion)
+                .toCollection(TreeSet::new);
+        return builder()
+                .from(this)
+                .majorVersions(newMajorVersions)
+                .extraVersions(newExtraVersions)
+                .build();
+    }
+
+    default SortedSet<String> allVersions() {
+        return ImmutableSortedSet.<String>naturalOrder()
+                .addAll(majorVersions().values())
+                .addAll(extraVersions())
+                .build();
+    }
+
+    private static int getMajorVersion(String newMajorVersion) {
+        int dotIndex = newMajorVersion.indexOf('.');
+        String majorVersionStr = dotIndex == -1 ? newMajorVersion : newMajorVersion.substring(0, dotIndex);
+        return Integer.parseInt(majorVersionStr);
+    }
+}

gradle-plugin-testing/src/main/java/com/palantir/gradle/plugintesting/PluginTestingExtension.java

@@ -30,7 +30,10 @@ public abstract class PluginTestingExtension {
 
     /**
      * Gradle versions to test against.
+     * @deprecated Use {@code gradle/gradle-test-versions.yml} to set Gradle versions to test against. This property
+     * will be removed as part of rolling out {@code gradle/gradle-test-versions.yml}.
      */
+    @Deprecated
     public abstract SetProperty<String> getGradleVersions();
 
     /**
@@ -41,8 +44,5 @@ public abstract class PluginTestingExtension {
     public PluginTestingExtension() {
         getIgnoreGradleDeprecations().convention(true);
         getConfigurationCacheEnabled().convention(false);
-        // TODO(#XXX): Should this be the latest gradle 8, or maybe whatever this plugin is compiled against?
-        // or is this the set of "milestone" versions and we dynamically add the version of the consuming project?
-        getGradleVersions().convention(GradleTestVersions.DEFAULT_TEST_GRADLE_VERSIONS);
     }
 }

gradle-plugin-testing/src/main/java/com/palantir/gradle/plugintesting/PluginTestingPlugin.java

@@ -15,6 +15,11 @@
  */
 package com.palantir.gradle.plugintesting;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.common.collect.ImmutableSet;
 import com.palantir.baseline.tasks.CheckUnusedDependenciesParentTask;
 import com.palantir.gradle.plugintesting.TestDependencyVersionsTask.TestDependency;
 import com.palantir.gradle.suppressibleerrorprone.SuppressibleErrorProneExtension;
@@ -24,6 +29,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import javax.inject.Inject;
 import org.gradle.api.Action;
 import org.gradle.api.NamedDomainObjectProvider;
 import org.gradle.api.Plugin;
@@ -32,15 +38,19 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.DependencyScopeConfiguration;
 import org.gradle.api.artifacts.ResolvableConfiguration;
+import org.gradle.api.file.ProjectLayout;
+import org.gradle.api.file.RegularFile;
 import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.testing.Test;
 import org.gradle.plugin.devel.plugins.JavaGradlePluginPlugin;
 import org.gradle.plugin.devel.tasks.PluginUnderTestMetadata;
+import org.gradle.util.GradleVersion;
 
-public class PluginTestingPlugin implements Plugin<Project> {
+public abstract class PluginTestingPlugin implements Plugin<Project> {
     /**
      * Used in tests to pick up the current version of this plugin.
      */
@@ -50,9 +60,16 @@ public class PluginTestingPlugin implements Plugin<Project> {
             List.of("plugin-testing-core", "configuration-cache-spec", "gradle-plugin-testing-junit");
 
     public static final Set<String> PATCHABLE_CHECKS = Set.of("GradleTestStringFormatting", "GradleTestPluginsBlock");
+    public static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory());
 
     private static final String MAVEN_GROUP = "com.palantir.gradle.plugintesting";
 
+    @Inject
+    protected abstract ProviderFactory getProviderFactory();
+
+    @Inject
+    protected abstract ProjectLayout getProjectLayout();
+
     private static String coreMavenCoordinates(String name) {
         return MAVEN_GROUP + ":" + name;
     }
@@ -71,7 +88,7 @@ public void apply(Project project) {
     }
 
     @SuppressWarnings("for-rollout:TaskDependsOn")
-    private static void doApply(Project project) {
+    private void doApply(Project project) {
         PluginTestingExtension testUtilsExt = project.getExtensions().getByType(PluginTestingExtension.class);
 
         setupErrorprones(project);
@@ -112,8 +129,18 @@ public void execute(Task _task) {
                             testDependenciesFileAbsolutePath.get());
 
                     // add system property for what versions of gradle should be used in tests
-                    String versions =
-                            String.join(",", testUtilsExt.getGradleVersions().get());
+                    Set<String> versionsFromConfig =
+                            readGradleTestingVersionsConfigFile().getOrElse(Set.of());
+                    Set<String> versionsFromExtension =
+                            testUtilsExt.getGradleVersions().get();
+                    Set<String> gradleTestVersions = ImmutableSet.<String>builder()
+                            .addAll(versionsFromConfig)
+                            .addAll(versionsFromExtension)
+                            .build();
+                    if (gradleTestVersions.isEmpty()) {
+                        gradleTestVersions = Set.of(GradleVersion.current().getVersion());
+                    }
+                    String versions = String.join(",", gradleTestVersions);
                     test.systemProperty(GradleTestVersions.TEST_GRADLE_VERSIONS_SYSTEM_PROPERTY, versions);
 
                     // add system property for whether to use configuration-cache by default in tests
@@ -207,4 +234,21 @@ private static void addGradlePluginForTestingConfigurations(Project project) {
                     .test(scope -> scope.from(gradlePluginForTestingResolvable.getName()));
         });
     }
+
+    private Provider<Set<String>> readGradleTestingVersionsConfigFile() {
+        RegularFile testVersionsConfigPath =
+                getProjectLayout().getProjectDirectory().file("gradle/gradle-test-versions.yml");
+
+        Provider<GradleTestVersionsConfig> config = getProviderFactory()
+                .fileContents(testVersionsConfigPath)
+                .getAsText()
+                .map(text -> {
+                    try {
+                        return YAML_MAPPER.readValue(text, new TypeReference<>() {});
+                    } catch (JsonProcessingException e) {
+                        throw new IllegalArgumentException("Failed to parse gradle/gradle-test-versions.yml", e);
+                    }
+                });
+        return config.map(GradleTestVersionsConfig::allVersions);
+    }
 }