From 23533f881f3045ee96b631f511b8399834aa20cb Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Wed, 28 Dec 2022 22:48:19 +0100 Subject: [PATCH 01/13] Show release notes URL from POM in PR body --- .../core/coursier/CoursierAlg.scala | 132 +++++++++--------- .../core/coursier/DependencyMetadata.scala | 47 +++++++ .../core/nurture/NurtureAlg.scala | 43 +++--- .../org/scalasteward/core/util/uri.scala | 10 +- .../scalasteward/core/vcs/VCSExtraAlg.scala | 8 +- .../org/scalasteward/core/TestSyntax.scala | 18 ++- .../core/coursier/CoursierAlgTest.scala | 112 ++++++++------- .../core/vcs/VCSExtraAlgTest.scala | 20 +++ 8 files changed, 236 insertions(+), 154 deletions(-) create mode 100644 modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala index e2c5062393..056595b041 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala @@ -16,15 +16,14 @@ package org.scalasteward.core.coursier +import cats.Parallel import cats.effect._ import cats.implicits._ -import cats.{Applicative, Parallel} import coursier.cache.{CachePolicy, FileCache} import coursier.core.{Authentication, Project} -import coursier.{Fetch, Info, Module, ModuleName, Organization} -import org.http4s.Uri +import coursier.{Fetch, Module, ModuleName, Organization} import org.scalasteward.core.data.Resolver.Credentials -import org.scalasteward.core.data.{Dependency, Resolver, Scope, Version} +import org.scalasteward.core.data.{Dependency, Resolver, Version} import org.scalasteward.core.util.uri import org.typelevel.log4cats.Logger @@ -32,16 +31,9 @@ import org.typelevel.log4cats.Logger * metadata. */ trait CoursierAlg[F[_]] { - def getArtifactUrl(dependency: Scope.Dependency): F[Option[Uri]] + def getMetadata(dependency: Dependency, resolvers: List[Resolver]): F[Option[DependencyMetadata]] def getVersions(dependency: Dependency, resolver: Resolver): F[List[Version]] - - final def getArtifactIdUrlMapping(dependencies: Scope.Dependencies)(implicit - F: Applicative[F] - ): F[Map[String, Uri]] = - dependencies.sequence - .traverseFilter(dep => getArtifactUrl(dep).map(_.map(dep.value.artifactId.name -> _))) - .map(_.toMap) } object CoursierAlg { @@ -50,64 +42,72 @@ object CoursierAlg { parallel: Parallel[F], F: Sync[F] ): CoursierAlg[F] = { - val fetch: Fetch[F] = Fetch[F](FileCache[F]()) + val fetch: Fetch[F] = + Fetch[F](FileCache[F]()) val cacheNoTtl: FileCache[F] = FileCache[F]().withTtl(None).withCachePolicies(List(CachePolicy.Update)) new CoursierAlg[F] { - override def getArtifactUrl(dependency: Scope.Dependency): F[Option[Uri]] = - convertToCoursierTypes(dependency).flatMap((getArtifactUrlImpl _).tupled) + override def getMetadata( + dependency: Dependency, + resolvers: List[Resolver] + ): F[Option[DependencyMetadata]] = + resolvers.traverseFilter(convertResolver(_).attempt.map(_.toOption)).flatMap { + repositories => + val csrDependency = toCoursierDependency(dependency) + getMetadataImpl(csrDependency, repositories, None) + } - private def getArtifactUrlImpl( + private def getMetadataImpl( dependency: coursier.Dependency, - repositories: List[coursier.Repository] - ): F[Option[Uri]] = { + repositories: List[coursier.Repository], + acc: Option[DependencyMetadata] + ): F[Option[DependencyMetadata]] = { val fetchArtifacts = fetch .withArtifactTypes(Set(coursier.Type.pom, coursier.Type.ivy)) .withDependencies(List(dependency)) .withRepositories(repositories) + fetchArtifacts.ioResult.attempt.flatMap { case Left(throwable) => - logger.debug(throwable)(s"Failed to fetch artifacts of $dependency").as(None) + logger.debug(throwable)(s"Failed to fetch artifacts of $dependency").as(acc) case Right(result) => val maybeProject = result.resolution.projectCache .get(dependency.moduleVersion) .map { case (_, project) => project } maybeProject.traverseFilter { project => - getScmUrlOrHomePage(project.info) match { - case Some(url) => F.pure(Some(url)) - case None => - getParentDependency(project).traverseFilter(getArtifactUrlImpl(_, repositories)) + val metadata = { + val current = metadataFrom(project) + acc.fold(current)(_.enrichWith(current)) } + if (metadata.homePage.isEmpty || metadata.scmUrl.isEmpty) { + parentOf(project) match { + case Some(parent) => getMetadataImpl(parent, repositories, Some(metadata)) + case None => F.pure(Some(metadata)) + } + } else F.pure(Some(metadata)) } } } override def getVersions(dependency: Dependency, resolver: Resolver): F[List[Version]] = - toCoursierRepository(resolver) match { - case Left(message) => - logger.error(message) >> F.raiseError(new Throwable(message)) - case Right(repository) => - val module = toCoursierModule(dependency) - repository.versions(module, cacheNoTtl.fetch).run.flatMap { - case Left(message) => - logger.debug(message) >> F.raiseError(new Throwable(message)) - case Right((versions, _)) => F.pure(versions.available.map(Version.apply).sorted) - } + convertResolver(resolver).flatMap { repository => + val module = toCoursierModule(dependency) + repository.versions(module, cacheNoTtl.fetch).run.flatMap { + case Left(message) => + logger.debug(message) >> F.raiseError[List[Version]](new Throwable(message)) + case Right((versions, _)) => + F.pure(versions.available.map(Version.apply).sorted) + } } - private def convertToCoursierTypes( - dependency: Scope.Dependency - ): F[(coursier.Dependency, List[coursier.Repository])] = - dependency.resolvers.traverseFilter(convertResolver).map { repositories => - (toCoursierDependency(dependency.value), repositories) - } - - private def convertResolver(resolver: Resolver): F[Option[coursier.Repository]] = + private def convertResolver(resolver: Resolver): F[coursier.Repository] = toCoursierRepository(resolver) match { - case Right(repository) => F.pure(Some(repository)) - case Left(message) => logger.error(s"Failed to convert $resolver: $message").as(None) + case Right(repository) => F.pure(repository) + case Left(message) => + logger.error(s"Failed to convert $resolver: $message") >> + F.raiseError[coursier.Repository](new Throwable(message)) } } } @@ -127,40 +127,40 @@ object CoursierAlg { private def toCoursierRepository(resolver: Resolver): Either[String, coursier.Repository] = resolver match { case Resolver.MavenRepository(_, location, creds, headers) => - Right( - coursier.maven.MavenRepository - .apply(location, toCoursierAuthentication(creds, headers)) - ) + val authentication = toCoursierAuthentication(creds, headers) + Right(coursier.maven.MavenRepository.apply(location, authentication)) case Resolver.IvyRepository(_, pattern, creds, headers) => - coursier.ivy.IvyRepository - .parse(pattern, authentication = toCoursierAuthentication(creds, headers)) + val authentication = toCoursierAuthentication(creds, headers) + coursier.ivy.IvyRepository.parse(pattern, authentication = authentication) } private def toCoursierAuthentication( credentials: Option[Credentials], headers: List[Resolver.Header] ): Option[Authentication] = - if (credentials.isEmpty && headers.isEmpty) { - None - } else { - Some( - new Authentication( - credentials.fold("")(_.user), - credentials.map(_.pass), - headers.map(h => (h.key, h.value)), - optional = false, - None, - httpsOnly = true, - passOnRedirect = false - ) + Option.when(credentials.nonEmpty || headers.nonEmpty) { + new Authentication( + credentials.fold("")(_.user), + credentials.map(_.pass), + headers.map(h => (h.key, h.value)), + optional = false, + realmOpt = None, + httpsOnly = true, + passOnRedirect = false ) } - private def getParentDependency(project: Project): Option[coursier.Dependency] = + private def metadataFrom(project: Project): DependencyMetadata = + DependencyMetadata( + homePage = uri.fromStringWithScheme(project.info.homePage), + scmUrl = project.info.scm.flatMap(_.url).flatMap(uri.fromStringWithScheme), + releaseNotesUrl = project.properties + .collectFirst { case ("releaseNotesUrl", value) => value } + .flatMap(uri.fromStringWithScheme) + ) + + private def parentOf(project: Project): Option[coursier.Dependency] = project.parent.map { case (module, version) => coursier.Dependency(module, version).withTransitive(false) } - - private def getScmUrlOrHomePage(info: Info): Option[Uri] = - uri.findBrowsableUrl(info.scm.flatMap(_.url).toList :+ info.homePage) } diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala new file mode 100644 index 0000000000..58ef386ab1 --- /dev/null +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2018-2022 Scala Steward contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.scalasteward.core.coursier + +import cats.Monad +import cats.syntax.all._ +import org.http4s.Uri +import org.scalasteward.core.util.uri + +final case class DependencyMetadata( + homePage: Option[Uri], + scmUrl: Option[Uri], + releaseNotesUrl: Option[Uri] +) { + def enrichWith(other: DependencyMetadata): DependencyMetadata = + DependencyMetadata( + homePage = homePage.orElse(other.homePage), + scmUrl = scmUrl.orElse(other.scmUrl), + releaseNotesUrl = releaseNotesUrl.orElse(other.releaseNotesUrl) + ) + + def filterUrls[F[_]](f: Uri => F[Boolean])(implicit F: Monad[F]): F[DependencyMetadata] = + for { + homePage <- homePage.filterA(f) + scmUrl <- scmUrl.filterA(f) + releaseNotesUrl <- releaseNotesUrl.filterA(f) + } yield DependencyMetadata(homePage, scmUrl, releaseNotesUrl) + + def repoUrl: Option[Uri] = { + val urls = scmUrl.toList ++ homePage.toList + urls.find(_.scheme.exists(uri.httpSchemes)).orElse(urls.headOption) + } +} diff --git a/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala index acd7111689..416228fd58 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala @@ -16,7 +16,7 @@ package org.scalasteward.core.nurture -import cats.Id +import cats.{Applicative, Id} import cats.effect.Concurrent import cats.implicits._ import org.scalasteward.core.application.Config.VCSCfg @@ -26,13 +26,12 @@ import org.scalasteward.core.data._ import org.scalasteward.core.edit.{EditAlg, EditAttempt} import org.scalasteward.core.git.{Branch, Commit, GitAlg} import org.scalasteward.core.repoconfig.PullRequestUpdateStrategy -import org.scalasteward.core.util.{Nel, UrlChecker} import org.scalasteward.core.util.logger.LoggerOps +import org.scalasteward.core.util.{Nel, UrlChecker} import org.scalasteward.core.vcs.data._ import org.scalasteward.core.vcs.{VCSApiAlg, VCSExtraAlg, VCSRepoAlg} import org.scalasteward.core.{git, util, vcs} import org.typelevel.log4cats.Logger -import cats.Applicative final class NurtureAlg[F[_]](config: VCSCfg)(implicit coursierAlg: CoursierAlg[F], @@ -199,25 +198,31 @@ final class NurtureAlg[F[_]](config: VCSCfg)(implicit _ <- logger.info(s"Create PR ${data.updateBranch.name}") dependenciesWithNextVersion = dependenciesUpdatedWithNextAndCurrentVersion(data.update) resolvers = data.repoData.cache.dependencyInfos.flatMap(_.resolvers) - dependencyScope = Scope( - value = dependenciesWithNextVersion.map { case (_, dependency) => dependency }, - resolvers = resolvers - ) - artifactIdToUrl <- coursierAlg.getArtifactIdUrlMapping(dependencyScope) - existingArtifactUrlsList <- artifactIdToUrl.toList.filterA { case (_, uri) => - urlChecker.exists(uri) - } - existingArtifactUrlsMap = existingArtifactUrlsList.toMap + dependencyToMetadata <- dependenciesWithNextVersion + .traverseFilter { case (_, dependency) => + coursierAlg + .getMetadata(dependency, resolvers) + .flatMap(_.traverse(_.filterUrls(urlChecker.exists))) + .map(_.tupleLeft(dependency)) + } + .map(_.toMap) + artifactIdToUrl = dependencyToMetadata.toList.mapFilter { case (dependency, metadata) => + metadata.repoUrl.tupleLeft(dependency.artifactId.name) + }.toMap releaseRelatedUrls <- dependenciesWithNextVersion.flatTraverse { case (currentVersion, dependency) => - existingArtifactUrlsMap - .get(dependency.artifactId.name) - .toList - .traverse(uri => + dependencyToMetadata.get(dependency).toList.flatTraverse { metadata => + metadata.repoUrl.toList.traverse { uri => vcsExtraAlg - .getReleaseRelatedUrls(uri, currentVersion, dependency.version) + .getReleaseRelatedUrls( + uri, + metadata.releaseNotesUrl, + currentVersion, + dependency.version + ) .tupleLeft(dependency.artifactId.name) - ) + } + } } filesWithOldVersion <- data.update @@ -229,7 +234,7 @@ final class NurtureAlg[F[_]](config: VCSCfg)(implicit data, branchName, edits, - existingArtifactUrlsMap, + artifactIdToUrl, releaseRelatedUrls.toMap, filesWithOldVersion, data.repoData.config.pullRequests.includeMatchedLabels diff --git a/modules/core/src/main/scala/org/scalasteward/core/util/uri.scala b/modules/core/src/main/scala/org/scalasteward/core/util/uri.scala index 845a46d43c..2846c3206c 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/util/uri.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/util/uri.scala @@ -41,11 +41,9 @@ object uri { val withUserInfo: Optional[Uri, UserInfo] = authorityWithUserInfo.compose(withAuthority) - private val httpSchemes: Set[Scheme] = - Set(Scheme.https, Scheme.http) + def fromStringWithScheme(s: String): Option[Uri] = + Uri.fromString(s).toOption.filter(_.scheme.isDefined) - def findBrowsableUrl(xs: List[String]): Option[Uri] = { - val urls = xs.flatMap(Uri.fromString(_).toList).filter(_.scheme.isDefined) - urls.find(_.scheme.exists(httpSchemes)).orElse(urls.headOption) - } + val httpSchemes: Set[Scheme] = + Set(Scheme.https, Scheme.http) } diff --git a/modules/core/src/main/scala/org/scalasteward/core/vcs/VCSExtraAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/vcs/VCSExtraAlg.scala index 842b957e5b..d3611535db 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/vcs/VCSExtraAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/vcs/VCSExtraAlg.scala @@ -27,6 +27,7 @@ import org.scalasteward.core.vcs trait VCSExtraAlg[F[_]] { def getReleaseRelatedUrls( repoUrl: Uri, + releaseNotesUrl: Option[Uri], currentVersion: Version, nextVersion: Version ): F[List[ReleaseRelatedUrl]] @@ -40,17 +41,18 @@ object VCSExtraAlg { new VCSExtraAlg[F] { override def getReleaseRelatedUrls( repoUrl: Uri, + releaseNotesUrl: Option[Uri], currentVersion: Version, nextVersion: Version ): F[List[ReleaseRelatedUrl]] = - vcs - .possibleReleaseRelatedUrls( + (releaseNotesUrl.toList.map(ReleaseRelatedUrl.CustomReleaseNotes.apply) ++ + vcs.possibleReleaseRelatedUrls( config.tpe, config.apiHost, repoUrl, currentVersion, nextVersion - ) + )) .filterA(releaseRelatedUrl => urlChecker.exists(releaseRelatedUrl.url)) } } diff --git a/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala b/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala index ad1fdb4bab..0a0e8029a5 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala @@ -5,19 +5,17 @@ import org.scalasteward.core.data._ import org.scalasteward.core.util.Nel object TestSyntax { + val sbtPluginReleases: IvyRepository = + IvyRepository( + "sbt-plugin-releases", + "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[defaultPattern]", + None, + Nil + ) + implicit class GenericOps[A](val self: A) extends AnyVal { def withMavenCentral: Scope[A] = Scope(self, List(Resolver.mavenCentral)) - - def withSbtPluginReleases: Scope[A] = { - val sbtPluginReleases = IvyRepository( - "sbt-plugin-releases", - "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[defaultPattern]", - None, - Nil - ) - Scope(self, List(sbtPluginReleases)) - } } implicit class StringOps(private val self: String) extends AnyVal { diff --git a/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala index 47ce72ae88..511daa01f5 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala @@ -4,88 +4,100 @@ import munit.CatsEffectSuite import org.http4s.syntax.literals._ import org.scalasteward.core.TestSyntax._ import org.scalasteward.core.buildtool.sbt.data.{SbtVersion, ScalaVersion} +import org.scalasteward.core.data.Resolver import org.scalasteward.core.mock.MockContext.context.coursierAlg import org.scalasteward.core.mock.MockState class CoursierAlgTest extends CatsEffectSuite { - test("getArtifactUrl: library") { + private val resolvers = List(Resolver.mavenCentral) + + private val emptyMetadata = DependencyMetadata(None, None, None) + + test("getMetadata: with homePage and scmUrl") { val dep = "org.typelevel".g % ("cats-effect", "cats-effect_2.12").a % "1.0.0" - coursierAlg.getArtifactUrl(dep.withMavenCentral).runA(MockState.empty).map { obtained => - assertEquals(obtained, Some(uri"https://github.com/typelevel/cats-effect")) - } + val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) + val expected = Some( + emptyMetadata.copy( + homePage = Some(uri"https://typelevel.org/cats-effect/"), + scmUrl = Some(uri"https://github.com/typelevel/cats-effect") + ) + ) + assertIO(obtained, expected) } - test("getArtifactUrl: defaults to homepage") { + test("getMetadata: homePage only") { val artifactId = ("play-ws-standalone-json", "play-ws-standalone-json_2.12").a val dep = "com.typesafe.play".g % artifactId % "2.1.0-M7" - coursierAlg.getArtifactUrl(dep.withMavenCentral).runA(MockState.empty).map { obtained => - assertEquals(obtained, Some(uri"https://github.com/playframework/play-ws")) - } + val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) + val expected = + Some(emptyMetadata.copy(homePage = Some(uri"https://github.com/playframework/play-ws"))) + assertIO(obtained, expected) } - test("getArtifactUrl: URL with no or invalid scheme 1") { + test("getMetadata: scmUrl without scheme") { val dep = "org.msgpack".g % "msgpack-core".a % "0.8.20" - coursierAlg.getArtifactUrl(dep.withMavenCentral).runA(MockState.empty).map { obtained => - assertEquals(obtained, Some(uri"http://msgpack.org/")) - } + val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) + val expected = Some(emptyMetadata.copy(homePage = Some(uri"http://msgpack.org/"))) + assertIO(obtained, expected) } - test("getArtifactUrl: URL with no or invalid scheme 2") { + test("getMetadata: scmUrl with git scheme") { val dep = "org.xhtmlrenderer".g % "flying-saucer-parent".a % "9.0.1" - coursierAlg.getArtifactUrl(dep.withMavenCentral).runA(MockState.empty).map { obtained => - assertEquals(obtained, Some(uri"http://code.google.com/p/flying-saucer/")) - } + val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) + val expected = Some( + emptyMetadata.copy( + homePage = Some(uri"http://code.google.com/p/flying-saucer/"), + scmUrl = Some(uri"git://github.com/flyingsaucerproject/flyingsaucer.git") + ) + ) + assertIO(obtained, expected) } - test("getArtifactUrl: from parent") { + test("getMetadata: homePage from parent") { val dep = "net.bytebuddy".g % "byte-buddy".a % "1.10.5" - coursierAlg.getArtifactUrl(dep.withMavenCentral).runA(MockState.empty).map { obtained => - assertEquals(obtained, Some(uri"https://bytebuddy.net")) - } + val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) + val expected = Some(emptyMetadata.copy(homePage = Some(uri"https://bytebuddy.net"))) + assertIO(obtained, expected) } - test("getArtifactUrl: minimal pom") { + test("getMetadata: minimal POM") { val dep = "altrmi".g % "altrmi-common".a % "0.9.6" - coursierAlg.getArtifactUrl(dep.withMavenCentral).runA(MockState.empty).map { obtained => - assertEquals(obtained, None) - } + val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) + val expected = Some(emptyMetadata) + assertIO(obtained, expected) } - test("getArtifactUrl: sbt plugin on Maven Central") { + test("getMetadata: sbt plugin on Maven Central") { val dep = ("org.xerial.sbt".g % "sbt-sonatype".a % "3.8") .copy(sbtVersion = Some(SbtVersion("1.0")), scalaVersion = Some(ScalaVersion("2.12"))) - coursierAlg.getArtifactUrl(dep.withMavenCentral).runA(MockState.empty).map { obtained => - assertEquals(obtained, Some(uri"https://github.com/xerial/sbt-sonatype")) - } + val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) + val expected = Some( + emptyMetadata.copy( + homePage = Some(uri"https://github.com/xerial/sbt-sonatype"), + scmUrl = Some(uri"https://github.com/xerial/sbt-sonatype") + ) + ) + assertIO(obtained, expected) } - test("getArtifactUrl: sbt plugin on sbt-plugin-releases") { + test("getMetadata: sbt plugin on sbt-plugin-releases") { val dep = ("com.github.gseitz".g % "sbt-release".a % "1.0.12") .copy(sbtVersion = Some(SbtVersion("1.0")), scalaVersion = Some(ScalaVersion("2.12"))) - coursierAlg.getArtifactUrl(dep.withSbtPluginReleases).runA(MockState.empty).map { obtained => - assertEquals(obtained, Some(uri"https://github.com/sbt/sbt-release")) - } + val obtained = coursierAlg.getMetadata(dep, List(sbtPluginReleases)).runA(MockState.empty) + val expected = + Some(emptyMetadata.copy(homePage = Some(uri"https://github.com/sbt/sbt-release"))) + assertIO(obtained, expected) } - test("getArtifactUrl: invalid scm URL but valid homepage") { + test("getMetadata: scmUrl with github scheme") { val dep = "com.github.japgolly.scalajs-react".g % ("core", "core_sjs1_2.13").a % "2.0.0-RC5" - coursierAlg.getArtifactUrl(dep.withMavenCentral).runA(MockState.empty).map { obtained => - assertEquals(obtained, Some(uri"https://github.com/japgolly/scalajs-react")) - } - } - - test("getArtifactIdUrlMapping") { - val deps = List( - "org.typelevel".g % ("cats-core", "cats-core_2.12").a % "1.6.0", - "org.typelevel".g % ("cats-effect", "cats-effect_2.12").a % "1.0.0" + val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) + val expected = Some( + emptyMetadata.copy( + homePage = Some(uri"https://github.com/japgolly/scalajs-react"), + scmUrl = Some(uri"github.com:japgolly/scalajs-react.git") + ) ) - coursierAlg.getArtifactIdUrlMapping(deps.withMavenCentral).runA(MockState.empty).map { - obtained => - val expected = Map( - "cats-core" -> uri"https://github.com/typelevel/cats", - "cats-effect" -> uri"https://github.com/typelevel/cats-effect" - ) - assertEquals(obtained, expected) - } + assertIO(obtained, expected) } } diff --git a/modules/core/src/test/scala/org/scalasteward/core/vcs/VCSExtraAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/vcs/VCSExtraAlgTest.scala index 92059aefcc..98770ec12a 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/vcs/VCSExtraAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/vcs/VCSExtraAlgTest.scala @@ -11,6 +11,7 @@ import org.scalasteward.core.TestInstances.ioLogger import org.scalasteward.core.TestSyntax._ import org.scalasteward.core.application.Config.VCSCfg import org.scalasteward.core.data.ReleaseRelatedUrl +import org.scalasteward.core.data.ReleaseRelatedUrl.CustomReleaseNotes import org.scalasteward.core.mock.MockConfig import org.scalasteward.core.util._ @@ -19,6 +20,7 @@ class VCSExtraAlgTest extends FunSuite { HttpRoutes.of[IO] { case HEAD -> Root / "foo" / "bar" / "compare" / "v0.1.0...v0.2.0" => Ok("exist") case HEAD -> Root / "foo" / "buz" / "compare" / "v0.1.0...v0.2.0" => PermanentRedirect() + case HEAD -> Root / "foo" / "buz" / "README.md" => Ok("exist") case _ => NotFound() } @@ -38,6 +40,7 @@ class VCSExtraAlgTest extends FunSuite { vcsExtraAlg .getReleaseRelatedUrls( repoUrl = uri"https://github.com/foo/foo", + releaseNotesUrl = None, currentVersion = updateFoo.currentVersion, nextVersion = updateFoo.nextVersion ) @@ -49,6 +52,7 @@ class VCSExtraAlgTest extends FunSuite { vcsExtraAlg .getReleaseRelatedUrls( repoUrl = uri"https://github.com/foo/bar", + releaseNotesUrl = None, currentVersion = updateBar.currentVersion, nextVersion = updateBar.nextVersion ) @@ -62,12 +66,25 @@ class VCSExtraAlgTest extends FunSuite { vcsExtraAlg .getReleaseRelatedUrls( repoUrl = uri"https://github.com/foo/buz", + releaseNotesUrl = None, currentVersion = updateBuz.currentVersion, nextVersion = updateBuz.nextVersion ) .unsafeRunSync(), List.empty ) + + assertEquals( + vcsExtraAlg + .getReleaseRelatedUrls( + repoUrl = uri"https://github.com/foo/buz", + releaseNotesUrl = Some(uri"https://github.com/foo/buz/README.md#changelog"), + currentVersion = updateBuz.currentVersion, + nextVersion = updateBuz.nextVersion + ) + .unsafeRunSync(), + List(CustomReleaseNotes(uri"https://github.com/foo/buz/README.md#changelog")) + ) } test("getBranchCompareUrl: github on prem") { @@ -84,6 +101,7 @@ class VCSExtraAlgTest extends FunSuite { githubOnPremVcsExtraAlg .getReleaseRelatedUrls( repoUrl = uri"https://github.on-prem.com/foo/foo", + releaseNotesUrl = None, currentVersion = updateFoo.currentVersion, nextVersion = updateFoo.nextVersion ) @@ -95,6 +113,7 @@ class VCSExtraAlgTest extends FunSuite { githubOnPremVcsExtraAlg .getReleaseRelatedUrls( repoUrl = uri"https://github.on-prem.com/foo/bar", + releaseNotesUrl = None, currentVersion = updateBar.currentVersion, nextVersion = updateBar.nextVersion ) @@ -110,6 +129,7 @@ class VCSExtraAlgTest extends FunSuite { githubOnPremVcsExtraAlg .getReleaseRelatedUrls( repoUrl = uri"https://github.on-prem.com/foo/buz", + releaseNotesUrl = None, currentVersion = updateFoo.currentVersion, nextVersion = updateFoo.nextVersion ) From 6ce91d5d2938a4626bfc17e6c3553095890921e3 Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Wed, 28 Dec 2022 23:58:49 +0100 Subject: [PATCH 02/13] Test getMetadata with an empty list of resolvers --- .../org/scalasteward/core/coursier/CoursierAlgTest.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala index 511daa01f5..82e0afed34 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala @@ -100,4 +100,11 @@ class CoursierAlgTest extends CatsEffectSuite { ) assertIO(obtained, expected) } + + test("getMetadata: no resolvers") { + val dep = "org.example".g % "foo".a % "1.0.0" + val obtained = coursierAlg.getMetadata(dep, List.empty).runA(MockState.empty) + val expected = None + assertIO(obtained, expected) + } } From 73cc23cd5d223812772ccb13110fc56f36facd35 Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Thu, 29 Dec 2022 00:40:50 +0100 Subject: [PATCH 03/13] Test getMetadata with a resolver with headers --- .../src/test/scala/org/scalasteward/core/TestSyntax.scala | 3 +-- .../org/scalasteward/core/coursier/CoursierAlgTest.scala | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala b/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala index 0a0e8029a5..e18c2bcf8d 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala @@ -9,8 +9,7 @@ object TestSyntax { IvyRepository( "sbt-plugin-releases", "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[defaultPattern]", - None, - Nil + None ) implicit class GenericOps[A](val self: A) extends AnyVal { diff --git a/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala index 82e0afed34..aa57f19918 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala @@ -107,4 +107,12 @@ class CoursierAlgTest extends CatsEffectSuite { val expected = None assertIO(obtained, expected) } + + test("getMetadata: resolver with headers") { + val dep = "org.typelevel".g % ("cats-effect", "cats-effect_2.12").a % "1.0.0" + val resolvers = + List(Resolver.mavenCentral.copy(headers = List(Resolver.Header("X-Foo", "bar")))) + val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty).map(_.isDefined) + assertIOBoolean(obtained) + } } From 0572e736b8567260e1199603e0ac0487b7b0d366 Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Thu, 29 Dec 2022 06:13:04 +0100 Subject: [PATCH 04/13] Add DependencyMetadataTest --- .../coursier/DependencyMetadataTest.scala | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 modules/core/src/test/scala/org/scalasteward/core/coursier/DependencyMetadataTest.scala diff --git a/modules/core/src/test/scala/org/scalasteward/core/coursier/DependencyMetadataTest.scala b/modules/core/src/test/scala/org/scalasteward/core/coursier/DependencyMetadataTest.scala new file mode 100644 index 0000000000..06ee49f09c --- /dev/null +++ b/modules/core/src/test/scala/org/scalasteward/core/coursier/DependencyMetadataTest.scala @@ -0,0 +1,28 @@ +package org.scalasteward.core.coursier + +import cats.Id +import cats.syntax.all._ +import munit.FunSuite +import org.http4s.implicits.http4sLiteralsSyntax + +class DependencyMetadataTest extends FunSuite { + test("filterUrls") { + val metadata = DependencyMetadata( + homePage = Some(uri"https://github.com/japgolly/scalajs-react"), + scmUrl = Some(uri"github.com:japgolly/scalajs-react.git"), + releaseNotesUrl = None + ) + val obtained = metadata.filterUrls(_.renderString.startsWith("http").pure[Id]) + assertEquals(obtained, metadata.copy(scmUrl = None)) + } + + test("repoUrl: scmUrl with non-http scheme") { + val homePage = Some(uri"https://github.com/japgolly/scalajs-react") + val metadata = DependencyMetadata( + homePage = homePage, + scmUrl = Some(uri"github.com:japgolly/scalajs-react.git"), + releaseNotesUrl = None + ) + assertEquals(metadata.repoUrl, homePage) + } +} From 6709e216287cc6af2a5452597b216e0b4426314f Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Mon, 2 Jan 2023 17:32:58 +0100 Subject: [PATCH 05/13] Extract preparePullRequest from createPullRequest and test it --- .../core/application/Context.scala | 1 + .../core/nurture/NurtureAlg.scala | 13 +++- .../core/nurture/NurtureAlgTest.scala | 73 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 modules/core/src/test/scala/org/scalasteward/core/nurture/NurtureAlgTest.scala diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala index be764b2eee..10dfabff0a 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala @@ -66,6 +66,7 @@ final class Context[F[_]](implicit val logger: Logger[F], val mavenAlg: MavenAlg[F], val millAlg: MillAlg[F], + val nurtureAlg: NurtureAlg[F], val pruningAlg: PruningAlg[F], val pullRequestRepository: PullRequestRepository[F], val refreshErrorAlg: RefreshErrorAlg[F], diff --git a/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala index 416228fd58..10f76ab837 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala @@ -193,9 +193,12 @@ final class NurtureAlg[F[_]](config: VCSCfg)(implicit _.updates.flatMap(dependenciesUpdatedWithNextAndCurrentVersion(_)) ) - private def createPullRequest(data: UpdateData, edits: List[EditAttempt]): F[ProcessResult] = + private[nurture] def preparePullRequest( + data: UpdateData, + edits: List[EditAttempt] + ): F[NewPullRequestData] = for { - _ <- logger.info(s"Create PR ${data.updateBranch.name}") + _ <- F.unit dependenciesWithNextVersion = dependenciesUpdatedWithNextAndCurrentVersion(data.update) resolvers = data.repoData.cache.dependencyInfos.flatMap(_.resolvers) dependencyToMetadata <- dependenciesWithNextVersion @@ -239,6 +242,12 @@ final class NurtureAlg[F[_]](config: VCSCfg)(implicit filesWithOldVersion, data.repoData.config.pullRequests.includeMatchedLabels ) + } yield requestData + + private def createPullRequest(data: UpdateData, edits: List[EditAttempt]): F[ProcessResult] = + for { + _ <- logger.info(s"Create PR ${data.updateBranch.name}") + requestData <- preparePullRequest(data, edits) pr <- vcsApiAlg.createPullRequest(data.repo, requestData) _ <- vcsApiAlg .labelPullRequest(data.repo, pr.number, requestData.labels) diff --git a/modules/core/src/test/scala/org/scalasteward/core/nurture/NurtureAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/nurture/NurtureAlgTest.scala new file mode 100644 index 0000000000..dd87ce5203 --- /dev/null +++ b/modules/core/src/test/scala/org/scalasteward/core/nurture/NurtureAlgTest.scala @@ -0,0 +1,73 @@ +package org.scalasteward.core.nurture + +import munit.CatsEffectSuite +import org.http4s.HttpApp +import org.http4s.dsl.Http4sDsl +import org.scalasteward.core.TestInstances._ +import org.scalasteward.core.TestSyntax._ +import org.scalasteward.core.data.{DependencyInfo, RepoData, UpdateData} +import org.scalasteward.core.edit.EditAttempt.UpdateEdit +import org.scalasteward.core.git.{Branch, Commit} +import org.scalasteward.core.mock.MockContext.context.nurtureAlg +import org.scalasteward.core.mock.{MockEff, MockState} +import org.scalasteward.core.repoconfig.RepoConfig +import org.scalasteward.core.vcs.data.{NewPullRequestData, Repo} + +class NurtureAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] { + test("preparePullRequest") { + val repo = Repo("scala-steward-org", "scala-steward") + val dependency = "org.typelevel".g % ("cats-effect", "cats-effect_2.13").a % "3.3.0" + val repoCache = dummyRepoCache.copy(dependencyInfos = + List(List(DependencyInfo(dependency, Nil)).withMavenCentral) + ) + val repoData = RepoData(repo, repoCache, RepoConfig.empty) + val fork = Repo("scala-steward", "scala-steward") + val update = (dependency %> "3.4.0").single + val baseBranch = Branch("main") + val updateBranch = Branch("update/cats-effect-3.4.0") + val updateData = UpdateData(repoData, fork, update, baseBranch, dummySha1, updateBranch) + val edits = List(UpdateEdit(update, Commit(dummySha1))) + val state = MockState.empty.copy(clientResponses = HttpApp { + case HEAD -> Root / "typelevel" / "cats-effect" => Ok() + case _ => NotFound() + }) + val obtained = nurtureAlg.preparePullRequest(updateData, edits).runA(state) + val expected = NewPullRequestData( + title = "Update cats-effect to 3.4.0", + body = + raw"""Updates [org.typelevel:cats-effect](https://github.com/typelevel/cats-effect) from 3.3.0 to 3.4.0. + | + | + |I'll automatically update this PR to resolve conflicts as long as you don't change it yourself. + | + |If you'd like to skip this version, you can just close this PR. If you have any feedback, just mention me in the comments below. + | + |Configure Scala Steward for your repository with a [`.scala-steward.conf`](https://github.com/scala-steward-org/scala-steward/blob/${org.scalasteward.core.BuildInfo.gitHeadCommit}/docs/repo-specific-configuration.md) file. + | + |Have a fantastic day writing Scala! + | + |
+ |Adjust future updates + | + |Add this to your `.scala-steward.conf` file to ignore future updates of this dependency: + |``` + |updates.ignore = [ { groupId = "org.typelevel", artifactId = "cats-effect" } ] + |``` + |Or, add this to slow down future updates of this dependency: + |``` + |dependencyOverrides = [{ + | pullRequests = { frequency = "@monthly" }, + | dependency = { groupId = "org.typelevel", artifactId = "cats-effect" } + |}] + |``` + |
+ | + |labels: library-update, early-semver-minor, semver-spec-minor, commit-count:1 + |""".stripMargin.trim, + head = "scala-steward:update/cats-effect-3.4.0", + base = baseBranch, + labels = List("library-update", "early-semver-minor", "semver-spec-minor", "commit-count:1") + ) + assertIO(obtained, expected) + } +} From 65de10ba3e231290ed15e7b1989b547405c729c6 Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Mon, 2 Jan 2023 19:51:02 +0100 Subject: [PATCH 06/13] Show GH release notes and version diff in preparePullRequest test --- .../org/scalasteward/core/nurture/NurtureAlgTest.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/core/src/test/scala/org/scalasteward/core/nurture/NurtureAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/nurture/NurtureAlgTest.scala index dd87ce5203..19091a31e3 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/nurture/NurtureAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/nurture/NurtureAlgTest.scala @@ -28,14 +28,17 @@ class NurtureAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] { val updateData = UpdateData(repoData, fork, update, baseBranch, dummySha1, updateBranch) val edits = List(UpdateEdit(update, Commit(dummySha1))) val state = MockState.empty.copy(clientResponses = HttpApp { - case HEAD -> Root / "typelevel" / "cats-effect" => Ok() - case _ => NotFound() + case HEAD -> Root / "typelevel" / "cats-effect" => Ok() + case HEAD -> Root / "typelevel" / "cats-effect" / "releases" / "tag" / "v3.4.0" => Ok() + case HEAD -> Root / "typelevel" / "cats-effect" / "compare" / "v3.3.0...v3.4.0" => Ok() + case _ => NotFound() }) val obtained = nurtureAlg.preparePullRequest(updateData, edits).runA(state) val expected = NewPullRequestData( title = "Update cats-effect to 3.4.0", body = raw"""Updates [org.typelevel:cats-effect](https://github.com/typelevel/cats-effect) from 3.3.0 to 3.4.0. + |[GitHub Release Notes](https://github.com/typelevel/cats-effect/releases/tag/v3.4.0) - [Version Diff](https://github.com/typelevel/cats-effect/compare/v3.3.0...v3.4.0) | | |I'll automatically update this PR to resolve conflicts as long as you don't change it yourself. From 762a82ddf50482f1e2e71e58ab10435ca22609e9 Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Mon, 2 Jan 2023 20:30:13 +0100 Subject: [PATCH 07/13] Fetch the parent POM if repoUrl is empty --- .../main/scala/org/scalasteward/core/coursier/CoursierAlg.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala index 28f0643b10..59b5471cf2 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala @@ -81,7 +81,7 @@ object CoursierAlg { val current = metadataFrom(project) acc.fold(current)(_.enrichWith(current)) } - if (metadata.homePage.isEmpty || metadata.scmUrl.isEmpty) { + if (metadata.repoUrl.isEmpty) { parentOf(project) match { case Some(parent) => getMetadataImpl(parent, repositories, Some(metadata)) case None => F.pure(Some(metadata)) From 2b5477a31e59d67782294dc2ec95c93ea82df82a Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Mon, 2 Jan 2023 21:06:05 +0100 Subject: [PATCH 08/13] Match releaseNotesURL key case-insensitive --- .../scala/org/scalasteward/core/coursier/CoursierAlg.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala index 59b5471cf2..d8db860f96 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala @@ -150,14 +150,16 @@ object CoursierAlg { ) } - private def metadataFrom(project: Project): DependencyMetadata = + private def metadataFrom(project: Project): DependencyMetadata = { + val releaseNotesUrlKey = "releaseNotesURL".toLowerCase DependencyMetadata( homePage = uri.fromStringWithScheme(project.info.homePage), scmUrl = project.info.scm.flatMap(_.url).flatMap(uri.fromStringWithScheme), releaseNotesUrl = project.properties - .collectFirst { case ("releaseNotesUrl", value) => value } + .collectFirst { case (key, value) if key.toLowerCase === releaseNotesUrlKey => value } .flatMap(uri.fromStringWithScheme) ) + } private def parentOf(project: Project): Option[coursier.Dependency] = project.parent.map { case (module, version) => From 148823575745e9f2159576a997e0440c9c490b5e Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Mon, 2 Jan 2023 21:12:09 +0100 Subject: [PATCH 09/13] Use equalsIgnoreCase --- .../scala/org/scalasteward/core/coursier/CoursierAlg.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala index d8db860f96..cd1bb396ae 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala @@ -150,16 +150,14 @@ object CoursierAlg { ) } - private def metadataFrom(project: Project): DependencyMetadata = { - val releaseNotesUrlKey = "releaseNotesURL".toLowerCase + private def metadataFrom(project: Project): DependencyMetadata = DependencyMetadata( homePage = uri.fromStringWithScheme(project.info.homePage), scmUrl = project.info.scm.flatMap(_.url).flatMap(uri.fromStringWithScheme), releaseNotesUrl = project.properties - .collectFirst { case (key, value) if key.toLowerCase === releaseNotesUrlKey => value } + .collectFirst { case (key, value) if key.equalsIgnoreCase("releaseNotesURL") => value } .flatMap(uri.fromStringWithScheme) ) - } private def parentOf(project: Project): Option[coursier.Dependency] = project.parent.map { case (module, version) => From 3c0e5ec210a00c0ef9067d0ae1d7e1ca0c453186 Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Mon, 2 Jan 2023 21:20:41 +0100 Subject: [PATCH 10/13] Change the property key to `info.releaseNotesUrl` --- .../main/scala/org/scalasteward/core/coursier/CoursierAlg.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala index cd1bb396ae..06da36fad7 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala @@ -155,7 +155,7 @@ object CoursierAlg { homePage = uri.fromStringWithScheme(project.info.homePage), scmUrl = project.info.scm.flatMap(_.url).flatMap(uri.fromStringWithScheme), releaseNotesUrl = project.properties - .collectFirst { case (key, value) if key.equalsIgnoreCase("releaseNotesURL") => value } + .collectFirst { case (key, value) if key.equalsIgnoreCase("info.releaseNotesUrl") => value } .flatMap(uri.fromStringWithScheme) ) From 1bf03d1cf4dd5c806f78f41afb53a8d296c760dc Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Tue, 3 Jan 2023 13:30:27 +0100 Subject: [PATCH 11/13] Deduplicate releaseRelatedUrls Duplicates could have happened if the URL from the POM was the same as one of the URLs constructed in possibleReleaseRelatedUrls. --- .../scalasteward/core/vcs/VCSExtraAlg.scala | 22 +++++++++++-------- .../core/vcs/VCSExtraAlgTest.scala | 18 +++++++++++---- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/scala/org/scalasteward/core/vcs/VCSExtraAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/vcs/VCSExtraAlg.scala index d3611535db..afdc391300 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/vcs/VCSExtraAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/vcs/VCSExtraAlg.scala @@ -44,15 +44,19 @@ object VCSExtraAlg { releaseNotesUrl: Option[Uri], currentVersion: Version, nextVersion: Version - ): F[List[ReleaseRelatedUrl]] = - (releaseNotesUrl.toList.map(ReleaseRelatedUrl.CustomReleaseNotes.apply) ++ - vcs.possibleReleaseRelatedUrls( - config.tpe, - config.apiHost, - repoUrl, - currentVersion, - nextVersion - )) + ): F[List[ReleaseRelatedUrl]] = { + val releaseRelatedUrls = + releaseNotesUrl.toList.map(ReleaseRelatedUrl.CustomReleaseNotes.apply) ++ + vcs.possibleReleaseRelatedUrls( + config.tpe, + config.apiHost, + repoUrl, + currentVersion, + nextVersion + ) + releaseRelatedUrls + .distinctBy(_.url) .filterA(releaseRelatedUrl => urlChecker.exists(releaseRelatedUrl.url)) + } } } diff --git a/modules/core/src/test/scala/org/scalasteward/core/vcs/VCSExtraAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/vcs/VCSExtraAlgTest.scala index e7cc5a5b3c..82b1bb5e06 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/vcs/VCSExtraAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/vcs/VCSExtraAlgTest.scala @@ -12,10 +12,11 @@ import org.scalasteward.core.mock.{MockEff, MockState} class VCSExtraAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] { private val state = MockState.empty.copy(clientResponses = HttpApp { - case HEAD -> Root / "foo" / "bar" / "README.md" => Ok() - case HEAD -> Root / "foo" / "bar" / "compare" / "v0.1.0...v0.2.0" => Ok() - case HEAD -> Root / "foo" / "buz" / "compare" / "v0.1.0...v0.2.0" => PermanentRedirect() - case _ => NotFound() + case HEAD -> Root / "foo" / "bar" / "README.md" => Ok() + case HEAD -> Root / "foo" / "bar" / "compare" / "v0.1.0...v0.2.0" => Ok() + case HEAD -> Root / "foo" / "bar1" / "blob" / "master" / "RELEASES.md" => Ok() + case HEAD -> Root / "foo" / "buz" / "compare" / "v0.1.0...v0.2.0" => PermanentRedirect() + case _ => NotFound() }) private val v1 = Version("0.1.0") @@ -46,6 +47,15 @@ class VCSExtraAlgTest extends CatsEffectSuite with Http4sDsl[MockEff] { assertIO(obtained, expected) } + test("getReleaseRelatedUrls: releaseNotesUrl is in possibleReleaseRelatedUrls") { + val releaseNotesUrl = uri"https://github.com/foo/bar1/blob/master/RELEASES.md" + val obtained = vcsExtraAlg + .getReleaseRelatedUrls(uri"https://github.com/foo/bar1", Some(releaseNotesUrl), v1, v2) + .runA(state) + val expected = List(CustomReleaseNotes(releaseNotesUrl)) + assertIO(obtained, expected) + } + test("getReleaseRelatedUrls: repoUrl permanent redirect") { val obtained = vcsExtraAlg.getReleaseRelatedUrls(uri"https://github.com/foo/buz", None, v1, v2).runA(state) From 7e0bec6a3cb42f5af754f6a1b2e9ceb1e93c9464 Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Tue, 3 Jan 2023 13:33:51 +0100 Subject: [PATCH 12/13] Change return type of getMetadata to F[DependencyMetadata] --- .../core/coursier/CoursierAlg.scala | 26 ++++------ .../core/coursier/DependencyMetadata.scala | 5 ++ .../core/nurture/NurtureAlg.scala | 6 +-- .../core/coursier/CoursierAlgTest.scala | 50 ++++++++----------- 4 files changed, 40 insertions(+), 47 deletions(-) diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala index 06da36fad7..2de887e5e5 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala @@ -31,7 +31,7 @@ import org.typelevel.log4cats.Logger * metadata. */ trait CoursierAlg[F[_]] { - def getMetadata(dependency: Dependency, resolvers: List[Resolver]): F[Option[DependencyMetadata]] + def getMetadata(dependency: Dependency, resolvers: List[Resolver]): F[DependencyMetadata] def getVersions(dependency: Dependency, resolver: Resolver): F[List[Version]] } @@ -52,18 +52,18 @@ object CoursierAlg { override def getMetadata( dependency: Dependency, resolvers: List[Resolver] - ): F[Option[DependencyMetadata]] = + ): F[DependencyMetadata] = resolvers.traverseFilter(convertResolver(_).attempt.map(_.toOption)).flatMap { repositories => val csrDependency = toCoursierDependency(dependency) - getMetadataImpl(csrDependency, repositories, None) + getMetadataImpl(csrDependency, repositories, DependencyMetadata.empty) } private def getMetadataImpl( dependency: coursier.Dependency, repositories: List[coursier.Repository], - acc: Option[DependencyMetadata] - ): F[Option[DependencyMetadata]] = { + acc: DependencyMetadata + ): F[DependencyMetadata] = { val fetchArtifacts = fetch .withArtifactTypes(Set(coursier.Type.pom, coursier.Type.ivy)) .withDependencies(List(dependency)) @@ -76,17 +76,13 @@ object CoursierAlg { val maybeProject = result.resolution.projectCache .get(dependency.moduleVersion) .map { case (_, project) => project } - maybeProject.traverseFilter { project => - val metadata = { - val current = metadataFrom(project) - acc.fold(current)(_.enrichWith(current)) + + maybeProject.fold(F.pure(acc)) { project => + val metadata = acc.enrichWith(metadataFrom(project)) + val recurse = Option.when(metadata.repoUrl.isEmpty)(()) + (recurse >> parentOf(project)).fold(F.pure(metadata)) { parent => + getMetadataImpl(parent, repositories, metadata) } - if (metadata.repoUrl.isEmpty) { - parentOf(project) match { - case Some(parent) => getMetadataImpl(parent, repositories, Some(metadata)) - case None => F.pure(Some(metadata)) - } - } else F.pure(Some(metadata)) } } } diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala index 58ef386ab1..7cba02f666 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala @@ -45,3 +45,8 @@ final case class DependencyMetadata( urls.find(_.scheme.exists(uri.httpSchemes)).orElse(urls.headOption) } } + +object DependencyMetadata { + val empty: DependencyMetadata = + DependencyMetadata(None, None, None) +} diff --git a/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala index 10f76ab837..27a018bd0a 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/nurture/NurtureAlg.scala @@ -202,11 +202,11 @@ final class NurtureAlg[F[_]](config: VCSCfg)(implicit dependenciesWithNextVersion = dependenciesUpdatedWithNextAndCurrentVersion(data.update) resolvers = data.repoData.cache.dependencyInfos.flatMap(_.resolvers) dependencyToMetadata <- dependenciesWithNextVersion - .traverseFilter { case (_, dependency) => + .traverse { case (_, dependency) => coursierAlg .getMetadata(dependency, resolvers) - .flatMap(_.traverse(_.filterUrls(urlChecker.exists))) - .map(_.tupleLeft(dependency)) + .flatMap(_.filterUrls(urlChecker.exists)) + .tupleLeft(dependency) } .map(_.toMap) artifactIdToUrl = dependencyToMetadata.toList.mapFilter { case (dependency, metadata) => diff --git a/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala index aa57f19918..17105375a4 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/coursier/CoursierAlgTest.scala @@ -11,16 +11,14 @@ import org.scalasteward.core.mock.MockState class CoursierAlgTest extends CatsEffectSuite { private val resolvers = List(Resolver.mavenCentral) - private val emptyMetadata = DependencyMetadata(None, None, None) + private val emptyMetadata = DependencyMetadata.empty test("getMetadata: with homePage and scmUrl") { val dep = "org.typelevel".g % ("cats-effect", "cats-effect_2.12").a % "1.0.0" val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) - val expected = Some( - emptyMetadata.copy( - homePage = Some(uri"https://typelevel.org/cats-effect/"), - scmUrl = Some(uri"https://github.com/typelevel/cats-effect") - ) + val expected = emptyMetadata.copy( + homePage = Some(uri"https://typelevel.org/cats-effect/"), + scmUrl = Some(uri"https://github.com/typelevel/cats-effect") ) assertIO(obtained, expected) } @@ -30,25 +28,23 @@ class CoursierAlgTest extends CatsEffectSuite { val dep = "com.typesafe.play".g % artifactId % "2.1.0-M7" val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) val expected = - Some(emptyMetadata.copy(homePage = Some(uri"https://github.com/playframework/play-ws"))) + emptyMetadata.copy(homePage = Some(uri"https://github.com/playframework/play-ws")) assertIO(obtained, expected) } test("getMetadata: scmUrl without scheme") { val dep = "org.msgpack".g % "msgpack-core".a % "0.8.20" val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) - val expected = Some(emptyMetadata.copy(homePage = Some(uri"http://msgpack.org/"))) + val expected = emptyMetadata.copy(homePage = Some(uri"http://msgpack.org/")) assertIO(obtained, expected) } test("getMetadata: scmUrl with git scheme") { val dep = "org.xhtmlrenderer".g % "flying-saucer-parent".a % "9.0.1" val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) - val expected = Some( - emptyMetadata.copy( - homePage = Some(uri"http://code.google.com/p/flying-saucer/"), - scmUrl = Some(uri"git://github.com/flyingsaucerproject/flyingsaucer.git") - ) + val expected = emptyMetadata.copy( + homePage = Some(uri"http://code.google.com/p/flying-saucer/"), + scmUrl = Some(uri"git://github.com/flyingsaucerproject/flyingsaucer.git") ) assertIO(obtained, expected) } @@ -56,14 +52,14 @@ class CoursierAlgTest extends CatsEffectSuite { test("getMetadata: homePage from parent") { val dep = "net.bytebuddy".g % "byte-buddy".a % "1.10.5" val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) - val expected = Some(emptyMetadata.copy(homePage = Some(uri"https://bytebuddy.net"))) + val expected = emptyMetadata.copy(homePage = Some(uri"https://bytebuddy.net")) assertIO(obtained, expected) } test("getMetadata: minimal POM") { val dep = "altrmi".g % "altrmi-common".a % "0.9.6" val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) - val expected = Some(emptyMetadata) + val expected = emptyMetadata assertIO(obtained, expected) } @@ -71,11 +67,9 @@ class CoursierAlgTest extends CatsEffectSuite { val dep = ("org.xerial.sbt".g % "sbt-sonatype".a % "3.8") .copy(sbtVersion = Some(SbtVersion("1.0")), scalaVersion = Some(ScalaVersion("2.12"))) val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) - val expected = Some( - emptyMetadata.copy( - homePage = Some(uri"https://github.com/xerial/sbt-sonatype"), - scmUrl = Some(uri"https://github.com/xerial/sbt-sonatype") - ) + val expected = emptyMetadata.copy( + homePage = Some(uri"https://github.com/xerial/sbt-sonatype"), + scmUrl = Some(uri"https://github.com/xerial/sbt-sonatype") ) assertIO(obtained, expected) } @@ -84,19 +78,16 @@ class CoursierAlgTest extends CatsEffectSuite { val dep = ("com.github.gseitz".g % "sbt-release".a % "1.0.12") .copy(sbtVersion = Some(SbtVersion("1.0")), scalaVersion = Some(ScalaVersion("2.12"))) val obtained = coursierAlg.getMetadata(dep, List(sbtPluginReleases)).runA(MockState.empty) - val expected = - Some(emptyMetadata.copy(homePage = Some(uri"https://github.com/sbt/sbt-release"))) + val expected = emptyMetadata.copy(homePage = Some(uri"https://github.com/sbt/sbt-release")) assertIO(obtained, expected) } test("getMetadata: scmUrl with github scheme") { val dep = "com.github.japgolly.scalajs-react".g % ("core", "core_sjs1_2.13").a % "2.0.0-RC5" val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty) - val expected = Some( - emptyMetadata.copy( - homePage = Some(uri"https://github.com/japgolly/scalajs-react"), - scmUrl = Some(uri"github.com:japgolly/scalajs-react.git") - ) + val expected = emptyMetadata.copy( + homePage = Some(uri"https://github.com/japgolly/scalajs-react"), + scmUrl = Some(uri"github.com:japgolly/scalajs-react.git") ) assertIO(obtained, expected) } @@ -104,7 +95,7 @@ class CoursierAlgTest extends CatsEffectSuite { test("getMetadata: no resolvers") { val dep = "org.example".g % "foo".a % "1.0.0" val obtained = coursierAlg.getMetadata(dep, List.empty).runA(MockState.empty) - val expected = None + val expected = emptyMetadata assertIO(obtained, expected) } @@ -112,7 +103,8 @@ class CoursierAlgTest extends CatsEffectSuite { val dep = "org.typelevel".g % ("cats-effect", "cats-effect_2.12").a % "1.0.0" val resolvers = List(Resolver.mavenCentral.copy(headers = List(Resolver.Header("X-Foo", "bar")))) - val obtained = coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty).map(_.isDefined) + val obtained = + coursierAlg.getMetadata(dep, resolvers).runA(MockState.empty).map(_.repoUrl.isDefined) assertIOBoolean(obtained) } } From b9b525a139c643f57edbd0d997241a8bc870419b Mon Sep 17 00:00:00 2001 From: "Frank S. Thomas" Date: Tue, 3 Jan 2023 15:46:59 +0100 Subject: [PATCH 13/13] fmt --- .../test/scala/org/scalasteward/core/TestSyntax.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala b/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala index e18c2bcf8d..b0f884bf53 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/TestSyntax.scala @@ -5,12 +5,10 @@ import org.scalasteward.core.data._ import org.scalasteward.core.util.Nel object TestSyntax { - val sbtPluginReleases: IvyRepository = - IvyRepository( - "sbt-plugin-releases", - "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[defaultPattern]", - None - ) + val sbtPluginReleases: IvyRepository = { + val pattern = "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/[defaultPattern]" + IvyRepository("sbt-plugin-releases", pattern, None) + } implicit class GenericOps[A](val self: A) extends AnyVal { def withMavenCentral: Scope[A] =