Add whitelist organizations option

docs/help.md

@@ -4,7 +4,7 @@ All command line arguments for the `scala-steward` application.
 
 ```
 Usage:
-    scala-steward --workspace <file> --repos-file <uri> [--repos-file <uri>]... [--git-author-name <string>] --git-author-email <string> [--git-author-signing-key <string>] --git-ask-pass <file> [--sign-commits] [--signoff] [--forge-type <forge-type>] [--forge-api-host <uri>] --forge-login <string> [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var <name=value>]... [--process-timeout <duration>] [--whitelist <string>]... [--read-only <string>]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size <integer>] [--repo-config <uri>]... [--disable-default-repo-config] [--scalafix-migrations <uri>]... [--disable-default-scalafix-migrations] [--artifact-migrations <uri>]... [--disable-default-artifact-migrations] [--cache-ttl <duration>] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers <integer>] [--gitlab-remove-source-branch] [--azure-repos-organization <string>] [--github-app-id <integer> --github-app-key-file <file>] [--url-checker-test-url <uri>]... [--default-maven-repo <string>]... [--refresh-backoff-period <duration>] [--exit-code-success-if-any-repo-succeeds]
+    scala-steward --workspace <file> --repos-file <uri> [--repos-file <uri>]... [--git-author-name <string>] --git-author-email <string> [--git-author-signing-key <string>] --git-ask-pass <file> [--sign-commits] [--signoff] [--forge-type <forge-type>] [--forge-api-host <uri>] --forge-login <string> [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var <name=value>]... [--process-timeout <duration>] [--whitelist <string>]... [--read-only <string>]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size <integer>] [--repo-config <uri>]... [--disable-default-repo-config] [--scalafix-migrations <uri>]... [--disable-default-scalafix-migrations] [--artifact-migrations <uri>]... [--disable-default-artifact-migrations] [--cache-ttl <duration>] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers <integer>] [--gitlab-remove-source-branch] [--azure-repos-organization <string>] [--github-app-id <integer> --github-app-key-file <file>] [--url-checker-test-url <uri>]... [--default-maven-repo <string>]... [--refresh-backoff-period <duration>] [--exit-code-success-if-any-repo-succeeds] [--whitelist-organization <string>]...
     scala-steward validate-repo-config
 
 
@@ -92,6 +92,8 @@ Options and flags:
         Period of time a failed build won't be triggered again; default: 0days
     --exit-code-success-if-any-repo-succeeds
         Whether the Scala Steward process should exit with success (exit code 0) if any repo succeeds; default: false
+    --whitelist-organization <string>
+        List of organizations to bypass UrlChecker
 
 Subcommands:
     validate-repo-config

modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala

@@ -153,6 +153,11 @@ object Cli {
     options[EnvVar]("env-var", help).orEmpty
   }
 
+  private val whiteListOrganizations: Opts[List[String]] = {
+    val help = s"List of organizations to bypass UrlChecker"
+    options[String]("whitelist-organization", help).orEmpty
+  }
+
   private val processTimeout: Opts[FiniteDuration] = {
     val default = 10.minutes
     val help =
@@ -351,7 +356,8 @@ object Cli {
     urlCheckerTestUrls,
     defaultMavenRepos,
     refreshBackoffPeriod,
-    exitCodePolicy
+    exitCodePolicy,
+    whiteListOrganizations
   ).mapN(Config.apply).map(Usage.Regular.apply)
 
   private val validateRepoConfig: Opts[Usage] =

modules/core/src/main/scala/org/scalasteward/core/application/Config.scala

@@ -64,7 +64,8 @@ final case class Config(
     urlCheckerTestUrls: Nel[Uri],
     defaultResolvers: List[Resolver],
     refreshBackoffPeriod: FiniteDuration,
-    exitCodePolicy: ExitCodePolicy
+    exitCodePolicy: ExitCodePolicy,
+    whiteListOrganizations: List[String]
 ) {
   def forgeSpecificCfg: ForgeSpecificCfg =
     forgeCfg.tpe match {

modules/core/src/main/scala/org/scalasteward/core/application/Context.scala

@@ -23,7 +23,6 @@ import eu.timepit.refined.types.numeric.PosInt
 import org.http4s.Uri
 import org.http4s.client.Client
 import org.http4s.headers.`User-Agent`
-import org.scalasteward.core.application.Config.ForgeCfg
 import org.scalasteward.core.buildtool.BuildToolDispatcher
 import org.scalasteward.core.buildtool.gradle.GradleAlg
 import org.scalasteward.core.buildtool.maven.MavenAlg
@@ -167,8 +166,7 @@ object Context {
       implicit val forgeApiAlg: ForgeApiAlg[F] = ForgeSelection
         .forgeApiAlg[F](config.forgeCfg, config.forgeSpecificCfg, forgeAuthAlg.authenticateApi)
       implicit val forgeRepoAlg: ForgeRepoAlg[F] = new ForgeRepoAlg[F](config)
-      implicit val forgeCfg: ForgeCfg = config.forgeCfg
-      implicit val updateInfoUrlFinder: UpdateInfoUrlFinder[F] = new UpdateInfoUrlFinder[F]
+      implicit val updateInfoUrlFinder: UpdateInfoUrlFinder[F] = new UpdateInfoUrlFinder[F](config)
       implicit val pullRequestRepository: PullRequestRepository[F] =
         new PullRequestRepository[F](pullRequestsStore)
       implicit val scalafixCli: ScalafixCli[F] = new ScalafixCli[F]

modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala

@@ -49,8 +49,8 @@ final case class DependencyMetadata(
     urls.find(_.scheme.exists(uri.httpSchemes)).orElse(urls.headOption)
   }
 
-  def forgeRepo(implicit config: ForgeCfg): Option[ForgeRepo] =
-    repoUrl.flatMap(ForgeRepo.fromRepoUrl)
+  def forgeRepo(config: ForgeCfg): Option[ForgeRepo] =
+    repoUrl.flatMap(ForgeRepo.fromRepoUrl(_)(config))
 }
 
 object DependencyMetadata {

modules/core/src/main/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinder.scala

@@ -19,17 +19,17 @@ package org.scalasteward.core.nurture
 import cats.Monad
 import cats.syntax.all.*
 import org.http4s.Uri
-import org.scalasteward.core.application.Config.ForgeCfg
 import org.scalasteward.core.coursier.DependencyMetadata
 import org.scalasteward.core.data.Version
 import org.scalasteward.core.forge.ForgeRepo
 import org.scalasteward.core.forge.ForgeType.*
 import org.scalasteward.core.nurture.UpdateInfoUrl.*
 import org.scalasteward.core.nurture.UpdateInfoUrlFinder.possibleUpdateInfoUrls
 import org.scalasteward.core.util.UrlChecker
+import org.scalasteward.core.application.Config
+import org.scalasteward.core.util.isWhitelisted
 
-final class UpdateInfoUrlFinder[F[_]](implicit
-    config: ForgeCfg,
+final class UpdateInfoUrlFinder[F[_]](config: Config)(implicit
     urlChecker: UrlChecker[F],
     F: Monad[F]
 ) {
@@ -38,10 +38,14 @@ final class UpdateInfoUrlFinder[F[_]](implicit
       versionUpdate: Version.Update
   ): F[List[UpdateInfoUrl]] = {
     val updateInfoUrls: List[UpdateInfoUrl] =
-      dependency.releaseNotesUrl.toList.map(CustomReleaseNotes.apply) ++
-        dependency.forgeRepo.toSeq.flatMap(forgeRepo =>
-          possibleUpdateInfoUrls(forgeRepo, versionUpdate)
-        )
+      if (dependency.releaseNotesUrl.exists(isWhitelisted(config.whiteListOrganizations, _)))
+        dependency.releaseNotesUrl.toList.map(CustomReleaseNotes.apply)
+      else
+        dependency.releaseNotesUrl.toList.map(CustomReleaseNotes.apply) ++
+          dependency
+            .forgeRepo(config.forgeCfg)
+            .toSeq
+            .flatMap(forgeRepo => possibleUpdateInfoUrls(forgeRepo, versionUpdate))
 
     updateInfoUrls
       .sorted(UpdateInfoUrl.updateInfoUrlOrder.toOrdering)

modules/core/src/main/scala/org/scalasteward/core/util/UrlChecker.scala

@@ -58,10 +58,19 @@ object UrlChecker {
           }
 
         private def status(url: Uri): F[Status] =
-          statusCache.cachingF(url.renderString)(None) {
-            val req = Request[F](method = Method.HEAD, uri = url)
-            modify(req).flatMap(urlCheckerClient.client.status)
-          }
+          isWhitelisted(config.whiteListOrganizations, url)
+            .pure[F]
+            .ifM(
+              Status.Ok.pure[F],
+              statusCache.cachingF(url.renderString)(None) {
+                val req = Request[F](method = Method.HEAD, uri = url)
+                modify(req)
+                  .flatTap(req =>
+                    logger.info(s"Checking if $url exists: ${req.asCurl(_ => false)}")
+                  )
+                  .flatMap(urlCheckerClient.client.status)
+              }
+            )
       }
     }
 }

modules/core/src/main/scala/org/scalasteward/core/util/package.scala

@@ -20,6 +20,7 @@ import cats.*
 import cats.syntax.all.*
 import fs2.Pipe
 import scala.collection.mutable.ListBuffer
+import org.http4s.Uri
 
 package object util {
   final type Nel[+A] = cats.data.NonEmptyList[A]
@@ -88,4 +89,10 @@ package object util {
     Left(s"Unexpected string '$s'. Expected one of: ${expected.mkString(", ")}.")
 
   def intellijThisImportIsUsed[A](a: A): Unit = ()
+
+  /** check if the url is in the organizations white list
+    */
+  def isWhitelisted(whitelist: List[String], url: Uri): Boolean =
+    whitelist.map(Uri.Path.unsafeFromString(_)).exists(url.path.startsWith(_))
+
 }

modules/core/src/test/scala/org/scalasteward/core/mock/MockConfig.scala

@@ -15,6 +15,8 @@ object MockConfig {
   val key = mockRoot / "rsa-4096-private.pem"
   key.overwrite(Source.fromResource("rsa-4096-private.pem").mkString)
 
+  val scalaStewardOrg = "scala-steward-org"
+
   private val args: List[String] = List(
     s"--workspace=$mockRoot/workspace",
     s"--repos-file=$reposFile",
@@ -30,7 +32,8 @@ object MockConfig {
     "--add-labels",
     "--github-app-id=1234",
     s"--github-app-key-file=$key",
-    "--refresh-backoff-period=1hour"
+    "--refresh-backoff-period=1hour",
+    s"--whitelist-organization=$scalaStewardOrg"
   )
   val Success(Cli.Usage.Regular(config)) = Cli.parseArgs(args): @unchecked
 }

modules/core/src/test/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinderTest.scala

@@ -12,7 +12,7 @@ import org.scalasteward.core.forge.ForgeType.*
 import org.scalasteward.core.forge.github.Repository
 import org.scalasteward.core.forge.{ForgeRepo, ForgeType}
 import org.scalasteward.core.mock.MockContext.context.*
-import org.scalasteward.core.mock.{GitHubAuth, MockEff, MockEffOps, MockState}
+import org.scalasteward.core.mock.{GitHubAuth, MockConfig, MockEff, MockEffOps, MockState}
 import org.scalasteward.core.nurture.UpdateInfoUrl.*
 import org.scalasteward.core.nurture.UpdateInfoUrlFinder.*
 
@@ -84,14 +84,16 @@ class UpdateInfoUrlFinderTest extends CatsEffectSuite with Http4sDsl[MockEff] {
     assertIO(obtained, List.empty)
   }
 
-  implicit private val config: ForgeCfg = ForgeCfg(
+  private val forgeConfig: ForgeCfg = ForgeCfg(
     ForgeType.GitHub,
     uri"https://github.on-prem.com/",
     "",
     doNotFork = false,
     addLabels = false
   )
-  private val onPremUpdateUrlFinder = new UpdateInfoUrlFinder[MockEff]
+  val config = MockConfig.config.copy(forgeCfg = forgeConfig)
+
+  private val onPremUpdateUrlFinder = new UpdateInfoUrlFinder[MockEff](config)
   private val gitHubFooBarRepo = ForgeRepo(GitHub, uri"https://github.com/foo/bar/")
   private val bitbucketFooBarRepo = ForgeRepo(Bitbucket, uri"https://bitbucket.org/foo/bar/")
   private val gitLabFooBarRepo = ForgeRepo(GitLab, uri"https://gitlab.com/foo/bar")
@@ -282,4 +284,12 @@ class UpdateInfoUrlFinderTest extends CatsEffectSuite with Http4sDsl[MockEff] {
     val expected = repoUrl / "browse" / "ReleaseNotes.md"
     assert(clue(obtained).contains(expected))
   }
+
+  test("possibleUpdateInfoUrls: CustomReleaseNotes Ok if whitelisted") {
+    val releaseNotesUrl = uri"https://github.com/scala-steward-org/foo/bar/blob/master/RELEASES.md"
+    val metadata = DependencyMetadata.empty.copy(releaseNotesUrl = releaseNotesUrl.some)
+    val obtained = updateInfoUrlFinder.findUpdateInfoUrls(metadata, versionUpdate).runA(state)
+    assertIO(obtained, List(CustomReleaseNotes(releaseNotesUrl)))
+  }
+
 }

modules/core/src/test/scala/org/scalasteward/core/util/utilTest.scala

@@ -6,6 +6,7 @@ import munit.ScalaCheckSuite
 import org.scalacheck.Gen
 import org.scalacheck.Prop.*
 import scala.collection.mutable.ListBuffer
+import org.http4s.Uri
 
 class utilTest extends ScalaCheckSuite {
   test("appendBounded") {
@@ -52,4 +53,16 @@ class utilTest extends ScalaCheckSuite {
     s.compile.drain.unsafeRunSync()
     assertEquals(count, n.toInt)
   }
+
+  test("isWhitelisted") {
+    assert(
+      isWhitelisted(List("org1", "org2"), Uri.unsafeFromString("https://github.com/org1/repo"))
+    )
+    assert(
+      !isWhitelisted(List("org1", "org2"), Uri.unsafeFromString("https://github.com/org3/repo"))
+    )
+    assert(
+      !isWhitelisted(List("org1", "org2"), Uri.unsafeFromString("https://github.com/org3/org1"))
+    )
+  }
 }