Implement remote cache

src/main/scala-2.12/PluginCompat.scala

@@ -3,13 +3,16 @@ package sbtassembly
 import java.nio.file.{ Path => NioPath }
 import java.util.jar.{ Manifest => JManifest }
 import sbt.*
+import sbt.Keys.*
+import sbt.Tags.Tag
 import sbt.internal.util.HNil
 import sbt.internal.util.Types.:+:
 import sbt.util.FileInfo.lastModified
 import sbt.util.Tracked.{ inputChanged, lastOutput }
 import xsbti.FileConverter
 
 private[sbtassembly] object PluginCompat {
+  type FileRef = java.io.File
   type Out = java.io.File
   type MainClass = sbt.Package.MainClass
 
@@ -22,13 +25,47 @@ private[sbtassembly] object PluginCompat {
     a.data.toPath()
   def toFile(a: Attributed[File])(implicit conv: FileConverter): File =
     a.data
-  def toOutput(x: File)(implicit conv: FileConverter): File =
-    x
+  def toFile(x: File): File = x
+  def toOutput(x: File)(implicit conv: FileConverter): File = x
   def toNioPaths(cp: Seq[Attributed[File]])(implicit conv: FileConverter): Vector[NioPath] =
     cp.map(_.data.toPath()).toVector
   def toFiles(cp: Seq[Attributed[File]])(implicit conv: FileConverter): Vector[File] =
     cp.map(_.data).toVector
 
+  trait AssemblyKeys0 {
+    lazy val assemblyOutputPath = taskKey[File]("output path of the über jar")
+  }
+
+  val assemblyTag = Tag("assembly")
+  def assemblyTask(key: TaskKey[Out])(
+    f: (
+      String,
+      File,
+      Classpath,
+      Classpath,
+      AssemblyOption,
+      Seq[PackageOption],
+      FileConverter,
+      File,
+      Logger
+    ) => Out
+  ): Def.Initialize[Task[Out]] = Def.task {
+    val t = (key / Keys.test).value
+    val s = (key / Keys.streams).value
+    val conv = fileConverter.value
+    f(
+      (key / AssemblyKeys.assemblyJarName).value.replaceAll(".jar", ""),
+      (key / AssemblyKeys.assemblyOutputPath).value,
+      (AssemblyKeys.assembly / fullClasspath).value,
+      (AssemblyKeys.assembly / externalDependencyClasspath).value,
+      (key / AssemblyKeys.assemblyOption).value,
+      (key / Keys.packageOptions).value,
+      conv,
+      s.cacheDirectory,
+      s.log
+    )
+  }.tag(assemblyTag)
+
   type CacheKey = FilesInfo[ModifiedFileInfo] :+:
     Map[String, (Boolean, String)] :+: // map of target paths that matched a merge strategy
     JManifest :+:

src/main/scala-3/PluginCompat.scala

@@ -4,11 +4,16 @@ import java.io.File
 import java.nio.file.{ Path => NioPath }
 import java.util.jar.{ Manifest => JManifest }
 import sbt.*
+import sbt.util.cacheLevel
+import sbt.Keys.*
+import sbt.Tags.Tag
 import sbt.librarymanagement.ModuleID
-import xsbti.{ FileConverter, HashedVirtualFileRef }
+import sjsonnew.*
+import xsbti.{ FileConverter, HashedVirtualFileRef, VirtualFile }
 
 object PluginCompat:
-  type Out = HashedVirtualFileRef
+  type FileRef = HashedVirtualFileRef
+  type Out = VirtualFile
   type JarManifest = PackageOption.JarManifest
   type MainClass = PackageOption.MainClass
   type ManifestAttributes = PackageOption.ManifestAttributes
@@ -24,13 +29,80 @@ object PluginCompat:
     conv.toPath(a.data)
   inline def toFile(a: Attributed[HashedVirtualFileRef])(implicit conv: FileConverter): File =
     toNioPath(a).toFile()
-  def toOutput(x: File)(implicit conv: FileConverter): HashedVirtualFileRef =
+  def toFile(p: NioPath): File =
+    p.toFile()
+  def toOutput(x: File)(implicit conv: FileConverter): VirtualFile =
     conv.toVirtualFile(x.toPath())
   def toNioPaths(cp: Seq[Attributed[HashedVirtualFileRef]])(implicit conv: FileConverter): Vector[NioPath] =
     cp.map(toNioPath).toVector
   inline def toFiles(cp: Seq[Attributed[HashedVirtualFileRef]])(implicit conv: FileConverter): Vector[File] =
     toNioPaths(cp).map(_.toFile())
 
+  trait AssemblyKeys0:
+    @cacheLevel(include = Array.empty)
+    lazy val assemblyOutputPath = taskKey[File]("output path of the über jar")
+  end AssemblyKeys0
+
+  val assemblyTag = Tag("assembly")
+  import sbt.util.CacheImplicits.given
+
+  private given forHashing: IsoLList.Aux[
+    AssemblyOption,
+    Boolean :*: Boolean :*: Boolean :*: Classpath :*: Boolean :*: Boolean :*:
+      Vector[String] :*: Option[Int] :*: Vector[String] :*: String :*: LNil
+  ] =
+    LList.iso(
+      { (v: AssemblyOption) =>
+        ("includeBin", v.includeBin) :*:
+          ("includeScala", v.includeScala) :*:
+          ("includeDependency", v.includeDependency) :*:
+          ("excludedJars", v.excludedJars) :*:
+          ("repeatableBuild", v.repeatableBuild) :*:
+          ("appendContentHash", v.appendContentHash) :*:
+          ("prependShellScript", v.prependShellScript match
+            case Some(xs) => xs.toVector
+            case None     => Vector.empty) :*:
+          ("maxHashLength", v.maxHashLength) :*:
+          ("shadeRules", v.shadeRules.toVector.map(_.toString)) :*:
+          ("scalaVersion", v.scalaVersion) :*:
+          LNil
+      },
+      {
+        case _ => ???
+      }
+    )
+
+  def assemblyTask(key: TaskKey[FileRef])(
+    f: (
+      String,
+      File,
+      Classpath,
+      Classpath,
+      AssemblyOption,
+      Seq[PackageOption],
+      FileConverter,
+      File,
+      Logger
+    ) => Out
+  ): Def.Initialize[Task[HashedVirtualFileRef]] = Def.cachedTask {
+    val s = (key / streams).value
+    val conv = fileConverter.value
+    val _ = AssemblyKeys.assemblyMetaBuildHash.value
+    val out: VirtualFile = f(
+      (key / AssemblyKeys.assemblyJarName).value.replaceAll(".jar", ""),
+      (key / AssemblyKeys.assemblyOutputPath).value,
+      (AssemblyKeys.assembly / fullClasspath).value,
+      (AssemblyKeys.assembly / externalDependencyClasspath).value,
+      (key / AssemblyKeys.assemblyOption).value,
+      (key / packageOptions).value,
+      conv,
+      s.cacheDirectory,
+      s.log
+    )
+    Def.declareOutput(out)
+    out: HashedVirtualFileRef
+  }.tag(assemblyTag)
+
   object HListFormats
   val Streamable = scala.reflect.io.Streamable
 

src/main/scala/sbtassembly/Assembly.scala

@@ -8,7 +8,6 @@ import sbt.internal.inc.classpath.ClasspathUtil
 import sbt.io.{ DirectoryFilter => _, IO => _, Path => _, Using }
 import sbt.util.{ FilesInfo, Level, ModifiedFileInfo }
 import sbt.{ File, Logger, _ }
-import sbt.Tags.Tag
 import CacheImplicits._
 import sbtassembly.AssemblyPlugin.autoImport.{ Assembly => _, _ }
 
@@ -38,8 +37,6 @@ object Assembly {
   val indent: String = " " * 2
   val newLineIndented: String = newLine + indent
 
-  val assemblyTag = Tag("assembly")
-
   private[sbtassembly] val scalaPre213Libraries = Vector(
     "scala-actors",
     "scala-compiler",
@@ -170,23 +167,6 @@ object Assembly {
     val jarName: String = s"$name${if (version.nonEmpty) "-" else ""}$version.jar"
   }
 
-  def assemblyTask(key: TaskKey[PluginCompat.Out]): Initialize[Task[PluginCompat.Out]] = Def.task {
-    val t = (key / test).value
-    val s = (key / streams).value
-    val conv = fileConverter.value
-    assemble(
-      (key / assemblyJarName).value.replaceAll(".jar", ""),
-      (key / assemblyOutputPath).value,
-      (assembly / fullClasspath).value,
-      (assembly / externalDependencyClasspath).value,
-      (key / assemblyOption).value,
-      (key / packageOptions).value,
-      conv,
-      s.cacheDirectory,
-      s.log
-    )
-  }.tag(assemblyTag)
-
   /**
    * Builds an assembly jar
    *

src/main/scala/sbtassembly/AssemblyKeys.scala

@@ -4,15 +4,14 @@ import com.eed3si9n.jarjarabrams
 import sbt.Keys.*
 import sbt.*
 
-trait AssemblyKeys {
-  lazy val assembly                  = taskKey[PluginCompat.Out]("Builds a deployable über JAR")
+trait AssemblyKeys extends PluginCompat.AssemblyKeys0 {
+  lazy val assembly                  = taskKey[PluginCompat.FileRef]("Builds a deployable über JAR")
   lazy val assembleArtifact          = settingKey[Boolean]("Enables (true) or disables (false) assembling an artifact")
   lazy val assemblyOption            = taskKey[AssemblyOption]("Configuration for making a deployable über JAR")
-  lazy val assemblyPackageScala      = taskKey[PluginCompat.Out]("Produces the Scala artifact")
-  lazy val assemblyPackageDependency = taskKey[PluginCompat.Out]("Produces the dependency artifact")
+  lazy val assemblyPackageScala      = taskKey[PluginCompat.FileRef]("Produces the Scala artifact")
+  lazy val assemblyPackageDependency = taskKey[PluginCompat.FileRef]("Produces the dependency artifact")
   lazy val assemblyJarName           = taskKey[String]("name of the über jar")
   lazy val assemblyDefaultJarName    = taskKey[String]("default name of the über jar")
-  lazy val assemblyOutputPath        = taskKey[File]("output path of the über jar")
   lazy val assemblyExcludedJars      = taskKey[Classpath]("list of excluded jars")
   lazy val assemblyMergeStrategy     = settingKey[String => MergeStrategy]("mapping from archive member path to merge strategy")
   lazy val assemblyShadeRules        = settingKey[Seq[jarjarabrams.ShadeRule]]("shading rules backed by jarjar")
@@ -22,6 +21,7 @@ trait AssemblyKeys {
   lazy val assemblyPrependShellScript = settingKey[Option[Seq[String]]]("A launch script to prepend to the über JAR")
   lazy val assemblyRepeatableBuild   = settingKey[Boolean]("If (true), builds the jar with a consistent hash (given the same inputs/assembly configuration) so it can be cached, but loses parallelism optimization. " +
                                                                        "If (false), builds the jar faster via parallelization, but loses hash consistency, and hence, cannot be cached")
+  lazy val assemblyMetaBuildHash     = taskKey[String]("Hash of metabuild")
 }
 
 object AssemblyKeys extends AssemblyKeys

src/main/scala/sbtassembly/AssemblyPlugin.scala

@@ -38,7 +38,22 @@ object AssemblyPlugin extends sbt.AutoPlugin {
     assemblyPrependShellScript := None,
     assemblyCacheOutput := true,
     assemblyRepeatableBuild := true,
-    concurrentRestrictions += Tags.limit(Assembly.assemblyTag, 1)
+    concurrentRestrictions += Tags.limit(PluginCompat.assemblyTag, 1),
+    assemblyMetaBuildHash := {
+      val extracted = Project.extract(state.value)
+      val metaBuildClasses = (for {
+        unit <- extracted.structure.units.values
+        sbtFiles <- unit.unit.definitions.dslDefinitions.sbtFiles
+        generated <- sbtFiles.generated
+      } yield PluginCompat.toFile(generated)).toVector.distinct
+      val metaBuildClasspath = (for {
+        unit <- extracted.structure.units.values
+        cp <- unit.classpath
+      } yield PluginCompat.toFile(cp)).toVector.distinct
+      val hashes = (metaBuildClasses ++ metaBuildClasspath)
+        .map(Assembly.hash).sorted
+      hashes.mkString("")
+    },
   )
 
   override lazy val projectSettings: Seq[Def.Setting[_]] = assemblySettings
@@ -56,9 +71,9 @@ object AssemblyPlugin extends sbt.AutoPlugin {
   )
 
   def baseAssemblySettings: Seq[sbt.Def.Setting[_]] = (Seq(
-    assembly := Assembly.assemblyTask(assembly).value,
-    assemblyPackageScala := Assembly.assemblyTask(assemblyPackageScala).value,
-    assemblyPackageDependency := Assembly.assemblyTask(assemblyPackageDependency).value,
+    assembly := PluginCompat.assemblyTask(assembly)(Assembly.assemble).value,
+    assemblyPackageScala := PluginCompat.assemblyTask(assemblyPackageScala)(Assembly.assemble).value,
+    assemblyPackageDependency := PluginCompat.assemblyTask(assemblyPackageDependency)(Assembly.assemble).value,
 
     // test
     assembly / test := {},
@@ -116,7 +131,6 @@ object AssemblyPlugin extends sbt.AutoPlugin {
         .withIncludeBin((packageBin / assembleArtifact).value)
         .withIncludeScala((assemblyPackageScala / assembleArtifact).value)
         .withIncludeDependency((assemblyPackageDependency / assembleArtifact).value)
-        .withMergeStrategy(assemblyMergeStrategy.value)
         .withExcludedJars(assemblyExcludedJars.value)
         .withCacheOutput(assemblyCacheOutput.value)
         .withAppendContentHash(assemblyAppendContentHash.value)

src/sbt-test/caching/caching/build.sbt

@@ -52,3 +52,8 @@ TaskKey[Unit]("fileCheck1") := {
 TaskKey[Unit]("fileCheck2") := {
   assert((crossTarget.value / "jarHash.txt").exists())
 }
+
+TaskKey[Unit]("hashCheck") := {
+  val x = assemblyMetaBuildHash.value
+  println(x)
+}

src/sbt-test/caching/caching/test

@@ -1,3 +1,5 @@
+> hashCheck
+
 # check if the file gets created
 > clean
 > assembly