Merge branch 'feat/OTA-5589/add-force-parameter-to-targets-upload-com…

…mand' into 'master'

OTA-5589 Add --force parameter to binary file upload. Fix createdAt for updated target item

Closes OTA-5589

See merge request olp/edge/ota/connect/back-end/ota-tuf!314

cli/src/main/scala/com/advancedtelematic/tuf/cli/Cli.scala

@@ -492,6 +492,10 @@ object Cli extends App with VersionInfo {
               .optional()
               .text("The timeout for the HTTP request of the upload, in seconds.")
               .toConfigParam('timeout),
+            opt[Unit]("force")
+              .action { (_, c) => c.copy(force = true) }
+              .text("Force upload of a binary file. This parameter skips checking whether the file has already been added to the targets.")
+              .hidden()
           )
           .text("""Uploads a binary to the repository.
                   |Note that this will not make the binary available on its own.

cli/src/main/scala/com/advancedtelematic/tuf/cli/CommandHandler.scala

@@ -3,12 +3,11 @@ package com.advancedtelematic.tuf.cli
 import java.net.URI
 import java.time.{Instant, Period, ZoneOffset}
 import java.util.concurrent.TimeUnit
-
 import cats.implicits._
 import com.advancedtelematic.libats.data.DataType.Checksum
 import com.advancedtelematic.libtuf.crypt.Sha256FileDigest
 import com.advancedtelematic.libtuf.data.ClientCodecs._
-import com.advancedtelematic.libtuf.data.ClientDataType.{ClientTargetItem, RootRole, TargetCustom}
+import com.advancedtelematic.libtuf.data.ClientDataType.{ClientTargetItem, RootRole, TargetCustom, TargetsRole}
 import com.advancedtelematic.libtuf.data.TufCodecs._
 import com.advancedtelematic.libtuf.data.TufDataType.TargetFormat.TargetFormat
 import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, RoleType, TargetFilename, TargetFormat, TargetName, TargetVersion, ValidTargetFilename}
@@ -40,12 +39,19 @@ object CommandHandler {
     refineV[ValidTargetFilename](s"${name.value}-${version.value}").leftMap(s => new IllegalArgumentException(s)).toTry
   }
 
+  private def targetItemCreatedAt(targetFilename: TargetFilename)(role: TargetsRole): Option[Instant] =
+    role.targets.get(targetFilename)
+      .flatMap(_.custom.flatMap(_.as[TargetCustom].toOption))
+      .map(_.createdAt)
+
   private def buildClientTarget(name: TargetName, version: TargetVersion, length: Long, checksum: Checksum,
-                                hardwareIds: List[HardwareIdentifier], uri: Option[URI], format: TargetFormat, cliUploaded: Boolean = false): Try[(TargetFilename, ClientTargetItem)] =
+                                hardwareIds: List[HardwareIdentifier], uri: Option[URI], format: TargetFormat,
+                                cliUploaded: Boolean = false, currentTargetsRole: Option[TargetsRole] = None): Try[(TargetFilename, ClientTargetItem)] =
     for {
       targetFilename <- targetFilenameFrom(name, version)
+      createdAt = currentTargetsRole.flatMap(targetItemCreatedAt(targetFilename)).getOrElse(Instant.now())
       newTarget = {
-        val custom = TargetCustom(name, version, hardwareIds, format.some, uri, cliUploaded = cliUploaded.some)
+        val custom = TargetCustom(name, version, hardwareIds, format.some, uri, cliUploaded = cliUploaded.some, createdAt = createdAt)
         val clientHashes = Map(checksum.method -> checksum.hash)
         ClientTargetItem(clientHashes, length, custom.asJson.some)
       }
@@ -115,7 +121,8 @@ object CommandHandler {
         config.checksum.valueOrConfigError,
         config.hardwareIds,
         config.targetUri,
-        config.targetFormat
+        config.targetFormat,
+        currentTargetsRole = tufRepo.readUnsignedRole[TargetsRole].toOption
       )
 
       itemT
@@ -126,6 +133,7 @@ object CommandHandler {
       val file = config.inputPath.valueOrConfigError
       val localFileChecksum = Sha256FileDigest.from(file)
 
+
       val itemT = buildClientTarget(
         config.targetName.valueOrConfigError,
         config.targetVersion.valueOrConfigError,
@@ -134,7 +142,8 @@ object CommandHandler {
         config.hardwareIds,
         uri = None,
         format = TargetFormat.BINARY,
-        cliUploaded = true
+        cliUploaded = true,
+        currentTargetsRole = tufRepo.readUnsignedRole[TargetsRole].toOption
       )
 
       for {
@@ -171,7 +180,7 @@ object CommandHandler {
       for {
         filename <- Future.fromTry(targetFilenameFrom(config.targetName.valueOrConfigError, config.targetVersion.valueOrConfigError))
         client <- repoServer
-        _ <- tufRepo.uploadTarget(client, filename, config.inputPath.valueOrConfigError, Duration(config.timeout, TimeUnit.SECONDS))
+        _ <- tufRepo.uploadTarget(client, filename, config.inputPath.valueOrConfigError, Duration(config.timeout, TimeUnit.SECONDS), config.force)
       } yield log.info("Target uploaded. You can now add this target to your targets with `targets add`")
 
     case PullTargets =>

cli/src/main/scala/com/advancedtelematic/tuf/cli/repo/TufRepo.scala

@@ -374,7 +374,7 @@ abstract class TufRepo[S <: TufServerClient](val repoPath: Path)(implicit ec: Ex
     rootRolesF
   }
 
-  def uploadTarget(repoClient: S, targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit]
+  def uploadTarget(repoClient: S, targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit]
 
   def moveRootOffline(repoClient: S,
                       newRootName: Option[KeyName],
@@ -544,8 +544,8 @@ class RepoServerRepo(repoPath: Path)(implicit ec: ExecutionContext) extends TufR
 
   }
 
-  override def uploadTarget(repoClient: ReposerverClient, targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] = {
-    repoClient.uploadTarget(targetFilename, inputPath, timeout)
+  override def uploadTarget(repoClient: ReposerverClient, targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit] = {
+    repoClient.uploadTarget(targetFilename, inputPath, timeout, force)
   }
 
   override def verifyUploadedBinary(repoClient: ReposerverClient, targetFilename: TargetFilename, localFileChecksum: Checksum): Future[Unit] = {
@@ -620,7 +620,7 @@ class DirectorRepo(repoPath: Path)(implicit ec: ExecutionContext) extends TufRep
   override def addTargetDelegation(name: DelegatedRoleName, key: List[TufKey], delegatedPaths: List[DelegatedPathPattern], threshold: Int): Try[Path] =
     Failure(CommandNotSupportedByRepositoryType(Director, "addTargetDelegation"))
 
-  override def uploadTarget(repoClient: DirectorClient, targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] =
+  override def uploadTarget(repoClient: DirectorClient, targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit] =
     Future.failed(CommandNotSupportedByRepositoryType(Director, "uploadTarget"))
 
   override def verifyUploadedBinary(reposerverClient: DirectorClient, targetFilename: TargetFilename, localFileChecksum: Checksum): Future[Unit] =

cli/src/test/scala/com/advancedtelematic/tuf/cli/CommandHandlerSpec.scala

@@ -6,7 +6,6 @@ import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.time.Instant
 import java.time.temporal.ChronoUnit
-
 import cats.data.Validated.Valid
 import cats.syntax.either._
 import cats.syntax.option._
@@ -16,9 +15,9 @@ import com.advancedtelematic.libtuf.data.ClientCodecs._
 import com.advancedtelematic.libtuf.data.ClientDataType
 import com.advancedtelematic.libtuf.data.ClientDataType.DelegatedPathPattern._
 import com.advancedtelematic.libtuf.data.ClientDataType.DelegatedRoleName._
-import com.advancedtelematic.libtuf.data.ClientDataType.{DelegatedPathPattern, DelegatedRoleName, TargetsRole}
+import com.advancedtelematic.libtuf.data.ClientDataType.{DelegatedPathPattern, DelegatedRoleName, TargetCustom, TargetsRole}
 import com.advancedtelematic.libtuf.data.TufCodecs._
-import com.advancedtelematic.libtuf.data.TufDataType.{Ed25519KeyType, KeyType, SignedPayload, TargetName, TargetVersion}
+import com.advancedtelematic.libtuf.data.TufDataType.{Ed25519KeyType, KeyType, SignedPayload, TargetFilename, TargetName, TargetVersion}
 import com.advancedtelematic.libtuf.data.ValidatedString._
 import com.advancedtelematic.tuf.cli.Commands._
 import com.advancedtelematic.tuf.cli.DataType.{KeyName, MutualTlsConfig, RepoConfig, TreehubConfig}
@@ -247,4 +246,40 @@ class CommandHandlerSpec extends CliSpec with KeyTypeSpecSupport with Inspectors
     method shouldBe HashMethod.SHA256
     checksum shouldBe Sha256FileDigest.from(uploadFilePath).hash
   }
+
+  test("adds the re-uploaded target to targets.json") {
+    val uploadFilePath = Files.createTempFile("s3upload-", ".txt")
+    Files.write(uploadFilePath, "“You who read me, are You sure of understanding my language“".getBytes(StandardCharsets.UTF_8))
+
+    val targetName = TargetName("uploaded-target")
+    val targetVersion = TargetVersion("0.0.1")
+    val targetFilename: TargetFilename = Refined.unsafeApply(s"${targetName.value}-${targetVersion.value}")
+
+    val config = Config(AddUploadedTarget, targetName = targetName.some, targetVersion = targetVersion.some, inputPath = uploadFilePath.some)
+
+    handler(config).futureValue
+
+    val role = tufRepo.readUnsignedRole[TargetsRole].get
+    val addedTarget = role.targets.get(targetFilename).value
+
+    addedTarget.length shouldBe uploadFilePath.toFile.length()
+    val (method, checksum) = addedTarget.hashes.head
+    method shouldBe HashMethod.SHA256
+    checksum shouldBe Sha256FileDigest.from(uploadFilePath).hash
+
+    val uploadUpdatedFilePath = Files.createTempFile("s3upload-", ".txt")
+    Files.write(uploadUpdatedFilePath, "“I am an updated file“".getBytes(StandardCharsets.UTF_8))
+
+    handler(config.copy(inputPath = uploadUpdatedFilePath.some)).futureValue
+
+    val updatedRole = tufRepo.readUnsignedRole[TargetsRole].get
+    val updatedTarget = updatedRole.targets.get(targetFilename).value
+
+    val (updatedMethod, updatedChecksum) = updatedTarget.hashes.head
+    updatedMethod shouldBe HashMethod.SHA256
+    updatedChecksum shouldBe Sha256FileDigest.from(uploadUpdatedFilePath).hash
+
+    addedTarget.custom.flatMap(_.as[TargetCustom].toOption).map(_.createdAt) shouldBe
+      updatedTarget.custom.flatMap(_.as[TargetCustom].toOption).map(_.createdAt)
+  }
 }

cli/src/test/scala/com/advancedtelematic/tuf/cli/util/CliSpec.scala

@@ -152,7 +152,7 @@ class FakeReposerverTufServerClient(val keyType: KeyType) extends ReposerverClie
     Option(pushedDelegations.get(name)).getOrElse(throw new IllegalArgumentException("[test] delegation not found"))
   }
 
-  override def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] = {
+  override def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit] = {
     _log.info(s"Received upload for $targetFilename, $inputPath")
     uploaded.put(targetFilename, inputPath)
     Future.successful(())

libtuf/src/main/scala/com/advancedtelematic/libtuf/http/TufServerHttpClient.scala

@@ -58,7 +58,7 @@ trait ReposerverClient extends TufServerClient {
 
   def pushTargets(role: SignedPayload[TargetsRole], previousChecksum: Option[Refined[String, ValidChecksum]]): Future[Unit]
 
-  def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit]
+  def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean = false): Future[Unit]
 
   def verifyUploadedBinary(targetFilename: TargetFilename, localFileChecksum: Checksum): Future[Unit]
 }
@@ -143,17 +143,17 @@ class ReposerverHttpClient(uri: URI, httpBackend: CliHttpBackend)(implicit ec: E
     execHttp[SignedPayload[TargetsRole]](req)().map(_.body)
   }
 
-  override def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] = {
+  override def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean = false): Future[Unit] = {
     val multipartUploadResult = for {
       inputFile <- Future.fromTry(Try(inputPath.toFile))
-      initResult <- initMultipartUpload(targetFilename, inputFile.length())
+      initResult <- initMultipartUpload(targetFilename, inputFile.length(), force)
       result <- s3MultipartUpload(targetFilename, inputFile, initResult.uploadId, initResult.partSize, timeout)
     } yield result
 
     multipartUploadResult.recoverWith {
       case e: CliHttpClientError if e.remoteError.code == ErrorCodes.Reposerver.NotImplemented =>
         //Multipart upload is not supported for Azure Blob Storage.
-        uploadByPreSignedUrl(targetFilename, inputPath, timeout)
+        uploadByPreSignedUrl(targetFilename, inputPath, timeout, force)
     }
   }
 
@@ -236,8 +236,8 @@ class ReposerverHttpClient(uri: URI, httpBackend: CliHttpBackend)(implicit ec: E
     Future.fromTry(uploadResult)
   }
 
-  private def uploadByPreSignedUrl(targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] = {
-    val req = basicRequest.put(apiUri(s"uploads/" + targetFilename.value))
+  private def uploadByPreSignedUrl(targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit] = {
+    val req = basicRequest.put(apiUri(s"uploads/" + targetFilename.value).params("force" -> force.toString))
       .body(inputPath)
       .readTimeout(timeout)
       .followRedirects(false)
@@ -326,9 +326,9 @@ class ReposerverHttpClient(uri: URI, httpBackend: CliHttpBackend)(implicit ec: E
     print(s"\u001B[100D$msg")
   }
 
-  private def initMultipartUpload(targetFilename: TargetFilename, fileSize: Long): Future[InitMultipartUploadResult] = {
+  private def initMultipartUpload(targetFilename: TargetFilename, fileSize: Long, force: Boolean): Future[InitMultipartUploadResult] = {
     val req = http
-      .post(apiUri(s"multipart/initiate/" + targetFilename.value).params("fileSize" -> fileSize.toString))
+      .post(apiUri(s"multipart/initiate/" + targetFilename.value).params("fileSize" -> fileSize.toString, "force" -> force.toString))
       .followRedirects(false)
 
     execHttp[InitMultipartUploadResult](req)().map(_.body)

reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoResource.scala

@@ -277,9 +277,9 @@ class RepoResource(keyserverClient: KeyserverClient, namespaceValidation: Namesp
         }
       } ~
       pathPrefix("uploads") {
-        (put & path(TargetFilenamePath) & withContentLengthCheck) { (filename, cl) =>
+        (put & path(TargetFilenamePath) & withContentLengthCheck & parameter('force.as[Boolean] ? false)) { (filename, cl, force) =>
           val f = async {
-            if(await(targetItemRepo.exists(repoId, filename)))
+            if(await(targetItemRepo.exists(repoId, filename)) && !force)
               throw new EntityAlreadyExists[TargetItem]()
             await(targetStore.buildStorageUrl(repoId, filename, cl))
           }
@@ -292,9 +292,9 @@ class RepoResource(keyserverClient: KeyserverClient, namespaceValidation: Namesp
         }
       } ~
       pathPrefix("multipart") {
-        (post & path("initiate" / TargetFilenamePath) & withMultipartUploadFileSizeCheck) { (fileName, _) =>
+        (post & path("initiate" / TargetFilenamePath) & withMultipartUploadFileSizeCheck & parameter('force.as[Boolean] ? false)) { (fileName, _, force) =>
           val rs = for {
-            exists <- targetItemRepo.exists(repoId, fileName)
+            exists <- if (force) Future.successful(false) else targetItemRepo.exists(repoId, fileName)
             result <- if (exists) Future.failed(new EntityAlreadyExists[TargetItem]()) else targetStore.initiateMultipartUpload(repoId, fileName)
           } yield result
           complete(rs)