diff --git a/cli/src/main/scala/com/advancedtelematic/tuf/cli/Cli.scala b/cli/src/main/scala/com/advancedtelematic/tuf/cli/Cli.scala index e251cf2e..dd38fb0f 100644 --- a/cli/src/main/scala/com/advancedtelematic/tuf/cli/Cli.scala +++ b/cli/src/main/scala/com/advancedtelematic/tuf/cli/Cli.scala @@ -492,6 +492,10 @@ object Cli extends App with VersionInfo { .optional() .text("The timeout for the HTTP request of the upload, in seconds.") .toConfigParam('timeout), + opt[Unit]("force") + .action { (_, c) => c.copy(force = true) } + .text("Force upload of a binary file. This parameter skips checking whether the file has already been added to the targets.") + .hidden() ) .text("""Uploads a binary to the repository. |Note that this will not make the binary available on its own. diff --git a/cli/src/main/scala/com/advancedtelematic/tuf/cli/CommandHandler.scala b/cli/src/main/scala/com/advancedtelematic/tuf/cli/CommandHandler.scala index 47a8760d..ecdee858 100644 --- a/cli/src/main/scala/com/advancedtelematic/tuf/cli/CommandHandler.scala +++ b/cli/src/main/scala/com/advancedtelematic/tuf/cli/CommandHandler.scala @@ -3,12 +3,11 @@ package com.advancedtelematic.tuf.cli import java.net.URI import java.time.{Instant, Period, ZoneOffset} import java.util.concurrent.TimeUnit - import cats.implicits._ import com.advancedtelematic.libats.data.DataType.Checksum import com.advancedtelematic.libtuf.crypt.Sha256FileDigest import com.advancedtelematic.libtuf.data.ClientCodecs._ -import com.advancedtelematic.libtuf.data.ClientDataType.{ClientTargetItem, RootRole, TargetCustom} +import com.advancedtelematic.libtuf.data.ClientDataType.{ClientTargetItem, RootRole, TargetCustom, TargetsRole} import com.advancedtelematic.libtuf.data.TufCodecs._ import com.advancedtelematic.libtuf.data.TufDataType.TargetFormat.TargetFormat import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, RoleType, TargetFilename, TargetFormat, TargetName, TargetVersion, ValidTargetFilename} @@ -40,12 +39,19 @@ object CommandHandler { refineV[ValidTargetFilename](s"${name.value}-${version.value}").leftMap(s => new IllegalArgumentException(s)).toTry } + private def targetItemCreatedAt(targetFilename: TargetFilename)(role: TargetsRole): Option[Instant] = + role.targets.get(targetFilename) + .flatMap(_.custom.flatMap(_.as[TargetCustom].toOption)) + .map(_.createdAt) + private def buildClientTarget(name: TargetName, version: TargetVersion, length: Long, checksum: Checksum, - hardwareIds: List[HardwareIdentifier], uri: Option[URI], format: TargetFormat, cliUploaded: Boolean = false): Try[(TargetFilename, ClientTargetItem)] = + hardwareIds: List[HardwareIdentifier], uri: Option[URI], format: TargetFormat, + cliUploaded: Boolean = false, currentTargetsRole: Option[TargetsRole] = None): Try[(TargetFilename, ClientTargetItem)] = for { targetFilename <- targetFilenameFrom(name, version) + createdAt = currentTargetsRole.flatMap(targetItemCreatedAt(targetFilename)).getOrElse(Instant.now()) newTarget = { - val custom = TargetCustom(name, version, hardwareIds, format.some, uri, cliUploaded = cliUploaded.some) + val custom = TargetCustom(name, version, hardwareIds, format.some, uri, cliUploaded = cliUploaded.some, createdAt = createdAt) val clientHashes = Map(checksum.method -> checksum.hash) ClientTargetItem(clientHashes, length, custom.asJson.some) } @@ -115,7 +121,8 @@ object CommandHandler { config.checksum.valueOrConfigError, config.hardwareIds, config.targetUri, - config.targetFormat + config.targetFormat, + currentTargetsRole = tufRepo.readUnsignedRole[TargetsRole].toOption ) itemT @@ -126,6 +133,7 @@ object CommandHandler { val file = config.inputPath.valueOrConfigError val localFileChecksum = Sha256FileDigest.from(file) + val itemT = buildClientTarget( config.targetName.valueOrConfigError, config.targetVersion.valueOrConfigError, @@ -134,7 +142,8 @@ object CommandHandler { config.hardwareIds, uri = None, format = TargetFormat.BINARY, - cliUploaded = true + cliUploaded = true, + currentTargetsRole = tufRepo.readUnsignedRole[TargetsRole].toOption ) for { @@ -171,7 +180,7 @@ object CommandHandler { for { filename <- Future.fromTry(targetFilenameFrom(config.targetName.valueOrConfigError, config.targetVersion.valueOrConfigError)) client <- repoServer - _ <- tufRepo.uploadTarget(client, filename, config.inputPath.valueOrConfigError, Duration(config.timeout, TimeUnit.SECONDS)) + _ <- tufRepo.uploadTarget(client, filename, config.inputPath.valueOrConfigError, Duration(config.timeout, TimeUnit.SECONDS), config.force) } yield log.info("Target uploaded. You can now add this target to your targets with `targets add`") case PullTargets => diff --git a/cli/src/main/scala/com/advancedtelematic/tuf/cli/repo/TufRepo.scala b/cli/src/main/scala/com/advancedtelematic/tuf/cli/repo/TufRepo.scala index 04f72530..92676ea2 100644 --- a/cli/src/main/scala/com/advancedtelematic/tuf/cli/repo/TufRepo.scala +++ b/cli/src/main/scala/com/advancedtelematic/tuf/cli/repo/TufRepo.scala @@ -374,7 +374,7 @@ abstract class TufRepo[S <: TufServerClient](val repoPath: Path)(implicit ec: Ex rootRolesF } - def uploadTarget(repoClient: S, targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] + def uploadTarget(repoClient: S, targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit] def moveRootOffline(repoClient: S, newRootName: Option[KeyName], @@ -544,8 +544,8 @@ class RepoServerRepo(repoPath: Path)(implicit ec: ExecutionContext) extends TufR } - override def uploadTarget(repoClient: ReposerverClient, targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] = { - repoClient.uploadTarget(targetFilename, inputPath, timeout) + override def uploadTarget(repoClient: ReposerverClient, targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit] = { + repoClient.uploadTarget(targetFilename, inputPath, timeout, force) } override def verifyUploadedBinary(repoClient: ReposerverClient, targetFilename: TargetFilename, localFileChecksum: Checksum): Future[Unit] = { @@ -620,7 +620,7 @@ class DirectorRepo(repoPath: Path)(implicit ec: ExecutionContext) extends TufRep override def addTargetDelegation(name: DelegatedRoleName, key: List[TufKey], delegatedPaths: List[DelegatedPathPattern], threshold: Int): Try[Path] = Failure(CommandNotSupportedByRepositoryType(Director, "addTargetDelegation")) - override def uploadTarget(repoClient: DirectorClient, targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] = + override def uploadTarget(repoClient: DirectorClient, targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit] = Future.failed(CommandNotSupportedByRepositoryType(Director, "uploadTarget")) override def verifyUploadedBinary(reposerverClient: DirectorClient, targetFilename: TargetFilename, localFileChecksum: Checksum): Future[Unit] = diff --git a/cli/src/test/scala/com/advancedtelematic/tuf/cli/CommandHandlerSpec.scala b/cli/src/test/scala/com/advancedtelematic/tuf/cli/CommandHandlerSpec.scala index 7bc89935..ff74be49 100644 --- a/cli/src/test/scala/com/advancedtelematic/tuf/cli/CommandHandlerSpec.scala +++ b/cli/src/test/scala/com/advancedtelematic/tuf/cli/CommandHandlerSpec.scala @@ -6,7 +6,6 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.time.Instant import java.time.temporal.ChronoUnit - import cats.data.Validated.Valid import cats.syntax.either._ import cats.syntax.option._ @@ -16,9 +15,9 @@ import com.advancedtelematic.libtuf.data.ClientCodecs._ import com.advancedtelematic.libtuf.data.ClientDataType import com.advancedtelematic.libtuf.data.ClientDataType.DelegatedPathPattern._ import com.advancedtelematic.libtuf.data.ClientDataType.DelegatedRoleName._ -import com.advancedtelematic.libtuf.data.ClientDataType.{DelegatedPathPattern, DelegatedRoleName, TargetsRole} +import com.advancedtelematic.libtuf.data.ClientDataType.{DelegatedPathPattern, DelegatedRoleName, TargetCustom, TargetsRole} import com.advancedtelematic.libtuf.data.TufCodecs._ -import com.advancedtelematic.libtuf.data.TufDataType.{Ed25519KeyType, KeyType, SignedPayload, TargetName, TargetVersion} +import com.advancedtelematic.libtuf.data.TufDataType.{Ed25519KeyType, KeyType, SignedPayload, TargetFilename, TargetName, TargetVersion} import com.advancedtelematic.libtuf.data.ValidatedString._ import com.advancedtelematic.tuf.cli.Commands._ import com.advancedtelematic.tuf.cli.DataType.{KeyName, MutualTlsConfig, RepoConfig, TreehubConfig} @@ -247,4 +246,40 @@ class CommandHandlerSpec extends CliSpec with KeyTypeSpecSupport with Inspectors method shouldBe HashMethod.SHA256 checksum shouldBe Sha256FileDigest.from(uploadFilePath).hash } + + test("adds the re-uploaded target to targets.json") { + val uploadFilePath = Files.createTempFile("s3upload-", ".txt") + Files.write(uploadFilePath, "“You who read me, are You sure of understanding my language“".getBytes(StandardCharsets.UTF_8)) + + val targetName = TargetName("uploaded-target") + val targetVersion = TargetVersion("0.0.1") + val targetFilename: TargetFilename = Refined.unsafeApply(s"${targetName.value}-${targetVersion.value}") + + val config = Config(AddUploadedTarget, targetName = targetName.some, targetVersion = targetVersion.some, inputPath = uploadFilePath.some) + + handler(config).futureValue + + val role = tufRepo.readUnsignedRole[TargetsRole].get + val addedTarget = role.targets.get(targetFilename).value + + addedTarget.length shouldBe uploadFilePath.toFile.length() + val (method, checksum) = addedTarget.hashes.head + method shouldBe HashMethod.SHA256 + checksum shouldBe Sha256FileDigest.from(uploadFilePath).hash + + val uploadUpdatedFilePath = Files.createTempFile("s3upload-", ".txt") + Files.write(uploadUpdatedFilePath, "“I am an updated file“".getBytes(StandardCharsets.UTF_8)) + + handler(config.copy(inputPath = uploadUpdatedFilePath.some)).futureValue + + val updatedRole = tufRepo.readUnsignedRole[TargetsRole].get + val updatedTarget = updatedRole.targets.get(targetFilename).value + + val (updatedMethod, updatedChecksum) = updatedTarget.hashes.head + updatedMethod shouldBe HashMethod.SHA256 + updatedChecksum shouldBe Sha256FileDigest.from(uploadUpdatedFilePath).hash + + addedTarget.custom.flatMap(_.as[TargetCustom].toOption).map(_.createdAt) shouldBe + updatedTarget.custom.flatMap(_.as[TargetCustom].toOption).map(_.createdAt) + } } diff --git a/cli/src/test/scala/com/advancedtelematic/tuf/cli/util/CliSpec.scala b/cli/src/test/scala/com/advancedtelematic/tuf/cli/util/CliSpec.scala index 56118843..312c0047 100644 --- a/cli/src/test/scala/com/advancedtelematic/tuf/cli/util/CliSpec.scala +++ b/cli/src/test/scala/com/advancedtelematic/tuf/cli/util/CliSpec.scala @@ -152,7 +152,7 @@ class FakeReposerverTufServerClient(val keyType: KeyType) extends ReposerverClie Option(pushedDelegations.get(name)).getOrElse(throw new IllegalArgumentException("[test] delegation not found")) } - override def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] = { + override def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit] = { _log.info(s"Received upload for $targetFilename, $inputPath") uploaded.put(targetFilename, inputPath) Future.successful(()) diff --git a/libtuf/src/main/scala/com/advancedtelematic/libtuf/http/TufServerHttpClient.scala b/libtuf/src/main/scala/com/advancedtelematic/libtuf/http/TufServerHttpClient.scala index 2fdd56c6..3021e102 100644 --- a/libtuf/src/main/scala/com/advancedtelematic/libtuf/http/TufServerHttpClient.scala +++ b/libtuf/src/main/scala/com/advancedtelematic/libtuf/http/TufServerHttpClient.scala @@ -58,7 +58,7 @@ trait ReposerverClient extends TufServerClient { def pushTargets(role: SignedPayload[TargetsRole], previousChecksum: Option[Refined[String, ValidChecksum]]): Future[Unit] - def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] + def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean = false): Future[Unit] def verifyUploadedBinary(targetFilename: TargetFilename, localFileChecksum: Checksum): Future[Unit] } @@ -143,17 +143,17 @@ class ReposerverHttpClient(uri: URI, httpBackend: CliHttpBackend)(implicit ec: E execHttp[SignedPayload[TargetsRole]](req)().map(_.body) } - override def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] = { + override def uploadTarget(targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean = false): Future[Unit] = { val multipartUploadResult = for { inputFile <- Future.fromTry(Try(inputPath.toFile)) - initResult <- initMultipartUpload(targetFilename, inputFile.length()) + initResult <- initMultipartUpload(targetFilename, inputFile.length(), force) result <- s3MultipartUpload(targetFilename, inputFile, initResult.uploadId, initResult.partSize, timeout) } yield result multipartUploadResult.recoverWith { case e: CliHttpClientError if e.remoteError.code == ErrorCodes.Reposerver.NotImplemented => //Multipart upload is not supported for Azure Blob Storage. - uploadByPreSignedUrl(targetFilename, inputPath, timeout) + uploadByPreSignedUrl(targetFilename, inputPath, timeout, force) } } @@ -236,8 +236,8 @@ class ReposerverHttpClient(uri: URI, httpBackend: CliHttpBackend)(implicit ec: E Future.fromTry(uploadResult) } - private def uploadByPreSignedUrl(targetFilename: TargetFilename, inputPath: Path, timeout: Duration): Future[Unit] = { - val req = basicRequest.put(apiUri(s"uploads/" + targetFilename.value)) + private def uploadByPreSignedUrl(targetFilename: TargetFilename, inputPath: Path, timeout: Duration, force: Boolean): Future[Unit] = { + val req = basicRequest.put(apiUri(s"uploads/" + targetFilename.value).params("force" -> force.toString)) .body(inputPath) .readTimeout(timeout) .followRedirects(false) @@ -326,9 +326,9 @@ class ReposerverHttpClient(uri: URI, httpBackend: CliHttpBackend)(implicit ec: E print(s"\u001B[100D$msg") } - private def initMultipartUpload(targetFilename: TargetFilename, fileSize: Long): Future[InitMultipartUploadResult] = { + private def initMultipartUpload(targetFilename: TargetFilename, fileSize: Long, force: Boolean): Future[InitMultipartUploadResult] = { val req = http - .post(apiUri(s"multipart/initiate/" + targetFilename.value).params("fileSize" -> fileSize.toString)) + .post(apiUri(s"multipart/initiate/" + targetFilename.value).params("fileSize" -> fileSize.toString, "force" -> force.toString)) .followRedirects(false) execHttp[InitMultipartUploadResult](req)().map(_.body) diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoResource.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoResource.scala index d43f1a5d..6043a4fc 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoResource.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoResource.scala @@ -277,9 +277,9 @@ class RepoResource(keyserverClient: KeyserverClient, namespaceValidation: Namesp } } ~ pathPrefix("uploads") { - (put & path(TargetFilenamePath) & withContentLengthCheck) { (filename, cl) => + (put & path(TargetFilenamePath) & withContentLengthCheck & parameter('force.as[Boolean] ? false)) { (filename, cl, force) => val f = async { - if(await(targetItemRepo.exists(repoId, filename))) + if(await(targetItemRepo.exists(repoId, filename)) && !force) throw new EntityAlreadyExists[TargetItem]() await(targetStore.buildStorageUrl(repoId, filename, cl)) } @@ -292,9 +292,9 @@ class RepoResource(keyserverClient: KeyserverClient, namespaceValidation: Namesp } } ~ pathPrefix("multipart") { - (post & path("initiate" / TargetFilenamePath) & withMultipartUploadFileSizeCheck) { (fileName, _) => + (post & path("initiate" / TargetFilenamePath) & withMultipartUploadFileSizeCheck & parameter('force.as[Boolean] ? false)) { (fileName, _, force) => val rs = for { - exists <- targetItemRepo.exists(repoId, fileName) + exists <- if (force) Future.successful(false) else targetItemRepo.exists(repoId, fileName) result <- if (exists) Future.failed(new EntityAlreadyExists[TargetItem]()) else targetStore.initiateMultipartUpload(repoId, fileName) } yield result complete(rs)