From d2355e7245562db875724f598febe5fa71f8643b Mon Sep 17 00:00:00 2001 From: SniXosha Date: Wed, 2 Jun 2021 16:04:24 +0300 Subject: [PATCH] Improve config_hash and image_digest resolving (#88) * Read and write configHash from/to all resources * Replace digest in all resources * Check config hash in plain apps --- build.gradle | 2 +- .../management/deploy/AppsDeployCommand.java | 3 +- .../checker/UpToDatePlainAppChecker.java | 17 ++-- .../smart/hash/ResourcesHashComputer.java | 28 +++--- .../smart/image/ImageVersionReplacer.java | 89 ++++++++----------- .../plain/PlainAppDescription.java | 22 +++++ .../application/plain/PlainApplication.java | 10 ++- .../smart/hash/ResourcesHashComputerTest.java | 10 +++ .../smart/image/ImageVersionReplacerTest.java | 11 +++ .../components/plain-app/deploy.yaml | 5 ++ .../plain-app/resources/resource1.yaml | 9 ++ .../plain-app/resources/resource2.yaml | 6 ++ 12 files changed, 137 insertions(+), 75 deletions(-) create mode 100644 src/main/java/io/osdf/core/application/plain/PlainAppDescription.java create mode 100644 src/test/resources/components/plain-app/deploy.yaml create mode 100644 src/test/resources/components/plain-app/resources/resource1.yaml create mode 100644 src/test/resources/components/plain-app/resources/resource2.yaml diff --git a/build.gradle b/build.gradle index 12d516c1..2818fd0c 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ plugins { group 'io.microconfig.osdf' -version '1.7.0' +version '1.7.1' sourceCompatibility = 11 diff --git a/src/main/java/io/osdf/actions/management/deploy/AppsDeployCommand.java b/src/main/java/io/osdf/actions/management/deploy/AppsDeployCommand.java index 5c8486a1..2b8a30a2 100644 --- a/src/main/java/io/osdf/actions/management/deploy/AppsDeployCommand.java +++ b/src/main/java/io/osdf/actions/management/deploy/AppsDeployCommand.java @@ -68,8 +68,7 @@ private void preprocessServices(List apps) { ResourcesHashComputer resourcesHashComputer = resourcesHashComputer(); ImageVersionReplacer tagReplacer = imageVersionReplacer(digestGetter(paths)); - apps.forEach(service -> resourcesHashComputer.insertIn(service.files())); apps.forEach(service -> tagReplacer.replaceFor(service.files())); + apps.forEach(service -> resourcesHashComputer.insertIn(service.files())); } - } diff --git a/src/main/java/io/osdf/actions/management/deploy/smart/checker/UpToDatePlainAppChecker.java b/src/main/java/io/osdf/actions/management/deploy/smart/checker/UpToDatePlainAppChecker.java index cf5b2536..ee6a0f65 100644 --- a/src/main/java/io/osdf/actions/management/deploy/smart/checker/UpToDatePlainAppChecker.java +++ b/src/main/java/io/osdf/actions/management/deploy/smart/checker/UpToDatePlainAppChecker.java @@ -1,11 +1,14 @@ package io.osdf.actions.management.deploy.smart.checker; -import io.osdf.common.yaml.YamlObject; import io.osdf.core.application.core.Application; -import io.osdf.core.application.core.description.CoreDescription; +import io.osdf.core.application.plain.PlainAppDescription; +import io.osdf.core.application.plain.PlainApplication; import java.util.Optional; +import static io.osdf.actions.management.deploy.smart.hash.ResourcesHashComputer.resourcesHashComputer; +import static io.osdf.core.application.plain.PlainApplication.plainApplication; + public class UpToDatePlainAppChecker implements UpToDateChecker { public static UpToDatePlainAppChecker upToDatePlainAppChecker() { return new UpToDatePlainAppChecker(); @@ -13,10 +16,14 @@ public static UpToDatePlainAppChecker upToDatePlainAppChecker() { @Override public boolean check(Application app) { - Optional description = app.coreDescription(); + PlainApplication plainApp = plainApplication(app); + Optional description = plainApp.loadDescription(PlainAppDescription.class, "plain"); if (description.isEmpty()) return false; - YamlObject deployProperties = app.files().deployProperties(); - return description.get().getConfigVersion().equals(deployProperties.get("config.version")); + String remoteHash = description.get().getConfigHash(); + if (remoteHash == null) return false; + + String localHash = resourcesHashComputer().currentHash(app.files()); + return remoteHash.equals(localHash); } } diff --git a/src/main/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputer.java b/src/main/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputer.java index 0fc34951..af734a7d 100644 --- a/src/main/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputer.java +++ b/src/main/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputer.java @@ -1,15 +1,15 @@ package io.osdf.actions.management.deploy.smart.hash; import io.osdf.common.utils.FileUtils; +import io.osdf.common.yaml.YamlObject; import io.osdf.core.application.core.files.ApplicationFiles; import io.osdf.core.cluster.resource.LocalClusterResource; import lombok.RequiredArgsConstructor; -import java.nio.file.Path; +import java.util.Objects; import static io.osdf.common.utils.FileUtils.readAll; import static io.osdf.common.utils.FileUtils.writeStringToFile; -import static io.osdf.common.yaml.YamlObject.yaml; import static java.util.stream.Collectors.joining; import static org.apache.commons.codec.digest.DigestUtils.md5Hex; @@ -22,8 +22,6 @@ public static ResourcesHashComputer resourcesHashComputer() { } public void insertIn(ApplicationFiles files) { - if (files.metadata().getMainResource() == null) return; - String currentHash = currentHash(files); if (currentHash == null || !currentHash.equals(HASH_PLACEHOLDER)) return; @@ -31,15 +29,19 @@ public void insertIn(ApplicationFiles files) { } public String currentHash(ApplicationFiles files) { - String resourcePath = files.metadata() - .getMainResource() - .getPath(); - return yaml(Path.of(resourcePath)).get("metadata.labels.configHash"); + return files.resources() + .stream() + .map(LocalClusterResource::path) + .map(YamlObject::yaml) + .map(yaml -> yaml.get("metadata.labels.configHash")) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); } private void computeAndInsertHash(ApplicationFiles files) { String hash = computeHash(files); - insertHash(Path.of(files.metadata().getMainResource().getPath()), hash); + insertHash(files, hash); } public String computeHash(ApplicationFiles files) { @@ -51,8 +53,10 @@ public String computeHash(ApplicationFiles files) { ); } - private void insertHash(Path path, String hash) { - String newContent = readAll(path).replace(HASH_PLACEHOLDER, hash); - writeStringToFile(path, newContent); + private void insertHash(ApplicationFiles files, String hash) { + files.resources().forEach(resource -> { + String newContent = readAll(resource.path()).replace(HASH_PLACEHOLDER, hash); + writeStringToFile(resource.path(), newContent); + }); } } diff --git a/src/main/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacer.java b/src/main/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacer.java index 5a4980c8..aa463b13 100644 --- a/src/main/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacer.java +++ b/src/main/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacer.java @@ -2,19 +2,13 @@ import io.osdf.actions.management.deploy.smart.image.digest.DigestGetter; import io.osdf.common.exceptions.OSDFException; -import io.osdf.common.yaml.YamlObject; import io.osdf.core.application.core.files.ApplicationFiles; -import io.osdf.core.application.core.files.metadata.LocalResourceMetadata; import lombok.RequiredArgsConstructor; -import java.nio.file.Path; -import java.util.List; import java.util.regex.Matcher; import static io.osdf.common.utils.FileUtils.readAll; import static io.osdf.common.utils.FileUtils.writeStringToFile; -import static io.osdf.common.yaml.YamlObject.yaml; -import static java.nio.file.Path.of; import static java.util.Arrays.stream; import static java.util.Objects.requireNonNullElse; import static java.util.regex.Pattern.compile; @@ -28,66 +22,37 @@ public static ImageVersionReplacer imageVersionReplacer(DigestGetter digestGette } public void replaceFor(ApplicationFiles files) { - if (files.metadata().getMainResource() == null) return; - String mainResourceContent = readAll(of(files.metadata().getMainResource().getPath())); - - String versionEntry = findVersionEntry(mainResourceContent); - if (versionEntry == null) return; - - String version = findVersion(versionEntry); - if (version == null) throw new OSDFException("Bad image version format: " + versionEntry); - - Boolean continueOnError = files.deployProperties() - .get("osdf.replaceVersionWithDigest.continueOnError"); - - getDigestAndReplace(files.metadata().getMainResource(), versionEntry, version, - requireNonNullElse(continueOnError, false)); + Boolean continueOnError = requireNonNullElse( + files.deployProperties().get("osdf.replaceVersionWithDigest.continueOnError"), false + ); + + files.resources().forEach(resource -> { + String newContent = replaceEntries(readAll(resource.path()), continueOnError); + writeStringToFile(resource.path(), newContent); + }); } - private void getDigestAndReplace(LocalResourceMetadata resource, String versionEntry, String version, - boolean continueOnError) { - YamlObject resourceYaml = yaml(of(resource.getPath())); - String imageUrl = imageUrl(resourceYaml); - if (imageUrl.contains("@")) return; - - String digest = getDigest(imageUrl.replace(versionEntry, version), continueOnError); - if (digest == null) return; + private String replaceEntries(String content, boolean continueOnError) { + while (true) { + ParsedEntry entry = findEntry(content); + if (entry == null) return content; - replaceTagInResource(of(resource.getPath()), imageUrl, imageUrl.replace(":" + versionEntry, "@" + digest)); - } - - private void replaceTagInResource(Path pathToResource, String imageUrl, String newImageUrl) { - String newContent = readAll(pathToResource) - .replace(imageUrl, newImageUrl); - writeStringToFile(pathToResource, newContent); + String digest = getDigest(entry.trueUrl(), continueOnError); + content = content.replace(entry.url, digest == null ? entry.trueUrl() : entry.urlWithDigest(digest)); + } } - private String findVersionEntry(String mainResourceContent) { - String lineWithImage = stream(mainResourceContent.split("\n")) + private ParsedEntry findEntry(String content) { + String lineWithImage = stream(content.split("\n")) .filter(line -> line.contains("IMAGE_DIGEST")) .findFirst() .orElse(null); if (lineWithImage == null) return null; - Matcher matcher = compile(".*().*").matcher(lineWithImage); - if (!matcher.matches()) return null; - return matcher.group(1); - } - - private String findVersion(String versionEntry) { - Matcher matcher = compile("").matcher(versionEntry); + Matcher matcher = compile(".*\\s+(.*()).*").matcher(lineWithImage); if (!matcher.matches()) return null; - return matcher.group(1); - } - private String imageUrl(YamlObject resourceYaml) { - return resourceYaml - .>get("spec.template.spec.containers") - .stream() - .map(YamlObject::yaml) - .map(container -> container.get("image")) - .findFirst() - .orElseThrow(() -> new OSDFException("Container not found")); + return new ParsedEntry(matcher.group(1), matcher.group(2), matcher.group(3)); } private String getDigest(String imageUrl, boolean continueOnError) { @@ -98,4 +63,20 @@ private String getDigest(String imageUrl, boolean continueOnError) { throw new OSDFException("Couldn't get digest for " + imageUrl + ": " + e.getMessage()); } } + + @RequiredArgsConstructor + static class ParsedEntry { + private final String url; + private final String versionEntry; + private final String version; + + public String trueUrl() { + return url.replace(versionEntry, version); + } + + public String urlWithDigest(String digest) { + return url.replace(":" + versionEntry, "@" + digest); + } + } + } diff --git a/src/main/java/io/osdf/core/application/plain/PlainAppDescription.java b/src/main/java/io/osdf/core/application/plain/PlainAppDescription.java new file mode 100644 index 00000000..5621f1af --- /dev/null +++ b/src/main/java/io/osdf/core/application/plain/PlainAppDescription.java @@ -0,0 +1,22 @@ +package io.osdf.core.application.plain; + +import io.osdf.core.application.core.files.ApplicationFiles; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import static io.osdf.actions.management.deploy.smart.hash.ResourcesHashComputer.resourcesHashComputer; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class PlainAppDescription { + private String configHash; + + public static PlainAppDescription from(ApplicationFiles files) { + String hash = resourcesHashComputer().currentHash(files); + return new PlainAppDescription(hash); + } +} diff --git a/src/main/java/io/osdf/core/application/plain/PlainApplication.java b/src/main/java/io/osdf/core/application/plain/PlainApplication.java index bc0785fd..29a47aad 100644 --- a/src/main/java/io/osdf/core/application/plain/PlainApplication.java +++ b/src/main/java/io/osdf/core/application/plain/PlainApplication.java @@ -1,6 +1,8 @@ package io.osdf.core.application.plain; +import io.osdf.common.exceptions.OSDFException; import io.osdf.core.application.core.AbstractApplication; +import io.osdf.core.application.core.Application; import io.osdf.core.application.core.description.CoreDescription; import io.osdf.core.application.core.files.ApplicationFiles; import io.osdf.core.connection.cli.ClusterCli; @@ -20,9 +22,15 @@ public static PlainApplication plainApplication(ApplicationFiles files, ClusterC return new PlainApplication(files, cli); } + public static PlainApplication plainApplication(Application app) { + if (!(app instanceof PlainApplication)) throw new OSDFException(app.name() + " is not a plain app"); + return (PlainApplication) app; + } + public void uploadDescription() { configMapLoader(cli).upload(super.descriptionConfigMapName(), of( - "core", CoreDescription.from(super.files()) + "core", CoreDescription.from(super.files()), + "plain", PlainAppDescription.from(super.files()) )); } } diff --git a/src/test/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputerTest.java b/src/test/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputerTest.java index ae639791..42f8d9db 100644 --- a/src/test/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputerTest.java +++ b/src/test/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputerTest.java @@ -31,4 +31,14 @@ void skipIfHashInsertionIsNotNeeded() throws IOException { String contentAfterInsertion = readString(files.getPath("resources/deployment.yaml")); assertEquals(originalContent, contentAfterInsertion); } + + @Test + void replaceAll() throws IOException { + ApplicationFiles files = applicationFilesFor("plain-app"); + + resourcesHashComputer().insertIn(files); + + assertFalse(readString(files.getPath("resources/resource1.yaml")).contains("")); + assertFalse(readString(files.getPath("resources/resource2.yaml")).contains("")); + } } \ No newline at end of file diff --git a/src/test/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacerTest.java b/src/test/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacerTest.java index 3e02839c..243e6edc 100644 --- a/src/test/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacerTest.java +++ b/src/test/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacerTest.java @@ -45,6 +45,17 @@ void digestNotAvailable() { assertThrows(OSDFException.class, () -> replacer.replaceFor(files)); } + @Test + void replaceAll() throws IOException { + ImageVersionReplacer replacer = imageVersionReplacer(digestGetter()); + ApplicationFiles files = applicationFilesFor("plain-app"); + + replacer.replaceFor(files); + + assertTrue(readString(files.getPath("resources/resource1.yaml")).contains("@sha256:digest")); + assertTrue(readString(files.getPath("resources/resource2.yaml")).contains("@sha256:digest")); + } + private DigestGetter digestGetter() { DigestGetter digestGetter = mock(DigestGetter.class); when(digestGetter.get(anyString())).thenReturn("sha256:digest"); diff --git a/src/test/resources/components/plain-app/deploy.yaml b/src/test/resources/components/plain-app/deploy.yaml new file mode 100644 index 00000000..3f5bfcd8 --- /dev/null +++ b/src/test/resources/components/plain-app/deploy.yaml @@ -0,0 +1,5 @@ +app: + version: latest + +config: + version: master \ No newline at end of file diff --git a/src/test/resources/components/plain-app/resources/resource1.yaml b/src/test/resources/components/plain-app/resources/resource1.yaml new file mode 100644 index 00000000..4527b432 --- /dev/null +++ b/src/test/resources/components/plain-app/resources/resource1.yaml @@ -0,0 +1,9 @@ +kind: FakeResource +metadata: + name: fake + labels: + configHash: +body: + image: url/image: + sameImage: url/image: + image2: url/image2: \ No newline at end of file diff --git a/src/test/resources/components/plain-app/resources/resource2.yaml b/src/test/resources/components/plain-app/resources/resource2.yaml new file mode 100644 index 00000000..ac7c9e58 --- /dev/null +++ b/src/test/resources/components/plain-app/resources/resource2.yaml @@ -0,0 +1,6 @@ +kind: FakeResource +metadata: + name: fake +body: + randomValue: + image: url/image: \ No newline at end of file