Improve config_hash and image_digest resolving (#88)

* Read and write configHash from/to all resources * Replace digest in all resources * Check config hash in plain apps
microconfig · Jun 2, 2021 · d2355e7 · d2355e7
1 parent f5f27f0
commit d2355e7
Show file tree

Hide file tree

Showing 12 changed files with 137 additions and 75 deletions.
diff --git a/build.gradle b/build.gradle
@@ -9,7 +9,7 @@ plugins {
 
 group 'io.microconfig.osdf'
 
-version '1.7.0'
+version '1.7.1'
 
 sourceCompatibility = 11
 

diff --git a/src/main/java/io/osdf/actions/management/deploy/AppsDeployCommand.java b/src/main/java/io/osdf/actions/management/deploy/AppsDeployCommand.java
@@ -68,8 +68,7 @@ private void preprocessServices(List<Application> apps) {
         ResourcesHashComputer resourcesHashComputer = resourcesHashComputer();
         ImageVersionReplacer tagReplacer = imageVersionReplacer(digestGetter(paths));
 
-        apps.forEach(service -> resourcesHashComputer.insertIn(service.files()));
         apps.forEach(service -> tagReplacer.replaceFor(service.files()));
+        apps.forEach(service -> resourcesHashComputer.insertIn(service.files()));
     }
-
 }
diff --git a/src/main/java/io/osdf/actions/management/deploy/smart/checker/UpToDatePlainAppChecker.java b/src/main/java/io/osdf/actions/management/deploy/smart/checker/UpToDatePlainAppChecker.java
@@ -1,22 +1,29 @@
 package io.osdf.actions.management.deploy.smart.checker;
 
-import io.osdf.common.yaml.YamlObject;
 import io.osdf.core.application.core.Application;
-import io.osdf.core.application.core.description.CoreDescription;
+import io.osdf.core.application.plain.PlainAppDescription;
+import io.osdf.core.application.plain.PlainApplication;
 
 import java.util.Optional;
 
+import static io.osdf.actions.management.deploy.smart.hash.ResourcesHashComputer.resourcesHashComputer;
+import static io.osdf.core.application.plain.PlainApplication.plainApplication;
+
 public class UpToDatePlainAppChecker implements UpToDateChecker {
     public static UpToDatePlainAppChecker upToDatePlainAppChecker() {
         return new UpToDatePlainAppChecker();
     }
 
     @Override
     public boolean check(Application app) {
-        Optional<CoreDescription> description = app.coreDescription();
+        PlainApplication plainApp = plainApplication(app);
+        Optional<PlainAppDescription> description = plainApp.loadDescription(PlainAppDescription.class, "plain");
         if (description.isEmpty()) return false;
 
-        YamlObject deployProperties = app.files().deployProperties();
-        return description.get().getConfigVersion().equals(deployProperties.get("config.version"));
+        String remoteHash = description.get().getConfigHash();
+        if (remoteHash == null) return false;
+
+        String localHash = resourcesHashComputer().currentHash(app.files());
+        return remoteHash.equals(localHash);
     }
 }
diff --git a/src/main/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputer.java b/src/main/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputer.java
@@ -1,15 +1,15 @@
 package io.osdf.actions.management.deploy.smart.hash;
 
 import io.osdf.common.utils.FileUtils;
+import io.osdf.common.yaml.YamlObject;
 import io.osdf.core.application.core.files.ApplicationFiles;
 import io.osdf.core.cluster.resource.LocalClusterResource;
 import lombok.RequiredArgsConstructor;
 
-import java.nio.file.Path;
+import java.util.Objects;
 
 import static io.osdf.common.utils.FileUtils.readAll;
 import static io.osdf.common.utils.FileUtils.writeStringToFile;
-import static io.osdf.common.yaml.YamlObject.yaml;
 import static java.util.stream.Collectors.joining;
 import static org.apache.commons.codec.digest.DigestUtils.md5Hex;
 
@@ -22,24 +22,26 @@ public static ResourcesHashComputer resourcesHashComputer() {
     }
 
     public void insertIn(ApplicationFiles files) {
-        if (files.metadata().getMainResource() == null) return;
-
         String currentHash = currentHash(files);
         if (currentHash == null || !currentHash.equals(HASH_PLACEHOLDER)) return;
 
         computeAndInsertHash(files);
     }
 
     public String currentHash(ApplicationFiles files) {
-        String resourcePath = files.metadata()
-                .getMainResource()
-                .getPath();
-        return yaml(Path.of(resourcePath)).get("metadata.labels.configHash");
+        return files.resources()
+                .stream()
+                .map(LocalClusterResource::path)
+                .map(YamlObject::yaml)
+                .map(yaml -> yaml.<String>get("metadata.labels.configHash"))
+                .filter(Objects::nonNull)
+                .findFirst()
+                .orElse(null);
     }
 
     private void computeAndInsertHash(ApplicationFiles files) {
         String hash = computeHash(files);
-        insertHash(Path.of(files.metadata().getMainResource().getPath()), hash);
+        insertHash(files, hash);
     }
 
     public String computeHash(ApplicationFiles files) {
@@ -51,8 +53,10 @@ public String computeHash(ApplicationFiles files) {
         );
     }
 
-    private void insertHash(Path path, String hash) {
-        String newContent = readAll(path).replace(HASH_PLACEHOLDER, hash);
-        writeStringToFile(path, newContent);
+    private void insertHash(ApplicationFiles files, String hash) {
+        files.resources().forEach(resource -> {
+            String newContent = readAll(resource.path()).replace(HASH_PLACEHOLDER, hash);
+            writeStringToFile(resource.path(), newContent);
+        });
     }
 }
diff --git a/src/main/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacer.java b/src/main/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacer.java
@@ -2,19 +2,13 @@
 
 import io.osdf.actions.management.deploy.smart.image.digest.DigestGetter;
 import io.osdf.common.exceptions.OSDFException;
-import io.osdf.common.yaml.YamlObject;
 import io.osdf.core.application.core.files.ApplicationFiles;
-import io.osdf.core.application.core.files.metadata.LocalResourceMetadata;
 import lombok.RequiredArgsConstructor;
 
-import java.nio.file.Path;
-import java.util.List;
 import java.util.regex.Matcher;
 
 import static io.osdf.common.utils.FileUtils.readAll;
 import static io.osdf.common.utils.FileUtils.writeStringToFile;
-import static io.osdf.common.yaml.YamlObject.yaml;
-import static java.nio.file.Path.of;
 import static java.util.Arrays.stream;
 import static java.util.Objects.requireNonNullElse;
 import static java.util.regex.Pattern.compile;
@@ -28,66 +22,37 @@ public static ImageVersionReplacer imageVersionReplacer(DigestGetter digestGette
     }
 
     public void replaceFor(ApplicationFiles files) {
-        if (files.metadata().getMainResource() == null) return;
-        String mainResourceContent = readAll(of(files.metadata().getMainResource().getPath()));
-
-        String versionEntry = findVersionEntry(mainResourceContent);
-        if (versionEntry == null) return;
-
-        String version = findVersion(versionEntry);
-        if (version == null) throw new OSDFException("Bad image version format: " + versionEntry);
-
-        Boolean continueOnError = files.deployProperties()
-                .get("osdf.replaceVersionWithDigest.continueOnError");
-
-        getDigestAndReplace(files.metadata().getMainResource(), versionEntry, version,
-                requireNonNullElse(continueOnError, false));
+        Boolean continueOnError = requireNonNullElse(
+                files.deployProperties().get("osdf.replaceVersionWithDigest.continueOnError"), false
+        );
+
+        files.resources().forEach(resource -> {
+            String newContent = replaceEntries(readAll(resource.path()), continueOnError);
+            writeStringToFile(resource.path(), newContent);
+        });
     }
 
-    private void getDigestAndReplace(LocalResourceMetadata resource, String versionEntry, String version,
-                                     boolean continueOnError) {
-        YamlObject resourceYaml = yaml(of(resource.getPath()));
-        String imageUrl = imageUrl(resourceYaml);
-        if (imageUrl.contains("@")) return;
-
-        String digest = getDigest(imageUrl.replace(versionEntry, version), continueOnError);
-        if (digest == null) return;
+    private String replaceEntries(String content, boolean continueOnError) {
+        while (true) {
+            ParsedEntry entry = findEntry(content);
+            if (entry == null) return content;
 
-        replaceTagInResource(of(resource.getPath()), imageUrl, imageUrl.replace(":" + versionEntry, "@" + digest));
-    }
-
-    private void replaceTagInResource(Path pathToResource, String imageUrl, String newImageUrl) {
-        String newContent = readAll(pathToResource)
-                .replace(imageUrl, newImageUrl);
-        writeStringToFile(pathToResource, newContent);
+            String digest = getDigest(entry.trueUrl(), continueOnError);
+            content = content.replace(entry.url, digest == null ? entry.trueUrl() : entry.urlWithDigest(digest));
+        }
     }
 
-    private String findVersionEntry(String mainResourceContent) {
-        String lineWithImage = stream(mainResourceContent.split("\n"))
+    private ParsedEntry findEntry(String content) {
+        String lineWithImage = stream(content.split("\n"))
                 .filter(line -> line.contains("IMAGE_DIGEST"))
                 .findFirst()
                 .orElse(null);
         if (lineWithImage == null) return null;
 
-        Matcher matcher = compile(".*(<IMAGE_DIGEST:.*>).*").matcher(lineWithImage);
-        if (!matcher.matches()) return null;
-        return matcher.group(1);
-    }
-
-    private String findVersion(String versionEntry) {
-        Matcher matcher = compile("<IMAGE_DIGEST:(.*)>").matcher(versionEntry);
+        Matcher matcher = compile(".*\\s+(.*(<IMAGE_DIGEST:(.*)>)).*").matcher(lineWithImage);
         if (!matcher.matches()) return null;
-        return matcher.group(1);
-    }
 
-    private String imageUrl(YamlObject resourceYaml) {
-        return resourceYaml
-                .<List<Object>>get("spec.template.spec.containers")
-                .stream()
-                .map(YamlObject::yaml)
-                .map(container -> container.<String>get("image"))
-                .findFirst()
-                .orElseThrow(() -> new OSDFException("Container not found"));
+        return new ParsedEntry(matcher.group(1), matcher.group(2), matcher.group(3));
     }
 
     private String getDigest(String imageUrl, boolean continueOnError) {
@@ -98,4 +63,20 @@ private String getDigest(String imageUrl, boolean continueOnError) {
             throw new OSDFException("Couldn't get digest for " + imageUrl + ": " + e.getMessage());
         }
     }
+
+    @RequiredArgsConstructor
+    static class ParsedEntry {
+        private final String url;
+        private final String versionEntry;
+        private final String version;
+
+        public String trueUrl() {
+            return url.replace(versionEntry, version);
+        }
+
+        public String urlWithDigest(String digest) {
+            return url.replace(":" + versionEntry, "@" + digest);
+        }
+    }
+
 }
diff --git a/src/main/java/io/osdf/core/application/plain/PlainAppDescription.java b/src/main/java/io/osdf/core/application/plain/PlainAppDescription.java
@@ -0,0 +1,22 @@
+package io.osdf.core.application.plain;
+
+import io.osdf.core.application.core.files.ApplicationFiles;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import static io.osdf.actions.management.deploy.smart.hash.ResourcesHashComputer.resourcesHashComputer;
+
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class PlainAppDescription {
+    private String configHash;
+
+    public static PlainAppDescription from(ApplicationFiles files) {
+        String hash = resourcesHashComputer().currentHash(files);
+        return new PlainAppDescription(hash);
+    }
+}
diff --git a/src/main/java/io/osdf/core/application/plain/PlainApplication.java b/src/main/java/io/osdf/core/application/plain/PlainApplication.java
@@ -1,6 +1,8 @@
 package io.osdf.core.application.plain;
 
+import io.osdf.common.exceptions.OSDFException;
 import io.osdf.core.application.core.AbstractApplication;
+import io.osdf.core.application.core.Application;
 import io.osdf.core.application.core.description.CoreDescription;
 import io.osdf.core.application.core.files.ApplicationFiles;
 import io.osdf.core.connection.cli.ClusterCli;
@@ -20,9 +22,15 @@ public static PlainApplication plainApplication(ApplicationFiles files, ClusterC
         return new PlainApplication(files, cli);
     }
 
+    public static PlainApplication plainApplication(Application app) {
+        if (!(app instanceof PlainApplication)) throw new OSDFException(app.name() + " is not a plain app");
+        return (PlainApplication) app;
+    }
+
     public void uploadDescription() {
         configMapLoader(cli).upload(super.descriptionConfigMapName(), of(
-                "core", CoreDescription.from(super.files())
+                "core", CoreDescription.from(super.files()),
+                "plain", PlainAppDescription.from(super.files())
         ));
     }
 }
diff --git a/src/test/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputerTest.java b/src/test/java/io/osdf/actions/management/deploy/smart/hash/ResourcesHashComputerTest.java
@@ -31,4 +31,14 @@ void skipIfHashInsertionIsNotNeeded() throws IOException {
         String contentAfterInsertion = readString(files.getPath("resources/deployment.yaml"));
         assertEquals(originalContent, contentAfterInsertion);
     }
+
+    @Test
+    void replaceAll() throws IOException {
+        ApplicationFiles files = applicationFilesFor("plain-app");
+
+        resourcesHashComputer().insertIn(files);
+
+        assertFalse(readString(files.getPath("resources/resource1.yaml")).contains("<CONFIG_HASH>"));
+        assertFalse(readString(files.getPath("resources/resource2.yaml")).contains("<CONFIG_HASH>"));
+    }
 }
diff --git a/src/test/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacerTest.java b/src/test/java/io/osdf/actions/management/deploy/smart/image/ImageVersionReplacerTest.java
@@ -45,6 +45,17 @@ void digestNotAvailable() {
         assertThrows(OSDFException.class, () -> replacer.replaceFor(files));
     }
 
+    @Test
+    void replaceAll() throws IOException {
+        ImageVersionReplacer replacer = imageVersionReplacer(digestGetter());
+        ApplicationFiles files = applicationFilesFor("plain-app");
+
+        replacer.replaceFor(files);
+
+        assertTrue(readString(files.getPath("resources/resource1.yaml")).contains("@sha256:digest"));
+        assertTrue(readString(files.getPath("resources/resource2.yaml")).contains("@sha256:digest"));
+    }
+
     private DigestGetter digestGetter() {
         DigestGetter digestGetter = mock(DigestGetter.class);
         when(digestGetter.get(anyString())).thenReturn("sha256:digest");

diff --git a/src/test/resources/components/plain-app/deploy.yaml b/src/test/resources/components/plain-app/deploy.yaml
@@ -0,0 +1,5 @@
+app:
+  version: latest
+
+config:
+  version: master
diff --git a/src/test/resources/components/plain-app/resources/resource1.yaml b/src/test/resources/components/plain-app/resources/resource1.yaml
@@ -0,0 +1,9 @@
+kind: FakeResource
+metadata:
+  name: fake
+  labels:
+    configHash: <CONFIG_HASH>
+body:
+  image: url/image:<IMAGE_DIGEST:version>
+  sameImage: url/image:<IMAGE_DIGEST:version>
+  image2: url/image2:<IMAGE_DIGEST:version>
diff --git a/src/test/resources/components/plain-app/resources/resource2.yaml b/src/test/resources/components/plain-app/resources/resource2.yaml
@@ -0,0 +1,6 @@
+kind: FakeResource
+metadata:
+  name: fake
+body:
+  randomValue: <CONFIG_HASH>
+  image: url/image:<IMAGE_DIGEST:version>