From 7f4f1b638cb79a67f7fc641dd71e78d771c8f323 Mon Sep 17 00:00:00 2001 From: James W <26191568+jgwoolley@users.noreply.github.com> Date: Wed, 29 Jan 2025 18:56:57 -0500 Subject: [PATCH] Implemented fix for some null pointers --- .gitignore | 1 + pom.xml | 10 +++++ .../java/com/yelloowstone/nf2t/cli/App.java | 38 ++++++++++++++----- .../yelloowstone/nf2t/cli/FlowFileResult.java | 12 +++--- .../nf2t/cli/FlowFileStreamResult.java | 17 +++++++-- .../com/yelloowstone/nf2t/cli/AppTest.java | 4 +- 6 files changed, 60 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index fc22153..19b5508 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.flowfilev3 *.adoc .vscode .classpath diff --git a/pom.xml b/pom.xml index d299462..3e2d79e 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,16 @@ + + org.slf4j + slf4j-nop + 2.0.16 + + + org.apache.nifi + nifi-expression-language + 1.23.2 + junit junit diff --git a/src/main/java/com/yelloowstone/nf2t/cli/App.java b/src/main/java/com/yelloowstone/nf2t/cli/App.java index 233c660..b6264b2 100644 --- a/src/main/java/com/yelloowstone/nf2t/cli/App.java +++ b/src/main/java/com/yelloowstone/nf2t/cli/App.java @@ -13,6 +13,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.UUID; import java.util.concurrent.Callable; import java.util.stream.Stream; @@ -22,6 +23,9 @@ import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.nifi.attribute.expression.language.PreparedQuery; +import org.apache.nifi.attribute.expression.language.Query; +import org.apache.nifi.attribute.expression.language.StandardEvaluationContext; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.util.FlowFilePackager; import org.apache.nifi.util.FlowFileUnpackager; @@ -76,7 +80,7 @@ private void unpackageFlowFileInputStream(final FlowFileStreamResult result, fin } if (contentPath != null && result.isUuidFilenames() && flowFileResult != null) { - String filename = flowFileResult.getAttributes().get(CoreAttributes.FILENAME.key()); + String filename = flowFileResult.getRawAttributeExpressions().get(CoreAttributes.FILENAME.key()); if (filename != null) { Path newContentPath = contentPath.getParent().resolve(filename); Files.move(contentPath, newContentPath); @@ -180,10 +184,8 @@ public Integer unpackageFlowFileStream( + FlowFileStreamResult.OUTPUTPATH_UNPACKAGE_DESCRIPTION, required = false) final String outputOption, @Option(names = { "-u", "--uuid" }, description = FlowFileStreamResult.UUID_DESCRIPTION, defaultValue = "true") final boolean uuidFilenames, - @Option(names = {"-r", "--results"}, description=FlowFileStreamResult.RESULTS_PATH_DESCRIPTION) final String resultsPath, - @Option(names = {"-a", "--attribute"}, description=FlowFileStreamResult.DEFAULT_ATTRIBUTES_DESCRIPTION, required=false) final Map attributes, - @Option(names = {"-k", "--keep-attributes"}, description=FlowFileStreamResult.KEEP_ATTRIBUTES_DESCRIPTION, defaultValue="true", required=false) final boolean keepAttributes) { - final FlowFileStreamResult result = createResult(version, extension, uuidFilenames, inputOption, outputOption, resultsPath, attributes, keepAttributes); + @Option(names = {"-r", "--results"}, description=FlowFileStreamResult.RESULTS_PATH_DESCRIPTION) final String resultsPath) { + final FlowFileStreamResult result = createResult(version, extension, uuidFilenames, inputOption, outputOption, resultsPath, new HashMap<>(), true); // Unpack Frequently Used Variables final Path inputPath = result.getInputPath(); @@ -249,8 +251,10 @@ public Integer packageFlowFileStream( + FlowFileStreamResult.INPUTPATH_PACKAGE_DESCRIPTION, required = true) final String inputOption, @Option(names = { "-o", "--out" }, description = "The output path." + FlowFileStreamResult.OUTPUTPATH_PACKAGE_DESCRIPTION, required = true) final String outputOption, - @Option(names = {"-r", "--results"}, description=FlowFileStreamResult.RESULTS_PATH_DESCRIPTION) final String resultsPath) { - final FlowFileStreamResult result = createResult(version, extension, true, inputOption, outputOption, resultsPath, new HashMap<>(), true); + @Option(names = {"-r", "--results"}, description=FlowFileStreamResult.RESULTS_PATH_DESCRIPTION) final String resultsPath, + @Option(names = {"-a", "--attribute"}, description=FlowFileStreamResult.DEFAULT_ATTRIBUTES_DESCRIPTION, required=false) final Map rawAttributeExpressions, + @Option(names = {"-k", "--keep-attributes"}, description=FlowFileStreamResult.KEEP_ATTRIBUTES_DESCRIPTION, defaultValue="true", required=false) final boolean keepAttributes) { + final FlowFileStreamResult result = createResult(version, extension, true, inputOption, outputOption, resultsPath, rawAttributeExpressions == null ? new HashMap<>(): rawAttributeExpressions, keepAttributes); // Unpack Frequently Used Variables final Path inputPath = result.getInputPath(); @@ -302,6 +306,12 @@ public Integer packageFlowFileStream( final FlowFilePackager packager = packageVersion.getPackager(); + final Map attributeExpressions = new HashMap<>(); + + for(final Entry attribute: result.getDefaultAttributes().entrySet()) { + attributeExpressions.put(attribute.getKey(), Query.prepare(attribute.getValue())); + } + try (OutputStream outputStream = Files.newOutputStream(outputPath)) { for (Path contentPath : contentPaths) { final SourceFile content = SourceFile.fromPath(null, outputPath); @@ -309,12 +319,18 @@ public Integer packageFlowFileStream( try { final long contentSize = Files.size(contentPath); final Map defaultAttributes = generateDefaultAttributes(contentPath, contentSize); + defaultAttributes.putAll(result.getDefaultAttributes()); final Map attributes = new HashMap<>(); if(result.keepAttributes) { attributes.putAll(defaultAttributes); } + final StandardEvaluationContext evaluationContext = new StandardEvaluationContext(defaultAttributes); + for(final Entry attribute: attributeExpressions.entrySet()) { + attributes.put(attribute.getKey(), attribute.getValue().evaluateExpressions(evaluationContext, null)); + } + try (InputStream inputStream = Files.newInputStream(contentPath)) { packager.packageFlowFile(inputStream, outputStream, attributes, contentSize); @@ -378,7 +394,7 @@ public boolean printResult(final FlowFileStreamResult result) { } public FlowFileStreamResult createResult(final int version, String extension, final boolean uuidFilenames, - final String inputOption, String outputOption, final String resultsOption, final Map defaultAttributes, boolean keepAttributes) { + final String inputOption, String outputOption, final String resultsOption, final Map attributeExpressions, boolean keepAttributes) { final Path inputPath = Paths.get(inputOption == null ? "." : inputOption); final Path outputPath = outputOption == null || outputOption.length() <= 0 ? null : Paths.get(outputOption); Path resultsPath = resultsOption == null ? null : Paths.get(resultsOption); @@ -392,14 +408,16 @@ public FlowFileStreamResult createResult(final int version, String extension, fi } long unixTime = System.currentTimeMillis() / 1000L; - return new FlowFileStreamResult(version, extension, uuidFilenames, inputPath, outputPath, resultsPath, unixTime, defaultAttributes, keepAttributes); + return new FlowFileStreamResult(version, extension, uuidFilenames, inputPath, outputPath, resultsPath, unixTime, attributeExpressions, keepAttributes); } public static Map generateDefaultAttributes(final Path path, final long contentSize) throws IOException { final Map attributes = new HashMap<>(); attributes.put(CoreAttributes.FILENAME.key(), path.getFileName().toString()); - attributes.put(CoreAttributes.PATH.key(), path.getParent().toString()); + if(path.getParent() != null) { + attributes.put(CoreAttributes.PATH.key(), path.getParent().toString()); + } attributes.put(CoreAttributes.ABSOLUTE_PATH.key(), path.toString()); attributes.put(FILE_SIZE_ATTRIBUTE, Long.toString(contentSize)); diff --git a/src/main/java/com/yelloowstone/nf2t/cli/FlowFileResult.java b/src/main/java/com/yelloowstone/nf2t/cli/FlowFileResult.java index 5cf810c..1b57bb3 100644 --- a/src/main/java/com/yelloowstone/nf2t/cli/FlowFileResult.java +++ b/src/main/java/com/yelloowstone/nf2t/cli/FlowFileResult.java @@ -16,19 +16,19 @@ public class FlowFileResult { @JsonProperty("contentPath") private SourceFile contentPath; - @JsonPropertyDescription("Represents the FlowFile Attributes.") + @JsonPropertyDescription("Represents the FlowFile attribute key, and value. The value is an FlowFile Attribute Expression, and can resolve variables from other attributes. Expressions should not reference other attributes that also contain expressions.") @JsonProperty("attributes") - private final Map attributes; + private final Map rawAttributeExpressions; @JsonPropertyDescription("Represents the size of the FlowFile Content.") @JsonProperty("contentSize") private final long contentSize; @JsonCreator - public FlowFileResult(@JsonProperty("flowFilePath") final SourceFile flowFilePath, @JsonProperty("contentPath") final SourceFile contentPath, @JsonProperty("attributes") final Map attributes, @JsonProperty("contentSize") final long size) { + public FlowFileResult(@JsonProperty("flowFilePath") final SourceFile flowFilePath, @JsonProperty("contentPath") final SourceFile contentPath, @JsonProperty("attributes") final Map rawAttributeExpressions, @JsonProperty("contentSize") final long size) { this.flowFilePath = flowFilePath; this.contentPath = contentPath; - this.attributes = attributes; + this.rawAttributeExpressions = rawAttributeExpressions; this.contentSize = size; } @@ -40,8 +40,8 @@ public SourceFile getContentPath() { return contentPath; } - public Map getAttributes() { - return attributes; + public Map getRawAttributeExpressions() { + return rawAttributeExpressions; } public long getContentSize() { diff --git a/src/main/java/com/yelloowstone/nf2t/cli/FlowFileStreamResult.java b/src/main/java/com/yelloowstone/nf2t/cli/FlowFileStreamResult.java index 714e68f..74f4a2c 100644 --- a/src/main/java/com/yelloowstone/nf2t/cli/FlowFileStreamResult.java +++ b/src/main/java/com/yelloowstone/nf2t/cli/FlowFileStreamResult.java @@ -75,9 +75,9 @@ public class FlowFileStreamResult { @JsonPropertyDescription(DEFAULT_ATTRIBUTES_DESCRIPTION) @JsonProperty("defaultAttributes") - final Map defaultAttributes; + final Map rawAttributeExpressions; - protected static final String KEEP_ATTRIBUTES_DESCRIPTION = "Keep default attributes generated by nf2t-cli."; + protected static final String KEEP_ATTRIBUTES_DESCRIPTION = "Keep default FlowFile attributes generated by nf2t-cli."; @JsonPropertyDescription(KEEP_ATTRIBUTES_DESCRIPTION) @JsonProperty("keepAttributes") @@ -87,7 +87,7 @@ public class FlowFileStreamResult { public FlowFileStreamResult(@JsonProperty("version") final int version, @JsonProperty("extension") String extension, @JsonProperty("uuidFilenames") final boolean uuidFilenames, @JsonProperty("inputPath") final Path inputPath, @JsonProperty("outputPath") final Path outputPath, @JsonProperty("resultsPath") Path resultsPath, - @JsonProperty("unixTime") final long unixTime, @JsonProperty("defaultAttributes") Map defaultAttributes, @JsonProperty("keepAttributes") boolean keepAttributes) { + @JsonProperty("unixTime") final long unixTime, @JsonProperty("defaultAttributes") Map rawAttributeExpressions, @JsonProperty("keepAttributes") boolean keepAttributes) { this.version = version; this.extension = extension; this.uuidFilenames = uuidFilenames; @@ -95,7 +95,7 @@ public FlowFileStreamResult(@JsonProperty("version") final int version, @JsonPro this.outputPath = outputPath; this.resultsPath = resultsPath; this.unixTime = unixTime; - this.defaultAttributes = defaultAttributes; + this.rawAttributeExpressions = rawAttributeExpressions; this.keepAttributes = keepAttributes; } @@ -142,4 +142,13 @@ public List getErrors() { public List getOutputFiles() { return outputFiles; } + + public Map getDefaultAttributes() { + return rawAttributeExpressions; + } + + public boolean isKeepAttributes() { + return keepAttributes; + } + } diff --git a/src/test/java/com/yelloowstone/nf2t/cli/AppTest.java b/src/test/java/com/yelloowstone/nf2t/cli/AppTest.java index e63e786..4af726a 100644 --- a/src/test/java/com/yelloowstone/nf2t/cli/AppTest.java +++ b/src/test/java/com/yelloowstone/nf2t/cli/AppTest.java @@ -39,7 +39,7 @@ public void testPackageFiles() throws Exception { final Path contentPath = environment.getContentPath(); final Path packagedPath = environment.getPackagedPath(); - final String sw = environment.execute(x -> new String[] {"package", "--version", Integer.toString(version), "--in", contentPath.toString(), "--out", packagedPath.toString()}); + final String sw = environment.execute(x -> new String[] {"package", "--version", Integer.toString(version), "--in", contentPath.toString(), "--out", packagedPath.toString(), "--attribute", "key=${foo}$${foo}$$${foo}", "--attribute", "key2=${size}"}); final FlowFileStreamResult result = reader.readValue(sw, FlowFileStreamResult.class); @@ -129,7 +129,7 @@ public void testUnpackageFiles() throws Exception { } final String sw = environment.execute(x -> new String[] {"unpackage", "--version", Integer.toString(version), - "--in", packagedPath.toString(), "--out", unpackagedPath.toString(), "--attribute", "key=value"}); + "--in", packagedPath.toString(), "--out", unpackagedPath.toString()}); System.out.println("\tResult:\n\t" + sw.toString()); final FlowFileStreamResult result = reader.readValue(sw.toString(), FlowFileStreamResult.class);