Upgrades 2024 11 + testsuite refactoring (#150)

Co-authored-by: Ondrej Zizka <ondrej.zizka@nexxiot.com>

pom.xml

@@ -19,10 +19,10 @@
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <version.kotlin>1.9.22</version.kotlin>
+        <version.kotlin>2.1.0</version.kotlin>
         <version.kotlin-dokka>1.9.10</version.kotlin-dokka>
         <version.jdk>11</version.jdk>
-        <version.jackson>2.16.1</version.jackson>
+        <version.jackson>2.18.2</version.jackson>
         <version.surefire>3.1.2</version.surefire>
 
         <jvm.args.debug /> <!-- Used by the -Pdebug profile. -->
@@ -71,10 +71,12 @@
                 <groupId>org.apache.maven.plugins</groupId><artifactId>maven-surefire-plugin</artifactId><version>${version.surefire}</version>
                 <configuration>
                     <trimStackTrace>false</trimStackTrace>
+                    <workingDirectory>${project.build.directory}/workdir</workingDirectory>
 
                     <systemPropertyVariables>
-                        <test.data.dir>${project.build.testSourceDirectory}/../data</test.data.dir>
                         <target.dir>${project.build.directory}</target.dir>
+                        <test.data.dir>${project.build.testSourceDirectory}/../data</test.data.dir>
+                        <test.output.dir>${project.build.directory}/testOutput</test.output.dir>
                         <textdb.allow_full_path>true</textdb.allow_full_path>
                     </systemPropertyVariables>
                 </configuration>
@@ -85,9 +87,10 @@
                 <configuration>
                     <trimStackTrace>false</trimStackTrace>
                     <systemPropertyVariables>
+                        <target.dir>${project.build.directory}</target.dir>
                         <test.cruncher.runCmd>${testRunCmd}</test.cruncher.runCmd>
                         <test.data.dir>${project.build.testSourceDirectory}/../data</test.data.dir>
-                        <target.dir>${project.build.directory}</target.dir>
+                        <test.output.dir>${project.build.directory}/testOutput</test.output.dir>
                         <textdb.allow_full_path>true</textdb.allow_full_path>
                     </systemPropertyVariables>
                 </configuration>
@@ -264,10 +267,10 @@
         <dependency><groupId>org.jetbrains.kotlin</groupId><artifactId>kotlin-stdlib-jdk8</artifactId><version>${version.kotlin}</version></dependency>
         <dependency><groupId>org.jetbrains.kotlin</groupId><artifactId>kotlin-reflect</artifactId><version>${version.kotlin}</version></dependency>
 
-        <dependency><groupId>org.hsqldb</groupId><artifactId>hsqldb</artifactId><version>2.7.2</version></dependency>
+        <dependency><groupId>org.hsqldb</groupId><artifactId>hsqldb</artifactId><version>2.7.4</version></dependency>
 
-        <dependency><groupId>org.apache.commons</groupId><artifactId>commons-lang3</artifactId><version>3.12.0</version></dependency>
-        <dependency><groupId>commons-io</groupId><artifactId>commons-io</artifactId><version>2.11.0</version></dependency>
+        <dependency><groupId>org.apache.commons</groupId><artifactId>commons-lang3</artifactId><version>3.17.0</version></dependency>
+        <dependency><groupId>commons-io</groupId><artifactId>commons-io</artifactId><version>2.18.0</version></dependency>
 
         <!-- Just for the JSON encoding. 355 kB.
         <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> <version>1.11</version> </dependency>
@@ -289,13 +292,13 @@
 
         <!-- Logging -->
         <dependency><groupId>org.slf4j</groupId><artifactId>slf4j-api</artifactId><version>2.0.9</version></dependency>
-        <dependency><groupId>ch.qos.logback</groupId><artifactId>logback-classic</artifactId><version>1.4.14</version></dependency>
-        <dependency><groupId>ch.qos.logback</groupId><artifactId>logback-core</artifactId><version>1.4.14</version></dependency>
+        <dependency><groupId>ch.qos.logback</groupId><artifactId>logback-classic</artifactId><version>1.5.12</version></dependency>
+        <dependency><groupId>ch.qos.logback</groupId><artifactId>logback-core</artifactId><version>1.5.12</version></dependency>
 
         <!-- Test deps -->
-        <dependency><groupId>org.junit.jupiter</groupId><artifactId>junit-jupiter</artifactId><version>5.10.1</version><scope>test</scope></dependency>
-        <dependency><groupId>org.assertj</groupId><artifactId>assertj-core</artifactId><version>3.25.2</version><scope>test</scope></dependency>
-        <dependency><groupId>org.apache.commons</groupId><artifactId>commons-csv</artifactId><version>1.10.0</version><scope>test</scope></dependency>
+        <dependency><groupId>org.junit.jupiter</groupId><artifactId>junit-jupiter</artifactId><version>5.11.3</version><scope>test</scope></dependency>
+        <dependency><groupId>org.assertj</groupId><artifactId>assertj-core</artifactId><version>3.26.3</version><scope>test</scope></dependency>
+        <dependency><groupId>org.apache.commons</groupId><artifactId>commons-csv</artifactId><version>1.12.0</version><scope>test</scope></dependency>
         <dependency><groupId>org.jetbrains.kotlin</groupId><artifactId>kotlin-test</artifactId><version>${version.kotlin}</version><scope>test</scope></dependency>
 
     </dependencies>

src/main/kotlin/cz/dynawest/csvcruncher/HsqlDbHelper.kt

@@ -98,7 +98,7 @@ class HsqlDbHelper(val jdbcConn: Connection) {
         }
         catch (ex: SQLSyntaxErrorException) {
             if (ex.message!!.contains("object not found:")) {
-                throw throwHintForObjectNotFound(ex, this)
+                throw throwHintForObjectNotFound(ex, this, sql)
             }
             throw CsvCruncherException("""
                 |    Seems your SQL contains errors:

src/main/kotlin/cz/dynawest/csvcruncher/app/OptionsEnums.kt

@@ -2,7 +2,6 @@ package cz.dynawest.csvcruncher.app
 
 import org.apache.commons.lang3.EnumUtils
 import java.util.*
-import java.util.function.Function
 import java.util.stream.Collectors
 
 class OptionsEnums {
@@ -56,10 +55,9 @@ class OptionsEnums {
         companion object {
             const val PARAM_NAME = "combineInputs"
             val optionValues: List<String>
-                get() = EnumUtils.getEnumList(CombineInputFiles::class.java).stream()
-                    .map(Function<CombineInputFiles, String> { it.optionValue })
-                    .filter { obj: String? -> Objects.nonNull(obj) }
-                    .collect(Collectors.toList())
+                get() = EnumUtils.getEnumList(CombineInputFiles::class.java)
+                    .map { it.optionValue }
+                    .filterNotNull()
         }
     }
 

src/main/kotlin/cz/dynawest/csvcruncher/converters/json/JsonFileFlattener.kt

@@ -62,7 +62,7 @@ class JsonFileFlattener : FileTabularizer {
 
             // Expect an array of objects -> rows
             // TODO: Or expect map of objects -> then the property name is a first column, and the objects props the further columns
-            val locationBefore = jsonParser.currentLocation
+            val locationBefore = jsonParser.currentLocation()
             val nextToken = jsonParser.nextToken()
             if (nextToken !== JsonToken.START_ARRAY) {
                 throw ItemsArraySproutNotFound("Items JSON Array not found after traversing over path '$sproutPath', found: $nextToken at $locationBefore")
@@ -79,11 +79,11 @@ class JsonFileFlattener : FileTabularizer {
         for (nextStep in itemsArrayPath) {
             val nextToken = jsonParser.nextToken()
             if (nextToken != JsonToken.START_OBJECT)
-                throw ItemsArraySproutNotFound(itemsArrayPath, jsonParser.currentLocation)
+                throw ItemsArraySproutNotFound(itemsArrayPath, jsonParser.currentLocation())
 
             do {
                 val nextFieldName = jsonParser.nextFieldName()
-                    ?: throw ItemsArraySproutNotFound(itemsArrayPath, jsonParser.currentLocation)
+                    ?: throw ItemsArraySproutNotFound(itemsArrayPath, jsonParser.currentLocation())
 
                 if (nextFieldName != nextStep.name) {
                     jsonParser.skipChildren()

src/main/kotlin/cz/dynawest/csvcruncher/util/FilesUtils.kt

@@ -428,7 +428,7 @@ object FilesUtils {
         do {
             if (!lineIterator.hasNext())
                 throw IllegalStateException("No first line with columns definition (format: [# ] [\"]<colName>[\"] [, ...]) in: " + file.path)
-            line = lineIterator.nextLine().trim { it <= ' ' }
+            line = lineIterator.next().trim { it <= ' ' }
         } while (line!!.startsWith(CSV_COMMENT_PREFIX))
 
         line = StringUtils.stripStart(line, "#")

src/main/kotlin/cz/dynawest/csvcruncher/util/HsqlDbTableCreator.kt

@@ -183,7 +183,7 @@ class HsqlDbTableCreator(private val hsqlDbHelper: HsqlDbHelper) {
 
     private fun setUpTableIndexes(tableName: String, indexedColumns: List<String>) {
         for (col in indexedColumns) {
-            val rnd = RandomStringUtils.randomAlphanumeric(3)
+            val rnd = RandomStringUtils.insecure().nextAlphanumeric(3)
             val sql = "CREATE INDEX ${quote("idx_${tableName}_$rnd")} ON ${quote(tableName)} (${quote(col)})"
             log.debug("Creating index, SQL: $sql")
             hsqlDbHelper.executeSql(sql, "Failed to create index: ")

src/main/kotlin/cz/dynawest/csvcruncher/util/HsqldbErrorHandling.kt

@@ -10,7 +10,7 @@ object HsqldbErrorHandling {
     /**
      * Analyzes the exception against the given DB connection and rethrows an exception with a message containing the available objects as a hint.
      */
-    fun throwHintForObjectNotFound(ex: SQLSyntaxErrorException, helper: HsqlDbHelper): CsvCruncherException {
+    fun throwHintForObjectNotFound(ex: SQLSyntaxErrorException, helper: HsqlDbHelper, sql: String): CsvCruncherException {
         val notFoundIsColumn = analyzeWhatWasNotFound(ex.message!!)
         val tableNames = helper.formatListOfAvailableTables(notFoundIsColumn)
         val hintMsg = if (notFoundIsColumn) """
@@ -23,6 +23,8 @@ object HsqldbErrorHandling {
             |  Looks like you are referring to a table that was not created.
             |  This could mean that you have a typo in the input file name,
             |  or maybe you use --combineInputs but try to use the original inputs.
+            |  Or it is this known bug: https://github.com/OndraZizka/csv-cruncher/issues/149
+            |  SQL: $sql
             |  These tables are actually available:
             |
             """.trimMargin()

src/test/kotlin/cz/dynawest/csvcruncher/CsvCruncherTestUtils.kt

@@ -1,27 +1,30 @@
 package cz.dynawest.csvcruncher
 
+import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testOutputDir
 import cz.dynawest.csvcruncher.app.App
 import org.apache.commons.csv.CSVFormat
 import org.apache.commons.csv.CSVRecord
 import org.apache.commons.lang3.StringUtils
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.TestInfo
 import java.io.*
 import java.nio.file.Path
 import java.nio.file.Paths
 
 object CsvCruncherTestUtils {
-    /**
-     * @return Path to the default test data dir.
-     */
-    val testDataDir: Path
-        get() = Paths.get(System.getProperty("user.dir")).resolve("src/test/data/")
+    //val testDataDir: Path get() = Paths.get(System.getProperty("user.dir")).resolve("src/test/data/")
 
-    /**
-     * @return Path to the default test output dir.
-     */
+    /** @return Path to the default test data dir. */
+    val testDataDir = Path.of(System.getProperty("test.data.dir"))
+        //.also { it.toFile().mkdirs() }
+        .also { it.toFile().isFile && throw Exception("SysProp test.data.dir needs to point to a directory with the tests data. Was: $it") }
+
+    /** @return Path to the default test output dir. */
     val testOutputDir: Path
-        get() = Paths.get(System.getProperty("user.dir")).resolve("target/testResults/")
+        get() = Paths.get(System.getProperty("test.output.dir"))
+            //.also { it.toFile().mkdirs() }
+            .also { it.toFile().isFile && throw Exception("SysProp test.output.dir needs to point to a directory, was: $it") }
 
     /**
      * Runs CSV Cruncher with the guven command, which is | separated arguments.
@@ -106,4 +109,9 @@ object CsvCruncherTestUtils {
             throw RuntimeException(e)
         }
     }
-}
+}
+
+
+fun TestInfo.defaultCsvOutputPath() = "$testOutputDir/${testClass.get().simpleName}_${testMethod.get().name}.csv".let { Path.of(it) }
+
+fun TestInfo.defaultJsonOutputPath() = "$testOutputDir/${testClass.get().simpleName}_${testMethod.get().name}.json".let { Path.of(it) }

src/test/kotlin/cz/dynawest/csvcruncher/app/OptionsParsingTest.kt

@@ -1,19 +1,23 @@
 package cz.dynawest.csvcruncher.app
 
+import cz.dynawest.csvcruncher.CsvCruncherTestUtils
+import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir
+import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testOutputDir
 import org.assertj.core.api.Assertions
 import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.TestInfo
 import java.nio.file.Path
 
 class OptionsParsingTest {
 
-    @Test fun testOptionsParsing() {
+    @Test fun testOptionsParsing(testInfo: TestInfo) {
 
         val command = "--json | --combineInputs" +
-            " |  --exclude=.*/LOAD.*\\.csv" +
-            " |  -in  | src/test/data/sample-collab/apollo_recording_group/" +
-            " |  -out | target/testResults/apollo_recording_group.csv" +
-            " |  -sql | SELECT * FROM apollo_recording_group" +
-            " |  -initSql | foo/init1.sql | foo/init2.sql"
+            " | --exclude=.*/LOAD.*\\.csv" +
+            " | -in  | ${CsvCruncherTestUtils.testDataDir}/sample-collab/apollo_recording_group/" +
+            " | -out | $testOutputDir/apollo_recording_group.csv" +
+            " | -sql | SELECT * FROM apollo_recording_group" +
+            " | -initSql | foo/init1.sql | foo/init2.sql"
 
         val arguments = command.split("|").map { it.trim() }
         val options = OptionsParser.parseArgs(arguments.toTypedArray())!!
@@ -27,10 +31,10 @@ class OptionsParsingTest {
 
         // Per import/export
         Assertions.assertThat(options.importArguments).size().isEqualTo(1)
-        Assertions.assertThat(options.importArguments).element(0).extracting{it.path}.isEqualTo(Path.of("src/test/data/sample-collab/apollo_recording_group/"))
+        Assertions.assertThat(options.importArguments).element(0).extracting{it.path}.isEqualTo(Path.of("$testDataDir/sample-collab/apollo_recording_group/"))
 
         Assertions.assertThat(options.exportArguments).size().isEqualTo(1)
-        Assertions.assertThat(options.exportArguments).element(0).extracting{it.path}.isEqualTo(Path.of("target/testResults/apollo_recording_group.csv"))
+        Assertions.assertThat(options.exportArguments).element(0).extracting{it.path}.isEqualTo(Path.of("$testOutputDir/apollo_recording_group.csv"))
         Assertions.assertThat(options.exportArguments).element(0).extracting{it.sqlQuery}.isEqualTo("SELECT * FROM apollo_recording_group")
 
         Assertions.assertThat(options.initSqlArguments).size().isEqualTo(2)

src/test/kotlin/cz/dynawest/csvcruncher/it/BasicIntegTest.kt

@@ -1,6 +1,7 @@
 package cz.dynawest.csvcruncher.it
 
 import cz.dynawest.csvcruncher.CsvCruncherTestUtils
+import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir
 import cz.dynawest.csvcruncher.defaultCsvOutputPath
 import cz.dynawest.csvcruncher.defaultJsonOutputPath
 import org.assertj.core.api.Assertions.assertThat
@@ -26,7 +27,7 @@ class BasicIntegTest {
 
     @Test
     fun singleImportSingleExportWithJson(testInfo: TestInfo) {
-        val inPath = Paths.get("src/test/data/eapBuilds.csv")
+        val inPath = Paths.get("$testDataDir/eapBuilds.csv")
         val outputPath = testInfo.defaultCsvOutputPath()
 
         val command =
@@ -41,7 +42,7 @@ class BasicIntegTest {
 
     @Test
     fun singleImportSingleExport_defaultSql(testInfo: TestInfo) {
-        val inPath = Paths.get("src/test/data/eapBuilds.csv")
+        val inPath = Paths.get("$testDataDir/eapBuilds.csv")
         val outputPath = testInfo.defaultCsvOutputPath()
 
         val command = """-in  | $inPath | -out | $outputPath"""
@@ -52,7 +53,7 @@ class BasicIntegTest {
 
     @Test
     fun singleImportSingleExport_jsonFileExtension(testInfo: TestInfo) {
-        val inPath = Paths.get("src/test/data/eapBuilds.csv")
+        val inPath = Paths.get("$testDataDir/eapBuilds.csv")
         val outputPath = testInfo.defaultJsonOutputPath()
 
         val command = """-in  | $inPath | -out | $outputPath"""
@@ -63,10 +64,10 @@ class BasicIntegTest {
 
     @Test
     fun test_initSqlScript(testInfo: TestInfo) {
-        val inPath = Paths.get("src/test/data/eapBuilds.csv")
+        val inPath = Paths.get("$testDataDir/eapBuilds.csv")
         val outputPath = testInfo.defaultCsvOutputPath()
 
-        val initSqlFile = Paths.get("src/test/data/init.sql")
+        val initSqlFile = Paths.get("$testDataDir/init.sql")
         val command =
             """-initSql | $initSqlFile | -in | $inPath | -out | $outputPath | -sql | SELECT jobName FROM eapBuilds LIMIT 1"""
         CsvCruncherTestUtils.runCruncherWithArguments(command)
@@ -76,7 +77,7 @@ class BasicIntegTest {
 
     @Test
     fun realData_json_redditAll(testInfo: TestInfo) {
-        val inPath = Paths.get("src/test/data/json/redditAll.json")
+        val inPath = Paths.get("$testDataDir/json/redditAll.json")
         val outputPath = testInfo.defaultCsvOutputPath()
 
         val command = """-in | $inPath | -itemsAt | /data/children | -out | $outputPath | -sql | SELECT * FROM REDDITALL_JSON"""
@@ -87,7 +88,7 @@ class BasicIntegTest {
 
     @Test
     fun test_basic_json_types(testInfo: TestInfo) {
-        val inPath = Paths.get("src/test/data/json/jsonTypes_all.json")
+        val inPath = Paths.get("$testDataDir/json/jsonTypes_all.json")
         val outputPath = testInfo.defaultCsvOutputPath()
 
         val command = """-in | $inPath | -itemsAt | / | -out | $outputPath | -sql | SELECT * FROM jsonTypes_all_json"""
@@ -98,7 +99,7 @@ class BasicIntegTest {
 
     @Test
     fun test_basic_json_noParentDirOfOutput(testInfo: TestInfo) {
-        val inPath = Paths.get("src/test/data/json/redditAll.json")
+        val inPath = Paths.get("$testDataDir/json/redditAll.json")
         val outputPath = testInfo.defaultJsonOutputPath()
         Path.of(outputPath.pathString + ".csv").deleteIfExists() // Temp file can remain after debugging
 
@@ -108,7 +109,7 @@ class BasicIntegTest {
 
     @Test @Disabled("Currently we just overwrite to simplify experimenting. Maybe let's have --noOverwrite?")
     fun testDoesNotOverwrite(testInfo: TestInfo) {
-        val inPath = Paths.get("src/test/data/json/redditAll.json")
+        val inPath = Paths.get("$testDataDir/json/redditAll.json")
         val outputPath = testInfo.defaultCsvOutputPath()
 
         val command = """-in | $inPath | -itemsAt | /data/children | -out | $outputPath | -sql | SELECT * FROM REDDITALL_JSON"""