Merge branch 'develop'

CHANGELOG.md

@@ -1,3 +1,15 @@
+## 3.11.0 - 14 November 2023
+
+- Enhancements
+	- \#340 Add readCsv() for large file/advanced csv processing
+	- \#336 Add parallelization option to queryToCsv()
+
+- Fixes
+  - \#339 csvToQuery() ignores trim setting when reading from file
+
+- Maintenance
+  - \#334 Upgrade excel-streaming-reader to 4.2.0
+
 ## 3.10.0 - 29 September 2023
 
 - Maintenance

ModuleConfig.cfc

@@ -4,7 +4,7 @@ component{
     this.author = "Julian Halliwell";
     this.webURL = "https://github.com/cfsimplicity/spreadsheet-cfml";
     this.description = "CFML Spreadsheet Library";
-    this.version = "3.10.0";
+    this.version = "3.11.0";
     this.autoMapModels = false;
 
     function configure(){

README.md

@@ -142,6 +142,7 @@ You may wish to place the spreadsheet library files in a central location with a
 * [isRowHidden](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/isRowHidden)
 * [isStreamingXmlFormat](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/isStreamingXmlFormat)
 * [isXmlFormat](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/isXmlFormat)
+* [readCsv](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/readCsv)
 * [readLargeFile](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/readLargeFile)
 * [removePrintGridlines](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/removePrintGridlines)
 * [renameSheet](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/renameSheet)

Spreadsheet.cfc

@@ -1,8 +1,8 @@
 component accessors="true"{
 
 	//"static"
-	property name="version" default="3.10.0" setter="false";
-	property name="osgiLibBundleVersion" default="5.2.4.0" setter="false"; //first 3 octets = POI version; increment 4th with other jar updates
+	property name="version" default="3.11.0" setter="false";
+	property name="osgiLibBundleVersion" default="5.2.4.1" setter="false"; //first 3 octets = POI version; increment 4th with other jar updates
 	property name="osgiLibBundleSymbolicName" default="spreadsheet-cfml" setter="false";
 	property name="exceptionType" default="cfsimplicity.spreadsheet" setter="false";
 	//configurable
@@ -166,6 +166,10 @@ component accessors="true"{
 		return getFormatHelper().getCachedCellStyles();
 	}
 
+	public boolean function engineSupportsParallelLoopProcessing(){
+		return ( !this.getIsACF() || ( this.getIsACF() && ( server.coldfusion.productVersion.ListFirst() >= 2021 ) ) );
+	}
+
 	/* MAIN PUBLIC API */
 
 	public Spreadsheet function addAutofilter( required workbook, string cellRange="", numeric row=1 ){
@@ -511,7 +515,7 @@ component accessors="true"{
 			Throw( type=this.getExceptionType() & ".invalidArgumentCombination", message="Invalid argument 'queryColumnTypes'.", detail="When specifying 'queryColumnTypes' as a struct you must also set the 'firstRowIsHeader' argument to true OR provide 'queryColumnNames'" );
 		var format = getCsvHelper().getFormat( arguments.delimiter?:"" );
 		var parsed = csvIsFile?
-			getCsvHelper().parseFromFile( arguments.filepath, format ):
+			getCsvHelper().parseFromFile( arguments.filepath, arguments.trim, format ):
 			getCsvHelper().parseFromString( arguments.csv, arguments.trim, format );
 		var data = parsed.data;
 		var maxColumnCount = parsed.maxColumnCount;
@@ -1092,13 +1096,11 @@ component accessors="true"{
 		return new( sheetName=arguments.sheetName, xmlFormat=true );
 	}
 
-	public string function queryToCsv( required query query, boolean includeHeaderRow=false, string delimiter="," ){
-		var data = getCsvHelper().queryToArrayForCsv( arguments.query, arguments.includeHeaderRow );
+	/* row order is not guaranteed if using more than one thread */
+	public string function queryToCsv( required query query, boolean includeHeaderRow=false, string delimiter=",", numeric threads=1 ){
+		var data = getCsvHelper().queryToArrayForCsv( arguments.query, arguments.includeHeaderRow, arguments.threads );
 		var builder = getStringHelper().newJavaStringBuilder();
-		var csvFormat = getCsvHelper().delimiterIsTab( arguments.delimiter )?
-			createJavaObject( "org.apache.commons.csv.CSVFormat" )[ JavaCast( "string", "TDF" ) ]
-			: createJavaObject( "org.apache.commons.csv.CSVFormat" )[ JavaCast( "string", "EXCEL" ) ]
-				.withDelimiter( JavaCast( "char", arguments.delimiter ) );
+		var csvFormat = getCsvHelper().getFormatForPrinting( arguments.delimiter );
 		createJavaObject( "org.apache.commons.csv.CSVPrinter" )
 			.init( builder, csvFormat )
 			.printRecords( data );
@@ -1188,6 +1190,10 @@ component accessors="true"{
 		return baos.toByteArray();
 	}
 
+	public any function readCsv( required string filepath ){
+		return New objects.ReadCsv( this, arguments.filepath );
+	}
+
 	public any function readLargeFile(
 		required string src
 		,string format="query"

box.json

@@ -1,7 +1,7 @@
 {
     "name" : "Spreadsheet CFML",
     "slug" : "spreadsheet-cfml",
-    "version" : "3.10.0",
+    "version" : "3.11.0",
     "shortDescription" : "CFML spreadsheet library",
     "author" : "Julian Halliwell",
     "location" : "forgeboxStorage",
@@ -28,7 +28,7 @@
         "testbox":"test/testbox/"
     },
     "type" : "modules",
-    "keywords" : [ "cfspreadsheet", "coldfusion", "lucee", "module", "spreadsheet" ],
+    "keywords" : [ "cfspreadsheet", "coldfusion", "lucee", "module", "spreadsheet", "csv" ],
     "private" : "false",
     "ignore" : [ ".github", "build", "src", "test", ".gitattributes", "CHANGELOG.md", "README.md" ]
 }

build/lib-osgi.mf

@@ -2,14 +2,14 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: Spreadsheet CFML
 Bundle-SymbolicName: spreadsheet-cfml
-Bundle-Version: 5.2.4.0
+Bundle-Version: 5.2.4.1
 Bundle-ClassPath: commons-codec-1.16.0.jar,
  commons-collections4-4.4.jar,
  commons-compress-1.24.0.jar,
  commons-csv-1.10.0.jar,
  commons-io-2.13.0.jar,
  commons-math3-3.6.1.jar,
- excel-streaming-reader-4.1.2.jar,
+ excel-streaming-reader-4.2.0.jar,
  log4j-api-2.20.0.jar,
  poi-5.2.4.jar,
  poi-ooxml-5.2.4.jar,

helpers/csv.cfc

@@ -1,13 +1,27 @@
 component extends="base"{
 
+	any function getFormatObject( string type="DEFAULT" ){
+		return library().createJavaObject( "org.apache.commons.csv.CSVFormat" )[ JavaCast( "string", arguments.type ) ];	
+	}
+
 	boolean function delimiterIsTab( required string delimiter ){
 		return ArrayFindNoCase( [ "#Chr( 9 )#", "\t", "tab" ], arguments.delimiter );//CF2016 doesn't support [].FindNoCase( needle )
 	}
 
 	any function getFormat( required string delimiter ){
 		if( arguments.delimiter.Len() )
 			return getCsvFormatForDelimiter( arguments.delimiter );
-		return library().createJavaObject( "org.apache.commons.csv.CSVFormat" )[ JavaCast( "string", "RFC4180" ) ].withIgnoreSurroundingSpaces();
+		var format = getFormatObject( "RFC4180" );
+		return format.builder()
+				.setIgnoreSurroundingSpaces( JavaCast( "boolean", true ) )
+				.build();
+	}
+
+	any function getFormatForPrinting( required string delimiter ){
+		if( delimiterIsTab( arguments.delimiter ) )
+			return getFormatObject( "TDF" );
+		var format = getFormatObject( "EXCEL" );
+		return format.builder().setDelimiter( arguments.delimiter ).build();
 	}
 
 	array function getColumnNames( required boolean firstRowIsHeader, required array data, required numeric maxColumnCount ){
@@ -24,22 +38,26 @@ component extends="base"{
 		return result;
 	}
 
-	array function queryToArrayForCsv( required query query, required boolean includeHeaderRow ){
+	/* row order is not guaranteed if using more than one thread */
+	array function queryToArrayForCsv( required query query, required boolean includeHeaderRow, numeric threads=1 ){
 		var result = [];
 		var columns = getQueryHelper()._QueryColumnArray( arguments.query );
 		if( arguments.includeHeaderRow )
 			result.Append( columns );
+		if( ( arguments.threads > 1 ) && !library().engineSupportsParallelLoopProcessing() )
+			getExceptionHelper().throwParallelOptionNotSupportedException();
+		if( arguments.threads > 1 ){
+			arguments.query.Each(
+				function( row ){
+					result.Append( getQueryRowValues( row, columns ) );
+				}
+				,true
+				,arguments.threads
+			);
+			return result;			
+		}
 		for( var row IN arguments.query ){
-			var rowValues = [];
-			for( var column IN columns ){
-				var cellValue = row[ column ];
-				if( getDateHelper().isDateObject( cellValue ) )
-					cellValue = DateTimeFormat( cellValue, library().getDateFormats().DATETIME );
-				if( IsValid( "integer", cellValue ) )
-					cellValue = JavaCast( "string", cellValue );// prevent CSV writer converting 1 to 1.0
-				rowValues.Append( cellValue );
-			}
-			result.Append( rowValues );
+			result.Append( getQueryRowValues( row, columns ) );
 		}
 		return result;
 	}
@@ -57,21 +75,22 @@ component extends="base"{
 		}
 	}
 
-	struct function parseFromFile( required string path, required any format ){
+	struct function parseFromFile( required string path, required boolean trim, required any format ){
 		getFileHelper()
 			.throwErrorIFfileNotExists( arguments.path )
 			.throwErrorIFnotCsvOrTextFile( arguments.path );
-		return parseFromString( FileRead( arguments.path ), false, arguments.format );
+		return parseFromString( FileRead( arguments.path ), arguments.trim, arguments.format );
 	}
 
 	/* Private */
-
 	private any function getCsvFormatForDelimiter( required string delimiter ){
 		if( delimiterIsTab( arguments.delimiter ) )
-			return library().createJavaObject( "org.apache.commons.csv.CSVFormat" )[ JavaCast( "string", "TDF" ) ];
-		return library().createJavaObject( "org.apache.commons.csv.CSVFormat" )[ JavaCast( "string", "RFC4180" ) ]
-			.withDelimiter( JavaCast( "char", arguments.delimiter ) )
-			.withIgnoreSurroundingSpaces();//stop spaces between fields causing problems with embedded lines
+			return getFormatObject( "TDF" );
+		var format = getFormatObject( "RFC4180" );
+		return format.builder()
+			.setDelimiter( arguments.delimiter )
+			.setIgnoreSurroundingSpaces( JavaCast( "boolean", true ) )//stop spaces between fields causing problems with embedded lines
+			.build();
 	}
 
 	private struct function dataFromParser( required any parser ){
@@ -81,17 +100,26 @@ component extends="base"{
 		};
 		var recordIterator = arguments.parser.iterator();
 		while( recordIterator.hasNext() ){
-			var row = [];
-			var columnNumber = 0;
-			var values = recordIterator.next().toList();
-			for( var value in values ){
-				columnNumber++;
-				result.maxColumnCount = Max( result.maxColumnCount, columnNumber );
-				row.Append( value );
-			}
-			result.data.Append( row );
+			var record = recordIterator.next();
+			result.maxColumnCount = Max( result.maxColumnCount, record.size() );
+			//ACF can't handle native arrays in QueryNew()
+			var values = ( library().getIsACF() )? record.toList(): record.values();
+			ArrayAppend( result.data, values );
 		}
 		return result;
 	}
 
+	private array function getQueryRowValues( required row, required array columns ){
+		var rowValues = [];
+		for( var column IN arguments.columns ){
+			var cellValue = arguments.row[ column ];
+			if( getDateHelper().isDateObject( cellValue ) )
+				cellValue = DateTimeFormat( cellValue, library().getDateFormats().DATETIME );
+			if( IsValid( "integer", cellValue ) )
+				cellValue = JavaCast( "string", cellValue );// prevent CSV writer converting 1 to 1.0
+			rowValues.Append( cellValue );
+		};
+		return rowValues;
+	}
+
 }

helpers/exception.cfc

@@ -30,4 +30,10 @@ component extends="base"{
 		Throw( type=library().getExceptionType() & ".nonExistentRow", message="Non-existent row", detail="Row #arguments.rowNumber# doesn't exist. You may need to create the row first by adding data to it." );
 	}
 
+	void function throwParallelOptionNotSupportedException(){
+		Throw( type=library().getExceptionType() & ".parallelOptionNotSupported", message="Parallel threads option not supported", detail="Your ColdFusion engine does not support parallel processing of loops via threads" );
+	}
+
+
+
 }

helpers/format.cfc

@@ -182,7 +182,7 @@ component extends="base"{
 		,required string setting
 	){
 		var font = 0;
-		var settingValue = arguments.format[ setting ];
+		var settingValue = arguments.format[ arguments.setting ];
 		switch( arguments.setting ){
 			case "alignment":
 				var alignment = arguments.cellStyle.getAlignment()[ JavaCast( "string", UCase( settingValue ) ) ];

helpers/query.cfc

@@ -147,7 +147,7 @@ component extends="base"{
 		cfloop( from=1, to=arguments.columnCount, index="local.colNum" ){
 			types[ colNum ] = "";
 			for( var row in arguments.data ){
-				if( row.IsEmpty() || ( row.Len() < colNum ) )
+				if( ArrayLen( row ) < colNum )
 					continue;//next column (ACF: empty values are sometimes just missing from the array??)
 				var value = row[ colNum ];
 				var detectedType = getDataTypeHelper().detectValueDataType( value );