Speed up scan (microsoft#916)

* Speed up scan

Avoid buffer pool allocation per GetNext and reuse memory to speed up scan

* nit

* avoid two counters for DBScan.

* update version

* Revert "update version"

This reverts commit 718059a.

* update version

Version.props

@@ -1,6 +1,6 @@
 <Project>
 	<!-- Versioning property for builds and packages -->
 	<PropertyGroup>
-		<VersionPrefix>1.0.50</VersionPrefix>
+		<VersionPrefix>1.0.51</VersionPrefix>
 	</PropertyGroup>
 </Project>

libs/server/Storage/Session/Common/ArrayKeyIterationFunctions.cs

@@ -312,11 +312,9 @@ internal sealed class MainStoreGetDBSize : IScanIteratorFunctions<SpanByte, Span
 
                 public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult)
                 {
-                    if (value.MetadataSize != 0 && MainSessionFunctions.CheckExpiry(ref value))
-                        cursorRecordResult = CursorRecordResult.Skip;
-                    else
+                    cursorRecordResult = CursorRecordResult.Skip;
+                    if (value.MetadataSize == 0 || !MainSessionFunctions.CheckExpiry(ref value))
                     {
-                        cursorRecordResult = CursorRecordResult.Accept;
                         ++info.count;
                     }
                     return true;
@@ -340,11 +338,9 @@ internal sealed class ObjectStoreGetDBSize : IScanIteratorFunctions<byte[], IGar
 
                 public bool SingleReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult)
                 {
-                    if (value.Expiration > 0 && ObjectSessionFunctions.CheckExpiry(value))
-                        cursorRecordResult = CursorRecordResult.Skip;
-                    else
+                    cursorRecordResult = CursorRecordResult.Skip;
+                    if (value.Expiration == 0 || !ObjectSessionFunctions.CheckExpiry(value))
                     {
-                        cursorRecordResult = CursorRecordResult.Accept;
                         ++info.count;
                     }
                     return true;

libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteScanIterator.cs

@@ -182,18 +182,28 @@ public unsafe bool GetNext(out RecordInfo recordInfo)
                 // We will return control to the caller, which means releasing epoch protection, and we don't want the caller to lock.
                 // Copy the entire record into bufferPool memory, so we do not have a ref to log data outside epoch protection.
                 // Lock to ensure no value tearing while copying to temp storage.
-                memory?.Return();
-                memory = null;
                 if (currentAddress >= headAddress || forceInMemory)
                 {
                     OperationStackContext<SpanByte, SpanByte, TStoreFunctions, SpanByteAllocator<TStoreFunctions>> stackCtx = default;
                     try
                     {
+                        if (memory == null)
+                        {
+                            memory = hlog.bufferPool.Get(recordSize);
+                        }
+                        else
+                        {
+                            if (memory.AlignedTotalCapacity < recordSize)
+                            {
+                                memory.Return();
+                                memory = hlog.bufferPool.Get(recordSize);
+                            }
+                        }
+
                         // GetKey() should work but for safety and consistency with other allocators use physicalAddress.
                         if (currentAddress >= headAddress && store is not null)
                             store.LockForScan(ref stackCtx, ref hlog._wrapper.GetKey(physicalAddress));
 
-                        memory = hlog.bufferPool.Get(recordSize);
                         unsafe
                         {
                             Buffer.MemoryCopy((byte*)currentPhysicalAddress, memory.aligned_pointer, recordSize, recordSize);

libs/storage/Tsavorite/cs/src/core/Utilities/BufferPool.cs

@@ -136,6 +136,11 @@ public void Return()
             pool?.Return(this);
         }
 
+        /// <summary>
+        /// Get the total aligned memory capacity of the buffer
+        /// </summary>
+        public int AlignedTotalCapacity => buffer.Length - offset;
+
         /// <summary>
         /// Get valid pointer
         /// </summary>