From a4bc0dd5d05f8dc14ee5bed1550ff43110d6e30f Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sat, 16 Nov 2024 22:28:33 +0200
Subject: [PATCH 1/9] Fix typos in error messages

Signed-off-by: Nir Soffer <nirsof@gmail.com>
---
 image/qcow2/qcow2.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/image/qcow2/qcow2.go b/image/qcow2/qcow2.go
index 8ee3370..1e8e730 100644
--- a/image/qcow2/qcow2.go
+++ b/image/qcow2/qcow2.go
@@ -562,7 +562,7 @@ func Open(ra io.ReaderAt, openWithType image.OpenWithType) (*Qcow2, error) {
 	var err error
 	img.Header, err = readHeader(r)
 	if err != nil {
-		return nil, fmt.Errorf("faild to read header: %w", err)
+		return nil, fmt.Errorf("failed to read header: %w", err)
 	}
 	img.errUnreadable = img.Header.Readable() // cache
 	if img.errUnreadable == nil {
@@ -596,7 +596,7 @@ func Open(ra io.ReaderAt, openWithType image.OpenWithType) (*Qcow2, error) {
 		// Load L1 table
 		img.l1Table, err = readL1Table(ra, img.Header.L1TableOffset, img.Header.L1Size)
 		if err != nil {
-			return img, fmt.Errorf("faild to read L1 table: %w", err)
+			return img, fmt.Errorf("failed to read L1 table: %w", err)
 		}
 
 		// Load decompressor

From 5864483d75403b7facca752de83a6837c743c590 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sat, 26 Oct 2024 20:19:36 +0300
Subject: [PATCH 2/9] Add Image.Extent() interface

Extent() return the next extent in the image, starting at the specified
offset, and limited by the specified length. An extent is a run of
clusters of with the same status.

Using Extent() we can iterate over the image and skip extents that reads
as zeros, instead of reading a zero buffer and detecting that the buffer
is full of zeros.

Benchmarking show 2 orders of magnitude speeded compared with zero
detection (10 TiB/s instead of 100 GiB/s)

    % go test -bench Benchmark0p/qcow2$/read
    Benchmark0p/qcow2/read-12      516     2503752 ns/op      107213.28 MB/s    1050537 B/op    39 allocs/op

    % go test -bench NextUnallocated
    BenchmarkNextUnallocated-12    116    10236186 ns/op    10489666.91 MB/s          0 B/op     0 allocs/op

Only qcow2 image has a real implementation. For other image formats we
treat all clusters as allocated, so Extent() always return one allocated
extent. For raw format we can implement Extents() later using SEEK_DATA
and SEEK_HOLE.

Extent() is more strict than ReadAt and fails when offset+length are
after the end of the image. We always know the length of the image so
requesting an extent after the end of the image is likely a bug in the
user code and failing fast may help to debug such issues.

Signed-off-by: Nir Soffer <nirsof@gmail.com>
---
 image/image.go       |  18 +++++
 image/qcow2/qcow2.go | 107 +++++++++++++++++++++++++
 image/raw/raw.go     |  13 ++++
 image/stub/stub.go   |   4 +
 qcow2reader_test.go  | 181 +++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 323 insertions(+)

diff --git a/image/image.go b/image/image.go
index 7ea497e..e7d1e52 100644
--- a/image/image.go
+++ b/image/image.go
@@ -8,10 +8,28 @@ import (
 // Type must be a "Backing file format name string" that appears in QCOW2.
 type Type string
 
+// Extent describes a byte range in the image with the same allocation,
+// compression, or zero status. Extents are aligned to the underlying file
+// system block size (e.g. 4k), or the image format cluster size (e.g. 64k). One
+// extent can describe one or more file system blocks or image clusters.
+type Extent struct {
+	// Offset from start of the image in bytes.
+	Start int64 `json:"start"`
+	// Length of this extent in bytes.
+	Length int64 `json:"length"`
+	// Set if this extent is allocated.
+	Allocated bool `json:"allocated"`
+	// Set if this extent is read as zeros.
+	Zero bool `json:"zero"`
+	// Set if this extent is compressed.
+	Compressed bool `json:"compressed"`
+}
+
 // Image implements [io.ReaderAt] and [io.Closer].
 type Image interface {
 	io.ReaderAt
 	io.Closer
+	Extent(start, length int64) (Extent, error)
 	Type() Type
 	Size() int64 // -1 if unknown
 	Readable() error
diff --git a/image/qcow2/qcow2.go b/image/qcow2/qcow2.go
index 1e8e730..582baf0 100644
--- a/image/qcow2/qcow2.go
+++ b/image/qcow2/qcow2.go
@@ -776,6 +776,9 @@ func (img *Qcow2) getClusterMeta(off int64, cm *clusterMeta) error {
 	if cm.L2Entry.compressed() {
 		cm.Compressed = true
 	} else {
+		// When using extended L2 clusters this is always false. To find which sub
+		// cluster is allocated/zero we need to iterate over the allocation bitmap in
+		// the extended l2 cluster entry.
 		cm.Zero = standardClusterDescriptor(desc).allZero()
 	}
 
@@ -966,6 +969,110 @@ func readZero(p []byte, off int64, sz uint64) (int, error) {
 	return l, err
 }
 
+// clusterStatus returns an extent describing a single cluster. off must be aligned to
+// cluster size.
+func (img *Qcow2) clusterStatus(off int64) (image.Extent, error) {
+	var cm clusterMeta
+	if err := img.getClusterMeta(off, &cm); err != nil {
+		return image.Extent{}, err
+	}
+
+	if !cm.Allocated {
+		// If there is no backing file, or the cluster cannot be in the backing file,
+		// return an unallocated cluster.
+		if img.backingImage == nil || off >= img.backingImage.Size() {
+			// Unallocated cluster reads as zeros.
+			unallocated := image.Extent{Start: off, Length: int64(img.clusterSize), Zero: true}
+			return unallocated, nil
+		}
+
+		// Get the cluster from the backing file.
+		length := int64(img.clusterSize)
+		if off+length > img.backingImage.Size() {
+			length = img.backingImage.Size() - off
+		}
+		parent, err := img.backingImage.Extent(off, length)
+		if err != nil {
+			return parent, err
+		}
+		// The backing image may be a raw image not aligned to cluster size.
+		parent.Length = int64(img.clusterSize)
+		return parent, nil
+	}
+
+	// Cluster present in this image.
+	allocated := image.Extent{
+		Start:      off,
+		Length:     int64(img.clusterSize),
+		Allocated:  true,
+		Compressed: cm.Compressed,
+		Zero:       cm.Zero,
+	}
+	return allocated, nil
+}
+
+// Return true if extents have the same status.
+func sameStatus(a, b image.Extent) bool {
+	return a.Allocated == b.Allocated && a.Zero == b.Zero && a.Compressed == b.Compressed
+}
+
+// Extent returns the next extent starting at the specified offset. An extent
+// describes one or more clusters having the same status. The maximum length of
+// the returned extent is limited by the specified length. The minimum length of
+// the returned extent is length of one cluster.
+func (img *Qcow2) Extent(start, length int64) (image.Extent, error) {
+	// Default to zero length non-existent cluster.
+	var current image.Extent
+
+	if img.errUnreadable != nil {
+		return current, img.errUnreadable
+	}
+	if img.clusterSize == 0 {
+		return current, errors.New("cluster size cannot be 0")
+	}
+	if start+length > int64(img.Header.Size) {
+		return current, errors.New("length out of bounds")
+	}
+
+	// Compute the clusterStart of the first cluster to query. This may be behind start.
+	clusterStart := start / int64(img.clusterSize) * int64(img.clusterSize)
+
+	remaining := length
+	for remaining > 0 {
+		clusterStatus, err := img.clusterStatus(clusterStart)
+		if err != nil {
+			return current, err
+		}
+
+		// First cluster: if start is not aligned to cluster size, clip the start.
+		if clusterStatus.Start < start {
+			clusterStatus.Start = start
+			clusterStatus.Length -= start - clusterStatus.Start
+		}
+
+		// Last cluster: if start+length is not aligned to cluster size, clip the end.
+		if remaining < int64(img.clusterSize) {
+			clusterStatus.Length -= int64(img.clusterSize) - remaining
+		}
+
+		if current.Length == 0 {
+			// First cluster: copy status to current.
+			current = clusterStatus
+		} else if sameStatus(current, clusterStatus) {
+			// Cluster with same status: extend current.
+			current.Length += clusterStatus.Length
+		} else {
+			// Start of next extent
+			break
+		}
+
+		clusterStart += int64(img.clusterSize)
+		remaining -= clusterStatus.Length
+	}
+
+	return current, nil
+}
+
 // ReadAt implements [io.ReaderAt].
 func (img *Qcow2) ReadAt(p []byte, off int64) (n int, err error) {
 	if img.errUnreadable != nil {
diff --git a/image/raw/raw.go b/image/raw/raw.go
index e0b7fcb..4d59fab 100644
--- a/image/raw/raw.go
+++ b/image/raw/raw.go
@@ -1,6 +1,7 @@
 package raw
 
 import (
+	"errors"
 	"io"
 	"os"
 
@@ -14,6 +15,18 @@ type Raw struct {
 	io.ReaderAt `json:"-"`
 }
 
+// Extent returns an allocated extent starting at the specified offset with
+// specified length. It is used when the speicfic image type does not implement
+// Extent(). The implementation is correct but inefficient. Fails if image size
+// is unknown.
+func (img *Raw) Extent(start, length int64) (image.Extent, error) {
+	if start+length > img.Size() {
+		return image.Extent{}, errors.New("length out of bounds")
+	}
+	// TODO: Implement using SEEK_HOLE/SEEK_DATA when supported by the file system.
+	return image.Extent{Start: start, Length: length, Allocated: true}, nil
+}
+
 func (img *Raw) Close() error {
 	if closer, ok := img.ReaderAt.(io.Closer); ok {
 		return closer.Close()
diff --git a/image/stub/stub.go b/image/stub/stub.go
index 103fb23..a595b35 100644
--- a/image/stub/stub.go
+++ b/image/stub/stub.go
@@ -25,6 +25,10 @@ type Stub struct {
 	t image.Type
 }
 
+func (img *Stub) Extent(start, length int64) (image.Extent, error) {
+	return image.Extent{}, fmt.Errorf("unimplemented type: %q", img.t)
+}
+
 func (img *Stub) ReadAt([]byte, int64) (int, error) {
 	return 0, img.Readable()
 }
diff --git a/qcow2reader_test.go b/qcow2reader_test.go
index ec8764c..4de86b2 100644
--- a/qcow2reader_test.go
+++ b/qcow2reader_test.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/lima-vm/go-qcow2reader"
 	"github.com/lima-vm/go-qcow2reader/convert"
+	"github.com/lima-vm/go-qcow2reader/image"
 	"github.com/lima-vm/go-qcow2reader/test/qemuimg"
 )
 
@@ -19,6 +20,186 @@ const (
 	GiB = int64(1) << 30
 )
 
+func TestExtentsUnallocated(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "image")
+	if err := qemuimg.Create(path, qemuimg.FormatQcow2, 4*GiB, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	img, err := qcow2reader.Open(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer img.Close()
+
+	t.Run("entire image", func(t *testing.T) {
+		actual, err := img.Extent(0, img.Size())
+		if err != nil {
+			t.Fatal(err)
+		}
+		expected := image.Extent{Start: 0, Length: img.Size(), Zero: true}
+		if actual != expected {
+			t.Fatalf("expected %+v, got %+v", expected, actual)
+		}
+	})
+	t.Run("same result", func(t *testing.T) {
+		r1, err := img.Extent(0, img.Size())
+		if err != nil {
+			t.Fatal(err)
+		}
+		r2, err := img.Extent(0, img.Size())
+		if err != nil {
+			t.Fatal(err)
+		}
+		if r1 != r2 {
+			t.Fatalf("expected %+v, got %+v", r1, r2)
+		}
+	})
+	t.Run("all segments", func(t *testing.T) {
+		for i := int64(0); i < img.Size(); i += 32 * MiB {
+			segment, err := img.Extent(i, 32*MiB)
+			if err != nil {
+				t.Fatal(err)
+			}
+			expected := image.Extent{Start: i, Length: 32 * MiB, Zero: true}
+			if segment != expected {
+				t.Fatalf("expected %+v, got %+v", expected, segment)
+			}
+		}
+	})
+	t.Run("start unaligned", func(t *testing.T) {
+		start := 32*MiB + 42
+		length := 32 * MiB
+		actual, err := img.Extent(start, length)
+		if err != nil {
+			t.Fatal(err)
+		}
+		expected := image.Extent{Start: start, Length: length, Zero: true}
+		if actual != expected {
+			t.Fatalf("expected %+v, got %+v", expected, actual)
+		}
+	})
+	t.Run("length unaligned", func(t *testing.T) {
+		start := 32 * MiB
+		length := 32*MiB - 42
+		actual, err := img.Extent(start, length)
+		if err != nil {
+			t.Fatal(err)
+		}
+		expected := image.Extent{Start: start, Length: length, Zero: true}
+		if actual != expected {
+			t.Fatalf("expected %+v, got %+v", expected, actual)
+		}
+	})
+	t.Run("start and length unaligned", func(t *testing.T) {
+		start := 32*MiB + 42
+		length := 32*MiB - 42
+		actual, err := img.Extent(start, length)
+		if err != nil {
+			t.Fatal(err)
+		}
+		expected := image.Extent{Start: start, Length: length, Zero: true}
+		if actual != expected {
+			t.Fatalf("expected %+v, got %+v", expected, actual)
+		}
+	})
+	t.Run("length after end of image", func(t *testing.T) {
+		start := img.Size() - 31*MiB
+		actual, err := img.Extent(start, 32*MiB)
+		if err == nil {
+			t.Fatal("out of bounds request did not fail")
+		}
+		var expected image.Extent
+		if actual != expected {
+			t.Fatalf("expected %+v, got %+v", expected, actual)
+		}
+	})
+	t.Run("start after end of image", func(t *testing.T) {
+		start := img.Size() + 1*MiB
+		actual, err := img.Extent(start, 32*MiB)
+		if err == nil {
+			t.Fatal("out of bounds request did not fail")
+		}
+		var expected image.Extent
+		if actual != expected {
+			t.Fatalf("expected %+v, got %+v", expected, actual)
+		}
+	})
+}
+
+func TestExtentsRaw(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "disk.img")
+	size := 4 * GiB
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	if err := f.Truncate(size); err != nil {
+		t.Fatal(err)
+	}
+	img, err := qcow2reader.Open(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer img.Close()
+
+	t.Run("entire image", func(t *testing.T) {
+		actual, err := img.Extent(0, img.Size())
+		if err != nil {
+			t.Fatal(err)
+		}
+		// Currently we always report raw images as fully allocated.
+		expected := image.Extent{Start: 0, Length: img.Size(), Allocated: true}
+		if actual != expected {
+			t.Fatalf("expected %+v, got %+v", expected, actual)
+		}
+	})
+	t.Run("length after end of image", func(t *testing.T) {
+		start := img.Size() - 31*MiB
+		actual, err := img.Extent(start, 32*MiB)
+		if err == nil {
+			t.Fatal("out of bounds request did not fail")
+		}
+		var expected image.Extent
+		if actual != expected {
+			t.Fatalf("expected %+v, got %+v", expected, actual)
+		}
+	})
+}
+
+func BenchmarkExtentsUnallocated(b *testing.B) {
+	path := filepath.Join(b.TempDir(), "image")
+	if err := qemuimg.Create(path, qemuimg.FormatQcow2, 100*GiB, "", ""); err != nil {
+		b.Fatal(err)
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		b.Fatal(err)
+	}
+	img, err := qcow2reader.Open(f)
+	if err != nil {
+		b.Fatal(err)
+	}
+	expected := image.Extent{Start: 0, Length: img.Size(), Zero: true}
+	resetBenchmark(b, img.Size())
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		actual, err := img.Extent(0, img.Size())
+		b.StopTimer()
+		if err != nil {
+			b.Fatal(err)
+		}
+		if actual != expected {
+			b.Fatalf("expected %+v, got %+v", expected, actual)
+		}
+	}
+}
+
 // Benchmark completely empty sparse image (0% utilization).  This is the best
 // case when we don't have to read any cluster from storage.
 func Benchmark0p(b *testing.B) {

From 55fbff069ba9fa94ec2c9f8fc81faa3287c42655 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sat, 2 Nov 2024 01:18:30 +0200
Subject: [PATCH 3/9] Test qcow2 images with different allocation

Add a helper for creating an image from list of extents. With this we
can create a raw or qcow2 image with allocation described by the
extents.

Add 4 tests, testing different allocation patterns:

- TestExtentsSome: some allocated clusters and some holes
- TestExtentsPartial: writing partial cluster allocates entire cluster
- TestExtentsMerge: consecutive extents of same type are merged
- TestExtentsZero: different extents types that read as zeros

For each test we verify qcow2 and qcow2 compressed (zlib) images.

Signed-off-by: Nir Soffer <nirsof@gmail.com>
---
 qcow2reader_test.go | 261 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 259 insertions(+), 2 deletions(-)

diff --git a/qcow2reader_test.go b/qcow2reader_test.go
index 4de86b2..bf879fc 100644
--- a/qcow2reader_test.go
+++ b/qcow2reader_test.go
@@ -7,17 +7,21 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	"github.com/lima-vm/go-qcow2reader"
 	"github.com/lima-vm/go-qcow2reader/convert"
 	"github.com/lima-vm/go-qcow2reader/image"
 	"github.com/lima-vm/go-qcow2reader/test/qemuimg"
+	"github.com/lima-vm/go-qcow2reader/test/qemuio"
 )
 
 const (
-	MiB = int64(1) << 20
-	GiB = int64(1) << 30
+	KiB         = int64(1) << 10
+	MiB         = int64(1) << 20
+	GiB         = int64(1) << 30
+	clusterSize = 64 * KiB
 )
 
 func TestExtentsUnallocated(t *testing.T) {
@@ -200,6 +204,259 @@ func BenchmarkExtentsUnallocated(b *testing.B) {
 	}
 }
 
+func TestExtentsSome(t *testing.T) {
+	extents := []image.Extent{
+		{Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		{Start: 1 * clusterSize, Length: 1 * clusterSize, Zero: true},
+		{Start: 2 * clusterSize, Length: 2 * clusterSize, Allocated: true},
+		{Start: 4 * clusterSize, Length: 96 * clusterSize, Zero: true},
+		{Start: 100 * clusterSize, Length: 8 * clusterSize, Allocated: true},
+		{Start: 108 * clusterSize, Length: 892 * clusterSize, Zero: true},
+		{Start: 1000 * clusterSize, Length: 16 * clusterSize, Allocated: true},
+		{Start: 1016 * clusterSize, Length: 8984 * clusterSize, Zero: true},
+	}
+	qcow2 := filepath.Join(t.TempDir(), "image")
+	if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	t.Run("qcow2", func(t *testing.T) {
+		actual, err := listExtents(qcow2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(extents, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+	t.Run("qcow2 zlib", func(t *testing.T) {
+		qcow2Zlib := qcow2 + ".zlib"
+		if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil {
+			t.Fatal(err)
+		}
+		actual, err := listExtents(qcow2Zlib)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(compressed(extents), actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+}
+
+func TestExtentsPartial(t *testing.T) {
+	// Writing part of of a cluster allocates entire cluster in the qcow2 image.
+	extents := []image.Extent{
+		{Start: 0 * clusterSize, Length: 1, Allocated: true},
+		{Start: 1 * clusterSize, Length: 98 * clusterSize, Zero: true},
+		{Start: 100*clusterSize - 1, Length: 1, Allocated: true},
+	}
+
+	// Listing extents works in cluster granularity.
+	full := []image.Extent{
+		{Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		{Start: 1 * clusterSize, Length: 98 * clusterSize, Zero: true},
+		{Start: 99 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+	}
+
+	qcow2 := filepath.Join(t.TempDir(), "image")
+	if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	t.Run("qcow2", func(t *testing.T) {
+		actual, err := listExtents(qcow2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(full, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+	t.Run("qcow2 zlib", func(t *testing.T) {
+		qcow2Zlib := qcow2 + ".zlib"
+		if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil {
+			t.Fatal(err)
+		}
+		actual, err := listExtents(qcow2Zlib)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(compressed(full), actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+}
+
+func TestExtentsMerge(t *testing.T) {
+	// Create image with consecutive extents of same type.
+	extents := []image.Extent{
+		{Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		{Start: 1 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		{Start: 2 * clusterSize, Length: 98 * clusterSize, Zero: true},
+		{Start: 100 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		{Start: 101 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+	}
+
+	// Extents with same type are merged.
+	merged := []image.Extent{
+		{Start: 0 * clusterSize, Length: 2 * clusterSize, Allocated: true},
+		{Start: 2 * clusterSize, Length: 98 * clusterSize, Zero: true},
+		{Start: 100 * clusterSize, Length: 2 * clusterSize, Allocated: true},
+	}
+
+	qcow2 := filepath.Join(t.TempDir(), "image")
+	if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	t.Run("qcow2", func(t *testing.T) {
+		actual, err := listExtents(qcow2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(merged, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+	t.Run("qcow2 zlib", func(t *testing.T) {
+		qcow2Zlib := qcow2 + ".zlib"
+		if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil {
+			t.Fatal(err)
+		}
+		actual, err := listExtents(qcow2Zlib)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(compressed(merged), actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+}
+
+func TestExtentsZero(t *testing.T) {
+	// Create image with different clusters that read as zeros.
+	extents := []image.Extent{
+		{Start: 0 * clusterSize, Length: 1000 * clusterSize, Allocated: true, Zero: true},
+		{Start: 1000 * clusterSize, Length: 1000 * clusterSize, Zero: true},
+	}
+
+	qcow2 := filepath.Join(t.TempDir(), "image")
+	if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	t.Run("qcow2", func(t *testing.T) {
+		actual, err := listExtents(qcow2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(extents, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+	t.Run("qcow2 zlib", func(t *testing.T) {
+		qcow2Zlib := qcow2 + ".zlib"
+		if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil {
+			t.Fatal(err)
+		}
+		actual, err := listExtents(qcow2Zlib)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// When converting to qcow2 images all clusters that read as zeros are
+		// converted to unallocated clusters.
+		converted := []image.Extent{
+			{Start: 0 * clusterSize, Length: 2000 * clusterSize, Zero: true},
+		}
+		if !slices.Equal(converted, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+}
+
+func compressed(extents []image.Extent) []image.Extent {
+	var res []image.Extent
+	for _, extent := range extents {
+		if extent.Allocated {
+			extent.Compressed = true
+		}
+		res = append(res, extent)
+	}
+	return res
+}
+
+func listExtents(path string) ([]image.Extent, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	img, err := qcow2reader.Open(f)
+	if err != nil {
+		return nil, err
+	}
+	defer img.Close()
+
+	var extents []image.Extent
+	var start int64
+
+	end := img.Size()
+	for start < end {
+		extent, err := img.Extent(start, end-start)
+		if err != nil {
+			return nil, err
+		}
+		if extent.Start != start {
+			return nil, fmt.Errorf("invalid extent start: %+v", extent)
+		}
+		if extent.Length <= 0 {
+			return nil, fmt.Errorf("invalid extent length: %+v", extent)
+		}
+		extents = append(extents, extent)
+		start += extent.Length
+	}
+	return extents, nil
+}
+
+// createTestImageWithExtents creates a n image with the allocation described
+// by extents.
+func createTestImageWithExtents(
+	path string,
+	format qemuimg.Format,
+	extents []image.Extent,
+	backingFile string,
+	backingFormat qemuimg.Format,
+) error {
+	lastExtent := extents[len(extents)-1]
+	size := lastExtent.Start + lastExtent.Length
+	if err := qemuimg.Create(path, format, size, backingFile, backingFormat); err != nil {
+		return err
+	}
+	for _, extent := range extents {
+		if !extent.Allocated {
+			continue
+		}
+		start := extent.Start
+		length := extent.Length
+		for length > 0 {
+			// qemu-io requires length < 2g.
+			n := length
+			if n >= 2*GiB {
+				n = 2*GiB - 64*KiB
+			}
+			if extent.Zero {
+				if err := qemuio.Zero(path, format, start, n); err != nil {
+					return err
+				}
+			} else {
+				if err := qemuio.Write(path, format, start, n, 0x55); err != nil {
+					return err
+				}
+			}
+			start += n
+			length -= n
+		}
+	}
+	return nil
+}
+
 // Benchmark completely empty sparse image (0% utilization).  This is the best
 // case when we don't have to read any cluster from storage.
 func Benchmark0p(b *testing.B) {

From 1982b2b375cbad02c1fdc6f0b6bba064af5e51cf Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sat, 16 Nov 2024 20:30:33 +0200
Subject: [PATCH 4/9] Test image with a backing file

We cannot have useful tests for raw backing file since we have only a
stub implementation reporting that the entire backing file is allocated,
so we test only qcow2 image backing file.

We test backing file in qcow2 and qcow2 compressed. When both images are
uncompressed, allocated extents from top and base are merged. When base
is compressed, allocated extents from top and based cannot be merged,
since all base clusters are compressed.

Compressed image with backing file may be possible to crate with
qemu-io, but is not real use case so we don't test it.

Signed-off-by: Nir Soffer <nirsof@gmail.com>
---
 qcow2reader_test.go | 94 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)

diff --git a/qcow2reader_test.go b/qcow2reader_test.go
index bf879fc..9b59229 100644
--- a/qcow2reader_test.go
+++ b/qcow2reader_test.go
@@ -371,6 +371,100 @@ func TestExtentsZero(t *testing.T) {
 	})
 }
 
+func TestExtentsBackingFile(t *testing.T) {
+	// Create an image with some clusters in the backing file, and some cluasters
+	// in the image. Accessing extents should present a unified view using both
+	// image and backing file.
+	tmpDir := t.TempDir()
+	baseExtents := []image.Extent{
+		{Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		{Start: 1 * clusterSize, Length: 9 * clusterSize, Zero: true},
+		{Start: 10 * clusterSize, Length: 2 * clusterSize, Allocated: true},
+		{Start: 12 * clusterSize, Length: 88 * clusterSize, Zero: true},
+		{Start: 100 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		{Start: 101 * clusterSize, Length: 899 * clusterSize, Zero: true},
+	}
+	topExtents := []image.Extent{
+		{Start: 0 * clusterSize, Length: 1 * clusterSize, Zero: true},
+		{Start: 1 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		{Start: 2 * clusterSize, Length: 9 * clusterSize, Zero: true},
+		{Start: 11 * clusterSize, Length: 2 * clusterSize, Allocated: true},
+		{Start: 13 * clusterSize, Length: 986 * clusterSize, Zero: true},
+		{Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+	}
+	baseRaw := filepath.Join(tmpDir, "base.raw")
+	if err := createTestImageWithExtents(baseRaw, qemuimg.FormatRaw, baseExtents, "", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("qcow2", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		baseQcow2 := filepath.Join(tmpDir, "base.qcow2")
+		if err := qemuimg.Convert(baseRaw, baseQcow2, qemuimg.FormatQcow2, qemuimg.CompressionNone); err != nil {
+			t.Fatal(err)
+		}
+		top := filepath.Join(tmpDir, "top.qcow2")
+		if err := createTestImageWithExtents(top, qemuimg.FormatQcow2, topExtents, baseQcow2, qemuimg.FormatQcow2); err != nil {
+			t.Fatal(err)
+		}
+		// When top and base are uncompressed, extents from to and based are merged.
+		expected := []image.Extent{
+			{Start: 0 * clusterSize, Length: 2 * clusterSize, Allocated: true},
+			{Start: 2 * clusterSize, Length: 8 * clusterSize, Zero: true},
+			{Start: 10 * clusterSize, Length: 3 * clusterSize, Allocated: true},
+			{Start: 13 * clusterSize, Length: 87 * clusterSize, Zero: true},
+			{Start: 100 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+			{Start: 101 * clusterSize, Length: 898 * clusterSize, Zero: true},
+			{Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		}
+		actual, err := listExtents(top)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(expected, actual) {
+			t.Fatalf("expected %v, got %v", expected, actual)
+		}
+	})
+	t.Run("qcow2 zlib", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		baseQcow2Zlib := filepath.Join(tmpDir, "base.qcow2")
+		if err := qemuimg.Convert(baseRaw, baseQcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil {
+			t.Fatal(err)
+		}
+		top := filepath.Join(tmpDir, "top.qcow2")
+		if err := createTestImageWithExtents(top, qemuimg.FormatQcow2, topExtents, baseQcow2Zlib, qemuimg.FormatQcow2); err != nil {
+			t.Fatal(err)
+		}
+		// When base is compressed, extents from to and based cannot be merged since
+		// allocated extents from base are compressed. When copying we can merge
+		// extents with different types that read as zero.
+		expected := []image.Extent{
+			// From base
+			{Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true, Compressed: true},
+			// From top
+			{Start: 1 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+			{Start: 2 * clusterSize, Length: 8 * clusterSize, Zero: true},
+			// From base
+			{Start: 10 * clusterSize, Length: 1 * clusterSize, Allocated: true, Compressed: true},
+			// From top (top clusters hide base clusters)
+			{Start: 11 * clusterSize, Length: 2 * clusterSize, Allocated: true},
+			{Start: 13 * clusterSize, Length: 87 * clusterSize, Zero: true},
+			// From base
+			{Start: 100 * clusterSize, Length: 1 * clusterSize, Allocated: true, Compressed: true},
+			{Start: 101 * clusterSize, Length: 898 * clusterSize, Zero: true},
+			// From top
+			{Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+		}
+		actual, err := listExtents(top)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(expected, actual) {
+			t.Fatalf("expected %v, got %v", expected, actual)
+		}
+	})
+}
+
 func compressed(extents []image.Extent) []image.Extent {
 	var res []image.Extent
 	for _, extent := range extents {

From 3358492c3259e5d990b49cec38895bf4c7fb41bd Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sat, 16 Nov 2024 22:04:59 +0200
Subject: [PATCH 5/9] Test short backing file

When adding a longer qcow2 image on top of an os image, we treat extents
after the end of the backing file as unallocated.

We test also the special case of raw backing file not aligned to cluster
size. When reading a short extent from the end of the backing file we
extend the length to full cluster.

Signed-off-by: Nir Soffer <nirsof@gmail.com>
---
 qcow2reader_test.go | 69 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/qcow2reader_test.go b/qcow2reader_test.go
index 9b59229..1f1d0aa 100644
--- a/qcow2reader_test.go
+++ b/qcow2reader_test.go
@@ -465,6 +465,75 @@ func TestExtentsBackingFile(t *testing.T) {
 	})
 }
 
+func TestExtentsBackingFileShort(t *testing.T) {
+	// Test the special case of shorter backing file. The typical use case is
+	// adding a large qcow2 image on top of a small os image.
+	baseExtents := []image.Extent{
+		{Start: 0 * clusterSize, Length: 10 * clusterSize, Allocated: true},
+		{Start: 10 * clusterSize, Length: 90 * clusterSize, Zero: true},
+	}
+	topExtents := []image.Extent{
+		{Start: 0 * clusterSize, Length: 5 * clusterSize, Zero: true},
+		{Start: 5 * clusterSize, Length: 10 * clusterSize, Allocated: true},
+		{Start: 15 * clusterSize, Length: 984 * clusterSize, Zero: true},
+		{Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+	}
+	expected := []image.Extent{
+		{Start: 0 * clusterSize, Length: 15 * clusterSize, Allocated: true},
+		{Start: 15 * clusterSize, Length: 984 * clusterSize, Zero: true},
+		{Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true},
+	}
+	tmpDir := t.TempDir()
+	base := filepath.Join(tmpDir, "base.qcow2")
+	if err := createTestImageWithExtents(base, qemuimg.FormatQcow2, baseExtents, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	top := filepath.Join(tmpDir, "top.qcow2")
+	if err := createTestImageWithExtents(top, qemuimg.FormatQcow2, topExtents, base, qemuimg.FormatQcow2); err != nil {
+		t.Fatal(err)
+	}
+	actual, err := listExtents(top)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(expected, actual) {
+		t.Fatalf("expected %v, got %v", expected, actual)
+	}
+}
+
+func TestExtentsBackingFileShortUnaligned(t *testing.T) {
+	// Test the special case of raw backing file not aligned to cluster size. When
+	// getting a short extent from the end of the backing file, we aligned the
+	// extent length to the top image cluster size.
+	blockSize := 4 * KiB
+	baseExtents := []image.Extent{
+		{Start: 0, Length: 100 * blockSize, Allocated: true},
+	}
+	topExtents := []image.Extent{
+		{Start: 0, Length: 10 * clusterSize, Zero: true},
+	}
+	expected := []image.Extent{
+		{Start: 0 * clusterSize, Length: 7 * clusterSize, Allocated: true},
+		{Start: 7 * clusterSize, Length: 3 * clusterSize, Zero: true},
+	}
+	tmpDir := t.TempDir()
+	base := filepath.Join(tmpDir, "base.raw")
+	if err := createTestImageWithExtents(base, qemuimg.FormatRaw, baseExtents, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	top := filepath.Join(tmpDir, "top.qcow2")
+	if err := createTestImageWithExtents(top, qemuimg.FormatQcow2, topExtents, base, qemuimg.FormatRaw); err != nil {
+		t.Fatal(err)
+	}
+	actual, err := listExtents(top)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(expected, actual) {
+		t.Fatalf("expected %v, got %v", expected, actual)
+	}
+}
+
 func compressed(extents []image.Extent) []image.Extent {
 	var res []image.Extent
 	for _, extent := range extents {

From 89056c64d473048aa52d9944506cd9d57cac0607 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sat, 2 Nov 2024 04:36:57 +0200
Subject: [PATCH 6/9] Add map sub command to example program

This is an example how the extent interface can be used, and useful tool
for debugging, similar to `qemu-img map`.

Example runs:

    % ./go-qcow2reader-example map /var/tmp/images/test.qcow2 | head -5
    {"start":0,"length":65536,"allocated":true,"zero":false,"compressed":false}
    {"start":65536,"length":983040,"allocated":false,"zero":true,"compressed":false}
    {"start":1048576,"length":131072,"allocated":true,"zero":false,"compressed":false}
    {"start":1179648,"length":655360,"allocated":false,"zero":true,"compressed":false}
    {"start":1835008,"length":131072,"allocated":true,"zero":false,"compressed":false}

    % ./go-qcow2reader-example map /var/tmp/images/test.zlib.qcow2 | head -5
    {"start":0,"length":65536,"allocated":true,"zero":false,"compressed":true}
    {"start":65536,"length":983040,"allocated":false,"zero":true,"compressed":false}
    {"start":1048576,"length":131072,"allocated":true,"zero":false,"compressed":true}
    {"start":1179648,"length":655360,"allocated":false,"zero":true,"compressed":false}
    {"start":1835008,"length":131072,"allocated":true,"zero":false,"compressed":true}

Signed-off-by: Nir Soffer <nirsof@gmail.com>
---
 cmd/go-qcow2reader-example/main.go |  5 +-
 cmd/go-qcow2reader-example/map.go  | 73 ++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 cmd/go-qcow2reader-example/map.go

diff --git a/cmd/go-qcow2reader-example/main.go b/cmd/go-qcow2reader-example/main.go
index 59ef338..435d8aa 100644
--- a/cmd/go-qcow2reader-example/main.go
+++ b/cmd/go-qcow2reader-example/main.go
@@ -41,7 +41,8 @@ func usage() {
 Available commands:
   info		show image information
   read		read image data and print to stdout
-  convert      convert image to raw format
+  convert	convert image to raw format
+  map		print image extents
 `
 	fmt.Fprintf(os.Stderr, usage, os.Args[0])
 	os.Exit(1)
@@ -71,6 +72,8 @@ func main() {
 		err = cmdRead(args)
 	case "convert":
 		err = cmdConvert(args)
+	case "map":
+		err = cmdMap(args)
 	default:
 		usage()
 	}
diff --git a/cmd/go-qcow2reader-example/map.go b/cmd/go-qcow2reader-example/map.go
new file mode 100644
index 0000000..bd6bd0b
--- /dev/null
+++ b/cmd/go-qcow2reader-example/map.go
@@ -0,0 +1,73 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/lima-vm/go-qcow2reader"
+	"github.com/lima-vm/go-qcow2reader/log"
+)
+
+func cmdMap(args []string) error {
+	var (
+		// Required
+		filename string
+
+		// Options
+		debug bool
+	)
+
+	fs := flag.NewFlagSet("map", flag.ExitOnError)
+	fs.Usage = func() {
+		fmt.Fprintf(fs.Output(), "Usage: %s map [OPTIONS...] FILE\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+	fs.BoolVar(&debug, "debug", false, "enable printing debug messages")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	if debug {
+		log.SetDebugFunc(logDebug)
+	}
+
+	switch len(fs.Args()) {
+	case 0:
+		return errors.New("no file was specified")
+	case 1:
+		filename = fs.Arg(0)
+	default:
+		return errors.New("too many files were specified")
+	}
+
+	f, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	img, err := qcow2reader.Open(f)
+	if err != nil {
+		return err
+	}
+	defer img.Close()
+
+	writer := bufio.NewWriter(os.Stdout)
+	encoder := json.NewEncoder(writer)
+
+	var start int64
+	end := img.Size()
+	for start < end {
+		extent, err := img.Extent(start, end-start)
+		if err != nil {
+			return err
+		}
+		encoder.Encode(extent)
+		start += extent.Length
+	}
+	return writer.Flush()
+}

From f7d86768cf8f14e70ad72a800972a434e0e5e4f2 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sun, 27 Oct 2024 01:04:18 +0200
Subject: [PATCH 7/9] Use Image.Extent() in converter
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Using Image.Extent() to skip zero extents instead of reading and
detecting zeros. We still detect zeroes in non-zero extents to convert
areas full of actual zeros to unallocated area in the target image.

This does not change much the performance for the Ubuntu compressed
image since most of the time is spent on reading and decompressing the
actual data.  Converting a large empty image is 2 orders of magnitude
faster, so I'm testing now 1 TiB image instead of 100 GiB image.

Example run with 1 TiB empty image:

    % hyperfine -w3 "qemu-img convert -f qcow2 -O raw -W /tmp/images/test.0p.qcow2 /tmp/tmp.img" \
                    "./go-qcow2reader-example convert /tmp/images/test.0p.qcow2 /tmp/tmp.img"
    Benchmark 1: qemu-img convert -f qcow2 -O raw -W /tmp/images/test.0p.qcow2 /tmp/tmp.img
      Time (mean ± σ):      14.0 ms ±   0.4 ms    [User: 11.8 ms, System: 2.0 ms]
      Range (min … max):    13.5 ms …  17.8 ms    181 runs

    Benchmark 2: ./go-qcow2reader-example convert /tmp/images/test.0p.qcow2 /tmp/tmp.img
      Time (mean ± σ):      20.6 ms ±   0.2 ms    [User: 118.9 ms, System: 2.2 ms]
      Range (min … max):    20.4 ms …  21.7 ms    130 runs

    Summary
      qemu-img convert -f qcow2 -O raw -W /tmp/images/test.0p.qcow2 /tmp/tmp.img ran
        1.48 ± 0.04 times faster than ./go-qcow2reader-example convert /tmp/images/test.0p.qcow2 /tmp/tmp.img

qemu-img is faster but 7 millisecond difference for 1 TiB image is not
very interesting.

Signed-off-by: Nir Soffer <nirsof@gmail.com>
---
 convert/convert.go | 96 ++++++++++++++++++++++++++++++----------------
 1 file changed, 63 insertions(+), 33 deletions(-)

diff --git a/convert/convert.go b/convert/convert.go
index 8cfef34..cb21448 100644
--- a/convert/convert.go
+++ b/convert/convert.go
@@ -6,18 +6,29 @@ import (
 	"fmt"
 	"io"
 	"sync"
+
+	"github.com/lima-vm/go-qcow2reader/image"
 )
 
+// The size of the buffer used to read data from non-zero extents of the image.
+// For best performance, the size should be aligned to the image cluster size or
+// the file system block size.
 const BufferSize = 1024 * 1024
 
-// Smaller value may increase the overhead of synchornizing multiple works.
+// To maxmimze performance we use multiple goroutines to read data from the
+// source image, decompress data, and write data to the target image. To
+// schedule the work to multiple goroutines, the image is split to multiple
+// segments, each processed by a single worker goroutine.
+//
+// Smaller value may increase the overhead of synchornizing multiple workers.
 // Larger value may be less efficient for smaller images. The default value
-// gives good results for the lima default Ubuntu image.
+// gives good results for the lima default Ubuntu image. Must be aligned to
+// BufferSize.
 const SegmentSize = 32 * BufferSize
 
 // For best I/O throughput we want to have enough in-flight requests, regardless
 // of number of cores. For best decompression we want to use one worker per
-// core, but too many workers is less effective. The default value gives good
+// core, but too many workers are less effective. The default value gives good
 // results with lima default Ubuntu image.
 const Workers = 8
 
@@ -129,10 +140,12 @@ func (c *Converter) reset(size int64) {
 	c.offset = 0
 }
 
-// Convert copy size bytes from io.ReaderAt to io.WriterAt. Unallocated areas or
-// areas full of zeros in the source are keep unallocated in the destination.
-// The destination must be new empty or full of zeroes.
-func (c *Converter) Convert(wa io.WriterAt, ra io.ReaderAt, size int64) error {
+// Convert copy size bytes from image to io.WriterAt. Unallocated extents in the
+// source image or read data which is all zeros are converted to unallocated
+// byte range in the target image. The target image must be new empty file or a
+// file full of zeroes. To get a sparse target image, the image must be a new
+// empty file, since Convert does not punch holes for zero ranges even if the
+// underlying file system supports hole punching.
 	c.reset(size)
 
 	zero := make([]byte, c.bufferSize)
@@ -151,40 +164,57 @@ func (c *Converter) Convert(wa io.WriterAt, ra io.ReaderAt, size int64) error {
 				}
 
 				for start < end {
-					// The last read may be shorter.
-					n := len(buf)
-					if end-start < int64(len(buf)) {
-						n = int(end - start)
+					// Get next extent in this segment.
+					extent, err := img.Extent(start, end-start)
+					if err != nil {
+						c.setError(err)
+						return
+					}
+					if extent.Zero {
+						start += extent.Length
+						continue
 					}
 
-					// Read more data.
-					nr, err := ra.ReadAt(buf[:n], start)
-					if err != nil {
-						if !errors.Is(err, io.EOF) {
-							c.setError(err)
-							return
+					// Consume data from this extent.
+					for extent.Length > 0 {
+						// The last read may be shorter.
+						n := len(buf)
+						if extent.Length < int64(len(buf)) {
+							n = int(extent.Length)
 						}
 
-						// EOF for the last read of the last segment is expected, but since we
-						// read exactly size bytes, we should never get a zero read.
-						if nr == 0 {
-							c.setError(errors.New("unexpected EOF"))
-							return
+						// Read more data.
+						nr, err := img.ReadAt(buf[:n], start)
+						if err != nil {
+							if !errors.Is(err, io.EOF) {
+								c.setError(err)
+								return
+							}
+
+							// EOF for the last read of the last segment is expected, but since we
+							// read exactly size bytes, we should never get a zero read.
+							if nr == 0 {
+								c.setError(errors.New("unexpected EOF"))
+								return
+							}
 						}
-					}
 
-					// If the data is all zeros we skip it to create a hole. Otherwise
-					// write the data.
-					if !bytes.Equal(buf[:nr], zero[:nr]) {
-						if nw, err := wa.WriteAt(buf[:nr], start); err != nil {
-							c.setError(err)
-							return
-						} else if nw != nr {
-							c.setError(fmt.Errorf("read %d, but wrote %d bytes", nr, nw))
-							return
+						// If the data is all zeros we skip it to create a hole. Otherwise
+						// write the data.
+						if !bytes.Equal(buf[:nr], zero[:nr]) {
+							if nw, err := wa.WriteAt(buf[:nr], start); err != nil {
+								c.setError(err)
+								return
+							} else if nw != nr {
+								c.setError(fmt.Errorf("read %d, but wrote %d bytes", nr, nw))
+								return
+							}
 						}
+
+						extent.Length -= int64(nr)
+						extent.Start += int64(nr)
+						start += int64(nr)
 					}
-					start += int64(nr)
 				}
 			}
 		}()

From c3792ed650458ffe93e684c35ec3ba647f9b1785 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sun, 17 Nov 2024 21:57:26 +0200
Subject: [PATCH 8/9] Support convert progress

Previously we always read the entire image so we could report progress
by wrapping the ReaderAt so every read updates the progress indicator.
Now we read only the allocated extents, so we need another way.

To report progress, you need to pass now a type implementing the Updater
interface. We could also use a function but this way seems cleaner.

Signed-off-by: Nir Soffer <nirsof@gmail.com>
---
 cmd/go-qcow2reader-example/convert.go |  2 +-
 convert/convert.go                    | 16 ++++++++++++++++
 qcow2reader_test.go                   |  2 +-
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/cmd/go-qcow2reader-example/convert.go b/cmd/go-qcow2reader-example/convert.go
index 863c308..81584b1 100644
--- a/cmd/go-qcow2reader-example/convert.go
+++ b/cmd/go-qcow2reader-example/convert.go
@@ -76,7 +76,7 @@ func cmdConvert(args []string) error {
 	if err != nil {
 		return err
 	}
-	if err := c.Convert(t, img, img.Size()); err != nil {
+	if err := c.Convert(t, img, img.Size(), nil); err != nil {
 		return err
 	}
 
diff --git a/convert/convert.go b/convert/convert.go
index cb21448..ae770ed 100644
--- a/convert/convert.go
+++ b/convert/convert.go
@@ -78,6 +78,14 @@ func (o *Options) Validate() error {
 	return nil
 }
 
+// Updater is an interface for tracking conversion progress.
+type Updater interface {
+	// Called from multiple goroutines after a byte range of length was converted.
+	// If the conversion is successfu, the total number of bytes will be the image
+	// virtual size.
+	Update(n int64)
+}
+
 type Converter struct {
 	// Read only after starting.
 	size        int64
@@ -146,6 +154,7 @@ func (c *Converter) reset(size int64) {
 // file full of zeroes. To get a sparse target image, the image must be a new
 // empty file, since Convert does not punch holes for zero ranges even if the
 // underlying file system supports hole punching.
+func (c *Converter) Convert(wa io.WriterAt, img image.Image, size int64, progress Updater) error {
 	c.reset(size)
 
 	zero := make([]byte, c.bufferSize)
@@ -172,6 +181,9 @@ func (c *Converter) reset(size int64) {
 					}
 					if extent.Zero {
 						start += extent.Length
+						if progress != nil {
+							progress.Update(extent.Length)
+						}
 						continue
 					}
 
@@ -211,6 +223,10 @@ func (c *Converter) reset(size int64) {
 							}
 						}
 
+						if progress != nil {
+							progress.Update(int64(nr))
+						}
+
 						extent.Length -= int64(nr)
 						extent.Start += int64(nr)
 						start += int64(nr)
diff --git a/qcow2reader_test.go b/qcow2reader_test.go
index 1f1d0aa..0bb60ee 100644
--- a/qcow2reader_test.go
+++ b/qcow2reader_test.go
@@ -809,7 +809,7 @@ func benchmarkConvert(b *testing.B, filename string) {
 	if err != nil {
 		b.Fatal(err)
 	}
-	err = c.Convert(dst, img, img.Size())
+	err = c.Convert(dst, img, img.Size(), nil)
 	if err != nil {
 		b.Fatal(err)
 	}

From 82009e9a91011b3c5a7ac804e667a9575522fa4c Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sun, 17 Nov 2024 23:56:54 +0200
Subject: [PATCH 9/9] Add progress bar to example program

Most users will want to integrate a progress bar and this serve as a
good example of how to do this. This also makes it more fun to work on
this project.

Ubuntu 24.04 image in qcow2 compressed format:

    % ./go-qcow2reader-example convert /var/tmp/images/test.zlib.qcow2 /tmp/tmp.img
    6.00 GiB / 6.00 GiB [-------------------------------------] 100.00% 2.86 GiB p/s

Ubuntu 24.04 image in qcow2 format:

    % ./go-qcow2reader-example convert /var/tmp/images/test.qcow2 /tmp/tmp.img
    6.00 GiB / 6.00 GiB [------------------------------------] 100.00% 15.73 GiB p/s

16 TiB empty qcow2 image:

    % ./go-qcow2reader-example convert /var/tmp/images/test.0p.qcow2 /tmp/tmp.img
    16.00 TiB / 16.00 TiB [---------------------------------] 100.00% 152.46 TiB p/s

The pb module brings too many dependencies. Maybe we can find another
library or write a very simple progress instead.

Signed-off-by: Nir Soffer <nirsof@gmail.com>
---
 cmd/go-qcow2reader-example/convert.go | 23 ++++++++++++++++++++++-
 cmd/go-qcow2reader-example/go.mod     | 11 +++++++++++
 cmd/go-qcow2reader-example/go.sum     | 18 ++++++++++++++++++
 3 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/cmd/go-qcow2reader-example/convert.go b/cmd/go-qcow2reader-example/convert.go
index 81584b1..590193b 100644
--- a/cmd/go-qcow2reader-example/convert.go
+++ b/cmd/go-qcow2reader-example/convert.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/cheggaaa/pb/v3"
 	"github.com/lima-vm/go-qcow2reader"
 	"github.com/lima-vm/go-qcow2reader/convert"
 	"github.com/lima-vm/go-qcow2reader/log"
@@ -76,7 +77,12 @@ func cmdConvert(args []string) error {
 	if err != nil {
 		return err
 	}
-	if err := c.Convert(t, img, img.Size(), nil); err != nil {
+
+	bar := newProgressBar(img.Size())
+	bar.Start()
+	defer bar.Finish()
+
+	if err := c.Convert(t, img, img.Size(), bar); err != nil {
 		return err
 	}
 
@@ -86,3 +92,18 @@ func cmdConvert(args []string) error {
 
 	return t.Close()
 }
+
+// progressBar adapts pb.ProgressBar to the Updater interface.
+type progressBar struct {
+	*pb.ProgressBar
+}
+
+func newProgressBar(size int64) *progressBar {
+	b := &progressBar{pb.New64(size)}
+	b.Set(pb.Bytes, true)
+	return b
+}
+
+func (b *progressBar) Update(n int64) {
+	b.ProgressBar.Add64(n)
+}
diff --git a/cmd/go-qcow2reader-example/go.mod b/cmd/go-qcow2reader-example/go.mod
index 1f10b56..e4d709a 100644
--- a/cmd/go-qcow2reader-example/go.mod
+++ b/cmd/go-qcow2reader-example/go.mod
@@ -3,8 +3,19 @@ module github.com/lima-vm/go-qcow2reader/cmd/go-qcow2reader-example
 go 1.22
 
 require (
+	github.com/cheggaaa/pb/v3 v3.1.5
 	github.com/klauspost/compress v1.16.5
 	github.com/lima-vm/go-qcow2reader v0.0.0-00010101000000-000000000000
 )
 
+require (
+	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/fatih/color v1.15.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+)
+
 replace github.com/lima-vm/go-qcow2reader => ../../
diff --git a/cmd/go-qcow2reader-example/go.sum b/cmd/go-qcow2reader-example/go.sum
index 1a08641..59daea5 100644
--- a/cmd/go-qcow2reader-example/go.sum
+++ b/cmd/go-qcow2reader-example/go.sum
@@ -1,2 +1,20 @@
+github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
+github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
+github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk=
+github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
 github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=