diff --git a/cmd/go-qcow2reader-example/convert.go b/cmd/go-qcow2reader-example/convert.go index 863c308..590193b 100644 --- a/cmd/go-qcow2reader-example/convert.go +++ b/cmd/go-qcow2reader-example/convert.go @@ -6,6 +6,7 @@ import ( "fmt" "os" + "github.com/cheggaaa/pb/v3" "github.com/lima-vm/go-qcow2reader" "github.com/lima-vm/go-qcow2reader/convert" "github.com/lima-vm/go-qcow2reader/log" @@ -76,7 +77,12 @@ func cmdConvert(args []string) error { if err != nil { return err } - if err := c.Convert(t, img, img.Size()); err != nil { + + bar := newProgressBar(img.Size()) + bar.Start() + defer bar.Finish() + + if err := c.Convert(t, img, img.Size(), bar); err != nil { return err } @@ -86,3 +92,18 @@ func cmdConvert(args []string) error { return t.Close() } + +// progressBar adapts pb.ProgressBar to the Updater interface. +type progressBar struct { + *pb.ProgressBar +} + +func newProgressBar(size int64) *progressBar { + b := &progressBar{pb.New64(size)} + b.Set(pb.Bytes, true) + return b +} + +func (b *progressBar) Update(n int64) { + b.ProgressBar.Add64(n) +} diff --git a/cmd/go-qcow2reader-example/go.mod b/cmd/go-qcow2reader-example/go.mod index 1f10b56..e4d709a 100644 --- a/cmd/go-qcow2reader-example/go.mod +++ b/cmd/go-qcow2reader-example/go.mod @@ -3,8 +3,19 @@ module github.com/lima-vm/go-qcow2reader/cmd/go-qcow2reader-example go 1.22 require ( + github.com/cheggaaa/pb/v3 v3.1.5 github.com/klauspost/compress v1.16.5 github.com/lima-vm/go-qcow2reader v0.0.0-00010101000000-000000000000 ) +require ( + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + golang.org/x/sys v0.6.0 // indirect +) + replace github.com/lima-vm/go-qcow2reader => ../../ diff --git a/cmd/go-qcow2reader-example/go.sum b/cmd/go-qcow2reader-example/go.sum index 1a08641..59daea5 100644 --- a/cmd/go-qcow2reader-example/go.sum +++ b/cmd/go-qcow2reader-example/go.sum @@ -1,2 +1,20 @@ +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk= +github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/cmd/go-qcow2reader-example/main.go b/cmd/go-qcow2reader-example/main.go index 59ef338..435d8aa 100644 --- a/cmd/go-qcow2reader-example/main.go +++ b/cmd/go-qcow2reader-example/main.go @@ -41,7 +41,8 @@ func usage() { Available commands: info show image information read read image data and print to stdout - convert convert image to raw format + convert convert image to raw format + map print image extents ` fmt.Fprintf(os.Stderr, usage, os.Args[0]) os.Exit(1) @@ -71,6 +72,8 @@ func main() { err = cmdRead(args) case "convert": err = cmdConvert(args) + case "map": + err = cmdMap(args) default: usage() } diff --git a/cmd/go-qcow2reader-example/map.go b/cmd/go-qcow2reader-example/map.go new file mode 100644 index 0000000..bd6bd0b --- /dev/null +++ b/cmd/go-qcow2reader-example/map.go @@ -0,0 +1,73 @@ +package main + +import ( + "bufio" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + + "github.com/lima-vm/go-qcow2reader" + "github.com/lima-vm/go-qcow2reader/log" +) + +func cmdMap(args []string) error { + var ( + // Required + filename string + + // Options + debug bool + ) + + fs := flag.NewFlagSet("map", flag.ExitOnError) + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "Usage: %s map [OPTIONS...] FILE\n", os.Args[0]) + flag.PrintDefaults() + } + fs.BoolVar(&debug, "debug", false, "enable printing debug messages") + if err := fs.Parse(args); err != nil { + return err + } + + if debug { + log.SetDebugFunc(logDebug) + } + + switch len(fs.Args()) { + case 0: + return errors.New("no file was specified") + case 1: + filename = fs.Arg(0) + default: + return errors.New("too many files were specified") + } + + f, err := os.Open(filename) + if err != nil { + return err + } + defer f.Close() + + img, err := qcow2reader.Open(f) + if err != nil { + return err + } + defer img.Close() + + writer := bufio.NewWriter(os.Stdout) + encoder := json.NewEncoder(writer) + + var start int64 + end := img.Size() + for start < end { + extent, err := img.Extent(start, end-start) + if err != nil { + return err + } + encoder.Encode(extent) + start += extent.Length + } + return writer.Flush() +} diff --git a/convert/convert.go b/convert/convert.go index 8cfef34..ae770ed 100644 --- a/convert/convert.go +++ b/convert/convert.go @@ -6,18 +6,29 @@ import ( "fmt" "io" "sync" + + "github.com/lima-vm/go-qcow2reader/image" ) +// The size of the buffer used to read data from non-zero extents of the image. +// For best performance, the size should be aligned to the image cluster size or +// the file system block size. const BufferSize = 1024 * 1024 -// Smaller value may increase the overhead of synchornizing multiple works. +// To maxmimze performance we use multiple goroutines to read data from the +// source image, decompress data, and write data to the target image. To +// schedule the work to multiple goroutines, the image is split to multiple +// segments, each processed by a single worker goroutine. +// +// Smaller value may increase the overhead of synchornizing multiple workers. // Larger value may be less efficient for smaller images. The default value -// gives good results for the lima default Ubuntu image. +// gives good results for the lima default Ubuntu image. Must be aligned to +// BufferSize. const SegmentSize = 32 * BufferSize // For best I/O throughput we want to have enough in-flight requests, regardless // of number of cores. For best decompression we want to use one worker per -// core, but too many workers is less effective. The default value gives good +// core, but too many workers are less effective. The default value gives good // results with lima default Ubuntu image. const Workers = 8 @@ -67,6 +78,14 @@ func (o *Options) Validate() error { return nil } +// Updater is an interface for tracking conversion progress. +type Updater interface { + // Called from multiple goroutines after a byte range of length was converted. + // If the conversion is successfu, the total number of bytes will be the image + // virtual size. + Update(n int64) +} + type Converter struct { // Read only after starting. size int64 @@ -129,10 +148,13 @@ func (c *Converter) reset(size int64) { c.offset = 0 } -// Convert copy size bytes from io.ReaderAt to io.WriterAt. Unallocated areas or -// areas full of zeros in the source are keep unallocated in the destination. -// The destination must be new empty or full of zeroes. -func (c *Converter) Convert(wa io.WriterAt, ra io.ReaderAt, size int64) error { +// Convert copy size bytes from image to io.WriterAt. Unallocated extents in the +// source image or read data which is all zeros are converted to unallocated +// byte range in the target image. The target image must be new empty file or a +// file full of zeroes. To get a sparse target image, the image must be a new +// empty file, since Convert does not punch holes for zero ranges even if the +// underlying file system supports hole punching. +func (c *Converter) Convert(wa io.WriterAt, img image.Image, size int64, progress Updater) error { c.reset(size) zero := make([]byte, c.bufferSize) @@ -151,40 +173,64 @@ func (c *Converter) Convert(wa io.WriterAt, ra io.ReaderAt, size int64) error { } for start < end { - // The last read may be shorter. - n := len(buf) - if end-start < int64(len(buf)) { - n = int(end - start) + // Get next extent in this segment. + extent, err := img.Extent(start, end-start) + if err != nil { + c.setError(err) + return + } + if extent.Zero { + start += extent.Length + if progress != nil { + progress.Update(extent.Length) + } + continue } - // Read more data. - nr, err := ra.ReadAt(buf[:n], start) - if err != nil { - if !errors.Is(err, io.EOF) { - c.setError(err) - return + // Consume data from this extent. + for extent.Length > 0 { + // The last read may be shorter. + n := len(buf) + if extent.Length < int64(len(buf)) { + n = int(extent.Length) } - // EOF for the last read of the last segment is expected, but since we - // read exactly size bytes, we should never get a zero read. - if nr == 0 { - c.setError(errors.New("unexpected EOF")) - return + // Read more data. + nr, err := img.ReadAt(buf[:n], start) + if err != nil { + if !errors.Is(err, io.EOF) { + c.setError(err) + return + } + + // EOF for the last read of the last segment is expected, but since we + // read exactly size bytes, we should never get a zero read. + if nr == 0 { + c.setError(errors.New("unexpected EOF")) + return + } } - } - // If the data is all zeros we skip it to create a hole. Otherwise - // write the data. - if !bytes.Equal(buf[:nr], zero[:nr]) { - if nw, err := wa.WriteAt(buf[:nr], start); err != nil { - c.setError(err) - return - } else if nw != nr { - c.setError(fmt.Errorf("read %d, but wrote %d bytes", nr, nw)) - return + // If the data is all zeros we skip it to create a hole. Otherwise + // write the data. + if !bytes.Equal(buf[:nr], zero[:nr]) { + if nw, err := wa.WriteAt(buf[:nr], start); err != nil { + c.setError(err) + return + } else if nw != nr { + c.setError(fmt.Errorf("read %d, but wrote %d bytes", nr, nw)) + return + } } + + if progress != nil { + progress.Update(int64(nr)) + } + + extent.Length -= int64(nr) + extent.Start += int64(nr) + start += int64(nr) } - start += int64(nr) } } }() diff --git a/image/image.go b/image/image.go index 7ea497e..e7d1e52 100644 --- a/image/image.go +++ b/image/image.go @@ -8,10 +8,28 @@ import ( // Type must be a "Backing file format name string" that appears in QCOW2. type Type string +// Extent describes a byte range in the image with the same allocation, +// compression, or zero status. Extents are aligned to the underlying file +// system block size (e.g. 4k), or the image format cluster size (e.g. 64k). One +// extent can describe one or more file system blocks or image clusters. +type Extent struct { + // Offset from start of the image in bytes. + Start int64 `json:"start"` + // Length of this extent in bytes. + Length int64 `json:"length"` + // Set if this extent is allocated. + Allocated bool `json:"allocated"` + // Set if this extent is read as zeros. + Zero bool `json:"zero"` + // Set if this extent is compressed. + Compressed bool `json:"compressed"` +} + // Image implements [io.ReaderAt] and [io.Closer]. type Image interface { io.ReaderAt io.Closer + Extent(start, length int64) (Extent, error) Type() Type Size() int64 // -1 if unknown Readable() error diff --git a/image/qcow2/qcow2.go b/image/qcow2/qcow2.go index 8ee3370..582baf0 100644 --- a/image/qcow2/qcow2.go +++ b/image/qcow2/qcow2.go @@ -562,7 +562,7 @@ func Open(ra io.ReaderAt, openWithType image.OpenWithType) (*Qcow2, error) { var err error img.Header, err = readHeader(r) if err != nil { - return nil, fmt.Errorf("faild to read header: %w", err) + return nil, fmt.Errorf("failed to read header: %w", err) } img.errUnreadable = img.Header.Readable() // cache if img.errUnreadable == nil { @@ -596,7 +596,7 @@ func Open(ra io.ReaderAt, openWithType image.OpenWithType) (*Qcow2, error) { // Load L1 table img.l1Table, err = readL1Table(ra, img.Header.L1TableOffset, img.Header.L1Size) if err != nil { - return img, fmt.Errorf("faild to read L1 table: %w", err) + return img, fmt.Errorf("failed to read L1 table: %w", err) } // Load decompressor @@ -776,6 +776,9 @@ func (img *Qcow2) getClusterMeta(off int64, cm *clusterMeta) error { if cm.L2Entry.compressed() { cm.Compressed = true } else { + // When using extended L2 clusters this is always false. To find which sub + // cluster is allocated/zero we need to iterate over the allocation bitmap in + // the extended l2 cluster entry. cm.Zero = standardClusterDescriptor(desc).allZero() } @@ -966,6 +969,110 @@ func readZero(p []byte, off int64, sz uint64) (int, error) { return l, err } +// clusterStatus returns an extent describing a single cluster. off must be aligned to +// cluster size. +func (img *Qcow2) clusterStatus(off int64) (image.Extent, error) { + var cm clusterMeta + if err := img.getClusterMeta(off, &cm); err != nil { + return image.Extent{}, err + } + + if !cm.Allocated { + // If there is no backing file, or the cluster cannot be in the backing file, + // return an unallocated cluster. + if img.backingImage == nil || off >= img.backingImage.Size() { + // Unallocated cluster reads as zeros. + unallocated := image.Extent{Start: off, Length: int64(img.clusterSize), Zero: true} + return unallocated, nil + } + + // Get the cluster from the backing file. + length := int64(img.clusterSize) + if off+length > img.backingImage.Size() { + length = img.backingImage.Size() - off + } + parent, err := img.backingImage.Extent(off, length) + if err != nil { + return parent, err + } + // The backing image may be a raw image not aligned to cluster size. + parent.Length = int64(img.clusterSize) + return parent, nil + } + + // Cluster present in this image. + allocated := image.Extent{ + Start: off, + Length: int64(img.clusterSize), + Allocated: true, + Compressed: cm.Compressed, + Zero: cm.Zero, + } + return allocated, nil +} + +// Return true if extents have the same status. +func sameStatus(a, b image.Extent) bool { + return a.Allocated == b.Allocated && a.Zero == b.Zero && a.Compressed == b.Compressed +} + +// Extent returns the next extent starting at the specified offset. An extent +// describes one or more clusters having the same status. The maximum length of +// the returned extent is limited by the specified length. The minimum length of +// the returned extent is length of one cluster. +func (img *Qcow2) Extent(start, length int64) (image.Extent, error) { + // Default to zero length non-existent cluster. + var current image.Extent + + if img.errUnreadable != nil { + return current, img.errUnreadable + } + if img.clusterSize == 0 { + return current, errors.New("cluster size cannot be 0") + } + if start+length > int64(img.Header.Size) { + return current, errors.New("length out of bounds") + } + + // Compute the clusterStart of the first cluster to query. This may be behind start. + clusterStart := start / int64(img.clusterSize) * int64(img.clusterSize) + + remaining := length + for remaining > 0 { + clusterStatus, err := img.clusterStatus(clusterStart) + if err != nil { + return current, err + } + + // First cluster: if start is not aligned to cluster size, clip the start. + if clusterStatus.Start < start { + clusterStatus.Start = start + clusterStatus.Length -= start - clusterStatus.Start + } + + // Last cluster: if start+length is not aligned to cluster size, clip the end. + if remaining < int64(img.clusterSize) { + clusterStatus.Length -= int64(img.clusterSize) - remaining + } + + if current.Length == 0 { + // First cluster: copy status to current. + current = clusterStatus + } else if sameStatus(current, clusterStatus) { + // Cluster with same status: extend current. + current.Length += clusterStatus.Length + } else { + // Start of next extent + break + } + + clusterStart += int64(img.clusterSize) + remaining -= clusterStatus.Length + } + + return current, nil +} + // ReadAt implements [io.ReaderAt]. func (img *Qcow2) ReadAt(p []byte, off int64) (n int, err error) { if img.errUnreadable != nil { diff --git a/image/raw/raw.go b/image/raw/raw.go index e0b7fcb..4d59fab 100644 --- a/image/raw/raw.go +++ b/image/raw/raw.go @@ -1,6 +1,7 @@ package raw import ( + "errors" "io" "os" @@ -14,6 +15,18 @@ type Raw struct { io.ReaderAt `json:"-"` } +// Extent returns an allocated extent starting at the specified offset with +// specified length. It is used when the speicfic image type does not implement +// Extent(). The implementation is correct but inefficient. Fails if image size +// is unknown. +func (img *Raw) Extent(start, length int64) (image.Extent, error) { + if start+length > img.Size() { + return image.Extent{}, errors.New("length out of bounds") + } + // TODO: Implement using SEEK_HOLE/SEEK_DATA when supported by the file system. + return image.Extent{Start: start, Length: length, Allocated: true}, nil +} + func (img *Raw) Close() error { if closer, ok := img.ReaderAt.(io.Closer); ok { return closer.Close() diff --git a/image/stub/stub.go b/image/stub/stub.go index 103fb23..a595b35 100644 --- a/image/stub/stub.go +++ b/image/stub/stub.go @@ -25,6 +25,10 @@ type Stub struct { t image.Type } +func (img *Stub) Extent(start, length int64) (image.Extent, error) { + return image.Extent{}, fmt.Errorf("unimplemented type: %q", img.t) +} + func (img *Stub) ReadAt([]byte, int64) (int, error) { return 0, img.Readable() } diff --git a/qcow2reader_test.go b/qcow2reader_test.go index ec8764c..0bb60ee 100644 --- a/qcow2reader_test.go +++ b/qcow2reader_test.go @@ -7,18 +7,619 @@ import ( "math/rand" "os" "path/filepath" + "slices" "testing" "github.com/lima-vm/go-qcow2reader" "github.com/lima-vm/go-qcow2reader/convert" + "github.com/lima-vm/go-qcow2reader/image" "github.com/lima-vm/go-qcow2reader/test/qemuimg" + "github.com/lima-vm/go-qcow2reader/test/qemuio" ) const ( - MiB = int64(1) << 20 - GiB = int64(1) << 30 + KiB = int64(1) << 10 + MiB = int64(1) << 20 + GiB = int64(1) << 30 + clusterSize = 64 * KiB ) +func TestExtentsUnallocated(t *testing.T) { + path := filepath.Join(t.TempDir(), "image") + if err := qemuimg.Create(path, qemuimg.FormatQcow2, 4*GiB, "", ""); err != nil { + t.Fatal(err) + } + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + img, err := qcow2reader.Open(f) + if err != nil { + t.Fatal(err) + } + defer img.Close() + + t.Run("entire image", func(t *testing.T) { + actual, err := img.Extent(0, img.Size()) + if err != nil { + t.Fatal(err) + } + expected := image.Extent{Start: 0, Length: img.Size(), Zero: true} + if actual != expected { + t.Fatalf("expected %+v, got %+v", expected, actual) + } + }) + t.Run("same result", func(t *testing.T) { + r1, err := img.Extent(0, img.Size()) + if err != nil { + t.Fatal(err) + } + r2, err := img.Extent(0, img.Size()) + if err != nil { + t.Fatal(err) + } + if r1 != r2 { + t.Fatalf("expected %+v, got %+v", r1, r2) + } + }) + t.Run("all segments", func(t *testing.T) { + for i := int64(0); i < img.Size(); i += 32 * MiB { + segment, err := img.Extent(i, 32*MiB) + if err != nil { + t.Fatal(err) + } + expected := image.Extent{Start: i, Length: 32 * MiB, Zero: true} + if segment != expected { + t.Fatalf("expected %+v, got %+v", expected, segment) + } + } + }) + t.Run("start unaligned", func(t *testing.T) { + start := 32*MiB + 42 + length := 32 * MiB + actual, err := img.Extent(start, length) + if err != nil { + t.Fatal(err) + } + expected := image.Extent{Start: start, Length: length, Zero: true} + if actual != expected { + t.Fatalf("expected %+v, got %+v", expected, actual) + } + }) + t.Run("length unaligned", func(t *testing.T) { + start := 32 * MiB + length := 32*MiB - 42 + actual, err := img.Extent(start, length) + if err != nil { + t.Fatal(err) + } + expected := image.Extent{Start: start, Length: length, Zero: true} + if actual != expected { + t.Fatalf("expected %+v, got %+v", expected, actual) + } + }) + t.Run("start and length unaligned", func(t *testing.T) { + start := 32*MiB + 42 + length := 32*MiB - 42 + actual, err := img.Extent(start, length) + if err != nil { + t.Fatal(err) + } + expected := image.Extent{Start: start, Length: length, Zero: true} + if actual != expected { + t.Fatalf("expected %+v, got %+v", expected, actual) + } + }) + t.Run("length after end of image", func(t *testing.T) { + start := img.Size() - 31*MiB + actual, err := img.Extent(start, 32*MiB) + if err == nil { + t.Fatal("out of bounds request did not fail") + } + var expected image.Extent + if actual != expected { + t.Fatalf("expected %+v, got %+v", expected, actual) + } + }) + t.Run("start after end of image", func(t *testing.T) { + start := img.Size() + 1*MiB + actual, err := img.Extent(start, 32*MiB) + if err == nil { + t.Fatal("out of bounds request did not fail") + } + var expected image.Extent + if actual != expected { + t.Fatalf("expected %+v, got %+v", expected, actual) + } + }) +} + +func TestExtentsRaw(t *testing.T) { + path := filepath.Join(t.TempDir(), "disk.img") + size := 4 * GiB + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if err := f.Truncate(size); err != nil { + t.Fatal(err) + } + img, err := qcow2reader.Open(f) + if err != nil { + t.Fatal(err) + } + defer img.Close() + + t.Run("entire image", func(t *testing.T) { + actual, err := img.Extent(0, img.Size()) + if err != nil { + t.Fatal(err) + } + // Currently we always report raw images as fully allocated. + expected := image.Extent{Start: 0, Length: img.Size(), Allocated: true} + if actual != expected { + t.Fatalf("expected %+v, got %+v", expected, actual) + } + }) + t.Run("length after end of image", func(t *testing.T) { + start := img.Size() - 31*MiB + actual, err := img.Extent(start, 32*MiB) + if err == nil { + t.Fatal("out of bounds request did not fail") + } + var expected image.Extent + if actual != expected { + t.Fatalf("expected %+v, got %+v", expected, actual) + } + }) +} + +func BenchmarkExtentsUnallocated(b *testing.B) { + path := filepath.Join(b.TempDir(), "image") + if err := qemuimg.Create(path, qemuimg.FormatQcow2, 100*GiB, "", ""); err != nil { + b.Fatal(err) + } + f, err := os.Open(path) + if err != nil { + b.Fatal(err) + } + img, err := qcow2reader.Open(f) + if err != nil { + b.Fatal(err) + } + expected := image.Extent{Start: 0, Length: img.Size(), Zero: true} + resetBenchmark(b, img.Size()) + for i := 0; i < b.N; i++ { + b.StartTimer() + actual, err := img.Extent(0, img.Size()) + b.StopTimer() + if err != nil { + b.Fatal(err) + } + if actual != expected { + b.Fatalf("expected %+v, got %+v", expected, actual) + } + } +} + +func TestExtentsSome(t *testing.T) { + extents := []image.Extent{ + {Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 1 * clusterSize, Length: 1 * clusterSize, Zero: true}, + {Start: 2 * clusterSize, Length: 2 * clusterSize, Allocated: true}, + {Start: 4 * clusterSize, Length: 96 * clusterSize, Zero: true}, + {Start: 100 * clusterSize, Length: 8 * clusterSize, Allocated: true}, + {Start: 108 * clusterSize, Length: 892 * clusterSize, Zero: true}, + {Start: 1000 * clusterSize, Length: 16 * clusterSize, Allocated: true}, + {Start: 1016 * clusterSize, Length: 8984 * clusterSize, Zero: true}, + } + qcow2 := filepath.Join(t.TempDir(), "image") + if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents, "", ""); err != nil { + t.Fatal(err) + } + t.Run("qcow2", func(t *testing.T) { + actual, err := listExtents(qcow2) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(extents, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) + t.Run("qcow2 zlib", func(t *testing.T) { + qcow2Zlib := qcow2 + ".zlib" + if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil { + t.Fatal(err) + } + actual, err := listExtents(qcow2Zlib) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(compressed(extents), actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) +} + +func TestExtentsPartial(t *testing.T) { + // Writing part of of a cluster allocates entire cluster in the qcow2 image. + extents := []image.Extent{ + {Start: 0 * clusterSize, Length: 1, Allocated: true}, + {Start: 1 * clusterSize, Length: 98 * clusterSize, Zero: true}, + {Start: 100*clusterSize - 1, Length: 1, Allocated: true}, + } + + // Listing extents works in cluster granularity. + full := []image.Extent{ + {Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 1 * clusterSize, Length: 98 * clusterSize, Zero: true}, + {Start: 99 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + } + + qcow2 := filepath.Join(t.TempDir(), "image") + if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents, "", ""); err != nil { + t.Fatal(err) + } + t.Run("qcow2", func(t *testing.T) { + actual, err := listExtents(qcow2) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(full, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) + t.Run("qcow2 zlib", func(t *testing.T) { + qcow2Zlib := qcow2 + ".zlib" + if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil { + t.Fatal(err) + } + actual, err := listExtents(qcow2Zlib) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(compressed(full), actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) +} + +func TestExtentsMerge(t *testing.T) { + // Create image with consecutive extents of same type. + extents := []image.Extent{ + {Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 1 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 2 * clusterSize, Length: 98 * clusterSize, Zero: true}, + {Start: 100 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 101 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + } + + // Extents with same type are merged. + merged := []image.Extent{ + {Start: 0 * clusterSize, Length: 2 * clusterSize, Allocated: true}, + {Start: 2 * clusterSize, Length: 98 * clusterSize, Zero: true}, + {Start: 100 * clusterSize, Length: 2 * clusterSize, Allocated: true}, + } + + qcow2 := filepath.Join(t.TempDir(), "image") + if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents, "", ""); err != nil { + t.Fatal(err) + } + t.Run("qcow2", func(t *testing.T) { + actual, err := listExtents(qcow2) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(merged, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) + t.Run("qcow2 zlib", func(t *testing.T) { + qcow2Zlib := qcow2 + ".zlib" + if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil { + t.Fatal(err) + } + actual, err := listExtents(qcow2Zlib) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(compressed(merged), actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) +} + +func TestExtentsZero(t *testing.T) { + // Create image with different clusters that read as zeros. + extents := []image.Extent{ + {Start: 0 * clusterSize, Length: 1000 * clusterSize, Allocated: true, Zero: true}, + {Start: 1000 * clusterSize, Length: 1000 * clusterSize, Zero: true}, + } + + qcow2 := filepath.Join(t.TempDir(), "image") + if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents, "", ""); err != nil { + t.Fatal(err) + } + t.Run("qcow2", func(t *testing.T) { + actual, err := listExtents(qcow2) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(extents, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) + t.Run("qcow2 zlib", func(t *testing.T) { + qcow2Zlib := qcow2 + ".zlib" + if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil { + t.Fatal(err) + } + actual, err := listExtents(qcow2Zlib) + if err != nil { + t.Fatal(err) + } + // When converting to qcow2 images all clusters that read as zeros are + // converted to unallocated clusters. + converted := []image.Extent{ + {Start: 0 * clusterSize, Length: 2000 * clusterSize, Zero: true}, + } + if !slices.Equal(converted, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) +} + +func TestExtentsBackingFile(t *testing.T) { + // Create an image with some clusters in the backing file, and some cluasters + // in the image. Accessing extents should present a unified view using both + // image and backing file. + tmpDir := t.TempDir() + baseExtents := []image.Extent{ + {Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 1 * clusterSize, Length: 9 * clusterSize, Zero: true}, + {Start: 10 * clusterSize, Length: 2 * clusterSize, Allocated: true}, + {Start: 12 * clusterSize, Length: 88 * clusterSize, Zero: true}, + {Start: 100 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 101 * clusterSize, Length: 899 * clusterSize, Zero: true}, + } + topExtents := []image.Extent{ + {Start: 0 * clusterSize, Length: 1 * clusterSize, Zero: true}, + {Start: 1 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 2 * clusterSize, Length: 9 * clusterSize, Zero: true}, + {Start: 11 * clusterSize, Length: 2 * clusterSize, Allocated: true}, + {Start: 13 * clusterSize, Length: 986 * clusterSize, Zero: true}, + {Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + } + baseRaw := filepath.Join(tmpDir, "base.raw") + if err := createTestImageWithExtents(baseRaw, qemuimg.FormatRaw, baseExtents, "", ""); err != nil { + t.Fatal(err) + } + + t.Run("qcow2", func(t *testing.T) { + tmpDir := t.TempDir() + baseQcow2 := filepath.Join(tmpDir, "base.qcow2") + if err := qemuimg.Convert(baseRaw, baseQcow2, qemuimg.FormatQcow2, qemuimg.CompressionNone); err != nil { + t.Fatal(err) + } + top := filepath.Join(tmpDir, "top.qcow2") + if err := createTestImageWithExtents(top, qemuimg.FormatQcow2, topExtents, baseQcow2, qemuimg.FormatQcow2); err != nil { + t.Fatal(err) + } + // When top and base are uncompressed, extents from to and based are merged. + expected := []image.Extent{ + {Start: 0 * clusterSize, Length: 2 * clusterSize, Allocated: true}, + {Start: 2 * clusterSize, Length: 8 * clusterSize, Zero: true}, + {Start: 10 * clusterSize, Length: 3 * clusterSize, Allocated: true}, + {Start: 13 * clusterSize, Length: 87 * clusterSize, Zero: true}, + {Start: 100 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 101 * clusterSize, Length: 898 * clusterSize, Zero: true}, + {Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + } + actual, err := listExtents(top) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(expected, actual) { + t.Fatalf("expected %v, got %v", expected, actual) + } + }) + t.Run("qcow2 zlib", func(t *testing.T) { + tmpDir := t.TempDir() + baseQcow2Zlib := filepath.Join(tmpDir, "base.qcow2") + if err := qemuimg.Convert(baseRaw, baseQcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil { + t.Fatal(err) + } + top := filepath.Join(tmpDir, "top.qcow2") + if err := createTestImageWithExtents(top, qemuimg.FormatQcow2, topExtents, baseQcow2Zlib, qemuimg.FormatQcow2); err != nil { + t.Fatal(err) + } + // When base is compressed, extents from to and based cannot be merged since + // allocated extents from base are compressed. When copying we can merge + // extents with different types that read as zero. + expected := []image.Extent{ + // From base + {Start: 0 * clusterSize, Length: 1 * clusterSize, Allocated: true, Compressed: true}, + // From top + {Start: 1 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + {Start: 2 * clusterSize, Length: 8 * clusterSize, Zero: true}, + // From base + {Start: 10 * clusterSize, Length: 1 * clusterSize, Allocated: true, Compressed: true}, + // From top (top clusters hide base clusters) + {Start: 11 * clusterSize, Length: 2 * clusterSize, Allocated: true}, + {Start: 13 * clusterSize, Length: 87 * clusterSize, Zero: true}, + // From base + {Start: 100 * clusterSize, Length: 1 * clusterSize, Allocated: true, Compressed: true}, + {Start: 101 * clusterSize, Length: 898 * clusterSize, Zero: true}, + // From top + {Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + } + actual, err := listExtents(top) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(expected, actual) { + t.Fatalf("expected %v, got %v", expected, actual) + } + }) +} + +func TestExtentsBackingFileShort(t *testing.T) { + // Test the special case of shorter backing file. The typical use case is + // adding a large qcow2 image on top of a small os image. + baseExtents := []image.Extent{ + {Start: 0 * clusterSize, Length: 10 * clusterSize, Allocated: true}, + {Start: 10 * clusterSize, Length: 90 * clusterSize, Zero: true}, + } + topExtents := []image.Extent{ + {Start: 0 * clusterSize, Length: 5 * clusterSize, Zero: true}, + {Start: 5 * clusterSize, Length: 10 * clusterSize, Allocated: true}, + {Start: 15 * clusterSize, Length: 984 * clusterSize, Zero: true}, + {Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + } + expected := []image.Extent{ + {Start: 0 * clusterSize, Length: 15 * clusterSize, Allocated: true}, + {Start: 15 * clusterSize, Length: 984 * clusterSize, Zero: true}, + {Start: 999 * clusterSize, Length: 1 * clusterSize, Allocated: true}, + } + tmpDir := t.TempDir() + base := filepath.Join(tmpDir, "base.qcow2") + if err := createTestImageWithExtents(base, qemuimg.FormatQcow2, baseExtents, "", ""); err != nil { + t.Fatal(err) + } + top := filepath.Join(tmpDir, "top.qcow2") + if err := createTestImageWithExtents(top, qemuimg.FormatQcow2, topExtents, base, qemuimg.FormatQcow2); err != nil { + t.Fatal(err) + } + actual, err := listExtents(top) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(expected, actual) { + t.Fatalf("expected %v, got %v", expected, actual) + } +} + +func TestExtentsBackingFileShortUnaligned(t *testing.T) { + // Test the special case of raw backing file not aligned to cluster size. When + // getting a short extent from the end of the backing file, we aligned the + // extent length to the top image cluster size. + blockSize := 4 * KiB + baseExtents := []image.Extent{ + {Start: 0, Length: 100 * blockSize, Allocated: true}, + } + topExtents := []image.Extent{ + {Start: 0, Length: 10 * clusterSize, Zero: true}, + } + expected := []image.Extent{ + {Start: 0 * clusterSize, Length: 7 * clusterSize, Allocated: true}, + {Start: 7 * clusterSize, Length: 3 * clusterSize, Zero: true}, + } + tmpDir := t.TempDir() + base := filepath.Join(tmpDir, "base.raw") + if err := createTestImageWithExtents(base, qemuimg.FormatRaw, baseExtents, "", ""); err != nil { + t.Fatal(err) + } + top := filepath.Join(tmpDir, "top.qcow2") + if err := createTestImageWithExtents(top, qemuimg.FormatQcow2, topExtents, base, qemuimg.FormatRaw); err != nil { + t.Fatal(err) + } + actual, err := listExtents(top) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(expected, actual) { + t.Fatalf("expected %v, got %v", expected, actual) + } +} + +func compressed(extents []image.Extent) []image.Extent { + var res []image.Extent + for _, extent := range extents { + if extent.Allocated { + extent.Compressed = true + } + res = append(res, extent) + } + return res +} + +func listExtents(path string) ([]image.Extent, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + img, err := qcow2reader.Open(f) + if err != nil { + return nil, err + } + defer img.Close() + + var extents []image.Extent + var start int64 + + end := img.Size() + for start < end { + extent, err := img.Extent(start, end-start) + if err != nil { + return nil, err + } + if extent.Start != start { + return nil, fmt.Errorf("invalid extent start: %+v", extent) + } + if extent.Length <= 0 { + return nil, fmt.Errorf("invalid extent length: %+v", extent) + } + extents = append(extents, extent) + start += extent.Length + } + return extents, nil +} + +// createTestImageWithExtents creates a n image with the allocation described +// by extents. +func createTestImageWithExtents( + path string, + format qemuimg.Format, + extents []image.Extent, + backingFile string, + backingFormat qemuimg.Format, +) error { + lastExtent := extents[len(extents)-1] + size := lastExtent.Start + lastExtent.Length + if err := qemuimg.Create(path, format, size, backingFile, backingFormat); err != nil { + return err + } + for _, extent := range extents { + if !extent.Allocated { + continue + } + start := extent.Start + length := extent.Length + for length > 0 { + // qemu-io requires length < 2g. + n := length + if n >= 2*GiB { + n = 2*GiB - 64*KiB + } + if extent.Zero { + if err := qemuio.Zero(path, format, start, n); err != nil { + return err + } + } else { + if err := qemuio.Write(path, format, start, n, 0x55); err != nil { + return err + } + } + start += n + length -= n + } + } + return nil +} + // Benchmark completely empty sparse image (0% utilization). This is the best // case when we don't have to read any cluster from storage. func Benchmark0p(b *testing.B) { @@ -208,7 +809,7 @@ func benchmarkConvert(b *testing.B, filename string) { if err != nil { b.Fatal(err) } - err = c.Convert(dst, img, img.Size()) + err = c.Convert(dst, img, img.Size(), nil) if err != nil { b.Fatal(err) }