Merge pull request containerd#1196 from mlaventure/update-windows-run…

…time

Update windows runtime

.appveyor.yml

@@ -0,0 +1,30 @@
+version: "{build}"
+
+image: Visual Studio 2017
+
+clone_folder: c:\gopath\src\github.com\containerd\containerd
+
+environment:
+  GOPATH: C:\gopath
+  CGO_ENABLED: 1
+
+before_build:
+  - choco install -y mingw
+  - choco install codecov
+
+build_script:
+  - bash.exe -lc "export PATH=/c/tools/mingw64/bin:/c/gopath/src/github.com/containerd/containerd/bin:$PATH ; mingw32-make.exe fmt"
+  - bash.exe -lc "export PATH=/c/tools/mingw64/bin:/c/gopath/src/github.com/containerd/containerd/bin:$PATH ; mingw32-make.exe vet"
+  - bash.exe -lc "export PATH=/c/tools/mingw64/bin:$PATH ; mingw32-make.exe build"
+  - bash.exe -lc "export PATH=/c/tools/mingw64/bin:$PATH ; mingw32-make.exe binaries"
+
+test_script:
+  # TODO: need an equivalent of TRAVIS_COMMIT_RANGE
+  # - GIT_CHECK_EXCLUDE="./vendor" TRAVIS_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE/.../..}" C:\MinGW\bin\mingw32-make.exe dco
+  - bash.exe -lc "export PATH=/c/tools/mingw64/bin:/c/gopath/src/github.com/containerd/containerd/bin:$PATH ; mingw32-make.exe integration"
+  - bash.exe -lc "export PATH=/c/tools/mingw64/bin:/c/gopath/src/github.com/containerd/containerd/bin:$PATH ; mingw32-make.exe coverage"
+  - bash.exe -lc "export PATH=/c/tools/mingw64/bin:/c/gopath/src/github.com/containerd/containerd/bin:$PATH ; mingw32-make.exe root-coverage"
+
+on_success:
+  # Note that, a Codecov upload token is not required.
+  - codecov -f coverage.txt

Makefile

@@ -9,15 +9,18 @@ VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always)
 REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
 
 ifneq "$(strip $(shell command -v go 2>/dev/null))" ""
-GOOS ?= $(shell go env GOOS)
+	GOOS ?= $(shell go env GOOS)
 else
-GOOS ?= $$GOOS
+	GOOS ?= $$GOOS
 endif
+
 WHALE = "🇩"
 ONI = "👹"
+FIX_PATH = $1
 ifeq ("$(OS)", "Windows_NT")
 	WHALE="+"
 	ONI="-"
+	FIX_PATH = $(subst /,\,$1)
 endif
 GOARCH ?= $(shell go env GOARCH)
 
@@ -44,7 +47,7 @@ GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",)
 GO_LDFLAGS=-ldflags "-X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PKG) $(EXTRA_LDFLAGS)"
 
 # Flags passed to `go test`
-TESTFLAGS ?=-parallel 8 -race
+TESTFLAGS ?=-parallel 8 -race -v
 
 .PHONY: clean all AUTHORS fmt vet lint dco build binaries test integration setup generate protos checkprotos coverage ci check help install uninstall vendor release
 .DEFAULT: default
@@ -88,7 +91,7 @@ vet: binaries ## run go vet
 
 fmt: ## run go fmt
 	@echo "$(WHALE) $@"
-	@test -z "$$(gofmt -s -l . | grep -v vendor/ | grep -v ".pb.go$$" | tee /dev/stderr)" || \
+	@test -z "$$(gofmt -s -l . | grep -Fv $(call FIX_PATH,'vendor/') | grep -v ".pb.go$$" | tee /dev/stderr)" || \
 		(echo "$(ONI) please format Go code with 'gofmt -s -w'" && false)
 	@test -z "$$(find . -path ./vendor -prune -o ! -name timestamp.proto ! -name duration.proto -name '*.proto' -type f -exec grep -Hn -e "^ " {} \; | tee /dev/stderr)" || \
 		(echo "$(ONI) please indent proto files with tabs only" && false)
@@ -97,7 +100,7 @@ fmt: ## run go fmt
 
 lint: ## run go lint
 	@echo "$(WHALE) $@"
-	@test -z "$$(golint ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)"
+	@test -z "$$(golint ./... | grep -Fv $(call FIX_PATH,'vendor/') | grep -v ".pb.go:" | tee /dev/stderr)"
 
 dco: ## dco check
 	@which git-validation > /dev/null 2>/dev/null || (echo "ERROR: git-validation not found" && false)
@@ -109,11 +112,11 @@ endif
 
 ineffassign: ## run ineffassign
 	@echo "$(WHALE) $@"
-	@test -z "$$(ineffassign . | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)"
+	@test -z "$$(ineffassign . | grep -Fv $(call FIX_PATH,'vendor/') | grep -v ".pb.go:" | tee /dev/stderr)"
 
 #errcheck: ## run go errcheck
 #	@echo "$(WHALE) $@"
-#	@test -z "$$(errcheck ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)"
+#	@test -z "$$(errcheck ./... | grep -Fv $(call FIX_PATH,'vendor/') | grep -v ".pb.go:" | tee /dev/stderr)"
 
 build: ## build the go packages
 	@echo "$(WHALE) $@"

archive/path_test.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package archive
 
 import (

archive/tar_test.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package archive
 
 import (

benchmark_test.go

@@ -20,7 +20,7 @@ func BenchmarkContainerCreate(b *testing.B) {
 		b.Error(err)
 		return
 	}
-	spec, err := GenerateSpec(WithImageConfig(ctx, image), WithProcessArgs("true"))
+	spec, err := GenerateSpec(WithImageConfig(ctx, image), withTrue())
 	if err != nil {
 		b.Error(err)
 		return
@@ -63,7 +63,7 @@ func BenchmarkContainerStart(b *testing.B) {
 		b.Error(err)
 		return
 	}
-	spec, err := GenerateSpec(WithImageConfig(ctx, image), WithProcessArgs("true"))
+	spec, err := GenerateSpec(WithImageConfig(ctx, image), withTrue())
 	if err != nil {
 		b.Error(err)
 		return

checkpoint_test.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package containerd
 
 import (

client.go

@@ -7,7 +7,6 @@ import (
 	"log"
 	"net/http"
 	"runtime"
-	"strconv"
 	"sync"
 	"time"
 
@@ -34,11 +33,9 @@ import (
 	imagesservice "github.com/containerd/containerd/services/images"
 	snapshotservice "github.com/containerd/containerd/services/snapshot"
 	"github.com/containerd/containerd/snapshot"
-	"github.com/containerd/containerd/typeurl"
 	pempty "github.com/golang/protobuf/ptypes/empty"
 	"github.com/opencontainers/image-spec/identity"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
@@ -48,13 +45,6 @@ import (
 func init() {
 	// reset the grpc logger so that it does not output in the STDIO of the calling process
 	grpclog.SetLogger(log.New(ioutil.Discard, "", log.LstdFlags))
-
-	// register TypeUrls for commonly marshaled external types
-	major := strconv.Itoa(specs.VersionMajor)
-	typeurl.Register(&specs.Spec{}, "opencontainers/runtime-spec", major, "Spec")
-	typeurl.Register(&specs.Process{}, "opencontainers/runtime-spec", major, "Process")
-	typeurl.Register(&specs.LinuxResources{}, "opencontainers/runtime-spec", major, "LinuxResources")
-	typeurl.Register(&specs.WindowsResources{}, "opencontainers/runtime-spec", major, "WindowsResources")
 }
 
 type clientOpts struct {

client_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"runtime"
 	"syscall"
 	"testing"
 	"time"
@@ -17,19 +18,14 @@ import (
 	"github.com/containerd/containerd/testutil"
 )
 
-const (
-	defaultRoot = "/var/lib/containerd-test"
-	testImage   = "docker.io/library/alpine:latest"
-)
-
 var (
 	address      string
 	noDaemon     bool
 	supportsCriu bool
 )
 
 func init() {
-	flag.StringVar(&address, "address", "/run/containerd-test/containerd.sock", "The address to the containerd socket for use in the tests")
+	flag.StringVar(&address, "address", defaultAddress, "The address to the containerd socket for use in the tests")
 	flag.BoolVar(&noDaemon, "no-daemon", false, "Do not start a dedicated daemon for the tests")
 	flag.Parse()
 }
@@ -57,11 +53,15 @@ func TestMain(m *testing.M) {
 	defer cancel()
 
 	if !noDaemon {
+		os.RemoveAll(defaultRoot)
+
 		// setup a new containerd daemon if !testing.Short
 		cmd = exec.Command("containerd",
 			"--root", defaultRoot,
 			"--address", address,
+			"--log-level", "debug",
 		)
+		cmd.Stdout = buf
 		cmd.Stderr = buf
 		if err := cmd.Start(); err != nil {
 			cmd.Wait()
@@ -94,14 +94,22 @@ func TestMain(m *testing.M) {
 	}).Info("running tests against containerd")
 
 	// pull a seed image
-	if _, err = client.Pull(ctx, testImage, WithPullUnpack); err != nil {
-		cmd.Process.Signal(syscall.SIGTERM)
-		cmd.Wait()
-		fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String())
+	if runtime.GOOS != "windows" { // TODO: remove once pull is supported on windows
+		if _, err = client.Pull(ctx, testImage, WithPullUnpack); err != nil {
+			cmd.Process.Signal(syscall.SIGTERM)
+			cmd.Wait()
+			fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String())
+			os.Exit(1)
+		}
+	}
+
+	if err := platformTestSetup(client); err != nil {
+		fmt.Fprintln(os.Stderr, "platform test setup failed", err)
 		os.Exit(1)
 	}
+
 	if err := client.Close(); err != nil {
-		fmt.Fprintln(os.Stderr, err)
+		fmt.Fprintln(os.Stderr, "failed to close client", err)
 	}
 
 	// run the test
@@ -110,13 +118,15 @@ func TestMain(m *testing.M) {
 	if !noDaemon {
 		// tear down the daemon and resources created
 		if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
-			fmt.Fprintln(os.Stderr, err)
+			if err := cmd.Process.Kill(); err != nil {
+				fmt.Fprintln(os.Stderr, "failed to signal containerd", err)
+			}
 		}
 		if err := cmd.Wait(); err != nil {
-			fmt.Fprintln(os.Stderr, err)
+			fmt.Fprintln(os.Stderr, "failed to wait for containerd", err)
 		}
 		if err := os.RemoveAll(defaultRoot); err != nil {
-			fmt.Fprintln(os.Stderr, err)
+			fmt.Fprintln(os.Stderr, "failed to remove test root dir", err)
 			os.Exit(1)
 		}
 		// only print containerd logs if the test failed
@@ -171,6 +181,11 @@ func TestNewClient(t *testing.T) {
 }
 
 func TestImagePull(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		// TODO: remove once Windows has a snapshotter
+		t.Skip("Windows does not have a snapshotter yet")
+	}
+
 	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)

client_unix.go

@@ -9,9 +9,6 @@ import (
 	"time"
 )
 
-// DefaultAddress is the default unix socket address
-const DefaultAddress = "/run/containerd/containerd.sock"
-
 func dialer(address string, timeout time.Duration) (net.Conn, error) {
 	address = strings.TrimPrefix(address, "unix://")
 	return net.DialTimeout("unix", address, timeout)

client_unix_test.go

@@ -0,0 +1,13 @@
+// +build !windows
+
+package containerd
+
+const (
+	defaultRoot    = "/var/lib/containerd-test"
+	defaultAddress = "/run/containerd-test/containerd.sock"
+	testImage      = "docker.io/library/alpine:latest"
+)
+
+func platformTestSetup(client *Client) error {
+	return nil
+}

client_windows.go

@@ -7,9 +7,6 @@ import (
 	winio "github.com/Microsoft/go-winio"
 )
 
-// DefaultAddress is the default unix socket address
-const DefaultAddress = `\\.\pipe\containerd-containerd`
-
 func dialer(address string, timeout time.Duration) (net.Conn, error) {
 	return winio.DialPipe(address, &timeout)
 }

client_windows_test.go

@@ -0,0 +1,87 @@
+package containerd
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	defaultAddress = `\\.\pipe\containerd-containerd-test`
+	testImage      = "docker.io/library/go:nanoserver"
+)
+
+var (
+	dockerLayerFolders []string
+
+	defaultRoot = filepath.Join(os.Getenv("programfiles"), "containerd", "root-test")
+)
+
+func platformTestSetup(client *Client) error {
+	var (
+		roots       []string
+		layerChains = make(map[string]string)
+	)
+	// Since we can't pull images yet, we'll piggyback on the default
+	// docker's images
+	wfPath := `C:\ProgramData\docker\windowsfilter`
+	wf, err := os.Open(wfPath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to access docker layers @ %s", wfPath)
+	}
+	defer wf.Close()
+	entries, err := wf.Readdirnames(0)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read %s entries", wfPath)
+	}
+
+	for _, fn := range entries {
+		layerChainPath := filepath.Join(wfPath, fn, "layerchain.json")
+		lfi, err := os.Stat(layerChainPath)
+		switch {
+		case err == nil && lfi.Mode().IsRegular():
+			f, err := os.OpenFile(layerChainPath, os.O_RDONLY, 0660)
+			if err != nil {
+				fmt.Fprintln(os.Stderr,
+					errors.Wrapf(err, "failed to open %s", layerChainPath))
+				continue
+			}
+			defer f.Close()
+			l := make([]string, 0)
+			if err := json.NewDecoder(f).Decode(&l); err != nil {
+				fmt.Fprintln(os.Stderr,
+					errors.Wrapf(err, "failed to decode %s", layerChainPath))
+				continue
+			}
+			switch {
+			case len(l) == 1:
+				layerChains[l[0]] = filepath.Join(wfPath, fn)
+			case len(l) > 1:
+				fmt.Fprintf(os.Stderr, "Too many entries in %s: %d", layerChainPath, len(l))
+			case len(l) == 0:
+				roots = append(roots, filepath.Join(wfPath, fn))
+			}
+		case os.IsNotExist(err):
+			// keep on going
+		default:
+			return errors.Wrapf(err, "error trying to access %s", layerChainPath)
+		}
+	}
+
+	// They'll be 2 roots, just take the first one
+	l := roots[0]
+	dockerLayerFolders = append(dockerLayerFolders, l)
+	for {
+		l = layerChains[l]
+		if l == "" {
+			break
+		}
+
+		dockerLayerFolders = append([]string{l}, dockerLayerFolders...)
+	}
+
+	return nil
+}