diff --git a/docker.Dockerfile b/docker.Dockerfile index d223bb8e6d..8d82dae732 100644 --- a/docker.Dockerfile +++ b/docker.Dockerfile @@ -12,30 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM marketplace.gcr.io/google/debian9 AS build +FROM marketplace.gcr.io/google/debian11 -ENV BAZEL_VERSION 0.19.2 -ENV BAZEL_ARCH linux_amd64_stripped +ENV DOCKER_TOOLS_TAG v1.0.0 RUN set -eux \ && apt-get update \ - && apt-get install git wget unzip python g++ -y + && apt-get install wget -y -# Install Bazel -RUN set -eux \ - && wget -q -O /bazel-installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" \ - && chmod +x /bazel-installer.sh \ - && /bazel-installer.sh RUN set -eux \ - && git clone https://github.com/GoogleCloudPlatform/runtimes-common.git --depth=1 \ - && cd runtimes-common \ - && bazel run //:gazelle \ - && bazel build versioning/scripts/dockerfiles:dockerfiles versioning/scripts/cloudbuild:cloudbuild \ - && cp bazel-bin/versioning/scripts/dockerfiles/${BAZEL_ARCH}/dockerfiles /bin/dockerfiles \ - && cp bazel-bin/versioning/scripts/cloudbuild/${BAZEL_ARCH}/cloudbuild /bin/cloudbuild - -FROM marketplace.gcr.io/google/debian9 - -COPY --from=build /bin/dockerfiles /bin/dockerfiles -COPY --from=build /bin/cloudbuild /bin/cloudbuild + && wget https://github.com/GoogleCloudPlatform/click-to-deploy/releases/download/${DOCKER_TOOLS_TAG}/cloudbuild -O /bin/cloudbuild \ + && wget https://github.com/GoogleCloudPlatform/click-to-deploy/releases/download/${DOCKER_TOOLS_TAG}/dockerfiles -O /bin/dockerfiles \ + && chmod +x /bin/cloudbuild \ + && chmod +x /bin/dockerfiles diff --git a/tools/WORKSPACE b/tools/WORKSPACE new file mode 100644 index 0000000000..9267df42bd --- /dev/null +++ b/tools/WORKSPACE @@ -0,0 +1,55 @@ +http_archive( + name = "io_bazel_rules_go", + sha256 = "1868ff68d6079e31b2f09b828b58d62e57ca8e9636edff699247c9108518570b", + url = "https://github.com/bazelbuild/rules_go/releases/download/0.11.1/rules_go-0.11.1.tar.gz", +) + +http_archive( + name = "bazel_gazelle", + sha256 = "92a3c59734dad2ef85dc731dbcb2bc23c4568cded79d4b87ebccd787eb89e8d0", + url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.11.0/bazel-gazelle-0.11.0.tar.gz", +) + +load("@io_bazel_rules_go//go:def.bzl", "go_repository", "go_rules_dependencies", "go_register_toolchains") + +# Declare Go direct dependencies. +go_repository( + name = "com_github_gopkg_v2", + importpath = "gopkg.in/yaml.v2", + tag = "v2.3.0", +) + +go_rules_dependencies() + +go_register_toolchains() + +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") + +gazelle_dependencies() + +git_repository( + name = "io_bazel_rules_docker", + commit = "8bbe2a8abd382641e65ff7127a3700a8530f02ce", + remote = "https://github.com/bazelbuild/rules_docker.git", +) + +git_repository( + name = "containerregistry", + commit = "6b250f0bae8cce028df939010ee3118c8f2977ba", + remote = "https://github.com/google/containerregistry", +) + +load( + "@io_bazel_rules_docker//docker:docker.bzl", + "docker_repositories", + "docker_pull", +) + +docker_repositories() + +load( + "@io_bazel_rules_docker//container:container.bzl", + "repositories", +) + +repositories() diff --git a/tools/cloudbuild-dockertools.yaml b/tools/cloudbuild-dockertools.yaml new file mode 100644 index 0000000000..51a5478273 --- /dev/null +++ b/tools/cloudbuild-dockertools.yaml @@ -0,0 +1,40 @@ +steps: +- id: Build Dockertools Docker Image + name: gcr.io/cloud-builders/docker + args: + - build + - --tag + - dockertools + - --file + - tools/dockertools.Dockerfile + - tools + +- id: Share dockertools using workspaces + name: dockertools + entrypoint: bash + args: + - -c + - | + cp /bin/cloudbuild /workspace/cloudbuild && + cp /bin/dockerfiles /workspace/dockerfiles && + cp /bin/cloudbuild /workspace/cloudbuild_$TAG_NAME && + cp /bin/dockerfiles /workspace/dockerfiles_$TAG_NAME + +- id: Create a new GitHub Release. + name: dockertools + entrypoint: bash + args: + - -c + - | + gh release create $TAG_NAME --notes "Dockertools" /workspace/{cloudbuild,dockerfiles} + secretEnv: ['GH_TOKEN'] + +artifacts: + objects: + location: 'gs://$PROJECT_ID-c2d/' + paths: ['/workspace/cloudbuild', '/workspace/dockerfiles', '/workspace/cloudbuild_$TAG_NAME', '/workspace/dockerfiles_$TAG_NAME'] + +availableSecrets: + secretManager: + - versionName: projects/$PROJECT_ID/secrets/GH_TOKEN/versions/1 + env: 'GH_TOKEN' diff --git a/tools/dockertools.Dockerfile b/tools/dockertools.Dockerfile new file mode 100644 index 0000000000..2478e194cd --- /dev/null +++ b/tools/dockertools.Dockerfile @@ -0,0 +1,42 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM marketplace.gcr.io/google/debian11 + +ENV BAZEL_VERSION 0.19.2 +ENV BAZEL_ARCH linux_amd64_stripped + +RUN set -eux \ + && apt-get update \ + && apt-get install git wget unzip python g++ curl -y + +# Install Bazel +RUN set -eux \ + && wget -q -O /bazel-installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" \ + && chmod +x /bazel-installer.sh \ + && /bazel-installer.sh + +# Install gh CLI +RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ + && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt update \ + && apt install gh + +RUN set -eux \ + && git clone https://github.com/GoogleCloudPlatform/click-to-deploy.git \ + && cd click-to-deploy/tools \ + && bazel build dockerversioning/scripts/dockerfiles:dockerfiles dockerversioning/scripts/cloudbuild:cloudbuild \ + && cp bazel-bin/dockerversioning/scripts/dockerfiles/${BAZEL_ARCH}/dockerfiles /bin/dockerfiles \ + && cp bazel-bin/dockerversioning/scripts/cloudbuild/${BAZEL_ARCH}/cloudbuild /bin/cloudbuild diff --git a/tools/dockerversioning/README.md b/tools/dockerversioning/README.md new file mode 100644 index 0000000000..a23846a348 --- /dev/null +++ b/tools/dockerversioning/README.md @@ -0,0 +1,70 @@ +# Description + +Versioning tools for Dockerfile source repos. Previously, these tools were in [runtimes-common](https://github.com/GoogleCloudPlatform/runtimes-common/tree/b39744e5a8588beba847a271e379b864a6ac2939), but have been moved to [click-to-deploy](https://github.com/GoogleCloudPlatform/click-to-deploy) for further development. + +- `dockerfiles` generates versionsed Dockerfiles from a common template. +- `cloudbuild` generates a configuration file to build these Dockerfiles using + [Google Container Builder](https://cloud.google.com/container-builder/docs/). + +# Installation + +- Install [Bazel, version 0.19.2](https://bazel.build) as the build tool. + +- Clone this repo: + +``` shell +git clone https://github.com/GoogleCloudPlatform/click-to-deploy.git +cd click-to-deploy/tools +``` + +- Build: + +``` shell +bazel run //:gazelle +bazel build dockerversioning/scripts/dockerfiles:dockerfiles +bazel build dockerversioning/scripts/cloudbuild:cloudbuild +``` + +- Set the path to the built scripts: + +``` shell +BAZEL_ARCH=linux_amd64_stripped +export PATH=$PATH:$PWD/bazel-bin/dockerversioning/scripts/dockerfiles/${BAZEL_ARCH}/ +export PATH=$PATH:$PWD/bazel-bin/dockerversioning/scripts/cloudbuild/${BAZEL_ARCH}/ +``` + +# Create `versions.yaml` + +At root of the Dockerfile source repo, add a file called `versions.yaml`. +Follow the format defined in `versions.go`. See an example on +[github](https://github.com/GoogleCloudPlatform/mysql-docker). + +Primary folders in the Dockerfile source repo: + +- `templates` contains `Dockerfile.template`, which is a Go template for + generating `Dockerfile`s. +- `tests` contains any tests that should be included in the generated cloud + build configuration. +- Version folders as defined in `versions.yaml`. The `Dockerfile`s are + generated into these folders. The folders should also contain all + supporting files for each version, for example `docker-entrypoint.sh` files. + +# Usage of `dockerfiles` command + +``` shell +cd path/to/dockerfile/repo +dockerfiles +``` + +# Usage of `cloudbuild` command + +``` shell +cd path/to/dockerfile/repo +cloudbuild > cloudbuild.yaml +``` + +You can use the generated `cloudbuild.yaml` file as followed: + +``` shell +gcloud container builds submit --config=cloudbuild.yaml . +``` diff --git a/tools/dockerversioning/scripts/cloudbuild/BUILD.bazel b/tools/dockerversioning/scripts/cloudbuild/BUILD.bazel new file mode 100644 index 0000000000..533484a60d --- /dev/null +++ b/tools/dockerversioning/scripts/cloudbuild/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/GoogleCloudPlatform/click-to-deploy/tools/dockerversioning/scripts/cloudbuild", + visibility = ["//visibility:private"], + deps = ["//dockerversioning/versions:go_default_library"], +) + +go_binary( + name = "cloudbuild", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/tools/dockerversioning/scripts/cloudbuild/main.go b/tools/dockerversioning/scripts/cloudbuild/main.go new file mode 100644 index 0000000000..14504afe71 --- /dev/null +++ b/tools/dockerversioning/scripts/cloudbuild/main.go @@ -0,0 +1,362 @@ +/* +Command line tool for generating a Cloud Build yaml file based on versions.yaml. +*/ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + "text/template" + + "github.com/GoogleCloudPlatform/click-to-deploy/tools/dockerversioning/versions" +) + +type cloudBuildOptions struct { + // Whether to restrict to a particular set of Dockerfile directories. + // If empty, all directories are used. + Directories []string + + // Whether to run tests as part of the build. + RunTests bool + + // Whether to require that image tags do not already exist in the repo. + RequireNewTags bool + + // Whether to push to all declared tags + FirstTagOnly bool + + // Optional timeout duration. If not specified, the Cloud Builder default timeout is used. + TimeoutSeconds int + + // Optional machine type used to run the build, must be one of: N1_HIGHCPU_8, N1_HIGHCPU_32, E2_HIGHCPU_8, E2_HIGHCPU_32. If not specified, the default machine is used. + MachineType string + + // Optional parallel build. If specified, images can be build on bigger machines in parallel. + EnableParallel bool + + // Forces parallel build. If specified, images are build on bigger machines in parallel. Overrides EnableParallel. + ForceParallel bool +} + +// TODO(huyhg): Replace "gcr.io/$PROJECT_ID/functional_test" with gcp-runtimes one. +const cloudBuildTemplateString = `steps: +{{- $parallel := .Parallel }} +{{- if .RequireNewTags }} +# Check if tags exist. +{{- range .Images }} + - name: gcr.io/gcp-runtimes/check_if_tag_exists + args: + - 'python' + - '/main.py' + - '--image={{ . }}' +{{- end }} +{{- end }} + +# Build images +{{- range .ImageBuilds }} +{{- if .Builder }} + - name: gcr.io/cloud-builders/docker + args: + - 'build' + - '--tag={{ .Tag }}' + - '{{ .Directory }}' +{{- if $parallel }} + waitFor: ['-'] + id: 'image-{{ .Tag }}' +{{- end }} +{{- else }} +{{- if .BuilderImage }} + - name: {{ .BuilderImage }} + args: {{ .BuilderArgs }} +{{- if $parallel }} + waitFor: ['image-{{ .BuilderImage }}'] + id: 'image-{{ .Tag }}' +{{- end }} +{{- else }} + - name: gcr.io/cloud-builders/docker + args: + - 'build' + - '--tag={{ .Tag }}' + - '{{ .Directory }}' +{{- if $parallel }} + waitFor: ['-'] + id: 'image-{{ .Tag }}' +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $imageIndex, $image := .ImageBuilds }} +{{- $primary := $image.Tag }} +{{- range $testIndex, $test := $image.StructureTests }} +{{- if and (eq $imageIndex 0) (eq $testIndex 0) }} + +# Run structure tests +{{- end}} + - name: gcr.io/gcp-runtimes/structure_test + args: + - '--image' + - '{{ $primary }}' + - '--config' + - '{{ $test }}' +{{- end }} +{{- end }} + +{{- range $imageIndex, $image := .ImageBuilds }} +{{- $primary := $image.Tag }} +{{- range $testIndex, $test := $image.FunctionalTests }} +{{- if and (eq $imageIndex 0) (eq $testIndex 0) }} + +# Run functional tests +{{- end }} + - name: gcr.io/$PROJECT_ID/functional_test + args: + - '--verbose' + - '--vars' + - 'IMAGE={{ $primary }}' + - '--vars' + - 'UNIQUE={{ $imageIndex }}-{{ $testIndex }}' + - '--test_spec' + - '{{ $test }}' +{{- if $parallel }} + waitFor: ['image-{{ $primary }}'] + id: 'test-{{ $primary }}-{{ $testIndex }}' +{{- end }} +{{- end }} + +{{- end }} + +# Add alias tags +{{- range $imageIndex, $image := .ImageBuilds }} +{{- $primary := $image.Tag }} +{{- range .Aliases }} + - name: gcr.io/cloud-builders/docker + args: + - 'tag' + - '{{ $primary }}' + - '{{ . }}' +{{- if $parallel }} + waitFor: + - 'image-{{ $primary }}' +{{- range $testIndex, $test := $image.FunctionalTests }} + - 'test-{{ $primary }}-{{ $testIndex }}' +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +images: +{{- range .AllImages }} + - '{{ . }}' +{{- end }} + +{{- if not (eq .TimeoutSeconds 0) }} + +timeout: {{ .TimeoutSeconds }}s +{{- end }} + +{{- if $parallel }} +options: + machineType: 'E2_HIGHCPU_8' +{{- else }} +{{- if .MachineType }} +options: + machineType: '{{ .MachineType }}' +{{- end }} +{{- end }} +` + +const testsDir = "tests" +const functionalTestsDir = "tests/functional_tests" +const structureTestsDir = "tests/structure_tests" +const testJsonSuffix = "_test.json" +const testYamlSuffix = "_test.yaml" +const workspacePrefix = "/workspace/" + +type imageBuildTemplateData struct { + Directory string + Tag string + Aliases []string + StructureTests []string + FunctionalTests []string + Builder bool + BuilderImage string + BuilderArgs []string + ImageNameFromBuilder string +} + +type cloudBuildTemplateData struct { + RequireNewTags bool + Parallel bool + ImageBuilds []imageBuildTemplateData + AllImages []string + TimeoutSeconds int + MachineType string +} + +func shouldParallelize(options cloudBuildOptions, numberOfVersions int, numberOfTests int) bool { + if options.ForceParallel { + return true + } + if !options.EnableParallel { + return false + } + return numberOfVersions > 1 || numberOfTests > 1 +} + +func newCloudBuildTemplateData( + registry string, spec versions.Spec, options cloudBuildOptions) cloudBuildTemplateData { + data := cloudBuildTemplateData{} + data.RequireNewTags = options.RequireNewTags + + // Determine the set of directories to operate on. + dirs := make(map[string]bool) + if len(options.Directories) > 0 { + for _, d := range options.Directories { + dirs[d] = true + } + } else { + for _, v := range spec.Versions { + dirs[v.Dir] = true + } + } + + // Extract tests to run. + var structureTests []string + var functionalTests []string + if options.RunTests { + // Legacy structure tests reside in the root tests/ directory. + structureTests = append(structureTests, readTests(testsDir)...) + structureTests = append(structureTests, readTests(structureTestsDir)...) + functionalTests = append(functionalTests, readTests(functionalTestsDir)...) + } + + // Extract a list of full image names to build. + for _, v := range spec.Versions { + if !dirs[v.Dir] { + continue + } + var images []string + for _, t := range v.Tags { + image := fmt.Sprintf("%v/%v:%v", registry, v.Repo, t) + images = append(images, image) + if options.FirstTagOnly { + break + } + } + // Ignore builder images from images list + if !v.Builder { + data.AllImages = append(data.AllImages, images...) + } + versionSTests, versionFTests := filterTests(structureTests, functionalTests, v) + // Enforce to use ImageNameFromBuilder as reference to create tags + if v.BuilderImage != "" { + BuilderImageFull := fmt.Sprintf("%v/%v", registry, v.BuilderImage) + data.ImageBuilds = append( + data.ImageBuilds, imageBuildTemplateData{v.Dir, v.ImageNameFromBuilder, images, versionSTests, versionFTests, v.Builder, BuilderImageFull, v.BuilderArgs, v.ImageNameFromBuilder}) + } else { + data.ImageBuilds = append( + data.ImageBuilds, imageBuildTemplateData{v.Dir, images[0], images[1:], versionSTests, versionFTests, v.Builder, v.BuilderImage, v.BuilderArgs, v.ImageNameFromBuilder}) + } + } + + data.TimeoutSeconds = options.TimeoutSeconds + data.MachineType = options.MachineType + data.Parallel = shouldParallelize(options, len(spec.Versions), len(functionalTests)) + return data +} + +func readTests(testsDir string) (tests []string) { + if info, err := os.Stat(testsDir); err == nil && info.IsDir() { + files, err := ioutil.ReadDir(testsDir) + check(err) + for _, f := range files { + if f.IsDir() { + continue + } + if strings.HasSuffix(f.Name(), testJsonSuffix) || strings.HasSuffix(f.Name(), testYamlSuffix) { + tests = append(tests, workspacePrefix+fmt.Sprintf("%s/%s", testsDir, f.Name())) + } + } + } + return +} + +func filterTests(structureTests []string, functionalTests []string, version versions.Version) (outStructureTests []string, outFunctionalTests []string) { + included := make(map[string]bool, len(structureTests)+len(functionalTests)) + for _, test := range append(structureTests, functionalTests...) { + included[test] = true + } + for _, excluded := range version.ExcludeTests { + if !included[workspacePrefix+excluded] { + log.Fatalf("No such test to exclude: %s", excluded) + } + included[workspacePrefix+excluded] = false + } + + outStructureTests = make([]string, 0, len(structureTests)) + for _, test := range structureTests { + if included[test] { + outStructureTests = append(outStructureTests, test) + } + } + outFunctionalTests = make([]string, 0, len(functionalTests)) + for _, test := range functionalTests { + if included[test] { + outFunctionalTests = append(outFunctionalTests, test) + } + } + return +} + +func renderCloudBuildConfig( + registry string, spec versions.Spec, options cloudBuildOptions) string { + data := newCloudBuildTemplateData(registry, spec, options) + tmpl, _ := template. + New("cloudBuildTemplate"). + Parse(cloudBuildTemplateString) + var result bytes.Buffer + tmpl.Execute(&result, data) + return result.String() +} + +func check(e error) { + if e != nil { + panic(e) + } +} + +func main() { + config := versions.LoadConfig("versions.yaml", "cloudbuild") + registryPtr := config.StringOption("registry", "gcr.io/$PROJECT_ID", "Registry, e.g: 'gcr.io/my-project'") + dirsPtr := config.StringOption("dirs", "", "Comma separated list of Dockerfile dirs to use.") + testsPtr := config.BoolOption("tests", true, "Run tests.") + newTagsPtr := config.BoolOption("new_tags", false, "Require that image tags do not already exist.") + firstTagOnly := config.BoolOption("first_tag", false, "Build only the first per version.") + timeoutPtr := config.IntOption("timeout", 0, "Timeout in seconds. If not set, the default Cloud Build timeout is used.") + machineTypePtr := config.StringOption("machineType","", "Optional machine type used to run the build, , must be one of: N1_HIGHCPU_8, N1_HIGHCPU_32, E2_HIGHCPU_8, E2_HIGHCPU_32. If not specified, the default machine is used.") + enableParallel := config.BoolOption("enable_parallel", false, "Enable parallel build and bigger VM") + forceParallel := config.BoolOption("force_parallel", false, "Force parallel build and bigger VM") + config.Parse() + + if *registryPtr == "" { + log.Fatalf("--registry flag is required") + } + + if strings.Contains(*registryPtr, ":") { + *registryPtr = strings.Replace(*registryPtr, ":", "/", 1) + } + + var dirs []string + if *dirsPtr != "" { + dirs = strings.Split(*dirsPtr, ",") + } + spec := versions.LoadVersions("versions.yaml") + options := cloudBuildOptions{dirs, *testsPtr, *newTagsPtr, *firstTagOnly, *timeoutPtr, *machineTypePtr, *enableParallel, *forceParallel} + result := renderCloudBuildConfig(*registryPtr, spec, options) + fmt.Println(result) +} \ No newline at end of file diff --git a/tools/dockerversioning/scripts/dockerfiles/BUILD.bazel b/tools/dockerversioning/scripts/dockerfiles/BUILD.bazel new file mode 100644 index 0000000000..819c9d2f5e --- /dev/null +++ b/tools/dockerversioning/scripts/dockerfiles/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/GoogleCloudPlatform/click-to-deploy/tools/dockerversioning/scripts/dockerfiles", + visibility = ["//visibility:private"], + deps = ["//dockerversioning/versions:go_default_library"], +) + +go_binary( + name = "dockerfiles", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/tools/dockerversioning/scripts/dockerfiles/main.go b/tools/dockerversioning/scripts/dockerfiles/main.go new file mode 100644 index 0000000000..ae2e67a61b --- /dev/null +++ b/tools/dockerversioning/scripts/dockerfiles/main.go @@ -0,0 +1,274 @@ +/* +Command line tool for updating Dockerfiles based on versions.yaml. +*/ +package main + +import ( + "bytes" + "flag" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "reflect" + "strings" + "text/template" + + "github.com/GoogleCloudPlatform/click-to-deploy/tools/dockerversioning/versions" +) + +type indentFormat string + +// indent replaces leading spaces in the input string with the +// value of the indentFormat object. +func (f indentFormat) indent(s string) string { + temp := strings.Split(s, "\n") + str := "" + for index, line := range temp { + if index > 0 { + str = str + "\n" + } + trimmed := strings.TrimLeft(line, " ") + diff := len(line) - len(trimmed) + prefix := strings.Repeat(string(f), diff) + str = str + prefix + trimmed + } + return str +} + +const keyServersRetryTemplate = `found='' && \ +for server in \ + pool.sks-keyservers.net \ + na.pool.sks-keyservers.net \ + eu.pool.sks-keyservers.net \ + oc.pool.sks-keyservers.net \ + ha.pool.sks-keyservers.net \ + hkp://p80.pool.sks-keyservers.net:80 \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu \ +; do \ + {{ . }} \ + && found=yes && break; \ +done; \ +test -n "$found"` + +func funcKeyServersRetryLoop(indentSequence string, cmd string) string { + f := indentFormat(indentSequence) + tmpl, err := template.New("retryTemplate").Parse(f.indent(keyServersRetryTemplate)) + check(err) + var result bytes.Buffer + tmpl.Execute(&result, cmd) + return funcIndent(indentSequence, string(result.Bytes())) +} + +func funcIndent(leading string, s string) string { + temp := strings.Split(s, "\n") + str := "" + for index, line := range temp { + if index > 0 { + str = str + "\n" + leading + } + str = str + line + } + return str +} + +func renderDockerfile(version versions.Version, tmpl template.Template) []byte { + var result bytes.Buffer + tmpl.Execute(&result, version) + return result.Bytes() +} + +func writeDockerfile(version versions.Version, data []byte, createDir bool) { + path := filepath.Join(version.Dir, "Dockerfile") + // Delete first to make sure file is created with the right mode. + deleteIfFileExists(path) + // Create nested directory structure if needed. + if createDir { + os.MkdirAll(version.Dir, os.ModePerm) + } + err := ioutil.WriteFile(path, data, 0644) + check(err) +} + +func findFilesToCopy(templateDir string, callback func(path string, fileInfo os.FileInfo)) { + filepath.Walk(templateDir, func(path string, info os.FileInfo, err error) error { + check(err) + if strings.HasSuffix(info.Name(), ".template") || info.IsDir() { + return nil + } + path, err = filepath.Rel(templateDir, path) + check(err) + callback(path, info) + return nil + }) +} + +func copyFiles(version versions.Version, templateDir string, createDir bool) { + findFilesToCopy(templateDir, func(filePath string, fileInfo os.FileInfo) { + data, err := ioutil.ReadFile(filepath.Join(templateDir, filePath)) + check(err) + + target := filepath.Join(version.Dir, filePath) + // Delete first to make sure file is created with the right mode. + deleteIfFileExists(target) + // Create nested directory structure if needed. + if createDir { + os.MkdirAll(path.Dir(target), os.ModePerm) + } + err = ioutil.WriteFile(target, data, fileInfo.Mode()) + check(err) + }) +} + +func deleteIfFileExists(path string) { + if fileInfo, err := os.Stat(path); err != nil { + if !os.IsNotExist(err) { + log.Fatalf("File %s exists but cannot be stat'ed", path) + } + } else { + if fileInfo.IsDir() { + log.Fatalf("%s is unexpectedly a directory", path) + } + err = os.Remove(path) + check(err) + } +} + +func verifyDockerfiles(version versions.Version, templateDir string, tmpl template.Template) (failureCount int) { + foundDockerfile := make(map[string]bool) + failureCount = 0 + warningCount := 0 + + data := renderDockerfile(version, tmpl) + + path := filepath.Join(version.Dir, "Dockerfile") + + dockerfile, err := ioutil.ReadFile(path) + check(err) + + foundDockerfile[path] = true + + if string(dockerfile) == string(data) { + log.Printf("%s: OK", path) + } else { + failureCount++ + log.Printf("%s: FAILED", path) + } + + err = filepath.Walk(templateDir, func(path string, info os.FileInfo, err error) error { + check(err) + if info.Name() == "Dockerfile" && !info.IsDir() && !foundDockerfile[path] { + warningCount++ + log.Printf("%s: UNIDENTIFIED (warning)", path) + } + return nil + }) + check(err) + + if failureCount == 0 && warningCount > 0 { + log.Print("Dockerfile verification completed: PASSED (with warnings)") + } else if failureCount == 0 { + log.Print("Dockerfile verification completed: PASSED") + } else { + log.Print("Dockerfile verification completed: FAILED") + } + + return +} + +func verifyCopiedFiles(version versions.Version, templateDir string) (failureCount int) { + failureCount = 0 + findFilesToCopy(templateDir, func(path string, sourceFileInfo os.FileInfo) { + failureCount++ + + source := filepath.Join(templateDir, path) + target := filepath.Join(version.Dir, path) + targetFileInfo, err := os.Stat(target) + if err != nil { + log.Printf("%s is expected but cannot be stat'ed", target) + log.Printf("Please, check accessability of %s", source) + return + } + + // Check mode for owner only. + sourcePerm := os.FileMode(sourceFileInfo.Mode().Perm() & 0700) + targetPerm := os.FileMode(targetFileInfo.Mode().Perm() & 0700) + if sourcePerm != targetPerm { + log.Printf("%s has wrong file mode %v, expected %v", target, targetPerm, sourcePerm) + return + } + + expected, err := ioutil.ReadFile(source) + check(err) + actual, err := ioutil.ReadFile(filepath.Join(version.Dir, path)) + if err != nil { + log.Printf("%s is expected but cannot be read", target) + return + } + + if !reflect.DeepEqual(expected, actual) { + log.Printf("%s content is different from its template", target) + return + } + + log.Printf("%s: OK", target) + + failureCount-- + }) + + if failureCount == 0 { + log.Print("Copied files verification completed: PASSED") + } else { + log.Print("Copied files verification completed: FAILED") + } + + return +} + +func check(e error) { + if e != nil { + panic(e) + } +} + +func main() { + defaultTemplateDirPtr := flag.String("template_dir", "templates", "Path to directory containing Dockerfile.template and any other files to copy over") + verifyPtr := flag.Bool("verify_only", false, "Verify dockerfiles") + createDirPtr := flag.Bool("create_directories", false, "Create new directories") + failureCount := 0 + flag.Parse() + + var spec versions.Spec + spec = versions.LoadVersions("versions.yaml") + + for _, version := range spec.Versions { + // Ignore version without Dir for possibility to use builder images + if version.Dir == "" { + continue + } + // templatePath - path to Dockerfile.template + templatePath := filepath.Join(*defaultTemplateDirPtr, version.TemplateSubDir, "Dockerfile.template") + templateData, err := ioutil.ReadFile(templatePath) + templateString := string(templateData) + check(err) + + tmpl, err := template. + New("dockerfileTemplate"). + Funcs(template.FuncMap{"KeyServersRetryLoop": funcKeyServersRetryLoop}). + Parse(templateString) + check(err) + + if *verifyPtr { + failureCount += verifyDockerfiles(version, filepath.Join(*defaultTemplateDirPtr, version.TemplateSubDir), *tmpl) + failureCount += verifyCopiedFiles(version, filepath.Join(*defaultTemplateDirPtr, version.TemplateSubDir)) + } else { + data := renderDockerfile(version, *tmpl) + writeDockerfile(version, data, *createDirPtr) + // if version.TemplateSubDir is empty then we default to 'templates' folder + copyFiles(version, filepath.Join(*defaultTemplateDirPtr, version.TemplateSubDir), *createDirPtr) + } + } + os.Exit(failureCount) +} diff --git a/tools/dockerversioning/versions/BUILD.bazel b/tools/dockerversioning/versions/BUILD.bazel new file mode 100644 index 0000000000..41925d87df --- /dev/null +++ b/tools/dockerversioning/versions/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["versions.go"], + importpath = "github.com/GoogleCloudPlatform/click-to-deploy/tools/dockerversioning/versions", + deps = [ + "@com_github_gopkg_v2//:go_default_library", + ], +) diff --git a/tools/dockerversioning/versions/versions.go b/tools/dockerversioning/versions/versions.go new file mode 100644 index 0000000000..64a4ba575c --- /dev/null +++ b/tools/dockerversioning/versions/versions.go @@ -0,0 +1,132 @@ +/* +Library for parsing versions.yaml file. +*/ +package versions + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "strconv" + + yaml "gopkg.in/yaml.v2" +) + +type Package struct { + Version string + Minor string + Major string + Gpg string + Sha1 string + Sha256 string + Sha512 string + Md5 string +} + +type Version struct { + Dir string + TemplateSubDir string `yaml:"templateSubDir"` + Repo string + Tags []string + From string + TemplateArgs map[string]string `yaml:"templateArgs"` + Packages map[string]Package + ExcludeTests []string `yaml:"excludeTests"` + Builder bool + BuilderImage string `yaml:"builderImage"` + BuilderArgs []string `yaml:"builderArgs"` + ImageNameFromBuilder string `yaml:"imageNameFromBuilder"` +} + +type Spec struct { + Versions []Version +} + +func ReadFile(path string) []byte { + data, err := ioutil.ReadFile(path) + if err != nil { + log.Fatalf("error: %v", err) + } + return []byte(data) +} + +func LoadVersions(path string) Spec { + spec := Spec{} + err := yaml.Unmarshal(ReadFile(path), &spec) + if err != nil { + log.Fatalf("error: %v", err) + } + + validateUniqueTags(spec) + + return spec +} + +// Config represents setting for a program call. Arguments can be provided in file, as a key-value +// map, or as a command-line parameters. +type Config map[string]string + +func LoadConfig(path, config string) Config { + var whole map[string]interface{} + err := yaml.Unmarshal(ReadFile(path), &whole) + if err != nil { + log.Fatalf("error: %v", err) + } + + if c, ok := whole[config]; ok { + configMap := map[string]string{} + mapInterface := c.(map[interface{}]interface{}) + for key, value := range mapInterface { + configMap[key.(string)] = fmt.Sprintf("%v", value) + } + return configMap + } + return map[string]string{} +} + +func (c Config) StringOption(name, defaultVal, helper string) *string { + if configVal, ok := c[name]; ok { + defaultVal = configVal + } + return flag.String(name, defaultVal, helper) +} + +func (c Config) BoolOption(name string, defaultVal bool, helper string) *bool { + if configVal, ok := c[name]; ok { + b, err := strconv.ParseBool(configVal) + if err != nil { + log.Fatalf("error: %v", err) + } + defaultVal = b + } + return flag.Bool(name, defaultVal, helper) +} + +func (c Config) IntOption(name string, defaultVal int, helper string) *int { + if configVal, ok := c[name]; ok { + i, err := strconv.Atoi(configVal) + if err != nil { + log.Fatalf("error: %v", err) + } + defaultVal = i + } + return flag.Int(name, defaultVal, helper) +} + +func (c Config) Parse() { + flag.Parse() +} + +func validateUniqueTags(spec Spec) { + repoTags := make(map[string]bool) + for _, version := range spec.Versions { + for _, tag := range version.Tags { + repoTag := fmt.Sprintf("%s:%s", version.Repo, tag) + if repoTags[repoTag] { + log.Fatalf("error: duplicate repo tag %v", repoTag) + } + repoTags[repoTag] = true + } + } +}