From 4967abcdb3a9ffcc5a38f671921943d415c52f61 Mon Sep 17 00:00:00 2001 From: Vladimir Jovin <58534614+dovvla@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:45:33 +0100 Subject: [PATCH] GH actions and docker fixes and refactor (#302) * Introduce docker bake and simplify build and push job Pin VSCode Server version in VNC image due to incompatilibities in newer versions (cpp tools and ros extensions specifically cannot be installed to vscode server during image build) Previously, the build and push job constitued of two parametrized jobs that were taking from the same template to build base and dependent vnc image. The .yml files themselves were somewhat convoluted and contained custom logic that should be unnecessary. Since buildx was already used in workflows, no reason not to use its really good features, like bake. Also, buildx has been installed by default since docker v23.0, and current is v27, so really, there is no reason not to use it, hence changes in the Makefile too (also left previous build goal as build_legacy goal for clarity). The convoluted logic of evaluating tags and targets manually, then passing them around to build and push jobs has been completely moved to docker-bake.hcl, which is way more extendable, and also can be used for both local and remote use. This also removes 2 .yml files which were dependent on each other, and instead makes the CI part of the system clean and simple. In addition, the job that actually builds the images will run faster now, as it is not explicitly sequential as before (basically, think that vnc job is now in some form of hot start mode, as it is not separated, so it can prepare its dependencies that are not dependent on the upstream image) * Refactor bake config file, much more readable, structured and extendible, modify Makefile, add initial Dockerfile.deploy * Enable push on new GH action, add some more comments to docker-bake.hcl --- .../build_and_push_image_workflow.yml | 63 ---------------- .github/workflows/build_and_push_images.yml | 38 +++++++--- docker/Dockerfile.base | 4 +- docker/Dockerfile.deploy | 71 +++++++++++++++++++ docker/Dockerfile.vnc | 3 +- docker/Makefile | 64 +++++++++++++---- docker/docker-bake.hcl | 50 +++++++++++++ 7 files changed, 201 insertions(+), 92 deletions(-) delete mode 100644 .github/workflows/build_and_push_image_workflow.yml create mode 100644 docker/Dockerfile.deploy create mode 100644 docker/docker-bake.hcl diff --git a/.github/workflows/build_and_push_image_workflow.yml b/.github/workflows/build_and_push_image_workflow.yml deleted file mode 100644 index daf9708b6..000000000 --- a/.github/workflows/build_and_push_image_workflow.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Deploy Images to GHCR - -on: - workflow_call: - inputs: - flavor: - required: false - default: "devel" - type: string - base_image: - required: false - default: "osrf/ros:humble-desktop" - type: string - -jobs: - push-mep3-image: - runs-on: ubuntu-latest - defaults: - run: - working-directory: "./docker" - - steps: - - name: "Checkout GitHub Action" - uses: actions/checkout@main - - - name: "Login to GitHub Container Registry" - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{github.actor}} - password: ${{secrets.GITHUB_TOKEN}} - - - name: "Configure Docker parameters" - id: configure_docker_params - run: | - #!/bin/bash - - case ${{inputs.flavor}} in - "vnc") - echo "dockerfile=Dockerfile.vnc" >> $GITHUB_OUTPUT - echo "image_name=mep3-vnc" >> $GITHUB_OUTPUT - ;; - *) - echo "dockerfile=Dockerfile.base" >> $GITHUB_OUTPUT - echo "image_name=mep3" >> $GITHUB_OUTPUT - ;; - esac - - - uses: docker/setup-buildx-action@v3 - - - name: Build and Push Docker Image - uses: docker/build-push-action@v5 - with: - context: docker - file: ./docker/${{ steps.configure_docker_params.outputs.dockerfile }} - build-args: | - BASE_IMAGE=${{ inputs.base_image }} - push: true - tags: | - ghcr.io/${{ github.repository_owner }}/${{ steps.configure_docker_params.outputs.image_name }}:latest - ghcr.io/${{ github.repository_owner }}/${{ steps.configure_docker_params.outputs.image_name }}:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/build_and_push_images.yml b/.github/workflows/build_and_push_images.yml index 0f371f124..33392f55e 100644 --- a/.github/workflows/build_and_push_images.yml +++ b/.github/workflows/build_and_push_images.yml @@ -4,15 +4,33 @@ on: - main paths: - "docker/**" - - ".github/workflows/build_and_push_image_workflow.yml" - ".github/workflows/build_and_push_images.yml" + jobs: - build-base: - uses: ./.github/workflows/build_and_push_image_workflow.yml - notify: - needs: build-base - name: build-vnc - uses: ./.github/workflows/build_and_push_image_workflow.yml - with: - flavor: vnc - base_image: "ghcr.io/${{ github.repository }}:${{ github.sha }}" + build_and_push_images_with_bake: + name: Build and push images with bake + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: "Login to GitHub Container Registry" + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + + - name: "Set env" + run: | + echo "COMMIT_SHA=${{github.sha}}" >> $GITHUB_ENV + echo "GITHUB_REPO=${{github.repository}}" >> $GITHUB_ENV + + - name: Build and push images with bake + uses: docker/bake-action@v5 + with: + workdir: docker + push: true diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 9dbbaf170..d0dad864d 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -59,8 +59,8 @@ RUN su memristor -c 'code --install-extension eamodio.gitlens' && \ su memristor -c 'code --install-extension redhat.vscode-xml' && \ su memristor -c 'code --install-extension ms-iot.vscode-ros' -# VS Code server -RUN su memristor -c 'curl -fsSL https://code-server.dev/install.sh | sh' && \ +# VS Code server, pinning version 4.22.0 as later versions seem to have broken the installation of cpp and ros extensions +RUN su memristor -c 'curl -fsSL https://code-server.dev/install.sh | sh -s -- --version 4.22.0 ' && \ su memristor -c 'code-server --install-extension eamodio.gitlens' && \ su memristor -c 'code-server --install-extension ms-python.python' && \ su memristor -c 'code-server --install-extension ms-vscode.cpptools-extension-pack' && \ diff --git a/docker/Dockerfile.deploy b/docker/Dockerfile.deploy new file mode 100644 index 000000000..3226f5df3 --- /dev/null +++ b/docker/Dockerfile.deploy @@ -0,0 +1,71 @@ +ARG BASE_IMAGE=osrf/ros:humble-simulation +FROM $BASE_IMAGE + +USER root + +# Essentials +RUN apt-get update && apt-get install --no-install-recommends -y -o Dpkg::Options::="--force-overwrite" \ + ros-humble-navigation2 \ + ros-humble-nav2-bringup \ + ros-humble-rviz2 \ + ros-humble-teleop-twist-keyboard \ + ros-humble-dynamixel-sdk \ + ros-humble-can-msgs \ + ros-humble-ruckig \ + ros-humble-laser-filters \ + ros-humble-domain-bridge \ + ros-humble-rmw-cyclonedds-cpp \ + ros-humble-ros2-control \ + ros-humble-ros2-controllers \ + ros-humble-rqt-common-plugins \ + ros-humble-webots-ros2 \ + ros-humble-dynamixel-workbench-toolbox \ + ros-humble-behaviortree-cpp \ + libopencv-dev \ + # TODO: Question which of these are necessary + python3-pip \ + python3-pil \ + alsa \ + libxshmfence1 \ + libgtk-3-dev \ + git \ + git-lfs \ + curl \ + wget \ + vim \ + rsync \ + dialog \ + fuse + +RUN python3 -m pip install scipy transforms3d + +#HOTFIX: https://github.com/ros-controls/ros2_controllers/issues/482 +RUN wget -O /tmp/diff_drive_controller.deb http://snapshots.ros.org/humble/2022-11-23/ubuntu/pool/main/r/ros-humble-diff-drive-controller/ros-humble-diff-drive-controller_2.12.0-1jammy.20221108.202153_amd64.deb && \ + apt install -y --allow-downgrades /tmp/diff_drive_controller.deb && \ + rm -f /tmp/diff_drive_controller.deb + +# User config +COPY ./config/bashrc /tmp/bashrc + +RUN mkdir -p /memristor && \ + cat /tmp/bashrc >> /memristor/.bashrc && \ + rm -f /tmp/bashrc && \ + mkdir -p /memristor/ros2_ws/src/mep3 + +# Set the working directory +WORKDIR /root/ros2_ws + +RUN git clone https://github.com/memristor/mep3 src/mep3 + +RUN touch src/mep3/mep3_simulation/COLCON_IGNORE + +RUN apt-get update && \ + apt-get install -y python3-vcstool && \ + rosdep update && \ + rosdep install --from-paths src --ignore-src -r -y + +# Build the packages +RUN . /opt/ros/humble/setup.sh && \ + colcon build --symlink-install --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo + +ENTRYPOINT ["/bin/bash", "-c", "source install/local_setup.bash && exec bash"] diff --git a/docker/Dockerfile.vnc b/docker/Dockerfile.vnc index 477adcc18..744ec50d9 100644 --- a/docker/Dockerfile.vnc +++ b/docker/Dockerfile.vnc @@ -1,5 +1,4 @@ -ARG BASE_IMAGE=ghcr.io/memristor/mep3 -FROM $BASE_IMAGE +FROM mep3 USER root diff --git a/docker/Makefile b/docker/Makefile index 6d38615ab..8349d4f68 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,9 +1,14 @@ +SHELL := $(shell which bash) +.SHELLFLAGS := -eu -o pipefail -c + + MAKEFLAGS+=--silent UID:=$(shell id -u) DOCKER_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) PROJECT_DIR:=$(shell dirname ${DOCKER_DIR}) NVIDIA_GPU:=$(shell (docker info | grep Runtimes | grep nvidia 1> /dev/null && command -v nvidia-smi 1>/dev/null 2>/dev/null && nvidia-smi | grep Processes 1>/dev/null 2>/dev/null) && echo '--runtime nvidia --gpus all' || echo '') -FLAVOR=devel +BUILDX_INSTALLED := $(shell docker buildx 1>/dev/null 2>&1 && echo true) +FLAVOR=base IMAGE=ghcr.io/memristor/mep3 .PHONY: all @@ -21,14 +26,43 @@ vnc: $(eval IMAGE=ghcr.io/memristor/mep3-vnc) true -build: - echo ${NO_CACHE_ARG} - DOCKER_BUILDKIT=1 docker build ${DOCKER_DIR} -f ${DOCKER_DIR}/Dockerfile.base -t mep3 ${DOCKER_ARGS} --build-arg UID=${UID} - [ ${FLAVOR} != 'devel' ] && \ - DOCKER_BUILDKIT=1 docker build ${DOCKER_DIR} -f ${DOCKER_DIR}/Dockerfile.${FLAVOR} -t ${IMAGE} ${DOCKER_ARGS} || \ - true +deploy: + $(eval FLAVOR=deploy) + $(eval IMAGE=ghcr.io/memristor/mep3-deploy) + true + +multiple: + $(eval FLAVOR=multiple) + true + +## Test if the provided command exists +exists/cmd/%: + @hash $(*) > /dev/null 2>&1 || (echo "ERROR: '$(*)' must be installed"; exit 1) + +## Test if the provided environment variable exists +exists/env/%: + @if [ -z '$($(*))' ]; then echo "ERROR: environment variable '$*' not set" && exit 1; fi + + +# Docker since version 23.0 has been including buildx by default, current live version is 27.2.1 as of 2024-09-16 +build: | exists/cmd/docker colors + if [ -n "${BUILDX_INSTALLED}" ]; then \ + [ ${FLAVOR} == 'multiple' ] && docker buildx bake && exit 0; \ + [ ${FLAVOR} == 'base' ] && docker buildx bake mep3 && exit 0; \ + docker buildx bake mep3-${FLAVOR} && exit 0; \ + else \ + printf '%b\n' "${RED}Docker buildx is not present, it is highly recommended to install newer version of docker\n${NC}" || \ + [ ${FLAVOR} = 'multiple' ] && \ + DOCKER_BUILDKIT=1 docker build ${DOCKER_DIR} -f ${DOCKER_DIR}/Dockerfile.base -t mep3 ${DOCKER_ARGS} --build-arg UID=${UID} && \ + DOCKER_BUILDKIT=1 docker build ${DOCKER_DIR} -f ${DOCKER_DIR}/Dockerfile.vnc -t mep3-vnc ${DOCKER_ARGS} && \ + DOCKER_BUILDKIT=1 docker build ${DOCKER_DIR} -f ${DOCKER_DIR}/Dockerfile.deploy -t mep3-deploy ${DOCKER_ARGS} && exit 0; \ + [ ${FLAVOR} = 'base' ] && \ + DOCKER_BUILDKIT=1 docker build ${DOCKER_DIR} -f ${DOCKER_DIR}/Dockerfile.${FLAVOR} -t mep3 ${DOCKER_ARGS} --build-arg UID=${UID} && exit 0; \ + [ ${FLAVOR} != 'base' ] && \ + DOCKER_BUILDKIT=1 docker build ${DOCKER_DIR} -f ${DOCKER_DIR}/Dockerfile.${FLAVOR} -t mep3-${FLAVOR} ${DOCKER_ARGS} && exit 0; \ + fi -run: test-nvidia +run: | exists/cmd/docker test-nvidia docker run \ --net=host \ --ipc=host \ @@ -45,7 +79,7 @@ run: test-nvidia -v ${PROJECT_DIR}:/memristor/ros2_ws/src/mep3:rw \ -d -it ${IMAGE} 1>/dev/null -test-nvidia: colors +test-nvidia: | exists/cmd/docker lspci | grep -qi nvidia && base64 --decode massage | unxz || true docker run --rm \ -e NVIDIA_DRIVER_CAPABILITIES=all ${NVIDIA_GPU} \ @@ -54,26 +88,26 @@ test-nvidia: colors printf '%b\n' "${RED}Detected NVIDIA GPU in system, but missing packets, look up NVIDIA GPU section in README!\n${NC}" || \ true -start-code-server: +start-code-server: | exists/cmd/docker docker exec -d -it mep3-${FLAVOR} bash -c 'pgrep code-server || code-server /memristor/ros2_ws/src/mep3' && \ xdg-open 'localhost:31415?folder=/memristor/ros2_ws/src/mep3' -stop-code-server: +stop-code-server: | exists/cmd/docker docker exec -it mep3-${FLAVOR} pkill -f code-server -exec: +exec: | exists/cmd/docker docker exec -it mep3-${FLAVOR} bash -destroy: +destroy: | exists/cmd/docker docker container kill mep3-${FLAVOR} 1>/dev/null || true docker container rm -f mep3-${FLAVOR} 1>/dev/null || true -setup-default: colors +setup-default: | exists/cmd/docker colors docker exec -it mep3-${FLAVOR} sh -c '/usr/bin/setup.sh --default' printf '%b\n%b\n' "${GREEN}Default setup complete!${NC}" \ "Run ${BOLD}make exec${NC} or ${BOLD}docker exec -it mep3-${FLAVOR}${NC} to access the container" -setup-interactive: colors +setup-interactive: | exists/cmd/docker colors docker exec -it mep3-${FLAVOR} sh -c '/usr/bin/setup.sh --interactive' printf '%b\n%b\n' "${GREEN}Interactive setup complete!${NC}" \ "Run ${BOLD}make exec${NC} or ${BOLD}docker exec -it mep3-${FLAVOR}${NC} to access the container" diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl new file mode 100644 index 000000000..76f88ff47 --- /dev/null +++ b/docker/docker-bake.hcl @@ -0,0 +1,50 @@ +variable "COMMIT_SHA" {} + +variable "GITHUB_REPO" {} + +variable "TARGET_IMAGE_NAME_MAPPING" { + default = { + "base" = "mep3" + "vnc" = "mep3-vnc" + "deploy" = "mep3-deploy" + } +} + +variable CONTEXTS_MAPPING { + default = { + "vnc" = { + "mep3" = "target:mep3" + } + } +} + +function "eval_tags" { + params = [image_name, commit_sha, github_repo] + result = [ + // If GITHUB_REPO is not set, then we are building locally + equal("", github_repo) ? image_name : "", + equal("", github_repo) && notequal("", commit_sha) ? "${image_name}:${commit_sha}" : "", + // otherwise, we are building on GitHub Actions + notequal("", github_repo) ? "ghcr.io/${github_repo}/${image_name}:latest" : "", + notequal("", github_repo) && notequal("", commit_sha) ? "ghcr.io/${github_repo}:${commit_sha}" : "" + ] +} + +target "default" { + name = lookup(TARGET_IMAGE_NAME_MAPPING, tgt, "") + matrix = { + tgt = keys(TARGET_IMAGE_NAME_MAPPING) + } + + tags = eval_tags(lookup(TARGET_IMAGE_NAME_MAPPING, tgt, ""), COMMIT_SHA, GITHUB_REPO) + + # Use Dockerfile.${tgt} for each target as defined by targets in TARGET_IMAGE_NAME_MAPPING + dockerfile = "Dockerfile.${tgt}" + + # Cache settings, enable caching from previous builds + cache-to = [format("%s%s", "type=gha,mode=max,scope=", lookup(TARGET_IMAGE_NAME_MAPPING, tgt, ""))] + cache-from = [format("%s%s", "type=gha,scope=", lookup(TARGET_IMAGE_NAME_MAPPING, tgt, ""))] + + # Contexts for dependent images, as defined in CONTEXTS_MAPPING graph, i.e. mep3-vnc builds atop mep3 + contexts = lookup(CONTEXTS_MAPPING, tgt, {}) +}