From 2db29e3206c9e3f8fe1b3504b7ecfca137fc3d99 Mon Sep 17 00:00:00 2001 From: Saheer Babu Date: Wed, 9 Jun 2021 11:40:41 +0100 Subject: [PATCH 01/17] github action workflow for docker management --- .github/workflows/ci_scripts/github_utils.py | 69 ++++ .../workflows/docker_management.branch.yml | 256 ++++++++++++++ .github/workflows/docker_management.prune.yml | 27 ++ .../workflows/docker_management.release.yml | 320 ++++++++++++++++++ .../workflows/docker_management.test-PR.yml | 62 ++++ docker_images/mbed-os-env/Dockerfile | 103 ++++++ docker_images/mbed-os-env/README.md | 49 +++ docker_images/mbed-os-env/test.sh | 17 + .../diagrams/docker-versioning.png | Bin 0 -> 31578 bytes .../docker_management/docker_management.md | 135 ++++++++ 10 files changed, 1038 insertions(+) create mode 100644 .github/workflows/ci_scripts/github_utils.py create mode 100644 .github/workflows/docker_management.branch.yml create mode 100644 .github/workflows/docker_management.prune.yml create mode 100644 .github/workflows/docker_management.release.yml create mode 100644 .github/workflows/docker_management.test-PR.yml create mode 100644 docker_images/mbed-os-env/Dockerfile create mode 100644 docker_images/mbed-os-env/README.md create mode 100755 docker_images/mbed-os-env/test.sh create mode 100644 docs/design-documents/docker_management/diagrams/docker-versioning.png create mode 100644 docs/design-documents/docker_management/docker_management.md diff --git a/.github/workflows/ci_scripts/github_utils.py b/.github/workflows/ci_scripts/github_utils.py new file mode 100644 index 00000000000..f96c3715a83 --- /dev/null +++ b/.github/workflows/ci_scripts/github_utils.py @@ -0,0 +1,69 @@ +import click +import requests +import logging +import sys +import time + +# delete images older than provided argument (number_of_days) +@click.command() +@click.pass_context +@click.option("-r", "--repository", required=True) +@click.option( + "-n", + "--number_of_days", + default="1", + help="number of days since image was created", + required=False, +) +def delete_old_images(ctx, repository, number_of_days): + s = requests.Session() + github_api_accept = "application/vnd.github.v3+json" + s.headers.update( + {"Authorization": f'token {ctx.obj["passwd"]}', "Accept": github_api_accept} + ) + r = s.get(f"https://api.github.com/user/packages/container/{repository}/versions") + versions = r.json() + + version_id = None + pattern = "%d.%m.%Y %H:%M:%S" + pattern = "%Y-%m-%dT%H:%M:%SZ" + current_time = time.time() + + for version in versions: + epoch = int(time.mktime(time.strptime(version["updated_at"], pattern))) + + if (current_time - epoch) / (24 * 60 * 60) > int(number_of_days): + version_id = version["id"] + logging.debug(f"deleteing image with version id {version_id}") + + url = f"https://api.github.com/user/packages/container/{repository}/versions/{version_id}" + resp = s.delete(url) + resp.raise_for_status() + + +@click.group() +@click.pass_context +@click.option("-u", "--username", required=False) +@click.option("-p", "--passwd", required=False) +@click.option("-v", "--verbose", is_flag=True, default=False) +def main(ctx, username, passwd, verbose): + ctx.obj = {"username": username, "passwd": passwd} + + if verbose: + logging.basicConfig( + stream=sys.stdout, + format="%(levelname)s %(asctime)s %(message)s", + datefmt="%m/%d/%Y %I:%M:%S %p", + ) + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.basicConfig( + format="%(levelname)s %(asctime)s %(message)s", + datefmt="%m/%d/%Y %I:%M:%S %p", + ) + logging.getLogger().setLevel(logging.INFO) + + +if __name__ == "__main__": + main.add_command(delete_old_images) + main() diff --git a/.github/workflows/docker_management.branch.yml b/.github/workflows/docker_management.branch.yml new file mode 100644 index 00000000000..1aa344095e1 --- /dev/null +++ b/.github/workflows/docker_management.branch.yml @@ -0,0 +1,256 @@ +name: Publish or Update docker image for head of branch +# This work flow is disabled for forked repositories. +# If you need to enable it, find references for github.repository_owner in this file +# and replace with ARMmbed with your organisation/account name +# Read more details in TODO: add design link + +on: + + schedule: + - cron: '15 4 * * *' + + push: + branches: + - master + + paths: + - requirements.txt + - docker_images/mbed-os-env/** + - .github/workflows/docker_management.branch.yml + + + workflow_dispatch: + + +jobs: + prepare-tags: + if: github.repository_owner == 'saheerb' || github.event_name != 'schedule' + runs-on: ubuntu-latest + + steps: + - + name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + + - + name: Print event name + shell: bash + run: echo ${{github.event_name}} + + - + name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - + name: Set UUID + id: generate-uuid + uses: filipstefansson/uuid-action@v1 + +# set docker tags we are building, and intending to publish +# dev-tag is temporary for testing purpose. This should be considered as unstable. +# dated-tag is created for versioning purpose +# prod-tag-latest could be used by customers, CI etc for keeping up to date + - + name: Get build information + shell: bash + run: | + mkdir -p build_info + date=$(date +"%Y.%m.%dT%H.%M.%S") + echo dev-${{ steps.extract_branch.outputs.branch }}-${date}-${{ steps.generate-uuid.outputs.uuid }} > build_info/dev_tag + echo ${{ steps.extract_branch.outputs.branch }}-${date} > build_info/prod_tag_dated + echo ${{ steps.extract_branch.outputs.branch }}-latest > build_info/prod_tag_latest + echo ${{ steps.extract_branch.outputs.branch }} > build_info/mbed_os_version + + # archiving and unarchiving are github actions ways to pass variables between jobs + - + name: Archive information + uses: actions/upload-artifact@v2 + with: + name: build-info + path: build_info + + + + build-container: + runs-on: ubuntu-latest + needs: prepare-tags + outputs: + DEV_DIGEST: ${{ steps.docker_info_dev.outputs.DIGEST }} + PROD_DIGEST: ${{ steps.docker_info_prod.outputs.DIGEST }} + + steps: + - + name: unarchive artefacts + uses: actions/download-artifact@v2 + with: + name: build-info + + - + name: Get build info from archive + shell: bash + id: build_info + run: | + value=`cat dev_tag` + echo "DEV TAG is $value" + echo "::set-output name=DOCKER_DEV_TAG::$value" + value=`cat prod_tag_dated` + echo "PROD TAG DATED is $value" + echo "::set-output name=DOCKER_PROD_TAG_DATED::$value" + value=`cat prod_tag_latest` + echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value" + echo "PROD TAG is $value" + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - + name: Checkout + uses: actions/checkout@v2 + + - + name: Build with remote cache + uses: docker/build-push-action@v2 + id: docker_build_dev + with: + context: . + platforms: linux/amd64,linux/arm64 + # load: true - not supported for multi arch + push: true + file: ./docker_images/mbed-os-env/Dockerfile + tags: ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} + cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_LATEST }} + cache-to: type=inline + + - + name: Find DOCKER DIGEST trying to publish (if exists in docker repository) + id: docker_info_prod + run: | + DIGEST=$(docker run quay.io/skopeo/stable --creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} inspect docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_LATEST }} | jq ".Digest") + echo "::set-output name=DIGEST::$DIGEST" + echo "Docker DIGEST: $DIGEST" + + - + name: Find DEV DOCKER DIGEST + id: docker_info_dev + run: | + DIGEST=$(docker run quay.io/skopeo/stable --creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} inspect docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} | jq ".Digest") + echo "Docker DIGEST: $DIGEST" + echo "::set-output name=DIGEST::$DIGEST" + echo "Docker DIGEST: $DIGEST" + + test-container: + runs-on: ubuntu-latest + needs: build-container + # no need to test (nor publish) if we already have the same image in dockerhub + if: needs.build-container.outputs.DEV_DIGEST != needs.build-container.outputs.PROD_DIGEST + outputs: + STATUS: ${{ steps.test.outputs.STATUS }} + strategy: + matrix: + platform: [linux/amd64, linux/arm64] + + steps: + - + name: unarchive artefacts + uses: actions/download-artifact@v2 + with: + name: build-info + + - + name: Get build info from archive + shell: bash + id: build_info + run: | + value=`cat dev_tag` + echo "TAG is $value" + echo "::set-output name=DOCKER_DEV_TAG::$value" + value=`cat prod_tag_dated` + echo "TAG is $value" + echo "::set-output name=DOCKER_PROD_TAG_DATED::$value" + value=`cat prod_tag_latest` + echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value" + value=`cat mbed_os_version` + echo "::set-output name=MBED_OS_VERSION::$value" + + - + name: Checkout + uses: actions/checkout@v2 + + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - + name: test the container + id: test + uses: addnab/docker-run-action@v3 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + options: -v ${{ github.workspace }}:/work -w=/work + image: ghcr.io/${{ github.actor }}/mbed-os-env-tmp@${{ steps.docker_info_dev.outputs.DIGEST }} + shell: bash + + run: | + set -e + echo "::set-output name=STATUS::failed" + uname -m + ./docker_images/mbed-os-env/test.sh ${{ steps.build_info.outputs.MBED_OS_VERSION }} + echo "::set-output name=STATUS::passed" + + + deploy-container: + runs-on: ubuntu-latest + needs: test-container + + steps: + - + name: unarchive artefacts + uses: actions/download-artifact@v2 + with: + name: build-info + + - + name: Get build info from archive + shell: bash + id: build_info + run: | + value=`cat dev_tag` + echo "TAG is $value" + echo "::set-output name=DOCKER_DEV_TAG::$value" + value=`cat prod_tag_dated` + echo "TAG is $value" + echo "::set-output name=DOCKER_PROD_TAG_DATED::$value" + value=`cat prod_tag_latest` + echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value" + + - + if: needs.test-container.outputs.STATUS == 'passed' + name: copy dev tag to prod + run: | + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_LATEST }} + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_DATED }} + + # move failed images to temporary name for troubleshooting + - if: needs.test-container.outputs.STATUS == 'failed' + name: Rename Dev Tag + run: | + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:failed-${{ steps.build_info.outputs.DOCKER_DEV_TAG }} diff --git a/.github/workflows/docker_management.prune.yml b/.github/workflows/docker_management.prune.yml new file mode 100644 index 00000000000..8891531b7dd --- /dev/null +++ b/.github/workflows/docker_management.prune.yml @@ -0,0 +1,27 @@ +name: Prune temporary docker images + +on: + schedule: + - cron: '15 4 * * *' + + workflow_dispatch: + +jobs: + prune-images: + if: github.repository_owner == 'saheerb' + runs-on: ubuntu-latest + + steps: + - + name: Checkout + uses: actions/checkout@v2 + + - + name: Delete old temporary images + run: | + # the following command may fail because github package doesn't allow + # deletion if only one image exists or if GITHUB_DELETE_IMAGE_TOKEN is not + # setup. This shouldn't create any alarm as temporary image deletion is + # not a critical activity. + python ./.github/workflows/ci_scripts/github_utils.py -u ${{ github.actor }} -p ${{ secrets.DOCKER_MANAGEMENT_TOKEN }} delete-old-images -r mbed-os-env-tmp | true + \ No newline at end of file diff --git a/.github/workflows/docker_management.release.yml b/.github/workflows/docker_management.release.yml new file mode 100644 index 00000000000..9ced2feab27 --- /dev/null +++ b/.github/workflows/docker_management.release.yml @@ -0,0 +1,320 @@ +name: Release or update docker image for a released mbed-os version + +# Read TODO: add design link before editing this file + +# This work flow is disabled for forked repositories. +# If you need to enable it, find references for github.repository_owner in this file +# and replace with ARMmbed with your organisation/account name + + +on: + push: + tags: + - mbed-os-6.[0-9]+.[0-9]+ + + schedule: + - cron: '15 4 * * *' + + workflow_dispatch: + inputs: + mbed_os_release_version: + description: 'mbed-os release version for which you want to update docker image.' + required: true + +jobs: + # this job finds the necessary tags to be applied to docker image + prepare-tags: + if: github.repository_owner == 'saheerb' + runs-on: ubuntu-latest + + steps: + - + name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - + # if manually triggered makes sure that input tag exists on the branch selected + # this is to avoid silly mistakes, for example: mbed-os-7.0.16-latest docker image + # getting released on mbed-os-6 branch + if: ${{ github.event_name == 'workflow_dispatch' }} + name: Sanity check tag of manual trigger + shell: bash + run: | + if [ -z $(git tag --merged ${GITHUB_REF} ${{ github.event.inputs.mbed_os_release_version }}) ]; then + echo "Check the tag name ${{ github.event.inputs.mbed_os_release_version }} is not found on branch ${GITHUB_REF}" + exit 1 + fi + + - + name: Set UUID + id: generate-uuid + uses: filipstefansson/uuid-action@v1 + + - + # set docker tags we are building, and intending to release + # this workflow can be executed when a tag is pushed, scheduled, or manual trigger + # depending on the trigger cache source and version of mbed-os will change + # + # when trigger is due to tag pushed (ie, a new mbed-os release is made), + # we are targeting to build docker image for the tag + # best bet for cache source is the image built for head of the branch + # when trigger is manual + # we are targeting to build docker image for the input tag in workflow + # best bet for cache source is input tag itself as it should already be there + # when trigger is scheduled + # we are targeting to build docker image for the last tag on the branch + # best bet for cache source is input tag itself as it should already be there + name: Get build information + shell: bash + run: | + mkdir -p build_info + date=$(date +"%Y.%m.%dT%H.%M.%S") + if [ "push" == "${{github.event_name}}" ];then + version=${GITHUB_REF#refs/tags/} + cache_source="master-latest" + elif [ "workflow_dispatch" == "${{github.event_name}}" ];then + version=${{ github.event.inputs.mbed_os_release_version }} + cache_source=${version}-latest + else + version=`git describe --tags --abbrev=0 --match mbed-os-[0-9]*.[0-9]*.[0-9]*` + cache_source=${version}-latest + fi + echo dev-${version}-${date}-${version} > build_info/dev_tag + echo ${version}-${date} > build_info/prod_tag_dated + echo ${version} > build_info/mbed_os_version + echo ${cache_source} > build_info/cache_source + + # archiving and unarchiving are github actions ways to pass variables between jobs + - + name: Archive information + uses: actions/upload-artifact@v2 + with: + name: build-info + path: build_info + + + build-container: + runs-on: ubuntu-latest + needs: prepare-tags + outputs: + DEV_DIGEST: ${{ steps.docker_info_dev.outputs.DIGEST }} + PROD_DIGEST: ${{ steps.docker_info_prod.outputs.DIGEST }} + + steps: + - + name: unarchive artefacts + uses: actions/download-artifact@v2 + with: + name: build-info + + # DOCKER_DEV_TAG is temporary image name. + # DOCKER_PROD_TAG_DATED is fixed tag + # prod-tag-latest could be used by customers, CI etc for keeping up to date + - + name: Get build info from archive + shell: bash + id: build_info + run: | + value=`cat dev_tag` + echo "TAG is $value" + echo "::set-output name=DOCKER_DEV_TAG::$value" + value=`cat prod_tag_dated` + echo "TAG is $value" + echo "::set-output name=DOCKER_PROD_TAG_DATED::$value" + value=`cat mbed_os_version` + echo "::set-output name=MBED_OS_VERSION::$value" + value=`cat cache_source` + echo "::set-output name=CACHE_SOURCE::$value" + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - + name: Checkout + uses: actions/checkout@v2 + with: + ref: refs/tags/${{ steps.build_info.outputs.MBED_OS_VERSION }} + + - + name: Build with remote cache + uses: docker/build-push-action@v2 + id: docker_build_dev + with: + context: . + platforms: linux/amd64,linux/arm64 + # load: true - not supported for multi arch + push: true + file: ./docker_images/mbed-os-env/Dockerfile + tags: ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} + cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.CACHE_SOURCE }} + cache-to: type=inline + + - + name: Find DOCKER DIGEST trying to publish (if exists in docker repository) + id: docker_info_prod + run: | + DIGEST=$(docker run quay.io/skopeo/stable --creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} inspect docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.CACHE_SOURCE }} | jq ".Digest") + echo "::set-output name=DIGEST::$DIGEST" + echo "Docker DIGEST: $DIGEST" + + - + name: Find DEV DOCKER DIGEST + id: docker_info_dev + run: | + DIGEST=$(docker run quay.io/skopeo/stable --creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} inspect docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} | jq ".Digest") + echo "Docker DIGEST: $DIGEST" + echo "::set-output name=DIGEST::$DIGEST" + echo "Docker DIGEST: $DIGEST" + + test-container: + runs-on: ubuntu-latest + needs: build-container + # no need to test if we already have the same image in dockerhub + if: needs.build-container.outputs.DEV_DIGEST != needs.build-container.outputs.PROD_DIGEST + outputs: + STATUS: ${{ steps.test.outputs.STATUS }} + strategy: + matrix: + platform: [linux/amd64, linux/arm64] + + steps: + - + name: unarchive artefacts + uses: actions/download-artifact@v2 + with: + name: build-info + + - + name: Get build info from archive + shell: bash + id: build_info + run: | + value=`cat dev_tag` + echo "TAG is $value" + echo "::set-output name=DOCKER_DEV_TAG::$value" + value=`cat prod_tag_dated` + echo "TAG is $value" + echo "::set-output name=DOCKER_PROD_TAG_DATED::$value" + value=`cat mbed_os_version` + echo "::set-output name=MBED_OS_VERSION::$value" + + - + name: Checkout + uses: actions/checkout@v2 + with: + ref: refs/tags/${{ steps.build_info.outputs.MBED_OS_VERSION }} + + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - + name: test the container + id: test + uses: addnab/docker-run-action@v3 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + options: -v ${{ github.workspace }}:/work -w=/work + image: ghcr.io/${{ github.actor }}/mbed-os-env-tmp@${{ steps.docker_info_dev.outputs.DIGEST }} + shell: bash + run: | + set -e + echo "::set-output name=STATUS::failed" + uname -m + ./docker_images/mbed-os-env/test.sh ${{ steps.build_info.outputs.MBED_OS_VERSION }} + echo "::set-output name=STATUS::passed" + + + deploy-container: + # this job is running always and to skip for forked repo + # add condition on every step. + env: + repo-owner: saheerb + runs-on: ubuntu-latest + needs: test-container + if: ${{ always() }} + + steps: + - + if: github.repository_owner == env.repo-owner + name: unarchive artefacts + uses: actions/download-artifact@v2 + with: + name: build-info + + - + if: github.repository_owner == env.repo-owner + name: Get build info from archive + shell: bash + id: build_info + run: | + value=`cat dev_tag` + echo "DEV TAG is $value" + echo "::set-output name=DOCKER_DEV_TAG::$value" + value=`cat prod_tag_dated` + echo "DATED PROD TAG is $value" + echo "::set-output name=DOCKER_PROD_TAG_DATED::$value" + value=`cat mbed_os_version` + echo "MBED OS VERSION is $value" + echo "::set-output name=MBED_OS_VERSION::$value" + + - + if: github.repository_owner == env.repo-owner + name: Checkout + uses: actions/checkout@v2 + with: + ref: refs/tags/${{ steps.build_info.outputs.MBED_OS_VERSION }} + + # even if tests skipped, as cache used is HEAD of branch, copy images to right tag + - + if: github.repository_owner == env.repo-owner && needs.test-container.result=='skipped' + name: copy dev tag to prod + run: | + set -x + echo ${{ needs.test-container.result }} + upto_patch_version=${{ steps.build_info.outputs.MBED_OS_VERSION }}-latest + upto_min_version=${upto_patch_version%.[0-9]*}-latest + upto_major_version=${upto_patch_version%.[0-9]*.[0-9]*}-latest + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${upto_patch_version} + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${upto_min_version} + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${upto_major_version} + + # when tests succeed copy to fixed tags + - + if: github.repository_owner == env.repo-owner && needs.test-container.outputs.STATUS == 'passed' + name: copy dev tag to prod dated tag + run: | + set -x + echo ${{ needs.test-container.result }} + upto_patch_version=${{ steps.build_info.outputs.MBED_OS_VERSION }}-latest + upto_min_version=${upto_patch_version%.[0-9]*}-latest + upto_major_version=${upto_patch_version%.[0-9]*.[0-9]*}-latest + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${upto_patch_version} + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${upto_min_version} + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${upto_major_version} + + # copy to fixed tag + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_DATED }} + + # when tests fail, rename to "failed-" for easy identification + - if: github.repository_owner == env.repo-owner && needs.test-container.outputs.STATUS == 'failed' + name: Rename Dev Tag + run: | + docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:failed-${{ steps.build_info.outputs.DOCKER_DEV_TAG }} diff --git a/.github/workflows/docker_management.test-PR.yml b/.github/workflows/docker_management.test-PR.yml new file mode 100644 index 00000000000..e6e8af5f926 --- /dev/null +++ b/.github/workflows/docker_management.test-PR.yml @@ -0,0 +1,62 @@ +name: Build and test image when PR is raised + +# This workflow is triggered when Dockerfile relatedt or github action itself changes are made in a PR +# The workflow is quite simple - builds and test the image. Release of newer version is done only when PR is merged. + +on: + pull_request: + branches: [ master ] + paths: + - docker_images/mbed-os-env/** + - .github/workflows/docker_management.* + - requirements.txt + +jobs: + + build-container: + runs-on: ubuntu-latest + + strategy: + matrix: + platform: [linux/amd64, linux/arm64] + steps: + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - + name: Checkout + uses: actions/checkout@v2 + + + # cache documentation: https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md + - + name: Build with remote cache + uses: docker/build-push-action@v2 + id: docker_build_dev + with: + context: . + platforms: ${{ matrix.platform }} + file: ./docker_images/mbed-os-env/Dockerfile + load: true + tags: mbed-os-env:a_pr_test + cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/mbed-os-env:master-latest + + + - + name: test the container + id: test + uses: addnab/docker-run-action@v2 + with: + options: -v ${{ github.workspace }}:/work -w=/work + image: mbed-os-env:a_pr_test + shell: bash + run: | + set -e + uname -m + ./docker_images/mbed-os-env/test.sh diff --git a/docker_images/mbed-os-env/Dockerfile b/docker_images/mbed-os-env/Dockerfile new file mode 100644 index 00000000000..570f750e012 --- /dev/null +++ b/docker_images/mbed-os-env/Dockerfile @@ -0,0 +1,103 @@ +# ------------------------------------------------------------------------------ +# Pull base image +FROM ubuntu:20.04 + +# ------------------------------------------------------------------------------ +# Arguments +ARG WORKDIR=/root + +# ------------------------------------------------------------------------------ +# Install tools via apt +RUN ls + +ADD requirements.txt . +ENV DEBIAN_FRONTEND=noninteractive +RUN set -x \ + && apt -y update \ + && apt -y install \ + git \ + wget \ + python3 \ + python3-dev \ + python3-setuptools \ + python3-usb \ + python3-pip \ + software-properties-common \ + build-essential \ + astyle \ + mercurial \ + ninja-build \ + libssl-dev \ + cargo \ + && apt clean && rm -rf /var/lib/apt/lists \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 \ + && : # last line + +# Cmake (Mbed OS requires >=3.19.0-rc3 version which is not available in Ubuntu 20.04 repository) +RUN set -x \ + && (cd /tmp \ + && [ "$(uname -m)" = "aarch64" ] && \ + CMAKE_SCRIPT="cmake-3.19.3-Linux-aarch64.sh" || \ + CMAKE_SCRIPT="cmake-3.19.3-Linux-x86_64.sh" \ + && wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/v3.19.3/${CMAKE_SCRIPT} \ + && sh ${CMAKE_SCRIPT} --exclude-subdir --prefix=/usr/local \ + && rm -rf ${CMAKE_SCRIPT}) \ + && exec bash \ + && : # last line + +# ------------------------------------------------------------------------------ +# Install Python modules (which are not included in requirements.txt) +RUN set -x \ + && pip3 install -U \ + mbed-cli \ + mbed-tools \ + && : # last line + +# Set up mbed environment +WORKDIR /tmp/ +COPY requirements.txt . +RUN set -x \ + && pip3 install -r requirements.txt \ + && : # last line + +# ------------------------------------------------------------------------------ +# Install arm-none-eabi-gcc +WORKDIR /opt/mbed-os-toolchain +RUN set -x \ + && [ "$(uname -m)" = "aarch64" ] && \ + TARBALL="gcc-arm-none-eabi-9-2019-q4-major-aarch64-linux.tar.bz2" || \ + TARBALL="gcc-arm-none-eabi-9-2019-q4-major-x86_64-linux.tar.bz2" \ + && wget -q https://developer.arm.com/-/media/Files/downloads/gnu-rm/9-2019q4/RC2.1/${TARBALL} \ + && tar -xjf ${TARBALL} \ + && rm ${TARBALL} \ + && : # last line + +# ------------------------------------------------------------------------------ +# Configure mbed build system +RUN set -x \ + && mbed config -G GCC_ARM_PATH /opt/mbed-os-toolchain/gcc-arm-none-eabi-9-2019-q4-major/bin/ \ + && mbed toolchain -G -s GCC_ARM \ + && : # last line + +# ------------------------------------------------------------------------------ +# Configure environment variables +ENV MBED_GCC_ARM_PATH=/opt/mbed-os-toolchain/gcc-arm-none-eabi-9-2019-q4-major/bin/ +ENV PATH="${PATH}:${MBED_GCC_ARM_PATH}" + +# ------------------------------------------------------------------------------ +# Display, check and save environment settings +# NOTE: using bash instead of Ubuntu default dash due to unsupport for pipefail +# Pipefail is crucial here, if the tools didn't install properly, docker build should not pass because of piping +RUN /bin/bash -c \ + "set -x -o pipefail \ + && arm-none-eabi-gcc --version | grep arm-none-eabi-gcc | tee env_settings \ + && cmake --version | grep version | tee -a env_settings \ + && python --version 2>&1 | tee -a env_settings \ + && (echo -n 'mbed-cli ' && mbed --version) | tee -a env_settings \ + && (echo -n 'mbed-greentea ' && mbedgt --version | grep ^[0-9]) | tee -a env_settings \ + && (echo -n 'mbed-host-tests ' && mbedhtrun --version) | tee -a env_settings \ + && (echo -n 'mbed-tools ' && mbed-tools --version) | tee -a env_settings \ + && : # LAST LINE" + + +WORKDIR /root diff --git a/docker_images/mbed-os-env/README.md b/docker_images/mbed-os-env/README.md new file mode 100644 index 00000000000..3c492b0733e --- /dev/null +++ b/docker_images/mbed-os-env/README.md @@ -0,0 +1,49 @@ +# mbed-OS development environment docker image + +This docker image is the official mbed-OS development environment. +* It is based on ubuntu 20.04 +* Python and cmake are installed +* arm-none-eabi-gcc toolchain is installed +* Latest released version of mbed-cli and GreenTea are installed +* All other mbed-OS dependency tools are installed. + +# How to use docker image: + +## Pull docker images +```bash +docker pull mbedos/mbed-os-env + +``` + +## Run mbed OS environment without HW support (build mbed images only) +launch docker by +```bash +docker run -it mbedos/mbed-os-env: +``` +Then you will have the container with mbed OS development environment. +you should be able to compile mbed commands/examples as recommenced in mbed documentations +e.g. +```bash +mbed import mbed-os-example-blinky +cd mbed-os-example-blinky +mbed compile -m -t GCC_ARM +``` + +## Run mbed OS environment with HW support (USB pass-through) +If you want to use this docker container connect and flash your targets. you need some extra command line option to pass-through your USB devices. +```bash +sudo docker run -it --privileged -v /dev/disk/by-id:/dev/disk/by-id -v /dev/serial/by-id:/dev/serial/by-id mbed-os-env: