diff --git a/build_openroad.sh b/build_openroad.sh index c6221c9867..5c1ff90e5e 100755 --- a/build_openroad.sh +++ b/build_openroad.sh @@ -206,6 +206,8 @@ EOF fi fi +echo "[INFO FLW-0028] Compiling with ${PROC} threads." + # Only add install prefix variables after parsing arguments. YOSYS_ARGS+=" PREFIX=${INSTALL_PATH}/yosys" OPENROAD_APP_ARGS+=" -D CMAKE_INSTALL_PREFIX=${INSTALL_PATH}/OpenROAD" diff --git a/etc/DockerHelper.sh b/etc/DockerHelper.sh index 83e1198684..d8831e0dee 100755 --- a/etc/DockerHelper.sh +++ b/etc/DockerHelper.sh @@ -24,8 +24,7 @@ usage: $0 [CMD] [OPTIONS] 'builder': os + packages to compile app + copy source code and build app -threads Max number of threads to use if compiling. - -sha Use git commit sha as the tag image. Default is - 'latest'. + Default = \$(nproc) -ci Install CI tools in image -h -help Show this message and exits -username Docker Username @@ -36,7 +35,6 @@ EOF } _setup() { - commitSha="$(git rev-parse HEAD | tr -cd 'a-zA-Z0-9-')" case "${os}" in "ubuntu20.04") osBaseImage="ubuntu:20.04" @@ -50,9 +48,10 @@ _setup() { ;; esac imageName="${IMAGE_NAME_OVERRIDE:-"${org}/flow-${os}-${target}"}" - imageTag="${commitSha}" - if [[ "${tag}" != "NONE" ]]; then + if [[ "${tag}" != "" ]]; then imageTag="${tag}" + else + imageTag=$(./etc/DockerTag.sh -dev) fi case "${target}" in "builder" | "master") @@ -95,43 +94,34 @@ _push() { echo "Missing required -password= argument" _help fi - docker login --username ${username} --password ${password} - if [[ "${tag}" == "NONE" ]]; then - tag="latest" + if [[ "${target}" != "dev" ]] && [[ "${target}" != "master" ]]; then + echo "Target ${target} is not valid candidate for push to Docker Hub." >&2 + _help fi - mkdir -p build - case "${target}" in - "dev" ) - ./etc/DockerHelper.sh create -os=${os} -ci -target=${target} \ - 2>&1 | tee build/create-${os}-${target}-${tag}.log - docker push ${imagePath} - ;; - "master" ) - # Create dev image needed as a base for builder image - ./etc/DockerHelper.sh create -os=${os} -target=dev \ - 2>&1 | tee build/create-${os}-dev-${target}-${tag}.log - # Create builder image - ./etc/DockerHelper.sh create -os=${os} -target=builder \ - 2>&1 | tee build/create-${os}-${target}-${tag}.log + docker login --username "${username}" --password "${password}" - docker push ${org}/flow-${os}-dev:${commitSha} + if [[ "${tag}" == "" ]]; then + tag=$(./etc/DockerTag.sh -dev) + fi - docker tag ${org}/flow-${os}-dev:${commitSha} ${org}/flow-${os}-dev:latest - docker push ${org}/flow-${os}-dev:latest + mkdir -p build + ./etc/DockerHelper.sh create -os=${os} -target=dev -tag=${tag} -ci \ + 2>&1 | tee build/create-${os}-dev-${tag}.log - docker tag ${org}/flow-${os}-builder:${commitSha} ${org}/orfs:${commitSha} - docker push ${org}/orfs:${commitSha} + docker push "${imageName}:${tag}" - docker tag ${org}/flow-${os}-builder:${commitSha} ${org}/orfs:${tag} - docker push ${org}/orfs:${tag} - ;; + if [[ "${target}" == "master" ]]; then + tag=$(./etc/DockerTag.sh -master) + # Create builder image + ./etc/DockerHelper.sh create -os=${os} -target=builder -tag=${tag} \ + 2>&1 | tee build/create-${os}-${target}-${tag}.log - *) - echo "Target ${target} is not valid candidate for push to Docker Hub." >&2 - _help - ;; - esac + docker tag ${org}/flow-${os}-builder:${tag} ${org}/orfs:${tag} + docker push ${org}/orfs:${tag} + docker tag ${org}/flow-${os}-builder:${tag} ${org}/orfs:${tag} + docker push ${org}/orfs:${tag} + fi } # @@ -161,7 +151,7 @@ fi os="ubuntu22.04" target="dev" numThreads="-1" -tag="NONE" +tag="" options="" while [ "$#" -gt 0 ]; do diff --git a/etc/DockerTag.sh b/etc/DockerTag.sh new file mode 100755 index 0000000000..d8718e769a --- /dev/null +++ b/etc/DockerTag.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +cd $(dirname $(realpath $0))/../ + +if [[ "$@" == "-dev" ]]; then + file_list=( + "./build_openroad.sh" + "./docker/Dockerfile.builder" + "./docker/Dockerfile.dev" + "./etc/DependencyInstaller.sh" + "./etc/DockerHelper.sh" + "./tools/OpenROAD/docker/Dockerfile.builder" + "./tools/OpenROAD/docker/Dockerfile.dev" + "./tools/OpenROAD/docker/Dockerfile.binary" + "./tools/OpenROAD/etc/DependencyInstaller.sh" + ) + cat "${file_list[@]}" | sha256sum | awk '{print substr($1, 1, 6)}' +elif [[ "$@" == "-master" ]]; then + git describe +else + echo "Usage: $0 {-dev|-master}" + exit 1 +fi diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 284ded0318..835adf3fac 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -194,17 +194,21 @@ def get_summary(status, text): ''' TODO: docs ''' - text += '-' * 30 + '\n' - text += ' ' * 7 + f"{status} designs\n" - text += '-' * 30 + '\n' + tmp = '' for name, data in design_list.items(): if data['status'] == status: content = gen_report(name, data) if content != '': - text += content + '\n' + tmp += content + '\n' # add empty line for readability if args.verbose >= 2 or status == STATUS_RED: - text += '\n' + tmp += '\n' + if tmp != '': + text += '-' * 30 + '\n' + text += ' ' * 7 + f"{status} designs\n" + text += '-' * 30 + '\n' + text += tmp + return text @@ -212,7 +216,8 @@ def write_summary(): ''' Write summary and individual report files with data on design_list ''' - summary = '=' * 60 + '\n' + summary = '\n' + summary += '=' * 60 + '\n' summary += ' ' * 26 + 'SUMMARY\n' summary += '=' * 60 + '\n' summary += f"\nNumber of designs: {len(design_list.keys())}\n\n" diff --git a/jenkins/public_tests_all.Jenkinsfile b/jenkins/public_tests_all.Jenkinsfile index 2c747b1c3d..56cb05a369 100644 --- a/jenkins/public_tests_all.Jenkinsfile +++ b/jenkins/public_tests_all.Jenkinsfile @@ -1,68 +1,29 @@ -@Library('utils@main') _ +@Library('utils@orfs-v2.0.1') _ -node { +node('ubuntu22') { - properties([ - copyArtifactPermission('${JOB_NAME},'+env.BRANCH_NAME), - ]); + properties([copyArtifactPermission('${JOB_NAME},'+env.BRANCH_NAME)]); stage('Checkout') { checkout scm; } - def commitHash = "none"; - def DOCKER_IMAGE_TAG = "latest"; + def DOCKER_IMAGE; stage('Build and Push Docker Image') { - if (isDependencyInstallerChanged(env.BRANCH_NAME)) { - commitHash = sh(script: 'git rev-parse HEAD', returnStdout: true); - commitHash = commitHash.replaceAll(/[^a-zA-Z0-9-]/, ''); - DOCKER_IMAGE_TAG = pushCIImage(env.BRANCH_NAME, commitHash); - } + DOCKER_IMAGE = dockerPush('ubuntu22.04', 'orfs'); + echo "Docker image is $DOCKER_IMAGE"; } - def DOCKER_IMAGE = "openroad/flow-ubuntu22.04-dev:${DOCKER_IMAGE_TAG}"; - docker.image("${DOCKER_IMAGE}").inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { - stage('Build ORFS and Stash bins') { - sh "git config --system --add safe.directory '*'"; - localBuild(); - } + stage('Build ORFS and Stash bins') { + buildBins(DOCKER_IMAGE); } stage('Run Tests') { - Map tasks = [failFast: false]; - def test_slugs = getTestSlugs("all"); - for (test in test_slugs) { - def currentSlug = test; // copy needed to correctly pass args to runTests - tasks["${test}"] = { - node { - catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - docker.image("${DOCKER_IMAGE}").inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { - sh "git config --system --add safe.directory '*'"; - checkout scm; - runTests(currentSlug); - } - } - } - } - } - parallel(tasks); + runTests(DOCKER_IMAGE, 'pr'); } - docker.image("${DOCKER_IMAGE}").inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { - sh "git config --system --add safe.directory '*'"; - stage('Report Summary') { - generateReportShortSummary(); - } - stage("Report HTML Table") { - generateReportHtmlTable(); - } - stage('Upload Metadata') { - uploadMetadata(env.BRANCH_NAME, commitHash); - } - stage('Send Report') { - def COMMIT_AUTHOR_EMAIL = sh(script: "git --no-pager show -s --format='%ae'", returnStdout: true).trim(); - sendEmail(env.BRANCH_NAME, COMMIT_AUTHOR_EMAIL); - } + stage ('Cleanup and Reporting') { + finalReport(DOCKER_IMAGE); } } diff --git a/jenkins/public_tests_small.Jenkinsfile b/jenkins/public_tests_small.Jenkinsfile deleted file mode 100644 index c48ecb3a61..0000000000 --- a/jenkins/public_tests_small.Jenkinsfile +++ /dev/null @@ -1,68 +0,0 @@ -@Library('utils@main') _ - -node { - - properties([ - copyArtifactPermission('${JOB_NAME},'+env.BRANCH_NAME), - ]); - - stage('Checkout') { - checkout scm; - } - - def commitHash = "none"; - def DOCKER_IMAGE_TAG = "latest"; - stage('Build and Push Docker Image') { - if (isDependencyInstallerChanged(env.BRANCH_NAME)) { - commitHash = sh(script: 'git rev-parse HEAD', returnStdout: true); - commitHash = commitHash.replaceAll(/[^a-zA-Z0-9-]/, ''); - DOCKER_IMAGE_TAG = pushCIImage(env.BRANCH_NAME, commitHash); - } - } - def DOCKER_IMAGE = "openroad/flow-ubuntu22.04-dev:${DOCKER_IMAGE_TAG}"; - - docker.image("${DOCKER_IMAGE}").inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { - stage('Build ORFS and Stash bins') { - sh "git config --system --add safe.directory '*'"; - localBuild(); - } - } - - stage('Run Tests') { - Map tasks = [failFast: false]; - def test_slugs = getTestSlugs("small"); - for (test in test_slugs) { - def currentSlug = test; // copy needed to correctly pass args to runTests - tasks["${test}"] = { - node { - catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - docker.image("${DOCKER_IMAGE}").inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { - sh "git config --system --add safe.directory '*'"; - checkout scm; - runTests(currentSlug); - } - } - } - } - } - parallel(tasks); - } - - docker.image("${DOCKER_IMAGE}").inside('--user=root --privileged -v /var/run/docker.sock:/var/run/docker.sock') { - sh "git config --system --add safe.directory '*'"; - stage('Report Summary') { - generateReportShortSummary(); - } - stage("Report HTML Table") { - generateReportHtmlTable(); - } - stage('Upload Metadata') { - uploadMetadata(env.BRANCH_NAME, commitHash); - } - stage('Send Report') { - def COMMIT_AUTHOR_EMAIL = sh(script: "git --no-pager show -s --format='%ae'", returnStdout: true).trim(); - sendEmail(env.BRANCH_NAME, COMMIT_AUTHOR_EMAIL); - } - } - -}