diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..7eb8f0fa --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,142 @@ +name: Benchmark + +on: + workflow_call: + inputs: + recipe_id_full: + required: true + type: string + + conan_extra_args: + required: false + default: "" + type: string + + benchmark_cmd: + required: true + type: string + + name: + required: true + type: string + + output_file_path: + required: true + type: string + + data_dir: + required: true + type: string + + tool: + required: true + type: string + +permissions: + contents: write + deployments: write + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + +jobs: + benchmark: + name: Run benchmark + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.head_ref }} + + - name: Checkout repo PR + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.base_ref }} + + - name: Sync pip requirements + run: wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt -O .github/workflows/requirements-runner.txt + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .github/workflows/requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .github/workflows/requirements-runner.txt + + - name: Install Linux system requirements for building + run: | + mkdir runner_scripts + wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/ubuntu_setup.sh -O runner_scripts/ubuntu_setup.sh + chmod +x runner_scripts/ubuntu_setup.sh + sudo ./runner_scripts/ubuntu_setup.sh + + - name: Setup pipeline caches + run: | + mkdir -p /home/runner/.conan/downloads + mkdir -p /home/runner/.conan/data + + - name: Create default Conan profile + run: conan profile new default --detect + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: /home/runner/.conan/data + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: /home/runner/.conan/downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Install dependencies + run: conan install . ${{ inputs.recipe_id_full }} -s build_type=Release --build=missing --update -g GitHubActionsRunEnv -g GitHubActionsBuildEnv ${{ inputs.conan_extra_args }} + + - name: Set Environment variables from Conan install (bash) + if: ${{ runner.os != 'Windows' }} + run: | + . ./activate_github_actions_runenv.sh + . ./activate_github_actions_buildenv.sh + working-directory: build/Release/generators + + - name: Build + run: | + cmake --preset release + cmake --build --preset release + + - name: Run benchmark + id: run-test + run: ${{ inputs.benchmark_cmd }} + working-directory: build/Release + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: ${{ inputs.name }} + output-file-path: ${{ inputs.output_file_path }} + gh-repository: github.com/Ultimaker/CuraEngineBenchmarks + gh-pages-branch: main + benchmark-data-dir-path: ${{ inputs.data_dir }} + tool: ${{ inputs.tool }} + github-token: ${{ secrets.CURA_BENCHMARK_PAT }} + auto-push: true + max-items-in-chart: 250 diff --git a/.github/workflows/check-actor.yml b/.github/workflows/check-actor.yml new file mode 100644 index 00000000..1b1ec269 --- /dev/null +++ b/.github/workflows/check-actor.yml @@ -0,0 +1,34 @@ +name: Check Actor + +on: + workflow_call: + outputs: + proceed: + description: "Is workflow started by a legit actor? No bots or forks" + value: ${{ jobs.check_actor.outputs.proceed }} + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + +jobs: + check_actor: + runs-on: ubuntu-latest + outputs: + proceed: ${{ steps.skip_check.outputs.proceed }} + steps: + - id: skip_check + run: | + if [[ "${{ github.actor }}" == *"[bot]"* ]]; then + echo "proceed=true" >> $GITHUB_OUTPUT + elif [[ "${{ github.event.pull_request }}" == "" ]]; then + echo "proceed=true" >> $GITHUB_OUTPUT + elif [[ "${{ github.event.pull_request.head.repo.fork }}" == "false" ]]; then + echo "proceed=true" >> $GITHUB_OUTPUT + else + echo "proceed=false" >> $GITHUB_OUTPUT + fi + shell: bash \ No newline at end of file diff --git a/.github/workflows/conan-package-create-linux.yml b/.github/workflows/conan-package-create-linux.yml new file mode 100644 index 00000000..c4e55883 --- /dev/null +++ b/.github/workflows/conan-package-create-linux.yml @@ -0,0 +1,110 @@ +name: Conan package create Linux + +on: + workflow_call: + inputs: + recipe_id_full: + required: true + type: string + + conan_internal: + required: false + default: false + type: boolean + + conan_extra_args: + required: false + default: "" + type: string + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + +jobs: + package-create: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.head_ref }} + + - name: Checkout repo PR + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.base_ref }} + + - name: Sync pip requirements + run: wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt -O .github/workflows/requirements-runner.txt + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .github/workflows/requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .github/workflows/requirements-runner.txt + + - name: Install Linux system requirements for building + run: | + mkdir -p runner_scripts + wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/ubuntu_setup.sh -O runner_scripts/ubuntu_setup.sh + chmod +x runner_scripts/ubuntu_setup.sh + sudo ./runner_scripts/ubuntu_setup.sh + + - name: Setup pipeline caches + run: | + mkdir -p /home/runner/.conan/downloads + mkdir -p /home/runner/.conan/data + + - name: Create default Conan profile + run: conan profile new default --detect + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Add Cura private Artifactory remote + if: ${{ inputs.conan_internal }} + run: conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/internal" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: /home/runner/.conan/data + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: /home/runner/.conan/downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Export the Package (binaries) + run: conan create . ${{ inputs.recipe_id_full }} --build=missing --update ${{ inputs.conan_extra_args }} + + - name: Upload the Package(s) + if: ${{ always() && !inputs.conan_internal }} + run: | + conan upload ${{ inputs.recipe_id_full }} -r cura --all -c + + - name: Upload the Package(s) to the private Artifactory + if: ${{ always() && inputs.conan_internal }} + run: | + conan upload ${{ inputs.recipe_id_full }} -r cura-private --all -c diff --git a/.github/workflows/conan-package-create-macos.yml b/.github/workflows/conan-package-create-macos.yml new file mode 100644 index 00000000..ac29efcd --- /dev/null +++ b/.github/workflows/conan-package-create-macos.yml @@ -0,0 +1,110 @@ +name: Conan package create Macos + +on: + workflow_call: + inputs: + recipe_id_full: + required: true + type: string + + conan_internal: + required: false + default: false + type: boolean + + conan_extra_args: + required: false + default: "" + type: string + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + +jobs: + package-create: + runs-on: macos-11 + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.head_ref }} + + - name: Checkout repo PR + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.base_ref }} + + - name: Sync pip requirements + run: wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt -O .github/workflows/requirements-runner.txt + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .github/workflows/requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .github/workflows/requirements-runner.txt + + - name: Install Macos system requirements for building + run: | + mkdir -p runner_scripts + wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/macos_setup.sh -O runner_scripts/macos_setup.sh + chmod +x runner_scripts/macos_setup.sh + ./runner_scripts/macos_setup.sh + + - name: Setup pipeline caches + run: | + mkdir -p /Users/runner/.conan/downloads + mkdir -p /Users/runner/.conan/data + + - name: Create default Conan profile + run: conan profile new default --detect + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Add Cura private Artifactory remote + if: ${{ inputs.conan_internal }} + run: conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/internal" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: /home/runner/.conan/data + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: /home/runner/.conan/downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Export the Package (binaries) + run: conan create . ${{ inputs.recipe_id_full }} --build=missing --update ${{ inputs.conan_extra_args }} + + - name: Upload the Package(s) + if: ${{ always() && !inputs.conan_internal }} + run: | + conan upload ${{ inputs.recipe_id_full }} -r cura --all -c + + - name: Upload the Package(s) to the private Artifactory + if: ${{ always() && inputs.conan_internal }} + run: | + conan upload ${{ inputs.recipe_id_full }} -r cura-private --all -c diff --git a/.github/workflows/conan-package-create-windows.yml b/.github/workflows/conan-package-create-windows.yml new file mode 100644 index 00000000..0321c055 --- /dev/null +++ b/.github/workflows/conan-package-create-windows.yml @@ -0,0 +1,107 @@ +name: Conan package create Windows + +on: + workflow_call: + inputs: + recipe_id_full: + required: true + type: string + + conan_internal: + required: false + default: false + type: boolean + + conan_extra_args: + required: false + default: "" + type: string + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + +jobs: + package-create: + runs-on: windows-2022 + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.head_ref }} + + - name: Checkout repo PR + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.base_ref }} + + - name: Sync pip requirements + run: curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt + working-directory: .github\workflows + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .\.github\workflows\requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .\.github\workflows\requirements-runner.txt + + - name: Setup pipeline caches + run: | + New-Item -ItemType Directory -Force -Path C:\Users\runneradmin\.conan\data + New-Item -ItemType Directory -Force -Path C:\.conan + New-Item -ItemType Directory -Force -Path C:\Users\runneradmin\.conan\downloads + + - name: Create default Conan profile + run: conan profile new default --detect + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Add Cura private Artifactory remote + if: ${{ inputs.conan_internal }} + run: conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/internal" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: | + C:\Users\runneradmin\.conan\data + C:\.conan + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: C:\Users\runneradmin\.conan\downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Export the Package (binaries) + run: conan create . ${{ inputs.recipe_id_full }} --build=missing --update ${{ inputs.conan_extra_args }} + + - name: Upload the Package(s) + if: ${{ always() && !inputs.conan_internal }} + run: | + conan upload ${{ inputs.recipe_id_full }} -r cura --all -c + + - name: Upload the Package(s) to the private Artifactory + if: ${{ always() && inputs.conan_internal }} + run: | + conan upload ${{ inputs.recipe_id_full }} -r cura-private --all -c diff --git a/.github/workflows/conan-package-release.yml b/.github/workflows/conan-package-release.yml index 06c78f75..afbeab24 100644 --- a/.github/workflows/conan-package-release.yml +++ b/.github/workflows/conan-package-release.yml @@ -2,6 +2,131 @@ name: Create Conan Package Release on: workflow_dispatch: + inputs: + organization: + description: 'Organization' + required: false + default: Ultimaker + type: string + + repository: + description: 'repository' + default: 'Cura' + type: choice + options: + - Cura + - Uranium + - fdm_materials + - Cura-binary-data + - CuraEngine + - CuraEngine_grpc_definitions + - CuraEngine_plugin_gradual_flow + - synsepalum-dulcificum + - libArcus + - libnest2d + - libSavitar + - pyarcus + - pynest2d + - pySavitar + + ref_name: + description: 'Git ref (main, 5.7, sha)' + required: false + default: 'main' + type: string + + version: + description: 'Semantic Version (5.7, 0.1.0-beta.1)' + required: true + type: string + + conan_release: + description: 'Create a release' + required: false + default: true + type: boolean + + conan_user: + description: 'Conan user (_, ultimaker, internal)' + required: false + default: '_' + type: string + + conan_channel: + description: 'Conan channel (_, testing, stable)' + required: false + default: '_' + type: string + + conan_internal: + description: 'UltiMaker private (internal)' + required: false + default: false + type: boolean + + conan_latest: + description: 'Create the latest alias' + required: false + default: true + type: boolean + + conan_extra_args: + description: 'Conan args: (--require-override)' + required: false + default: '' + type: string + + workflow_call: + inputs: + organization: + required: false + default: Ultimaker + type: string + + repository: + required: true + type: string + + ref_name: + required: false + default: '' + type: string + + version: + required: true + type: string + + conan_user: + required: false + default: '_' + type: string + + conan_channel: + required: false + default: '_' + type: string + + conan_internal: + required: false + default: false + type: boolean + + conan_latest: + required: false + default: true + type: boolean + + conan_extra_args: + required: false + default: '' + type: string + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} jobs: package-export: @@ -10,3 +135,94 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v4 + with: + repository: ${{ inputs.organization }}/${{ inputs.repository }} + ref: ${{ inputs.ref_name }} + fetch-depth: 1 + token: ${{ secrets.CURA_CONAN_PACKAGE_RELEASE_PAT }} + + - name: Sync pip requirements + run: | + mkdir -p .github/workflows + cd .github/workflows + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .github/workflows/requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .github/workflows/requirements-runner.txt + + - name: Setup pipeline caches + run: | + mkdir -p /home/runner/.conan/downloads + mkdir -p /home/runner/.conan/data + + - name: Create default Conan profile + run: conan profile new default --detect + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Add Cura private Artifactory remote + if: ${{ inputs.conan_internal }} + run: conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/internal" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: /home/runner/.conan/data + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: /home/runner/.conan/downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Install pyyaml + run: pip install pyyaml + + - name: Get basic project info + run: | + echo "CONAN_PROJECT_NAME=$(conan inspect . | awk '/^name:/ {print $2}')" >> $GITHUB_ENV + + - id: get-conan-broadcast-data + name: Get Conan broadcast data + run: | + mkdir runner_scripts + wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/get_conan_broadcast_data.py -O runner_scripts/get_conan_broadcast_data.py + python runner_scripts/get_conan_broadcast_data.py --user ${{ inputs.conan_user }} --channel ${{ inputs.conan_channel }} --project_name $CONAN_PROJECT_NAME --release ${{ inputs.conan_release }} --version ${{ inputs.version }} + + - name: Export the Package + run: conan export . ${{ steps.get-conan-broadcast-data.outputs.recipe_id_full }} ${{ inputs.conan_extra_args }} + + - name: Create the latest alias + if: ${{ inputs.conan_latest }} + run: conan alias ${{ steps.get-conan-broadcast-data.outputs.recipe_id_latest }} ${{ steps.get-conan-broadcast-data.outputs.recipe_id_full }} + + - name: Upload the Package(s) + if: ${{ ! inputs.conan_internal }} + run: conan upload ${{ steps.get-conan-broadcast-data.outputs.recipe_id_full }} -r cura --all -c + + - name: Upload the latest Package(s) + if: ${{ ! inputs.conan_internal && inputs.conan_latest }} + run: conan upload ${{ steps.get-conan-broadcast-data.outputs.recipe_id_latest }} -r cura -c + + - name: Upload the Package(s) to the private Artifactory + if: ${{ inputs.conan_internal }} + run: conan upload ${{ steps.get-conan-broadcast-data.outputs.recipe_id_full }} -r cura-private --all -c + + - name: Upload the latest Package(s) to the private Artifactory + if: ${{ inputs.conan_internal && inputs.conan_latest }} + run: conan upload ${{ steps.get-conan-broadcast-data.outputs.recipe_id_latest }} -r cura-private -c \ No newline at end of file diff --git a/.github/workflows/conan-recipe-export.yml b/.github/workflows/conan-recipe-export.yml new file mode 100644 index 00000000..931c402b --- /dev/null +++ b/.github/workflows/conan-recipe-export.yml @@ -0,0 +1,113 @@ +name: Export Conan Recipe to server + +on: + workflow_call: + inputs: + recipe_id_full: + required: true + type: string + + recipe_id_latest: + required: false + type: string + + conan_internal: + required: false + default: false + type: boolean + + conan_extra_args: + required: false + default: "" + type: string + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + +jobs: + package-export: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.head_ref }} + + - name: Checkout repo PR + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.base_ref }} + + - name: Sync pip requirements + run: wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt -O .github/workflows/requirements-runner.txt + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .github/workflows/requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .github/workflows/requirements-runner.txt + + - name: Setup pipeline caches + run: | + mkdir -p /home/runner/.conan/downloads + mkdir -p /home/runner/.conan/data + + - name: Create default Conan profile + run: conan profile new default --detect + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Add Cura private Artifactory remote + if: ${{ inputs.conan_internal }} + run: conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/internal" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: /home/runner/.conan/data + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: /home/runner/.conan/downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Export the Package + run: conan export . ${{ inputs.recipe_id_full }} ${{ inputs.conan_extra_args }} + + - name: Create the latest alias + if: always() + run: conan alias ${{ inputs.recipe_id_latest }} ${{ inputs.recipe_id_full }} + + - name: Upload the Package(s) + if: ${{ always() && !inputs.conan_internal}} + run: | + conan upload ${{ inputs.recipe_id_full }} -r cura --all -c + conan upload ${{ inputs.recipe_id_latest }} -r cura -c + + - name: Upload the Package(s) to the private Artifactory + if: ${{ always() && inputs.conan_internal }} + run: | + conan upload ${{ inputs.recipe_id_full }} -r cura-private --all -c + conan upload ${{ inputs.recipe_id_latest }} -r cura-private -c diff --git a/.github/workflows/conan-recipe-version.yml b/.github/workflows/conan-recipe-version.yml new file mode 100644 index 00000000..931202d7 --- /dev/null +++ b/.github/workflows/conan-recipe-version.yml @@ -0,0 +1,97 @@ +name: Get Conan Recipe Version + +on: + workflow_call: + inputs: + project_name: + required: true + type: string + + user: + required: false + default: ultimaker + type: string + + release: + required: false + default: false + type: boolean + + version: + required: false + default: '' + type: string + + outputs: + recipe_id_full: + description: "The full Conan recipe id: /@/" + value: ${{ jobs.get-semver.outputs.recipe_id_full }} + + recipe_id_latest: + description: "The full Conan recipe aliased (latest) id: /(latest)@/" + value: ${{ jobs.get-semver.outputs.recipe_id_latest }} + + recipe_semver_full: + description: "The full semver ..-+" + value: ${{ jobs.get-semver.outputs.semver_full }} + + is_release_branch: + description: "is current branch a release branch?" + value: ${{ jobs.get-semver.outputs.release_branch }} + + user: + description: "The conan user" + value: ${{ jobs.get-semver.outputs.user }} + + channel: + description: "The conan channel" + value: ${{ jobs.get-semver.outputs.channel }} + + project_name: + description: "The conan projectname" + value: ${{ inputs.project_name }} + +jobs: + get-semver: + + runs-on: ubuntu-latest + + outputs: + recipe_id_full: ${{ steps.get-conan-broadcast-data.outputs.recipe_id_full }} + recipe_id_latest: ${{ steps.get-conan-broadcast-data.outputs.recipe_id_latest }} + semver_full: ${{ steps.get-conan-broadcast-data.outputs.semver_full }} + is_release_branch: ${{ steps.get-conan-broadcast-data.outputs.is_release_branch }} + user: ${{ steps.get-conan-broadcast-data.outputs.user }} + channel: ${{ steps.get-conan-broadcast-data.outputs.channel }} + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + fetch-depth: 1 + sparse-checkout: 'conandata.yml' + ref: ${{ github.head_ref }} + + - name: Checkout repo PR + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + fetch-depth: 1 + sparse-checkout: 'conandata.yml' + ref: ${{ github.base_ref }} + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: "3.12.x" + + - name: Install pyyaml + run: pip install pyyaml + + - id: get-conan-broadcast-data + name: Get Conan broadcast data + run: | + mkdir runner_scripts + wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/get_conan_broadcast_data.py -O runner_scripts/get_conan_broadcast_data.py + python runner_scripts/get_conan_broadcast_data.py --user ${{ inputs.user }} --project_name "${{ inputs.project_name }}" --sha "${{ github.sha }}" --event_name "${{ github.event_name }}" --base_ref "${{ github.base_ref }}" --ref_name "${{ github.ref_name }}" --release "${{ inputs.release }}" --version "${{ inputs.version }}" \ No newline at end of file diff --git a/.github/workflows/cura-installer-default-value.yml b/.github/workflows/cura-installer-default-value.yml new file mode 100644 index 00000000..0ed5ed2d --- /dev/null +++ b/.github/workflows/cura-installer-default-value.yml @@ -0,0 +1,63 @@ +name: Get default installer values + +on: + workflow_call: + inputs: + cura_conan_version: + required: true + type: string + + latest_release: + required: true + type: string + + latest_release_schedule_hour: + required: true + type: number + + latest_release_tag: + required: true + type: string + + outputs: + cura_conan_version: + description: "The Conan version used to build Cura" + value: ${{ jobs.default_values.outputs.cura_conan_version }} + + release_tag: + description: "The git tag used to upload the Artifacts to the release `nightly` or `nightly-5.6 (for example)" + value: ${{ jobs.default_values.outputs.release_tag }} + +jobs: + default_values: + runs-on: ubuntu-latest + outputs: + cura_conan_version: ${{ steps.default.outputs.cura_conan_version }} + release_tag: ${{ steps.default.outputs.release_tag }} + + steps: + - name: Output default values + id: default + shell: python + run: | + import os + import datetime + + if "${{ github.event_name }}" != "schedule": + cura_conan_version = "${{ inputs.cura_conan_version }}" + else: + now = datetime.datetime.now() + cura_conan_version = "cura/latest@ultimaker/stable" if now.hour == int(${{ inputs.latest_release_schedule_hour }}) else "cura/latest@ultimaker/testing" + + release_tag = f"${{ inputs.latest_release_tag }}-${{ inputs.latest_release }}" if "/stable" in cura_conan_version else "${{ inputs.latest_release_tag }}" + + # Set cura_conan_version environment variable + output_env = os.environ["GITHUB_OUTPUT"] + content = "" + if os.path.exists(output_env): + with open(output_env, "r") as f: + content = f.read() + with open(output_env, "w") as f: + f.write(content) + f.writelines(f"cura_conan_version={cura_conan_version}\n") + f.writelines(f"release_tag={release_tag}\n") \ No newline at end of file diff --git a/.github/workflows/cura-installer-linux.yml b/.github/workflows/cura-installer-linux.yml new file mode 100644 index 00000000..293e4d04 --- /dev/null +++ b/.github/workflows/cura-installer-linux.yml @@ -0,0 +1,214 @@ +name: Linux Installer +run-name: ${{ inputs.cura_conan_version }} for Linux-${{ inputs.architecture }} by @${{ github.actor }} + +on: + workflow_call: + inputs: + cura_conan_version: + description: 'Cura Conan Version' + default: 'cura/latest@ultimaker/testing' + required: true + type: string + + conan_args: + description: 'Conan args: eq.: --require-override' + default: '' + required: false + type: string + + enterprise: + description: 'Build Cura as an Enterprise edition' + default: false + required: true + type: boolean + + staging: + description: 'Use staging API' + default: false + required: true + type: boolean + + architecture: + description: 'Architecture' + required: true + default: 'X64' + type: string + + operating_system: + description: 'OS' + required: true + default: 'ubuntu-22.04' + type: string + + conan_internal: + required: false + default: false + type: boolean + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + CURA_CONAN_VERSION: ${{ inputs.cura_conan_version }} + ENTERPRISE: ${{ inputs.enterprise }} + STAGING: ${{ inputs.staging }} + +jobs: + cura-installer-create: + runs-on: ${{ inputs.operating_system }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Sync pip requirements + run: wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt -O .github/workflows/requirements-runner.txt + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .github/workflows/requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .github/workflows/requirements-runner.txt + + - name: Install Linux system requirements for building + run: | + mkdir runner_scripts + cd runner_scripts + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/ubuntu_setup.sh + chmod +x ubuntu_setup.sh + sudo ./ubuntu_setup.sh + + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/appimage_setup.sh + cd .. + source ./runner_scripts/appimage_setup.sh + + - name: Setup pipeline caches + run: | + mkdir -p /home/runner/.conan/downloads + mkdir -p /home/runner/.conan/data + + - name: Create default Conan profile + run: conan profile new default --detect + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Add Cura private Artifactory remote + if: ${{ inputs.conan_internal }} + run: conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/internal" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: /home/runner/.conan/data + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: /home/runner/.conan/downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Configure GPG Key Linux (Bash) + run: echo -n "$GPG_PRIVATE_KEY" | base64 --decode | gpg --import + + - name: Create the Packages (Bash) + run: conan install $CURA_CONAN_VERSION ${{ inputs.conan_args }} --build=missing --update -if cura_inst -g VirtualPythonEnv -o cura:enterprise=$ENTERPRISE -o cura:staging=$STAGING -o cura:internal=${{ inputs.conan_internal }} -c tools.build:skip_test=True + + - name: Set Environment variables for Cura (bash) + run: | + . ./cura_inst/bin/activate_github_actions_env.sh + . ./cura_inst/bin/activate_github_actions_version_env.sh + + - name: Create the Cura dist + run: pyinstaller ./cura_inst/UltiMaker-Cura.spec + + - name: Get installer helper scripts + run: | + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/cura_installer_filename.py + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/summarize_used_dependencies.py + working-directory: runner_scripts + + - name: Output the name file name and extension + id: filename + run: python runner_scripts/cura_installer_filename.py --os ${{ runner.os }} --arch ${{ inputs.architecture }} --enterprise ${{ inputs.enterprise }} --internal ${{ inputs.conan_internal }} + + - name: Summarize the used dependencies + run: python runner_scripts/summarize_used_dependencies.py --installer_filename ${{ steps.filename.outputs.INSTALLER_FILENAME }} + + - name: Create the Linux AppImage (Bash) + run: | + python ../cura_inst/packaging/AppImage-builder/create_appimage.py ./UltiMaker-Cura $CURA_VERSION_FULL "${{ steps.filename.outputs.INSTALLER_FILENAME }}.AppImage" + chmod +x "${{ steps.filename.outputs.INSTALLER_FILENAME }}.AppImage" + working-directory: dist + + - name: Remove internal packages before uploading + run: | + conan remove "*@internal/*" -f || true + conan remove "cura_private_data*" -f || true + + - name: Upload the Package(s) + if: ${{ always() && ! inputs.conan_internal }} + run: conan upload "*" -r cura --all -c + + - name: Upload the AppImage + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.filename.outputs.INSTALLER_FILENAME }}-AppImage + path: | + dist/${{ steps.filename.outputs.INSTALLER_FILENAME }}.AppImage + retention-days: 5 + + - name: Upload the asc + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.filename.outputs.INSTALLER_FILENAME }}-asc + path: | + dist/${{ steps.filename.outputs.INSTALLER_FILENAME }}.AppImage.asc + retention-days: 5 + + - name: Setup Sentry CLI + uses: mathieu-bour/setup-sentry-cli@v1 + with: + version: latest + token: ${{ secrets.CURAENGINE_SENTRY_TOKEN }} + organization: ultimaker-o7 + project: curaengine + + - name: Upload the debug symbols + run: | + sentry-cli debug-files upload UltiMaker-Cura/CuraEngine + sentry-cli debug-files upload UltiMaker-Cura/libArcus.* + sentry-cli debug-files upload UltiMaker-Cura/libpolyclipping.* + working-directory: dist + + - name: Write the run info + shell: python + run: | + import os + with open("run_info.sh", "w") as f: + f.writelines(f'echo "CURA_VERSION_FULL={os.environ["CURA_VERSION_FULL"]}" >> $GITHUB_ENV\n') + + - name: Upload the run info + uses: actions/upload-artifact@v3 + with: + name: linux-run-info + path: | + run_info.sh + retention-days: 5 + diff --git a/.github/workflows/cura-installer-macos.yml b/.github/workflows/cura-installer-macos.yml new file mode 100644 index 00000000..3593e686 --- /dev/null +++ b/.github/workflows/cura-installer-macos.yml @@ -0,0 +1,249 @@ +name: Macos Installer +run-name: ${{ inputs.cura_conan_version }} for Macos-${{ inputs.architecture }} by @${{ github.actor }} + +on: + workflow_call: + inputs: + cura_conan_version: + description: 'Cura Conan Version' + default: 'cura/latest@ultimaker/testing' + required: true + type: string + + conan_args: + description: 'Conan args: eq.: --require-override' + default: '' + required: false + type: string + + enterprise: + description: 'Build Cura as an Enterprise edition' + default: false + required: true + type: boolean + + staging: + description: 'Use staging API' + default: false + required: true + type: boolean + + architecture: + description: 'Architecture' + required: true + default: 'ARM64' + type: string + + operating_system: + description: 'OS' + required: true + default: 'self-hosted-ARM64' + type: string + + conan_internal: + required: false + default: false + type: boolean + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + CODESIGN_IDENTITY: ${{ secrets.CODESIGN_IDENTITY }} + MAC_NOTARIZE_USER: ${{ secrets.MAC_NOTARIZE_USER }} + MAC_NOTARIZE_PASS: ${{ secrets.MAC_NOTARIZE_PASS }} + MACOS_CERT_P12: ${{ secrets.MACOS_CERT_P12 }} + MACOS_CERT_INSTALLER_P12: ${{ secrets.MACOS_CERT_INSTALLER_P12 }} + MACOS_CERT_USER: ${{ secrets.MACOS_CERT_USER }} + MACOS_CERT_PASSPHRASE: ${{ secrets.MACOS_CERT_PASSPHRASE }} + CURA_CONAN_VERSION: ${{ inputs.cura_conan_version }} + ENTERPRISE: ${{ inputs.enterprise }} + STAGING: ${{ inputs.staging }} + +jobs: + cura-installer-create: + runs-on: ${{ inputs.operating_system }} + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.head_ref }} + + - name: Checkout repo PR + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.base_ref }} + + - name: Sync pip requirements + run: curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt + working-directory: .github/workflows + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .github/workflows/requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .github/workflows/requirements-runner.txt + + - name: Install Macos system requirements for building + run: | + mkdir runner_scripts + cd runner_scripts + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/macos_setup.sh + chmod +x macos_setup.sh + ./macos_setup.sh + + - name: Setup pipeline caches + run: | + mkdir -p /Users/runner/.conan/downloads + mkdir -p /Users/runner/.conan/data + + - name: Create default Conan profile + run: conan profile new default --detect --force + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Add Cura private Artifactory remote + if: ${{ inputs.conan_internal }} + run: conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/internal" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: /home/runner/.conan/data + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: /home/runner/.conan/downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Remove Macos keychain (Bash) + run: security delete-keychain signing_temp.keychain || true + + - name: Configure Macos keychain Developer Cert(Bash) + id: macos-keychain-developer-cert + uses: apple-actions/import-codesign-certs@v1 + with: + keychain-password: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }} + p12-file-base64: ${{ secrets.MACOS_CERT_P12 }} + p12-password: ${{ secrets.MACOS_CERT_PASSPHRASE }} + + - name: Configure Macos keychain Installer Cert (Bash) + id: macos-keychain-installer-cert + uses: apple-actions/import-codesign-certs@v1 + with: + keychain-password: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }} + create-keychain: false # keychain is created in previous use of action. + p12-file-base64: ${{ secrets.MACOS_CERT_INSTALLER_P12 }} + p12-password: ${{ secrets.MACOS_CERT_PASSPHRASE }} + + - name: Remove private Artifactory + if: ${{ ! inputs.conan_internal }} + run: conan remote remove cura-private || true + + - name: Create the Packages (Bash) + run: conan install $CURA_CONAN_VERSION ${{ inputs.conan_args }} --build=missing --update -if cura_inst -g VirtualPythonEnv -o cura:enterprise=$ENTERPRISE -o cura:staging=$STAGING -o cura:internal=${{ inputs.conan_internal }} -c tools.build:skip_test=True + + - name: Set Environment variables for Cura (bash) + run: | + . ./cura_inst/bin/activate_github_actions_env.sh + . ./cura_inst/bin/activate_github_actions_version_env.sh + + - name: Unlock Macos keychain (Bash) + run: security unlock -p $TEMP_KEYCHAIN_PASSWORD signing_temp.keychain + env: + TEMP_KEYCHAIN_PASSWORD: ${{ steps.macos-keychain-developer-cert.outputs.keychain-password }} + + - name: Create the Cura dist + run: pyinstaller ./cura_inst/UltiMaker-Cura.spec + + - name: Get installer helper scripts + run: | + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/cura_installer_filename.py + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/summarize_used_dependencies.py + working-directory: runner_scripts + + - name: Output the name file name and extension + id: filename + run: python runner_scripts/cura_installer_filename.py --os ${{ runner.os }} --arch ${{ inputs.architecture }} --enterprise ${{ inputs.enterprise }} --internal ${{ inputs.conan_internal }} + + - name: Summarize the used dependencies + run: python runner_scripts/summarize_used_dependencies.py --installer_filename ${{ steps.filename.outputs.INSTALLER_FILENAME }} + + - name: Create the Macos dmg (Bash) + run: python ../cura_inst/packaging/MacOS/build_macos.py --source_path ../cura_inst --dist_path . --cura_conan_version $CURA_CONAN_VERSION --filename "${{ steps.filename.outputs.INSTALLER_FILENAME }}" --build_dmg --build_pkg --app_name "$CURA_APP_NAME" + working-directory: dist + + - name: Remove internal packages before uploading + run: | + conan remove "*@internal/*" -f || true + conan remove "cura_private_data*" -f || true + + - name: Upload the Package(s) + if: ${{ always() && ! inputs.conan_internal }} + run: conan upload "*" -r cura --all -c + + - name: Upload the dmg + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.filename.outputs.INSTALLER_FILENAME }}-dmg + path: | + dist/${{ steps.filename.outputs.INSTALLER_FILENAME }}.dmg + retention-days: 5 + + - name: Upload the pkg + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.filename.outputs.INSTALLER_FILENAME }}-pkg + path: | + dist/${{ steps.filename.outputs.INSTALLER_FILENAME }}.pkg + retention-days: 5 + + - name: Setup Sentry CLI + uses: mathieu-bour/setup-sentry-cli@v1 + with: + version: latest + token: ${{ secrets.CURAENGINE_SENTRY_TOKEN }} + organization: ultimaker-o7 + project: curaengine + + - name: Upload the debug symbols + run: | + sentry-cli debug-files upload UltiMaker-Cura/CuraEngine + sentry-cli debug-files upload UltiMaker-Cura/libArcus.* + sentry-cli debug-files upload UltiMaker-Cura/libpolyclipping.* + working-directory: dist + + - name: Write the run info + shell: python + run: | + import os + with open("run_info.sh", "w") as f: + f.writelines(f'echo "CURA_VERSION_FULL={os.environ["CURA_VERSION_FULL"]}" >> $GITHUB_ENV\n') + + - name: Upload the run info + uses: actions/upload-artifact@v3 + with: + name: macos-run-info + path: | + run_info.sh + retention-days: 5 diff --git a/.github/workflows/cura-installer-windows.yml b/.github/workflows/cura-installer-windows.yml new file mode 100644 index 00000000..b44731d1 --- /dev/null +++ b/.github/workflows/cura-installer-windows.yml @@ -0,0 +1,244 @@ +name: Windows Installer +run-name: ${{ inputs.cura_conan_version }} for Windows-${{ inputs.architecture }} by @${{ github.actor }} + +on: + workflow_call: + inputs: + cura_conan_version: + description: 'Cura Conan Version' + default: 'cura/latest@ultimaker/testing' + required: true + type: string + + conan_args: + description: 'Conan args: eq.: --require-override' + default: '' + required: false + type: string + + enterprise: + description: 'Build Cura as an Enterprise edition' + default: false + required: true + type: boolean + + staging: + description: 'Use staging API' + default: false + required: true + type: boolean + + architecture: + description: 'Architecture' + required: true + default: 'X64' + type: string + + operating_system: + description: 'OS' + required: true + default: 'windows-2022' + type: string + + conan_internal: + required: false + default: false + type: boolean + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + WIN_CERT_INSTALLER_CER: ${{ secrets.WIN_CERT_INSTALLER_CER }} + WIN_CERT_INSTALLER_CER_PASS: ${{ secrets.WIN_CERT_INSTALLER_CER_PASS }} + CURA_CONAN_VERSION: ${{ inputs.cura_conan_version }} + ENTERPRISE: ${{ inputs.enterprise }} + STAGING: ${{ inputs.staging }} + +jobs: + cura-installer-create: + runs-on: ${{ inputs.operating_system }} + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.head_ref }} + + - name: Checkout repo PR + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.base_ref }} + + - name: Sync pip requirements + run: curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt + working-directory: .github\workflows + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .\.github\workflows\requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .\.github\workflows\requirements-runner.txt + + - name: Setup pipeline caches + run: | + New-Item -ItemType Directory -Force -Path C:\Users\runneradmin\.conan\data + New-Item -ItemType Directory -Force -Path C:\.conan + New-Item -ItemType Directory -Force -Path C:\Users\runneradmin\.conan\downloads + + - name: Create default Conan profile + run: conan profile new default --detect + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Add Cura private Artifactory remote + if: ${{ inputs.conan_internal }} + run: conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/internal" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: | + C:\Users\runneradmin\.conan\data + C:\.conan + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: C:\Users\runneradmin\.conan\downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Create the Packages (Powershell) + run: conan install $Env:CURA_CONAN_VERSION ${{ inputs.conan_args }} --build=missing --update -if cura_inst -g VirtualPythonEnv -o cura:enterprise=$Env:ENTERPRISE -o cura:staging=$Env:STAGING -o cura:internal=${{ inputs.conan_internal }} -c tools.build:skip_test=True + + - name: Set Environment variables for Cura (Powershell) + run: | + echo "${Env:WIX}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + .\cura_inst\Scripts\activate_github_actions_env.ps1 + .\cura_inst\Scripts\activate_github_actions_version_env.ps1 + + - name: Create the Cura dist + run: pyinstaller ./cura_inst/UltiMaker-Cura.spec + + - name: Get installer helper scripts + run: | + New-Item -ItemType Directory -Force -Path runner_scripts + cd runner_scripts + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/cura_installer_filename.py + curl -O https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/summarize_used_dependencies.py + + - name: Output the name file name and extension + id: filename + run: python runner_scripts/cura_installer_filename.py --os ${{ runner.os }} --arch ${{ inputs.architecture }} --enterprise ${{ inputs.enterprise }} --internal ${{ inputs.conan_internal }} + + - name: Summarize the used dependencies + run: python runner_scripts/summarize_used_dependencies.py --installer_filename ${{ steps.filename.outputs.INSTALLER_FILENAME }} + + - name: Create PFX certificate from BASE64_PFX_CONTENT secret + id: create-pfx + env: + PFX_CONTENT: ${{ secrets.WIN_CERT_INSTALLER_CER }} + run: | + $pfxPath = Join-Path -Path $env:RUNNER_TEMP -ChildPath "cert.pfx"; + $encodedBytes = [System.Convert]::FromBase64String($env:PFX_CONTENT); + Set-Content $pfxPath -Value $encodedBytes -AsByteStream; + echo "PFX_PATH=$pfxPath" >> $env:GITHUB_OUTPUT; + + - name: Create the Windows msi installer (Powershell) + run: | + python ..\cura_inst\packaging\msi\create_windows_msi.py ..\cura_inst .\UltiMaker-Cura "${{steps.filename.outputs.INSTALLER_FILENAME }}.msi" "$Env:CURA_APP_NAME" + working-directory: dist + + - name: Sign the Windows msi installer (Powershell) + env: + PFX_PATH: ${{ steps.create-pfx.outputs.PFX_PATH }} + run: | + & "C:/Program Files (x86)/Windows Kits/10/bin/10.0.17763.0/x86/signtool.exe" sign /f $Env:PFX_PATH /p "$Env:WIN_CERT_INSTALLER_CER_PASS" /fd SHA256 /t http://timestamp.digicert.com "${{steps.filename.outputs.INSTALLER_FILENAME }}.msi" + working-directory: dist + + - name: Create the Windows exe installer (Powershell) + run: | + python ..\cura_inst\packaging\NSIS\create_windows_installer.py ../cura_inst . "${{steps.filename.outputs.INSTALLER_FILENAME }}.exe" + working-directory: dist + + - name: Sign the Windows exe installer (Powershell) + env: + PFX_PATH: ${{ steps.create-pfx.outputs.PFX_PATH }} + run: | + & "C:/Program Files (x86)/Windows Kits/10/bin/10.0.17763.0/x86/signtool.exe" sign /f $Env:PFX_PATH /p "$Env:WIN_CERT_INSTALLER_CER_PASS" /fd SHA256 /t http://timestamp.digicert.com "${{steps.filename.outputs.INSTALLER_FILENAME }}.exe" + working-directory: dist + + - name: Remove internal packages before uploading + run: | + conan remove "*@internal/*" -f || true + conan remove "cura_private_data*" -f || true + + - name: Upload the Package(s) + if: ${{ always() && ! inputs.conan_internal }} + run: conan upload "*" -r cura --all -c + + - name: Upload the msi + uses: actions/upload-artifact@v3 + with: + name: ${{steps.filename.outputs.INSTALLER_FILENAME }}-msi + path: | + dist/${{steps.filename.outputs.INSTALLER_FILENAME }}.msi + retention-days: 5 + + - name: Upload the exe + uses: actions/upload-artifact@v3 + with: + name: ${{steps.filename.outputs.INSTALLER_FILENAME }}-exe + path: | + dist/${{steps.filename.outputs.INSTALLER_FILENAME }}.exe + retention-days: 5 + + - name: Setup Sentry CLI + uses: mathieu-bour/setup-sentry-cli@v1 + with: + version: latest + token: ${{ secrets.CURAENGINE_SENTRY_TOKEN }} + organization: ultimaker-o7 + project: curaengine + + - name: Upload the debug symbols + run: | + sentry-cli debug-files upload .\UltiMaker-Cura\CuraEngine.exe + sentry-cli debug-files upload .\UltiMaker-Cura\Arcus.dll + sentry-cli debug-files upload .\UltiMaker-Cura\polyclipping.dll + working-directory: dist + + # NOTE: The extension is .sh, since this isn't going to build-environment, so not on the Win build image. + - name: Write the run info + shell: python + run: | + import os + with open("run_info.sh", "w") as f: + f.writelines(f'echo "CURA_VERSION_FULL={os.environ["CURA_VERSION_FULL"]}" >> $GITHUB_ENV\n') + + # NOTE: The extension is .sh, since this isn't going to build-environment, so not on the Win build image. + - name: Upload the run info + uses: actions/upload-artifact@v3 + with: + name: windows-run-info + path: | + run_info.sh + retention-days: 5 diff --git a/.github/workflows/process-pull-request.yml b/.github/workflows/process-pull-request.yml new file mode 100644 index 00000000..021106da --- /dev/null +++ b/.github/workflows/process-pull-request.yml @@ -0,0 +1,25 @@ +name: Process Pull Request + +on: + workflow_call: + inputs: + labels: + required: false + default: "PR: Community Contribution :crown:" + type: string + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + +jobs: + add_label: + runs-on: ubuntu-latest + steps: + - uses: actions-ecosystem/action-add-labels@v1 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + labels: ${{ inputs.labels }} \ No newline at end of file diff --git a/.github/workflows/requirements-runner.txt b/.github/workflows/requirements-runner.txt new file mode 100644 index 00000000..23af0b09 --- /dev/null +++ b/.github/workflows/requirements-runner.txt @@ -0,0 +1,2 @@ +conan>=1.60.2,<2.0.0 +sip<=6.7.12 \ No newline at end of file diff --git a/.github/workflows/slack-notify.yml b/.github/workflows/slack-notify.yml new file mode 100644 index 00000000..b265a4fe --- /dev/null +++ b/.github/workflows/slack-notify.yml @@ -0,0 +1,53 @@ +name: Get Conan Recipe Version + +on: + workflow_call: + inputs: + success: + required: true + type: boolean + + success_title: + required: true + type: string + + success_body: + required: true + type: string + + failure_title: + required: true + type: string + + failure_body: + required: true + type: string + +jobs: + slackNotification: + name: Slack Notification + + runs-on: ubuntu-latest + + steps: + - name: Slack notify on-success + if: ${{ inputs.success }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_USERNAME: ${{ github.repository }} + SLACK_COLOR: green + SLACK_ICON: https://github.com/Ultimaker/Cura/blob/main/icons/cura-128.png?raw=true + SLACK_TITLE: ${{ inputs.success_title }} + SLACK_MESSAGE: ${{ inputs.success_body }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + + - name: Slack notify on-failure + if: ${{ !inputs.success }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_USERNAME: ${{ github.repository }} + SLACK_COLOR: red + SLACK_ICON: https://github.com/Ultimaker/Cura/blob/main/icons/cura-128.png?raw=true + SLACK_TITLE: ${{ inputs.failure_title }} + SLACK_MESSAGE: ${{ inputs.failure_body }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/unit-test-post.yml b/.github/workflows/unit-test-post.yml new file mode 100644 index 00000000..c5166603 --- /dev/null +++ b/.github/workflows/unit-test-post.yml @@ -0,0 +1,95 @@ +name: unit-test-post + +on: + workflow_call: + inputs: + event: + required: true + type: string + + conclusion: + required: true + type: string + +permissions: + issues: write + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + +jobs: + publish-test-results: + if: ${{ inputs.event == 'pull_request' && inputs.conclusion == 'success' }} + runs-on: ubuntu-latest + + steps: + - name: Download analysis results + uses: actions/github-script@v3.1.0 + with: + script: | + let artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{github.event.workflow_run.id }}, + }); + let matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "test-result" + })[0]; + let download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: "zip", + }); + let fs = require("fs"); + fs.writeFileSync("${{github.workspace}}/test-result.zip", Buffer.from(download.data)); + + - name: Set environment variables + run: | + mkdir pr_env + unzip test-result.zip -d pr_env + echo "pr_id=$(cat pr_env/pr-id.txt)" >> $GITHUB_ENV + echo "pr_head_repo=$(cat pr_env/pr-head-repo.txt)" >> $GITHUB_ENV + echo "pr_head_ref=$(cat pr_env/pr-head-ref.txt)" >> $GITHUB_ENV + + - uses: actions/checkout@v3 + with: + repository: ${{ env.pr_head_repo }} + ref: ${{ env.pr_head_ref }} + persist-credentials: false + + - name: Redownload analysis results + uses: actions/github-script@v3.1.0 + with: + script: | + let artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{github.event.workflow_run.id }}, + }); + let matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "test-result" + })[0]; + let download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: "zip", + }); + let fs = require("fs"); + fs.writeFileSync("${{github.workspace}}/test-result.zip", Buffer.from(download.data)); + + - name: Extract analysis results + run: | + mkdir -p tests + unzip test-result.zip -d tests + + - name: Publish Unit Test Results + id: test-results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + files: "tests/**/*.xml" + + - name: Conclusion + run: echo "Conclusion is ${{ steps.test-results.outputs.json && fromJSON( steps.test-results.outputs.json ).conclusion }}" diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml new file mode 100644 index 00000000..f767e903 --- /dev/null +++ b/.github/workflows/unit-test.yml @@ -0,0 +1,150 @@ +--- +name: unit-test + +on: + workflow_call: + inputs: + recipe_id_full: + required: true + type: string + + conan_extra_args: + required: false + default: '' + type: string + + unit_test_cmd: + required: true + type: string + + conan_generator_dir: + required: false + default: './build/Release/generators' + type: string + + unit_test_dir: + required: true + type: string + + build: + required: false + default: false + type: boolean + +permissions: + contents: read + +env: + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_USER }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASS }} + +jobs: + testing: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.head_ref }} + + - name: Checkout repo PR + uses: actions/checkout@v4 + if: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + fetch-depth: 1 + ref: ${{ github.base_ref }} + + - name: Sync pip requirements + run: wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/.github/workflows/requirements-runner.txt -O .github/workflows/requirements-runner.txt + + - name: Setup Python and pip + uses: actions/setup-python@v4 + with: + python-version: 3.11.x + cache: pip + cache-dependency-path: .github/workflows/requirements-runner.txt + + - name: Install Python requirements and Create default Conan profile + run: pip install -r .github/workflows/requirements-runner.txt + + - name: Install Linux system requirements for building + run: | + mkdir runner_scripts + wget https://raw.githubusercontent.com/Ultimaker/cura-workflows/main/runner_scripts/ubuntu_setup.sh -O runner_scripts/ubuntu_setup.sh + chmod +x runner_scripts/ubuntu_setup.sh + sudo ./runner_scripts/ubuntu_setup.sh + + - name: Setup pipeline caches + run: | + mkdir -p /home/runner/.conan/downloads + mkdir -p /home/runner/.conan/data + + - name: Create default Conan profile + run: conan profile new default --detect + + - name: Get Conan configuration + run: | + conan config install https://github.com/Ultimaker/conan-config.git + conan config install https://github.com/Ultimaker/conan-config.git -a "-b runner/${{ runner.os }}/${{ runner.arch }}" + + - name: Cache Conan packages + uses: actions/cache@v3 + with: + path: /home/runner/.conan/data + key: ${{ runner.os }}-conan-data-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-data- + + - name: Cache Conan downloads + uses: actions/cache@v3 + with: + path: /home/runner/.conan/downloads + key: ${{ runner.os }}-conan-downloads-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-conan-downloads- + + - name: Install dependencies + run: conan install . ${{ inputs.recipe_id_full }} -s build_type=Release --build=missing --update -g GitHubActionsRunEnv -g GitHubActionsBuildEnv ${{ inputs.conan_extra_args }} + + - name: Set Environment variables from Conan install (bash) + run: | + for file in activate_github_actions*.sh; do + if [ -f "$file" ]; then + . "./$file" + fi + done + echo "$PATH" >> $GITHUB_PATH + working-directory: ${{ inputs.conan_generator_dir }} + + - name: Build + if: ${{ inputs.build == true }} + run: | + cmake --preset release + cmake --build --preset release + + - name: Run Unit Test + id: run-test + run: ${{ inputs.unit_test_cmd }} + working-directory: ${{ inputs.unit_test_dir }} + + - name: Save PR metadata + if: always() + run: | + echo ${{ github.event.number }} > pr-id.txt + echo ${{ github.event.pull_request.head.repo.full_name }} > pr-head-repo.txt + echo ${{ github.event.pull_request.head.ref }} > pr-head-ref.txt + working-directory: ${{ inputs.unit_test_dir }} + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-result + path: | + ${{ inputs.unit_test_dir }}/**/*.xml + ${{ inputs.unit_test_dir }}/pr-id.txt + ${{ inputs.unit_test_dir }}/pr-head-repo.txt + ${{ inputs.unit_test_dir }}/pr-head-ref.txt diff --git a/doc/buildsystem introduction.pdf b/doc/buildsystem introduction.pdf new file mode 100644 index 00000000..bf2b7c23 Binary files /dev/null and b/doc/buildsystem introduction.pdf differ diff --git a/runner_scripts/appimage_setup.sh b/runner_scripts/appimage_setup.sh new file mode 100644 index 00000000..4eb39593 --- /dev/null +++ b/runner_scripts/appimage_setup.sh @@ -0,0 +1,14 @@ +# Get the AppImage tool +sudo apt install coreutils binutils patchelf desktop-file-utils fakeroot fuse squashfs-tools strace util-linux zsync libgdk-pixbuf2.0-dev -y + +wget --no-check-certificate --quiet https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage -O $GITHUB_WORKSPACE/appimagetool +chmod +x $GITHUB_WORKSPACE/appimagetool +echo "APPIMAGETOOL_LOCATION=$GITHUB_WORKSPACE/appimagetool" >> $GITHUB_ENV + +# Get the AppImage builder +wget --no-check-certificate --quiet -O $GITHUB_WORKSPACE/appimage-builder-x86_64.AppImage https://github.com/AppImageCrafters/appimage-builder/releases/download/v1.1.0/appimage-builder-1.1.0-x86_64.AppImage +chmod +x appimage-builder-x86_64.AppImage +echo "APPIMAGEBUILDER_LOCATION=$GITHUB_WORKSPACE/appimage-builder-x86_64.AppImage" >> $GITHUB_ENV + +# Make sure these tools can be found on the path +echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH \ No newline at end of file diff --git a/runner_scripts/cura_installer_filename.py b/runner_scripts/cura_installer_filename.py new file mode 100644 index 00000000..2e2fa527 --- /dev/null +++ b/runner_scripts/cura_installer_filename.py @@ -0,0 +1,36 @@ +import argparse +import os + + +def set_installer_filename(args): + os_name = {"Linux": "linux", "Windows": "win64", "macOS": "macos"}.get(args.os) + enterprise = "-Enterprise" if args.enterprise == "true" else "" + internal = "-Internal" if args.internal == "true" else "" + + installer_filename_args = ["UltiMaker-Cura", os.getenv('CURA_VERSION_FULL')] + if args.enterprise == "true": + installer_filename_args.append("Enterprise") + if args.internal == "true": + installer_filename_args.append("Internal") + installer_filename_args.append(os_name) + installer_filename_args.append(args.architecture) + installer_filename = "-".join(installer_filename_args) + output_env = os.environ["GITHUB_OUTPUT"] + content = "" + if os.path.exists(output_env): + with open(output_env, "r") as f: + content = f.read() + + with open(output_env, "w") as f: + f.write(content) + f.writelines(f"INSTALLER_FILENAME={installer_filename}\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description = 'Set the installer filename') + parser.add_argument('--os', type = str, help = 'OS') + parser.add_argument('--architecture', type = str, help = 'Architecture') + parser.add_argument('--enterprise', type = str, help = 'Enterprise') + parser.add_argument('--internal', type = str, help = 'Internal') + args = parser.parse_args() + set_installer_filename(args) diff --git a/runner_scripts/get_conan_broadcast_data.py b/runner_scripts/get_conan_broadcast_data.py new file mode 100644 index 00000000..18e612b5 --- /dev/null +++ b/runner_scripts/get_conan_broadcast_data.py @@ -0,0 +1,76 @@ +import argparse +import os +import yaml + +def get_conan_broadcast_data(args): + with open("conandata.yml", "r") as f: + conan_data = yaml.safe_load(f) + version = conan_data["version"] if args.version == '' else args.version + project_name = args.project_name + + if args.release == "true": + actual_version = version + user = "_" + channel = "_" + is_release_branch = True + else: + build_metadata = "" + if "+" in version: + build_metadata += f"+{version.split('+')[1]}" + version = version.split("+")[0] + elif args.sha: + build_metadata += f"+{args.sha[:6]}" + + actual_version = f"{version}{build_metadata}" + user = args.user.lower() + is_release_branch = False + if args.channel: + channel = args.channel.lower() + else: + ref_name = args.base_ref if args.event_name == "pull_request" else args.ref_name + if "beta" in version and args.event_name != "pull_request" and ref_name == '.'.join(version.split('.')[:2]): + channel = "stable" + is_release_branch = True + else: + if ref_name in ("main", "master"): + channel = 'testing' + else: + channel = "_".join(ref_name.replace("-", "_").split("_")[:2]).lower() + + # %% Set the environment output + output_env = os.environ["GITHUB_OUTPUT"] + with open(output_env, "a") as f: + f.writelines(f"name={project_name}\n") + f.writelines(f"version={actual_version}\n") + f.writelines(f"channel={channel}\n") + f.writelines(f"recipe_id_full={project_name}/{actual_version}@{user}/{channel}\n") + f.writelines(f"recipe_id_latest={project_name}/latest@{user}/{channel}\n") + f.writelines(f"semver_full={actual_version}\n") + f.writelines(f"is_release_branch={str(is_release_branch).lower()}\n") + + summary_env = os.environ["GITHUB_STEP_SUMMARY"] + with open(summary_env, "w") as f: + f.writelines(f"# {project_name}\n") + f.writelines(f"name={project_name}\n") + f.writelines(f"version={actual_version}\n") + f.writelines(f"channel={channel}\n") + f.writelines(f"recipe_id_full={project_name}/{actual_version}@{user}/{channel}\n") + f.writelines(f"recipe_id_latest={project_name}/latest@{user}/{channel}\n") + f.writelines(f"semver_full={actual_version}\n") + f.writelines(f"is_release_branch={str(is_release_branch).lower()}\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description = 'Get Conan broadcast data') + parser.add_argument('--user', type = str, help = 'User input') + parser.add_argument('--channel', type = str, help = 'User channel') + parser.add_argument('--project_name', type = str, help = 'Name of the project') + parser.add_argument('--sha', type = str, help = 'Commit SHA') + parser.add_argument('--event_name', type = str, help = 'Github event name') + parser.add_argument('--base_ref', type = str, help = 'Github base reference') + parser.add_argument('--ref_name', type = str, help = 'Github name reference') + parser.add_argument('--release', type = str, help = 'Is a release') + parser.add_argument('--version', type = str, help = 'User override version') + + args = parser.parse_args() + get_conan_broadcast_data(args) diff --git a/runner_scripts/macos_setup.sh b/runner_scripts/macos_setup.sh new file mode 100644 index 00000000..b7e5765b --- /dev/null +++ b/runner_scripts/macos_setup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +brew install autoconf automake ninja wget \ No newline at end of file diff --git a/runner_scripts/summarize_used_dependencies.py b/runner_scripts/summarize_used_dependencies.py new file mode 100644 index 00000000..0d266196 --- /dev/null +++ b/runner_scripts/summarize_used_dependencies.py @@ -0,0 +1,30 @@ +import argparse +import os + +from cura.CuraVersion import ConanInstalls, PythonInstalls + + +def set_gh_output_used_dependencies(args): + summary_env = os.environ["GITHUB_STEP_SUMMARY"] + content = "" + if os.path.exists(summary_env): + with open(summary_env, "r") as f: + content = f.read() + + with open(summary_env, "w") as f: + f.write(content) + f.writelines(f"# {args.installer_filename}\n") + f.writelines("## Conan packages:\n") + for dep_name, dep_info in ConanInstalls.items(): + f.writelines(f"`{dep_name} {dep_info['version']} {dep_info['revision']}`\n") + + f.writelines("## Python modules:\n") + for dep_name, dep_info in PythonInstalls.items(): + f.writelines(f"`{dep_name} {dep_info['version']}`\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description = 'Set the installer filename') + parser.add_argument('--installer_filename', type = str, help = 'INSTALLER_FILENAME') + args = parser.parse_args() + set_gh_output_used_dependencies(args) diff --git a/runner_scripts/ubuntu_setup.sh b/runner_scripts/ubuntu_setup.sh new file mode 100644 index 00000000..a21225af --- /dev/null +++ b/runner_scripts/ubuntu_setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +add-apt-repository ppa:ubuntu-toolchain-r/test -y +apt update +apt upgrade +apt install build-essential checkinstall libegl-dev zlib1g-dev libssl-dev ninja-build autoconf libx11-dev libx11-xcb-dev libfontenc-dev libice-dev libsm-dev libxau-dev libxaw7-dev libxcomposite-dev libxcursor-dev libxdamage-dev libxdmcp-dev libxext-dev libxfixes-dev libxi-dev libxinerama-dev libxkbfile-dev libxmu-dev libxmuu-dev libxpm-dev libxrandr-dev libxrender-dev libxres-dev libxss-dev libxt-dev libxtst-dev libxv-dev libxvmc-dev libxxf86vm-dev xtrans-dev libxcb-render0-dev libxcb-render-util0-dev libxcb-xkb-dev libxcb-icccm4-dev libxcb-image0-dev libxcb-keysyms1-dev libxcb-randr0-dev libxcb-shape0-dev libxcb-sync-dev libxcb-xfixes0-dev libxcb-xinerama0-dev xkb-data libxcb-dri3-dev uuid-dev libxcb-util-dev libxkbcommon-x11-dev libxcb-cursor-dev pkg-config flex bison -y + +apt install g++-13 gcc-13 -y +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 13 +update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 13 diff --git a/runner_scripts/upload_conan_recipes.py b/runner_scripts/upload_conan_recipes.py new file mode 100644 index 00000000..24ea911d --- /dev/null +++ b/runner_scripts/upload_conan_recipes.py @@ -0,0 +1,46 @@ +import argparse +import os +import re +import yaml + +from pathlib import Path + +def upload_changed_recipes(args): + files = args.Files + configs = dict(zip([ str(f).split("/")[1] for f in files ], [ Path(*str(f).split("/")[:2]).joinpath("config.yml") for f in files ])) + + packages = [] + channel = "stable" if "main" in args.branch else re.match(r"CURA-\d*", args.branch)[0].lower().replace("-", "_") + + for name, config in configs.items(): + if not config.exists(): + continue + + versions = {} + with open(config, "r") as f: + versions = yaml.safe_load(f)["versions"] + + for version, data in versions.items(): + conanfile = config.parent.joinpath(data["folder"], "conanfile.py") + package = f"{name}/{version}@{args.user}/{channel}" + create_cmd = f"conan create {conanfile} {package}" + os.system(create_cmd) + upload_cmd = f"conan upload {package} -r {args.remote} -c" + os.system(upload_cmd) + packages.append(package) + + summary_env = os.environ["GITHUB_STEP_SUMMARY"] + with open(summary_env, "w") as f: + f.writelines(f"# Created and Uploaded to remote {args.remote}\n") + for package in packages: + f.writelines(f"{package}\n") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description = 'Upload all the changed recipes in the recipe folder') + parser.add_argument('--user', type = str, help = 'User') + parser.add_argument('--branch', type = str, help = 'Branch') + parser.add_argument('--remote', type = str, help = 'Remote') + parser.add_argument("Files", metavar="F", type=str, nargs="+", help="Files or directories to format") + + args = parser.parse_args() + upload_changed_recipes(args) diff --git a/scripts/remove_conan_artifacts.py b/scripts/remove_conan_artifacts.py new file mode 100644 index 00000000..904ae6ee --- /dev/null +++ b/scripts/remove_conan_artifacts.py @@ -0,0 +1,41 @@ +from bs4 import BeautifulSoup +import requests +import os +import subprocess +import argparse + +def remove_conan_binaries(query: str, remote: str): + # Search for packages given the query + conan_search_command = f"conan search {query} --raw -r {remote}" + process = subprocess.run(conan_search_command.split(), capture_output=True) + packages = process.stdout.decode('utf-8').splitlines() + for package in packages: + conan_search_command = f"conan search {package} --table packages.html -r {remote}" + os.system(conan_search_command) + try: + # Read the generated HTML file + with open("packages.html", "r") as f: + contents = f.read() + # Create a BeautifulSoup object and specify the parser + soup = BeautifulSoup(contents, 'html.parser') + # Find the table that contains the package ids + table = soup.find_all('table')[-1] + # Iterate over rows in the table skip the header row + rows = table.find_all('tr')[1:] + package_ids = [row.find_all('td')[1].text for row in rows if len(row.find_all('td')) > 1] + finally: + os.remove("packages.html") + # Remove each binary package (but not the recipe) + for package_id in package_ids: + print(f"Removing binaries {package_id} for {package}") + remove_command = f"conan remove -p {package_id} -f -r {remote} {package}" + # print(remove_command) + os.system(remove_command) + +# Parse arguments +parser = argparse.ArgumentParser(description="Remove Conan Binaries") +parser.add_argument("--query", type=str, required=True, help="Query for packages to remove") +parser.add_argument("--remote", type=str, required=True, help="Remote to remove from") + +args = parser.parse_args() +remove_conan_binaries(args.query, args.remote) \ No newline at end of file diff --git a/scripts/transfer_conan_recipes.py b/scripts/transfer_conan_recipes.py new file mode 100644 index 00000000..209a3576 --- /dev/null +++ b/scripts/transfer_conan_recipes.py @@ -0,0 +1,94 @@ +from bs4 import BeautifulSoup +import requests +import os +import subprocess +import argparse +import re + +from conan.tools.scm import Version + + +def list_all_editable_packages(): + conan_search_command = "conan search --raw" + process = subprocess.run(conan_search_command.split(), capture_output = True) + packages = process.stdout.decode('utf-8').splitlines() + return packages + + +def clear_local_cache(): + conan_remove_command = "conan remove \"*\" -f" + os.system(conan_remove_command) + + +def get_remotes(): + conan_remotes_command = "conan remote list" + process = subprocess.run(conan_remotes_command.split(), capture_output = True) + remotes = [remote.split(":")[0] for remote in process.stdout.decode('utf-8').splitlines()] + return remotes + + +def disable_all_remotes(): + remotes = get_remotes() + for remote in remotes: + conan_disable_command = f"conan remote disable {remote}" + os.system(conan_disable_command) + + +def enable_remote(remote: str): + conan_enable_command = f"conan remote enable {remote}" + os.system(conan_enable_command) + + +def get_recipes_from_remote(query: str, remote: str): + disable_all_remotes() + enable_remote(remote) + + conan_search_command = f"conan search {query} --raw -r {remote}" + process = subprocess.run(conan_search_command.split(), capture_output = True) + packages = process.stdout.decode('utf-8').splitlines() + return packages + + +def upload_package_filter(package): + p = package.split("/") + if p[0] in ["arcus", "cura", "cura_binary_data", "curaengine", "curaengine_grpc_definitions", "curaengine_plugin_gradual_flow", "curaengine_plugin_infill_generate", "curaengine_plugin_postprocess", "curaengine_simplify_plugin", + "dulcificum", "fdm_materials", "nest2d", "pyarcus", "pynest2d", "pyprojecttoolchain", "pysavitar", "savitar", "scripta", "sipbuildtool", "spatial", "standardprojectsettings", "translationextractor", "umbase", + "umspatial", "umspatialbackend", "uranium"]: + return True + return False + + +def download_packages(packages: list, remote: str): + for package in packages: + print(f"Downloading {package}") + conan_download_command = f"conan download {package} -r {remote}" + os.system(conan_download_command) + + +def upload_packages(packages: list, remote: str): + for package in packages: + print(f"Uploading {package}") + conan_upload_command = f"conan upload {package} -r {remote}" + os.system(conan_upload_command) + + +def main(): + # Parse arguments + parser = argparse.ArgumentParser(description = "Transfer Conan Recipes") + parser.add_argument("--source-remote", type = str, required = True, help = "source remote") + parser.add_argument("--target-remote", type = str, required = True, help = "target remote") + + args = parser.parse_args() + + clear_local_cache() + packages = [] + for package in get_recipes_from_remote("*", args.source_remote): + if upload_package_filter(package): + packages.append(package) + + download_packages(packages, args.source_remote) + upload_packages(packages, args.target_remote) + + +if __name__ == "__main__": + main()