Merge pull request #1763 from nextcloud/feature/cicd/build-lxc

Build LXC images in CICD pipeline
nextcloud · Apr 15, 2023 · 6285936 · 6285936
2 parents d2c480a + f72b11d
commit 6285936
Show file tree

Hide file tree

Showing 9 changed files with 195 additions and 117 deletions.
diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml
@@ -10,10 +10,14 @@ on:
         type: string
         default: 'x86'
     outputs:
-      artifact_name:
+      lxd_artifact_name:
         value: "${{ jobs.build-current.outputs.artifact_name }}"
-      artifact_file:
+      lxd_artifact_file:
         value: "${{ jobs.build-current.outputs.artifact_file }}"
+      lxc_artifact_name:
+        value: "${{ jobs.convert-to-lxc-image.outputs.artifact_name }}"
+      lxc_artifact_file:
+        value: "${{ jobs.convert-to-lxc-image.outputs.artifact_file }}"
   push:
     branches:
       - "**"
@@ -168,7 +172,7 @@ jobs:
           path: "output/${{ steps.pack-lxd.outputs.artifact_file }}"
           if-no-files-found: error
 
-  update-previous:
+  test-update:
     needs:
       - build-previous
     runs-on: ubuntu-20.04
@@ -193,6 +197,14 @@ jobs:
               nictype: bridged
               type: nic
           EOF
+      - name: Setup Firefox
+        uses: browser-actions/setup-firefox@latest
+      - name: Setup GeckoDriver
+        uses: ChlodAlejandro/setup-geckodriver@latest
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup Selenium
+        run: pip install selenium
       - name: Checkout code
         uses: actions/checkout@v3
         with:
@@ -241,82 +253,12 @@ jobs:
           fi
           
           lxc stop ncp
-      - name: Pack LXD image
-        id: pack-lxd
-        run: |
-          set -x
-          . ./build/buildlib.sh
-          ARTIFACT_FILE="NextCloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}"
           lxc publish -q ncp -f --alias "ncp/updated"
-          mkdir -p output
-          lxc image export -q "ncp/updated" "output/${ARTIFACT_FILE}"
-          echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT
-      - name: upload LXD image to artifact store
-        uses: actions/upload-artifact@v3
-        with:
-          name: "${{ env.ARTIFACT_NAME }}"
-          path: "output/${{ steps.pack-lxd.outputs.artifact_file }}"
-          if-no-files-found: error
-
-  test-fresh-install:
-    needs:
-      - determine-runner
-      - build-current
-    runs-on: ${{ needs.determine-runner.outputs.runner_label }}
-    env:
-      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }}
-      ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          ref: "${{ env.VERSION }}"
-      - name: Cleanup lxd
-        run: test -z "$(lxc profile device show default | grep eth0)" || lxc profile device remove default eth0
-      - uses: whywaita/setup-lxd@v1
-        with:
-          lxd_version: latest/stable
-      - name: Fix LXD
-        continue-on-error: true
-        run: |
-          lxc profile create network
-          cat <<EOF | lxc profile edit network
-          devices:
-            eth0:
-              name: eth0
-              network: lxdbr0
-              type: nic
-          EOF
-      - name: Setup Firefox
-        uses: browser-actions/setup-firefox@latest
-        continue-on-error: true
-      - name: Setup GeckoDriver
-        uses: ChlodAlejandro/setup-geckodriver@latest
-        continue-on-error: true
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Setup Selenium
-        run: pip install selenium
-      - name: download LXD image from artifact store
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ env.ARTIFACT_NAME }}
-      - name: Remove old lxd images
-        if: ${{ inputs.arch == 'arm64' }}
-        run: |
-          for img in $(lxc image list -c f -f csv)
-          do
-            lxc image delete "$img"
-          done
-      - name: Launch ncp container
+      - name: Relaunch container
         run: |
           set -x
-          . ./build/buildlib.sh
           lxc delete -q -f ncp || true
-          lxc image import -q "./${ARTIFACT_FILE?}" --alias "ncp/test" || true
-          systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp || \
-          sudo systemd-run --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp 
+          systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/updated" ncp
           lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
           sleep 30
           ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
@@ -350,23 +292,27 @@ jobs:
           }
           lxc stop ncp
 
-  test-update:
+  test-fresh-install:
     needs:
-      - update-previous
-    runs-on: ubuntu-20.04
+      - determine-runner
+      - build-current
+    runs-on: ${{ needs.determine-runner.outputs.runner_label }}
     env:
       VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      ARTIFACT_NAME: ${{ needs.update-previous.outputs.artifact_name }}
-      ARTIFACT_FILE: ${{ needs.update-previous.outputs.artifact_file }}
+      ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }}
+      ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
         with:
           ref: "${{ env.VERSION }}"
+      - name: Cleanup lxd
+        run: test -z "$(lxc profile device show default | grep eth0)" || lxc profile device remove default eth0
       - uses: whywaita/setup-lxd@v1
         with:
           lxd_version: latest/stable
       - name: Fix LXD
+        continue-on-error: true
         run: |
           lxc profile create network
           cat <<EOF | lxc profile edit network
@@ -378,8 +324,10 @@ jobs:
           EOF
       - name: Setup Firefox
         uses: browser-actions/setup-firefox@latest
+        continue-on-error: true
       - name: Setup GeckoDriver
         uses: ChlodAlejandro/setup-geckodriver@latest
+        continue-on-error: true
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Selenium
@@ -388,12 +336,21 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: ${{ env.ARTIFACT_NAME }}
+      - name: Remove old lxd images
+        if: ${{ inputs.arch == 'arm64' }}
+        run: |
+          for img in $(lxc image list -c f -f csv)
+          do
+            lxc image delete "$img"
+          done
       - name: Launch ncp container
         run: |
           set -x
+          . ./build/buildlib.sh
           lxc delete -q -f ncp || true
-          lxc image import -q "./${ARTIFACT_FILE?}" --alias "ncp/test"
-          systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp
+          lxc image import -q "./${ARTIFACT_FILE?}" --alias "ncp/test" || true
+          systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp || \
+          sudo systemd-run --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp 
           lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
           sleep 30
           ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
@@ -426,3 +383,40 @@ jobs:
             exit 1
           }
           lxc stop ncp
+
+  convert-to-lxc-image:
+    needs:
+      - determine-runner
+      - build-current
+    runs-on: ${{ needs.determine-runner.outputs.runner_label }}
+    outputs:
+      artifact_name: "${{ steps.lxd-to-lxc.outputs.artifact_name }}"
+      artifact_file: "${{ steps.lxd-to-lxc.outputs.artifact_file }}"
+    env:
+      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      LXD_ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }}
+      LXD_ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }}
+    steps:
+      - name: download LXD image from artifact store
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.LXD_ARTIFACT_NAME }}
+      - name: convert to LXC image
+        id: lxd-to-lxc
+        run: |
+          mkdir repackage output
+          cd repackage
+          sudo tar xpf "../${LXD_ARTIFACT_FILE?}"
+          sudo rm -rf ./rootfs/dev
+          LXC_ARTIFACT_FILE="${LXD_ARTIFACT_FILE//LXD/LXC_EXPERIMENTAL}"
+          sudo tar cpzf "../output/${LXC_ARTIFACT_FILE?}" -C rootfs/ .
+          cd ..
+          sudo chown "$(id -un):" "./output/${LXC_ARTIFACT_FILE}"
+          echo "artifact_file=${LXC_ARTIFACT_FILE?}" >> $GITHUB_OUTPUT
+          echo "artifact_name=${LXD_ARTIFACT_NAME//lxd/lxc}" >> $GITHUB_OUTPUT
+      - name: upload LXD image to artifact store
+        uses: actions/upload-artifact@v3
+        with:
+          name: "${{ steps.lxd-to-lxc.outputs.artifact_name }}"
+          path: "output/${{ steps.lxd-to-lxc.outputs.artifact_file }}"
+          if-no-files-found: error
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -164,9 +164,10 @@ jobs:
     uses: ./.github/workflows/publish-image.yml
     with:
       git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      artifact_id: "${{ needs.lxd-x86.outputs.artifact_name }}"
-      artifact_file: "${{ needs.lxd-x86.outputs.artifact_file }}"
+      artifact_id: "${{ needs.lxd-x86.outputs.lxd_artifact_name }}"
+      artifact_file: "${{ needs.lxd-x86.outputs.lxd_artifact_file }}"
       dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
+
   lxd-arm64-release:
     needs:
       - github-release
@@ -175,8 +176,32 @@ jobs:
     uses: ./.github/workflows/publish-image.yml
     with:
       git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      artifact_id: "${{ needs.lxd-arm64.outputs.artifact_name }}"
-      artifact_file: "${{ needs.lxd-arm64.outputs.artifact_file }}"
+      artifact_id: "${{ needs.lxd-arm64.outputs.lxd_artifact_name }}"
+      artifact_file: "${{ needs.lxd-arm64.outputs.lxd_artifact_file }}"
+      dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
+
+  lxc-x86-release:
+    needs:
+      - github-release
+      - lxd-x86
+    if: ${{ inputs.lxd || github.event_name != 'workflow_dispatch' }}
+    uses: ./.github/workflows/publish-image.yml
+    with:
+      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      artifact_id: "${{ needs.lxd-x86.outputs.lxc_artifact_name }}"
+      artifact_file: "${{ needs.lxd-x86.outputs.lxc_artifact_file }}"
+      dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
+
+  lxc-arm64-release:
+    needs:
+      - github-release
+      - lxd-arm64
+    if: ${{ inputs.lxd || github.event_name != 'workflow_dispatch' }}
+    uses: ./.github/workflows/publish-image.yml
+    with:
+      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      artifact_id: "${{ needs.lxd-arm64.outputs.lxc_artifact_name }}"
+      artifact_file: "${{ needs.lxd-arm64.outputs.lxc_artifact_file }}"
       dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
 
   raspberrypi-release:

diff --git a/bin/nc-broadcast b/bin/nc-broadcast
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+if [[ ${EUID} -ne 0 ]]; then
+  printf "Must be run as root. Try 'sudo %s'\n" "$( basename "$0" )"
+  exit 1
+fi
+
+if [[ " $* " =~ " "(--help|-h)" " ]]
+then
+  echo 'Description:
+  Generate notifications for all Nextcloud users
+
+Usage:
+  nc-broadcast <short-message> [options]
+
+Arguments:
+  short-message                    Short message to be sent to the user (max. 255 characters)
+
+Options:
+  -l, --long-message=LONG-MESSAGE  Long message to be sent to the users (max. 4000 characters) [default: ""]
+  -h, --help                       Display this help message
+  -q, --quiet                      Do not output any message
+  -V, --version                    Display ncc/occ version
+      --ansi|--no-ansi             Force (or disable --no-ansi) ANSI output
+  -n, --no-interaction             Do not ask any interactive question
+      --no-warnings                Skip global warnings, show command output only
+  -v|vv|vvv, --verbose             Increase the verbosity of messages: 1 for normal output, 2 for more verbose output and 3 for debug'
+  exit 0
+fi
+
+ncc user:list --output=json | jq -r 'keys[]' | while read -r user
+do
+  echo "Sending notification to $user"
+  ncc notification:generate "${@:2}" "$user" "$1"
+done
+echo "All users have been notified."
diff --git a/bin/ncp/NETWORKING/SSH.sh b/bin/ncp/NETWORKING/SSH.sh
@@ -8,7 +8,12 @@
 #
 
 
-install() { :; }
+install() {
+  apt-get update
+  apt-get install -y --no-install-recommends openssh-server
+  systemctl stop    ssh
+  systemctl disable ssh
+ }
 
 is_active()
 {

diff --git a/bin/ncp/SECURITY/fail2ban.sh b/bin/ncp/SECURITY/fail2ban.sh
@@ -24,6 +24,7 @@
 install()
 {
   apt-get update
+  apt-get install --no-install-recommends -y python3-systemd
   apt-get install --no-install-recommends -y fail2ban whois
   update-rc.d fail2ban disable
   rm -f /etc/fail2ban/jail.d/defaults-debian.conf
@@ -123,6 +124,7 @@ action = %($ACTION)s
 enabled  = true
 port     = ssh
 filter   = sshd
+backend  = systemd
 logpath  = /var/log/auth.log
 maxretry = $MAXRETRY
 
@@ -135,16 +137,18 @@ port     = http,https
 filter   = nextcloud
 logpath  = $NCLOG
 maxretry = $MAXRETRY
+backend  = auto
 
 #
 # UFW
 #
 [ufwban]
 enabled = true
-port = ssh, http, https
-filter = ufwban
+port    = ssh, http, https
+filter  = ufwban
 logpath = /var/log/ufw.log
-action = ufw
+action  = ufw
+backend = auto
 EOF
 
   cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local

diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh
@@ -26,6 +26,16 @@ CURRENT_NC_MAJ="${CURRENT_NC_VERSION%%.*}"
   exit 1
 }
 
+DOCKER_DISCONTINUATION_ALERT="ATTENTION: NextcloudPi docker is being discontinued after Nextcloud 25! Minor NC updates and security updates and fixes will be provided until 2023/11.
+Learn more in the official announcement: https://help.nextcloud.com/t/nextcloudpi-planning-to-discontinue-its-docker-version-with-nc-25"
+
+echo -e "
+\033[1;31m$DOCKER_DISCONTINUATION_ALERT\033[0m
+
+Continue in 5 seconds..."
+
+sleep 5
+
 
 # wrapper to simulate update-rc.d
 cat > /usr/local/sbin/update-rc.d <<'EOF'
@@ -75,6 +85,9 @@ fi
 
 # wait for trap from 'docker stop'
 echo "Init done"
+
+[[ -f /data/docker_discontinuation_alert_sent ]] || \
+  ( . /usr/local/etc/library.sh; notify_admin "$DOCKER_DISCONTINUATION_ALERT" && touch /data/docker_discontinuation_alert_sent )
 while true; do sleep 0.5; done