From 7299cb02587a6cc0c36e4e2ad425f03acc8d84bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 15 Apr 2023 02:02:11 +0200 Subject: [PATCH 1/6] CICD: Extract LXC images from LXD images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build-lxd.yml: Merge jobs update-previous and test-update - build-lxd.yml: Add job for converting LXD images to LXC images - release.yml: Release LXC images Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-lxd.yml | 158 +++++++++++++++----------------- .github/workflows/release.yml | 33 ++++++- 2 files changed, 105 insertions(+), 86 deletions(-) diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 2fa013930..d99cb01b8 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -10,10 +10,14 @@ on: type: string default: 'x86' outputs: - artifact_name: + lxd_artifact_name: value: "${{ jobs.build-current.outputs.artifact_name }}" - artifact_file: + lxd_artifact_file: value: "${{ jobs.build-current.outputs.artifact_file }}" + lxc_artifact_name: + value: "${{ jobs.convert-to-lxc-image.outputs.artifact_name }}" + lxc_artifact_file: + value: "${{ jobs.convert-to-lxc-image.outputs.artifact_file }}" push: branches: - "**" @@ -168,7 +172,7 @@ jobs: path: "output/${{ steps.pack-lxd.outputs.artifact_file }}" if-no-files-found: error - update-previous: + test-update: needs: - build-previous runs-on: ubuntu-20.04 @@ -193,6 +197,14 @@ jobs: nictype: bridged type: nic EOF + - name: Setup Firefox + uses: browser-actions/setup-firefox@latest + - name: Setup GeckoDriver + uses: ChlodAlejandro/setup-geckodriver@latest + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Selenium + run: pip install selenium - name: Checkout code uses: actions/checkout@v3 with: @@ -241,82 +253,12 @@ jobs: fi lxc stop ncp - - name: Pack LXD image - id: pack-lxd - run: | - set -x - . ./build/buildlib.sh - ARTIFACT_FILE="NextCloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}" lxc publish -q ncp -f --alias "ncp/updated" - mkdir -p output - lxc image export -q "ncp/updated" "output/${ARTIFACT_FILE}" - echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT - - name: upload LXD image to artifact store - uses: actions/upload-artifact@v3 - with: - name: "${{ env.ARTIFACT_NAME }}" - path: "output/${{ steps.pack-lxd.outputs.artifact_file }}" - if-no-files-found: error - - test-fresh-install: - needs: - - determine-runner - - build-current - runs-on: ${{ needs.determine-runner.outputs.runner_label }} - env: - VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" - ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }} - ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: "${{ env.VERSION }}" - - name: Cleanup lxd - run: test -z "$(lxc profile device show default | grep eth0)" || lxc profile device remove default eth0 - - uses: whywaita/setup-lxd@v1 - with: - lxd_version: latest/stable - - name: Fix LXD - continue-on-error: true - run: | - lxc profile create network - cat </dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' sleep 30 ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" @@ -350,23 +292,27 @@ jobs: } lxc stop ncp - test-update: + test-fresh-install: needs: - - update-previous - runs-on: ubuntu-20.04 + - determine-runner + - build-current + runs-on: ${{ needs.determine-runner.outputs.runner_label }} env: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" - ARTIFACT_NAME: ${{ needs.update-previous.outputs.artifact_name }} - ARTIFACT_FILE: ${{ needs.update-previous.outputs.artifact_file }} + ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }} + ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }} steps: - name: Checkout code uses: actions/checkout@v3 with: ref: "${{ env.VERSION }}" + - name: Cleanup lxd + run: test -z "$(lxc profile device show default | grep eth0)" || lxc profile device remove default eth0 - uses: whywaita/setup-lxd@v1 with: lxd_version: latest/stable - name: Fix LXD + continue-on-error: true run: | lxc profile create network cat </dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' sleep 30 ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" @@ -426,3 +383,40 @@ jobs: exit 1 } lxc stop ncp + + convert-to-lxc-image: + needs: + - determine-runner + - build-current + runs-on: ${{ needs.determine-runner.outputs.runner_label }} + outputs: + artifact_name: "${{ steps.lxd-to-lxc.outputs.artifact_name }}" + artifact_file: "${{ steps.lxd-to-lxc.outputs.artifact_file }}" + env: + VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + LXD_ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }} + LXD_ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }} + steps: + - name: download LXD image from artifact store + uses: actions/download-artifact@v3 + with: + name: ${{ env.LXD_ARTIFACT_NAME }} + - name: convert to LXC image + id: lxd-to-lxc + run: | + mkdir repackage output + cd repackage + sudo tar xpf "../${LXD_ARTIFACT_FILE?}" + sudo rm -rf ./rootfs/dev + LXC_ARTIFACT_FILE="${LXD_ARTIFACT_FILE//LXD/LXC_EXPERIMENTAL}" + sudo tar cpzf "../output/${LXC_ARTIFACT_FILE?}" -C rootfs/ . + cd .. + sudo chown "$(id -un):" "./output/${LXC_ARTIFACT_FILE}" + echo "artifact_file=${LXC_ARTIFACT_FILE?}" >> $GITHUB_OUTPUT + echo "artifact_name=${LXD_ARTIFACT_NAME//lxd/lxc}" >> $GITHUB_OUTPUT + - name: upload LXD image to artifact store + uses: actions/upload-artifact@v3 + with: + name: "${{ steps.lxd-to-lxc.outputs.artifact_name }}" + path: "output/${{ steps.lxd-to-lxc.outputs.artifact_file }}" + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff7f6a40b..9874c7e03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -164,9 +164,10 @@ jobs: uses: ./.github/workflows/publish-image.yml with: git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" - artifact_id: "${{ needs.lxd-x86.outputs.artifact_name }}" - artifact_file: "${{ needs.lxd-x86.outputs.artifact_file }}" + artifact_id: "${{ needs.lxd-x86.outputs.lxd_artifact_name }}" + artifact_file: "${{ needs.lxd-x86.outputs.lxd_artifact_file }}" dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} + lxd-arm64-release: needs: - github-release @@ -175,8 +176,32 @@ jobs: uses: ./.github/workflows/publish-image.yml with: git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" - artifact_id: "${{ needs.lxd-arm64.outputs.artifact_name }}" - artifact_file: "${{ needs.lxd-arm64.outputs.artifact_file }}" + artifact_id: "${{ needs.lxd-arm64.outputs.lxd_artifact_name }}" + artifact_file: "${{ needs.lxd-arm64.outputs.lxd_artifact_file }}" + dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} + + lxc-x86-release: + needs: + - github-release + - lxd-x86 + if: ${{ inputs.lxd || github.event_name != 'workflow_dispatch' }} + uses: ./.github/workflows/publish-image.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + artifact_id: "${{ needs.lxd-x86.outputs.lxc_artifact_name }}" + artifact_file: "${{ needs.lxd-x86.outputs.lxc_artifact_file }}" + dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} + + lxc-arm64-release: + needs: + - github-release + - lxd-arm64 + if: ${{ inputs.lxd || github.event_name != 'workflow_dispatch' }} + uses: ./.github/workflows/publish-image.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + artifact_id: "${{ needs.lxd-arm64.outputs.lxc_artifact_name }}" + artifact_file: "${{ needs.lxd-arm64.outputs.lxc_artifact_file }}" dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} raspberrypi-release: From ae5ab8df90d0e65f9454af7441cb036f14ae85cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 15 Apr 2023 02:02:59 +0200 Subject: [PATCH 2/6] ncp.sh: Don't disable root/webui user for container images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- ncp.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ncp.sh b/ncp.sh index 2904f775c..a47b53444 100644 --- a/ncp.sh +++ b/ncp.sh @@ -132,8 +132,10 @@ EOF ## NCP USER FOR AUTHENTICATION id -u "$WEBADMIN" &>/dev/null || useradd --home-dir /nonexistent "$WEBADMIN" echo -e "$WEBPASSWD\n$WEBPASSWD" | passwd "$WEBADMIN" - chsh -s /usr/sbin/nologin "$WEBADMIN" - chsh -s /usr/sbin/nologin root + is_docker || is_lxc || { + chsh -s /usr/sbin/nologin "$WEBADMIN" + chsh -s /usr/sbin/nologin root + } ## NCP LAUNCHER mkdir -p /home/www From 4dd2dcef6a75f181ace57bc81f961499e1339b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 15 Apr 2023 02:06:01 +0200 Subject: [PATCH 3/6] update.sh: Support some previously excluded apps on LXD/LXC containers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fail2ban.sh: Use systemd backend for ssh jail (auth.log might not be available) - SSH.sh: Make sure openssh-server is installed newly supported on LXC/LXD are: - nc-autoupdate-ncp - nc-update - nc-datadir - nc-database - UFW - nc-audit - SSH - fail2ban - nc-nextcloud - nc-init - samba Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/NETWORKING/SSH.sh | 7 +++++- bin/ncp/SECURITY/fail2ban.sh | 10 +++++--- update.sh | 47 ++++++++++++++++++------------------ 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/bin/ncp/NETWORKING/SSH.sh b/bin/ncp/NETWORKING/SSH.sh index 710d628cc..12f1642ce 100644 --- a/bin/ncp/NETWORKING/SSH.sh +++ b/bin/ncp/NETWORKING/SSH.sh @@ -8,7 +8,12 @@ # -install() { :; } +install() { + apt-get update + apt-get install -y --no-install-recommends openssh-server + systemctl stop ssh + systemctl disable ssh + } is_active() { diff --git a/bin/ncp/SECURITY/fail2ban.sh b/bin/ncp/SECURITY/fail2ban.sh index a2daade22..e2c133a46 100644 --- a/bin/ncp/SECURITY/fail2ban.sh +++ b/bin/ncp/SECURITY/fail2ban.sh @@ -24,6 +24,7 @@ install() { apt-get update + apt-get install --no-install-recommends -y python3-systemd apt-get install --no-install-recommends -y fail2ban whois update-rc.d fail2ban disable rm -f /etc/fail2ban/jail.d/defaults-debian.conf @@ -123,6 +124,7 @@ action = %($ACTION)s enabled = true port = ssh filter = sshd +backend = systemd logpath = /var/log/auth.log maxretry = $MAXRETRY @@ -135,16 +137,18 @@ port = http,https filter = nextcloud logpath = $NCLOG maxretry = $MAXRETRY +backend = auto # # UFW # [ufwban] enabled = true -port = ssh, http, https -filter = ufwban +port = ssh, http, https +filter = ufwban logpath = /var/log/ufw.log -action = ufw +action = ufw +backend = auto EOF cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local diff --git a/update.sh b/update.sh index ee514d977..223c9fa50 100755 --- a/update.sh +++ b/update.sh @@ -25,44 +25,40 @@ fi CONFDIR=/usr/local/etc/ncp-config.d UPDATESDIR=updates -# don't make sense in a docker container -EXCL_DOCKER=" -nc-autoupdate-ncp -nc-update +# don't make sense in containers +EXCL_CONTAINER=" nc-automount nc-format-USB -nc-datadir -nc-database nc-ramlogs nc-swapfile nc-static-IP nc-wifi -UFW nc-snapshot nc-snapshot-auto nc-snapshot-sync nc-restore-snapshot -nc-audit nc-hdd-monitor nc-hdd-test nc-zram -SSH -fail2ban NFS " -if is_docker &>/dev/null; then -# in docker, just remove the volume for this -EXCL_DOCKER+=" +# don't make sense in a docker container +EXCL_DOCKER=" +$EXCL_CONTAINER +nc-autoupdate-ncp +nc-update +nc-datadir +nc-database +UFW +nc-audit +SSH +fail2ban nc-nextcloud nc-init -" - -# better use a designated container -EXCL_DOCKER+=" samba " -fi + # check running apt or apt-get pgrep -x "apt|apt-get" &>/dev/null && { echo "apt is currently running. Try again later"; exit 1; } @@ -74,11 +70,14 @@ source /usr/local/etc/library.sh mkdir -p "$CONFDIR" # prevent installing some ncp-apps in the containerized versions -if is_docker || is_lxc; then - for opt in $EXCL_DOCKER; do - touch $CONFDIR/$opt.cfg - done -fi + +EXCL_APPS="" +is_docker && EXCL_APPS="$EXCL_DOCKER" +is_lxc && EXCL_APPS="$EXCL_CONTAINER" + +for opt in $EXCL_APPS; do + touch $CONFDIR/$opt.cfg +done # copy all files in bin and etc cp -r bin/* /usr/local/bin/ @@ -166,7 +165,7 @@ chown -R www-data: /var/www/nextcloud/apps/nextcloudpi # remove unwanted ncp-apps for containerized versions if is_docker || is_lxc; then - for opt in $EXCL_DOCKER; do + for opt in $EXCL_APPS; do rm $CONFDIR/$opt.cfg find /usr/local/bin/ncp -name "$opt.sh" -exec rm '{}' \; done From ab501b3f51091e8546a185d0182e7276c71e0fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 15 Apr 2023 02:06:25 +0200 Subject: [PATCH 4/6] ncp-metrics.cfg.sh: Exclude snapshots from backup monitoring on LXD/LXC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- etc/ncp-templates/ncp-metrics.cfg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ncp-templates/ncp-metrics.cfg.sh b/etc/ncp-templates/ncp-metrics.cfg.sh index 505ccee71..f107a1c2b 100644 --- a/etc/ncp-templates/ncp-metrics.cfg.sh +++ b/etc/ncp-templates/ncp-metrics.cfg.sh @@ -51,7 +51,7 @@ EOF EOF } -is_docker || { +is_docker || is_lxc || { DATADIR=$( get_nc_config_value datadirectory ) || { echo "ERROR: Could not get data directory. Is NextCloud running?" >&2 From 0de39f155ee3ee07bc6429db3612a3afc36e3a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 15 Apr 2023 02:06:36 +0200 Subject: [PATCH 5/6] Add nc-broadcast utility script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/nc-broadcast | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 bin/nc-broadcast diff --git a/bin/nc-broadcast b/bin/nc-broadcast new file mode 100755 index 000000000..7a6d78c17 --- /dev/null +++ b/bin/nc-broadcast @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +if [[ ${EUID} -ne 0 ]]; then + printf "Must be run as root. Try 'sudo %s'\n" "$( basename "$0" )" + exit 1 +fi + +if [[ " $* " =~ " "(--help|-h)" " ]] +then + echo 'Description: + Generate notifications for all Nextcloud users + +Usage: + nc-broadcast [options] + +Arguments: + short-message Short message to be sent to the user (max. 255 characters) + +Options: + -l, --long-message=LONG-MESSAGE Long message to be sent to the users (max. 4000 characters) [default: ""] + -h, --help Display this help message + -q, --quiet Do not output any message + -V, --version Display ncc/occ version + --ansi|--no-ansi Force (or disable --no-ansi) ANSI output + -n, --no-interaction Do not ask any interactive question + --no-warnings Skip global warnings, show command output only + -v|vv|vvv, --verbose Increase the verbosity of messages: 1 for normal output, 2 for more verbose output and 3 for debug' + exit 0 +fi + +ncc user:list --output=json | jq -r 'keys[]' | while read -r user +do + echo "Sending notification to $user" + ncc notification:generate "${@:2}" "$user" "$1" +done +echo "All users have been notified." From f72b11d938d415a4b08483f91519b8c843f47d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 15 Apr 2023 02:17:53 +0200 Subject: [PATCH 6/6] run-parts.sh: Notify docker users about discontinuation of NCP docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/debian-ncp/run-parts.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index 2a0588906..35b545cfd 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -26,6 +26,16 @@ CURRENT_NC_MAJ="${CURRENT_NC_VERSION%%.*}" exit 1 } +DOCKER_DISCONTINUATION_ALERT="ATTENTION: NextcloudPi docker is being discontinued after Nextcloud 25! Minor NC updates and security updates and fixes will be provided until 2023/11. +Learn more in the official announcement: https://help.nextcloud.com/t/nextcloudpi-planning-to-discontinue-its-docker-version-with-nc-25" + +echo -e " +\033[1;31m$DOCKER_DISCONTINUATION_ALERT\033[0m + +Continue in 5 seconds..." + +sleep 5 + # wrapper to simulate update-rc.d cat > /usr/local/sbin/update-rc.d <<'EOF' @@ -75,6 +85,9 @@ fi # wait for trap from 'docker stop' echo "Init done" + +[[ -f /data/docker_discontinuation_alert_sent ]] || \ + ( . /usr/local/etc/library.sh; notify_admin "$DOCKER_DISCONTINUATION_ALERT" && touch /data/docker_discontinuation_alert_sent ) while true; do sleep 0.5; done