From ff5fdb65d473a320dc4554e556593bdecbee7887 Mon Sep 17 00:00:00 2001 From: coltkondo Date: Mon, 26 Sep 2022 22:11:18 -0400 Subject: [PATCH 01/32] Fixed: Nextcloudpi icon pointing to ownyoursbits.com #1571 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- ncp-web/index.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ncp-web/index.php b/ncp-web/index.php index e7e3f5390..17045474a 100644 --- a/ncp-web/index.php +++ b/ncp-web/index.php @@ -134,7 +134,7 @@
- "> + ">
@@ -207,7 +207,7 @@
- "> + ">
From d94ca6b109eb8c2ea17ed401afac704930217482 Mon Sep 17 00:00:00 2001 From: MB-Finski <64466176+MB-Finski@users.noreply.github.com> Date: Sun, 2 Oct 2022 22:46:51 +0300 Subject: [PATCH 02/32] Temporarily place backup to BASEDIR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improves docker compatibility. Placing the backup file in /var/www/ will cause a failure in rolling back the update on containerized setups ("Can only restore from ext/btrfs/zfs filesystems"). The host filesystem is most likely compatible so using $BASEDIR as the backup location *should* solve this issue for most users. Signed-off-by: MB-Finski <64466176+MB-Finski@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-update-nc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index 60e8486f0..28fa51825 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -94,7 +94,7 @@ wget -q "$URL" -O nextcloud.tar.bz2 || { echo "Error downloading"; exit 1; } # backup #################### -BKPDIR=/var/www/ +BKPDIR="$BASEDIR" WITH_DATA=no COMPRESSED=yes LIMIT=0 From ff8877db65d73b13f8babd734d122ed79b02cd78 Mon Sep 17 00:00:00 2001 From: MB-Finski <64466176+MB-Finski@users.noreply.github.com> Date: Sun, 2 Oct 2022 22:09:38 +0300 Subject: [PATCH 03/32] Change backup directory for docker compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve docker compatibility: Change the backup directory for ./nextcloud/data to reside within the host file system (basedir) so as to avoid moving the data folder back and forth between the docker container and the host file system. In situations where the nextcloud data takes up more than the free available space on the host system, this may lead to loosing some or all of the data. Signed-off-by: MB-Finski <64466176+MB-Finski@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-update-nc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index 28fa51825..7d1930276 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -165,7 +165,7 @@ rollback() { rm -rf /var/www/nextcloud.tar.bz2 "$BASEDIR"/nextcloud-old echo "Rolling back to backup $BKP..." local TMPDATA - TMPDATA="$( mktemp -d "/var/www/ncp-data.XXXXXX" )" || { echo "Failed to create temp dir" >&2; exit 1; } + TMPDATA="$( mktemp -d "$BASEDIR/recovery/ncp-data.XXXXXX" )" || { echo "Failed to create temp dir" >&2; exit 1; } [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && mv -T "$DATADIR" "$TMPDATA" ncp-restore "$BKP" || { echo "Rollback failed! Data left at $TMPDATA"; exit 1; } [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && { rm -rf "$DATADIR"; mv -T "$TMPDATA" "$DATADIR"; } From 5178e5553f7215fc2b4f2e3b061ea0cfc017fc78 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 17:30:55 +0100 Subject: [PATCH 04/32] release.yml: Don't tag docker images as :latest on prereleases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/release.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 776524b79..1547f51eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -283,7 +283,7 @@ jobs: docker tag "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}" "ownyourbits/nextcloudpi-${arch}:${version?}" docker tag "ownyourbits/nextcloudpi-${arch}:${version?}" "ownyourbits/nextcloudpi-${arch}:latest" docker push "ownyourbits/nextcloudpi-${arch}:${version?}" - docker push "ownyourbits/nextcloudpi-${arch}:latest" + [[ "$version" =~ ^v[0-9]+'.'[0-9]+'.'[0-9]+$ ]] && docker push "ownyourbits/nextcloudpi-${arch}:latest" done docker manifest create ownyourbits/nextcloudpi:${version?} \ @@ -294,6 +294,10 @@ jobs: - name: Create manifest and push as latest to docker hub run: | + [[ "$version" =~ ^v[0-9]+'.'[0-9]+'.'[0-9]+$ ]] || { + echo "not tagging latest - is a pre-/beta-release" + exit 0 + } docker manifest create ownyourbits/nextcloudpi:latest \ ownyourbits/nextcloudpi-armhf:latest \ ownyourbits/nextcloudpi-x86:latest \ From 97085dbc3f211ee86a8fa22eba847a300c7c17e2 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:01:26 +0100 Subject: [PATCH 05/32] nc-restore: Fix potential dataloss when restoring backup (fixes #1417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - nc-restore: Keep old data directory if present - nc-restore: Temporarily move and restore user files during nc-restore Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/BACKUPS/nc-backup.sh | 1 + bin/ncp/BACKUPS/nc-restore.sh | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/bin/ncp/BACKUPS/nc-backup.sh b/bin/ncp/BACKUPS/nc-backup.sh index 240ab4091..b25c3318d 100644 --- a/bin/ncp/BACKUPS/nc-backup.sh +++ b/bin/ncp/BACKUPS/nc-backup.sh @@ -114,6 +114,7 @@ tar $compress_arg -cf "$destfile" \ --exclude "nextcloud/data/*/files_versions/*" \ --exclude "nextcloud/data/ncp-update-backups" \ --exclude "nextcloud/data/__groupfolders" \ + --exclude "nextcloud/.data_*" \ -C $basedir nextcloud/ \ || { echo "error generating backup" diff --git a/bin/ncp/BACKUPS/nc-restore.sh b/bin/ncp/BACKUPS/nc-restore.sh index 36a158cc9..2dcd39a3a 100644 --- a/bin/ncp/BACKUPS/nc-restore.sh +++ b/bin/ncp/BACKUPS/nc-restore.sh @@ -63,9 +63,31 @@ tar $compress_arg -xf "$BACKUPFILE" -C "$TMPDIR" || exit 1 ## RESTORE FILES echo "restore files..." +[[ -d "$NCDIR/data" ]] && { + DATA_BKP_DIR="$(mktemp -d -p "$NCDIR/.." nc-data-XXXXXX)" + mv -T "$NCDIR/data" "$DATA_BKP_DIR/" +} rm -rf "$NCDIR" mv -T "$TMPDIR"/nextcloud "$NCDIR" || { echo "Error restoring base files"; exit 1; } +if [[ -n "$DATA_BKP_DIR" ]] +then + echo "Restoring $NCDIR/data..." + target="$NCDIR/data" + [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" + mv -T "$DATA_BKP_DIR" "$target" +else +fi + +if [[ -n "$DATA_BKP_DIR" ]] +then + echo "Restoring $NCDIR/data..." + target="$NCDIR/data" + [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" + mv -T "$DATA_BKP_DIR" "$target" +else +fi + # update NC database password to this instance sed -i "s|'dbpassword' =>.*|'dbpassword' => '$DBPASSWD',|" /var/www/nextcloud/config/config.php From 73213342d12f57cf5cf5fee4e89d37fcde41a39e Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Tue, 4 Oct 2022 00:26:27 +0200 Subject: [PATCH 06/32] library.sh: Make notify_push setup in set-nc-domain more robust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- etc/library.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/etc/library.sh b/etc/library.sh index ee94cf3c8..e43a8f456 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -159,7 +159,12 @@ function set-nc-domain() ncc config:system:set trusted_proxies 13 --value="${domain}" ncc config:system:set trusted_proxies 14 --value="$(dig +short "${domain}")" sleep 5 # this seems to be required in the VM for some reason. We get `http2 error: protocol error` after ncp-upgrade-nc - ncc notify_push:setup "${url}/push" + for try in {1..3} + do + echo "Setup notify_push (attempt ${try}/3)" + ncc notify_push:setup "${url}/push" + sleep 5 + done fi } From bd835dcd82d6aa157851ca898384f24d709f2c9b Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:05:28 +0100 Subject: [PATCH 07/32] Fix installation and pipeline errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 47 +++++++++++++++++++++++++++++- bin/ncp/CONFIG/nc-datadir.sh | 6 +++- bin/ncp/NETWORKING/SSH.sh | 2 +- build/build-SD-rpi.sh | 2 -- build/docker/Dockerfile | 10 ++++++- etc/library.sh | 33 +++++++++++++++++++++ etc/ncp-config.d/nc-datadir.cfg | 2 +- install.sh | 8 +++++ 8 files changed, 103 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index e42579ceb..97e50abf6 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -67,6 +67,41 @@ jobs: VERSION: "${{ inputs.git_ref || github.ref }}" ARCH: "${{ inputs.arch || 'x86' }}" steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" + fetch-depth: 0 + - name: Get previous release + id: get_previous_version + shell: bash + run: | + set -ex + if [[ -n "${{ github.base_ref }}" ]] + then + version="${{ github.base_ref }}" + elif [[ "${{ env.VERSION }}" == "refs/heads/devel" ]] + then + version="latest" + else + if [[ "${{ github.ref_type }}" != "tag" ]] || ! git describe --tags > /dev/null + then + git fetch -fu --tags origin ${{ env.VERSION }}:${{ env.VERSION }} + fi + version="$(git describe --tags || true)" + [[ "$version" =~ .*-.*-.* ]] || { + git checkout HEAD~1 + version="$(git describe --tags || echo latest)" + } + version="${version%-*-*}" + fi + curl "https://hub.docker.com/v2/repositories/ownyourbits/nextcloudpi-${ARCH}/tags/${version}" | grep 'errinfo' > /dev/null && { + echo "Image ownyourbits/nextcloudpi-${ARCH}:${version} does not exist - falling back to :latest" + version="latest" + } + echo "Previous version is '$version'" + echo "PREVIOUS_VERSION=$version" >> "$GITHUB_ENV" + echo "::set-output name=previous_version::${version}" - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -75,7 +110,17 @@ jobs: docker volume create ncdata platform=linux/${ARCH?} [[ "$ARCH" == 'x86' ]] && platform=linux/amd64 - docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ownyourbits/nextcloudpi:latest + tmpdir=$(mktemp -d) + pushd "$tmpdir" + cat < ./Dockerfile + FROM --platform=$platform ownyourbits/nextcloudpi:${{ steps.get_previous_version.outputs.previous_version }} + + RUN echo 'Mutex posixsem' >> /etc/apache2/mods-available/ssl.conf + EOF + docker build . -t ncp-testing -f Dockerfile + popd + rm -r "$tmpdir" + docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing - name: Checkout code uses: actions/checkout@v3 diff --git a/bin/ncp/CONFIG/nc-datadir.sh b/bin/ncp/CONFIG/nc-datadir.sh index 3a0061d4c..8d55a98f9 100644 --- a/bin/ncp/CONFIG/nc-datadir.sh +++ b/bin/ncp/CONFIG/nc-datadir.sh @@ -147,7 +147,11 @@ configure() ( . "${BINDIR?}/SYSTEM/metrics.sh" - reload_metrics_config + reload_metrics_config || { + echo 'WARN: There was an issue reloading ncp metrics. This might not affect your installation, + but keep it in mind if there is an issue with metrics.' + true + } ) echo "The NC data directory has been moved successfully." diff --git a/bin/ncp/NETWORKING/SSH.sh b/bin/ncp/NETWORKING/SSH.sh index 62242e974..227ce47b8 100644 --- a/bin/ncp/NETWORKING/SSH.sh +++ b/bin/ncp/NETWORKING/SSH.sh @@ -39,7 +39,7 @@ configure() echo -e "$PASS\n$CONFIRM" | passwd "$USER" || return 1 # Reenable pi user - usermod "$USER" -s /bin/bash + chsh -s /bin/bash "$USER" # Check for insecure default pi password ( taken from old jessie method ) # TODO Due to Debian bug #1003151 with mkpasswd this feature is not working properly at the moment - https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1837456.html diff --git a/build/build-SD-rpi.sh b/build/build-SD-rpi.sh index d7cc0b709..b7a2d0f19 100755 --- a/build/build-SD-rpi.sh +++ b/build/build-SD-rpi.sh @@ -79,8 +79,6 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ # harden SSH further for Raspbian sed -i 's|^#PermitRootLogin .*|PermitRootLogin no|' /etc/ssh/sshd_config - # default user 'pi' for SSH - # cleanup source etc/library.sh && run_app_unsafe post-inst.sh rm /etc/resolv.conf diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 1f813becb..7d9351a08 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -144,6 +144,7 @@ COPY ncp-previewgenerator /tmp/ncp-build/ncp-previewgenerator/ COPY build/docker /tmp/ncp-build/build/docker/ COPY ncp.sh update.sh post-inst.sh /tmp/ncp-build/ COPY etc/ncp-config.d/nc-init.cfg /usr/local/etc/ncp-config.d/nc-init-copy.cfg +COPY etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/ncp-config.d/nc-datadir.cfg RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ set -e; \ @@ -160,13 +161,20 @@ touch /.docker-image; \ apt-get update; \ apt-get install --no-install-recommends -y wget ca-certificates; \ -# install nextcloudpi +# install nextcloudpiset_app_param nc-datadir.sh DATADIR '/data-ro/ncdata' \ source /usr/local/etc/library.sh; \ cd /tmp/ncp-build/; \ install_app ncp.sh; \ mv /usr/local/etc/ncp-config.d/nc-init-copy.cfg /usr/local/etc/ncp-config.d/nc-init.cfg; \ run_app_unsafe bin/ncp/CONFIG/nc-init.sh; \ +echo 'Moving data directory to a more sensible location' \ +mkdir -p /opt/ncdata \ +set_app_param nc-datadir.sh DATADIR '/data-ro/ncdata' \ +run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh \ +set_app_param nc-datadir.sh DATADIR '/data/ncdata' \ +# Remove nc-datadir.cfg as it's disabled in docker +rm /usr/local/etc/nc-datadir.cfg \ sed -i 's|data-ro|data|' /data-ro/nextcloud/config/config.php; \ # fix default paths diff --git a/etc/library.sh b/etc/library.sh index e43a8f456..80327d56a 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -246,6 +246,39 @@ find_app_param() jq -r ".params[$p_num].value" < "$cfg_file" } +set_app_param() +{ + local script="${1?}" + local param_id="${2?}" + local param_value="${3?}" + local ncp_app="$(basename "$script" .sh)" + local cfg_file="$CFGDIR/$ncp_app.cfg" + + grep -q '[\\&#;'"'"'`|*?~<>^"()[{}$&[:space:]]' <<< "${param_value}" && { echo "Invalid characters in field ${vars[$i]}"; return 1; } + + cfg="$(cat "$cfg_file")" + + local len="$(jq '.params | length' <<<"$cfg")" + local param_found=false + + for (( i = 0 ; i < len ; i++ )); do + # check for invalid characters + [[ "$(jq -r ".params[$i].id" <<<"$cfg")" == "$param_id" ]] && { + cfg="$(jq ".params[$i].value = \"${param_value}\"" <<<"$cfg")" + param_found=true + } + + done + + [[ "$param_found" == "true" ]] || { + echo "Did not find parameter '${param_id}' in configuration of app '$(basename "$script" .sh)'" + return 1 + } + + echo "$cfg" > "$cfg_file" + +} + # receives a script file, no security checks function run_app_unsafe() { diff --git a/etc/ncp-config.d/nc-datadir.cfg b/etc/ncp-config.d/nc-datadir.cfg index f387c3e3c..7ec1cd724 100644 --- a/etc/ncp-config.d/nc-datadir.cfg +++ b/etc/ncp-config.d/nc-datadir.cfg @@ -9,7 +9,7 @@ { "id": "DATADIR", "name": "Data directory", - "value": "/media/USBdrive/ncdata", + "value": "/opt/ncdata", "suggest": "/media/USBdrive/ncdata", "default": "/var/www/nextcloud/data", "type": "directory" diff --git a/install.sh b/install.sh index b7a459798..ace574556 100644 --- a/install.sh +++ b/install.sh @@ -67,6 +67,14 @@ rm /usr/local/etc/ncp-config.d/nc-nextcloud.cfg # armbian overlay is ro systemctl restart mysqld # TODO this shouldn't be necessary, but somehow it's needed in Debian 9.6. Fixme install_app ncp.sh run_app_unsafe bin/ncp/CONFIG/nc-init.sh +echo 'Moving data directory to a more sensible location' +mkdir -p /opt/ncdata +[[ -f "/usr/local/etc/nc-datadir.cfg" ]] || { + should_rm_datadir_cfg=true + cp etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/nc-datadir.cfg +} +run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh +[[ -z "$should_rm_datadir_cfg" ]] || rm /usr/local/etc/nc-datadir.cfg rm /.ncp-image # skip on Armbian / Vagrant / LXD ... From 814949343978ffb8083b69a453cb415dcbc5a718 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:07:46 +0100 Subject: [PATCH 08/32] SSH: Add option to enable sudo for ssh user. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Disable root login Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/NETWORKING/SSH.sh | 5 +++++ etc/ncp-config.d/SSH.cfg | 6 ++++++ ncp.sh | 1 + 3 files changed, 12 insertions(+) diff --git a/bin/ncp/NETWORKING/SSH.sh b/bin/ncp/NETWORKING/SSH.sh index 227ce47b8..5d3bbc352 100644 --- a/bin/ncp/NETWORKING/SSH.sh +++ b/bin/ncp/NETWORKING/SSH.sh @@ -41,6 +41,11 @@ configure() # Reenable pi user chsh -s /bin/bash "$USER" + [[ "$SUDO" == "yes" ]] && { + usermod -aG sudo "$USER" + echo "Enabled sudo for $USER" + } + # Check for insecure default pi password ( taken from old jessie method ) # TODO Due to Debian bug #1003151 with mkpasswd this feature is not working properly at the moment - https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1837456.html #local SHADOW SALT HASH diff --git a/etc/ncp-config.d/SSH.cfg b/etc/ncp-config.d/SSH.cfg index 031fb9ee3..388540265 100644 --- a/etc/ncp-config.d/SSH.cfg +++ b/etc/ncp-config.d/SSH.cfg @@ -29,6 +29,12 @@ "name": "Confirm", "value": "1234", "type": "password" + }, + { + "id": "SUDO", + "name": "Enable sudo", + "value": "no", + "type": "bool" } ] } diff --git a/ncp.sh b/ncp.sh index de9d2f7b8..9aab8c11c 100644 --- a/ncp.sh +++ b/ncp.sh @@ -132,6 +132,7 @@ EOF id -u "$WEBADMIN" &>/dev/null || useradd --home-dir /nonexistent "$WEBADMIN" echo -e "$WEBPASSWD\n$WEBPASSWD" | passwd "$WEBADMIN" chsh -s /usr/sbin/nologin "$WEBADMIN" + chsh -s /usr/sbin/nologin root ## NCP LAUNCHER mkdir -p /home/www From 2072f77720bc9b93949b517549fcb3c42067f89b Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:11:52 +0100 Subject: [PATCH 09/32] Installer and pipeline fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 157 +++++++++++++++--------- .github/workflows/build-lxd.yml | 167 ++++++++++++++++++++++---- .github/workflows/build-sd-images.yml | 6 +- .github/workflows/vm-tests.yml | 4 +- bin/ncp-report | 8 +- bin/ncp/BACKUPS/nc-restore.sh | 3 +- bin/ncp/CONFIG/nc-database.sh | 2 +- bin/ncp/CONFIG/nc-datadir.sh | 4 +- build/build-LXD.sh | 5 +- build/build-SD-armbian.sh | 3 +- install.sh | 9 +- 11 files changed, 271 insertions(+), 97 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 97e50abf6..3205c9a4d 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -30,14 +30,17 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.ref }}" ARCH: "${{ inputs.arch || 'x86' }}" + LOG_DCKR: "\\033[1;34mDCKR::\\033[0m" + LOG_NCP: "\\033[1;36m~NCP::\\033[0m" + LOG_CICD: "\\033[1;35mCICD::\\033[0m" + LOG_TEST: "\\033[1;33mTEST::\\033[0m" + LOG_DIAG: "\\033[1;31mDIAG::\\033[0m" steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Checkout code uses: actions/checkout@v3 @@ -66,7 +69,14 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.ref }}" ARCH: "${{ inputs.arch || 'x86' }}" + LOG_DCKR: "\\033[1;34mDCKR::\\033[0m" + LOG_NCP: "\\033[1;36m~NCP::\\033[0m" + LOG_CICD: "\\033[1;35mCICD::\\033[0m" + LOG_TEST: "\\033[1;33mTEST::\\033[0m" + LOG_DIAG: "\\033[1;31mDIAG::\\033[0m" steps: + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Checkout code uses: actions/checkout@v3 with: @@ -101,9 +111,12 @@ jobs: } echo "Previous version is '$version'" echo "PREVIOUS_VERSION=$version" >> "$GITHUB_ENV" - echo "::set-output name=previous_version::${version}" + echo "previous_version=${version}" >> $GITHUB_OUTPUT - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 + + - name: Enable qemu SUID + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --credential yes - name: Start ncp container run: | @@ -117,10 +130,10 @@ jobs: RUN echo 'Mutex posixsem' >> /etc/apache2/mods-available/ssl.conf EOF - docker build . -t ncp-testing -f Dockerfile + docker buildx build --platform "$platform" . -t ncp-testing -f Dockerfile popd rm -r "$tmpdir" - docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing + docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing localhost - name: Checkout code uses: actions/checkout@v3 @@ -137,10 +150,10 @@ jobs: - name: Activate container working-directory: ./tests run: | - docker logs nextcloudpi |& awk '{ print "CONTAINER::" $0 }' - echo 'CONTAINER:: ==========' - docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & - docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" + echo -e "${LOG_DCKR} ==========" + docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & + docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & cmd=(python activation_tests.py --no-gui localhost 8443 4443) [[ "${ARCH?}" == "x86" ]] || { sleep 30; cmd+=(--timeout 300); } @@ -148,10 +161,13 @@ jobs: for attempt in {1..3} do echo ":: Activation Tests (attempt $attempt/3) ::" - "${cmd[@]}" || { - tail -n 20 geckodriver.log >&2 || true - echo "CI:: Activation test (attempt $attempt/3) failed!" - docker exec nextcloudpi bash /usr/local/bin/ncp-diag + "${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + echo -e "${LOG_CICD} Activation test (attempt $attempt/3) failed!" + docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + cat /proc/sys/fs/binfmt_misc/qemu-aarch64 |& awk "{ print \"${LOG_DIAG} (qemu-aarch64) \" \$0 }" || true + cat /proc/sys/fs/binfmt_misc/qemu-arm |& awk "{ print \"${LOG_DIAG} (qemu-arm) \" \$0 }" || true sleep 12 continue } @@ -159,21 +175,24 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: Activation test failed in all attempts!" + echo -e "${LOG_CICD} Activation test failed in all attempts!" + echo "Creating error report..." + docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt + echo "done." exit 1 } - echo "CI:: Activation test successful" + echo -e "${LOG_CICD} Activation test successful" - name: Start new container run: | docker stop nextcloudpi - docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing + docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing localhost - name: Integration Tests working-directory: ./tests run: | - docker logs nextcloudpi |& awk '{ print "CONTAINER::" $0 }' - echo 'CONTAINER:: ==========' - docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & - docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" + echo -e "${LOG_DCKR} ==========" + docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & + docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & [[ "${ARCH?}" == "x86" ]] || sleep 30 sleep 20 @@ -182,8 +201,9 @@ jobs: for attempt in {1..3} do echo ":: System Tests (attempt $attempt/3) ::" - python system_tests.py --no-ping --non-interactive || { - echo "CI:: System test (attempt $attempt) failed!" + python system_tests.py --no-ping --non-interactive || awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + echo -e "${LOG_CICD} System test (attempt $attempt) failed!" sleep 12 continue } @@ -191,18 +211,19 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: System test failed in all attempts!" + echo -e "${LOG_CICD} System test failed in all attempts!" exit 1 } - echo "CI:: System test successful" + echo -e "${LOG_CICD} System test successful" success=false for attempt in {1..3} do echo ":: Nextcloud Tests (attempt $attempt/3) ::" - python nextcloud_tests.py --no-gui localhost 8443 4443 || { - tail -n 20 geckodriver.log >&2 || true - echo "CI:: Nextcloud test (attempt $attempt/3) failed!" + python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + echo -e "${LOG_CICD} Nextcloud test (attempt $attempt/3) failed!" sleep 12 continue } @@ -210,12 +231,21 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: Nextcloud test failed in all attempts!" + echo -e "${LOG_CICD} Nextcloud test failed in all attempts!" + echo -e "${LOG_CICD} Creating error report..." + docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt + echo -e "${LOG_CICD} done." exit 1 } - echo "CI:: Nextcloud test successful" + echo -e "${LOG_CICD} Nextcloud test successful" - docker exec nextcloudpi bash /usr/local/bin/ncp-diag + docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" + - name: "Upload error report" + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ github.run_id }}-docker-${{ env.ARCH }}-update-test-error-report + path: tests/error-report.txt test: needs: @@ -224,16 +254,21 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.ref }}" ARCH: "${{ inputs.arch || 'x86' }}" + LOG_DCKR: "\\033[1;34mDCKR::\\033[0m" + LOG_NCP: "\\033[1;36m~NCP::\\033[0m" + LOG_CICD: "\\033[1;35mCICD::\\033[0m" + LOG_TEST: "\\033[1;33mTEST::\\033[0m" + LOG_DIAG: "\\033[1;31mDIAG::\\033[0m" steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Login to docker run: | echo "${{ secrets.DOCKER_PASSWORD_INTERNAL }}" | docker login -u "${{ secrets.DOCKER_LOGIN_INTERNAL }}" --password-stdin - name: Start ncp container run: | - docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing + docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing localhost - name: Checkout code uses: actions/checkout@v3 @@ -250,10 +285,10 @@ jobs: - name: Integration Tests working-directory: ./tests run: | - docker logs nextcloudpi |& awk '{ print "CONTAINER::" $0 }' - echo 'CONTAINER:: ==========' - docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & - docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" + echo -e "${LOG_DCKR} ==========" + docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & + docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & cmd=(python activation_tests.py --no-gui localhost 8443 4443) [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) @@ -261,10 +296,11 @@ jobs: for attempt in {1..3} do echo ":: Activation Tests (attempt $attempt/3) ::" - "${cmd[@]}" || { - tail -n 20 geckodriver.log >&2 || true - echo "CI:: Activation test (attempt $attempt/3) failed!" - docker exec nextcloudpi bash /usr/local/bin/ncp-diag + "${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + echo -e "${LOG_CICD} Activation test (attempt $attempt/3) failed!" + docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true sleep 12 continue } @@ -272,17 +308,18 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: Activation test failed in all attempts!" + echo -e "${LOG_CICD} Activation test failed in all attempts!" exit 1 } - echo "CI:: Activation test successful" + echo -e "${LOG_CICD} Activation test successful" success=false for attempt in {1..3} do echo ":: System Tests (attempt $attempt/3) ::" - python system_tests.py --no-ping --non-interactive || { - echo "CI:: System test (attempt $attempt) failed!" + python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + echo -e "${LOG_CICD} System test (attempt $attempt) failed!" sleep 5 continue } @@ -290,18 +327,19 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: System test failed in all attempts!" + echo -e "${LOG_CICD} System test failed in all attempts!" exit 1 } - echo "CI:: System test successful" + echo -e "${LOG_CICD} System test successful" success=false for attempt in {1..3} do echo ":: Nextcloud Tests (attempt $attempt/3) ::" - python nextcloud_tests.py --no-gui localhost 8443 4443 || { - tail -n 20 geckodriver.log >&2 || true - echo "CI:: Nextcloud test (attempt $attempt/3) failed!" + python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + echo -e "${LOG_CICD} Nextcloud test (attempt $attempt/3) failed!" sleep 5 continue } @@ -309,10 +347,19 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: Nextcloud test failed in all attempts!" + echo -e "${LOG_CICD} Nextcloud test failed in all attempts!" + echo -e "${LOG_CICD} Creating error report..." + docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt + echo -e "${LOG_CICD} done." exit 1 } - echo "CI:: Nextcloud test successful" + echo -e "${LOG_CICD} Nextcloud test successful" - docker exec nextcloudpi bash /usr/local/bin/ncp-diag + docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + - name: "Upload error report" + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ github.run_id }}-docker-${{ env.ARCH }}-install-test-error-report + path: tests/error-report.txt diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 223a8ffbe..ef1ccab39 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -19,8 +19,8 @@ on: - "devel" jobs: - build: - runs-on: ubuntu-latest + build-current: + runs-on: ubuntu-20.04 outputs: artifact_name: "${{ env.ARTIFACT_NAME }}" artifact_file: "${{ steps.pack-lxd.outputs.artifact_file }}" @@ -35,6 +35,28 @@ jobs: - uses: whywaita/setup-lxd@v1 with: lxd_version: latest/stable + +# - name: Fix LXD +# run: | +# ip address +# ip route list +# lxc profile create network +# cat <> $GITHUB_OUTPUT - name: upload LXD image to artifact store uses: actions/upload-artifact@v3 with: @@ -53,7 +75,7 @@ jobs: if-no-files-found: error build-previous: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 outputs: artifact_name: "${{ env.ARTIFACT_NAME }}" artifact_file: "${{ steps.pack-lxd.outputs.artifact_file }}" @@ -61,6 +83,7 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" ARTIFACT_NAME: "${{ github.run_id }}-lxd-image-previous" + LXD_EXTRA_PROFILE: network steps: - name: Checkout code uses: actions/checkout@v3 @@ -93,10 +116,21 @@ jobs: echo "Previous version is '$version'" git checkout "$version" echo "VERSION=$version" >> "$GITHUB_ENV" - echo "::set-output name=previous_version::${version}" + echo "previous_version=${version}" >> $GITHUB_OUTPUT - uses: whywaita/setup-lxd@v1 with: lxd_version: latest/stable + - name: Fix LXD + run: | + lxc profile create network + cat <> $GITHUB_OUTPUT - name: upload LXD image to artifact store uses: actions/upload-artifact@v3 with: @@ -117,7 +151,7 @@ jobs: update-previous: needs: - build-previous - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 outputs: artifact_name: "${{ env.ARTIFACT_NAME }}" artifact_file: "${{ steps.pack-lxd.outputs.artifact_file }}" @@ -128,6 +162,17 @@ jobs: - uses: whywaita/setup-lxd@v1 with: lxd_version: latest/stable + - name: Fix LXD + run: | + lxc profile create network + cat <> $GITHUB_OUTPUT - name: upload LXD image to artifact store uses: actions/upload-artifact@v3 with: @@ -193,23 +238,89 @@ jobs: path: "output/${{ steps.pack-lxd.outputs.artifact_file }}" if-no-files-found: error - test: + test-fresh-install: + needs: + - build-current + runs-on: ubuntu-20.04 + env: + VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }} + ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" + - uses: whywaita/setup-lxd@v1 + with: + lxd_version: latest/stable + - name: Fix LXD + run: | + lxc profile create network + cat </dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + sleep 30 + ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="${ip/% *}" + echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts + - name: Test LXD Image + working-directory: ./tests + run: | + lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || { + echo "Activation test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + lxc exec ncp -- "tail -n20 /var/log/ncp.log" + exit 1 + } + python system_tests.py --non-interactive || { + echo "System test failed!" + exit 1 + } + python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || { + echo "Nextcloud test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + lxc exec ncp -- "tail -n20 /var/log/ncp.log" + exit 1 + } + lxc stop ncp + + test-update: needs: - - build - update-previous - strategy: - matrix: - build: - - source: install - artifact_name: ${{ needs.build.outputs.artifact_name }} - artifact_file: ${{ needs.build.outputs.artifact_file }} - - source: update - artifact_name: ${{ needs.update-previous.outputs.artifact_name }} - artifact_file: ${{ needs.update-previous.outputs.artifact_file }} - fail-fast: false - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + ARTIFACT_NAME: ${{ needs.update-previous.outputs.artifact_name }} + ARTIFACT_FILE: ${{ needs.update-previous.outputs.artifact_file }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -218,6 +329,16 @@ jobs: - uses: whywaita/setup-lxd@v1 with: lxd_version: latest/stable + - name: Fix LXD + run: | + lxc profile create network + cat </dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' sleep 30 diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index 08af6d47c..168dc4e67 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -68,7 +68,7 @@ jobs: artifacts=("armbian/output/images/Armbian"*.img) mkdir -p output mv "${artifacts[0]}" "output/$IMG" - echo "::set-output name=artifact_file::${IMG}" + echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: "Build Armbian (2nd attempt)" if: ${{ inputs.board_id != 'raspberrypi' && steps.build-armbian.outcome == 'failure' }} @@ -87,7 +87,7 @@ jobs: artifacts=("armbian/output/images/Armbian"*.img) mkdir -p output mv "${artifacts[0]}" "output/$IMG" - echo "::set-output name=artifact_file::${IMG}" + echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: "Upload Armbian logs" if: ${{ inputs.board_id != 'raspberrypi' && failure() }} @@ -116,7 +116,7 @@ jobs: echo "Retrying ($i out of 10)" done - echo "::set-output name=artifact_file::${IMG}" + echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: upload image to artifact store uses: actions/upload-artifact@v3 diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml index f2ff3cd39..743ae5b4d 100644 --- a/.github/workflows/vm-tests.yml +++ b/.github/workflows/vm-tests.yml @@ -81,7 +81,7 @@ jobs: version="${version%-*-*}" fi echo "Previous version is '$version'" - echo "::set-output name=previous_version::${version}" + echo "previous_version=${version}" >> $GITHUB_OUTPUT - run: | set -x mkdir -p ./.ssh @@ -387,7 +387,7 @@ jobs: steps: - name: Delete old snapshots run: | - for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -20) + for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -16) do echo "Deleting snapshot '$snapshot'..." hcloud image delete "$snapshot" diff --git a/bin/ncp-report b/bin/ncp-report index e598b2439..7ca5af9c1 100755 --- a/bin/ncp-report +++ b/bin/ncp-report @@ -45,19 +45,19 @@ close_summary ## open_summary "Nextcloud configuration" -sudo -u www-data php /var/www/nextcloud/occ config:list system +sudo -u www-data php /var/www/nextcloud/occ config:list system 2>&1 || true close_summary ## open_summary "HTTPd logs" -tail -20 /var/log/apache2/error.log +tail -20 /var/log/apache2/error.log 2>&1 || true close_summary ## open_summary "Database logs" -tail -20 /var/log/mysql/*.log +tail -20 /var/log/mysql/*.log 2>&1 || true close_summary ## @@ -66,7 +66,7 @@ DATADIR="$( grep datadirectory /var/www/nextcloud/config/config.php | awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1 )" open_summary "Nextcloud logs" -tail -20 "$DATADIR"/nextcloud.log +tail -20 "$DATADIR"/nextcloud.log 2>&1 || true close_summary # License diff --git a/bin/ncp/BACKUPS/nc-restore.sh b/bin/ncp/BACKUPS/nc-restore.sh index 2dcd39a3a..97d88e6b0 100644 --- a/bin/ncp/BACKUPS/nc-restore.sh +++ b/bin/ncp/BACKUPS/nc-restore.sh @@ -31,7 +31,7 @@ DIR="$( cd "$( dirname "$BACKUPFILE" )" &>/dev/null && pwd )" #abspath [[ "$DIR" =~ "$NCDIR" ]] && { echo "Refusing to restore from $NCDIR"; exit 1; } TMPDIR="$( mktemp -d "$( dirname "$BACKUPFILE" )"/ncp-restore.XXXXXX )" || { echo "Failed to create temp dir" >&2; exit 1; } -grep -q -e ext -e btrfs -e zfs <( stat -fc%T "$TMPDIR" ) || { echo "Can only restore from ext/btrfs/zfs filesystems" >&2; exit 1; } +grep -q -e ext -e btrfs -e zfs <( stat -fc%T "$TMPDIR" ) || { echo "Can only restore from ext/btrfs/zfs filesystems (found '$(stat -fc%T "${TMPDIR}")" >&2; exit 1; } TMPDIR="$( cd "$TMPDIR" &>/dev/null && pwd )" || { echo "$TMPDIR not found"; exit 1; } #abspath cleanup(){ local RET=$?; echo "Cleanup..."; rm -rf "${TMPDIR}"; trap "" EXIT; exit $RET; } @@ -77,6 +77,7 @@ then [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" mv -T "$DATA_BKP_DIR" "$target" else + echo "Not overwriting data directory" fi if [[ -n "$DATA_BKP_DIR" ]] diff --git a/bin/ncp/CONFIG/nc-database.sh b/bin/ncp/CONFIG/nc-database.sh index 54b10d358..013cdf44f 100644 --- a/bin/ncp/CONFIG/nc-database.sh +++ b/bin/ncp/CONFIG/nc-database.sh @@ -37,7 +37,7 @@ configure() local BASEDIR=$( dirname "$DBDIR" ) mkdir -p "$BASEDIR" - grep -q -e ext -e btrfs <( stat -fc%T "$BASEDIR" ) || { echo -e "Only ext/btrfs filesystems can hold the data directory"; return 1; } + grep -q -e ext -e btrfs <( stat -fc%T "$BASEDIR" ) || { echo -e "Only ext/btrfs filesystems can hold the data directory (found '$(stat -fc%T "${BASEDIR}")"; return 1; } sudo -u mysql test -x "$BASEDIR" || { echo -e "ERROR: the user mysql does not have access permissions over $BASEDIR"; return 1; } diff --git a/bin/ncp/CONFIG/nc-datadir.sh b/bin/ncp/CONFIG/nc-datadir.sh index 8d55a98f9..df7869b11 100644 --- a/bin/ncp/CONFIG/nc-datadir.sh +++ b/bin/ncp/CONFIG/nc-datadir.sh @@ -79,8 +79,8 @@ configure() ENCDIR="${BASEDIR}"/ncdata_enc # checks - grep -q -e ext -e btrfs <( stat -fc%T "${BASEDIR}" ) || { - echo -e "Only ext/btrfs filesystems can hold the data directory" + [[ "$DISABLE_FS_CHECK" == 1 ]] || grep -q -e ext -e btrfs <( stat -fc%T "${BASEDIR}" ) || { + echo -e "Only ext/btrfs filesystems can hold the data directory (found '$(stat -fc%T "${BASEDIR}")')" return 1 } diff --git a/build/build-LXD.sh b/build/build-LXD.sh index c695e769e..e34402e10 100755 --- a/build/build-LXD.sh +++ b/build/build-LXD.sh @@ -32,7 +32,10 @@ prepare_dirs # tmp cache output ## BUILD NCP lxc delete -f ncp 2>/dev/null || true -systemd-run --user --scope -p "Delegate=yes" lxc launch -q images:debian/bullseye ncp +LXC_LAUNCH=(lxc launch -p default) +[[ -n "$LXD_EXTRA_PROFILE" ]] && LXC_LAUNCH+=(-p "$LXD_EXTRA_PROFILE") +LXC_LAUNCH+=(-q 'images:debian/bullseye' ncp) +systemd-run --user --scope -p "Delegate=yes" "${LXC_LAUNCH[@]}" lxc config device add ncp buildcode disk source="$(pwd)" path=/build lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' lxc exec ncp -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh' diff --git a/build/build-SD-armbian.sh b/build/build-SD-armbian.sh index fddc30c5a..1a955c56e 100755 --- a/build/build-SD-armbian.sh +++ b/build/build-SD-armbian.sh @@ -69,7 +69,8 @@ EXTRA_CONF=build/armbian/"config-$BOARD".conf rm -rf armbian/output/images mkdir -p armbian/userpatches sed -e '/docker.*run/s/-it//' armbian/config/templates/config-docker.conf > armbian/userpatches/config-docker.conf -docker pull "ghcr.io/armbian/build:$(cut -d"." -f1-2 < armbian/VERSION)-$(dpkg --print-architecture)" +docker pull "ghcr.io/armbian/build:$(cut -d"." -f1-2 < armbian/VERSION)-$(dpkg --print-architecture)" \ + || docker pull "ghcr.io/armbian/build:latest-$(dpkg --print-architecture)" armbian/compile.sh docker ncp rm "$CONF" diff --git a/install.sh b/install.sh index ace574556..31044e2bb 100644 --- a/install.sh +++ b/install.sh @@ -68,13 +68,14 @@ systemctl restart mysqld # TODO this shouldn't be necessary, but somehow it's ne install_app ncp.sh run_app_unsafe bin/ncp/CONFIG/nc-init.sh echo 'Moving data directory to a more sensible location' +df -h mkdir -p /opt/ncdata -[[ -f "/usr/local/etc/nc-datadir.cfg" ]] || { +[[ -f "/usr/local/etc/ncp-config.d/nc-datadir.cfg" ]] || { should_rm_datadir_cfg=true - cp etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/nc-datadir.cfg + cp etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/ncp-config.d/nc-datadir.cfg } -run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh -[[ -z "$should_rm_datadir_cfg" ]] || rm /usr/local/etc/nc-datadir.cfg +DISABLE_FS_CHECK=1 NCPCFG="/usr/local/etc/ncp.cfg" run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh +[[ -z "$should_rm_datadir_cfg" ]] || rm /usr/local/etc/ncp-config.d/nc-datadir.cfg rm /.ncp-image # skip on Armbian / Vagrant / LXD ... From cde1b440637f0af6a83e5bf58e3d6568708654dd Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 4 Jan 2023 14:48:14 +0100 Subject: [PATCH 10/32] Move ncp-backup and ncp-restore to separate files (instead relying on cat during installation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-backup | 104 ++++++++++++++++++ bin/ncp-restore | 184 +++++++++++++++++++++++++++++++ bin/ncp/BACKUPS/nc-backup.sh | 110 +------------------ bin/ncp/BACKUPS/nc-restore.sh | 198 +--------------------------------- 4 files changed, 292 insertions(+), 304 deletions(-) create mode 100755 bin/ncp-backup create mode 100755 bin/ncp-restore diff --git a/bin/ncp-backup b/bin/ncp-backup new file mode 100755 index 000000000..f7e22a431 --- /dev/null +++ b/bin/ncp-backup @@ -0,0 +1,104 @@ +#!/bin/bash +set -eE +source /usr/local/etc/library.sh + +if [ "$1" = "--help" ]; then + echo "$0 " + echo " and are 'yes' or 'no'" + echo " integer >= 1 or 0 for no limit" + exit 0 +fi + +destdir="${1:-/media/USBdrive/ncp-backups}" +includedata="${2:-no}" +compress="${3:-no}" +backuplimit="${4:-0}" + +destfile="$destdir"/nextcloud-bkp_$( date +"%Y%m%d_%s" ).tar +dbbackup=nextcloud-sqlbkp_$( date +"%Y%m%d" ).bak +[[ -f /.docker-image ]] && basedir=/data || basedir=/var/www + +[[ "$compress" == "yes" ]] && destfile="$destfile".gz + +datadir=$( get_nc_config_value datadirectory ) || { + echo "Error reading data directory. Is NextCloud running and configured?"; + exit 1; +} + +cleanup(){ local ret=$?; rm -f "${dbbackup}" ; restore_maintenance_mode; exit $ret; } +fail() { local ret=$?; echo "Abort..." ; rm -f "${dbbackup}" "${destfile}"; restore_maintenance_mode; exit $ret; } +trap cleanup EXIT +trap fail INT TERM HUP ERR + +echo "check free space..." # allow at least ~500 extra MiB +mkdir -p "$destdir" +nsize=$(du -sb "$basedir/nextcloud" | awk '{ print $1 }') +[[ -d "$basedir/nextcloud"/data ]] && \ + dsize=$(du -sb "$basedir/nextcloud"/data | awk '{ print $1 }') +margin=$((500*1024*1024)) # safety margin for database and some extra +if [[ "$includedata" == "yes" ]]; then + datasize=$(du -sb "$datadir" | awk '{ print $1 }') + size=$((nsize - dsize + margin + datasize)) +else + size=$((nsize - dsize + margin)) +fi +free=$( df -B1 "$destdir" | tail -1 | awk '{ print $4 }' ) + +[ $size -ge $free ] && { + echo "free space check failed. Need $size Bytes"; + exit 1; +} + +# delete older backups +[[ $backuplimit != 0 ]] && { + numbkps=$( ls "$destdir"/nextcloud-bkp_* 2>/dev/null | wc -l ) + [[ $numbkps -ge $backuplimit ]] && \ + ls -t $destdir/nextcloud-bkp_* | tail -$(( numbkps - backuplimit + 1 )) | while read -r f; do + echo "clean up old backup $f" + rm "$f" + done +} + +# database +save_maintenance_mode +cd "$basedir" || exit 1 +echo "backup database..." +mysqldump -u root --single-transaction nextcloud > "$dbbackup" + +# files +echo "backup files..." +[[ "$includedata" == "yes" ]] && data="$(basename "$datadir")" +[[ "$compress" == "yes" ]] && compress_arg="-I pigz" +mkdir -p "$destdir" +tar $compress_arg -cf "$destfile" \ +\ + "$dbbackup" \ +\ + --exclude "$data/.opcache" \ + --exclude "$data/{access,error,nextcloud}.log" \ + --exclude "$data/access.log" \ + --exclude "$data/ncp-update-backups" \ + -C "$(dirname "$datadir"/)" $data \ +\ + --exclude "nextcloud/data/*/files/*" \ + --exclude "nextcloud/data/.opcache" \ + --exclude "nextcloud/data/{access,error,nextcloud}.log" \ + --exclude "nextcloud/data/access.log" \ + --exclude "nextcloud/data/appdata_*/preview/*" \ + --exclude "nextcloud/data/*/uploads/*" \ + --exclude "nextcloud/data/*/cache/*" \ + --exclude "nextcloud/data/*/files_trashbin/*" \ + --exclude "nextcloud/data/*/files_versions/*" \ + --exclude "nextcloud/data/ncp-update-backups" \ + --exclude "nextcloud/data/__groupfolders" \ + --exclude "nextcloud/.data_*" \ + -C $basedir nextcloud/ \ + || { + echo "error generating backup" + exit 1 + } +rm "$dbbackup" +chmod 640 "$destfile" +chown :www-data "$destfile" + +echo "backup $destfile generated" \ No newline at end of file diff --git a/bin/ncp-restore b/bin/ncp-restore new file mode 100755 index 000000000..9b63df43d --- /dev/null +++ b/bin/ncp-restore @@ -0,0 +1,184 @@ +#!/bin/bash +set -eE + +BACKUPFILE="$1" + +DBADMIN=ncadmin +DBPASSWD="$( grep password /root/.my.cnf | sed 's|password=||' )" + +source /usr/local/etc/library.sh # sets PHPVER + +DIR="$( cd "$( dirname "$BACKUPFILE" )" &>/dev/null && pwd )" #abspath + +[[ -f /.docker-image ]] && NCDIR=/data/nextcloud || NCDIR=/var/www/nextcloud + +[[ $# -eq 0 ]] && { echo "missing first argument" ; exit 1; } +[[ -f "$BACKUPFILE" ]] || { echo "$BACKUPFILE not found" ; exit 1; } +[[ "$DIR" =~ "$NCDIR".* ]] && { echo "Refusing to restore from $NCDIR"; exit 1; } + +TMPDIR="$( mktemp -d "$( dirname "$BACKUPFILE" )"/ncp-restore.XXXXXX )" || { echo "Failed to create temp dir" >&2; exit 1; } +grep -q -e ext -e btrfs -e zfs <( stat -fc%T "$TMPDIR" ) || { echo "Can only restore from ext/btrfs/zfs filesystems (found '$(stat -fc%T "${TMPDIR}")" >&2; exit 1; } + +TMPDIR="$( cd "$TMPDIR" &>/dev/null && pwd )" || { echo "$TMPDIR not found"; exit 1; } #abspath +cleanup(){ local RET=$?; echo "Cleanup..."; rm -rf "${TMPDIR}"; trap "" EXIT; exit $RET; } +trap cleanup INT TERM HUP ERR EXIT +rm -rf "$TMPDIR" && mkdir -p "$TMPDIR" + +[[ "$BACKUPFILE" =~ .*".tar.gz" ]] && compress_arg="-I pigz" + +# CHECK FREE SPACE IN $TMPDIR +echo "check free space..." # allow at least ~100 extra MiB +extractedsize=$(tar $compress_arg -tvf "$BACKUPFILE" | awk '{s+=$3} END{printf "%.0f", (s/1024)}') # Size of extracted files in "KB" +size=$(($extractedsize + 100*1024)) +free=$( df "$TMPDIR" | tail -1 | awk '{ print $4 }' ) +[ $size -ge $free ] && { + echo "free space check failed. Need $size KB in $TMPDIR"; + exit 1; +} + +# EXTRACT FILES +echo "extracting backup file $BACKUPFILE..." +tar $compress_arg -xf "$BACKUPFILE" -C "$TMPDIR" || exit 1 + +## SANITY CHECKS +[[ -d "$TMPDIR"/nextcloud ]] && [[ -f "$( ls "$TMPDIR"/nextcloud-sqlbkp_*.bak 2>/dev/null )" ]] || { + echo "invalid backup file. Abort" + exit 1 +} + +## RESTORE FILES + +echo "restore files..." +[[ -d "$NCDIR/data" ]] && { + DATA_BKP_DIR="$(mktemp -d -p "$NCDIR/.." nc-data-XXXXXX)" + mv -T "$NCDIR/data" "$DATA_BKP_DIR/" +} +rm -rf "$NCDIR" +mv -T "$TMPDIR"/nextcloud "$NCDIR" || { echo "Error restoring base files"; exit 1; } + +if [[ -n "$DATA_BKP_DIR" ]] +then + echo "Restoring $NCDIR/data..." + target="$NCDIR/data" + [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" + mv -T "$DATA_BKP_DIR" "$target" +else + echo "Dataless backup detected - not restoring data" +fi + +# update NC database password to this instance +sed -i "s|'dbpassword' =>.*|'dbpassword' => '$DBPASSWD',|" /var/www/nextcloud/config/config.php + +# update redis credentials +REDISPASS="$( grep "^requirepass" /etc/redis/redis.conf | cut -f2 -d' ' )" +[[ "$REDISPASS" != "" ]] && \ + sed -i "s|'password'.*|'password' => '$REDISPASS',|" /var/www/nextcloud/config/config.php +service redis-server restart + +## RE-CREATE DATABASE TABLE + +echo "restore database..." +mysql -u root </dev/null && { + rmdir "$DATADIR" || exit 1 + btrfs subvolume create "$DATADIR" || exit 1 + } + chown www-data: "$DATADIR" + TMPDATA="$TMPDIR/$( basename "$DATADIR" )" + mv "$TMPDATA"/* "$TMPDATA"/.[!.]* "$DATADIR" || exit 1 + rmdir "$TMPDATA" || exit 1 + + ncc maintenance:mode --off + +### INCLUDEDATA=no situation + +else + echo "No datadir found in backup" + + [[ -e "$DATADIR" ]] || { + echo "${DATADIR} not found. Resetting to ${NCDIR}/data" + DATADIR="$NCDIR"/data + mkdir -p "${DATADIR}" + touch "${DATADIR}"/.ocdata + chown -R www-data: "${DATADIR}" + } + + ncc maintenance:mode --off + ncc files:scan --all + + # cache needs to be cleaned as of NC 12 + NEED_RESTART=1 +fi + +sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR}"/config/config.php + +# Just in case we moved the opcache dir +install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" + +# tmp upload dir +mkdir -p "$DATADIR/tmp" +chown www-data: "$DATADIR/tmp" +ncc config:system:set tempdirectory --value "$DATADIR/tmp" +sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/cli/php.ini +sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini +sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini + +# logs +ncc config:system:set logfile --value="$DATADIR/nextcloud.log" + +# update fail2ban logpath +[[ -f /etc/fail2ban/jail.conf ]] && { + sed -i "s|logpath =.*|logpath = $DATADIR/nextcloud.log|" /etc/fail2ban/jail.conf + pgrep fail2ban &>/dev/null && service fail2ban restart +} + +# refresh nextcloud trusted domains +bash /usr/local/bin/nextcloud-domain.sh + +# update the systems data-fingerprint +ncc maintenance:data-fingerprint + +# refresh thumbnails +ncc files:scan-app-data + +# restart PHP if needed +[[ "$NEED_RESTART" == "1" ]] && { + clear_opcache + bash -c "sleep 3; service php${PHPVER}-fpm restart" &>/dev/null & +} \ No newline at end of file diff --git a/bin/ncp/BACKUPS/nc-backup.sh b/bin/ncp/BACKUPS/nc-backup.sh index b25c3318d..1b08ba745 100644 --- a/bin/ncp/BACKUPS/nc-backup.sh +++ b/bin/ncp/BACKUPS/nc-backup.sh @@ -19,114 +19,8 @@ install() apt-get update apt-get install -y --no-install-recommends pigz - cat > /usr/local/bin/ncp-backup <<'EOF' -#!/bin/bash -set -eE -source /usr/local/etc/library.sh - -if [ "$1" = "--help" ]; then - echo "$0 " - echo " and are 'yes' or 'no'" - echo " integer >= 1 or 0 for no limit" - exit 0 -fi - -destdir="${1:-/media/USBdrive/ncp-backups}" -includedata="${2:-no}" -compress="${3:-no}" -backuplimit="${4:-0}" - -destfile="$destdir"/nextcloud-bkp_$( date +"%Y%m%d_%s" ).tar -dbbackup=nextcloud-sqlbkp_$( date +"%Y%m%d" ).bak -occ="sudo -u www-data php /var/www/nextcloud/occ" -[[ -f /.docker-image ]] && basedir=/data || basedir=/var/www - -[[ "$compress" == "yes" ]] && destfile="$destfile".gz - -datadir=$( get_nc_config_value datadirectory ) || { - echo "Error reading data directory. Is NextCloud running and configured?"; - exit 1; -} - -cleanup(){ local ret=$?; rm -f "${dbbackup}" ; restore_maintenance_mode; exit $ret; } -fail() { local ret=$?; echo "Abort..." ; rm -f "${dbbackup}" "${destfile}"; restore_maintenance_mode; exit $ret; } -trap cleanup EXIT -trap fail INT TERM HUP ERR - -echo "check free space..." # allow at least ~500 extra MiB -mkdir -p "$destdir" -nsize=$(du -sb "$basedir/nextcloud" | awk '{ print $1 }') -[[ -d "$basedir/nextcloud"/data ]] && \ - dsize=$(du -sb "$basedir/nextcloud"/data | awk '{ print $1 }') -margin=$((500*1024*1024)) # safety margin for database and some extra -if [[ "$includedata" == "yes" ]]; then - datasize=$(du -sb "$datadir" | awk '{ print $1 }') - size=$((nsize - dsize + margin + datasize)) -else - size=$((nsize - dsize + margin)) -fi -free=$( df -B1 "$destdir" | tail -1 | awk '{ print $4 }' ) - -[ $size -ge $free ] && { - echo "free space check failed. Need $size Bytes"; - exit 1; -} - -# delete older backups -[[ $backuplimit != 0 ]] && { - numbkps=$( ls "$destdir"/nextcloud-bkp_* 2>/dev/null | wc -l ) - [[ $numbkps -ge $backuplimit ]] && \ - ls -t $destdir/nextcloud-bkp_* | tail -$(( numbkps - backuplimit + 1 )) | while read -r f; do - echo "clean up old backup $f" - rm "$f" - done -} - -# database -save_maintenance_mode -cd "$basedir" || exit 1 -echo "backup database..." -mysqldump -u root --single-transaction nextcloud > "$dbbackup" - -# files -echo "backup files..." -[[ "$includedata" == "yes" ]] && data="$(basename "$datadir")" -[[ "$compress" == "yes" ]] && compress_arg="-I pigz" -mkdir -p "$destdir" -tar $compress_arg -cf "$destfile" \ -\ - "$dbbackup" \ -\ - --exclude "$data/.opcache" \ - --exclude "$data/{access,error,nextcloud}.log" \ - --exclude "$data/access.log" \ - --exclude "$data/ncp-update-backups" \ - -C "$(dirname "$datadir"/)" $data \ -\ - --exclude "nextcloud/data/*/files/*" \ - --exclude "nextcloud/data/.opcache" \ - --exclude "nextcloud/data/{access,error,nextcloud}.log" \ - --exclude "nextcloud/data/access.log" \ - --exclude "nextcloud/data/appdata_*/preview/*" \ - --exclude "nextcloud/data/*/uploads/*" \ - --exclude "nextcloud/data/*/cache/*" \ - --exclude "nextcloud/data/*/files_trashbin/*" \ - --exclude "nextcloud/data/*/files_versions/*" \ - --exclude "nextcloud/data/ncp-update-backups" \ - --exclude "nextcloud/data/__groupfolders" \ - --exclude "nextcloud/.data_*" \ - -C $basedir nextcloud/ \ - || { - echo "error generating backup" - exit 1 - } -rm "$dbbackup" -chmod 640 "$destfile" -chown :www-data "$destfile" - -echo "backup $destfile generated" -EOF - chmod +x /usr/local/bin/ncp-backup + ln -s /usr/local/bin/ncp-backup /usr/local/bin/ncp/ncp-backup + chmod +x /usr/local/bin/ncp/ncp-backup } configure() diff --git a/bin/ncp/BACKUPS/nc-restore.sh b/bin/ncp/BACKUPS/nc-restore.sh index 97d88e6b0..322fab9d6 100644 --- a/bin/ncp/BACKUPS/nc-restore.sh +++ b/bin/ncp/BACKUPS/nc-restore.sh @@ -11,202 +11,8 @@ install() { - cat > /usr/local/bin/ncp-restore <<'EOF' -#!/bin/bash -set -eE - -BACKUPFILE="$1" - -DBADMIN=ncadmin -DBPASSWD="$( grep password /root/.my.cnf | sed 's|password=||' )" - -source /usr/local/etc/library.sh # sets PHPVER - -DIR="$( cd "$( dirname "$BACKUPFILE" )" &>/dev/null && pwd )" #abspath - -[[ -f /.docker-image ]] && NCDIR=/data/nextcloud || NCDIR=/var/www/nextcloud - -[[ $# -eq 0 ]] && { echo "missing first argument" ; exit 1; } -[[ -f "$BACKUPFILE" ]] || { echo "$BACKUPFILE not found" ; exit 1; } -[[ "$DIR" =~ "$NCDIR" ]] && { echo "Refusing to restore from $NCDIR"; exit 1; } - -TMPDIR="$( mktemp -d "$( dirname "$BACKUPFILE" )"/ncp-restore.XXXXXX )" || { echo "Failed to create temp dir" >&2; exit 1; } -grep -q -e ext -e btrfs -e zfs <( stat -fc%T "$TMPDIR" ) || { echo "Can only restore from ext/btrfs/zfs filesystems (found '$(stat -fc%T "${TMPDIR}")" >&2; exit 1; } - -TMPDIR="$( cd "$TMPDIR" &>/dev/null && pwd )" || { echo "$TMPDIR not found"; exit 1; } #abspath -cleanup(){ local RET=$?; echo "Cleanup..."; rm -rf "${TMPDIR}"; trap "" EXIT; exit $RET; } -trap cleanup INT TERM HUP ERR EXIT -rm -rf "$TMPDIR" && mkdir -p "$TMPDIR" - -[[ "$BACKUPFILE" =~ ".tar.gz" ]] && compress_arg="-I pigz" - -# CHECK FREE SPACE IN $TMPDIR -echo "check free space..." # allow at least ~100 extra MiB -extractedsize=$(tar $compress_arg -tvf "$BACKUPFILE" | awk '{s+=$3} END{printf "%.0f", (s/1024)}') # Size of extracted files in "KB" -size=$(($extractedsize + 100*1024)) -free=$( df "$TMPDIR" | tail -1 | awk '{ print $4 }' ) -[ $size -ge $free ] && { - echo "free space check failed. Need $size KB in $TMPDIR"; - exit 1; -} - -# EXTRACT FILES -echo "extracting backup file $BACKUPFILE..." -tar $compress_arg -xf "$BACKUPFILE" -C "$TMPDIR" || exit 1 - -## SANITY CHECKS -[[ -d "$TMPDIR"/nextcloud ]] && [[ -f "$( ls "$TMPDIR"/nextcloud-sqlbkp_*.bak 2>/dev/null )" ]] || { - echo "invalid backup file. Abort" - exit 1 -} - -## RESTORE FILES - -echo "restore files..." -[[ -d "$NCDIR/data" ]] && { - DATA_BKP_DIR="$(mktemp -d -p "$NCDIR/.." nc-data-XXXXXX)" - mv -T "$NCDIR/data" "$DATA_BKP_DIR/" -} -rm -rf "$NCDIR" -mv -T "$TMPDIR"/nextcloud "$NCDIR" || { echo "Error restoring base files"; exit 1; } - -if [[ -n "$DATA_BKP_DIR" ]] -then - echo "Restoring $NCDIR/data..." - target="$NCDIR/data" - [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" - mv -T "$DATA_BKP_DIR" "$target" -else - echo "Not overwriting data directory" -fi - -if [[ -n "$DATA_BKP_DIR" ]] -then - echo "Restoring $NCDIR/data..." - target="$NCDIR/data" - [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" - mv -T "$DATA_BKP_DIR" "$target" -else -fi - -# update NC database password to this instance -sed -i "s|'dbpassword' =>.*|'dbpassword' => '$DBPASSWD',|" /var/www/nextcloud/config/config.php - -# update redis credentials -REDISPASS="$( grep "^requirepass" /etc/redis/redis.conf | cut -f2 -d' ' )" -[[ "$REDISPASS" != "" ]] && \ - sed -i "s|'password'.*|'password' => '$REDISPASS',|" /var/www/nextcloud/config/config.php -service redis-server restart - -## RE-CREATE DATABASE TABLE - -echo "restore database..." -mysql -u root </dev/null && { - rmdir "$DATADIR" || exit 1 - btrfs subvolume create "$DATADIR" || exit 1 - } - chown www-data: "$DATADIR" - TMPDATA="$TMPDIR/$( basename "$DATADIR" )" - mv "$TMPDATA"/* "$TMPDATA"/.[!.]* "$DATADIR" || exit 1 - rmdir "$TMPDATA" || exit 1 - - ncc maintenance:mode --off - -### INCLUDEDATA=no situation - -else - echo "No datadir found in backup" - - [[ -e "$DATADIR" ]] || { - echo "${DATADIR} not found. Resetting to ${NCDIR}/data" - DATADIR="$NCDIR"/data - mkdir -p "${DATADIR}" - touch "${DATADIR}"/.ocdata - chown -R www-data: "${DATADIR}" - } - - ncc maintenance:mode --off - ncc files:scan --all - - # cache needs to be cleaned as of NC 12 - NEED_RESTART=1 -fi - -sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR}"/config/config.php - -# Just in case we moved the opcache dir -install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" - -# tmp upload dir -mkdir -p "$DATADIR/tmp" -chown www-data: "$DATADIR/tmp" -ncc config:system:set tempdirectory --value "$DATADIR/tmp" -sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/cli/php.ini -sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini -sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini - -# logs -ncc config:system:set logfile --value="$DATADIR/nextcloud.log" - -# update fail2ban logpath -[[ -f /etc/fail2ban/jail.conf ]] && { - sed -i "s|logpath =.*|logpath = $DATADIR/nextcloud.log|" /etc/fail2ban/jail.conf - pgrep fail2ban &>/dev/null && service fail2ban restart -} - -# refresh nextcloud trusted domains -bash /usr/local/bin/nextcloud-domain.sh - -# update the systems data-fingerprint -ncc maintenance:data-fingerprint - -# refresh thumbnails -ncc files:scan-app-data - -# restart PHP if needed -[[ "$NEED_RESTART" == "1" ]] && { - clear_opcache - bash -c "sleep 3; service php${PHPVER}-fpm restart" &>/dev/null & -} -EOF - chmod +x /usr/local/bin/ncp-restore + ln -s /usr/local/bin/ncp-restore /usr/local/bin/ncp/ncp-restore + chmod +x /usr/local/bin/ncp/ncp-restore } configure() From b69ad715076c35e2dfc6d50b0a967afe1beac0e2 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:12:59 +0100 Subject: [PATCH 11/32] Add basic integration test for nc-backup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- tests/NcpRobotLib.py | 178 +++++++++++++++++++++++++++++++++++++ tests/nc_backup_test.robot | 23 +++++ tests/requirements.txt | 2 + 3 files changed, 203 insertions(+) create mode 100644 tests/NcpRobotLib.py create mode 100644 tests/nc_backup_test.robot create mode 100644 tests/requirements.txt diff --git a/tests/NcpRobotLib.py b/tests/NcpRobotLib.py new file mode 100644 index 000000000..cc44939ba --- /dev/null +++ b/tests/NcpRobotLib.py @@ -0,0 +1,178 @@ +import json +import os +import re +import subprocess +from subprocess import PIPE +import random +import string +import time +from typing import Dict +from robot.api import FatalError, Error, logger +import tempfile + + +class NcpRobotLib: + ROBOT_LIBRARY_SCOPE = "SUITE" + + def __init__(self, http_port=80, https_port=443, webui_port=4443): + self._http_port = http_port + self._https_port = https_port + self._webui_port = webui_port + self._docker: Dict[str, any] = { + 'container': None + } + + def setup_ncp(self, instance_type, version): + if instance_type == 'docker': + container_name = 'nextcloudpi-' + ''.join(random.choice(string.digits) for i in range(6)) + image = version if ':' in version else f"ownyourbits/nextcloudpi:{version}" + d_run = subprocess.run([ + 'docker', 'run', '-d', + '-p', f'127.0.0.1:{self._http_port}:80', + '-p', f'127.0.0.1:{self._https_port}:443', + '-p', f'127.0.0.1:{self._webui_port}:4443', + '--name', container_name, + image, 'localhost' + ], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if d_run.returncode != 0: + raise FatalError(f"Failed to start test instance: \n'{d_run.stderr}'") + + self._docker['container'] = container_name + logger.info("Waiting for container startup") + time.sleep(20) + self.ncp_is_running() + + else: + raise FatalError(f"Invalid instance type '{instance_type}'") + + def ncp_is_running(self): + + d_run = subprocess.run(['docker', 'container', 'ls', '--filter', f'name={self._docker["container"]}'], + stdout=PIPE, stderr=PIPE) + re_string = r'.* Up.* ' + self._docker['container'] + r'.*' + docker_success_re = re.compile(re_string, flags=re.DOTALL) + + if not docker_success_re.match(d_run.stdout.decode('utf-8')): + raise AssertionError(f"Failed to start test instance: 'Container status is not 'Up'") + + def destroy_ncp(self): + # time.sleep(60) + if self._docker['container'] is None: + raise Error("Container not running") + + _ = subprocess.run(['docker', 'stop', self._docker['container']]) + try: + self.ncp_is_running() + raise FatalError("Failed to stop instance - ncp is still running") + except AssertionError: + pass + finally: + d_run = subprocess.run(['docker', 'rm', self._docker['container']]) + err = None + if d_run.returncode != 0: + err = FatalError(f"Failed to delete container {self._docker['container']}") + self._docker['container'] = None + if err is not None: + raise FatalError(err) + + def create_backup_in(self, target, *args): + logger.info(f"Creating backup in {target}...") + self.ncp_is_running() + backup_cfg = { + "id": "nc-backup", + "name": "nc-backup", + "title": "nc-backup", + "description": "Backup this NC instance to a file", + "info": "This will always include the current Nextcloud directory and the Database.\nYou can choose to include or exclude NC-data.", + "infotitle": "", + "params": [ + { + "id": "DESTDIR", + "name": "Destination directory", + "value": target, + "suggest": "/media/USBdrive/ncp-backups" + }, + { + "id": "INCLUDEDATA", + "name": "Include data", + "value": "no" if 'dataless' in args else "yes", + "type": "bool" + }, + { + "id": "COMPRESS", + "name": "Compress", + "value": "yes" if 'compressed' in args else "no", + "type": "bool" + }, + { + "id": "BACKUPLIMIT", + "name": "Number of backups to keep", + "value": "99", + "suggest": "4" + } + ] + } + temp_dir = tempfile.TemporaryDirectory() + tmp_file_path = os.path.join(temp_dir.name, 'nc-backup.cfg') + with open(tmp_file_path, 'w') as f: + json.dump(backup_cfg, f) + + backup_config_path = '/data/ncp/nc-backup.cfg' + d_run = self.run_on_ncp('rm', backup_config_path) + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"Unexpected error: {d_run.stderr}") + + d_run = subprocess.run(['docker', 'cp', tmp_file_path, f'{self._docker["container"]}:{backup_config_path}']) + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"Unexpected error: {d_run.stderr}") + + temp_dir.cleanup() + + d_run = self.run_on_ncp('chown', 'root:www-data', backup_config_path) + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"Unexpected error: {d_run.stderr}") + + d_run = self.run_on_ncp('chmod', '660', backup_config_path) + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"Unexpected error: {d_run.stderr}") + + d_run = self.run_on_ncp('bash', '-c', "set -x " + "&& source /usr/local/etc/library.sh " + "&& echo \"ncp library loaded.\" " + "&& run_app nc-backup") + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"An error occurred while creating backup: \n{d_run.stderr}") + + def run_on_ncp(self, *command): + run_cmd = ['docker', 'exec', self._docker['container']] + run_cmd.extend(command) + logger.debug(f"Executing: '{run_cmd}'") + return subprocess.run(run_cmd, stdout=PIPE, stderr=PIPE) + + def assert_file_exists_in_archive(self, backup_path, file, silent=False): + if not silent: + logger.info(f"Checking if '{file}' exists in any archive in '{backup_path}'") + d_run = self.run_on_ncp('bash', '-c', f'compgen -G "{backup_path}"') + if d_run.returncode != 0: + raise AssertionError(f'No such backup: {backup_path}') + archive = d_run.stdout.decode('utf-8').split('\n')[0] + d_run = self.run_on_ncp('tar', '-tf', archive) + if d_run.returncode != 0: + raise AssertionError(f'Error reading archive: {archive}') + match = re.search(file, d_run.stdout.decode('utf-8')) + if match is None: + raise AssertionError(f'File not found in backup: {file}') + + def assert_file_exists_not_in_archive(self, backup_path, file): + logger.info(f"Checking that '{file}' does not exist in any archive in '{backup_path}'") + try: + self.assert_file_exists_in_archive(backup_path, file, True) + raise AssertionError(f"File unexpectedly found in backup: {file}") + except AssertionError: + pass diff --git a/tests/nc_backup_test.robot b/tests/nc_backup_test.robot new file mode 100644 index 000000000..89cd0d028 --- /dev/null +++ b/tests/nc_backup_test.robot @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +*** Settings *** +Library NcpRobotLib.py 8080 8443 4443 +Suite Setup Setup Ncp docker latest +Suite Teardown Destroy Ncp +Test Teardown Run on Ncp rm -rf /opt/ncp-backups/*.tar + +*** Test Cases *** + +Create dataless Backup + #[Documentation] Creates and validates a backup using nc-backup + Create Backup in /opt/ncp-backups dataless + Assert file exists in archive /opt/ncp-backups/*.tar nextcloud/config/config.php + Assert file exists in archive /opt/ncp-backups/*.tar nextcloud-sqlbkp_.*.bak + Assert file exists not in archive /opt/ncp-backups/*.tar ncdata/files/ncp + +Create Backup with data + #[Documentation] Creates and validates a backup using nc-backup + Create Backup in /opt/ncp-backups + Assert file exists in archive /opt/ncp-backups/*.tar nextcloud/config/config.php + Assert file exists in archive /opt/ncp-backups/*.tar nextcloud-sqlbkp_.*.bak + Assert file exists in archive /opt/ncp-backups/*.tar data/ncp/files/Nextcloud.png diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 000000000..6adc4eb50 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +selenium~=4.7 +robotframework~=6.0 \ No newline at end of file From 64fa7d4e1010bb0c560aa67fc7fd624ffeea71a0 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:14:43 +0100 Subject: [PATCH 12/32] Fix setting up datadir at new default location during docker build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/CONFIG/nc-datadir.sh | 18 +++++++++++------- build/docker/Dockerfile | 15 ++++++++------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/bin/ncp/CONFIG/nc-datadir.sh b/bin/ncp/CONFIG/nc-datadir.sh index df7869b11..d2658edbd 100644 --- a/bin/ncp/CONFIG/nc-datadir.sh +++ b/bin/ncp/CONFIG/nc-datadir.sh @@ -21,13 +21,13 @@ install() } tmpl_opcache_dir() { - DATADIR="$(get_nc_config_value datadirectory)" + DATADIR="$(get_nc_config_value datadirectory || find_app_param nc-datadir DATADIR)" echo -n "${DATADIR}/.opcache" #[[ $( stat -fc%d / ) == $( stat -fc%d "$DATADIR" ) ]] && echo "/tmp" || echo "${DATADIR}/.opcache" } tmpl_tmp_upload_dir() { - DATADIR="$(get_nc_config_value datadirectory)" + DATADIR="$(get_nc_config_value datadirectory || find_app_param nc-datadir DATADIR)" echo -n "${DATADIR}/tmp" } @@ -101,7 +101,7 @@ configure() ## COPY cd /var/www/nextcloud - save_maintenance_mode + [[ "$BUILD_MODE" == 1 ]] || save_maintenance_mode echo "moving data directory from ${SRCDIR} to ${BASEDIR}..." @@ -124,13 +124,17 @@ configure() chown www-data: "${DATADIR}" # datadir - sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR?}"/config/config.php - ncc config:system:set logfile --value="${DATADIR}/nextcloud.log" + ncc config:system:set datadirectory --value="${DATADIR}" \ + || sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR?}"/config/config.php + + ncc config:system:set logfile --value="${DATADIR}/nextcloud.log" \ + || sed -i "s|'logfile' =>.*|'logfile' => '${DATADIR}/nextcloud.log',|" "${NCDIR?}"/config/config.php set_ncpcfg datadir "${DATADIR}" # tmp upload dir create_tmp_upload_dir - ncc config:system:set tempdirectory --value "$DATADIR/tmp" + ncc config:system:set tempdirectory --value "$DATADIR/tmp" \ + || sed -i "s|'tempdirectory' =>.*|'tempdirectory' => '${DATADIR}/tmp',|" "${NCDIR?}"/config/config.php sed -i "s|^;\?upload_tmp_dir =.*$|uploadtmp_dir = ${DATADIR}/tmp|" /etc/php/"${PHPVER?}"/cli/php.ini sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = ${DATADIR}/tmp|" /etc/php/"${PHPVER}"/fpm/php.ini sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = ${DATADIR}/tmp|" /etc/php/"${PHPVER}"/fpm/php.ini @@ -143,7 +147,7 @@ configure() [[ -f /etc/fail2ban/jail.local ]] && \ sed -i "s|logpath =.*nextcloud.log|logpath = ${DATADIR}/nextcloud.log|" /etc/fail2ban/jail.local - restore_maintenance_mode + [[ "$BUILD_MODE" == 1 ]] || restore_maintenance_mode ( . "${BINDIR?}/SYSTEM/metrics.sh" diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 7d9351a08..2d7a6795f 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -144,7 +144,7 @@ COPY ncp-previewgenerator /tmp/ncp-build/ncp-previewgenerator/ COPY build/docker /tmp/ncp-build/build/docker/ COPY ncp.sh update.sh post-inst.sh /tmp/ncp-build/ COPY etc/ncp-config.d/nc-init.cfg /usr/local/etc/ncp-config.d/nc-init-copy.cfg -COPY etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/ncp-config.d/nc-datadir.cfg +COPY etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/ncp-config.d/nc-datadir-copy.cfg RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ set -e; \ @@ -168,13 +168,14 @@ install_app ncp.sh; \ mv /usr/local/etc/ncp-config.d/nc-init-copy.cfg /usr/local/etc/ncp-config.d/nc-init.cfg; \ run_app_unsafe bin/ncp/CONFIG/nc-init.sh; \ -echo 'Moving data directory to a more sensible location' \ -mkdir -p /opt/ncdata \ -set_app_param nc-datadir.sh DATADIR '/data-ro/ncdata' \ -run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh \ -set_app_param nc-datadir.sh DATADIR '/data/ncdata' \ +echo 'Moving data directory to a more sensible location'; \ +mv /usr/local/etc/ncp-config.d/nc-datadir-copy.cfg /usr/local/etc/ncp-config.d/nc-datadir.cfg; \ +mkdir -p /data-ro/ncdata; \ +set_app_param nc-datadir.sh DATADIR '/data-ro/ncdata'; \ +BUILD_MODE=1 DISABLE_FS_CHECK=1 run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh; \ +set_app_param nc-datadir.sh DATADIR '/data/ncdata'; \ # Remove nc-datadir.cfg as it's disabled in docker -rm /usr/local/etc/nc-datadir.cfg \ +rm /usr/local/etc/ncp-config.d/nc-datadir.cfg; \ sed -i 's|data-ro|data|' /data-ro/nextcloud/config/config.php; \ # fix default paths From f8e416f6d528de7660d68737b329fc63ad9ea522 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:16:51 +0100 Subject: [PATCH 13/32] docker-build.yml: Fix docker update-test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 3205c9a4d..2b9a36875 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -130,7 +130,7 @@ jobs: RUN echo 'Mutex posixsem' >> /etc/apache2/mods-available/ssl.conf EOF - docker buildx build --platform "$platform" . -t ncp-testing -f Dockerfile + docker buildx build --platform "$platform" -t ncp-testing -f Dockerfile -o type=image,name=ncp-testing . popd rm -r "$tmpdir" docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing localhost From 0c3ac35e39a14f38d0d188adb3e09ba1d24d85fc Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 4 Jan 2023 21:16:58 +0100 Subject: [PATCH 14/32] ncp-restore: Don't assume data directory inside nc directory on docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-restore | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bin/ncp-restore b/bin/ncp-restore index 9b63df43d..9f78459e6 100755 --- a/bin/ncp-restore +++ b/bin/ncp-restore @@ -62,8 +62,6 @@ then target="$NCDIR/data" [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" mv -T "$DATA_BKP_DIR" "$target" -else - echo "Dataless backup detected - not restoring data" fi # update NC database password to this instance @@ -93,10 +91,9 @@ mysql -u root nextcloud < "$TMPDIR"/nextcloud-sqlbkp_*.bak || { echo "Error res ## RESTORE DATADIR -if is_docker; then +DATADIR="$(get_nc_config_value datadirectory)" +if is_docker && [[ "$DATADIR" == "/var/www/nextcloud/data" ]]; then DATADIR=/data/nextcloud/data -else - DATADIR="$(get_nc_config_value datadirectory)" fi [[ "$DATADIR" == "" ]] && { echo "Error reading data directory"; exit 1; } From 731e25e36bb7a9526354735dbcc2d8fc79c7a73f Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 4 Jan 2023 22:24:48 +0100 Subject: [PATCH 15/32] nc-datadir.sh: Prevent btrfs commands from being executed on docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/CONFIG/nc-datadir.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ncp/CONFIG/nc-datadir.sh b/bin/ncp/CONFIG/nc-datadir.sh index d2658edbd..aa4c99e5e 100644 --- a/bin/ncp/CONFIG/nc-datadir.sh +++ b/bin/ncp/CONFIG/nc-datadir.sh @@ -106,7 +106,7 @@ configure() echo "moving data directory from ${SRCDIR} to ${BASEDIR}..." # use subvolumes, if BTRFS - [[ "$(stat -fc%T "${BASEDIR}")" == "btrfs" ]] && { + [[ "$(stat -fc%T "${BASEDIR}")" == "btrfs" ]] && ! is_docker && { echo "BTRFS filesystem detected" rmdir "${BASEDIR}" btrfs subvolume create "${BASEDIR}" From 23f698da203c3d322b606ecab1d70356dade59c9 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:19:32 +0100 Subject: [PATCH 16/32] 000ncp: Install templates during docker startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - docker-build.yml: Fix update-test job Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 4 ++-- build/docker/nextcloudpi/000ncp | 8 +++++++- etc/library.sh | 3 +++ etc/ncp-templates/php/opcache.ini.sh | 6 +++++- etc/ncp-templates/php/pool.d.www.conf.sh | 2 +- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 2b9a36875..d9d921673 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -130,7 +130,7 @@ jobs: RUN echo 'Mutex posixsem' >> /etc/apache2/mods-available/ssl.conf EOF - docker buildx build --platform "$platform" -t ncp-testing -f Dockerfile -o type=image,name=ncp-testing . + docker buildx build --platform "$platform" -t ncp-testing -f Dockerfile --load . popd rm -r "$tmpdir" docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing localhost @@ -201,7 +201,7 @@ jobs: for attempt in {1..3} do echo ":: System Tests (attempt $attempt/3) ::" - python system_tests.py --no-ping --non-interactive || awk "{ print \"${LOG_TEST} \" \$0 }" + python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { echo -e "${LOG_CICD} System test (attempt $attempt) failed!" sleep 12 diff --git a/build/docker/nextcloudpi/000ncp b/build/docker/nextcloudpi/000ncp index ac0983518..0bd6b8c3a 100755 --- a/build/docker/nextcloudpi/000ncp +++ b/build/docker/nextcloudpi/000ncp @@ -21,7 +21,13 @@ persistent_cfg /etc/cron.weekly } persistent_cfg /usr/local/bin /data/bin +install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" +install_template "php/90-ncp.ini.sh" "/etc/php/${PHPVER}/fpm/conf.d/90-ncp.ini" +install_template "php/pool.d.www.conf.sh" "/etc/php/${PHPVER}/fpm/pool.d/www.conf" +#install_template "mysql/91-ncp.cnf.sh" "/etc/mysql/mariadb.conf.d/91-ncp.cnf" +install_template "ncp-metrics.cfg.sh" "/usr/local/etc/ncp-metrics.cfg" + touch /var/log/ncp.log -tail -f -n0 /data/nextcloud/data/nextcloud.log & +{ tail -f -n0 /data/ncdata/data/nextcloud.log || tail -f -n0 /data/nextcloud/data/nextcloud.log; } & exit 0 diff --git a/etc/library.sh b/etc/library.sh index 80327d56a..46d5265db 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -216,6 +216,9 @@ install_template() { local template="${1?}" local target="${2?}" local bkp="$(mktemp)" + + echo "Installing template '$template'..." + mkdir -p "$(dirname "$target")" [[ -f "$target" ]] && cp -a "$target" "$bkp" { diff --git a/etc/ncp-templates/php/opcache.ini.sh b/etc/ncp-templates/php/opcache.ini.sh index 7bbef81b9..d92badb2a 100644 --- a/etc/ncp-templates/php/opcache.ini.sh +++ b/etc/ncp-templates/php/opcache.ini.sh @@ -5,11 +5,15 @@ source /usr/local/etc/library.sh PHPVER="${PHPVER?ERROR: PHPVER variable unset!}" -if [[ "$1" == "--defaults" ]] || ! [[ -f "${BINDIR}/CONFIG/nc-datadir.sh" ]] +if [[ "$1" == "--defaults" ]] || ! [[ -f "${BINDIR}/CONFIG/nc-datadir.sh" ]] && ! is_docker then echo "INFO: Restoring template to default settings" >&2 TMP_DIR="/tmp" +elif is_docker +then + DATADIR="$(get_nc_config_value datadirectory || echo '/data/ncdata/data')" + TMP_DIR="$DATADIR/.opcache" else TMP_DIR="$(source "${BINDIR}/CONFIG/nc-datadir.sh"; tmpl_opcache_dir)" fi diff --git a/etc/ncp-templates/php/pool.d.www.conf.sh b/etc/ncp-templates/php/pool.d.www.conf.sh index 451b5afff..c4f8f3f59 100644 --- a/etc/ncp-templates/php/pool.d.www.conf.sh +++ b/etc/ncp-templates/php/pool.d.www.conf.sh @@ -5,7 +5,7 @@ source /usr/local/etc/library.sh PHPVER="${PHPVER?ERROR: PHPVER variable unset!}" -if [[ "$1" == "--defaults" ]] || ! [[ -f "${BINDIR}/CONFIG/nc-datadir.sh" ]] +if [[ "$1" == "--defaults" ]] || ! [[ -f "${BINDIR}/CONFIG/nc-limits.sh" ]] then echo "INFO: Restoring template to default settings" >&2 From eee84431a118d396120017cadcfaf855bb6ccad0 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 16:14:54 +0100 Subject: [PATCH 17/32] Fix release pipeline for lxd and docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build-docker.yml: Increase retries for integration tests to 5 - build-lxd.yml: Fix empty output variables Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 26 +++++++++++++------------- .github/workflows/build-lxd.yml | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index d9d921673..a3d228516 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -158,13 +158,13 @@ jobs: [[ "${ARCH?}" == "x86" ]] || { sleep 30; cmd+=(--timeout 300); } success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: Activation Tests (attempt $attempt/3) ::" + echo ":: Activation Tests (attempt $attempt/5) ::" "${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true - echo -e "${LOG_CICD} Activation test (attempt $attempt/3) failed!" + echo -e "${LOG_CICD} Activation test (attempt $attempt/5) failed!" docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true cat /proc/sys/fs/binfmt_misc/qemu-aarch64 |& awk "{ print \"${LOG_DIAG} (qemu-aarch64) \" \$0 }" || true cat /proc/sys/fs/binfmt_misc/qemu-arm |& awk "{ print \"${LOG_DIAG} (qemu-arm) \" \$0 }" || true @@ -193,14 +193,14 @@ jobs: echo -e "${LOG_DCKR} ==========" docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & - [[ "${ARCH?}" == "x86" ]] || sleep 30 + [[ "${ARCH?}" == "x86" ]] || sleep 120 sleep 20 success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: System Tests (attempt $attempt/3) ::" + echo ":: System Tests (attempt $attempt/5) ::" python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { echo -e "${LOG_CICD} System test (attempt $attempt) failed!" @@ -219,7 +219,7 @@ jobs: success=false for attempt in {1..3} do - echo ":: Nextcloud Tests (attempt $attempt/3) ::" + echo ":: Nextcloud Tests (attempt $attempt/5) ::" python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true @@ -293,9 +293,9 @@ jobs: [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: Activation Tests (attempt $attempt/3) ::" + echo ":: Activation Tests (attempt $attempt/5) ::" "${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true @@ -314,9 +314,9 @@ jobs: echo -e "${LOG_CICD} Activation test successful" success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: System Tests (attempt $attempt/3) ::" + echo ":: System Tests (attempt $attempt/5) ::" python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { echo -e "${LOG_CICD} System test (attempt $attempt) failed!" @@ -333,9 +333,9 @@ jobs: echo -e "${LOG_CICD} System test successful" success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: Nextcloud Tests (attempt $attempt/3) ::" + echo ":: Nextcloud Tests (attempt $attempt/5) ::" python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index ef1ccab39..27a895598 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -7,9 +7,9 @@ on: type: string outputs: artifact_name: - value: "${{ jobs.build.outputs.artifact_name }}" + value: "${{ jobs.build-current.outputs.artifact_name }}" artifact_file: - value: "${{ jobs.build.outputs.artifact_file }}" + value: "${{ jobs.build-current.outputs.artifact_file }}" push: branches: - "**" From 224b64bcea85a938778a2256f143e5e5458bd60c Mon Sep 17 00:00:00 2001 From: "Victor-ray, S" <12261439+ZendaiOwl@users.noreply.github.com> Date: Sun, 1 Jan 2023 13:07:30 +0100 Subject: [PATCH 18/32] =?UTF-8?q?=E0=BF=93=E2=9D=AF=20Fixed=20ncp.sh,=20re?= =?UTF-8?q?moved=20the=20pi=20user=20/sbin/nologin=20shell=20as=20that=20u?= =?UTF-8?q?ser=20no=20longer=20exists=20by=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Victor-ray, S <12261439+ZendaiOwl@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- ncp.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ncp.sh b/ncp.sh index 9aab8c11c..2904f775c 100644 --- a/ncp.sh +++ b/ncp.sh @@ -1,6 +1,6 @@ #!/bin/bash -# NextCloudPi additions to Raspbian +# NextcloudPi additions to Raspbian # # Copyleft 2017 by Ignacio Nunez Hernanz # GPL licensed (see end of file) * Use at your own risk! @@ -25,12 +25,13 @@ install() $APTINSTALL git dialog whiptail jq file lsb-release mkdir -p "$CONFDIR" "$BINDIR" - # include option in raspi-config (only Raspbian) + # This has changed, pi user no longer exists by default, the user needs to create it with Raspberry Pi imager + # The raspi-config layout and options have also changed + # https://github.com/RPi-Distro/raspi-config/blob/master/raspi-config test -f /usr/bin/raspi-config && { - sed -i '/Change User Password/i"0 NextCloudPi Configuration" "Configuration of NextCloudPi" \\' /usr/bin/raspi-config - sed -i '/1\\ \*) do_change_pass ;;/i0\\ *) ncp-config ;;' /usr/bin/raspi-config - # Disable raspberry pi default user - usermod pi -s /sbin/nologin + # shellcheck disable=SC1003 + sed -i '/S3 Password/i "S0 NextcloudPi Configuration" "Configuration of NextcloudPi" \\' /usr/bin/raspi-config + sed -i '/S3\\ \*) do_change_pass ;;/i S0\\ *) ncp-config ;;' /usr/bin/raspi-config } # add the ncc shortcut @@ -205,7 +206,7 @@ EOF chmod g+w /var/run/.ncp-latest-version # Install all ncp-apps - ALLOW_UPDATE_SCRIPT=1 bin/ncp-update $BRANCH || exit $? + ALLOW_UPDATE_SCRIPT=1 bin/ncp-update "$BRANCH" || exit $? # LIMIT LOG SIZE grep -q maxsize /etc/logrotate.d/apache2 || sed -i /weekly/amaxsize2M /etc/logrotate.d/apache2 @@ -254,7 +255,7 @@ EOF ## tag image is_docker && local DOCKER_TAG="_docker" is_lxc && local DOCKER_TAG="_lxc" - echo "NextCloudPi${DOCKER_TAG}_$( date "+%m-%d-%y" )" > /usr/local/etc/ncp-baseimage + echo "NextcloudPi${DOCKER_TAG}_$( date "+%m-%d-%y" )" > /usr/local/etc/ncp-baseimage ## SSH hardening if [[ -f /etc/ssh/sshd_config ]]; then From 7303d4775924285887de2f970d3f7f38698ef9d1 Mon Sep 17 00:00:00 2001 From: "Victor-ray, S" <12261439+ZendaiOwl@users.noreply.github.com> Date: Sun, 1 Jan 2023 11:35:49 +0100 Subject: [PATCH 19/32] =?UTF-8?q?=E0=BF=93=E2=9D=AF=20Updated=20the=20READ?= =?UTF-8?q?ME=20a=20little=20bit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Victor-ray, S <12261439+ZendaiOwl@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8350366a1..c0960b2a0 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ English | [Traditional Chinese 繁體中文](i18n/README-zh_TW.md) | [Simplified Chinese 简体中文](i18n/README-zh_CN.md) -# NextCloudPi [![chatroom icon](https://patrolavia.github.io/telegram-badge/chat.png)](https://t.me/NextCloudPi) [![forums icon](https://img.shields.io/badge/help-forums-blue.svg)](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=N8PJHSEQF4G7Y&lc=US&item_name=Own%20Your%20Bits&item_number=NextCloudPi&no_note=1&no_shipping=1¤cy_code=EUR&bn=PP%2dDonationsBF%3abtn_donate_LG%2egif%3aNonHosted) [![blog](https://img.shields.io/badge/follow-blog-orange.svg)](https://ownyourbits.com) +# NextcloudPi [![chatroom icon](https://patrolavia.github.io/telegram-badge/chat.png)](https://t.me/NextcloudPi) [![forums icon](https://img.shields.io/badge/help-forums-blue.svg)](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=N8PJHSEQF4G7Y&lc=US&item_name=Own%20Your%20Bits&item_number=NextcloudPi&no_note=1&no_shipping=1¤cy_code=EUR&bn=PP%2dDonationsBF%3abtn_donate_LG%2egif%3aNonHosted) [![blog](https://img.shields.io/badge/follow-blog-orange.svg)](https://ownyourbits.com) -![NC Logo](https://ownyourbits.com/wp-content/uploads/2017/11/ncp-square.png) +![NCP Logo](https://github.com/nextcloud/nextcloudpi/blob/master/ncp-app/img/app.svg) -This is the build code for [NextCloudPi](https://nextcloudpi.com). +This is the build code for [NextcloudPi](https://nextcloudpi.com). -NextCloudPi is a ready to use image for Virtual Machines, Raspberry Pi, Odroid HC1, rock64 and other boards [(⇒Downloads)](https://github.com/nextcloud/nextcloudpi/releases). +NextcloudPi is a ready to use image for Virtual Machines, Raspberry Pi, Odroid HC1, rock64 and other boards [(⇒Downloads)](https://github.com/nextcloud/nextcloudpi/releases). -This code also generates the NextCloudPi [docker image](https://hub.docker.com/r/ownyourbits/nextcloudpi), LXD and VM, and includes an installer for any Debian based system. +This code also generates the NextcloudPi [docker image](https://hub.docker.com/r/ownyourbits/nextcloudpi), LXD and VM, and includes an installer for any Debian based system. Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.com) @@ -42,7 +42,7 @@ Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.co ## Extras * Setup wizard - * NextCloudPi Web Panel + * NextcloudPi Web Panel * Wi-Fi ready * Ram logs * Automatic security updates, activated by default. @@ -98,7 +98,7 @@ docker run -d -p 4443:4443 -p 443:443 -p 80:80 -v ncdata:/data --name nextcloudp ## Run in LXD ``` -lxc import NextCloudPi_LXD_09-29-21.tar.bz +lxc import NextcloudPi_LXD_09-29-21.tar.bz lxc start ncp ``` @@ -118,7 +118,7 @@ build/build-SD-rpi.sh ./build-SD-armbian.sh odroidxu4 # supported board code name ``` -In order to generate the Docker images +In order to generate the Docker images, you'll also need to change the username, repo and tags to match your credentials at Docker Hub. ``` git clone https://github.com/nextcloud/nextcloudpi.git @@ -134,7 +134,7 @@ build/build-docker.sh arm64 build/build-LXD.sh ``` -NextCloudPi can be installed in any architecture running the latest Debian +NextcloudPi can be installed in any architecture running the latest Debian _Note: this assumes a clean Debian install, and there is no rollback method_ @@ -150,4 +150,4 @@ https://hub.docker.com/r/ownyourbits/nextcloudpi ## Contact -You can find us in the [forums](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) and a [Telegram group](https://t.me/NextCloudPi) +You can find us in the [forums](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) and a [Telegram group](https://t.me/NextcloudPi) From 2f71dbec2ed24afad7fdf472ba36db1c6a2d4889 Mon Sep 17 00:00:00 2001 From: "Victor-ray, S" <12261439+ZendaiOwl@users.noreply.github.com> Date: Sun, 1 Jan 2023 11:43:20 +0100 Subject: [PATCH 20/32] =?UTF-8?q?=E0=BF=93=E2=9D=AF=20Updated=20the=20READ?= =?UTF-8?q?ME=20a=20little=20bit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Victor-ray, S <12261439+ZendaiOwl@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c0960b2a0..0c5064b1c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ English | [Traditional Chinese 繁體中文](i18n/README-zh_TW.md) | [Simplified # NextcloudPi [![chatroom icon](https://patrolavia.github.io/telegram-badge/chat.png)](https://t.me/NextcloudPi) [![forums icon](https://img.shields.io/badge/help-forums-blue.svg)](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=N8PJHSEQF4G7Y&lc=US&item_name=Own%20Your%20Bits&item_number=NextcloudPi&no_note=1&no_shipping=1¤cy_code=EUR&bn=PP%2dDonationsBF%3abtn_donate_LG%2egif%3aNonHosted) [![blog](https://img.shields.io/badge/follow-blog-orange.svg)](https://ownyourbits.com) - ![NCP Logo](https://github.com/nextcloud/nextcloudpi/blob/master/ncp-app/img/app.svg) This is the build code for [NextcloudPi](https://nextcloudpi.com). @@ -14,6 +13,7 @@ This code also generates the NextcloudPi [docker image](https://hub.docker.com/r Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.com) --- + [![VM Integration Tests](https://github.com/nextcloud/nextcloudpi/workflows/VM%20Integration%20Tests/badge.svg)](https://github.com/nextcloud/nextcloudpi/actions/workflows/vm-tests.yml) [![Docker Integration Tests](https://github.com/nextcloud/nextcloudpi/actions/workflows/build-docker.yml/badge.svg)](https://github.com/nextcloud/nextcloudpi/actions/workflows/build-docker.yml) @@ -80,13 +80,13 @@ Extras can be activated and configured using the web interface at HTTPS port 444 ![ncp-web](https://user-images.githubusercontent.com/21343324/136853829-f4e99ec0-6307-431f-b4c7-21b2330cae7f.png) -, or from the command line from +Or from the command line using ``` sudo ncp-config ``` -![NCP-config](https://ownyourbits.com/wp-content/uploads/2017/03/ncp-conf-700x456.jpg) +![NCP-config](https://help.nextcloud.com/uploads/default/original/3X/b/3/b3d157022a32296ab54428b14b5df02104a91f18.png) ## Run in docker From 298f0f2e931dc1e256cd0e9befe5f188929c94c0 Mon Sep 17 00:00:00 2001 From: "Victor-ray, S" <12261439+ZendaiOwl@users.noreply.github.com> Date: Wed, 4 Jan 2023 13:58:02 +0100 Subject: [PATCH 21/32] =?UTF-8?q?=E0=BF=93=E2=9D=AF=20Changed=20NextCloud?= =?UTF-8?q?=20to=20Nextcloud?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Victor-ray, S <12261439+ZendaiOwl@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- README.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0c5064b1c..f0a5c2be5 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,8 @@ Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.co * Automatic NCP updates * Automatic Nextcloud updates * Update notifications - * NextCloud backup and restore - * NextCloud online installation + * Nextcloud backup and restore + * Nextcloud online installation * Format USB drive to BTRFS * BTRFS snapshots * Automatic BTRFS snapshots @@ -104,15 +104,27 @@ lxc start ncp ## How to build -Install git, docker, qemu-user-static, chroot and all the usual building tools. +Install: + +- `git` +- `docker` +- `build-essential` +- `qemu` +- `qemu-user-static` +- `chroot` +- `jq` +- `psmisc` +- `procps` + +and all the usual building tools. ``` git clone https://github.com/nextcloud/nextcloudpi.git cd nextcloudpi -build/build-SD-rpi.sh +./build/build-SD-rpi.sh ``` -, or for an Armbian based board +### Armbian-based board ``` ./build-SD-armbian.sh odroidxu4 # supported board code name @@ -128,16 +140,18 @@ build/build-docker.sh armhf build/build-docker.sh arm64 ``` -, for LXD +### LXD ``` -build/build-LXD.sh +./build/build-LXD.sh ``` NextcloudPi can be installed in any architecture running the latest Debian _Note: this assumes a clean Debian install, and there is no rollback method_ +### Curl install scripts + ``` # curl -sSL https://raw.githubusercontent.com/nextcloud/nextcloudpi/master/install.sh | bash ``` From 2655c5d5a153a0a129f17b5ead1b8dc453f5b6b9 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Fri, 6 Jan 2023 01:01:17 +0100 Subject: [PATCH 22/32] Update nextcloud to version 24.0.5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- README.md | 2 +- etc/ncp-config.d/nc-nextcloud.cfg | 2 +- etc/ncp.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f0a5c2be5..43604f720 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.co ## Features * Debian/Raspbian 11 Bullseye - * Nextcloud 24.0.5 + * Nextcloud 24.0.8 * Apache 2.4.25, with HTTP2 enabled * PHP 7.4 * MariaDB 10 diff --git a/etc/ncp-config.d/nc-nextcloud.cfg b/etc/ncp-config.d/nc-nextcloud.cfg index 2421b1643..38e9c782c 100644 --- a/etc/ncp-config.d/nc-nextcloud.cfg +++ b/etc/ncp-config.d/nc-nextcloud.cfg @@ -9,7 +9,7 @@ { "id": "VER", "name": "Version", - "value": "24.0.5" + "value": "24.0.8" }, { "id": "MAXFILESIZE", diff --git a/etc/ncp.cfg b/etc/ncp.cfg index 5e204b2ed..8d60b13a6 100644 --- a/etc/ncp.cfg +++ b/etc/ncp.cfg @@ -1,5 +1,5 @@ { - "nextcloud_version": "24.0.5", + "nextcloud_version": "24.0.8", "php_version": "8.1", "release": "bullseye" } From da2dedc4500547544dd43318fe653f49d58d2afe Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Fri, 6 Jan 2023 19:04:59 +0100 Subject: [PATCH 23/32] Fix erroneously created symlinks in /usr/local/bin/ncp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/BACKUPS/nc-backup.sh | 3 --- bin/ncp/BACKUPS/nc-restore.sh | 6 +----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/bin/ncp/BACKUPS/nc-backup.sh b/bin/ncp/BACKUPS/nc-backup.sh index 1b08ba745..8df7c3f3d 100644 --- a/bin/ncp/BACKUPS/nc-backup.sh +++ b/bin/ncp/BACKUPS/nc-backup.sh @@ -18,9 +18,6 @@ install() { apt-get update apt-get install -y --no-install-recommends pigz - - ln -s /usr/local/bin/ncp-backup /usr/local/bin/ncp/ncp-backup - chmod +x /usr/local/bin/ncp/ncp-backup } configure() diff --git a/bin/ncp/BACKUPS/nc-restore.sh b/bin/ncp/BACKUPS/nc-restore.sh index 322fab9d6..05eca5009 100644 --- a/bin/ncp/BACKUPS/nc-restore.sh +++ b/bin/ncp/BACKUPS/nc-restore.sh @@ -9,11 +9,7 @@ # More at https://ownyourbits.com/2017/02/13/nextcloud-ready-raspberry-pi-image/ # -install() -{ - ln -s /usr/local/bin/ncp-restore /usr/local/bin/ncp/ncp-restore - chmod +x /usr/local/bin/ncp/ncp-restore -} +install() { :; } configure() { From ef4ec175ff298e8ea66c88e57684837c94ca8011 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 7 Jan 2023 19:08:13 +0100 Subject: [PATCH 24/32] ncp-restore: Improve clarity of output regarding data conflicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-restore | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/ncp-restore b/bin/ncp-restore index 9f78459e6..8c5ceda20 100755 --- a/bin/ncp-restore +++ b/bin/ncp-restore @@ -58,9 +58,12 @@ mv -T "$TMPDIR"/nextcloud "$NCDIR" || { echo "Error restoring base files"; exit if [[ -n "$DATA_BKP_DIR" ]] then - echo "Restoring $NCDIR/data..." target="$NCDIR/data" - [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" + [[ -d "$target" ]] && { + target="$NCDIR/.data_$(date -I)" + echo "DATA CONFLICT! Your old '$NCDIR/data' directory can be found in '$target' after the restore script is done. The new '$NCDIR/data' directory will be restored from the backup." + } + echo "Restoring old '$NCDIR/data' to '$target'..." mv -T "$DATA_BKP_DIR" "$target" fi @@ -105,8 +108,8 @@ NUMFILES=2 if [[ $( ls "$TMPDIR" | wc -l ) -eq $NUMFILES ]]; then [[ -e "$DATADIR" ]] && { - echo "backing up existing $DATADIR to $DATADIR-$( date "+%m-%d-%y" )..." - mv "$DATADIR" "$DATADIR-$( date "+%m-%d-%y" )" || exit 1 + echo "backing up existing $DATADIR to $DATADIR-$( date -I )..." + mv "$DATADIR" "$DATADIR-$( date -I )" || exit 1 } echo "restore datadir to $DATADIR..." From 71f8a2ef81eb7168ad841fa290e63d0710062e63 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 7 Jan 2023 19:17:48 +0100 Subject: [PATCH 25/32] ncp-restore: Use new default data directory location when resetting datadir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-restore | 6 ++++-- etc/ncp-templates/php/opcache.ini.sh | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/ncp-restore b/bin/ncp-restore index 8c5ceda20..a7e4badfa 100755 --- a/bin/ncp-restore +++ b/bin/ncp-restore @@ -132,8 +132,10 @@ else echo "No datadir found in backup" [[ -e "$DATADIR" ]] || { - echo "${DATADIR} not found. Resetting to ${NCDIR}/data" - DATADIR="$NCDIR"/data + echo -n "${DATADIR} not found." + DATADIR="/data/ncdata/data" + is_docker || DATADIR="/opt/data" + echo "Resetting to ${DATADIR}" mkdir -p "${DATADIR}" touch "${DATADIR}"/.ocdata chown -R www-data: "${DATADIR}" diff --git a/etc/ncp-templates/php/opcache.ini.sh b/etc/ncp-templates/php/opcache.ini.sh index d92badb2a..26d9b2f56 100644 --- a/etc/ncp-templates/php/opcache.ini.sh +++ b/etc/ncp-templates/php/opcache.ini.sh @@ -18,6 +18,8 @@ else TMP_DIR="$(source "${BINDIR}/CONFIG/nc-datadir.sh"; tmpl_opcache_dir)" fi +mkdir -p "$TMP_DIR" + cat < Date: Sat, 7 Jan 2023 19:26:22 +0100 Subject: [PATCH 26/32] ncp-restore: Add command for merging data directory with backup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-restore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ncp-restore b/bin/ncp-restore index a7e4badfa..2bc8696cc 100755 --- a/bin/ncp-restore +++ b/bin/ncp-restore @@ -61,7 +61,7 @@ then target="$NCDIR/data" [[ -d "$target" ]] && { target="$NCDIR/.data_$(date -I)" - echo "DATA CONFLICT! Your old '$NCDIR/data' directory can be found in '$target' after the restore script is done. The new '$NCDIR/data' directory will be restored from the backup." + echo "DATA CONFLICT! Your old '$NCDIR/data' directory can be found in '$target' after the restore script is done. The new '$NCDIR/data' directory will be restored from the backup. You can manually merge these directories with this command (run nc-fix-permissions and nc-scan afterwards): cp -rn '$target/'* '$target/'.[!.]* '$NCDIR/data/'" } echo "Restoring old '$NCDIR/data' to '$target'..." mv -T "$DATA_BKP_DIR" "$target" From 35c775be2f32ff4b372a9508f590bcab0850ebb0 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 7 Jan 2023 21:29:48 +0100 Subject: [PATCH 27/32] opcache.ini.sh: Fix template populating /data/... during docker build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- etc/ncp-templates/php/opcache.ini.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etc/ncp-templates/php/opcache.ini.sh b/etc/ncp-templates/php/opcache.ini.sh index 26d9b2f56..53a5c0de0 100644 --- a/etc/ncp-templates/php/opcache.ini.sh +++ b/etc/ncp-templates/php/opcache.ini.sh @@ -12,14 +12,13 @@ then TMP_DIR="/tmp" elif is_docker then - DATADIR="$(get_nc_config_value datadirectory || echo '/data/ncdata/data')" + DATADIR="/data-ro/ncdata/data" + [[ "$DOCKERBUILD" == 1 ]] || DATADIR="$(get_nc_config_value datadirectory || echo '/data/ncdata/data')" TMP_DIR="$DATADIR/.opcache" else TMP_DIR="$(source "${BINDIR}/CONFIG/nc-datadir.sh"; tmpl_opcache_dir)" fi -mkdir -p "$TMP_DIR" - cat < Date: Sat, 7 Jan 2023 21:42:32 +0100 Subject: [PATCH 28/32] build-lxd.yml, build-docker.yml: Use GH API token for downloading gecko driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 8 ++++++-- .github/workflows/build-lxd.yml | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index a3d228516..f54232c5a 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -143,7 +143,9 @@ jobs: - name: Setup Firefox uses: browser-actions/setup-firefox@latest - name: Setup GeckoDriver - uses: browser-actions/setup-geckodriver@latest + uses: ChlodAlejandro/setup-geckodriver@latest + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Selenium run: pip install selenium @@ -278,7 +280,9 @@ jobs: - name: Setup Firefox uses: browser-actions/setup-firefox@latest - name: Setup GeckoDriver - uses: browser-actions/setup-geckodriver@latest + uses: ChlodAlejandro/setup-geckodriver@latest + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Selenium run: pip install selenium diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 27a895598..f911093b3 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -267,7 +267,9 @@ jobs: - name: Setup Firefox uses: browser-actions/setup-firefox@latest - name: Setup GeckoDriver - uses: browser-actions/setup-geckodriver@latest + uses: ChlodAlejandro/setup-geckodriver@latest + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Selenium run: pip install selenium - name: download LXD image from artifact store @@ -342,7 +344,9 @@ jobs: - name: Setup Firefox uses: browser-actions/setup-firefox@latest - name: Setup GeckoDriver - uses: browser-actions/setup-geckodriver@latest + uses: ChlodAlejandro/setup-geckodriver@latest + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Selenium run: pip install selenium - name: download LXD image from artifact store From 9fdd7822cda73a0dea5a76806979d76df7b5e6c7 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 8 Jan 2023 15:37:45 +0100 Subject: [PATCH 29/32] build-docker.yml, build-sd-images.yml: Increase retries and timeouts during tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 10 +++++----- .github/workflows/build-sd-images.yml | 18 +++++++++--------- bin/ncp-restore | 9 +++++---- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index f54232c5a..6936d5ad7 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -157,7 +157,7 @@ jobs: docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & cmd=(python activation_tests.py --no-gui localhost 8443 4443) - [[ "${ARCH?}" == "x86" ]] || { sleep 30; cmd+=(--timeout 300); } + [[ "${ARCH?}" == "x86" ]] || { sleep 60; cmd+=(--timeout 300); } success=false for attempt in {1..5} @@ -195,7 +195,7 @@ jobs: echo -e "${LOG_DCKR} ==========" docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & - [[ "${ARCH?}" == "x86" ]] || sleep 120 + [[ "${ARCH?}" == "x86" ]] || sleep 240 sleep 20 @@ -294,7 +294,7 @@ jobs: docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & cmd=(python activation_tests.py --no-gui localhost 8443 4443) - [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) + [[ "${ARCH?}" == "x86" ]] || { sleep 60; cmd+=(--timeout 300); } success=false for attempt in {1..5} @@ -324,7 +324,7 @@ jobs: python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { echo -e "${LOG_CICD} System test (attempt $attempt) failed!" - sleep 5 + sleep 12 continue } success=true @@ -344,7 +344,7 @@ jobs: [[ ${PIPESTATUS[0]} -eq 0 ]] || { tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true echo -e "${LOG_CICD} Nextcloud test (attempt $attempt/3) failed!" - sleep 5 + sleep 12 continue } success=true diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index 168dc4e67..5fdd37fdc 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -201,9 +201,9 @@ jobs: } success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: Activation Tests (attempt $attempt/3) ::" + echo ":: Activation Tests (attempt $attempt/5) ::" python tests/activation_tests.py -t 300 --no-gui "$ip" 443 4443 || { echo "Activation test failed!" echo "Geckodriver logs:" @@ -212,7 +212,7 @@ jobs: echo "mysql: " sudo systemd-run --wait -P --machine=ncp bash /usr/local/bin/ncp-diag sudo systemd-run --wait -P --machine=ncp systemctl status mysql - sleep 6 + sleep 12 continue } success=true @@ -224,12 +224,12 @@ jobs: } success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: System Tests (attempt $attempt/3) ::" + echo ":: System Tests (attempt $attempt/5) ::" sudo python tests/system_tests.py --non-interactive || { echo "System test failed!" - sleep 6 + sleep 12 continue } success=true @@ -241,9 +241,9 @@ jobs: } success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: Nextcloud Tests (attempt $attempt/3) ::" + echo ":: Nextcloud Tests (attempt $attempt/5) ::" python tests/nextcloud_tests.py --no-gui "$ip" 443 4443 || { echo "Nextcloud test failed!" echo "Geckodriver logs:" @@ -251,7 +251,7 @@ jobs: echo "================" echo "ncp.log: " sudo systemd-run --wait -P --machine=ncp ncp /bin/bash -c "tail -n20 /var/log/ncp.log" || true - sleep 6 + sleep 12 continue } success=true diff --git a/bin/ncp-restore b/bin/ncp-restore index 2bc8696cc..aea4c064d 100755 --- a/bin/ncp-restore +++ b/bin/ncp-restore @@ -88,6 +88,7 @@ CREATE USER '$DBADMIN'@'localhost' IDENTIFIED BY '$DBPASSWD'; GRANT ALL PRIVILEGES ON nextcloud.* TO $DBADMIN@localhost; EXIT EOFMYSQL +# shellcheck disable=SC2181 [ $? -ne 0 ] && { echo "Error configuring nextcloud database"; exit 1; } mysql -u root nextcloud < "$TMPDIR"/nextcloud-sqlbkp_*.bak || { echo "Error restoring nextcloud database"; exit 1; } @@ -157,9 +158,9 @@ install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache mkdir -p "$DATADIR/tmp" chown www-data: "$DATADIR/tmp" ncc config:system:set tempdirectory --value "$DATADIR/tmp" -sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/cli/php.ini -sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini -sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini +sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" "/etc/php/${PHPVER}/cli/php.ini" +sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" "/etc/php/${PHPVER}/fpm/php.ini" +sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = $DATADIR/tmp|" "/etc/php/${PHPVER}/fpm/php.ini" # logs ncc config:system:set logfile --value="$DATADIR/nextcloud.log" @@ -183,4 +184,4 @@ ncc files:scan-app-data [[ "$NEED_RESTART" == "1" ]] && { clear_opcache bash -c "sleep 3; service php${PHPVER}-fpm restart" &>/dev/null & -} \ No newline at end of file +} From 13129b694e4d8684cb201eaa8c65783ab39357c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 8 Jan 2023 18:23:29 +0100 Subject: [PATCH 30/32] nc-snapshot-auto.sh: Add missing import of library.sh in /etc/cron.hourly/btrfs-snp (#1689) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/BACKUPS/nc-snapshot-auto.sh | 3 +++ updates/1.50.5.sh | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 updates/1.50.5.sh diff --git a/bin/ncp/BACKUPS/nc-snapshot-auto.sh b/bin/ncp/BACKUPS/nc-snapshot-auto.sh index 51e26ce60..a0ccef221 100644 --- a/bin/ncp/BACKUPS/nc-snapshot-auto.sh +++ b/bin/ncp/BACKUPS/nc-snapshot-auto.sh @@ -26,6 +26,9 @@ configure() cat > /etc/cron.hourly/btrfs-snp < Date: Sun, 8 Jan 2023 20:56:57 +0100 Subject: [PATCH 31/32] Update changelog.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- changelog.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/changelog.md b/changelog.md index 83a2a1388..aead65d47 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,24 @@ +## [v1.50.5](https://github.com/nextcloud/nextcloudpi/tree/v1.50.5) (2023-01-08) Various fixes and update to NC 24.0.8 + +### Changes + +- Move default location of data directory out of nextcloud directory (partial fix for [#1417](https://github.com/nextcloud/nextcloudpi/issues/1417)) +- ssh: Add option to add ssh user to sudo group +- Disable root user for login +- Install various templates on docker container start (improves config persistence) + +### Fixes + +- Keep old data directory if present (partial fix for [#1417](https://github.com/nextcloud/nextcloudpi/issues/1417)) +- Temporarily move and restore user files during nc-restore (partial fix for [#1417](https://github.com/nextcloud/nextcloudpi/issues/1417)) +- Improve robustness of the setup process for notify_push + +### Development related + +- build-docker.yml: Fix update-test on architectures other than x86 +- Many fixes and improvements for the CICD pipelines +- Add basic integration test for nc-backup + ## [v1.50.4](https://github.com/nextcloud/nextcloudpi/tree/v1.50.4) (2022-12-17) Installer Fix - Don't install deprecated PHP packages during installation (`php-json`) From d9cb6543cd7b1c308c34f3a73714efdbbe916880 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 8 Jan 2023 21:02:04 +0100 Subject: [PATCH 32/32] Update Readme.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 43604f720..06a139690 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,11 @@ Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.co * Debian/Raspbian 11 Bullseye * Nextcloud 24.0.8 - * Apache 2.4.25, with HTTP2 enabled - * PHP 7.4 + * Apache, with HTTP2 enabled + * PHP 8.1 * MariaDB 10 * Redis memory cache - * ncp-config for easy setup ( RAM logs, USB drive and more ) + * ncp-config TUI for easy setup ( RAM logs, USB drive and more ) * Automatic redirection to HTTPS * ACPU PHP cache * PHP Zend OPcache enabled with file cache