v1.57.0

.github/actions/create-test-instance/action.yml

@@ -13,7 +13,7 @@ inputs:
   server_type:
     description: Server type to use for hetzner servers
     required: true
-    default: "cx22"
+    default: "cx23"
 
 outputs:
   server_address:

.github/workflows/build-sd-images.yml

@@ -155,12 +155,14 @@ jobs:
         run: |
           set -ex
           mkdir -p output
+          sudo apt-get update
           sudo apt-get install -y pv
           wget -nv "${{ inputs.test_image_url }}" -O - | pv -n --bytes --rate --timer -i 3 > "output/${ARTIFACT_FILE?}"
       - name: Prepare test
         run: |
           set -x
           mv "output/${ARTIFACT_FILE?}" ncp.img
+          sudo apt-get update
           sudo apt-get install -y systemd-container
           python3 -m venv ./.venv
           . ./.venv/bin/activate

bin/ncp-diag

@@ -15,7 +15,12 @@ source /usr/local/etc/library.sh
 # Distro, NCP version and tag
 echo "NextcloudPi version|$( cat /usr/local/etc/ncp-version )"
 [[ -f /usr/local/etc/ncp-baseimage ]] && echo "NextcloudPi image|$( cat /usr/local/etc/ncp-baseimage )"
-echo "OS|$(sed 's| \\n \\l||' /etc/issue). $(uname -r) ($(uname -m))"
+if [[ -r /etc/os-release ]]; then
+  . /etc/os-release
+  echo "OS|$PRETTY_NAME. $(uname -r) ($(uname -m))"
+else
+  echo "OS|unknown. $(uname -r) ($(uname -m))"
+fi
 
 # Data
 DATADIR="$( grep datadirectory /var/www/nextcloud/config/config.php |

bin/ncp-update-nc

@@ -22,6 +22,11 @@ set -eE${DBG}
 BIN="${0##*/}"
 VER="$1"
 [[ "$VER" == "" ]] && { echo "Usage ${BIN} <version>"; exit 1; }
+extra_args=()
+if [[ " $* " =~ " "--allow-incompatible-apps" " ]]
+then
+  extra_args+=(--allow-incompatible-apps)
+fi
 
 connect_to_nc_update() {
   tail -n 100 -f "/var/log/ncp-update-nc.log" &
@@ -54,7 +59,7 @@ fi
 
 systemctl reset-failed ncp-update-nc 2>/dev/null ||:
 systemd-run -u 'ncp-update-nc' --service-type=oneshot --no-block -p TimeoutStartSec="24h" -p TimeoutStopSec="1h" \
-  bash -c "set -o pipefail; DBG='${DBG:-}' /usr/local/bin/ncp-update-nc.d/update-nc.sh '${VER}' |& tee /var/log/ncp-update-nc.log"
+  bash -c "set -o pipefail; DBG='${DBG:-}' /usr/local/bin/ncp-update-nc.d/update-nc.sh '${VER}' ${extra_args[*]} |& tee /var/log/ncp-update-nc.log"
 sleep 1
 
 if ! [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|inactive|activating|deactivating)$ ]]

bin/ncp-update-nc.d/update-nc.sh

@@ -94,6 +94,27 @@ URL="https://download.nextcloud.com/server/releases/nextcloud-$TARGET_VERSION.ta
 echo "Download Nextcloud $TARGET_VERSION..."
 wget -q "$URL" -O nextcloud.tar.bz2 || { echo "Error downloading"; exit 1; }
 
+# Check if installed apps are compatible
+server_apps="$(tar -tf nextcloud.tar.bz2 --exclude='nextcloud/apps/*/*' | grep '^nextcloud/apps' | sed -e 's|^nextcloud/apps/||g' -e 's|/$||g')"
+compatible_apps="$(curl "https://apps.nextcloud.com/api/v1/platform/$TARGET_VERSION/apps.json" | jq '.[].id' -r)"
+enabled_apps="$(ncc app:list --output json | jq '.enabled | keys | .[]' -r)"
+
+incompatible="$(comm -2 -3 <( echo "${enabled_apps?}" | sort ) <( { echo "${compatible_apps?}"; echo "${server_apps?}"; echo "nextcloudpi"; } | sort ))"
+
+if [[ -n "$incompatible" ]]
+then
+  if [[ " ${*} " =~ " "--allow-incompatible-apps" " ]]
+  then
+    echo "WARNING: The following apps are incompatible with the new Nextcloud version and will be disabled (update is running with --allow-incompatible-apps):"
+    echo "$incompatible"
+  else
+    echo "ERROR: Some installed apps are incompatible with the new Nextcloud version. Aborting update (run with --allow-incompatible-apps to ignore this)."
+    echo "Incompatible apps:"
+    echo "$incompatible"
+    exit 1
+  fi
+fi
+
 # backup
 ####################
 BKPDIR="$BASEDIR"
@@ -205,9 +226,7 @@ fi
 
 # use the correct version for custom apps
 NCVER="$(nc_version)"
-if is_more_recent_than "21.0.0" "${NCVER}"; then
-  NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc20
-else
+if ! is_more_recent_than "21.0.0" "${NCVER}"; then
   # Install notify_push if not installed
   if ! is_app_enabled notify_push; then
     ncc app:install notify_push
@@ -227,10 +246,7 @@ else
     }
 
   fi
-  NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc21
 fi
-rm -rf /var/www/nextcloud/apps/previewgenerator
-ln -snf "${NCPREV}" /var/www/nextcloud/apps/previewgenerator
 
 if ! is_more_recent_than "24.0.0" "${NCVER}" && is_more_recent_than "8.1.0" "${PHPVER}.0"
 then

bin/ncp/CONFIG/nc-init.sh

@@ -177,28 +177,26 @@ EOF
   # ncp-previewgenerator
   local ncver
   ncver="$(ncc status 2>/dev/null | grep "version:" | awk '{ print $3 }')"
-  if is_more_recent_than "21.0.0" "${ncver}"; then
-    local ncprev=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc20
-  else
+  if ! is_more_recent_than "21.0.0" "${ncver}"; then
     ncc app:install notify_push
     ncc app:enable  notify_push
     test -f /.ncp-image || start_notify_push # don't start during build
-    local ncprev=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc21
   fi
-  ln -snf "${ncprev}" /var/www/nextcloud/apps/previewgenerator
-  chown -R www-data: /var/www/nextcloud/apps/previewgenerator
-  ncc app:enable previewgenerator
 
   # previews
-  ncc config:app:set previewgenerator squareSizes --value="32 256"
-  ncc config:app:set previewgenerator widthSizes  --value="256 384"
-  ncc config:app:set previewgenerator heightSizes --value="256"
+
+  ncc app:install previewgenerator
+  ncc app:enable previewgenerator
+  ncc config:app:set --value="64 256" previewgenerator squareSizes
+  ncc config:app:set --value="256 4096" previewgenerator fillWidthHeightSizes
+  ncc config:app:set --value="64 256 1024" previewgenerator widthSizes
+  ncc config:app:set --value="64 256 1024" previewgenerator heightSizes
   ncc config:system:set preview_max_x --value 2048
   ncc config:system:set preview_max_y --value 2048
   ncc config:system:set jpeg_quality --value 60
-  ncc config:app:set preview jpeg_quality --value="60"
 
   # other
+  ncc config:system:set serverid --value="$((RANDOM % 1024))" --type=integer
   ncc config:system:set overwriteprotocol --value=https
   ncc config:system:set overwrite.cli.url --value="https://nextcloudpi/"
 

bin/ncp/CONFIG/nc-previews-auto.sh

@@ -8,48 +8,33 @@
 
 isactive()
 {
-  [[ -f "/etc/cron.d/nc-previews-auto" ]]
+  [[ -f "/etc/cron.hourly/ncp-previewgenerator" ]]
 }
 
 configure()
 {
+  # Disable during build
+  ! [[ -f /.ncp-image ]] || return 0
+
   [[ "$ACTIVE" != "yes" ]] && {
-    rm -f /etc/cron.d/nc-previews-auto
-    service cron restart
+    rm -f "/etc/cron.hourly/ncp-previewgenerator"
     echo "Automatic preview generation disabled"
     return 0
   }
 
-  grep -qP "^\d+$" <<<"$RUNTIME" || { echo "Invalid RUNTIME value $RUNTIME"; return 1; }
-  RUNTIME=$((RUNTIME*60))
-
-  echo "0  2  *  *  *  root  /usr/local/bin/nc-previews" >  /etc/cron.d/nc-previews-auto
-  chmod 644 /etc/cron.d/nc-previews-auto
-
-  cat > /usr/local/bin/nc-previews <<EOF
-#!/bin/bash
-echo -e "\n[ nc-previews-auto ]" >> /var/log/ncp.log
-(
-    for i in \$(seq 1 \$(nproc)); do
-      ionice -c3 nice -n20 /usr/local/bin/ncc preview:pre-generate -n -vvv &
-    done
-    wait
-) 2>&1 >>/var/log/ncp.log &
-
-PID=\$!
-[[ "$RUNTIME" != 0 ]] && {
-  for i in \$(seq 1 1 $RUNTIME); do
-    sleep 1
-    kill -0 "\$PID" &>/dev/null || break
-  done
-  pkill -f preview:pre-generate
-  pkill -f preview:generate-all
-}
-wait "\$PID"
-EOF
-chmod +x /usr/local/bin/nc-previews
-
-  service cron restart
+  ncc app:getpath previewgenerator > /dev/null || ncc app:install previewgenerator
+  is_app_enabled previewgenerator || ncc app:enable previewgenerator
+  ncc config:app:set --value="64 256" previewgenerator squareSizes
+  ncc config:app:set --value="256 4096" previewgenerator fillWidthHeightSizes
+  ncc config:app:set --value="64 256 1024" previewgenerator widthSizes
+  ncc config:app:set --value="64 256 1024" previewgenerator heightSizes
+  ncc config:app:set --value=false --type=boolean previewgenerator job_disabled
+  ncc config:app:set --value=3000 --type=integer previewgenerator job_max_execution_time
+  ncc config:app:set --value=0 --type=integer previewgenerator job_max_previews
+
+  mkdir -p /etc/cron.hourly
+  install_template cron.hourly/ncp-previewgenerator.sh /etc/cron.hourly/ncp-previewgenerator
+  chmod +x /etc/cron.hourly/ncp-previewgenerator
   echo "Automatic preview generation enabled"
   return 0
 }

bin/ncp/NETWORKING/nc-trusted-proxies.sh

@@ -7,11 +7,23 @@
 #
 #
 
+tmpl_trusted_proxies_list() {
+  (
+  . /usr/local/etc/library.sh
+    for param in PROXY{1,2,3}
+    do
+      proxy="$(find_app_param nc-trusted-proxies "$param")"
+      [[ -z "$proxy" ]] || echo "$proxy"
+    done
+  )
+}
+
 configure()
 {
   [[ "$PROXY1" != "" ]] && ncc config:system:set trusted_proxies 0 --value="$PROXY1"
   [[ "$PROXY2" != "" ]] && ncc config:system:set trusted_proxies 1 --value="$PROXY2"
   [[ "$PROXY3" != "" ]] && ncc config:system:set trusted_proxies 2 --value="$PROXY3"
+  install_template nextcloud.conf.sh /etc/apache2/sites-available/001-nextcloud.conf
 
   exit 0
 }

bin/ncp/TOOLS/nc-previews.sh

@@ -8,39 +8,111 @@
 # More at nextcloudpi.com
 #
 
+GENERATE_LOG="/var/log/ncp-generate-previews.log"
+GENERATE_JOB_ID="ncp-generate-previews"
+
+connect_to_preview_generation() {
+  tail -n 100 -f "${GENERATE_LOG}" &
+  tail_pid=$!
+  trap "kill '$tail_pid'" EXIT
+  while [[ "$(systemctl is-active "${GENERATE_JOB_ID}" ||:)" =~ ^(active|activating|deactivating)$ ]]
+  do
+    sleep 3
+  done
+
+  if [[ "$(systemctl is-active "${GENERATE_JOB_ID}" ||:)" == "inactive" ]]
+  then
+    echo "Preview generation finished successfully."
+    return 0
+  elif [[ "$(systemctl is-active "${GENERATE_JOB_ID}" ||:)" == "failed" ]]
+  then
+    echo "Preview generation failed (or was installed already)."
+    return 1
+  else
+    echo "Preview generation was not found or failed (unexpected status: '$(systemctl is-active "${GENERATE_JOB_ID}" ||:)')"
+  fi
+}
+
 
 configure()
 {
-  pgrep -af preview:pre-generate &>/dev/null || pgrep -af preview:generate-all &>/dev/null && {
-    echo "nc-previews is already running"
-    return 1
-  }
+  # Disable during build
+  ! [[ -f /.ncp-image ]] || return 0
+
+
+  if [[ "$(systemctl is-active "${GENERATE_JOB_ID}" ||:)" =~ ^(active|activating|deactivating)$ ]]
+  then
+    echo "Existing preview generation process detected. Connecting..."
+    connect_to_preview_generation
+    exit $?
+  fi
+
+  if ! [[ -f /.ncp-image ]]
+  then
+    ncc app:getpath previewgenerator > /dev/null || ncc app:install previewgenerator
+    is_app_enabled previewgenerator || ncc app:enable previewgenerator
+    ncc config:app:set --value="64 256" previewgenerator squareSizes
+    ncc config:app:set --value="256 4096" previewgenerator fillWidthHeightSizes
+    ncc config:app:set --value="64 256 1024" previewgenerator widthSizes
+    ncc config:app:set --value="64 256 1024" previewgenerator heightSizes
+    if is_app_enabled memories
+    then
+      ncc config:app:set --value="256 4096" previewgenerator coverWidthHeightSizes
+    else
+      ncc config:app:set --value="" previewgenerator coverWidthHeightSizes
+    fi
+  fi
+
+  tmpscript="$(mktemp /run/ncp-preview-generate.XXXXXX)"
+
+  PROC="$(nproc)"
+  if [[ "$PROC" -gt 3 ]]
+  then
+    PROC="$((PROC-2))"
+  else
+    PROC=1
+  fi
 
   [[ "$CLEAN" == "yes" ]] && {
-    local datadir
-    datadir=$( get_nc_config_value datadirectory ) || {
-      echo "data directory not found";
-      return 1;
-    }
-
-    rm -r "$datadir"/appdata_*/preview/* &>/dev/null
-    mysql nextcloud <<<"delete from ${DB_PREFIX?}filecache where path like \"appdata_%/preview/%\""
-    ncc files:scan-app-data -n
+    if ! is_more_recent_than "$(nc_version)" 30.99.99
+    then
+      echo "ERROR: CLEAN not supported for Nextcloud < 31 (was $(nc_version))"
+      return
+    else
+      echo 'echo "Cleaning old previews. This can take a while ..."' >> "$tmpscript"
+      echo 'ncc preview:cleanup' >> "$tmpscript"
+    fi
   }
 
   [[ "$INCREMENTAL" == "yes" ]] && {
-    for i in $(seq 1 $(nproc)); do
-      ncc preview:pre-generate -n -vvv &
-    done
-    wait
+    cat <<EOF >> "$tmpscript"
+for _ in $PROC; do
+  ncc preview:pre-generate -n -vvv &
+done
+wait
+EOF
     return
   }
 
-  for i in $(seq 1 $(nproc)); do
-    [[ "$PATH1" != "" ]] && PATH_ARG=(-p "$PATH1")
-    ncc preview:generate-all -n -v ${PATH_ARG[@]} &
-  done
-  wait
+  [[ "$PATH1" != "" ]] && PATH_ARG=(-p "$PATH1")
+  echo "ncc preview:generate-all -w \"${PROC}\" -n -vv " "${PATH_ARG[@]}" >> "$tmpscript"
+
+  systemctl reset-failed "${GENERATE_JOB_ID}" 2>/dev/null ||:
+  systemd-run -u "${GENERATE_JOB_ID}" --service-type=oneshot --no-block -p TimeoutStartSec="72h" -p TimeoutStopSec="1h" \
+    bash -c "bash '$tmpscript' |& tee '$GENERATE_LOG'"
+  sleep 1
+
+  if ! [[ "$(systemctl is-active "${GENERATE_JOB_ID}" ||:)" =~ ^(active|inactive|activating|deactivating)$ ]]
+  then
+    echo "Failed to start preview generation job"
+    [[ -f "${GENERATE_LOG}" ]] && cat "${GENERATE_LOG}"
+    systemctl status --no-pager "${GENERATE_JOB_ID}" ||:
+    exit 1
+  fi
+
+  echo "Preview generation started. You can safely close this session, the job will keep running in the background."
+
+  [[ "${PREVIEW_GENERATION_DETACH:-false}" == "true" ]] || connect_to_preview_generation
 }
 
 install() { :; }