Merge branch 'next' into newarchitecture

.github/workflows/deploy.yml

@@ -48,13 +48,13 @@ jobs:
         uses: actions/checkout@v6
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           push: true
           file: ./Dockerfile-core

.github/workflows/docker-image.yml

@@ -26,13 +26,13 @@ jobs:
           echo ${{ steps.vars.outputs.tag }}
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           context: .
           push: true
@@ -57,13 +57,13 @@ jobs:
           echo ${{ steps.vars.outputs.tag }}
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           push: true
           file: ./Dockerfile-api
@@ -88,13 +88,13 @@ jobs:
           echo ${{ steps.vars.outputs.tag }}
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           push: true
           file: ./Dockerfile-frontend
@@ -119,13 +119,13 @@ jobs:
           echo ${{ steps.vars.outputs.tag }}
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
           push: true
           file: ./Dockerfile-core

.github/workflows/release_notice.yml

@@ -14,7 +14,7 @@ jobs:
         run: echo "$GITHUB_CONTEXT"
       - name: Send custom JSON data to Slack workflow
         id: slack
-        uses: slackapi/slack-github-action@v2.1.1
+        uses: slackapi/slack-github-action@v3.0.1
         with:
           # This data can be any valid JSON from a previous step in the GitHub Action
           webhook: ${{ secrets.SLACK_RELEASE }}

CHANGELOG.md

@@ -1,3 +1,116 @@
+## Version 25.03.X
+Enterprise Fixes:
+- [flow] Optimize timeline period query
+
+Dependencies:
+- Bump follow-redirects from 1.15.11 to 1.16.0
+- Bump get-random-values from 4.1.1 to 4.1.2
+- Revert @vitejs/plugin-legacy from 8.0.1 to 7.2.1
+
+## Version 25.03.42
+Fixes:
+- [alerts] Fixed alert jobs using system's timezone instead of application's
+- [core] Fixed duplicate conditional in form field template
+
+Enterprise Fixes:
+- [data-manager] Fix notification message after editing user property
+- [white-labeling] Update newsletter setting description
+
+Dependencies:
+- Bump @vitejs/plugin-legacy from 7.2.1 to 8.0.1
+- Bump ejs from 5.0.1 to 5.0.2
+- Bump node-forge from 1.3.3 to 1.4.0 in /plugins/push
+
+## Version 25.03.41
+Fixes:
+- [push] Fix: Cannot create a push notification when configuration _id is a string
+- [star-rating] Fixed widget asset path with subdirectory
+
+Enterprise Fixes:
+- [journeys] Fix: prevent users entered stat to minus value for race conditions
+- [surveys] Fixed widget asset path with subdirectory
+- [journey-engin] Added new image handling mechanism for modal and half-modal content blocks
+
+Dependencies:
+- Bump axios from 1.13.5 to 1.15.0
+- Bump basic-ftp from 5.2.0 to 5.2.1
+- Bump brace-expansion from 1.1.12 to 1.1.13
+- Bump brace-expansion from 2.0.2 to 2.0.3 in /plugins/hooks
+- Bump cypress from 15.13.0 to 15.13.1 in /ui-tests
+- Bump docker/login-action from 4.0.0 to 4.1.0 in the actions group
+- Bump file-type and jimp
+- Bump lodash from 4.17.23 to 4.18.1
+- Bump nodemailer from 8.0.2 to 8.0.5
+- Bump path-to-regexp from 0.1.12 to 0.1.13
+- Bump pdfjs-dist from 5.5.207 to 5.6.205 in /ui-tests
+- Bump picomatch from 4.0.3 to 4.0.4
+- Bump rate-limiter-flexible from 9.1.1 to 11.0.0
+- Bump sass from 1.98.0 to 1.99.0
+- Bump sass-embedded from 1.98.0 to 1.99.0
+- Bump swiper from 12.1.2 to 12.1.3
+- Bump vite from 7.3.1 to 7.3.2
+
+## Version 25.03.40
+Fixes:
+- [hooks] Implement domain/ip address validation for hooks with http effect
+- [reports] Hardcoded default secret for the e-mails converted to be randomly generated
+
+Enterprise fixes:
+- [drill] Hide redacted user properties in filters
+- [oidc] Using sub as fallback as user identifier when there's no email
+
+Dependencies:
+- Bump countly-sdk-web from 26.1.0 to 26.1.1
+- Bump ejs from 4.0.1 to 5.0.1
+- Bump express-rate-limit from 8.3.0 to 8.3.1
+- Bump fast-xml-parser from 5.4.1 to 5.5.7 in /plugins/push
+- Bump flatted from 3.3.4 to 3.4.2
+- Bump moment-timezone from 0.6.0 to 0.6.1 in /bin/scripts/timezones
+- Bump nodemailer from 8.0.1 to 8.0.2
+- Bump puppeteer from 24.38.0 to 24.39.0
+- Bump sass from 1.97.3 to 1.98.0
+
+## Version 25.03.39
+Fixes:
+- [core] Fixed replaceDatabaseString incorrectly replacing "countly" in the MongoDB username when it appears in the connection URL
+- [dashboards] Unescape event segment values in meta
+- [push] Using Android specific content for Huawei messages as well
+
+Enterprise fixes:
+- [data-manager] Fix validation approval button label
+- [data-manager] Fix validation table column names
+
+## Version 25.03.38
+Fixes:
+- [push] Fixed the property name by changing it from link to url for message buttons for Huawei messages
+- [web] Use Client Hints
+
+## Version 25.03.37
+Fixes:
+- [core] Update home page download notification text
+- [data-manager] Add search and checkboxes in event selector when creating event group
+
+Enterprise fixes:
+- [funnels] Use lsid in same session funnel calculation
+- [users] Export drill data on user export
+- [users] Fix export query when there is profile group filter
+
+Dependencies:
+- Bump @faker-js/faker from 10.2.0 to 10.3.0 in /ui-tests
+- Bump axios from 1.13.2 to 1.13.5
+- Bump basic-ftp from 5.1.0 to 5.2.0
+- Bump cypress from 15.10.0 to 15.11.0 in /ui-tests
+- Bump fast-xml-parser and @google-cloud/storage in /plugins/push
+- Bump geoip-lite from 1.4.10 to 2.0.0
+- Bump minimatch from 9.0.5 to 9.0.9 in /api/utils/countly-request
+- Bump minimatch from 9.0.5 to 9.0.9 in /plugins/hooks
+- Bump nodemailer from 7.0.13 to 8.0.1
+- Bump qs from 6.14.1 to 6.14.2
+- Bump rollup from 4.56.0 to 4.59.0 in /plugins/journey_engine
+- Bump swiper from 12.0.3 to 12.1.2 in /plugins/content
+- Bump systeminformation from 5.30.5 to 5.31.1 in /ui-tests
+- Bump tar-stream from 3.1.7 to 3.1.8 in /plugins/system-utility
+
 ## Version 25.03.36
 Enterprise fixes:
 - [journey] Workflow fixes
@@ -37,7 +150,7 @@ Enterprise Fixes:
 ## Version 25.03.33
 Fixes:
 - [compliance-hub] Correctly merge user history on user merge
-- [core] Ensured usage of local fonts. 
+- [core] Ensured usage of local fonts.
 - [events] Fixed issue with page reloading on error
 - [push] re-schedule on a message update for auto triggers as well
 - [remote-config] Fix searching for parameter in experiment variants

Dockerfile-api

@@ -54,7 +54,7 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo
 	# preinstall
 	cp -n ./api/config.sample.js ./api/config.js && \
 	cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \
-	HOME=/tmp npm install -g npm@latest && \
+	HOME=/tmp npm install -g npm@10 && \
 	HOME=/tmp npm install --unsafe-perm=true --allow-root && \
 	HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \
 	./bin/docker/preinstall.sh && \

Dockerfile-centos-api

@@ -50,15 +50,17 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo
 	yum group install -y "Development Tools" && \
 	yum install -y epel-release && \
 	yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \
-	yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \
-	yum install -y wget openssl-devel make git unzip bzip2 && \
+	curl -L -O -J "https://box.tools.count.ly/public.php/dav/files/Wj8opzNdyE5DyDX/?accept=zip" && \
+	yum install -y raven-release.el8.noarch.rpm && \
+	yum install -y ipa-gothic-fonts && \
+	yum install -y wget openssl-devel make git libsqlite* sqlite unzip bzip2 && \
 	# modify standard distribution
 	./bin/docker/modify.sh && \
 	\
 	# preinstall
 	cp -n ./api/config.sample.js ./api/config.js && \
 	cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \
-	HOME=/tmp npm install -g npm@latest && \
+	HOME=/tmp npm install -g npm@10 && \
 	HOME=/tmp npm install --unsafe-perm=true --allow-root && \
 	HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \
 	./bin/docker/preinstall.sh && \

Dockerfile-centos-frontend

@@ -48,16 +48,18 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo
         yum group install -y "Development Tools" && \
         yum install -y epel-release && \
         yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \
-	      yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \
-        yum install -y wget openssl-devel make git unzip bzip2 && \
+        curl -L -O -J "https://box.tools.count.ly/public.php/dav/files/Wj8opzNdyE5DyDX/?accept=zip" && \
+        yum install -y raven-release.el8.noarch.rpm && \
+        yum install -y ipa-gothic-fonts && \
+        yum install -y wget openssl-devel make git sqlite libsqlite* unzip bzip2 && \
         # modify standard distribution
         ./bin/docker/modify.sh && \
         \
 	# preinstall
 	cp -n ./frontend/express/public/javascripts/countly/countly.config.sample.js ./frontend/express/public/javascripts/countly/countly.config.js && \
 	cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \
 	cp -n ./api/config.sample.js ./api/config.js && \
-	HOME=/tmp npm install -g npm@latest && \
+	HOME=/tmp npm install -g npm@10 && \
 	HOME=/tmp npm install --unsafe-perm=true --allow-root && \
 	HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \
 	./bin/docker/preinstall.sh && \

Dockerfile-core

@@ -27,7 +27,10 @@ USER root
 WORKDIR /opt/countly
 COPY . .
 
-RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \
+RUN sed -i 's|http://|https://|g' /etc/apt/sources.list && \
+    find /etc/apt/sources.list.d -name '*.list' -exec sed -i 's|http://|https://|g' {} + || true && \
+    echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries && \
+    useradd -r -M -U -d /opt/countly -s /bin/false countly && \
     apt-get update && \
     apt-get install -y \
         # standard

Dockerfile-frontend

@@ -52,7 +52,7 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo
 	cp -n ./api/config.sample.js ./api/config.js && \
 	cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \
 	cp -n ./frontend/express/public/javascripts/countly/countly.config.sample.js ./frontend/express/public/javascripts/countly/countly.config.js && \
-	HOME=/tmp npm install -g npm@latest && \
+	HOME=/tmp npm install -g npm@10 && \
 	HOME=/tmp npm install --unsafe-perm=true --allow-root && \
 	HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \
 	./bin/docker/preinstall.sh && \

api/parts/data/exports.ts

@@ -287,7 +287,20 @@ function output(params: Params, data: unknown, filename: string, type: string):
 
     if (type === 'xlsx' || type === 'xls') {
         params.res.writeHead?.(200, headers);
-        (data as { pipe: (res: unknown) => void }).pipe(params.res);
+        const outputStream = data as {
+            on?: (event: string, callback: (err: Error) => void) => void;
+            pipe: (res: unknown) => void;
+        };
+        outputStream.on?.('error', function(streamErr: Error) {
+            common.log('exports').e(streamErr);
+            if (!params.res.headersSent) {
+                common.returnMessage(params, 500, 'Export stream error');
+            }
+            else {
+                params.res.end();
+            }
+        });
+        outputStream.pipe(params.res);
     }
     else {
         common.returnRaw(params, 200, data, headers);
@@ -490,6 +503,15 @@ function streamExport(
     }
     else if (type === 'xlsx' || type === 'xls') {
         const xc = new XLSXTransformStream();
+        xc.on('error', function(streamErr: Error) {
+            common.log('exports').e(streamErr);
+            if (!params.res.headersSent) {
+                common.returnMessage(params, 500, 'Export stream error');
+            }
+            else {
+                params.res.end();
+            }
+        });
         xc.pipe(params.res);
         if (listAtEnd === false) {
             xc.write(paramList);

api/parts/mgmt/apps.js

@@ -857,6 +857,10 @@ function deleteAllAppData(appId, fromAppDelete, params, app) {
     }
     common.db.collection('app_users' + appId).drop(function() {
         if (!fromAppDelete) {
+            common.db.collection('metric_changes' + appId).drop(function() {
+                common.db.collection('metric_changes' + appId).ensureIndex({ts: 1, "cc.o": 1}, { background: true }, function() {});
+                common.db.collection('metric_changes' + appId).ensureIndex({uid: 1}, { background: true }, function() {});
+            });
             //Removes old app_user_merges collection
             common.db.collection('app_user_merges' + appId).drop(function() {});
             if (params.qstring.args.period === "reset") {