Merge branch 'master' into rating-consent

Author: widatama

.github/workflows/main.yml

@@ -242,91 +242,13 @@ jobs:
           /sbin/my_init &
           node bin/scripts/test.connection.js
           npx grunt mochaTest       
-  ui-test-dashboard:
-    runs-on: ubuntu-latest
-
-    services:
-      mongodb:
-        image: mongo:8.0
-        options: >-
-          --health-cmd mongosh
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 27017:27017
-
-    container:
-      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
-      env:
-        COUNTLY_CONFIG__MONGODB_HOST: mongodb
-        COUNTLY_CONFIG_API_PREVENT_JOBS: true
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Chrome
-        shell: bash
-        run: |
-          apt update
-          apt install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb wget
-          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -O /tmp/chrome.deb
-          apt install -y /tmp/chrome.deb
-
-      - name: Copy code
-        shell: bash
-        run: cp -rf ./* /opt/countly
-
-      - name: Prepare files to use correct MongoDB host
-        shell: bash
-        run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
-
-      - name: NPM install
-        shell: bash
-        working-directory: /opt/countly
-        run: npm install
-
-      - name: Prepare environment
-        shell: bash
-        working-directory: /opt/countly
-        run: |
-          sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js
-          cp "./plugins/plugins.default.json" "/opt/countly/plugins/plugins.json"
-          npm install
-          sudo countly task dist-all
-          bash bin/backup/import_events.sh
-          bash bin/scripts/countly.prepare.ce.tests.sh
-          countly add_user '${{ secrets.CYPRESS_USER_EMAIL }}' '${{ secrets.CYPRESS_USER_PASSWORD }}'
-          mongosh --host mongodb --eval 'db.getSiblingDB("countly").members.updateOne({username: "${{ secrets.CYPRESS_USER_EMAIL }}"}, {$set: {username: "${{ secrets.CYPRESS_USER_USERNAME }}", subscribe_newsletter: true}});'
-          mongosh --host mongodb --eval 'db.getSiblingDB("countly").plugins.updateOne({_id: "plugins"}, {$set: {"frontend.countly_tracking": true}});'
-          cd ui-tests
-          echo '{"username": "${{ secrets.CYPRESS_USER_USERNAME }}","email": "${{ secrets.CYPRESS_USER_EMAIL }}","password": "${{ secrets.CYPRESS_USER_PASSWORD }}"}' > cypress/fixtures/user.json
-          sed -i 's/00000000-0000-0000-0000-000000000000/${{ secrets.CYPRESS_KEY }}/g' package.json
-          cp cypress.config.sample.js cypress.config.js
-          sed -i 's/000000/${{ secrets.CYPRESS_PROJECT_ID }}/g' cypress.config.js
 
-      - name: Run UI tests
-        shell: bash
-        working-directory: /opt/countly
-        run: |
-          /sbin/my_init &
-          cd ui-tests
-          npm install
-          xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
-          npm run cy:run:dashboard
+  ui-tests:
+    runs-on: ubuntu-22.04-4core
 
-      - name: Upload UI tests artifacts
-        if: ${{ failure() }}
-        shell: bash
-        working-directory: /opt/countly/ui-tests/cypress
-        run: |
-          ARTIFACT_ARCHIVE_NAME="$(date '+%Y%m%d-%H.%M')_${GITHUB_REPOSITORY#*/}_CI#${{ github.run_number }}.tar.gz"
-          mkdir -p screenshots videos
-          tar zcvf "$ARTIFACT_ARCHIVE_NAME" screenshots videos
-          curl -o /tmp/uploader.log -u "${{ secrets.BOX_UPLOAD_AUTH }}" ${{ secrets.BOX_UPLOAD_PATH }} -T "$ARTIFACT_ARCHIVE_NAME"
-
-  ui-test-onboarding:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test_type: [dashboard, onboarding]
 
     services:
       mongodb:
@@ -377,7 +299,13 @@ jobs:
           cp "./plugins/plugins.default.json" "/opt/countly/plugins/plugins.json"
           npm install
           sudo countly task dist-all
+          bash bin/backup/import_events.sh
           bash bin/scripts/countly.prepare.ce.tests.sh
+          if [ "${{ matrix.test_type }}" = "dashboard" ]; then
+            countly add_user '${{ secrets.CYPRESS_USER_EMAIL }}' '${{ secrets.CYPRESS_USER_PASSWORD }}'
+            mongosh --host mongodb --eval 'db.getSiblingDB("countly").members.updateOne({username: "${{ secrets.CYPRESS_USER_EMAIL }}"}, {$set: {username: "${{ secrets.CYPRESS_USER_USERNAME }}", subscribe_newsletter: true}});'
+            mongosh --host mongodb --eval 'db.getSiblingDB("countly").plugins.updateOne({_id: "plugins"}, {$set: {"frontend.countly_tracking": true}});'
+          fi
           cd ui-tests
           echo '{"username": "${{ secrets.CYPRESS_USER_USERNAME }}","email": "${{ secrets.CYPRESS_USER_EMAIL }}","password": "${{ secrets.CYPRESS_USER_PASSWORD }}"}' > cypress/fixtures/user.json
           sed -i 's/00000000-0000-0000-0000-000000000000/${{ secrets.CYPRESS_KEY }}/g' package.json
@@ -391,15 +319,18 @@ jobs:
           /sbin/my_init &
           cd ui-tests
           npm install
-          xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
-          npm run cy:run:onboarding
+          if [ "${{ matrix.test_type }}" = "dashboard" ]; then
+            xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" npm run cy:run:dashboard
+          else
+            xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" npm run cy:run:onboarding
+          fi
 
       - name: Upload UI tests artifacts
         if: ${{ failure() }}
         shell: bash
         working-directory: /opt/countly/ui-tests/cypress
         run: |
-          ARTIFACT_ARCHIVE_NAME="$(date '+%Y%m%d-%H.%M')_${GITHUB_REPOSITORY#*/}_CI#${{ github.run_number }}.tar.gz"
+          ARTIFACT_ARCHIVE_NAME="$(date '+%Y%m%d-%H.%M')_${GITHUB_REPOSITORY#*/}_CI#${{ github.run_number }}_${{ matrix.test_type }}.tar.gz"
           mkdir -p screenshots videos
           tar zcvf "$ARTIFACT_ARCHIVE_NAME" screenshots videos
           curl -o /tmp/uploader.log -u "${{ secrets.BOX_UPLOAD_AUTH }}" ${{ secrets.BOX_UPLOAD_PATH }} -T "$ARTIFACT_ARCHIVE_NAME"

CHANGELOG.md

@@ -2,6 +2,10 @@
 Fixes:
 - [star-rating] Fix consent text limit counter
 
+Enterprise Fixes:
+- [active-users] Fixed bug related to selecting calculation ranges. As a result, some dates were previously calculated on incomplete data set.
+
+
 ## Version 25.03.6
 Enterprise Features:
 - [journey_engine] Publish toggle is converted into button and rearranged
@@ -4736,4 +4740,4 @@ This version provides several features and bugfixes to both server and SDKs. The
     admin or user. An admin of an application can edit application settings. 
     A user of an application can only view analytics for that application 
     and cannot edit its settings.
-  * Added csfr protection to all methods provided through app.js.
+  * Added csfr protection to all methods provided through app.js.

api/utils/countly-request/package-lock.json

@@ -29,8 +29,6 @@ pluginManager.dbConnection().then((countlyDb) => {
                 () => countlyDb.collection('app_users' + app._id).ensureIndex({"tsd": 1}, { background: true }, cb),
                 () => countlyDb.collection('app_users' + app._id).ensureIndex({"did": 1}, { background: true }, cb),
                 () => countlyDb.collection('app_users' + app._id).dropIndex("lac_1_ls_1", cb),
-                () => countlyDb.collection('app_user_merges' + app._id).ensureIndex({cd: 1}, {expireAfterSeconds: 60 * 60 * 3, background: true}, cb),
-                () => countlyDb.collection('metric_changes' + app._id).ensureIndex({ts: -1}, { background: true }, cb),
                 () => countlyDb.collection('metric_changes' + app._id).ensureIndex({ts: 1, "cc.o": 1}, { background: true }, cb),
                 () => countlyDb.collection('metric_changes' + app._id).ensureIndex({uid: 1}, { background: true }, cb)
             ];

bin/scripts/add_indexes.js

@@ -0,0 +1,66 @@
+/**
+ * MongoDB script to delete all collections from a database
+ * that start with "drill_events" string, but excluding the exact "drill_events" collection
+ * 
+ * 
+ * Server: MongoDB
+ * Path: any
+ * Command: mongosh -u uname -p 'password' --authenticationDatabase admin delete_old_drill_events.js
+ */
+
+/* global db, print, quit */
+
+// Set the database name
+const dbName = "countly_drill";
+
+// Set to true for dry run, false to actually delete collections
+const dryRun = true;
+
+print(`Using database: ${dbName}`);
+var cly = db;
+// If we need to switch to a different database
+if (db.getName() !== dbName) {
+    cly = db.getSiblingDB(dbName);
+}
+
+console.log(`Operating on database: ${dbName}`);
+
+// Get all collection names that start with "drill_events" but are not exactly "drill_events"
+const collections = cly.getCollectionInfos().map(info => info.name).filter(collName =>
+    collName.startsWith("drill_events") && collName !== "drill_events"
+);
+
+// Print the collections that will be deleted
+print("Collections to be deleted:");
+collections.forEach(collName => print(`- ${collName}`));
+
+// Ask for confirmation
+if (collections.length === 0) {
+    print("No matching collections found to delete.");
+    quit();
+}
+
+print(`\nFound ${collections.length} collections to delete.`);
+
+
+if (dryRun) {
+    print("Dry run mode is enabled. No collections will be deleted.");
+    print("To proceed with deletion, set 'dryRun' to false in the script.");
+    quit();
+}
+
+// Delete each collection
+let deletedCount = 0;
+collections.forEach(collName => {
+    try {
+        cly[collName].drop();
+        print(`Dropped collection: ${collName}`);
+        deletedCount++;
+    }
+    catch (error) {
+        print(`Error dropping collection ${collName}: ${error}`);
+    }
+});
+
+// Print summary
+print(`\nOperation completed. Deleted ${deletedCount} out of ${collections.length} collections.`);

bin/scripts/data-cleanup/delete_old_drill_events.js

@@ -2,13 +2,17 @@
 *  Sharding Countly collections when DB requires authentication, provide it to authDB.auth command in the code
 *  Server: mongodb
 *  Path: any
-*  Command: mongosh < sharding.js
+*  Command: mongosh -u uname -p 'password' --authenticationDatabase admin sharding.js
 */
 
-/* global Mongo, print, printjson */
-var COUNTLY_DRILL = 'countly_drill',
-    COUNTLY = 'countly',
-    COUNT_TO_SHARD = 100000;
+/* global db, print, printjson */
+
+// Set countly_drill database name
+const COUNTLY_DRILL = 'countly_drill';
+// Set countly database name
+const COUNTLY = 'countly';
+// Set the threshold for sharding collections
+const COUNT_TO_SHARD = 100000;
 
 var EXCEPTIONS = [
     /^system\./,
@@ -27,14 +31,8 @@ var COUNTLY_TO_SHARD = [
     "feedback",
 ];
 
-var conn = new Mongo(),
-    authDB = conn.getDB('admin');
-
-// need to update this info
-authDB.auth('<username>', '<password>');
-
-var cly = conn.getDB(COUNTLY),
-    drill = conn.getDB(COUNTLY_DRILL);
+var cly = db.getSiblingDB(COUNTLY),
+    drill = db.getSiblingDB(COUNTLY_DRILL);
 
 var clyCollections = cly.getCollectionNames(), collections = clyCollections.concat(drill.getCollectionNames()), check = [];
 
@@ -61,8 +59,7 @@ check.forEach(function(c) {
     var db = clyCollections.indexOf(c) === -1 ? drill : cly,
         dbName = clyCollections.indexOf(c) === -1 ? COUNTLY_DRILL : COUNTLY,
         count = db[c].count(),
-        capped = db[c].stats()['capped'],
-        status = db[c].getShardVersion().ok;
+        capped = db[c].stats()['capped'];
 
     COUNTLY_TO_SHARD.some((e) => {
         if (c.indexOf(e) == 0) {
@@ -71,7 +68,7 @@ check.forEach(function(c) {
         }
     });
 
-    if (!capped && count > COUNT_TO_SHARD && !status && !exceptional) {
+    if (!capped && count > COUNT_TO_SHARD && !exceptional) {
         print('Creating hashed index & enabling sharding for collection "' + c + '"... ');
 
         db.getCollection(c).createIndex({ _id: 'hashed' });