diff --git a/.codacy.yml b/.codacy.yml index 458d2f39617..766d982a8f6 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -7,11 +7,9 @@ exclude_paths: - 'frontend/express/public/localization/**' - 'frontend/express/public/fonts/**' - 'frontend/express/public/images/**' - - 'frontend/express/public/stylesheets/amaranjs/**' - 'frontend/express/public/stylesheets/font-awesome/**' - 'frontend/express/public/stylesheets/ionicons/**' - 'frontend/express/public/stylesheets/material/**' - - 'frontend/express/public/stylesheets/selectize/**' - 'bin/backup/**' - 'bin/upgrade/**' - 'bin/**' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e21a6099b11..932bf96340a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -161,7 +161,9 @@ jobs: - name: Copy code shell: bash - run: cp -rf ./* /opt/countly + run: | + cp -rf ./* /opt/countly + cp ./bin/docker/postinstall.sh /etc/my_init.d - name: Remove plugin tests shell: bash @@ -287,7 +289,12 @@ jobs: - name: Copy code shell: bash - run: cp -rf ./* /opt/countly + run: | + rm -rf /opt/countly/frontend + rm -rf /opt/countly/plugins/old-ui-compatibility + cp -rf ./* /opt/countly + cp /opt/countly/frontend/express/config.sample.js /opt/countly/frontend/express/config.js + cp /opt/countly/frontend/express/public/javascripts/countly/countly.config.sample.js /opt/countly/frontend/express/public/javascripts/countly/countly.config.js - name: Prepare files to use correct MongoDB host shell: bash @@ -341,3 +348,82 @@ jobs: mkdir -p screenshots videos downloads tar zcvf "$ARTIFACT_ARCHIVE_NAME" screenshots videos downloads curl -o /tmp/uploader.log -u "${{ secrets.BOX_UPLOAD_AUTH }}" ${{ secrets.BOX_UPLOAD_PATH }} -T "$ARTIFACT_ARCHIVE_NAME" + + ui-test-sdk: + runs-on: ubuntu-latest + + services: + mongodb: + image: mongo:8.0 + options: >- + --health-cmd mongosh + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 27017:27017 + + container: + image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }} + env: + COUNTLY_CONFIG__MONGODB_HOST: mongodb + COUNTLY_CONFIG_API_PREVENT_JOBS: true + + steps: + - uses: actions/checkout@v2 + + - name: Install Chrome + shell: bash + run: | + apt update + apt install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb wget + wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -O /tmp/chrome.deb + apt install -y /tmp/chrome.deb + + - name: Copy code + shell: bash + run: cp -rf ./* /opt/countly + + - name: Prepare files to use correct MongoDB host + shell: bash + run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh" + + - name: NPM install + shell: bash + working-directory: /opt/countly + run: npm install + + - name: Prepare environment + shell: bash + working-directory: /opt/countly + run: | + sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js + cp "./plugins/plugins.default.json" "/opt/countly/plugins/plugins.json" + npm install + sudo countly task dist-all + bash bin/scripts/countly.prepare.ce.tests.sh + cd ui-tests + echo '{"username": "${{ secrets.CYPRESS_USER_USERNAME }}","email": "${{ secrets.CYPRESS_USER_EMAIL }}","password": "${{ secrets.CYPRESS_USER_PASSWORD }}"}' > cypress/fixtures/user.json + sed -i 's/00000000-0000-0000-0000-000000000000/${{ secrets.CYPRESS_KEY }}/g' package.json + cp cypress.config.sample.js cypress.config.js + sed -i 's/000000/${{ secrets.CYPRESS_PROJECT_ID }}/g' cypress.config.js + + - name: Run UI tests + shell: bash + working-directory: /opt/countly + run: | + /sbin/my_init & + cd ui-tests + npm install + xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \ + npm run cy:run:sdk + + - name: Upload UI tests artifacts + if: ${{ failure() }} + shell: bash + working-directory: /opt/countly/ui-tests/cypress + run: | + ARTIFACT_ARCHIVE_NAME="$(date '+%Y%m%d-%H.%M')_${GITHUB_REPOSITORY#*/}_CI#${{ github.run_number }}_ui_test_sdk.tar.gz" + mkdir -p screenshots videos + tar zcvf "$ARTIFACT_ARCHIVE_NAME" screenshots videos + curl -o /tmp/uploader.log -u "${{ secrets.BOX_UPLOAD_AUTH }}" ${{ secrets.BOX_UPLOAD_PATH }} -T "$ARTIFACT_ARCHIVE_NAME" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f30ab32692..20733525b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,44 @@ -## Version 25.03.xx +## 25.xx +Dependencies: +- Remove SQLite + +## Version 25.03.28 +Fixes: +- [alerts] Add alert interval validation in the frontend +- [events] Correctly navigate to event groupmin events menu + +Enterprise Fixes: +- [applications] Ensure application management list reorders after create/update +- [concurrent_users] Fix email check for alert +- [dashboards] Keep dashboard sidebar sorted alphabetically after additions +- [data-manager] Correctly show last triggered for events if data masking is enabled + +## Version 25.03.27 +Fixes: +- [core-vis] Fix chart legend click event +- [push] Fixed the options of the request being made during mime detection +- [views] Fix view name that is displayed in view table +- [data-manager] Fix last modified data for event and segment + +Enterprise Fixes: +- [concurrent_users] Fix alert threshold comparison +- [dashboards] Add setting to disable public dashboards +- [surveys] Handle multiple survey submission from same user based on survey visibility +- [users] Display user property limits in user profiles when exceeded +- [users] Set correct users widget table rows amount according to selected setting + +## Version 25.03.26 Fixes: - [push] Fixed timeout setting - [security] Fixed injection possibility on res.expose Enterprise Fixes: +- [data-manager] Fixed bug when merging events with ampersand symbol in the name - [groups] Add logs for user updates - [nps] Sort widgets by internal name and search by name or internal name - [surveys] Change question map log to debug log - [surveys] Sort widgets by internal name and search by name or internal name -Enterprise Fixes: -- [data-manager] Fixed bug when merging events with ampersand symbol in the name - Dependencies: - Bump axios from 1.12.2 to 1.13.1 in /plugins/cognito - Bump csvtojson from 1.1.12 to 2.0.14 @@ -32,6 +59,7 @@ Fixes: Enterprise Fixes: - [ab-testing] Add script for fixing variant cohort - [groups] Fix user permission update after updating user group permission +- [funnels] Fixed delete confirmation using correct button copy ## Version 25.03.24 Fixes: @@ -64,7 +92,6 @@ Enterprise Fixes: - [users] Add survey section to user feedback page - [users] Fixed uploading user profile pictures - ## Version 25.03.22 Fixes: - [alerts] Fix: Migrate alerts to the new events model @@ -142,7 +169,6 @@ Enterprise Fixes: Dependencies: - Bump puppeteer from 24.16.2 to 24.17.0 - ## Version 25.03.16 Enterprise Fixes: - [journeys] Fix for skip threshold check in concurrent requests @@ -152,7 +178,6 @@ Dependencies: - Bump get-random-values from 3.0.0 to 4.0.0 - Bump puppeteer from 24.16.1 to 24.16.2 - ## Version 25.03.15 Enterprise Fixes: - [cohorts] Unescape segmentation properties options to prevent duplicated values @@ -186,7 +211,6 @@ Enterprise Fixes: - [flows] Showing correct state for disabled flows - [surveys] Move "not likely" label next to 0 on mobile screens - ## Version 25.03.12 Features: - [plugins] Add configuration warning tags to settings UI @@ -209,7 +233,6 @@ Dependencies: - Bump puppeteer from 24.14.0 to 24.15.0 - Bump supertest from 7.1.3 to 7.1.4 - ## Version 25.03.11 Fixes: - [core] Fix mongo connection url parsing @@ -305,7 +328,6 @@ Fixes: - [hooks] Added null check for incoming data - [push] Fix external drawer initialization - [times-of-day] Fix chart component - Enterprise Fixes: - [content] Asset URL was wrongly constructed when user switches between apps - [ab-testing] Updates diff --git a/Dockerfile-api b/Dockerfile-api index a5dfa82d30e..3ab8a02bdaa 100644 --- a/Dockerfile-api +++ b/Dockerfile-api @@ -49,7 +49,6 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo dpkg -i /tmp/tini.deb && \ \ # modify standard distribution - apt-get update && apt-get install -y sqlite3 && \ ./bin/docker/modify.sh && \ \ # preinstall diff --git a/Dockerfile-centos-api b/Dockerfile-centos-api index 0352a049c9c..336b3694644 100644 --- a/Dockerfile-centos-api +++ b/Dockerfile-centos-api @@ -51,7 +51,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo yum install -y epel-release && \ yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \ yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \ - yum install -y wget openssl-devel make git libsqlite* sqlite unzip bzip2 && \ + yum install -y wget openssl-devel make git unzip bzip2 && \ # modify standard distribution ./bin/docker/modify.sh && \ \ diff --git a/Dockerfile-centos-frontend b/Dockerfile-centos-frontend index 06ff48a1c79..1a0f5510ea4 100644 --- a/Dockerfile-centos-frontend +++ b/Dockerfile-centos-frontend @@ -49,7 +49,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo yum install -y epel-release && \ yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \ yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \ - yum install -y wget openssl-devel make git sqlite libsqlite* unzip bzip2 && \ + yum install -y wget openssl-devel make git unzip bzip2 && \ # modify standard distribution ./bin/docker/modify.sh && \ \ diff --git a/Dockerfile-core b/Dockerfile-core index b3a807d8b5c..43daa5ee45b 100644 --- a/Dockerfile-core +++ b/Dockerfile-core @@ -1,4 +1,4 @@ -FROM phusion/baseimage:focal-1.2.0 +FROM phusion/baseimage:jammy-1.0.4 ARG COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,logger,systemlogs,populator,reports,crashes,push,star-rating,slipping-away-users,compare,server-stats,dbviewer,times-of-day,compliance-hub,alerts,onboarding,consolidate,remote-config,hooks,dashboards,sdk,data-manager,guides # Countly Enterprise: @@ -29,7 +29,7 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \ apt-get update && \ apt-get install -y \ # standard - build-essential libkrb5-dev git sqlite3 wget sudo \ + build-essential libkrb5-dev git wget sudo \ # nginx nginx \ # puppeteer @@ -38,7 +38,7 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \ libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils \ # push / nghttp2 gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev \ - libevent-dev libjansson-dev libjemalloc-dev cython python3-dev python-setuptools && \ + libevent-dev libjansson-dev libjemalloc-dev python3-dev python-setuptools && \ # node wget -qO- https://deb.nodesource.com/setup_20.x | bash - && \ # data_migration (mongo clients) @@ -76,8 +76,8 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \ chown -R countly:countly /opt/countly && \ # cleanup npm remove -y --no-save mocha nyc should supertest && \ - apt-get remove -y build-essential libkrb5-dev sqlite3 wget \ - gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev libevent-dev libjansson-dev libjemalloc-dev cython python3-dev python-setuptools && \ + apt-get remove -y build-essential libkrb5-dev wget \ + gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev libevent-dev libjansson-dev libjemalloc-dev python3-dev python-setuptools && \ apt-get install -y gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* ~/.npm ~/.cache && \ diff --git a/Dockerfile-frontend b/Dockerfile-frontend index 063cea69afe..88aaefba51f 100644 --- a/Dockerfile-frontend +++ b/Dockerfile-frontend @@ -46,7 +46,6 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo dpkg -i /tmp/tini.deb && \ \ # modify standard distribution - apt-get update && apt-get install -y sqlite3 && \ ./bin/docker/modify.sh && \ \ # preinstall diff --git a/Gruntfile.js b/Gruntfile.js index 4c7c1a98986..3331af6f83f 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -14,21 +14,8 @@ module.exports = function(grunt) { dom: { src: [ 'frontend/express/public/javascripts/dom/jquery/jquery.js', - 'frontend/express/public/javascripts/dom/jquery.form.js', - 'frontend/express/public/javascripts/dom/tipsy/jquery.tipsy.js', - 'frontend/express/public/javascripts/dom/jquery.noisy.min.js', - 'frontend/express/public/javascripts/dom/jquery.sticky.headers.js', - 'frontend/express/public/javascripts/dom/jqueryui/jquery-ui.js', - 'frontend/express/public/javascripts/dom/jqueryui/jquery-ui-i18n.js', 'frontend/express/public/javascripts/dom/gridstack/gridstack-h5.js', - 'frontend/express/public/javascripts/dom/slimScroll.min.js', - 'frontend/express/public/javascripts/dom/jquery.easing.1.3.js', - 'frontend/express/public/javascripts/dom/dataTables/js/jquery.dataTables.js', - 'frontend/express/public/javascripts/dom/dataTables/js/ZeroClipboard.js', - 'frontend/express/public/javascripts/dom/dataTables/js/TableTools.js', 'frontend/express/public/javascripts/dom/pace/pace.min.js', - 'frontend/express/public/javascripts/dom/drop/tether.min.js', - 'frontend/express/public/javascripts/dom/drop/drop.min.js' ], dest: 'frontend/express/public/javascripts/min/countly.dom.concat.js' }, @@ -39,22 +26,13 @@ module.exports = function(grunt) { 'frontend/express/public/javascripts/utils/lodash.merge.js', 'frontend/express/public/javascripts/utils/prefixfree.min.js', 'frontend/express/public/javascripts/utils/moment/moment-with-locales.min.js', - 'frontend/express/public/javascripts/utils/handlebars.js', 'frontend/express/public/javascripts/utils/backbone-min.js', 'frontend/express/public/javascripts/utils/jquery.i18n.properties.js', - 'frontend/express/public/javascripts/utils/jstz.min.js', 'frontend/express/public/javascripts/utils/store+json2.min.js', 'frontend/express/public/javascripts/utils/jquery.idle-timer.js', - 'frontend/express/public/javascripts/utils/textcounter.min.js', 'frontend/express/public/javascripts/utils/initialAvatar.js', - 'frontend/express/public/javascripts/utils/jquery.amaran.min.js', - 'frontend/express/public/javascripts/utils/jquery.titlealert.js', - 'frontend/express/public/javascripts/utils/jquery.hoverIntent.minified.js', - 'frontend/express/public/javascripts/utils/tooltipster/tooltipster.bundle.min.js', 'frontend/express/public/javascripts/utils/highlight/highlight.pack.js', - 'frontend/express/public/javascripts/utils/dropzone.js', 'frontend/express/public/javascripts/utils/webfont.js', - 'frontend/express/public/javascripts/utils/selectize.min.js', 'frontend/express/public/javascripts/utils/leaflet.js', 'frontend/express/public/javascripts/utils/js-deep-equals.unsorted.min.js', 'frontend/express/public/javascripts/utils/polyfill/es6-promise.auto.min.js', @@ -77,16 +55,8 @@ module.exports = function(grunt) { 'frontend/express/public/javascripts/utils/vue/vue-json-pretty.min.js', 'frontend/express/public/javascripts/utils/jquery.xss.js', 'frontend/express/public/javascripts/countly/countly.common.js', - 'frontend/express/public/javascripts/utils/simpleUpload.min.js', - 'frontend/express/public/javascripts/utils/jsoneditor/codemirror.js', - 'frontend/express/public/javascripts/utils/jsoneditor/javascript.min.js', - 'frontend/express/public/javascripts/utils/jsoneditor/json2.js', - 'frontend/express/public/javascripts/utils/jsoneditor/jsonlint.js', - 'frontend/express/public/javascripts/utils/jsoneditor/minify.json.js', - 'frontend/express/public/javascripts/utils/jsoneditor/jsoneditor.js', 'frontend/express/public/javascripts/utils/Sortable.min.js', 'frontend/express/public/javascripts/utils/vue/vuedraggable.umd.min.js', - 'frontend/express/public/javascripts/utils/countly.checkbox.js', 'frontend/express/public/javascripts/utils/lodash.mergeWith.js', 'frontend/express/public/javascripts/utils/element-tiptap.umd.min.js' ], @@ -94,21 +64,6 @@ module.exports = function(grunt) { }, visualization: { src: [ - 'frontend/express/public/javascripts/visualization/jquery.peity.min.js', - 'frontend/express/public/javascripts/visualization/jquery.sparkline.js', - 'frontend/express/public/javascripts/visualization/flot/jquery.flot.js', - 'frontend/express/public/javascripts/visualization/flot/jquery.flot.tickrotor.js', - 'frontend/express/public/javascripts/visualization/flot/jquery.flot.pie.js', - 'frontend/express/public/javascripts/visualization/flot/jquery.flot.resize.js', - 'frontend/express/public/javascripts/visualization/flot/jquery.flot.stack.js', - 'frontend/express/public/javascripts/visualization/flot/jquery.flot.spline.js', - 'frontend/express/public/javascripts/visualization/flot/jquery.flot.crosshair.js', - 'frontend/express/public/javascripts/visualization/flot/jquery.flot.orderBars.js', - 'frontend/express/public/javascripts/visualization/flot/jquery.flot.navigate.js', - 'frontend/express/public/javascripts/visualization/gauge.min.js', - 'frontend/express/public/javascripts/visualization/d3/d3.min.js', - 'frontend/express/public/javascripts/visualization/rickshaw/rickshaw.min.js', - 'frontend/express/public/javascripts/visualization/rickshaw/rickshaw.x.axis.js' ], dest: 'frontend/express/public/javascripts/min/countly.visualization.concat.js' }, @@ -159,7 +114,6 @@ module.exports = function(grunt) { 'frontend/express/public/javascripts/countly/vue/components/progress.js', 'frontend/express/public/javascripts/countly/vue/directives/scroll-shadow.js', 'frontend/express/public/javascripts/countly/vue/legacy.js', - 'frontend/express/public/javascripts/countly/countly.vue.legacy.js', 'frontend/express/public/javascripts/countly/countly.token.manager.js', 'frontend/express/public/javascripts/countly/countly.version.history.js', 'frontend/express/public/javascripts/countly/countly.analytics.js', @@ -229,18 +183,10 @@ module.exports = function(grunt) { 'frontend/express/public/stylesheets/main.min.css': [ 'frontend/express/public/stylesheets/main.css', 'frontend/express/public/stylesheets/vue/clyvue.css', - 'frontend/express/public/stylesheets/vue/vue-json-pretty.css', - 'frontend/express/public/stylesheets/amaranjs/amaran.min.css', - 'frontend/express/public/stylesheets/selectize/selectize.css', 'frontend/express/public/stylesheets/leaflet/leaflet.css', - 'frontend/express/public/stylesheets/jsoneditor/codemirror.css', - 'frontend/express/public/stylesheets/countly-checkbox/countly.checkbox.css', - 'frontend/express/public/javascripts/dom/tipsy/tipsy.css', + 'frontend/express/public/stylesheets/vue/vue-json-pretty.css', 'frontend/express/public/javascripts/dom/gridstack/gridstack.css', - 'frontend/express/public/javascripts/visualization/rickshaw/rickshaw.min.css', 'frontend/express/public/javascripts/dom/pace/pace-theme-flash.css', - 'frontend/express/public/javascripts/dom/drop/drop-theme-countly.min.css', - 'frontend/express/public/javascripts/utils/tooltipster/tooltipster.bundle.min.css', 'frontend/express/public/stylesheets/bulma/bulma-custom.css', 'frontend/express/public/stylesheets/styles/manifest2.css', 'frontend/express/public/stylesheets/vue/element-tiptap.css', diff --git a/api/parts/data/fetch.js b/api/parts/data/fetch.js index 210c277571a..83ea1d8c974 100644 --- a/api/parts/data/fetch.js +++ b/api/parts/data/fetch.js @@ -137,6 +137,8 @@ fetch.fetchEventGroups = function(params) { fetch.fetchMergedEventGroups = function(params) { const { qstring: { event } } = params; fetch.getMergedEventGroups(params, event, {}, function(result) { + result = result || {}; + result.eventName = params.qstring.event; common.returnOutput(params, result); }); }; diff --git a/api/parts/data/usage.js b/api/parts/data/usage.js index 93fc8b978eb..2c6fbbc5090 100644 --- a/api/parts/data/usage.js +++ b/api/parts/data/usage.js @@ -7,7 +7,7 @@ var usage = {}, common = require('./../../utils/common.js'), geoip = require('geoip-lite'), - geocoder = require('offline-geocoder')(), + geocoder = require('./../../../bin/offline-geocoder/src/index.js')(), log = require('../../utils/log.js')('api:usage'), async = require('async'), plugins = require('../../../plugins/pluginManager.js'), diff --git a/api/parts/mgmt/event_groups.js b/api/parts/mgmt/event_groups.js index 39d1928d825..f736f01d117 100644 --- a/api/parts/mgmt/event_groups.js +++ b/api/parts/mgmt/event_groups.js @@ -39,6 +39,10 @@ const create = (params) => { 'type': 'Boolean' } }; + if (!params.qstring.args) { + common.returnMessage(params, 400, 'Error: args not found'); + return false; + } params.qstring.args = JSON.parse(params.qstring.args); const {obj, errors} = common.validateArgs(params.qstring.args, argProps, true); if (!obj) { @@ -59,7 +63,14 @@ const create = (params) => { /** * Event Groups CRUD - The function updating which created `Event Groups` data by `_id` - * @param {Object} params - + * @param {Object} params - params object containing the query string and other parameters + * @returns {Boolean} + * This function updates the event groups based on the provided parameters. + * It handles different update scenarios: + * 1. If `args` is provided, it updates the event group with the specified `_id`. + * 2. If `event_order` is provided, it updates the order of the events in the group. + * 3. If `update_status` is provided, it updates the status of the specified event groups. + * 4. If none of these parameters are found, it returns a 400 error indicating that the required arguments are not found. */ const update = (params) => { if (params.qstring.args) { @@ -72,7 +83,7 @@ const update = (params) => { common.returnMessage(params, 200, 'Success'); }); } - if (params.qstring.event_order) { + else if (params.qstring.event_order) { params.qstring.event_order = JSON.parse(params.qstring.event_order); var bulkArray = []; params.qstring.event_order.forEach(function(id, index) { @@ -91,7 +102,7 @@ const update = (params) => { common.returnMessage(params, 200, 'Success'); }); } - if (params.qstring.update_status) { + else if (params.qstring.update_status) { params.qstring.update_status = JSON.parse(params.qstring.update_status); params.qstring.status = JSON.parse(params.qstring.status); var idss = params.qstring.update_status; @@ -145,6 +156,10 @@ const update = (params) => { } ); } + else { + common.returnMessage(params, 400, 'Error: args not found'); + return false; + } }; /** @@ -152,6 +167,10 @@ const update = (params) => { * @param {Object} params - */ const remove = async(params) => { + if (!params.qstring.args) { + common.returnMessage(params, 400, 'Error: args not found'); + return false; + } params.qstring.args = JSON.parse(params.qstring.args); var idss = params.qstring.args; common.db.collection(COLLECTION_NAME).remove({_id: { $in: params.qstring.args }}, (error) =>{ diff --git a/bin/countly.install_rhel.sh b/bin/countly.install_rhel.sh index aa4df20be87..b0011d1203f 100644 --- a/bin/countly.install_rhel.sh +++ b/bin/countly.install_rhel.sh @@ -14,7 +14,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" bash "$DIR/scripts/logo.sh"; # prerequisite per release -sudo dnf install -y wget openssl-devel make git sqlite unzip bzip2 +sudo dnf install -y wget openssl-devel make git unzip bzip2 sudo dnf install -y python3-pip sudo pip3 install pip --upgrade @@ -87,7 +87,7 @@ sudo systemctl start sendmail > /dev/null || echo "sendmail service does not exi #install npm modules npm config set prefix "$DIR/../.local/" -( cd "$DIR/.."; npm install argon2; npm install sqlite3 --build-from-source; npm install; ) +( cd "$DIR/.."; npm install argon2; npm install; ) #install numactl sudo dnf install -y numactl @@ -156,6 +156,8 @@ node "$DIR/scripts/install_plugins" #load city data into database nodejs "$DIR/scripts/loadCitiesInDb.js" +nodejs "$DIR/offline-geocoder/scripts/download_geonames_data.js" +nodejs "$DIR/offline-geocoder/scripts/import_geonames_mongodb.js" #get web sdk sudo countly update sdk-web diff --git a/bin/countly.install_travis.sh b/bin/countly.install_travis.sh index 8668f2d3b4f..c1c11eb215a 100644 --- a/bin/countly.install_travis.sh +++ b/bin/countly.install_travis.sh @@ -19,7 +19,7 @@ apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 #update package index apt-get update -apt-get install -y build-essential git sqlite3 unzip shellcheck +apt-get install -y build-essential git unzip shellcheck if apt-cache pkgnames | grep -q python-software-properties; then apt-get install -y python-software-properties @@ -116,6 +116,8 @@ bash "$DIR/scripts/countly.install.plugins.sh" #load city data into database nodejs "$DIR/scripts/loadCitiesInDb.js" +nodejs "$DIR/offline-geocoder/scripts/download_geonames_data.js" +nodejs "$DIR/offline-geocoder/scripts/import_geonames_mongodb.js" #compile scripts for production countly task dist-all diff --git a/bin/countly.install_ubuntu.sh b/bin/countly.install_ubuntu.sh index 159ee61be5e..22f2c8175c7 100644 --- a/bin/countly.install_ubuntu.sh +++ b/bin/countly.install_ubuntu.sh @@ -16,7 +16,7 @@ bash "$DIR/scripts/logo.sh"; #update package index sudo apt-get update -sudo apt-get install -y wget build-essential libkrb5-dev git sqlite3 unzip bzip2 shellcheck curl gnupg2 ca-certificates lsb-release +sudo apt-get install -y wget build-essential libkrb5-dev git unzip bzip2 shellcheck curl gnupg2 ca-certificates lsb-release if [[ "$UBUNTU_YEAR" = "22" ]]; then sudo apt-get install -y python2 python2-dev @@ -93,7 +93,7 @@ sudo apt-get install -y sendmail #install npm modules npm config set prefix "$DIR/../.local/" -( cd "$DIR/.."; npm install argon2; npm install sqlite3 --build-from-source; npm install; ) +( cd "$DIR/.."; npm install argon2; npm install; ) #install mongodb if ! command -v mongod &> /dev/null; then @@ -158,6 +158,8 @@ bash "$DIR/scripts/countly.install.plugins.sh" #load city data into database nodejs "$DIR/scripts/loadCitiesInDb.js" +nodejs "$DIR/offline-geocoder/scripts/download_geonames_data.js" +nodejs "$DIR/offline-geocoder/scripts/import_geonames_mongodb.js" #get web sdk sudo countly update sdk-web diff --git a/bin/docker/modify.sh b/bin/docker/modify.sh index 7681a04b408..3a6c26f77fc 100755 --- a/bin/docker/modify.sh +++ b/bin/docker/modify.sh @@ -17,12 +17,26 @@ if [ "${COUNTLY_CONTAINER}" != "frontend" ]; then # Run ab-testing models compilation if it's there if [ -d /opt/countly/plugins/ab-testing ]; then if [ "${ID}" == "debian" ] || [ "${ID}" == "ubuntu" ]; then - apt-get install -y python3-pip + echo "Debian noninteractive" + export DEBIAN_FRONTEND=noninteractive + export TZ=Etc/UTC + + apt-get -y update + apt-get install -y software-properties-common build-essential python3-dev libncurses*-dev libsqlite3-dev libreadline6-dev libgdbm-dev zlib1g-dev libbz2-dev sqlite3 tk-dev zip libssl-dev libncurses5-dev liblzma-dev lsb-core lsb-release + + export LC_ALL="en_US.UTF-8" + export LC_CTYPE="en_US.UTF-8" + export -n CC + export -n CXX + add-apt-repository -y ppa:deadsnakes/ppa + apt -y install python3.12 python3.12-dev + curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py + python3.12 get-pip.py else yum install -y python36 python36-libs python36-devel python36-pip fi + ln -s /usr/bin/python3 /usr/bin/python # shellcheck disable=SC1091 - python3.8 -m pip install -r /opt/countly/plugins/ab-testing/api/bayesian/requirements.txt - cd /opt/countly/plugins/ab-testing/api/bayesian && python3.8 model.py + python3.12 -m pip install -r "/opt/countly/plugins/ab-testing/api/bayesian/requirements_docker.txt" && sudo python3.12 "/opt/countly/plugins/ab-testing/api/bayesian/models/cmdstanpy_model.py" fi fi diff --git a/bin/docker/postinstall.sh b/bin/docker/postinstall.sh index 22af77a8a10..0c3c4fa994b 100755 --- a/bin/docker/postinstall.sh +++ b/bin/docker/postinstall.sh @@ -20,4 +20,6 @@ else #load city data into database node "/opt/countly/bin/scripts/loadCitiesInDb.js" + node "/opt/countly/bin/offline-geocoder/scripts/download_geonames_data.js" + node "/opt/countly/bin/offline-geocoder/scripts/import_geonames_mongodb.js" fi diff --git a/bin/docker/preinstall.sh b/bin/docker/preinstall.sh index d4ffbc851ad..a22054638b8 100755 --- a/bin/docker/preinstall.sh +++ b/bin/docker/preinstall.sh @@ -15,6 +15,8 @@ plugins="${plugins::-1}]" node ./node_modules/geoip-lite/scripts/updatedb.js license_key="$GEOIP" +node "./bin/offline-geocoder/scripts/download_geonames_data.js" + echo "$plugins" > /opt/countly/plugins/plugins.json (cd /opt/countly && npx grunt dist-all && rm -rf /opt/countly/plugins/plugins.json) diff --git a/bin/offline-geocoder/LICENSE b/bin/offline-geocoder/LICENSE new file mode 100644 index 00000000000..7e66944210d --- /dev/null +++ b/bin/offline-geocoder/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2016 Luca Spiller + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bin/offline-geocoder/README.md b/bin/offline-geocoder/README.md new file mode 100644 index 00000000000..554bbcc6adf --- /dev/null +++ b/bin/offline-geocoder/README.md @@ -0,0 +1,84 @@ +# Offline Geocoder + +Node library for reverse geocoding. Designed to be used offline (for example +embedded in a desktop or mobile application) - no web requests are made to +perform a lookup. + +## Data + +This uses data from the [GeoNames project](http://www.geonames.org/), which is +free to use under the [Creative Commons Attribution 3.0 license](http://creativecommons.org/licenses/by/3.0/). +To enable this to work offline, the data is imported into a SQLite database +which is roughly 12 MB, so easily embeddable within an application. + +By default it uses the `cities1000` dataset which contains details of all +worldwide cities with a population of at least 1000 people. Depending on your +needs you may get better performance or accuracy by using one of their other +datasets. + +The GeoNames data is limited to city-level granularity, so if you need street +level accuracy this won't work for you. Also most data is only available in +English. Take a look at the +[OpenStreetMap Nominatim project](https://github.com/twain47/Nominatim) for a +similar tool with a lot more features. + +The advantages of this working offline are you don't need to pay or obtain a +license key, and it's fast. On my meager laptop I can perform around 300 +lookups per second with a single process. + +## Installation + +``` +npm install --save offline-geocoder +``` + +You also need to obtain a database which isn't included in the package, to +generate your own take a look in `scripts`. + +## Usage + +When you initialize the library you need to pass the location of the database: + +```javascript +const geocoder = require('offline-geocoder')({ database: 'data/geodata.db' }) +``` + +### Reverse Geocoding + +To perform a revese geocode lookup just pass the coordinates: + +```javascript +geocoder.reverse(41.89, 12.49) + .then(function(result) { + console.log(result) + }) + .catch(function(error) { + console.error(error) + }) +``` + +Which outputs: + +``` +{ id: 3169070, + name: 'Rome', + formatted: 'Rome, Latium, Italy', + country: { id: 'IT', name: 'Italy' }, + admin1: { id: 7, name: 'Latium' }, + coordinates: { latitude: 41.89193, longitude: 12.51133 } } +``` + +The library also has a callback interface: + +```javascript +geocoder.reverse(41.89, 12.49, function(error, result) { + console.log(result) +}) +``` + +## License + +This library is licensed under [the MIT license](https://github.com/lucaspiller/offline-geocoder/blob/master/LICENSE). + +You don't need to give this library attribution, but you must do so for +GeoNames if you use their data! diff --git a/bin/offline-geocoder/scripts/download_geonames_data.js b/bin/offline-geocoder/scripts/download_geonames_data.js new file mode 100644 index 00000000000..1ab270407c4 --- /dev/null +++ b/bin/offline-geocoder/scripts/download_geonames_data.js @@ -0,0 +1,212 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const path = require('path'); +const https = require('https'); +const { createWriteStream, promises: fsPromises } = require('fs'); +const yauzl = require('yauzl'); // Pure JavaScript unzip implementation + +// Get the project root directory (directory of the script) +const PROJECT_ROOT = path.resolve(__dirname, '..'); +const DATA_DIR = path.join(PROJECT_ROOT, "data"); +const DATA = "cities1000.txt"; +const ADMIN1 = "admin1CodesASCII.txt"; +const COUNTRIES = "countryInfo.txt"; + +// Helper function to download a file using Node.js +function downloadFile(url, destination) { + console.log(`Downloading ${url} to ${destination}...`); + return new Promise((resolve, reject) => { + const file = createWriteStream(destination); + + https.get(url, response => { + console.log(`Response status code: ${response.statusCode}`); + if (response.statusCode !== 200) { + console.log(`Failed to download ${url}: ${response.statusCode}`); + reject(new Error(`Failed to download ${url}: ${response.statusCode}`)); + return; + } + + response.pipe(file); + + file.on("error", err => { + console.log(`Error writing to file ${destination}:`, err); + fs.unlink(destination, () => {}); + reject(err); + }); + + file.on('finish', () => { + console.log(`Downloaded ${url} to ${destination}`); + file.close(); + resolve(); + }); + }).on('error', err => { + console.log(`Error downloading ${url}:`, err); + fs.unlink(destination, () => {}); + reject(err); + }); + + file.on('error', err => { + console.log(`Error writing to file ${destination}:`, err); + fs.unlink(destination, () => {}); + reject(err); + }); + }); +} + +// Helper function to unzip a file using yauzl (pure JS implementation) +async function unzipFile(zipFile, destination) { + console.log(`Unzipping ${zipFile} to ${destination}...`); + try { + // Read zip file into memory + const zipBuffer = await fsPromises.readFile(zipFile); + + // Open zip file from buffer + const zipfile = await new Promise((resolve, reject) => { + yauzl.fromBuffer(zipBuffer, { lazyEntries: true }, (err, zipfile) => { + if (err) { + reject(err); + } + else { + resolve(zipfile); + } + }); + }); + + return new Promise((resolve, reject) => { + zipfile.on('entry', (entry) => { + // Skip directory entries + if (/\/$/.test(entry.fileName)) { + zipfile.readEntry(); + return; + } + + // Check if this is the file we want (cities1000.txt) + if (entry.fileName === path.basename(destination)) { + zipfile.openReadStream(entry, (err, readStream) => { + if (err) { + reject(err); + return; + } + + // Extract file to destination + const writeStream = fs.createWriteStream(destination); + + readStream.on('end', () => { + zipfile.readEntry(); + }); + + writeStream.on('finish', () => { + // We found and extracted the file we needed + resolve(); + }); + + readStream.pipe(writeStream); + }); + } + else { + // Not the file we're looking for, continue to next entry + zipfile.readEntry(); + } + }); + + zipfile.on('error', (err) => { + console.log(`Error reading zip file ${zipFile}:`, err); + reject(err); + }); + + zipfile.on('end', () => { + // If we get here without resolving, we didn't find the file + resolve(); + }); + + // Start reading entries + zipfile.readEntry(); + }); + } + catch (err) { + console.log(`Error unzipping file ${zipFile}:`, err); + return Promise.reject(err); + } +} + +// Download files if they don't exist +async function downloadFiles() { + console.log("Checking and downloading necessary files..."); + + // Create data directory if it doesn't exist + if (!fs.existsSync(DATA_DIR)) { + fs.mkdirSync(DATA_DIR); + } + + const dataFilePath = path.join(DATA_DIR, DATA); + const admin1FilePath = path.join(DATA_DIR, ADMIN1); + const countriesFilePath = path.join(DATA_DIR, COUNTRIES); + + if (!fs.existsSync(dataFilePath)) { + console.log("Downloading cities from Geonames..."); + const zipFile = path.join(DATA_DIR, "cities1000.zip"); + + try { + console.log("before downloading cities1000.zip"); + await downloadFile("https://download.geonames.org/export/dump/cities1000.zip", zipFile); + console.log("after downloading cities1000.zip"); + console.log("Extracting cities1000.zip..."); + await unzipFile(zipFile, dataFilePath); + console.log("Extraction complete, removing zip file..."); + await fsPromises.unlink(zipFile); + console.log(`Extracted ${dataFilePath}`); + } + catch (error) { + console.log("Error extracting zip file:", error); + throw error; + } + } + else { + console.log(`Using existing ${dataFilePath}`); + } + + if (!fs.existsSync(admin1FilePath)) { + console.log("Downloading admin1 from Geonames..."); + try { + await downloadFile("https://download.geonames.org/export/dump/admin1CodesASCII.txt", admin1FilePath); + } + catch (error) { + console.log("Error downloading admin1:", error); + throw error; + } + } + else { + console.log(`Using existing ${admin1FilePath}`); + } + + if (!fs.existsSync(countriesFilePath)) { + console.log("Downloading countries from Geonames..."); + try { + await downloadFile("https://download.geonames.org/export/dump/countryInfo.txt", countriesFilePath); + } + catch (error) { + console.log("Error downloading countries:", error); + throw error; + } + } + else { + console.log(`Using existing ${countriesFilePath}`); + } + + console.log("All files downloaded successfully!"); +} + +// Main execution +async function main() { + try { + await downloadFiles(); + console.log("Done downloading data!"); + } + catch (err) { + console.error("Error:", err); + process.exit(1); + } +} + +main(); diff --git a/bin/offline-geocoder/scripts/import_geonames_mongodb.js b/bin/offline-geocoder/scripts/import_geonames_mongodb.js new file mode 100644 index 00000000000..94b7bfc8146 --- /dev/null +++ b/bin/offline-geocoder/scripts/import_geonames_mongodb.js @@ -0,0 +1,180 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const path = require('path'); +const pluginManager = require("../../../plugins/pluginManager.js"); + +// Get the project root directory (directory of the script) +const PROJECT_ROOT = path.resolve(__dirname, '..'); +const DATA_DIR = path.join(PROJECT_ROOT, "data"); +const DATA = path.join(DATA_DIR, "cities1000.txt"); +const ADMIN1 = path.join(DATA_DIR, "admin1CodesASCII.txt"); +const COUNTRIES = path.join(DATA_DIR, "countryInfo.txt"); +const COLLECTION_PREFIX = "geocoder_"; + +// Parse TSV files and insert into MongoDB +async function processTsvFiles() { + console.log("Processing TSV files and inserting into MongoDB..."); + + // Check if data files exist + if (!fs.existsSync(DATA)) { + throw new Error(`Data file ${DATA} does not exist. Please run download_geonames_data.js first.`); + } + if (!fs.existsSync(ADMIN1)) { + throw new Error(`Data file ${ADMIN1} does not exist. Please run download_geonames_data.js first.`); + } + if (!fs.existsSync(COUNTRIES)) { + throw new Error(`Data file ${COUNTRIES} does not exist. Please run download_geonames_data.js first.`); + } + + const db = await pluginManager.dbConnection(); + + // Create collections if they don't exist + const features = db.collection(COLLECTION_PREFIX + 'features'); + const coordinates = db.collection(COLLECTION_PREFIX + 'coordinates'); + const admin1 = db.collection(COLLECTION_PREFIX + 'admin1'); + const countries = db.collection(COLLECTION_PREFIX + 'countries'); + + // Drop collections if they exist + await Promise.all([ + features.drop().catch(() => {}), + coordinates.drop().catch(() => {}), + admin1.drop().catch(() => {}), + countries.drop().catch(() => {}) + ]); + + // Process features and coordinates + console.log("Processing features and coordinates..."); + const featureData = fs.readFileSync(DATA, 'utf8').split('\n'); + const featuresArray = []; + const coordinatesArray = []; + + featureData.forEach(line => { + if (!line.trim()) { + return; + } + + const fields = line.split('\t'); + if (fields.length < 19) { + return; + } + + const id = parseInt(fields[0]); + const name = fields[1].replace(/"/g, '').replace(/;/g, ''); + const countryId = fields[8]; + const admin1Id = parseInt(fields[10]); + const tz = fields[17]; + + featuresArray.push({ + id, + name, + country_id: countryId, + admin1_id: admin1Id, + tz + }); + + coordinatesArray.push({ + feature_id: id, + latitude: parseFloat(fields[4]), + longitude: parseFloat(fields[5]) + }); + }); + + // Process admin1 + console.log("Processing admin1 data..."); + const admin1Data = fs.readFileSync(ADMIN1, 'utf8').split('\n'); + const admin1Array = []; + + admin1Data.forEach(line => { + if (!line.trim()) { + return; + } + + const fields = line.split('\t'); + if (fields.length < 3) { + return; + } + + const idParts = fields[0].split('.'); + const countryId = idParts[0]; + const id = parseInt(idParts[1]); + const name = fields[1].replace(/"/g, '').replace(/;/g, ''); + + admin1Array.push({ + country_id: countryId, + id, + name + }); + }); + + // Process countries + console.log("Processing country data..."); + const countriesData = fs.readFileSync(COUNTRIES, 'utf8').split('\n'); + const countriesArray = []; + + countriesData.forEach(line => { + if (!line.trim() || line.startsWith('#')) { + return; + } + + const fields = line.split('\t'); + if (fields.length < 5) { + return; + } + + countriesArray.push({ + id: fields[0], + name: fields[4] + }); + }); + + // Insert data into collections + console.log("Inserting data into MongoDB collections..."); + if (featuresArray.length > 0) { + console.log(`Inserting ${featuresArray.length} features...`); + await features.insertMany(featuresArray); + } + + if (coordinatesArray.length > 0) { + console.log(`Inserting ${coordinatesArray.length} coordinates...`); + await coordinates.insertMany(coordinatesArray); + } + + if (admin1Array.length > 0) { + console.log(`Inserting ${admin1Array.length} admin1 records...`); + await admin1.insertMany(admin1Array); + } + + if (countriesArray.length > 0) { + console.log(`Inserting ${countriesArray.length} countries...`); + await countries.insertMany(countriesArray); + } + + // Create indexes + console.log("Creating indexes..."); + await coordinates.createIndex({ latitude: 1, longitude: 1 }); + await coordinates.createIndex({ feature_id: 1 }); + await features.createIndex({ id: 1 }); + await features.createIndex({ name: 1, country_id: 1 }); + await admin1.createIndex({ country_id: 1, id: 1 }); + await countries.createIndex({ id: 1 }); + + console.log(`Created MongoDB collections with ${featuresArray.length} features.`); + + // Clean up + db.close(); +} + +// Main execution +async function main() { + try { + await processTsvFiles(); + console.log("Done importing data into MongoDB!"); + } + catch (err) { + console.error("Error:", err); + process.exit(1); + } +} + +main(); diff --git a/bin/offline-geocoder/src/index.js b/bin/offline-geocoder/src/index.js new file mode 100644 index 00000000000..6a86842008f --- /dev/null +++ b/bin/offline-geocoder/src/index.js @@ -0,0 +1,58 @@ +"use strict"; + +const reverse = require('./reverse'); +const findLocation = require('./location').find; +const pluginManager = require("../../../plugins/pluginManager.js"); + +const NO_DB = 'Database connection failed'; + +function Geocoder(options) { + var geocoder = function(options) { + this.options = options || {}; + + if (this.options.dbUrl === undefined) { + this.options.dbUrl = 'mongodb://localhost:27017'; + } + + if (this.options.dbName === undefined) { + this.options.dbName = 'countly'; + } + + if (this.options.collectionPrefix === undefined) { + this.options.collectionPrefix = 'geocoder_'; + } + + this.dbClient = null; + this.dbConnected = false; + + // Setup MongoDB connection + this.connect(); + }; + + geocoder.prototype.connect = async function() { + this.db = await pluginManager.dbConnection(); + this.dbConnected = true; + }; + + geocoder.prototype.reverse = async function(latitude, longitude, callback) { + if (!this.dbConnected) { + if (callback) { + callback(NO_DB); + } + return Promise.reject(NO_DB); + } + return reverse(this, latitude, longitude, callback); + }; + + geocoder.prototype.location = async function(locationId, locationCountryId) { + if (!this.dbConnected) { + return Promise.reject(NO_DB); + } + + return findLocation(this, locationId, locationCountryId); + }; + + return new geocoder(options); +} + +module.exports = Geocoder; diff --git a/bin/offline-geocoder/src/location.js b/bin/offline-geocoder/src/location.js new file mode 100644 index 00000000000..b52134d44cf --- /dev/null +++ b/bin/offline-geocoder/src/location.js @@ -0,0 +1,94 @@ +"use strict"; + +async function find(geocoder, locationId, locationCountryId) { + const featuresCollection = geocoder.db.collection(geocoder.options.collectionPrefix + 'features'); + const coordinatesCollection = geocoder.db.collection(geocoder.options.collectionPrefix + 'coordinates'); + const admin1Collection = geocoder.db.collection(geocoder.options.collectionPrefix + 'admin1'); + const countriesCollection = geocoder.db.collection(geocoder.options.collectionPrefix + 'countries'); + + let query; + if (typeof locationCountryId === 'string') { + query = { name: locationId, country_id: locationCountryId }; + } + else { + query = { id: locationId }; + } + + // Get the feature + const feature = await featuresCollection.findOne(query); + + if (!feature) { + return; + } + + // Get related data + const coordinates = await coordinatesCollection.findOne({ feature_id: feature.id }); + const country = await countriesCollection.findOne({ id: feature.country_id }); + const admin1 = await admin1Collection.findOne({ + country_id: feature.country_id, + id: feature.admin1_id + }); + + // Combine the results to match the expected schema + const result = { + id: feature.id, + name: feature.name, + tz: feature.tz, + admin1_id: admin1 ? admin1.id : null, + admin1_name: admin1 ? admin1.name : null, + country_id: country ? country.id : null, + country_name: country ? country.name : null, + latitude: coordinates ? coordinates.latitude : null, + longitude: coordinates ? coordinates.longitude : null + }; + + return formatResult([result]); +} + +function formatResult(rows) { + const row = rows[0]; + + if (row === undefined) { + return undefined; + } + else { + return format(row); + } +} + +function format(result) { + // Construct the formatted name consisting of the name, admin1 name and + // country name. Some features don't have an admin1, and others may have the + // same name as the feature, so this handles that. + let nameParts = []; + nameParts.push(result.name); + if (result.admin1_name && result.admin1_name != result.name) { + nameParts.push(result.admin1_name); + } + nameParts.push(result.country_name); + const formattedName = nameParts.join(', '); + + return { + id: result.id, + name: result.name, + tz: result.tz, + formatted: formattedName, + country: { + id: result.country_id, + name: result.country_name + }, + admin1: { + id: result.admin1_id, + name: result.admin1_name, + }, + coordinates: { + latitude: result.latitude, + longitude: result.longitude + } + }; +} + +module.exports = { + find: find, + format: format +}; diff --git a/bin/offline-geocoder/src/reverse.js b/bin/offline-geocoder/src/reverse.js new file mode 100644 index 00000000000..129ca30b005 --- /dev/null +++ b/bin/offline-geocoder/src/reverse.js @@ -0,0 +1,101 @@ +"use strict"; + +const formatLocation = require('./location').format; + +// This finds the closest feature based upon Pythagoras's theorem. It is an +// approximation, and won't provide results as accurate as the haversine +// formula, but trades that for performance. For our use case this is good +// enough as the data is just an approximation of the centre point of a +// feature. +// +// The scale parameter accounts for the fact that 1 degree in longitude is +// different at the poles vs the equator. +// +// Based upon http://stackoverflow.com/a/7261601/155715 +async function findFeature(geocoder, latitude, longitude, callback) { + const coordinatesCollection = geocoder.db.collection(geocoder.options.collectionPrefix + 'coordinates'); + const featuresCollection = geocoder.db.collection(geocoder.options.collectionPrefix + 'features'); + const admin1Collection = geocoder.db.collection(geocoder.options.collectionPrefix + 'admin1'); + const countriesCollection = geocoder.db.collection(geocoder.options.collectionPrefix + 'countries'); + + const scale = Math.pow(Math.cos(latitude * Math.PI / 180), 2); + + // First find the closest coordinates + const nearestCoords = await coordinatesCollection.find({ + latitude: { $gte: latitude - 1.5, $lte: latitude + 1.5 }, + longitude: { $gte: longitude - 1.5, $lte: longitude + 1.5 } + }).toArray(); + if (!nearestCoords || nearestCoords.length === 0) { + if (typeof (callback) === 'function') { + callback(undefined, {}); + } + else { + return {}; + } + } + + // Calculate distances and find the closest one + nearestCoords.forEach(coord => { + coord.distance = (latitude - coord.latitude) * (latitude - coord.latitude) + + (longitude - coord.longitude) * (longitude - coord.longitude) * scale; + }); + + nearestCoords.sort((a, b) => a.distance - b.distance); + const closest = nearestCoords[0]; + + // Get the feature + const feature = await featuresCollection.findOne({ id: closest.feature_id }); + if (!feature) { + if (typeof (callback) === 'function') { + callback(undefined, {}); + } + else { + return {}; + } + } + + // Get related data + const country = await countriesCollection.findOne({ id: feature.country_id }); + const admin1 = await admin1Collection.findOne({ + country_id: feature.country_id, + id: feature.admin1_id + }); + + // Combine the results to match the expected schema + const result = { + id: feature.id, + name: feature.name, + tz: feature.tz, + admin1_id: admin1 ? admin1.id : null, + admin1_name: admin1 ? admin1.name : null, + country_id: country ? country.id : null, + country_name: country ? country.name : null, + latitude: closest.latitude, + longitude: closest.longitude + }; + + const formattedResult = formatResult([result]); + if (typeof (callback) === 'function') { + callback(undefined, formattedResult); + } + else { + return formattedResult; + } +} + +function formatResult(rows) { + const row = rows[0]; + + if (!row || row === undefined) { + return {}; + } + else { + return formatLocation(row); + } +} + +function Reverse(geocoder, latitude, longitude, callback) { + return findFeature(geocoder, latitude, longitude, callback); +} + +module.exports = Reverse; diff --git a/bin/scripts/adjust_stats.js b/bin/scripts/adjust_stats.js new file mode 100644 index 00000000000..eb69d7354d3 --- /dev/null +++ b/bin/scripts/adjust_stats.js @@ -0,0 +1,136 @@ +/** + * Script to check statistics for Adjust data in Countly. + * + * This script checks: + * 1. Number of documents in the adjust collection for the specified app_id + * 2. Number of users in app_users{APP_ID} collection with custom.adjust_id + * 3. Number of adjust_install events in the drill collection + * + * Location: + * Place this script in the `bin/scripts` directory of your Countly installation. + * + * Usage: + * 1. Replace the `APP_ID` variable with the desired app's ID. + * 2. Run the script using Node.js: + * ``` + * node /var/countly/bin/scripts/adjust_stats.js + * ``` + */ + +// Define the APP_ID variable +const APP_ID = '5ab0c3ef92938d0e61cf77f4'; + +const plugins = require('../../plugins/pluginManager.js'); + +(async() => { + console.log(`Checking Adjust statistics for APP_ID: ${APP_ID}`); + + try { + // Connect to countly database + const db = await plugins.dbConnection("countly"); + + // Connect to countly_drill database + const drillDb = await plugins.dbConnection("countly_drill"); + + console.log('Connected to databases successfully.'); + + // 1. Check how many documents are in adjust collection for this app_id + console.log('\n--- Checking adjust collection ---'); + + // Define date range for filtering (July 17-22, 2025) + const startDate = new Date('2025-07-17T00:00:00.000Z'); + const endDate = new Date('2025-07-22T23:59:59.999Z'); + console.log(`Date range filter: ${startDate.toISOString()} to ${endDate.toISOString()}`); + + const adjustQuery = { + app_id: APP_ID, + cd: { + $gte: startDate, + $lte: endDate + } + }; + + const adjustCount = await db.collection('adjust').countDocuments(adjustQuery); + console.log(`Documents in adjust collection for app_id ${APP_ID} (${startDate.toDateString()} - ${endDate.toDateString()}): ${adjustCount}`); + + // 1a. Check unique amount of adjust_id values in adjust collection + const uniqueAdjustIds = await db.collection('adjust').distinct('adjust_id', adjustQuery); + console.log(`Unique adjust_id values in adjust collection for app_id ${APP_ID} (${startDate.toDateString()} - ${endDate.toDateString()}): ${uniqueAdjustIds.length}`); + + // 1b. Breakdown by event property in adjust collection + console.log('\n--- Event breakdown in adjust collection ---'); + const eventBreakdown = await db.collection('adjust').aggregate([ + { $match: adjustQuery }, + { $group: { _id: "$event", count: { $sum: 1 } } }, + { $sort: { count: -1 } } + ]).toArray(); + + console.log('Event breakdown:'); + eventBreakdown.forEach(item => { + console.log(` ${item._id}: ${item.count}`); + }); + + // 2. Check how many users in app_users{APP_ID} collection have custom.adjust_id value + console.log('\n--- Checking app_users collection ---'); + const appUsersCollection = 'app_users' + APP_ID; + + // Use the same date range but convert to seconds for fac field + const appUsersQuery = { + 'custom.adjust_id': { $exists: true }, + fac: { + $gte: Math.floor(startDate.getTime() / 1000), + $lte: Math.floor(endDate.getTime() / 1000) + } + }; + + const usersWithAdjustId = await db.collection(appUsersCollection).countDocuments(appUsersQuery); + console.log(`Users with custom.adjust_id in ${appUsersCollection} (${startDate.toDateString()} - ${endDate.toDateString()}): ${usersWithAdjustId}`); + + // 2a. Check unique custom.adjust_id values in app_users collection + const uniqueUserAdjustIds = await db.collection(appUsersCollection).distinct('custom.adjust_id', appUsersQuery); + console.log(`Unique custom.adjust_id values in ${appUsersCollection} (${startDate.toDateString()} - ${endDate.toDateString()}): ${uniqueUserAdjustIds.length}`); + + // 3. Check how many adjust_install events are in drill collection + console.log('\n--- Checking drill collection for adjust_install events ---'); + const drillCollectionName = 'drill_events'; + console.log(`Drill collection name: ${drillCollectionName}`); + + // Use the same date range but convert to milliseconds for ts field + const drillQuery = { + "a": APP_ID, + "e": "adjust_install", + ts: { + $gte: startDate.getTime(), + $lte: endDate.getTime() + } + }; + + const adjustInstallEvents = await drillDb.collection(drillCollectionName).countDocuments(drillQuery); + console.log(`adjust_install events in drill collection (${startDate.toDateString()} - ${endDate.toDateString()}): ${adjustInstallEvents}`); + + // 3a. Check unique custom.adjust_id values in drill collection + const uniqueDrillAdjustIds = await drillDb.collection(drillCollectionName).distinct('custom.adjust_id', drillQuery); + console.log(`Unique custom.adjust_id values in drill collection (${startDate.toDateString()} - ${endDate.toDateString()}): ${uniqueDrillAdjustIds.length}`); + + // Summary + console.log('\n--- SUMMARY ---'); + console.log(`APP_ID: ${APP_ID}`); + console.log(`Date range: ${startDate.toDateString()} - ${endDate.toDateString()}`); + console.log(`Adjust collection documents: ${adjustCount}`); + console.log(`Unique adjust_id values: ${uniqueAdjustIds.length}`); + console.log(`Users with adjust_id: ${usersWithAdjustId}`); + console.log(`Unique custom.adjust_id values in app_users: ${uniqueUserAdjustIds.length}`); + console.log(`adjust_install events in drill collection: ${adjustInstallEvents}`); + console.log(`Unique custom.adjust_id values in drill collection: ${uniqueDrillAdjustIds.length}`); + + console.log('\nStatistics check completed.'); + + } + catch (error) { + console.error('Error during statistics check:', error); + } + finally { + console.log('Terminating the process...'); + process.exit(0); + } +})(); diff --git a/bin/scripts/export-data/setting_limits_and_real_values.js b/bin/scripts/export-data/setting_limits_and_real_values.js index 7cbb0e0bfa3..482c25578f8 100644 --- a/bin/scripts/export-data/setting_limits_and_real_values.js +++ b/bin/scripts/export-data/setting_limits_and_real_values.js @@ -19,8 +19,8 @@ const DEFAULT_LIMITS = { view_name_limit: 128, view_segment_limit: 100, view_segment_value_limit: 10, - //custom_prop_limit: 20, - custom_property_limit: 20, + custom_prop_limit: 20, + //custom_property_limit: 20, custom_prop_value_limit: 50, }; Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) { @@ -56,8 +56,8 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" view_name_limit: pluginsCollectionPlugins?.views?.view_name_limit || DEFAULT_LIMITS.view_name_limit, view_segment_limit: pluginsCollectionPlugins?.views?.segment_limit || DEFAULT_LIMITS.view_segment_limit, view_segment_value_limit: pluginsCollectionPlugins?.views?.segment_value_limit || DEFAULT_LIMITS.view_segment_value_limit, - //custom_prop_limit: pluginsCollectionPlugins?.users?.custom_prop_limit || DEFAULT_LIMITS.custom_prop_limit, - custom_property_limit: pluginsCollectionPlugins?.drill?.custom_property_limit || DEFAULT_LIMITS.custom_property_limit, + custom_prop_limit: pluginsCollectionPlugins?.users?.custom_prop_limit || DEFAULT_LIMITS.custom_prop_limit, + //custom_property_limit: pluginsCollectionPlugins?.drill?.custom_property_limit || DEFAULT_LIMITS.custom_property_limit, custom_prop_value_limit: pluginsCollectionPlugins?.users?.custom_set_limit || DEFAULT_LIMITS.custom_prop_value_limit, }; // GETTING REAL DATA PER APP @@ -247,10 +247,10 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" }); app_results['View Segments Unique Values'] = {"default": defaultVal, "set": currentVal, "real": realVal}; // USER PROPERTIES - //defaultVal = DEFAULT_LIMITS.custom_prop_limit; - //currentVal = CURRENT_LIMITS.custom_prop_limit; - defaultVal = DEFAULT_LIMITS.custom_property_limit; - currentVal = CURRENT_LIMITS.custom_property_limit; + defaultVal = DEFAULT_LIMITS.custom_prop_limit; + currentVal = CURRENT_LIMITS.custom_prop_limit; + //defaultVal = DEFAULT_LIMITS.custom_property_limit; + //currentVal = CURRENT_LIMITS.custom_property_limit; realVal = customPropsPerApp && customPropsPerApp[0]?.customPropertiesCount || 0; app_results['Max user custom properties'] = {"default": defaultVal, "set": currentVal, "real": realVal}; // VALUES IN AN ARRAY FOR ONE USER PROPERTY diff --git a/bin/scripts/generate-api-docs.js b/bin/scripts/generate-api-docs.js new file mode 100755 index 00000000000..90f827c18dd --- /dev/null +++ b/bin/scripts/generate-api-docs.js @@ -0,0 +1,36 @@ +#!/usr/bin/env node + +/** + * This script runs the API documentation generation process: + * 1. Merge all OpenAPI specs into one file + * 2. Generate Swagger UI HTML documentation + */ + +const { execFileSync } = require('child_process'); +const path = require('path'); + +console.log('🚀 Starting API documentation generation process...'); + +// Paths to the scripts +const scriptsDir = __dirname; +const mergeScript = path.join(scriptsDir, 'merge-openapi.js'); +const swaggerScript = path.join(scriptsDir, 'generate-swagger-ui.js'); + +try { + // Step 1: Merge OpenAPI specs + console.log('\n📑 Step 1: Merging OpenAPI specifications...'); + execFileSync('node', [mergeScript], { stdio: 'inherit' }); + + // Step 2: Generate Swagger UI documentation + console.log('\n📙 Step 2: Generating Swagger UI documentation...'); + execFileSync('node', [swaggerScript], { stdio: 'inherit' }); + + console.log('\n✅ API documentation generation completed successfully!'); + console.log('📊 Documentation is available in the doc/api directory:'); + console.log(' - Swagger UI: doc/api/swagger-ui-api.html'); + +} +catch (error) { + console.error('\n❌ Error during API documentation generation:', error.message); + process.exit(1); +} \ No newline at end of file diff --git a/bin/scripts/generate-swagger-ui.js b/bin/scripts/generate-swagger-ui.js new file mode 100755 index 00000000000..dc36919cf86 --- /dev/null +++ b/bin/scripts/generate-swagger-ui.js @@ -0,0 +1,124 @@ +#!/usr/bin/env node + +/** + * This script generates HTML documentation using Swagger UI from the merged OpenAPI specification. + */ + +const fs = require('fs'); +const path = require('path'); + +// Configuration +const outputDir = path.join(__dirname, '../../doc/api'); +const mergedSpecPath = path.join(outputDir, 'openapi-merged.json'); +const outputHtmlPath = path.join(outputDir, 'index.html'); + +// Ensure the merged spec exists +if (!fs.existsSync(mergedSpecPath)) { + console.error(`Merged OpenAPI spec not found at ${mergedSpecPath}`); + console.error('Please run merge-openapi.js first'); + process.exit(1); +} + +console.log('Generating Swagger UI HTML documentation...'); + +// Create a simple HTML file with Swagger UI +const swaggerUiHtml = ` + + +
+ + +| " + item.series.moreInfo[p].label + ": | " + item.series.moreInfo[p].value + "% | "; - } - } - else { - for (p = 0; p < 5; p = p + 1) { - tooltipcontent += "
| " + item.series.moreInfo[p].label + " : | " + item.series.moreInfo[p].value + "% |
| ... | |
| (and " + (item.series.moreInfo.length - 5) + " other) | |
" + jQuery.i18n.map["export.format-if-possible-explain"] + "
"); - }, - contentAsHTML: true, - functionInit: function(instance2) { - instance2.content("" + jQuery.i18n.map["export.format-if-possible-explain"] + "
"); - } - }); - - if (data && data.fields && Object.keys(data.fields).length > 0) { - dialog.find(".export-format-option").css("display", "none"); - if (dialog.find(".export-columns-selector:visible").length > 0) { - if (dialog.find(".export-all-columns").hasClass("fa-check-square")) { - //export all columns no need for projections - for (var filed in data.fields) { - if (data.fields[filed].to === "time") { - dialog.find(".export-format-option").css("display", "block"); - } - } - } - else { - var projection = {}; - - var checked = dialog.find('.columns-wrapper .fa-check-square'); - for (var kz = 0; kz < checked.length; kz++) { - projection[$(checked[kz]).data("index")] = true; - } - - if (instance && instance.fixProjectionParams) { - projection = instance.fixProjectionParams(projection); - } - - for (var filed2 in data.fields) { - if (data.fields[filed2].to === "time" && projection[filed2]) { - dialog.find(".export-format-option").css("display", "block"); - } - } - - } - } - } - else { - dialog.find(".export-format-option").css("display", "none"); - } - } - /** - * Displays database export dialog - * @param {number} count - total count of documents to export - * @param {object} data - data for export query to use when constructing url - * @param {boolean} asDialog - open it as dialog - * @param {boolean} exportByAPI - export from api request, export from db when set to false - * @param {boolean} instance - optional. Reference to table to get correct colum names(only if there is need to select columns to export) There must be changes made in table settings to allow it. (table.addColumnExportSelector = true and each column must have columnsSelectorIndex value as field in db) - * @returns {object} jQuery object reference to dialog - * @example - * var dialog = CountlyHelpers.export(300000); - * //later when done - * CountlyHelpers.removeDialog(dialog); - */ - CountlyHelpers.export = function(count, data, asDialog, exportByAPI, instance) { - //var hardLimit = countlyGlobal.config.export_limit; - //var pages = Math.ceil(count / hardLimit); - var dialog = $("#cly-export").clone(); - var type = "csv"; - //var page = 0; - var tableCols; - - var formatData = data.formatFields || ""; - try { - formatData = JSON.parse(formatData); - } - catch (e) { - formatData = {}; - } - dialog.removeAttr("id"); - /*dialog.find(".details").text(jQuery.i18n.prop("export.export-number", (count + "").replace(/(\d)(?=(\d{3})+$)/g, '$1 '), pages)); - if (count <= hardLimit) { - dialog.find(".cly-select").hide(); - } - else { - dialog.find(".select-items > div").append('| '+this._get(a,"weekHeader")+" | ":"";for(var S=0;S<7;S++){var T=(S+y)%7;R+="=5?' class="ui-datepicker-week-end"':"")+">"+''+C[T]+" | "}Q+=R+"'+this._get(a,"calculateWeek")(Y)+" | ":"";for(var S=0;S<7;S++){var ba=F?F.apply(a.input?a.input[0]:null,[Y]):[!0,""],bb=Y.getMonth()!=n,bc=bb&&!H||!ba[0]||l&&Y"+(bb&&!G?" ":bc?''+Y.getDate()+"":''+Y.getDate()+"")+" | ",Y.setDate(Y.getDate()+1),Y=this._daylightSavingAdjust(Y)}Q+=_+""}n++,n>11&&(n=0,o++),Q+="
|---|
' + inst._getRegional('rgbR') + '
' + inst._getRegional('rgbG') + '
' + inst._getRegional('rgbB') + '
' + inst._getRegional('alphaA') + '
HEX
{{getData[key].name}}
- ++ {{ getData[key].name }} + + + Applied to SDKs + +
+ +