Merge pull request #587 from Swetrix/feature/v2-analytics-api

feat: Per-dimension stats API

Author: Blaumaus

.gitignore

@@ -78,3 +78,4 @@ DESIGN.md
 DESIGN.json
 
 next-env.d.ts
+**/tsconfig.tsbuildinfo

backend/apps/cloud/src/analytics/analytics.controller.ts

@@ -43,16 +43,16 @@ import {
   DataType,
   getLowestPossibleTimeBucket,
 } from './analytics.service'
+import { normalizeFiltersToV1Json } from './v2/query/filters.translator'
 import { VALID_PERIODS } from './decorators/validate-period.decorator'
 import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
 import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'
 import { AuthenticationGuard } from '../auth/guards/authentication.guard'
 import { PageviewsDto } from './dto/pageviews.dto'
 import { EventsDto } from './dto/events.dto'
-import { GetDataDto, ChartRenderMode, TimeBucketType } from './dto/getData.dto'
+import { GetDataDto, ChartRenderMode } from './dto/getData.dto'
 import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
 import { GetPagePropertyMetaDto } from './dto/get-page-property-meta.dto'
-import { GetUserFlowDto } from './dto/getUserFlow.dto'
 import { GetJourneysDto } from './dto/get-journeys.dto'
 import { GetJourneySessionsDto } from './dto/get-journey-sessions.dto'
 import { GetFunnelsDto } from './dto/getFunnels.dto'
@@ -76,13 +76,7 @@ import {
   DataDeletionPreviewDto,
 } from './dto/data-deletion.dto'
 import { GetVersionFiltersDto } from './dto/get-version-filters.dto'
-import {
-  IFunnel,
-  IGetFunnel,
-  IPageProperty,
-  IUserFlow,
-  PerfMeasure,
-} from './interfaces'
+import { IFunnel, IGetFunnel, IPageProperty, PerfMeasure } from './interfaces'
 import { GetSessionsDto } from './dto/get-sessions.dto'
 import { GetSessionDto } from './dto/get-session.dto'
 import {
@@ -115,7 +109,7 @@ import { LiveVisitorsDto } from './dto/live-visitors.dto'
 import { GetHeartbeatStatsDto } from './dto/get-heartbeat-stats'
 import { GetKeywordsDto } from './dto/get-keywords.dto'
 import { GetBotStatsDto } from './dto/get-bot-stats.dto'
-import { GSCService } from '../project/gsc.service'
+import { GSC_ALL_TIME_DAYS, GSCService } from '../project/gsc.service'
 import { GetProfileIdDto, GetSessionIdDto } from './dto/get-id.dto'
 import { ExperimentService } from '../experiment/experiment.service'
 import {
@@ -135,9 +129,6 @@ const BOT_RESPONSE = { message: 'Bot traffic detected, request is ignored' }
 
 const ONLINE_VISITORS_WINDOW_MINUTES = 5 // minutes
 
-// Maximum range for "all time" GSC queries (~16 months, GSC's data retention)
-const GSC_ALL_TIME_DAYS = 480
-
 // Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
 const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
 const isPerformanceValid = (perf: any) => {
@@ -1022,59 +1013,6 @@ export class AnalyticsController {
     return this.getData(data, uid, {}, true)
   }
 
-  @Get('user-flow')
-  @Auth(true, true)
-  async getUserFlow(
-    @Query() data: GetUserFlowDto,
-    @CurrentUserId() uid: string,
-    @Headers() headers: { 'x-password'?: string },
-  ): Promise<IUserFlow | { appliedFilters: any[] }> {
-    const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, filters } = data
-
-    await this.analyticsService.checkProjectAccess(
-      pid,
-      uid,
-      headers['x-password'],
-    )
-
-    await this.analyticsService.checkBillingAccess(pid)
-
-    this.logger.log(
-      `pid: ${pid}, period: ${period}`,
-      'GET /analytics/user-flow',
-    )
-
-    let diff
-
-    if (period === 'all') {
-      const res = await this.analyticsService.calculateTimeBucketForAllTime(
-        pid,
-        'pageview',
-      )
-
-      diff = res.diff
-    }
-
-    const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
-    const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
-      from,
-      to,
-      null,
-      period,
-      safeTimezone,
-      diff,
-    )
-
-    const [filtersQuery, filtersParams, appliedFilters] =
-      this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true)
-
-    const params = { pid, groupFrom, groupTo, ...filtersParams }
-
-    const flow = await this.analyticsService.getUserFlow(params, filtersQuery)
-
-    return { ...flow, appliedFilters }
-  }
-
   @Get('journeys')
   @Auth(true, true)
   async getJourneys(
@@ -1122,7 +1060,11 @@ export class AnalyticsController {
     )
 
     const [filtersQuery, filtersParams, appliedFilters] =
-      this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true)
+      this.analyticsService.getFiltersQuery(
+        normalizeFiltersToV1Json(filters, 'traffic'),
+        DataType.ANALYTICS,
+        true,
+      )
 
     const params = { pid, groupFrom, groupTo, ...filtersParams }
 
@@ -1198,7 +1140,7 @@ export class AnalyticsController {
     )
 
     const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
-      filters || '[]',
+      normalizeFiltersToV1Json(filters, 'traffic'),
       DataType.ANALYTICS,
     )
     const params = { pid, groupFrom, groupTo, ...filtersParams }
@@ -1251,139 +1193,6 @@ export class AnalyticsController {
     return { keywords }
   }
 
-  @Get('gsc-dashboard')
-  @Auth(true, true)
-  async getGSCDashboard(
-    @Query() data: GetKeywordsDto,
-    @CurrentUserId() uid: string,
-    @Headers() headers: { 'x-password'?: string },
-  ) {
-    const {
-      pid,
-      period,
-      from,
-      to,
-      timezone = DEFAULT_TIMEZONE,
-      timeBucket,
-      filters,
-    } = data
-
-    await this.analyticsService.checkProjectAccess(
-      pid,
-      uid,
-      headers['x-password'],
-    )
-
-    await this.analyticsService.checkBillingAccess(pid)
-
-    const defaultTimeBucket = ['1h', 'today', 'yesterday', '1d'].includes(
-      period,
-    )
-      ? TimeBucketType.HOUR
-      : TimeBucketType.DAY
-    const finalTimeBucket = (timeBucket || defaultTimeBucket) as TimeBucketType
-    let groupTimeBucket: TimeBucketType | null = finalTimeBucket
-    if (period === 'all' || period === 'custom') {
-      groupTimeBucket = null
-    }
-
-    this.logger.log(
-      `pid: ${pid}, period: ${period}, timeBucket: ${finalTimeBucket}, filters: ${filters}`,
-      'GET /analytics/gsc-dashboard',
-    )
-
-    const diff = period === 'all' ? GSC_ALL_TIME_DAYS : undefined
-
-    const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
-    const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
-      from,
-      to,
-      groupTimeBucket,
-      period,
-      safeTimezone,
-      diff,
-    )
-
-    return this.gscService.getDashboard(
-      pid,
-      groupFrom,
-      groupTo,
-      finalTimeBucket,
-      filters,
-    )
-  }
-
-  @Get('gsc-details')
-  @Auth(true, true)
-  async getGSCDetails(
-    @Query() data: GetKeywordsDto,
-    @CurrentUserId() uid: string,
-    @Headers() headers: { 'x-password'?: string },
-  ) {
-    const {
-      pid,
-      period,
-      from,
-      to,
-      timezone = DEFAULT_TIMEZONE,
-      page,
-      query,
-    } = data
-
-    this.logger.log(
-      `pid: ${pid}, period: ${period}, page: ${page}, query: ${query}`,
-      'GET /analytics/gsc-details',
-    )
-
-    await this.analyticsService.checkProjectAccess(
-      pid,
-      uid,
-      headers['x-password'],
-    )
-
-    await this.analyticsService.checkBillingAccess(pid)
-
-    const diff = period === 'all' ? GSC_ALL_TIME_DAYS : undefined
-
-    const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
-    const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
-      from,
-      to,
-      null,
-      period,
-      safeTimezone,
-      diff,
-    )
-
-    if (page) {
-      const keywords = await this.gscService.getKeywords(
-        pid,
-        groupFrom,
-        groupTo,
-        50,
-        0,
-        undefined,
-        page,
-      )
-      return { type: 'queries', data: keywords }
-    }
-
-    if (query) {
-      const pages = await this.gscService.getTopPages(
-        pid,
-        groupFrom,
-        groupTo,
-        50,
-        0,
-        undefined,
-        query,
-      )
-      return { type: 'pages', data: pages }
-    }
-
-    return { type: 'none', data: [] }
-  }
-
   @Get('birdseye')
   @Auth(true, true)
   // returns overall short statistics per project
@@ -2578,7 +2387,10 @@ export class AnalyticsController {
     )
 
     const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
-      this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
+      this.analyticsService.getFiltersQuery(
+        normalizeFiltersToV1Json(filters, 'traffic'),
+        DataType.ANALYTICS,
+      )
 
     let timeBucket
     let diff
@@ -3044,7 +2856,7 @@ export class AnalyticsController {
 
     return this.analyticsService.getDataDeletionPreview(
       body.pid,
-      body.filters || '[]',
+      normalizeFiltersToV1Json(body.filters, 'traffic'),
       body.from || null,
       body.to || null,
     )
@@ -3068,7 +2880,7 @@ export class AnalyticsController {
 
     await this.analyticsService.deleteData(
       body.pid,
-      body.filters || '[]',
+      normalizeFiltersToV1Json(body.filters, 'traffic'),
       body.from || null,
       body.to || null,
       body.types,

backend/apps/cloud/src/analytics/analytics.module.ts

@@ -21,6 +21,16 @@ import {
   SESSION_REPLAY_EXPORT_QUEUE,
 } from './session-replay-export.service'
 import { SessionReplayExportProcessor } from './session-replay-export.processor'
+import { AnalyticsV2Service } from './v2/analytics-v2.service'
+import { SeoV2Service } from './v2/seo-v2.service'
+import { TrafficV2Controller } from './v2/controllers/traffic-v2.controller'
+import { PerformanceV2Controller } from './v2/controllers/performance-v2.controller'
+import { CaptchaV2Controller } from './v2/controllers/captcha-v2.controller'
+import { ErrorsV2Controller } from './v2/controllers/errors-v2.controller'
+import { SessionsV2Controller } from './v2/controllers/sessions-v2.controller'
+import { ProfilesV2Controller } from './v2/controllers/profiles-v2.controller'
+import { ProjectV2Controller } from './v2/controllers/project-v2.controller'
+import { SeoV2Controller } from './v2/controllers/seo-v2.controller'
 
 @Module({
   imports: [
@@ -34,6 +44,8 @@ import { SessionReplayExportProcessor } from './session-replay-export.processor'
   ],
   providers: [
     AnalyticsService,
+    AnalyticsV2Service,
+    SeoV2Service,
     BotDetectionService,
     SaltService,
     SessionReplayS3Service,
@@ -44,6 +56,16 @@ import { SessionReplayExportProcessor } from './session-replay-export.processor'
     PublicProjectCacheInterceptor,
   ],
   exports: [AnalyticsService, BotDetectionService, SaltService],
-  controllers: [AnalyticsController],
+  controllers: [
+    AnalyticsController,
+    TrafficV2Controller,
+    PerformanceV2Controller,
+    CaptchaV2Controller,
+    ErrorsV2Controller,
+    SessionsV2Controller,
+    ProfilesV2Controller,
+    ProjectV2Controller,
+    SeoV2Controller,
+  ],
 })
 export class AnalyticsModule {}