Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/api/src/agent/handlers/chart-handler.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { User } from '@databuddy/auth';
import type { Website } from '@databuddy/shared';
import type { Website } from '@databuddy/shared/types';
import type { z } from 'zod';
import type { AIResponseJsonSchema } from '../prompts/agent';
import { executeQuery } from '../utils/query-executor';
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/agent/handlers/metric-handler.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { User } from '@databuddy/auth';
import type { Website } from '@databuddy/shared';
import type { Website } from '@databuddy/shared/types';
import type { z } from 'zod';
import type { AIResponseJsonSchema } from '../prompts/agent';
import { executeQuery } from '../utils/query-executor';
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/agent/processor.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { User } from '@databuddy/auth';
import type { Website } from '@databuddy/shared';
import type { Website } from '@databuddy/shared/types';
import { handleChartResponse } from './handlers/chart-handler';
import { handleMetricResponse } from './handlers/metric-handler';
import { comprehensiveUnifiedPrompt } from './prompts/agent';
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/lib/website-utils.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { auth } from '@databuddy/auth';
import { db, userPreferences, websites } from '@databuddy/db';
import { cacheable } from '@databuddy/redis';
import type { Website } from '@databuddy/shared';
import type { Website } from '@databuddy/shared/types';
import { eq } from 'drizzle-orm';
import {
getApiKeyFromHeader,
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/query/types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { QueryBuilderMeta } from '@databuddy/shared';
import type { QueryBuilderMeta } from '@databuddy/shared/types';

export const FilterOperators = {
eq: '=',
Expand Down
3 changes: 2 additions & 1 deletion apps/api/src/query/utils.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { getCountryCode, getCountryName, referrers } from '@databuddy/shared';
import { getCountryCode, getCountryName } from '@databuddy/shared/country-codes';
import { referrers } from '@databuddy/shared/lists';
import { mapScreenResolutionToDeviceType } from './screen-resolution-to-device-type';
import type { SimpleQueryConfig } from './types';

Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/routes/query.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { filterOptions } from '@databuddy/shared';
import { filterOptions } from '@databuddy/shared/lists';
import { Elysia, t } from 'elysia';
import { getCachedWebsiteDomain, getWebsiteDomain } from '../lib/website-utils';
// import { createRateLimitMiddleware } from '../middleware/rate-limit';
Expand Down
12 changes: 10 additions & 2 deletions apps/basket/src/utils/user-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
* and platform identification.
*/

import { bots, logger } from '@databuddy/shared';
import { logger } from '@databuddy/shared/utils';
import { bots } from '@databuddy/shared/lists';
import { UAParser } from 'ua-parser-js';

export interface UserAgentInfo {
Expand Down Expand Up @@ -76,6 +77,13 @@ export function parseUserAgent(userAgent: string): {
}
}

// Precompile bot regexes once to avoid per-request overhead
const compiledBotRegexes = bots.map((bot) => ({
name: bot.name,
category: bot.category,
regex: new RegExp(bot.regex, 'i'),
}));

Comment on lines +80 to +86

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Short-circuit on missing User-Agent before running regex checks

Currently, detection scans all compiled regexes before checking for a missing UA, doing unnecessary work for bot/malformed requests. Early-return to avoid the scan.

 export function detectBot(
   userAgent: string,
   request: Request
 ): {
   isBot: boolean;
   reason?: string;
   category?: string;
   botName?: string;
 } {
-  const ua = userAgent || '';
-
-  const detectedBot = compiledBotRegexes.find((bot) => bot.regex.test(ua));
-  if (detectedBot) {
-    return {
-      isBot: true,
-      reason: 'known_bot_user_agent',
-      category: 'Known Bot',
-      botName: detectedBot.name,
-    };
-  }
-
-  if (!userAgent) {
+  if (!userAgent) {
     return {
       isBot: true,
       reason: 'missing_user_agent',
       category: 'Missing Headers',
     };
   }
+  const ua = userAgent || '';
+
+  const detectedBot = compiledBotRegexes.find((bot) => bot.regex.test(ua));
+  if (detectedBot) {
+    return {
+      isBot: true,
+      reason: 'known_bot_user_agent',
+      category: 'Known Bot',
+      botName: detectedBot.name,
+    };
+  }

Also applies to: 96-116

🧰 Tools
🪛 ast-grep (0.38.6)

[warning] 83-83: Regular expression constructed from variable input detected. This can lead to Regular Expression Denial of Service (ReDoS) attacks if the variable contains malicious patterns. Use libraries like 'recheck' to validate regex safety or use static patterns.
Context: new RegExp(bot.regex, 'i')
Note: [CWE-1333] Inefficient Regular Expression Complexity [REFERENCES]
- https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS
- https://cwe.mitre.org/data/definitions/1333.html

(regexp-from-variable)

🤖 Prompt for AI Agents
In apps/basket/src/utils/user-agent.ts around lines 80 to 86 (and also apply
same change to the logic at lines 96-116), the code compiles and iterates all
bot regexes before checking whether the incoming User-Agent header exists;
change the function(s) to short-circuit immediately when userAgent is
null/undefined/empty (e.g., if (!userAgent) return the appropriate
non-bot/malformed result) before running any regex tests, and ensure any callers
still receive the correct return shape when UA is missing.

💡 Verification agent

🧩 Analysis chain

Mitigate potential ReDoS risk from variable-driven regex patterns

Static analysis correctly flags new RegExp(bot.regex, 'i'). While these patterns are internal (not user-supplied), an adversarial UA can still trigger worst-case backtracking. Consider:

  • Using a safe regex engine (e.g., RE2-backed solutions) for untrusted inputs.
  • Pre-validating patterns using a linter (safe-regex) at build-time.
  • Defensive guards: quick substring heuristics before regex pass to reduce scan volume.

I can add a build-time check that scans packages/shared/src/lists/bots.ts for potentially catastrophic patterns and surfaces a CI warning. Want me to draft that script?


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "1) Locate user-agent.ts containing `compiledBotRegexes`..."
ua_file=$(rg --hidden -n --no-line-number "compiledBotRegexes" -S --files-with-matches | head -n1 || true)
if [ -n "$ua_file" ]; then
  echo "Found: $ua_file"
  echo "---- start: $ua_file (1-240) ----"
  sed -n '1,240p' "$ua_file"
  echo "---- end snippet ----"
else
  echo "compiledBotRegexes not found."
fi

echo
echo "2) Try to locate bots list file (common exports: `const bots =`, `export const bots`, `export default bots`)..."
bots_file=$(rg --hidden -n --no-line-number "export .*bots|const bots =|export default .*bots" -S --files-with-matches | head -n1 || true)
if [ -n "$bots_file" ]; then
  echo "Found bots list file: $bots_file"
  echo "---- start: $bots_file (1-400) ----"
  sed -n '1,400p' "$bots_file"
  echo "---- end snippet ----"
else
  echo "No bots list file found with those patterns. Searching for files named bots.ts..."
  bots_file=$(fd --hidden -IH bots.ts || true)
  if [ -n "$bots_file" ]; then
    echo "fd found: $bots_file"
    echo "---- start: $bots_file (1-400) ----"
    sed -n '1,400p' "$bots_file"
    echo "---- end snippet ----"
  else
    echo "No bots.ts found."
  fi
fi

echo
echo "3) Search repo for other dynamic RegExp usages (`new RegExp(...)`)..."
rg --hidden -n "new RegExp\(" -S || true

echo
echo "4) Search for usages of bot.regex or similar references..."
rg --hidden -n "bot\.regex|bots\]" -S || true

echo
echo "5) If bots file found, search it for suspicious constructs (backrefs, nested quantifiers)..."
if [ -n "${bots_file:-}" ]; then
  echo "Searching for backreferences (\\1, \\2, ...):"
  rg -n "\\\\[0-9]" "$bots_file" || true
  echo "Searching for patterns that look like nested quantifiers (e.g., '(.+)+', '(.+)*', '(.+){2,}'):"
  rg -n "\\(\\.\\+\\)\\+|\\(\\.\\+\\)\\*|\\(\\.\\+\\)\\{[0-9]" "$bots_file" || true
  echo "Searching for other risky tokens like '(.+)' followed by '+' or '*' in the same literal:"
  rg -n "\\)\\+" "$bots_file" || true
fi

echo
echo "Script finished."

Length of output: 6413


🏁 Script executed:

#!/bin/bash
set -euo pipefail

dir="packages/shared/src/lists"
echo "1) List files in $dir"
ls -la "$dir" || true
echo

echo "2) Locate bots file by scanning exports/definitions"
bots_file=$(rg --hidden -n --no-line-number "(^|\s)(export\s+const\s+bots|const\s+bots\s*=|export\s+default\s+bots)" -S --files-with-matches | head -n1 || true)
if [ -z "$bots_file" ]; then
  # fallback: any file named bots.*
  bots_file=$(fd --hidden -IH "^bots\\." "$dir" || true)
fi

if [ -n "$bots_file" ]; then
  echo "Found bots file: $bots_file"
  echo "---- start: $bots_file (1-400) ----"
  sed -n '1,400p' "$bots_file"
  echo "---- end snippet ----"
else
  echo "No explicit bots file found via patterns. Searching for any files exporting lists named 'bots'..."
  rg --hidden -n "bots\\s*[:=]\\s*\\[" -S || true
fi

echo
if [ -n "$bots_file" ]; then
  echo "3) Show lines with regex declarations in $bots_file"
  rg -n "regex\\s*:\\s*['\"]" "$bots_file" || true

  echo
  echo "4) Quick heuristics for risky regex constructs in $bots_file"
  echo "- Backreferences (\\1, \\2, ...):"
  rg -n "\\\\[0-9]" "$bots_file" || true

  echo "- Nested/repeated quantifiers (e.g. '(.+)+', '(.*)+', '(.+)*'):"
  rg -n "\\(\\.\\+\\)\\+|\\(\\.\\*\\)\\+|\\(\\.\\+\\)\\*|\\(\\.\\*\\)\\*|\\(\\.\\+\\)\\{[0-9]" "$bots_file" || true

  echo "- Any ')+' occurrences (simple indicator of repeating groups):"
  rg -n "\\)\\+" "$bots_file" || true

  echo "- Very long regex literals (>200 chars):"
  awk '{
    if (match($0,/regex[[:space:]]*:[[:space:]]*["'\'']/)) {
      line=$0
      # accumulate continued lines until closing quote found
      q=substr($0, RSTART+RLENGTH)
      if (gsub(/["'\'']/,"",q)==0) {
        # attempt simple length check on the whole line
        if (length(line) > 200) print NR ":" line
      } else {
        if (length(line) > 200) print NR ":" line
      }
    }
  }' "$bots_file" || true
fi

echo
echo "Script finished."

Length of output: 38333


Action: low immediate ReDoS risk — recommend adding a CI safe-regex scan (I can draft it)

Short summary:

  • I inspected apps/basket/src/utils/user-agent.ts and packages/shared/src/lists/bots.ts (device-detector-derived).
  • Heuristic checks for backreferences, nested/repeated quantifiers, ')+' patterns and very long regex literals returned no obvious ReDoS patterns.
  • Because these patterns are internal, the immediate risk is low — but they run against untrusted UA strings, so defensive CI/runtime checks are still worthwhile.

Locations:

  • apps/basket/src/utils/user-agent.ts — compiledBotRegexes: new RegExp(bot.regex, 'i') (around line ~80).
  • packages/shared/src/lists/bots.ts — source list of regex patterns.

Recommended (minimal, prioritized):

  • Add a build-time safe-regex scan (e.g., safe-regex / safe-regex-cli or custom checker) on packages/shared/src/lists/bots.ts to surface risky patterns in CI.
  • Optional runtime guards: keep UA length checks (already present), consider a fast substring pre-check (e.g., require "bot"/known token) before running full regexes as defence-in-depth.

Want me to draft the build-time script that scans packages/shared/src/lists/bots.ts and emits a CI warning/fail for risky patterns?

🧰 Tools
🪛 ast-grep (0.38.6)

[warning] 83-83: Regular expression constructed from variable input detected. This can lead to Regular Expression Denial of Service (ReDoS) attacks if the variable contains malicious patterns. Use libraries like 'recheck' to validate regex safety or use static patterns.
Context: new RegExp(bot.regex, 'i')
Note: [CWE-1333] Inefficient Regular Expression Complexity [REFERENCES]
- https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS
- https://cwe.mitre.org/data/definitions/1333.html

(regexp-from-variable)

🤖 Prompt for AI Agents
In apps/basket/src/utils/user-agent.ts around lines 80–86 the code compiles
external bot regexes with new RegExp(bot.regex, 'i'); add a CI build-time
safe-regex scan against packages/shared/src/lists/bots.ts (use safe-regex or
safe-regex-cli or a small custom checker) that fails or warns on patterns
flagged as ReDoS-risky; additionally add a defensive runtime guard: keep the
existing UA length check, wrap regex compilation/tests in try/catch and
optionally add a cheap substring token pre-check (e.g., look for "bot" or known
tokens) before running the full regex tests to reduce exposure.

export function detectBot(
userAgent: string,
request: Request
Expand All @@ -87,7 +95,7 @@ export function detectBot(
} {
const ua = userAgent || '';

const detectedBot = bots.find((bot) => new RegExp(bot.regex, 'i').test(ua));
const detectedBot = compiledBotRegexes.find((bot) => bot.regex.test(ua));
if (detectedBot) {
return {
isBot: true,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use client';

import type { Website } from '@databuddy/shared';
import type { Website } from '@databuddy/shared/types';
import { GlobeIcon } from '@phosphor-icons/react';
import { cn } from '@/lib/utils';

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use client';

import { getCountryCode, getCountryName } from '@databuddy/shared';
import { getCountryCode, getCountryName } from '@databuddy/shared/country-codes';
import { QuestionIcon } from '@phosphor-icons/react';
import {
AlertTriangle,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use client';

import { type DynamicQueryFilter, filterOptions } from '@databuddy/shared';
import type { DynamicQueryFilter } from '@databuddy/shared/types';
import { filterOptions } from '@databuddy/shared/lists';
import { FunnelIcon } from '@phosphor-icons/react';
import { useParams } from 'next/navigation';
import { Suspense, useCallback, useState } from 'react';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import type {
DateRange as BaseDateRange,
DynamicQueryFilter,
Website,
} from '@databuddy/shared';
} from '@databuddy/shared/types';

// Extended date range with granularity
export interface DateRange extends BaseDateRange {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use client';

import type { DateRange, DynamicQueryFilter } from '@databuddy/shared';
import type { DateRange, DynamicQueryFilter } from '@databuddy/shared/types';
import { ArrowClockwiseIcon, BugIcon } from '@phosphor-icons/react';
import { use, useCallback, useEffect, useMemo, useState } from 'react';
import { toast } from 'sonner';
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use client';

import { filterOptions } from '@databuddy/shared';
import { filterOptions } from '@databuddy/shared/lists';
import {
DragDropContext,
Draggable,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use client';

import { filterOptions, type GoalFilter } from '@databuddy/shared';
import type { GoalFilter } from '@databuddy/shared/types';
import { filterOptions } from '@databuddy/shared/lists';
import {
Eye,
MouseMiddleClick,
Expand Down
2 changes: 1 addition & 1 deletion apps/dashboard/app/(main)/websites/[id]/map/page.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use client';

import type { LocationData } from '@databuddy/shared';
import type { LocationData } from '@databuddy/shared/types';
import { GlobeIcon, MapPinIcon, QuestionIcon } from '@phosphor-icons/react';
import dynamic from 'next/dynamic';
import Image from 'next/image';
Expand Down
3 changes: 2 additions & 1 deletion apps/dashboard/app/(main)/websites/[id]/page.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use client';

import { type DynamicQueryFilter, filterOptions } from '@databuddy/shared';
import type { DynamicQueryFilter } from '@databuddy/shared/types';
import { filterOptions } from '@databuddy/shared/lists';
import { ArrowClockwiseIcon, WarningIcon, XIcon } from '@phosphor-icons/react';
import { useQueryClient } from '@tanstack/react-query';
import dayjs from 'dayjs';
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { DateRange } from '@databuddy/shared';
import type { DateRange } from '@databuddy/shared/types';
import { type UseQueryOptions, useQuery } from '@tanstack/react-query';
import { useMemo } from 'react';

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { ProcessedMiniChartData, Website } from '@databuddy/shared';
import type { ProcessedMiniChartData, Website } from '@databuddy/shared/types';
import {
ArrowRightIcon,
MinusIcon,
Expand Down
2 changes: 1 addition & 1 deletion apps/dashboard/components/analytics/map-component.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use client';

import type { CountryData, LocationData } from '@databuddy/shared';
import type { CountryData, LocationData } from '@databuddy/shared/types';
import { scalePow } from 'd3-scale';
import type { Feature, GeoJsonObject } from 'geojson';
import type { Layer, Map as LeafletMap } from 'leaflet';
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { Website } from '@databuddy/shared';
import type { Website } from '@databuddy/shared/types';
import { CaretLeftIcon, PlanetIcon } from '@phosphor-icons/react';
import Link from 'next/link';
import { Button } from '@/components/ui/button';
Expand Down
4 changes: 2 additions & 2 deletions apps/dashboard/hooks/use-dynamic-query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ import type {
RevenueBreakdownData,
RevenueSummaryData,
RevenueTrendData,
} from '@databuddy/shared';
import { getCountryCode, getCountryName } from '@databuddy/shared';
} from '@databuddy/shared/types';
import { getCountryCode, getCountryName } from '@databuddy/shared/country-codes';
import {
type UseInfiniteQueryOptions,
type UseQueryOptions,
Expand Down
2 changes: 1 addition & 1 deletion apps/dashboard/hooks/use-filters.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { DynamicQueryFilter } from '@databuddy/shared';
import type { DynamicQueryFilter } from '@databuddy/shared/types';
import { useCallback } from 'react';

export const operatorOptions = [
Expand Down
2 changes: 1 addition & 1 deletion apps/dashboard/hooks/use-funnels.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { DateRange } from '@databuddy/shared';
import type { DateRange } from '@databuddy/shared/types';
import { useQueries, useQueryClient } from '@tanstack/react-query';
import { useMemo } from 'react';
import { trpc } from '@/lib/trpc';
Expand Down
2 changes: 1 addition & 1 deletion apps/dashboard/hooks/use-goals.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { GoalFilter } from '@databuddy/shared';
import type { GoalFilter } from '@databuddy/shared/types';
import { useQueryClient } from '@tanstack/react-query';
import { useMemo } from 'react';
import { trpc } from '@/lib/trpc';
Expand Down
2 changes: 1 addition & 1 deletion apps/dashboard/hooks/use-preferences.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import {
convertToTimezone,
formatDate,
getBrowserTimezone,
} from '@databuddy/shared';
} from '@databuddy/shared/utils/date-utils';
import { useCallback } from 'react';
import { trpc } from '@/lib/trpc';

Expand Down
2 changes: 1 addition & 1 deletion apps/docs/app/(home)/api/actions.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use server';

import type { QueryBuilderMeta } from '@databuddy/shared';
import type { QueryBuilderMeta } from '@databuddy/shared/types';

interface QueryConfig {
allowedFilters: string[];
Expand Down
2 changes: 1 addition & 1 deletion apps/docs/app/(home)/api/query-type-dialog.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use client';

import type { QueryBuilderMeta } from '@databuddy/shared';
import type { QueryBuilderMeta } from '@databuddy/shared/types';
import { CheckCircleIcon, CopyIcon } from '@phosphor-icons/react';
import { useEffect, useState } from 'react';
import { Badge } from '@/components/ui/badge';
Expand Down
2 changes: 1 addition & 1 deletion apps/docs/app/(home)/api/query-types-grid.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use client';

import type { QueryBuilderMeta } from '@databuddy/shared';
import type { QueryBuilderMeta } from '@databuddy/shared/types';
import { CaretRightIcon } from '@phosphor-icons/react';
import { Card, CardContent } from '@/components/ui/card';
import { QueryTypeDialog } from './query-type-dialog';
Expand Down
2 changes: 1 addition & 1 deletion apps/docs/app/(home)/blog/[slug]/page.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { formatDate } from '@databuddy/shared';
import { formatDate } from '@databuddy/shared/utils/date-utils';
import {
ArrowLeftIcon,
CalendarIcon,
Expand Down
Loading
Loading