feat: optimize data processing for ai code tracker

Signed-off-by: Gašper Grom <gasper.grom@gmail.com>

Author: gaspergrom

services/libs/tinybird/datasources/ai_code_tracker_commits_ds.datasource

@@ -0,0 +1,17 @@
+DESCRIPTION >
+    - `ai_code_tracker_commits_ds` contains only authored-commit activities, pre-filtered from the full activities table.
+    - Populated daily by `ai_code_tracker_commits_copy.pipe`.
+    - Stores only the fields needed for AI tool detection: timestamp, title, body, attributes.
+    - Reduces the dataset from ~1B rows to only commits, with sorting keys optimized for the AI pattern matching step.
+
+TAGS "Report"
+
+SCHEMA >
+    `timestamp` DateTime,
+    `title` String DEFAULT '',
+    `body` String DEFAULT '',
+    `attributes` String DEFAULT ''
+
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYear(timestamp)
+ENGINE_SORTING_KEY timestamp

services/libs/tinybird/pipes/ai_code_tracker_commits_copy.pipe

@@ -0,0 +1,21 @@
+DESCRIPTION >
+    - `ai_code_tracker_commits_copy.pipe` extracts only authored-commit rows from `activities_deduplicated_ds`.
+    - Runs daily to populate `ai_code_tracker_commits_ds` with a small subset (~commits only) of the full 1B+ activities table.
+    - This intermediate datasource is then used by `ai_code_tracker_copy.pipe` for fast AI pattern matching.
+
+TAGS "Report"
+
+NODE ai_code_tracker_commits_copy_result
+SQL >
+    SELECT
+        a.timestamp,
+        a.title,
+        a.body,
+        a.attributes
+    FROM activities_deduplicated_ds a
+    WHERE a.type = 'authored-commit'
+
+TYPE COPY
+TARGET_DATASOURCE ai_code_tracker_commits_ds
+COPY_MODE replace
+COPY_SCHEDULE 0 2 * * *

services/libs/tinybird/pipes/ai_code_tracker_copy.pipe

@@ -1,6 +1,6 @@
 DESCRIPTION >
     - `ai_code_tracker_copy.pipe` materializes AI-assisted commit counts into `ai_code_tracker_ds`.
-    - Runs daily to scan `activities_deduplicated_ds` for commits matching AI tool patterns.
+    - Runs daily (after commits copy) reading from `ai_code_tracker_commits_ds` (only commits, not all 1B+ activities).
     - Classifies commits by AI tool based on title, body, and attributes content.
     - Stores monthly aggregates per tool plus total commits per month (toolKey = '__total__').
     - Uses multiSearchAnyCaseInsensitive for fast single-pass pre-filtering before classification.
@@ -9,34 +9,30 @@ TAGS "Report"
 
 NODE ai_code_tracker_copy_totals
 DESCRIPTION >
-    Total commits per month - no LIKE scanning needed, just a simple count
+    Total commits per month - simple count, no string scanning
 SQL >
     SELECT
-        toStartOfMonth(a.timestamp) AS monthStart,
+        toStartOfMonth(timestamp) AS monthStart,
         '__total__' AS toolKey,
         count() AS commitCount
-    FROM activities_deduplicated_ds a
-    WHERE a.type = 'authored-commit'
+    FROM ai_code_tracker_commits_ds
     GROUP BY monthStart
 
 NODE ai_code_tracker_copy_prefilter
 DESCRIPTION >
-    Fast pre-filter: only keep commits that contain ANY AI keyword in title, body, or attributes.
-    multiSearchAnyCaseInsensitive does a single pass instead of dozens of separate LIKE checks.
+    Fast pre-filter: only keep commits containing ANY AI keyword.
+    multiSearchAnyCaseInsensitive does a single pass instead of dozens of separate checks.
 SQL >
     SELECT
-        toStartOfMonth(a.timestamp) AS monthStart,
-        a.title,
-        a.body,
-        a.attributes
-    FROM activities_deduplicated_ds a
+        toStartOfMonth(timestamp) AS monthStart,
+        title,
+        body,
+        attributes
+    FROM ai_code_tracker_commits_ds
     WHERE
-        a.type = 'authored-commit'
-        AND (
-            multiSearchAnyCaseInsensitive(a.title, ['copilot', 'chatgpt', 'claude', 'cursor', 'codewhisperer', 'gemini', 'codeium', 'aider', 'devin', 'tabnine', 'ai-generated', 'ai generated']) != 0
-            OR multiSearchAnyCaseInsensitive(a.body, ['copilot', 'chatgpt', 'claude', 'cursor', 'codewhisperer', 'gemini', 'codeium', 'aider', 'devin', 'tabnine', 'ai-generated', 'ai generated', 'co-authored-by']) != 0
-            OR multiSearchAnyCaseInsensitive(a.attributes, ['copilot', 'ai-generated']) != 0
-        )
+        multiSearchAnyCaseInsensitive(title, ['copilot', 'chatgpt', 'claude', 'cursor', 'codewhisperer', 'gemini', 'codeium', 'aider', 'devin', 'tabnine', 'ai-generated', 'ai generated']) != 0
+        OR multiSearchAnyCaseInsensitive(body, ['copilot', 'chatgpt', 'claude', 'cursor', 'codewhisperer', 'gemini', 'codeium', 'aider', 'devin', 'tabnine', 'ai-generated', 'ai generated', 'co-authored-by']) != 0
+        OR multiSearchAnyCaseInsensitive(attributes, ['copilot', 'ai-generated']) != 0
 
 NODE ai_code_tracker_copy_classify
 DESCRIPTION >
@@ -45,7 +41,7 @@ SQL >
     SELECT
         monthStart,
         multiIf(
-            positionCaseInsensitive(title, 'github copilot') > 0 OR positionCaseInsensitive(body, 'co-authored-by') > 0 AND positionCaseInsensitive(body, 'copilot') > 0 OR positionCaseInsensitive(attributes, 'copilot') > 0, 'github-copilot',
+            positionCaseInsensitive(title, 'github copilot') > 0 OR (positionCaseInsensitive(body, 'co-authored-by') > 0 AND positionCaseInsensitive(body, 'copilot') > 0) OR positionCaseInsensitive(attributes, 'copilot') > 0, 'github-copilot',
             positionCaseInsensitive(title, 'cursor') > 0 OR positionCaseInsensitive(body, 'cursor') > 0, 'cursor',
             positionCaseInsensitive(title, 'claude') > 0 OR positionCaseInsensitive(body, 'claude') > 0, 'claude',
             positionCaseInsensitive(title, 'chatgpt') > 0 OR positionCaseInsensitive(body, 'chatgpt') > 0, 'chatgpt',