Skip to content

Commit cc396e5

Browse files
themaroltthemarolt
committed
Merge branch 'main' into deprecate-questdb-CM-420
2 parents daabe9c + 1b008fa commit cc396e5

10 files changed

Lines changed: 145 additions & 47 deletions

File tree

‎backend/src/database/migrations/U1759157837__add_activity_types.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
-- The UP migration only added data, no need to delete it.

‎backend/src/database/migrations/V1759157837__add_activity_types.sql‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
-- GitLab platform (Code Contributions)
2+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
3+
('authored-commit', 'gitlab', true, false);
4+
5+
-- Fix Groups.io platform name
6+
UPDATE "activityTypes" SET platform = 'groupsio' WHERE platform = 'groups.io';
7+
8+
-- Discourse platform (Collaboration)
9+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
10+
('create_topic', 'discourse', false, true),
11+
('message_in_topic', 'discourse', false, true);
12+
13+
14+
-- Insert other activity types
15+
16+
-- Dev.to
17+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
18+
('comment', 'devto', false, false);
19+
20+
-- Discord
21+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
22+
('joined_guild', 'discord', false, false);
23+
24+
-- Discourse
25+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
26+
('join', 'discourse', false, false),
27+
('like', 'discourse', false, false);
28+
29+
-- Github
30+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
31+
('pull_request-assigned', 'github', false, false),
32+
('fork', 'github', false, false),
33+
('star', 'github', false, false),
34+
('unstar', 'github', false, false);
35+
36+
-- Gitlab
37+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
38+
('merge_request-assigned', 'gitlab', false, false),
39+
('fork', 'gitlab', false, false),
40+
('star', 'gitlab', false, false);
41+
42+
-- Groups.io
43+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
44+
('member_join', 'groupsio', false, false),
45+
('member_leave', 'groupsio', false, false);
46+
47+
-- HackerNews
48+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
49+
('post', 'hackernews', false, false),
50+
('comment', 'hackernews', false, false);
51+
52+
-- LinkedIn
53+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
54+
('comment', 'linkedin', false, false),
55+
('reaction', 'linkedin', false, false);
56+
57+
-- Reddit
58+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
59+
('post', 'reddit', false, false),
60+
('comment', 'reddit', false, false);
61+
62+
-- Slack
63+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
64+
('channel_joined', 'slack', false, false);
65+
66+
-- Twitter
67+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration") VALUES
68+
('hashtag', 'twitter', false, false),
69+
('mention', 'twitter', false, false),
70+
('follow', 'twitter', false, false);

‎frontend/src/modules/admin/modules/integration/pages/integration-list.page.vue‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@
7272
<template #default="{ progress, progressError }">
7373
<div v-if="platformsByStatus.length > 0" class="flex flex-col gap-6">
7474
<lf-tooltip
75-
v-if="isTeamUser()"
75+
v-if="isTeamUser"
7676
class="ml-auto"
7777
placement="top"
7878
content-class="!max-w-76 !p-3 !text-start"
@@ -182,6 +182,13 @@ const getIntegrationCountPerStatus = computed<Record<string, number>>(() => {
182182
return statusCount;
183183
});
184184
185+
const authStore = useAuthStore();
186+
const userId = computed(() => authStore.user?.id);
187+
const teamUserIds = computed(() => config.permissions.teamUserIds);
188+
const env = computed(() => config.env);
189+
190+
const isTeamUser = computed(() => env.value !== 'production' || teamUserIds.value?.includes(userId.value));
191+
185192
onMounted(() => {
186193
localStorage.setItem('segmentId', id);
187194
localStorage.setItem('segmentGrandparentId', grandparentId);
@@ -201,15 +208,7 @@ const updateGithubVersion = () => {
201208
if (githubIntegration) {
202209
return !!githubIntegration.isNango;
203210
}
204-
return !!isTeamUser();
205-
};
206-
207-
const isTeamUser = () => {
208-
const authStore = useAuthStore();
209-
const userId = authStore.user?.id;
210-
return (
211-
config.permissions.teamUserIds?.includes(userId) || config.env === 'local'
212-
);
211+
return !!isTeamUser.value;
213212
};
214213
</script>
215214

‎frontend/src/modules/admin/modules/projects/components/view/lf-projects-table.vue‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,11 @@ const props = defineProps({
130130
const { hasPermission, hasAccessToSegmentId } = usePermissions();
131131
132132
const authStore = useAuthStore();
133-
const isTeamUser = computed(() => config.env !== 'production' || config.permissions.teamUserIds?.includes(authStore.user?.id));
133+
const userId = computed(() => authStore.user?.id);
134+
const teamUserIds = computed(() => config.permissions.teamUserIds);
135+
const env = computed(() => config.env);
136+
137+
const isTeamUser = computed(() => env.value !== 'production' || teamUserIds.value?.includes(userId.value));
134138
135139
</script>
136140

‎frontend/src/modules/admin/pages/admin-panel.page.vue‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,11 @@ const changeView = (view: string) => {
125125
126126
const isAdminUser = computed(() => roles.value.includes(LfRole.admin));
127127
128-
const isTeamUser = computed(() => config.env !== 'production' || config.permissions.teamUserIds?.includes(authStore.user?.id));
128+
const userId = computed(() => authStore.user?.id);
129+
const teamUserIds = computed(() => config.permissions.teamUserIds);
130+
const env = computed(() => config.env);
131+
132+
const isTeamUser = computed(() => env.value !== 'production' || teamUserIds.value?.includes(userId.value));
129133
130134
const changeAdminPanelView = (view: string) => {
131135
switch (view) {

‎scripts/cli‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -719,8 +719,8 @@ function kill_all_containers() {
719719

720720
function service_start() {
721721
for SERVICE in $CONTAINERS; do
722-
if [[ ${#INGORED_SERVICES[@]} -ne 0 ]]; then
723-
for IGNORED_SERVICE in "${INGORED_SERVICES[@]}"; do
722+
if [[ ${#IGNORED_SERVICES[@]} -ne 0 ]]; then
723+
for IGNORED_SERVICE in "${IGNORED_SERVICES[@]}"; do
724724
if [[ "$SERVICE" == "${IGNORED_SERVICE}" ]]; then
725725
SKIP=1
726726
break
@@ -802,17 +802,17 @@ while test $# -gt 0; do
802802
exit
803803
;;
804804
start-e2e)
805-
declare -a INGORED_SERVICES=("job-generator" "data-sink-worker" "integration-run-worker" "integration-stream-worker")
805+
declare -a IGNORED_SERVICES=("job-generator" "data-sink-worker" "integration-run-worker" "integration-stream-worker")
806806
start
807807
exit
808808
;;
809809
start-be)
810-
declare -a INGORED_SERVICES=("frontend")
810+
declare -a IGNORED_SERVICES=("frontend")
811811
start
812812
exit
813813
;;
814814
start-dev)
815-
# INGORED_SERVICES=("python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker")
815+
# IGNORED_SERVICES=("python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker")
816816
DEV=1
817817
start
818818
exit
@@ -823,31 +823,31 @@ while test $# -gt 0; do
823823
exit
824824
;;
825825
clean-start-dev)
826-
# INGORED_SERVICES=("python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker")
826+
# IGNORED_SERVICES=("python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker")
827827
CLEAN_START=1
828828
DEV=1
829829
start
830830
exit
831831
;;
832832
start-backend)
833-
declare -a INGORED_SERVICES=("frontend")
833+
declare -a IGNORED_SERVICES=("frontend")
834834
start
835835
exit
836836
;;
837837
start-backend-dev)
838-
declare -a INGORED_SERVICES=("frontend")
838+
declare -a IGNORED_SERVICES=("frontend")
839839
DEV=1
840840
start
841841
exit
842842
;;
843843
clean-start-backend)
844-
declare -a INGORED_SERVICES=("frontend")
844+
declare -a IGNORED_SERVICES=("frontend")
845845
CLEAN_START=1
846846
start
847847
exit
848848
;;
849849
clean-start-backend-dev)
850-
declare -a INGORED_SERVICES=("frontend")
850+
declare -a IGNORED_SERVICES=("frontend")
851851
CLEAN_START=1
852852
DEV=1
853853
start
@@ -859,7 +859,7 @@ while test $# -gt 0; do
859859
exit
860860
;;
861861
service-restart-be)
862-
declare -a INGORED_SERVICES=("frontend")
862+
declare -a IGNORED_SERVICES=("frontend")
863863
kill_all_containers
864864
service_start
865865
exit
@@ -878,7 +878,7 @@ while test $# -gt 0; do
878878
exit
879879
;;
880880
clean-start-fe-dev)
881-
INGORED_SERVICES=("frontend" "python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker")
881+
IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker")
882882
CLEAN_START=1
883883
DEV=1
884884
start

‎services/apps/git_integration/src/crowdgit/services/clone/clone_service.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -256,16 +256,16 @@ async def _calculate_batch_depth(self, repo_path: str, remote: str) -> int:
256256
total_branches_tags = len(total_branches_tags.splitlines())
257257
if total_branches_tags <= 200:
258258
# Small repo, get a decent amount of history
259-
calculated_depth = 500
259+
calculated_depth = 200
260260
elif total_branches_tags <= 1000:
261261
# Medium repo, get a moderate amount of history
262-
calculated_depth = 300
262+
calculated_depth = 100
263263
elif total_branches_tags <= 5000:
264264
# Large repo, get less history
265-
calculated_depth = 20
265+
calculated_depth = 10
266266
else:
267267
# Very large repo, get a minimal history
268-
calculated_depth = 10
268+
calculated_depth = 5
269269
self.logger.info(
270270
f"total_branches_tags={total_branches_tags}, calculated_depth={calculated_depth}"
271271
)

‎services/apps/git_integration/src/crowdgit/services/commit/commit_service.py‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import time
88
import uuid
99
from concurrent.futures import ProcessPoolExecutor
10+
from concurrent.futures.process import BrokenProcessPool
1011
from decimal import Decimal
1112
from typing import Any
1213

@@ -69,6 +70,7 @@ def _get_or_create_pool(self) -> ProcessPoolExecutor:
6970
def cleanup_process_pool(self):
7071
"""Cleanup process pool to prevent resource leaks"""
7172
if self.process_pool:
73+
self.logger.info("Cleaning up process pool")
7274
self.process_pool.shutdown(wait=False)
7375
self.process_pool = None
7476

@@ -659,26 +661,44 @@ async def _process_activities_from_commits(
659661

660662
executor = self._get_or_create_pool()
661663

662-
futures = [
663-
loop.run_in_executor(
664-
executor,
665-
CommitService.process_commits_chunk,
666-
chunk,
667-
repo_path,
668-
edge_commit_hash,
669-
remote,
670-
segment_id,
671-
integration_id,
672-
)
673-
for chunk in chunks
674-
]
664+
try:
665+
futures = [
666+
loop.run_in_executor(
667+
executor,
668+
CommitService.process_commits_chunk,
669+
chunk,
670+
repo_path,
671+
edge_commit_hash,
672+
remote,
673+
segment_id,
674+
integration_id,
675+
)
676+
for chunk in chunks
677+
]
678+
self.logger.info(f"Submitted {len(futures)} tasks to process pool")
679+
except Exception as e:
680+
if isinstance(e, BrokenProcessPool):
681+
self.logger.warning("BrokenProcessPool during task submission, cleaning up")
682+
self.cleanup_process_pool()
683+
raise
675684

676685
# Save each chunk's activities as they complete
686+
completed_chunks = 0
677687
for future in asyncio.as_completed(futures):
678-
chunk_activities_db, chunk_activities_queue = await future
679-
if chunk_activities_db and chunk_activities_queue:
680-
await batch_insert_activities(chunk_activities_db)
681-
await self.queue_service.send_batch_activities(chunk_activities_queue)
688+
try:
689+
chunk_activities_db, chunk_activities_queue = await future
690+
completed_chunks += 1
691+
self.logger.info(f"Chunk {completed_chunks}/{len(futures)} completed")
692+
if chunk_activities_db and chunk_activities_queue:
693+
await batch_insert_activities(chunk_activities_db)
694+
await self.queue_service.send_batch_activities(chunk_activities_queue)
695+
except Exception as e:
696+
if isinstance(e, BrokenProcessPool):
697+
self.logger.warning(
698+
f"BrokenProcessPool after {completed_chunks}/{len(futures)} chunks, cleaning up"
699+
)
700+
self.cleanup_process_pool()
701+
raise
682702

683703
@staticmethod
684704
def _validate_commit_structure(commit_lines: list[str]) -> bool:

‎services/apps/git_integration/src/crowdgit/settings.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def load_env_var(key: str, required=True, default=None):
1818
REPOSITORY_UPDATE_INTERVAL_HOURS = int(
1919
load_env_var("REPOSITORY_UPDATE_INTERVAL_HOURS", default=24)
2020
)
21-
MAX_WORKER_PROCESSES = int(load_env_var("MAX_WORKER_PROCESSES", default=min(4, os.cpu_count())))
21+
MAX_WORKER_PROCESSES = int(load_env_var("MAX_WORKER_PROCESSES", default=min(3, os.cpu_count())))
2222
DEFAULT_TENANT_ID = load_env_var(
2323
"CROWD_SSO_LF_TENANT_ID", default="875c38bd-2b1b-4e91-ad07-0cfbabb4c49f"
2424
)

‎services/libs/data-access-layer/src/organizations/organizations.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,7 @@ export async function getTimeseriesOfActiveOrganizations(
471471
const query = `
472472
SELECT
473473
COUNT_DISTINCT("organizationId") AS count,
474-
DATE_TRUNC('day', timestamp)
474+
DATE_TRUNC('day', timestamp) as date
475475
FROM "activityRelations"
476476
WHERE "organizationId" IS NOT NULL
477477
${params.segmentIds ? 'AND "segmentId" IN ($(segmentIds:csv))' : ''}

0 commit comments

Comments
 (0)