feat: detect and store repo license via licensee IN-1105

Signed-off-by: Gašper Grom <gasper.grom@gmail.com>

Author: gaspergrom

backend/src/database/migrations/U1778154987__addLicenseToRepositories.sql

@@ -0,0 +1 @@
+ALTER TABLE public.repositories DROP COLUMN license;

backend/src/database/migrations/V1778154987__addLicenseToRepositories.sql

@@ -0,0 +1 @@
+ALTER TABLE public.repositories ADD COLUMN license VARCHAR(255);

scripts/services/docker/Dockerfile.git_integration

@@ -83,10 +83,13 @@ RUN apt-get update && apt-get install -y \
     ca-certificates \
     git \
     ripgrep \
+    ruby \
+    ruby-dev \
+    build-essential \
     --no-install-recommends \
-    && rm -rf /var/lib/apt/lists/* \
-    && apt-get clean \
-    && apt-get autoremove -y
+    && gem install licensee -v '10.0.0' --no-document \
+    && apt-get remove --autoremove -y ruby-dev build-essential \
+    && rm -rf /var/lib/apt/lists/*
 
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \

services/apps/git_integration/src/crowdgit/database/crud.py

@@ -283,6 +283,17 @@ async def update_last_processed_commit(repo_id: str, commit_hash: str, branch: s
     return str(result)
 
 
+async def update_repository_license(repository_id: str, license_spdx: str | None) -> None:
+    sql_query = """
+    UPDATE public.repositories
+        SET license = $1,
+        "updatedAt" = NOW()
+    WHERE id = $2
+      AND ($1 IS NOT NULL OR license IS NULL)
+    """
+    await execute(sql_query, (license_spdx, repository_id))
+
+
 async def mark_repo_as_processed(repo_id: str, repo_state: RepositoryState):
     sql_query = """
     UPDATE git."repositoryProcessing"

services/apps/git_integration/src/crowdgit/server.py

@@ -8,6 +8,7 @@
 from crowdgit.services import (
     CloneService,
     CommitService,
+    LicenseService,
     MaintainerService,
     QueueService,
     SoftwareValueService,
@@ -28,6 +29,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     software_value_service = SoftwareValueService()
     vulnerability_scanner_service = VulnerabilityScannerService()
     maintainer_service = MaintainerService()
+    license_service = LicenseService()
 
     worker_task = None
     worker = RepositoryWorker(
@@ -36,6 +38,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
         software_value_service=software_value_service,
         vulnerability_scanner_service=vulnerability_scanner_service,
         maintainer_service=maintainer_service,
+        license_service=license_service,
         queue_service=queue_service,
     )
     logger.info("Repo worker initialized")

services/apps/git_integration/src/crowdgit/services/__init__.py

@@ -1,6 +1,7 @@
 from crowdgit.services.base.base_service import BaseService
 from crowdgit.services.clone.clone_service import CloneService
 from crowdgit.services.commit.commit_service import CommitService
+from crowdgit.services.license.license_service import LicenseService
 from crowdgit.services.maintainer.maintainer_service import MaintainerService
 from crowdgit.services.queue.queue_service import QueueService
 from crowdgit.services.software_value.software_value_service import SoftwareValueService
@@ -12,6 +13,7 @@
     "BaseService",
     "CloneService",
     "CommitService",
+    "LicenseService",
     "SoftwareValueService",
     "VulnerabilityScannerService",
     "MaintainerService",

services/apps/git_integration/src/crowdgit/services/license/__init__.py

@@ -0,0 +1,3 @@
+from crowdgit.services.license.license_service import LicenseService
+
+__all__ = ["LicenseService"]

services/apps/git_integration/src/crowdgit/services/license/license_service.py

@@ -0,0 +1,40 @@
+import json
+
+from crowdgit.errors import CommandExecutionError, CommandTimeoutError
+from crowdgit.services.base.base_service import BaseService
+from crowdgit.services.utils import run_shell_command
+
+
+class LicenseService(BaseService):
+    """Detects SPDX license from a cloned repository using the licensee gem."""
+
+    async def detect(self, repo_path: str) -> str | None:
+        """Run licensee against repo_path and return the SPDX identifier, or None."""
+        try:
+            output = await run_shell_command(["licensee", "detect", "--json", repo_path])
+        except CommandExecutionError:
+            self.logger.info(f"licensee found no license in {repo_path}")
+            return None
+        except CommandTimeoutError as e:
+            self.logger.warning(f"licensee timed out: {repr(e)}")
+            return None
+        except FileNotFoundError as e:
+            self.logger.warning(f"licensee binary not found in PATH: {repr(e)}")
+            return None
+        except Exception as e:
+            self.logger.warning(f"licensee failed: {repr(e)}")
+            return None
+
+        try:
+            data = json.loads(output)
+            matched = data.get("matched_license") or {}
+            spdx_id = matched.get("spdx_id")
+            confidence = matched.get("confidence")
+            if spdx_id:
+                self.logger.info(f"License detected: {spdx_id} (confidence={confidence}) in {repo_path}")
+            else:
+                self.logger.info(f"No SPDX license matched in {repo_path}")
+            return spdx_id
+        except Exception as e:
+            self.logger.warning(f"Failed to parse licensee output: {repr(e)}")
+            return None

services/apps/git_integration/src/crowdgit/worker/repository_worker.py

@@ -7,6 +7,7 @@
     mark_repo_as_processed,
     release_repo,
     update_last_processed_commit,
+    update_repository_license,
 )
 from crowdgit.enums import RepositoryState
 from crowdgit.errors import (
@@ -22,6 +23,7 @@
 from crowdgit.services import (
     CloneService,
     CommitService,
+    LicenseService,
     MaintainerService,
     QueueService,
     SoftwareValueService,
@@ -46,13 +48,15 @@ def __init__(
         software_value_service: SoftwareValueService,
         vulnerability_scanner_service: VulnerabilityScannerService,
         maintainer_service: MaintainerService,
+        license_service: LicenseService,
         queue_service: QueueService,
     ):
         self.clone_service = clone_service
         self.commit_service = commit_service
         self.software_value_service = software_value_service
         self.vulnerability_scanner_service = vulnerability_scanner_service
         self.maintainer_service = maintainer_service
+        self.license_service = license_service
         self.queue_service = queue_service
         self._shutdown = False
 
@@ -236,6 +240,8 @@ async def _process_single_repository(self, repository: Repository):
                         repository.id, batch_info.repo_path, repository.url
                     )
                     await self.maintainer_service.process_maintainers(repository, batch_info)
+                    license_spdx = await self.license_service.detect(batch_info.repo_path)
+                    await update_repository_license(repository.id, license_spdx)
                 await self.commit_service.process_single_batch_commits(
                     repository,
                     batch_info,