fix: skip llm parsing for oversized project affiliation files (#4319)

Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com>

Author: skwowet

services/apps/git_integration/src/crowdgit/services/affiliation/affiliation_service.py

@@ -52,6 +52,7 @@ class AffiliationService(BaseService):
 
     MAX_CHUNK_SIZE = 5000
     MAX_CONCURRENT_CHUNKS = 3
+    MAX_FILE_SIZE_BYTES = 1_000_000
     FILE_PICKER_PREVIEW_MAX_CHARS = 400
     FILE_PICKER_BATCH_SIZE = 20
 
@@ -74,14 +75,19 @@ class AffiliationService(BaseService):
     )
 
     @staticmethod
-    async def read_text_file(file_path: str) -> str:
-        async with aiofiles.open(file_path, "rb") as f:
-            return safe_decode(await f.read())
-
-    @staticmethod
-    def compute_file_hash(content: str) -> str:
-        """SHA-256 hex digest of UTF-8 file content (not a Git blob SHA)."""
-        return hashlib.sha256(content.encode("utf-8")).hexdigest()
+    async def compute_file_hash_from_path(
+        file_path: str, *, retain_content: bool = False
+    ) -> tuple[bytes | None, str]:
+        """SHA-256 of raw file bytes. Set retain_content to decode and parse under the size limit."""
+        hasher = hashlib.sha256()
+        chunks: list[bytes] = []
+        async with aiofiles.open(file_path, "rb") as affiliation_file:
+            while file_chunk := await affiliation_file.read(1024 * 1024):
+                hasher.update(file_chunk)
+                if retain_content:
+                    chunks.append(file_chunk)
+        file_bytes = b"".join(chunks) if retain_content else None
+        return file_bytes, hasher.hexdigest()
 
     @classmethod
     def is_text_file_path(cls, relative_path: str) -> bool:
@@ -955,15 +961,42 @@ async def process_affiliations(
                 raise AffiliationFileNotFoundError(ai_cost=ai_cost)
 
             file_path_on_disk = os.path.join(batch_info.repo_path, latest_file_path)
-            content = await self.read_text_file(file_path_on_disk)
-            file_hash = self.compute_file_hash(content)
-            latest_file_hash = file_hash
-
-            affiliations, parse_cost = await self.resolve_snapshot(
-                registry,
-                content,
-                file_hash,
-            )
+            file_size_bytes = await aiofiles.os.path.getsize(file_path_on_disk)
+            # Too big for llm — mark unusable and move on.
+            if file_size_bytes > self.MAX_FILE_SIZE_BYTES:
+                # Steady state: already gave up on this file; getsize is enough.
+                if (
+                    registry
+                    and registry.status == AffiliationRegistryStatus.UNUSABLE.value
+                    and registry.file_path == latest_file_path
+                    and registry.file_hash
+                ):
+                    file_hash = registry.file_hash
+                    latest_file_hash = file_hash
+                    affiliations = []
+                    parse_cost = 0.0
+                else:
+                    # First hit: hash it so the registry can mark it as unusable.
+                    _, file_hash = await self.compute_file_hash_from_path(file_path_on_disk)
+                    latest_file_hash = file_hash
+                    raise AffiliationAnalysisError(
+                        retain_file_hash=True,
+                        error_message=(
+                            f"Affiliation file {latest_file_path!r} is too large for LLM parsing "
+                            f"({file_size_bytes} bytes > {self.MAX_FILE_SIZE_BYTES} bytes)"
+                        ),
+                    )
+            else:
+                file_bytes, file_hash = await self.compute_file_hash_from_path(
+                    file_path_on_disk, retain_content=True
+                )
+                content = safe_decode(file_bytes)
+                latest_file_hash = file_hash
+                affiliations, parse_cost = await self.resolve_snapshot(
+                    registry,
+                    content,
+                    file_hash,
+                )
             ai_cost += parse_cost
 
             if repository.parent_repo: