feat: implement section extractor

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

services/apps/git_integration/src/crowdgit/services/maintainer/maintainer_service.py

@@ -37,6 +37,7 @@
 from crowdgit.models.service_execution import ServiceExecution
 from crowdgit.services.base.base_service import BaseService
 from crowdgit.services.maintainer.bedrock import invoke_bedrock
+from crowdgit.services.maintainer.section_extractor import SectionExtractor
 from crowdgit.services.utils import run_shell_command
 from crowdgit.settings import MAINTAINER_RETRY_INTERVAL_DAYS, MAINTAINER_UPDATE_INTERVAL_HOURS
 
@@ -93,6 +94,7 @@ class MaintainerService(BaseService):
         "code_owners",
         "emeritus",
         "workgroup",
+        "readme",
     }
 
     VALID_EXTENSIONS = {
@@ -132,6 +134,12 @@ class MaintainerService(BaseService):
     STEM_MATCH_SCORE = 50
     PARTIAL_STEM_SCORE = 25
 
+    # Files in KNOWN_PATHS that still need section filtering (contain non-governance content)
+    SECTION_FILTERED_PATHS = {"readme.md", "governance.md"}
+    SCORING_KEYWORDS_SET = frozenset(SCORING_KEYWORDS)
+
+    _section_extractor = SectionExtractor()
+
     def make_role(self, title: str):
         title = title.lower()
         title = (
@@ -586,6 +594,16 @@ async def analyze_and_build_result(self, filename: str, content: str) -> Maintai
                 f"Skipping README file '{filename}': no governance keyword found in content"
             )
             raise MaintanerAnalysisError(error_code=ErrorCode.NO_MAINTAINER_FOUND)
+
+        fname = os.path.basename(filename).lower()
+        if fname not in self.KNOWN_PATHS or fname in self.SECTION_FILTERED_PATHS:
+            extracted = self._section_extractor.extract(fname, content, self.SCORING_KEYWORDS_SET)
+            if extracted:
+                self.logger.info(f"Using extracted sections for '{filename}' {extracted}")
+                content = extracted
+            else:
+                self.logger.debug(f"No sections extracted for '{filename}', using full content")
+
         result = await self.analyze_file_content(filename, content)
 
         if not result.output.info:

services/apps/git_integration/src/crowdgit/services/maintainer/section_extractor.py

@@ -0,0 +1,90 @@
+import os
+import re
+
+import tomllib
+import yaml
+
+
+class SectionExtractor:
+    """
+    Extracts relevant sections from file content based on file format and governance keywords.
+    Returns None when no relevant sections are found or format is unsupported (caller falls back to full content).
+    """
+
+    def extract(self, filename: str, content: str, keywords: set[str]) -> str | None:
+        """
+        Returns extracted relevant section text, or None if no sections found.
+        filename should be the basename (lowercased).
+        """
+        ext = os.path.splitext(filename)[1]
+        if ext in (".md", ".markdown"):
+            return self._extract_markdown_sections(content, keywords)
+        elif ext in (".yaml", ".yml"):
+            return self._extract_yaml_sections(content, keywords)
+        elif ext == ".toml":
+            return self._extract_toml_sections(content, keywords)
+        return None
+
+    def _extract_markdown_sections(self, content: str, keywords: set[str]) -> str | None:
+        """
+        Splits content on `#`-style heading lines only (# / ## / ###...).
+        Includes a section if its heading text contains any keyword.
+        Returns joined matching sections, or None if none match.
+        """
+        heading_pattern = re.compile(r"^#{1,6}\s+", re.MULTILINE)
+        # Split into (heading_line, body) pairs; first element may be pre-heading content
+        parts = heading_pattern.split(content)
+        headings = heading_pattern.findall(content)
+
+        # parts[0] is text before the first heading (skip it)
+        # parts[1..] correspond to headings[0..]
+        matching_sections = []
+        for i, heading_marker in enumerate(headings):
+            block = parts[i + 1]  # block starts right after the heading marker
+            # The first line of block is the heading text
+            first_newline = block.find("\n")
+            heading_text = block[:first_newline].strip() if first_newline != -1 else block.strip()
+            if any(kw in heading_text.lower() for kw in keywords):
+                matching_sections.append(f"{heading_marker}{block}")
+
+        return "".join(matching_sections) if matching_sections else None
+
+    def _extract_yaml_sections(self, content: str, keywords: set[str]) -> str | None:
+        """
+        Parses YAML and returns top-level keys whose name contains any keyword, serialized back to YAML.
+        Returns None if no keys match or parsing fails.
+        """
+        try:
+            data = yaml.safe_load(content)
+        except yaml.YAMLError:
+            return None
+
+        if not isinstance(data, dict):
+            return None
+
+        matching = {k: v for k, v in data.items() if any(kw in str(k).lower() for kw in keywords)}
+        if not matching:
+            return None
+
+        return yaml.dump(matching, default_flow_style=False, allow_unicode=True)
+
+    def _extract_toml_sections(self, content: str, keywords: set[str]) -> str | None:
+        """
+        Parses TOML and returns top-level keys whose name contains any keyword,
+        serialized as Python repr key=value lines (not valid TOML syntax).
+        Returns None if no keys match or parsing fails.
+        """
+        try:
+            data = tomllib.loads(content)
+        except tomllib.TOMLDecodeError:
+            return None
+
+        matching = {k: v for k, v in data.items() if any(kw in k.lower() for kw in keywords)}
+        if not matching:
+            return None
+
+        # Serialize matching keys back as simple TOML representation
+        lines = []
+        for k, v in matching.items():
+            lines.append(f"{k} = {repr(v)}")
+        return "\n".join(lines)