Merge pull request #36 from GoodbyePlanet/feat/additional-language-parsers

feat: Add support for most popular languages

Author: GoodbyePlanet

.gitignore

@@ -24,4 +24,7 @@ build/
 blog.md
 
 # Local config (use config.example.yaml as a template)
-config.yaml
+config.yaml
+
+# Claude Code local settings
+.claude/settings.local.json

README.md

@@ -25,16 +25,9 @@ and stale entries for deleted files are cleaned up automatically. Pass `force: t
 
 Language is detected automatically from file extension or filename — no configuration needed.
 
-| Language                    | Extracted symbols / extras                                                                                                                                                                                                                           |
-|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Go**                      | `function`, `method` (with receiver), `struct`, `interface`, `type`                                                                                                                                                                                  |
-| **Java**                    | `class`, `interface`, `enum`, `record`, `method`; Spring stereotypes (`controller`, `service`, `repository`, `component`, `configuration`, `entity`, `exception_handler`); HTTP routes from `@RequestMapping`/`@GetMapping`/etc.; Lombok annotations |
-| **Python**                  | `class`, `function`, `method`, `dataclass`, `pydantic_model`, `api_route` (FastAPI), `lifecycle_hook`; async detection                                                                                                                               |
-| **TypeScript / React**      | `class`, `interface`, `type`, `function`, `react_component`, `react_hook`; `React.memo` detection; JSDoc preserved                                                                                                                                   |
-| **Dockerfile**              | `from`, `run`, `copy`, `env`, `expose`, `entrypoint`, `cmd`                                                                                                                                                                                          |
-| **Docker Compose**          | `service`                                                                                                                                                                                                                                            |
-| **Markdown**                | `heading` (with hierarchy)                                                                                                                                                                                                                           |
-| **JSON / HTML / CSS / XML** | structural elements; XML parser is Maven/Spring-aware (`dependency`, `plugin`, `bean`)                                                                                                                                                               |
+Go, Java, Python, TypeScript / JavaScript (React), Rust, C#, C, C++, Ruby, PHP, Kotlin, Scala, Swift, Dart, Bash, SQL, Lua, R, Dockerfile, Docker Compose, Markdown, JSON, HTML, CSS, XML.
+
+Most parsers are framework-aware where it matters — Spring stereotypes and HTTP routes for Java/Kotlin, FastAPI/Pydantic for Python, ASP.NET for C#, Rails for Ruby, Laravel/Symfony for PHP, React/SwiftUI/Flutter widgets, etc. See `server/parser/` for the per-language extraction details.
 
 ## Setup
 
@@ -200,7 +193,7 @@ server/
 ├── main.py          # MCP server entry point + lifespan
 ├── config.py        # Settings and service configuration
 ├── state.py         # Shared store singletons
-├── parser/          # Tree-sitter parsers (Go, Java, Python, TypeScript, Dockerfile, Compose, Markdown, JSON, HTML, CSS, XML)
+├── parser/          # Tree-sitter parsers (Go, Java, Python, TypeScript, Rust, C#, C, C++, Ruby, PHP, Kotlin, Scala, Swift, Dart, Bash, SQL, Lua, R, Dockerfile, Compose, Markdown, JSON, HTML, CSS, XML)
 ├── embeddings/      # Jina Code V2 embedding client (batched, async)
 ├── indexer/         # GitHub fetcher, code indexing pipeline, git history pipeline
 ├── store/           # Qdrant vector store (code_symbols and git_commits)

pyproject.toml

@@ -19,6 +19,19 @@ dependencies = [
     "tree-sitter-css>=0.23.0",
     "tree-sitter-xml>=0.7.0",
     "tree-sitter-yaml>=0.6.0",
+    "tree-sitter-rust>=0.24.0",
+    "tree-sitter-c-sharp>=0.23.0",
+    "tree-sitter-c>=0.24.0",
+    "tree-sitter-cpp>=0.23.0",
+    "tree-sitter-ruby>=0.23.0",
+    "tree-sitter-php>=0.24.0",
+    "tree-sitter-kotlin>=1.1.0",
+    "tree-sitter-scala>=0.26.0",
+    "tree-sitter-swift>=0.7.0",
+    "tree-sitter-bash>=0.25.0",
+    "tree-sitter-sql>=0.3.0",
+    "tree-sitter-lua>=0.5.0",
+    "tree-sitter-language-pack>=1.5.0,<1.6.0",
     "pydantic-settings>=2.14.0",
     "pyyaml>=6.0",
     "httpx>=0.28.0",

server/main.py

@@ -4,7 +4,9 @@
 from contextlib import asynccontextmanager
 from typing import AsyncIterator
 
+import uvicorn
 from mcp.server.fastmcp import FastMCP
+from starlette.applications import Starlette
 
 from server.config import settings
 from server.embeddings.bm25 import BM25SparseProvider, close_sparse_embedding_provider
@@ -51,15 +53,29 @@ async def lifespan(_: FastMCP) -> AsyncIterator[None]:
     logger.info("semcode MCP server stopped.")
 
 
+# For streamable-http we drive the Starlette app's lifespan ourselves (see main),
+# so the per-MCP-session lifespan would re-init the store on every client connect.
 mcp = FastMCP(
     "semcode",
     instructions="Semantic code search across microservices codebases. Supports Go, Java, Python, and TypeScript/React.",
-    lifespan=lifespan,
+    lifespan=lifespan if settings.mcp_transport != "streamable-http" else None,
     host=settings.mcp_host,
     port=settings.mcp_port,
 )
 
 
+def _wrap_http_lifespan(app: Starlette) -> None:
+    original = app.router.lifespan_context
+
+    @asynccontextmanager
+    async def combined(scope_app: Starlette) -> AsyncIterator[None]:
+        async with lifespan(mcp):
+            async with original(scope_app):
+                yield
+
+    app.router.lifespan_context = combined
+
+
 def main() -> None:
     from server.tools.search import register_search_tools
     from server.tools.index import register_index_tools
@@ -77,7 +93,17 @@ def main() -> None:
     register_system_prompts(mcp)
     register_http_routes(mcp)
 
-    mcp.run(transport=settings.mcp_transport)
+    if settings.mcp_transport == "streamable-http":
+        app = mcp.streamable_http_app()
+        _wrap_http_lifespan(app)
+        uvicorn.run(
+            app,
+            host=settings.mcp_host,
+            port=settings.mcp_port,
+            log_level=mcp.settings.log_level.lower(),
+        )
+    else:
+        mcp.run(transport=settings.mcp_transport)
 
 
 if __name__ == "__main__":

server/parser/_spring_annotations.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+# Shared between Java and Kotlin parsers — both target Spring/JPA on the JVM.
+
+SPRING_STEREOTYPES: dict[str, str] = {
+    "RestController": "controller",
+    "Controller": "controller",
+    "Service": "service",
+    "Repository": "repository",
+    "Component": "component",
+    "Configuration": "configuration",
+    "RestControllerAdvice": "exception_handler",
+    "ControllerAdvice": "exception_handler",
+    "Entity": "entity",
+    "MappedSuperclass": "entity",
+}
+
+HTTP_METHOD_ANNOTATIONS: dict[str, str | None] = {
+    "GetMapping": "GET",
+    "PostMapping": "POST",
+    "PutMapping": "PUT",
+    "DeleteMapping": "DELETE",
+    "PatchMapping": "PATCH",
+    "RequestMapping": None,  # method determined from attributes
+}
+
+LOMBOK_ANNOTATIONS: set[str] = {
+    "Getter",
+    "Setter",
+    "Data",
+    "Builder",
+    "NoArgsConstructor",
+    "AllArgsConstructor",
+    "RequiredArgsConstructor",
+    "Slf4j",
+    "ToString",
+    "EqualsAndHashCode",
+    "Value",
+}

server/parser/bash.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import tree_sitter_bash
+from tree_sitter import Language, Node, Parser
+
+from server.parser.base import CodeSymbol, _node_text
+
+BASH_LANGUAGE = Language(tree_sitter_bash.language())
+
+
+def _docstring(node: Node, source: bytes) -> str | None:
+    prev = node.prev_sibling
+    lines: list[str] = []
+    while prev is not None:
+        if prev.type == "comment":
+            lines.insert(0, _node_text(prev, source))
+            prev = prev.prev_sibling
+            continue
+        if prev.type in ("\n", " "):
+            prev = prev.prev_sibling
+            continue
+        break
+    return "\n".join(lines) if lines else None
+
+
+def _parse_function(node: Node, source: bytes, file_path: str) -> CodeSymbol | None:
+    name_node = node.child_by_field_name("name")
+    if name_node is None:
+        for child in node.children:
+            if child.type == "word":
+                name_node = child
+                break
+    if name_node is None:
+        return None
+
+    return CodeSymbol(
+        name=_node_text(name_node, source),
+        symbol_type="function",
+        language="bash",
+        source=_node_text(node, source),
+        file_path=file_path,
+        start_line=node.start_point[0] + 1,
+        end_line=node.end_point[0] + 1,
+        signature=_node_text(node, source).split("{", 1)[0].strip(),
+        docstring=_docstring(node, source),
+    )
+
+
+class BashParser:
+    def __init__(self) -> None:
+        self._parser = Parser(BASH_LANGUAGE)
+
+    def supported_extensions(self) -> list[str]:
+        return [".sh", ".bash"]
+
+    def language(self) -> str:
+        return "bash"
+
+    def parse_file(self, source: bytes, file_path: str) -> list[CodeSymbol]:
+        tree = self._parser.parse(source)
+        symbols: list[CodeSymbol] = []
+        for child in tree.root_node.children:
+            if child.type == "function_definition":
+                sym = _parse_function(child, source, file_path)
+                if sym:
+                    symbols.append(sym)
+        return symbols