Skip to content

Commit fdd433d

Browse files
committed
feat: Add support for most popular languages
1 parent 8917250 commit fdd433d

50 files changed

Lines changed: 4278 additions & 48 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

README.md

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,9 @@ and stale entries for deleted files are cleaned up automatically. Pass `force: t
2525

2626
Language is detected automatically from file extension or filename — no configuration needed.
2727

28-
| Language | Extracted symbols / extras |
29-
|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
30-
| **Go** | `function`, `method` (with receiver), `struct`, `interface`, `type` |
31-
| **Java** | `class`, `interface`, `enum`, `record`, `method`; Spring stereotypes (`controller`, `service`, `repository`, `component`, `configuration`, `entity`, `exception_handler`); HTTP routes from `@RequestMapping`/`@GetMapping`/etc.; Lombok annotations |
32-
| **Python** | `class`, `function`, `method`, `dataclass`, `pydantic_model`, `api_route` (FastAPI), `lifecycle_hook`; async detection |
33-
| **TypeScript / React** | `class`, `interface`, `type`, `function`, `react_component`, `react_hook`; `React.memo` detection; JSDoc preserved |
34-
| **Dockerfile** | `from`, `run`, `copy`, `env`, `expose`, `entrypoint`, `cmd` |
35-
| **Docker Compose** | `service` |
36-
| **Markdown** | `heading` (with hierarchy) |
37-
| **JSON / HTML / CSS / XML** | structural elements; XML parser is Maven/Spring-aware (`dependency`, `plugin`, `bean`) |
28+
Go, Java, Python, TypeScript / JavaScript (React), Rust, C#, C, C++, Ruby, PHP, Kotlin, Scala, Swift, Dart, Bash, SQL, Lua, R, Dockerfile, Docker Compose, Markdown, JSON, HTML, CSS, XML.
29+
30+
Most parsers are framework-aware where it matters — Spring stereotypes and HTTP routes for Java/Kotlin, FastAPI/Pydantic for Python, ASP.NET for C#, Rails for Ruby, Laravel/Symfony for PHP, React/SwiftUI/Flutter widgets, etc. See `server/parser/` for the per-language extraction details.
3831

3932
## Setup
4033

@@ -200,7 +193,7 @@ server/
200193
├── main.py # MCP server entry point + lifespan
201194
├── config.py # Settings and service configuration
202195
├── state.py # Shared store singletons
203-
├── parser/ # Tree-sitter parsers (Go, Java, Python, TypeScript, Dockerfile, Compose, Markdown, JSON, HTML, CSS, XML)
196+
├── parser/ # Tree-sitter parsers (Go, Java, Python, TypeScript, Rust, C#, C, C++, Ruby, PHP, Kotlin, Scala, Swift, Dart, Bash, SQL, Lua, R, Dockerfile, Compose, Markdown, JSON, HTML, CSS, XML)
204197
├── embeddings/ # Jina Code V2 embedding client (batched, async)
205198
├── indexer/ # GitHub fetcher, code indexing pipeline, git history pipeline
206199
├── store/ # Qdrant vector store (code_symbols and git_commits)

pyproject.toml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,19 @@ dependencies = [
1919
"tree-sitter-css>=0.23.0",
2020
"tree-sitter-xml>=0.7.0",
2121
"tree-sitter-yaml>=0.6.0",
22+
"tree-sitter-rust>=0.24.0",
23+
"tree-sitter-c-sharp>=0.23.0",
24+
"tree-sitter-c>=0.24.0",
25+
"tree-sitter-cpp>=0.23.0",
26+
"tree-sitter-ruby>=0.23.0",
27+
"tree-sitter-php>=0.24.0",
28+
"tree-sitter-kotlin>=1.1.0",
29+
"tree-sitter-scala>=0.26.0",
30+
"tree-sitter-swift>=0.7.0",
31+
"tree-sitter-bash>=0.25.0",
32+
"tree-sitter-sql>=0.3.0",
33+
"tree-sitter-lua>=0.5.0",
34+
"tree-sitter-language-pack>=1.5.0,<1.6.0",
2235
"pydantic-settings>=2.14.0",
2336
"pyyaml>=6.0",
2437
"httpx>=0.28.0",
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
from __future__ import annotations
2+
3+
# Shared between Java and Kotlin parsers — both target Spring/JPA on the JVM.
4+
5+
SPRING_STEREOTYPES: dict[str, str] = {
6+
"RestController": "controller",
7+
"Controller": "controller",
8+
"Service": "service",
9+
"Repository": "repository",
10+
"Component": "component",
11+
"Configuration": "configuration",
12+
"RestControllerAdvice": "exception_handler",
13+
"ControllerAdvice": "exception_handler",
14+
"Entity": "entity",
15+
"MappedSuperclass": "entity",
16+
}
17+
18+
HTTP_METHOD_ANNOTATIONS: dict[str, str | None] = {
19+
"GetMapping": "GET",
20+
"PostMapping": "POST",
21+
"PutMapping": "PUT",
22+
"DeleteMapping": "DELETE",
23+
"PatchMapping": "PATCH",
24+
"RequestMapping": None, # method determined from attributes
25+
}
26+
27+
LOMBOK_ANNOTATIONS: set[str] = {
28+
"Getter",
29+
"Setter",
30+
"Data",
31+
"Builder",
32+
"NoArgsConstructor",
33+
"AllArgsConstructor",
34+
"RequiredArgsConstructor",
35+
"Slf4j",
36+
"ToString",
37+
"EqualsAndHashCode",
38+
"Value",
39+
}

server/parser/bash.py

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
from __future__ import annotations
2+
3+
import tree_sitter_bash
4+
from tree_sitter import Language, Node, Parser
5+
6+
from server.parser.base import CodeSymbol, _node_text
7+
8+
BASH_LANGUAGE = Language(tree_sitter_bash.language())
9+
10+
11+
def _docstring(node: Node, source: bytes) -> str | None:
12+
prev = node.prev_sibling
13+
lines: list[str] = []
14+
while prev is not None:
15+
if prev.type == "comment":
16+
lines.insert(0, _node_text(prev, source))
17+
prev = prev.prev_sibling
18+
continue
19+
if prev.type in ("\n", " "):
20+
prev = prev.prev_sibling
21+
continue
22+
break
23+
return "\n".join(lines) if lines else None
24+
25+
26+
def _parse_function(node: Node, source: bytes, file_path: str) -> CodeSymbol | None:
27+
name_node = node.child_by_field_name("name")
28+
if name_node is None:
29+
for child in node.children:
30+
if child.type == "word":
31+
name_node = child
32+
break
33+
if name_node is None:
34+
return None
35+
36+
return CodeSymbol(
37+
name=_node_text(name_node, source),
38+
symbol_type="function",
39+
language="bash",
40+
source=_node_text(node, source),
41+
file_path=file_path,
42+
start_line=node.start_point[0] + 1,
43+
end_line=node.end_point[0] + 1,
44+
signature=_node_text(node, source).split("{", 1)[0].strip(),
45+
docstring=_docstring(node, source),
46+
)
47+
48+
49+
class BashParser:
50+
def __init__(self) -> None:
51+
self._parser = Parser(BASH_LANGUAGE)
52+
53+
def supported_extensions(self) -> list[str]:
54+
return [".sh", ".bash"]
55+
56+
def language(self) -> str:
57+
return "bash"
58+
59+
def parse_file(self, source: bytes, file_path: str) -> list[CodeSymbol]:
60+
tree = self._parser.parse(source)
61+
symbols: list[CodeSymbol] = []
62+
for child in tree.root_node.children:
63+
if child.type == "function_definition":
64+
sym = _parse_function(child, source, file_path)
65+
if sym:
66+
symbols.append(sym)
67+
return symbols

server/parser/c.py

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
from __future__ import annotations
2+
3+
import tree_sitter_c
4+
from tree_sitter import Language, Node, Parser
5+
6+
from server.parser.base import CodeSymbol, _node_text
7+
8+
C_LANGUAGE = Language(tree_sitter_c.language())
9+
10+
11+
def _docstring(node: Node, source: bytes) -> str | None:
12+
prev = node.prev_sibling
13+
while prev is not None:
14+
if prev.type == "comment":
15+
text = _node_text(prev, source)
16+
if text.startswith("/**") or text.startswith("///"):
17+
return text
18+
break
19+
if prev.type in ("\n", " "):
20+
prev = prev.prev_sibling
21+
continue
22+
break
23+
return None
24+
25+
26+
def _function_name(declarator: Node, source: bytes) -> str | None:
27+
"""Drill through pointer/parenthesized declarators to the bare identifier."""
28+
while declarator is not None:
29+
if declarator.type == "function_declarator":
30+
inner = declarator.child_by_field_name("declarator")
31+
if inner is None:
32+
return None
33+
declarator = inner
34+
continue
35+
if declarator.type in ("identifier", "field_identifier"):
36+
return _node_text(declarator, source)
37+
if declarator.type == "destructor_name":
38+
for child in declarator.children:
39+
if child.type == "identifier":
40+
return "~" + _node_text(child, source)
41+
return None
42+
# pointer_declarator, parenthesized_declarator, …
43+
inner = declarator.child_by_field_name("declarator")
44+
if inner is None:
45+
return None
46+
declarator = inner
47+
return None
48+
49+
50+
def _signature(node: Node, source: bytes) -> str:
51+
body = node.child_by_field_name("body")
52+
if body:
53+
return (
54+
source[node.start_byte : body.start_byte]
55+
.decode("utf-8", errors="replace")
56+
.strip()
57+
)
58+
return _node_text(node, source).split("{", 1)[0].split(";", 1)[0].strip()
59+
60+
61+
def _parse_function(
62+
node: Node,
63+
source: bytes,
64+
file_path: str,
65+
language: str,
66+
parent_name: str | None = None,
67+
package: str | None = None,
68+
) -> CodeSymbol | None:
69+
declarator = node.child_by_field_name("declarator")
70+
if declarator is None:
71+
return None
72+
name = _function_name(declarator, source)
73+
if name is None:
74+
return None
75+
76+
sym_type = "method" if parent_name else "function"
77+
78+
return CodeSymbol(
79+
name=name,
80+
symbol_type=sym_type,
81+
language=language,
82+
source=_node_text(node, source),
83+
file_path=file_path,
84+
start_line=node.start_point[0] + 1,
85+
end_line=node.end_point[0] + 1,
86+
parent_name=parent_name,
87+
package=package,
88+
signature=_signature(node, source),
89+
docstring=_docstring(node, source),
90+
)
91+
92+
93+
def _parse_typedef(
94+
node: Node, source: bytes, file_path: str, language: str
95+
) -> CodeSymbol | None:
96+
declarator = node.child_by_field_name("declarator")
97+
if declarator is None:
98+
return None
99+
name = _function_name(declarator, source)
100+
if name is None:
101+
# typedef sometimes has identifier directly
102+
for child in node.children:
103+
if child.type == "type_identifier":
104+
name = _node_text(child, source)
105+
break
106+
if not name:
107+
return None
108+
109+
return CodeSymbol(
110+
name=name,
111+
symbol_type="type",
112+
language=language,
113+
source=_node_text(node, source),
114+
file_path=file_path,
115+
start_line=node.start_point[0] + 1,
116+
end_line=node.end_point[0] + 1,
117+
signature=_node_text(node, source).split(";", 1)[0].strip(),
118+
docstring=_docstring(node, source),
119+
)
120+
121+
122+
_TYPED_NAMED_NODE = {
123+
"struct_specifier": "struct",
124+
"enum_specifier": "enum",
125+
"union_specifier": "union",
126+
}
127+
128+
129+
def _parse_named_type(
130+
node: Node, source: bytes, file_path: str, language: str
131+
) -> CodeSymbol | None:
132+
name_node = node.child_by_field_name("name")
133+
if name_node is None:
134+
return None
135+
return CodeSymbol(
136+
name=_node_text(name_node, source),
137+
symbol_type=_TYPED_NAMED_NODE[node.type],
138+
language=language,
139+
source=_node_text(node, source),
140+
file_path=file_path,
141+
start_line=node.start_point[0] + 1,
142+
end_line=node.end_point[0] + 1,
143+
signature=_node_text(node, source).split("{", 1)[0].strip(),
144+
docstring=_docstring(node, source),
145+
)
146+
147+
148+
def _walk_c(
149+
container: Node,
150+
source: bytes,
151+
file_path: str,
152+
symbols: list[CodeSymbol],
153+
) -> None:
154+
for child in container.children:
155+
if child.type == "function_definition":
156+
sym = _parse_function(child, source, file_path, "c")
157+
if sym:
158+
symbols.append(sym)
159+
elif child.type in _TYPED_NAMED_NODE:
160+
sym = _parse_named_type(child, source, file_path, "c")
161+
if sym:
162+
symbols.append(sym)
163+
elif child.type == "type_definition":
164+
sym = _parse_typedef(child, source, file_path, "c")
165+
if sym:
166+
symbols.append(sym)
167+
168+
169+
class CParser:
170+
def __init__(self) -> None:
171+
self._parser = Parser(C_LANGUAGE)
172+
173+
def supported_extensions(self) -> list[str]:
174+
return [".c", ".h"]
175+
176+
def language(self) -> str:
177+
return "c"
178+
179+
def parse_file(self, source: bytes, file_path: str) -> list[CodeSymbol]:
180+
tree = self._parser.parse(source)
181+
symbols: list[CodeSymbol] = []
182+
_walk_c(tree.root_node, source, file_path, symbols)
183+
return symbols

0 commit comments

Comments
 (0)