-
Notifications
You must be signed in to change notification settings - Fork 18
Expand file tree
/
Copy pathdocument.py
More file actions
64 lines (51 loc) · 2.12 KB
/
Copy pathdocument.py
File metadata and controls
64 lines (51 loc) · 2.12 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
import hashlib
import re
from dataclasses import dataclass, field
from datetime import datetime
from typing import Optional
from .chunk import Chunk
@dataclass
class Document:
GENERATED_TITLE_MAX_CHARS = 100
id: str | None = None
content: str = ""
uri: str | None = None
metadata: dict = field(default_factory=dict)
created_at: datetime | None = None
updated_at: datetime | None = None
chunks: list["Chunk"] = field(default_factory=list)
def hash(self) -> str:
"""Generate a hash for the document content using SHA-3 for maximum collision resistance"""
return hashlib.sha3_256(self.content.encode()).hexdigest()
def get_title(self) -> Optional[str]:
"""Extract title from metadata if available"""
if self.metadata and "title" in self.metadata:
return self.metadata["title"]
if self.metadata and "generated" in self.metadata:
if "title" in self.metadata["generated"]:
return self.metadata["generated"]["title"]
return None
def set_generated_title(self):
"""Set a generated title in metadata"""
if "generated" not in self.metadata:
self.metadata["generated"] = {}
self.metadata["generated"]["title"] = self.extract_document_title(
fallback_first_line=True
)
def extract_document_title(self, fallback_first_line: bool = False) -> str | None:
"""Extract title from markdown content.
Args:
text: The markdown text to extract the title from.
fallback_first_line: If True, use the first non-empty line as title if no heading is found.
"""
# Look for first level-1 heading
match = re.search(r"^# (.+)$", self.content, re.MULTILINE)
if match:
return match.group(1).strip()
# Fallback: first non-empty line with at least one word
if fallback_first_line:
for line in self.content.splitlines():
line = line.strip()
if line and re.search(r"\w", line):
return line[: self.GENERATED_TITLE_MAX_CHARS]
return None