discovery-crawler/parser.py at main · Darky-Github/discovery-crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
from bs4 import BeautifulSoup
import re

CHUNK_SIZE = 1800

def clean_text(text):
    text = re.sub(r"\s+", " ", text)
    return text.strip()

def chunk_text(text, size=CHUNK_SIZE):
    chunks = []

    for i in range(0, len(text), size):
        chunk = text[i:i + size].strip()

        if len(chunk) > 120:
            chunks.append(chunk)

    return chunks

def parse(html):
    soup = BeautifulSoup(html, "html.parser")

    for tag in soup([
        "script",
        "style",
        "noscript",
        "svg",
        "footer",
        "nav",
        "aside",
        "form"
    ]):
        tag.decompose()

    title = ""

    if soup.title:
        title = clean_text(soup.title.text)

    text = soup.get_text(" ", strip=True)

    text = clean_text(text)

    text = text[:15000]

    chunks = chunk_text(text)

    links = []

    for a in soup.find_all("a"):
        href = a.get("href")

        if href:
            links.append(href)

    return {
        "title": title,
        "chunks": chunks,
        "links": links
    }