1+ import ipaddress
2+ import socket
3+ from urllib .parse import urljoin , urlparse
4+
5+ import requests
16from bs4 import BeautifulSoup
7+ import trafilatura
8+
9+ # Hostnames that should never be fetched server-side, regardless of resolution.
10+ _BLOCKED_HOSTNAMES = {"localhost" }
11+
12+ # Hard cap on a fetched body. Generous for an article, but bounds the memory and
13+ # worker time a single (possibly hostile) response can consume.
14+ MAX_FETCH_BYTES = 2_000_000 # 2 MB
15+
16+
17+ class UnsafeURLError (Exception ):
18+ """Raised when a URL (or a redirect target) points at a non-public host."""
19+
20+
21+ def _is_public_ip (ip_str : str ) -> bool :
22+ """True only for globally routable addresses. ``is_global`` rejects
23+ loopback, private, link-local, multicast, reserved and unspecified ranges
24+ (e.g. 127.0.0.1, 10.x, 169.254.169.254, ::1) in one check."""
25+ try :
26+ return ipaddress .ip_address (ip_str ).is_global
27+ except ValueError :
28+ return False
29+
30+
31+ def _url_host_is_safe (url : str ) -> bool :
32+ """True if ``url`` is an http(s) URL whose host resolves only to public IPs.
33+
34+ Blocks SSRF to internal/loopback/link-local targets. Resolves the hostname
35+ and requires *every* returned address to be public, so a name that maps to
36+ a private IP is rejected. IP-literal hosts are checked directly.
37+ """
38+ try :
39+ parsed = urlparse (url )
40+ except ValueError :
41+ return False
42+ if parsed .scheme not in "http" , "https" ) or not parsed .hostname :
43+ return False
44+ host = parsed .hostname
45+ if host .lower () in _BLOCKED_HOSTNAMES :
46+ return False
47+ # IP literal: check it directly without a DNS lookup.
48+ try :
49+ ipaddress .ip_address (host )
50+ return _is_public_ip (host )
51+ except ValueError :
52+ pass
53+ # Hostname: resolve and require ALL addresses to be public.
54+ try :
55+ infos = socket .getaddrinfo (host , parsed .port , proto = socket .IPPROTO_TCP )
56+ except socket .gaierror :
57+ return False
58+ addresses = {info [4 ][0 ] for info in infos }
59+ return bool (addresses ) and all (_is_public_ip (addr ) for addr in addresses )
60+
61+
62+ def safe_get (
63+ url : str , * , timeout : float = 10 , max_redirects : int = 5
64+ ) -> requests .Response :
65+ """GET ``url`` with SSRF protection.
66+
67+ Redirects are followed manually so each hop's host is re-validated — a
68+ public URL can otherwise 302 to an internal one. The response body is capped
69+ at ``MAX_FETCH_BYTES`` (checked against ``Content-Length`` and again while
70+ streaming, since the header can be absent or wrong). Raises ``UnsafeURLError``
71+ if the URL or any redirect target isn't a public http(s) host, or the body
72+ exceeds the cap; network failures propagate as ``requests.RequestException``.
73+
74+ Residual gap: DNS rebinding (host resolves public at check time, private at
75+ connect time) is not closed — that needs pinning the validated IP into the
76+ connection. TODO: the calling endpoint is not yet login-gated or
77+ rate-limited; close this gap (or add those controls) before relying on it.
78+ """
79+ for _ in range (max_redirects + 1 ):
80+ if not _url_host_is_safe (url ):
81+ raise UnsafeURLError (url )
82+ resp = requests .get (url , timeout = timeout , allow_redirects = False , stream = True )
83+ if resp .is_redirect :
84+ location = resp .headers .get ("Location" )
85+ resp .close ()
86+ if not location :
87+ resp ._content = b""
88+ return resp
89+ url = urljoin (url , location )
90+ continue
91+ # Reject early if the server declares an oversized body, then enforce the
92+ # cap while streaming in case Content-Length is missing or understated.
93+ declared = resp .headers .get ("Content-Length" )
94+ if declared and declared .isdigit () and int (declared ) > MAX_FETCH_BYTES :
95+ resp .close ()
96+ raise UnsafeURLError (f"response too large: { url } )
97+ chunks , total = [], 0
98+ for chunk in resp .iter_content (8192 ):
99+ total += len (chunk )
100+ if total > MAX_FETCH_BYTES :
101+ resp .close ()
102+ raise UnsafeURLError (f"response too large: { url } )
103+ chunks .append (chunk )
104+ # Populate the body so callers can use resp.text/.content normally despite
105+ # stream=True (which otherwise defers — and would bypass — the read).
106+ resp ._content = b"" .join (chunks )
107+ return resp
108+ raise UnsafeURLError (f"too many redirects: { url } )
2109
3110
4111def extract_content (html : str ) -> str :
@@ -11,3 +118,34 @@ def extract_content(html: str) -> str:
11118 # drop blank lines
12119 minimized = [line for line in lines if line ]
13120 return "\n " .join (minimized )
121+
122+
123+ def extract_article (html : str , url : str | None = None ) -> tuple [str , str ]:
124+ """Extract ``(title, body)`` of the main article from an arbitrary web page.
125+
126+ trafilatura isolates the main content and strips boilerplate (navigation,
127+ footers, comments, ads). That keeps the summarizer focused — and cheaper —
128+ versus feeding it the whole page. When trafilatura can't isolate an article
129+ (unusual template, very thin page) the body falls back to the naive
130+ visible-text dump in ``extract_content`` so server-rendered HTML still
131+ yields something.
132+
133+ Returns ``("", "")`` only when even the fallback is empty — callers should
134+ treat an empty body as "couldn't read the page" and degrade gracefully
135+ (skip auto-summarization / surface an inline error) rather than summarizing
136+ junk. Note: pages rendered client-side (SPAs), paywalled, or behind anti-bot
137+ blocks won't yield text here regardless of which extractor runs.
138+ """
139+ body = (
140+ trafilatura .extract (html , url = url , include_comments = False , include_tables = False )
141+ or ""
142+ ).strip ()
143+ if not body :
144+ body = extract_content (html )
145+
146+ title = ""
147+ metadata = trafilatura .extract_metadata (html , default_url = url )
148+ if metadata and metadata .title :
149+ title = metadata .title .strip ()
150+
151+ return title , body
0 commit comments