Skip to content

Commit 6a07dc2

Browse files
lzwjavaclaude
lzwjava
and
claude
committed
refactor(web_search): use centralized HTTP session, remove hardcoded proxy
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 792de00 commit 6a07dc2

2 files changed

Lines changed: 99 additions & 81 deletions

File tree

‎iclaw/web_search.py‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,11 @@
1-
import os
21
from concurrent.futures import ThreadPoolExecutor, as_completed
32
from urllib.parse import parse_qs, urlparse
43

54
import requests
65
from bs4 import BeautifulSoup
76
from readability import Document
87

9-
# Configuration
10-
DEFAULT_PROXY = {"http": "http://127.0.0.1:7890", "https": "http://127.0.0.1:7890"}
11-
PROXY = {
12-
"http": os.environ.get("HTTP_PROXY", DEFAULT_PROXY["http"]),
13-
"https": os.environ.get("HTTPS_PROXY", DEFAULT_PROXY["https"]),
14-
}
8+
from iclaw import http
159

1610
HEADERS = {
1711
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
@@ -25,7 +19,7 @@ def search_ddg(query, num_results=10):
2519
url = f"https://html.duckduckgo.com/html/?q={query}"
2620

2721
try:
28-
res = requests.get(url, headers=HEADERS, proxies=PROXY, timeout=10)
22+
res = http.get_session().get(url, headers=HEADERS, timeout=10)
2923
res.raise_for_status()
3024
except Exception as e:
3125
print(f"[web search] Error searching DDG: {e}")
@@ -60,9 +54,7 @@ def search_startpage(query, num_results=20):
6054
}
6155

6256
try:
63-
res = requests.get(
64-
url, params=params, headers=HEADERS, proxies=PROXY, timeout=10
65-
)
57+
res = http.get_session().get(url, params=params, headers=HEADERS, timeout=10)
6658
res.raise_for_status()
6759
except Exception as e:
6860
print(f"[web search] Error searching Startpage: {e}")
@@ -93,13 +85,14 @@ def search_bing(query, num_results=20):
9385
url = f"https://www.bing.com/search?q={query}&setmkt=en-US&setlang=en-US&cc=US"
9486

9587
try:
96-
# Use a session to maintain cookies
88+
shared = http.get_session()
9789
session = requests.Session()
98-
# Set cookies to force US market and English language
90+
session.proxies = dict(shared.proxies)
91+
session.verify = shared.verify
9992
session.cookies.set("SRCHHPGUSR", "SRCHLANG=EN&WLS=2", domain=".bing.com")
10093
session.cookies.set("_EDGE_S", "mkt=en-us", domain=".bing.com")
10194

102-
res = session.get(url, headers=HEADERS, proxies=PROXY, timeout=10)
95+
res = session.get(url, headers=HEADERS, timeout=10)
10396
res.raise_for_status()
10497
except Exception as e:
10598
print(f"[web search] Error searching Bing: {e}")
@@ -124,8 +117,7 @@ def search_bing(query, num_results=20):
124117

125118
def extract_text_from_url(url):
126119
try:
127-
session = requests.Session()
128-
res = session.get(url, headers=HEADERS, proxies=PROXY, timeout=15)
120+
res = http.get_session().get(url, headers=HEADERS, timeout=15)
129121
res.encoding = res.apparent_encoding
130122

131123
if res.status_code != 200:

‎tests/test_web_search.py‎

Lines changed: 91 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -4,44 +4,53 @@
44

55

66
class TestWebSearch(unittest.TestCase):
7-
@patch("iclaw.web_search.requests.get")
8-
def test_search_ddg(self, mock_get):
9-
mock_get.return_value = MagicMock(
7+
@patch("iclaw.web_search.http.get_session")
8+
def test_search_ddg(self, mock_gs):
9+
mock_session = MagicMock()
10+
mock_gs.return_value = mock_session
11+
mock_session.get.return_value = MagicMock(
1012
ok=True,
1113
text='<html><div class="result__title"><a class="result__a" href="u">T</a></div></html>',
1214
)
1315
results = web_search.search_ddg("q", num_results=1)
1416
self.assertEqual(len(results), 1)
1517
self.assertEqual(results[0]["url"], "u")
1618

17-
@patch("iclaw.web_search.requests.get")
18-
def test_search_ddg_with_redirect_url(self, mock_get):
19-
mock_get.return_value = MagicMock(
19+
@patch("iclaw.web_search.http.get_session")
20+
def test_search_ddg_with_redirect_url(self, mock_gs):
21+
mock_session = MagicMock()
22+
mock_gs.return_value = mock_session
23+
mock_session.get.return_value = MagicMock(
2024
ok=True,
2125
text='<html><div class="result__title"><a class="result__a" href="//duckduckgo.com/l/?uddg=http%3A%2F%2Fexample.com">T</a></div></html>',
2226
)
2327
results = web_search.search_ddg("q", num_results=1)
2428
self.assertEqual(len(results), 1)
2529
self.assertEqual(results[0]["url"], "http://example.com")
2630

27-
@patch("iclaw.web_search.requests.get")
28-
def test_search_ddg_double_slash_url(self, mock_get):
29-
mock_get.return_value = MagicMock(
31+
@patch("iclaw.web_search.http.get_session")
32+
def test_search_ddg_double_slash_url(self, mock_gs):
33+
mock_session = MagicMock()
34+
mock_gs.return_value = mock_session
35+
mock_session.get.return_value = MagicMock(
3036
ok=True,
3137
text='<html><div class="result__title"><a class="result__a" href="//example.com/page">T</a></div></html>',
3238
)
3339
results = web_search.search_ddg("q", num_results=1)
3440
self.assertEqual(results[0]["url"], "https://example.com/page")
3541

36-
@patch("iclaw.web_search.requests.get", side_effect=Exception("network error"))
37-
def test_search_ddg_error(self, mock_get):
42+
@patch("iclaw.web_search.http.get_session")
43+
def test_search_ddg_error(self, mock_gs):
44+
mock_gs.return_value.get.side_effect = Exception("network error")
3845
with patch("sys.stdout"):
3946
results = web_search.search_ddg("q")
4047
self.assertEqual(results, [])
4148

42-
@patch("iclaw.web_search.requests.get")
43-
def test_search_startpage(self, mock_get):
44-
mock_get.return_value = MagicMock(
49+
@patch("iclaw.web_search.http.get_session")
50+
def test_search_startpage(self, mock_gs):
51+
mock_session = MagicMock()
52+
mock_gs.return_value = mock_session
53+
mock_session.get.return_value = MagicMock(
4554
ok=True,
4655
text='<html><div class="result"><a class="result-link" href="http://u"><div class="wgl-title">T</div></a></div></html>',
4756
)
@@ -50,14 +59,20 @@ def test_search_startpage(self, mock_get):
5059
self.assertEqual(results[0]["url"], "http://u")
5160
self.assertEqual(results[0]["title"], "T")
5261

53-
@patch("iclaw.web_search.requests.get", side_effect=Exception("err"))
54-
def test_search_startpage_error(self, mock_get):
62+
@patch("iclaw.web_search.http.get_session")
63+
def test_search_startpage_error(self, mock_gs):
64+
mock_gs.return_value.get.side_effect = Exception("err")
5565
with patch("sys.stdout"):
5666
results = web_search.search_startpage("q")
5767
self.assertEqual(results, [])
5868

69+
@patch("iclaw.web_search.http.get_session")
5970
@patch("iclaw.web_search.requests.Session")
60-
def test_search_bing(self, mock_session):
71+
def test_search_bing(self, mock_session, mock_gs):
72+
mock_shared = MagicMock()
73+
mock_shared.proxies = {}
74+
mock_shared.verify = True
75+
mock_gs.return_value = mock_shared
6176
mock_s = MagicMock()
6277
mock_session.return_value = mock_s
6378
mock_s.get.return_value = MagicMock(
@@ -69,8 +84,13 @@ def test_search_bing(self, mock_session):
6984
self.assertEqual(results[0]["url"], "http://u")
7085
self.assertEqual(results[0]["title"], "T")
7186

87+
@patch("iclaw.web_search.http.get_session")
7288
@patch("iclaw.web_search.requests.Session")
73-
def test_search_bing_double_slash(self, mock_session):
89+
def test_search_bing_double_slash(self, mock_session, mock_gs):
90+
mock_shared = MagicMock()
91+
mock_shared.proxies = {}
92+
mock_shared.verify = True
93+
mock_gs.return_value = mock_shared
7494
mock_s = MagicMock()
7595
mock_session.return_value = mock_s
7696
mock_s.get.return_value = MagicMock(
@@ -80,75 +100,81 @@ def test_search_bing_double_slash(self, mock_session):
80100
results = web_search.search_bing("q", num_results=1)
81101
self.assertEqual(results[0]["url"], "https://u.com")
82102

103+
@patch("iclaw.web_search.http.get_session")
83104
@patch("iclaw.web_search.requests.Session")
84-
def test_search_bing_error(self, mock_session):
105+
def test_search_bing_error(self, mock_session, mock_gs):
106+
mock_shared = MagicMock()
107+
mock_shared.proxies = {}
108+
mock_shared.verify = True
109+
mock_gs.return_value = mock_shared
85110
mock_s = MagicMock()
86111
mock_session.return_value = mock_s
87112
mock_s.get.side_effect = Exception("err")
88113
with patch("sys.stdout"):
89114
results = web_search.search_bing("q")
90115
self.assertEqual(results, [])
91116

92-
@patch("iclaw.web_search.requests.Session")
93-
def test_extract_text(self, mock_session):
94-
mock_s = MagicMock()
95-
mock_session.return_value = mock_s
96-
mock_s.get.return_value = MagicMock(
117+
@patch("iclaw.web_search.http.get_session")
118+
def test_extract_text(self, mock_gs):
119+
mock_session = MagicMock()
120+
mock_gs.return_value = mock_session
121+
mock_session.get.return_value = MagicMock(
97122
ok=True,
98123
status_code=200,
99124
text='<html><div id="firstHeading">T</div></html>',
100125
apparent_encoding="u8",
101126
)
102-
mock_s.get.return_value.apparent_encoding = "u8"
103127
self.assertIn(
104128
"T", web_search.extract_text_from_url("https://en.wikipedia.org/wiki/T")
105129
)
106130

107-
@patch("iclaw.web_search.requests.Session")
108-
def test_extract_text_non_200(self, mock_session):
109-
mock_s = MagicMock()
110-
mock_session.return_value = mock_s
111-
mock_s.get.return_value = MagicMock(status_code=404, apparent_encoding="utf-8")
131+
@patch("iclaw.web_search.http.get_session")
132+
def test_extract_text_non_200(self, mock_gs):
133+
mock_session = MagicMock()
134+
mock_gs.return_value = mock_session
135+
mock_session.get.return_value = MagicMock(
136+
status_code=404, apparent_encoding="utf-8"
137+
)
112138
result = web_search.extract_text_from_url("http://example.com")
113139
self.assertIn("404", result)
114140

115-
@patch("iclaw.web_search.requests.Session")
116-
def test_extract_text_exception(self, mock_session):
117-
mock_s = MagicMock()
118-
mock_session.return_value = mock_s
119-
mock_s.get.side_effect = Exception("timeout")
141+
@patch("iclaw.web_search.http.get_session")
142+
def test_extract_text_exception(self, mock_gs):
143+
mock_session = MagicMock()
144+
mock_gs.return_value = mock_session
145+
mock_session.get.side_effect = Exception("timeout")
120146
result = web_search.extract_text_from_url("http://example.com")
121147
self.assertIn("Error", result)
122148

123-
@patch("iclaw.web_search.requests.Session")
124-
def test_extract_text_zhihu(self, mock_session):
125-
mock_s = MagicMock()
126-
mock_session.return_value = mock_s
127-
mock_s.get.return_value = MagicMock(
149+
@patch("iclaw.web_search.http.get_session")
150+
def test_extract_text_zhihu(self, mock_gs):
151+
mock_session = MagicMock()
152+
mock_gs.return_value = mock_session
153+
mock_session.get.return_value = MagicMock(
128154
status_code=200,
129155
text='<html><body><h1 class="QuestionHeader-title">Question</h1><div class="RichContent-inner">Answer text</div></body></html>',
130156
apparent_encoding="utf-8",
131157
)
132158
result = web_search.extract_text_from_url("https://zhihu.com/question/123")
133159
self.assertIn("Question", result)
134160

135-
@patch("iclaw.web_search.requests.Session")
136-
def test_extract_text_github(self, mock_session):
137-
mock_s = MagicMock()
138-
mock_session.return_value = mock_s
139-
mock_s.get.return_value = MagicMock(
161+
@patch("iclaw.web_search.http.get_session")
162+
def test_extract_text_github(self, mock_gs):
163+
mock_session = MagicMock()
164+
mock_gs.return_value = mock_session
165+
mock_session.get.return_value = MagicMock(
140166
status_code=200,
141167
text='<html><body><div class="repository-content">Repo content here</div></body></html>',
142168
apparent_encoding="utf-8",
143169
)
144170
result = web_search.extract_text_from_url("https://github.com/user/repo")
145171
self.assertIn("Repo content", result)
146172

147-
@patch("iclaw.web_search.requests.Session")
148-
def test_extract_text_baidu_zhidao(self, mock_session):
149-
mock_s = MagicMock()
150-
mock_session.return_value = mock_s
151-
mock_s.get.return_value = MagicMock(
173+
@patch("iclaw.web_search.http.get_session")
174+
def test_extract_text_baidu_zhidao(self, mock_gs):
175+
mock_session = MagicMock()
176+
mock_gs.return_value = mock_session
177+
mock_session.get.return_value = MagicMock(
152178
status_code=200,
153179
text='<html><body><div class="wgt-best-content">Best answer</div></body></html>',
154180
apparent_encoding="utf-8",
@@ -159,11 +185,11 @@ def test_extract_text_baidu_zhidao(self, mock_session):
159185
self.assertIn("Best answer", result)
160186

161187
@patch("iclaw.web_search.Document")
162-
@patch("iclaw.web_search.requests.Session")
163-
def test_extract_text_generic_readability(self, mock_session, mock_doc):
164-
mock_s = MagicMock()
165-
mock_session.return_value = mock_s
166-
mock_s.get.return_value = MagicMock(
188+
@patch("iclaw.web_search.http.get_session")
189+
def test_extract_text_generic_readability(self, mock_gs, mock_doc):
190+
mock_session = MagicMock()
191+
mock_gs.return_value = mock_session
192+
mock_session.get.return_value = MagicMock(
167193
status_code=200,
168194
text="<html><body><p>Hello world content that is long enough to pass the 100 char threshold and needs to be extracted properly by the readability library for the test to pass.</p></body></html>",
169195
apparent_encoding="utf-8",
@@ -173,11 +199,11 @@ def test_extract_text_generic_readability(self, mock_session, mock_doc):
173199
self.assertIn("Hello world", result)
174200

175201
@patch("iclaw.web_search.Document")
176-
@patch("iclaw.web_search.requests.Session")
177-
def test_extract_text_generic_fallback(self, mock_session, mock_doc):
178-
mock_s = MagicMock()
179-
mock_session.return_value = mock_s
180-
mock_s.get.return_value = MagicMock(
202+
@patch("iclaw.web_search.http.get_session")
203+
def test_extract_text_generic_fallback(self, mock_gs, mock_doc):
204+
mock_session = MagicMock()
205+
mock_gs.return_value = mock_session
206+
mock_session.get.return_value = MagicMock(
181207
status_code=200,
182208
text="<html><body><main>Main content</main></body></html>",
183209
apparent_encoding="utf-8",
@@ -187,11 +213,11 @@ def test_extract_text_generic_fallback(self, mock_session, mock_doc):
187213
self.assertIn("Main content", result)
188214

189215
@patch("iclaw.web_search.Document")
190-
@patch("iclaw.web_search.requests.Session")
191-
def test_extract_text_body_fallback(self, mock_session, mock_doc):
192-
mock_s = MagicMock()
193-
mock_session.return_value = mock_s
194-
mock_s.get.return_value = MagicMock(
216+
@patch("iclaw.web_search.http.get_session")
217+
def test_extract_text_body_fallback(self, mock_gs, mock_doc):
218+
mock_session = MagicMock()
219+
mock_gs.return_value = mock_session
220+
mock_session.get.return_value = MagicMock(
195221
status_code=200,
196222
text="<html><body><p>Body text</p></body></html>",
197223
apparent_encoding="utf-8",

0 commit comments

Comments
 (0)