Skip to content

Commit 7e2567b

Browse files
ReikaNekoUwUReikaNekoUwU
authored
Add files via upload
1 parent acdb5d9 commit 7e2567b

8 files changed

Lines changed: 168 additions & 186 deletions

‎PyWWW/pywwwget_chatgpt.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1008,6 +1008,10 @@ def upload_file_to_pysftp_string(data, url):
10081008
# HTTP helpers (download only)
10091009
# --------------------------
10101010

1011+
# --------------------------
1012+
# HTTP helpers (download only)
1013+
# --------------------------
1014+
10111015
def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10121016
if headers is None:
10131017
headers = {}
@@ -1048,7 +1052,11 @@ def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10481052
auth = (username, password) if (username and password) else None
10491053
r = requests.get(rebuilt_url, headers=headers, auth=auth, stream=True, timeout=(5, 60))
10501054
r.raise_for_status()
1051-
shutil.copyfileobj(r.raw, httpfile)
1055+
r.raw.decode_content = True
1056+
#shutil.copyfileobj(r.raw, httpfile)
1057+
for chunk in r.iter_content(chunk_size=1024 * 1024):
1058+
if chunk:
1059+
httpfile.write(chunk)
10521060

10531061
# HTTPX
10541062
elif usehttp == "httpx" and havehttpx:
@@ -1078,7 +1086,7 @@ def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10781086
auth_headers = urllib3.make_headers(basic_auth="{}:{}".format(username, password))
10791087
headers.update(auth_headers)
10801088
# Request with preload_content=False to get a file-like object
1081-
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False)
1089+
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False, decode_content=True)
10821090
shutil.copyfileobj(resp, httpfile)
10831091
resp.release_conn()
10841092

‎PyWWW/pywwwget_deepseek.py‎

Lines changed: 62 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1214,131 +1214,106 @@ def upload_file_to_pysftp_string(data, url):
12141214
return upload_file_to_sftp_string(data, url)
12151215

12161216
# --------------------------
1217-
# HTTP Protocol Implementation
1217+
# HTTP helpers (download only)
12181218
# --------------------------
12191219

12201220
def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
1221-
"""Download file from HTTP/HTTPS server."""
12221221
if headers is None:
12231222
headers = {}
1224-
12251223
p = urlparse(url)
1226-
1227-
# Extract auth from URL
12281224
username = unquote(p.username) if p.username else None
12291225
password = unquote(p.password) if p.password else None
1230-
1231-
# Remove auth from URL for libraries that don't support URL auth
1226+
1227+
# Strip auth from URL
12321228
netloc = p.hostname or ""
12331229
if p.port:
12341230
netloc += ":" + str(p.port)
12351231
rebuilt_url = urlunparse((p.scheme, netloc, p.path, p.params, p.query, p.fragment))
1236-
1237-
# Parse query string for resume options
1232+
1233+
# HTTP resume: ?resume=1&resume_to=/path
12381234
qs = parse_qs(p.query or "")
12391235
resume = _qflag(qs, "resume", False)
12401236
resume_to = _qstr(qs, "resume_to", None)
1241-
1242-
# Handle resume
12431237
httpfile = MkTempFile()
12441238
resume_off = 0
1245-
12461239
if resume and resume_to:
12471240
try:
12481241
if os.path.exists(resume_to):
12491242
httpfile = open(resume_to, "ab+")
12501243
httpfile.seek(0, 2)
1251-
resume_off = httpfile.tell()
1244+
resume_off = int(httpfile.tell())
12521245
else:
12531246
_ensure_dir(os.path.dirname(resume_to) or ".")
12541247
httpfile = open(resume_to, "wb+")
12551248
resume_off = 0
12561249
except Exception:
12571250
httpfile = MkTempFile()
12581251
resume_off = 0
1259-
12601252
if resume_off and "Range" not in headers and "range" not in headers:
12611253
headers["Range"] = "bytes=%d-" % resume_off
1262-
1263-
try:
1264-
# Requests library
1265-
if usehttp == "requests" and haverequests:
1254+
1255+
# Requests
1256+
if usehttp == "requests" and haverequests:
1257+
auth = (username, password) if (username and password) else None
1258+
r = requests.get(rebuilt_url, headers=headers, auth=auth, stream=True, timeout=(5, 60))
1259+
r.raise_for_status()
1260+
r.raw.decode_content = True
1261+
#shutil.copyfileobj(r.raw, httpfile)
1262+
for chunk in r.iter_content(chunk_size=1024 * 1024):
1263+
if chunk:
1264+
httpfile.write(chunk)
1265+
1266+
# HTTPX
1267+
elif usehttp == "httpx" and havehttpx:
1268+
with httpx.Client(follow_redirects=True, timeout=60.0) as client:
12661269
auth = (username, password) if (username and password) else None
1267-
r = requests.get(rebuilt_url, headers=headers, auth=auth,
1268-
stream=True, timeout=(5, 60))
1270+
r = client.get(rebuilt_url, headers=headers, auth=auth)
12691271
r.raise_for_status()
1270-
1271-
for chunk in r.iter_content(chunk_size=8192):
1272+
for chunk in r.iter_bytes():
12721273
if chunk:
12731274
httpfile.write(chunk)
1274-
1275-
# HTTPX library
1276-
elif usehttp == "httpx" and havehttpx:
1277-
with httpx.Client(follow_redirects=True, timeout=60.0) as client:
1278-
auth = (username, password) if (username and password) else None
1279-
r = client.get(rebuilt_url, headers=headers, auth=auth)
1280-
r.raise_for_status()
1281-
1282-
for chunk in r.iter_bytes():
1283-
if chunk:
1284-
httpfile.write(chunk)
1285-
1286-
# Mechanize library
1287-
elif usehttp == "mechanize" and havemechanize:
1288-
br = mechanize.Browser()
1289-
br.set_handle_robots(False)
1290-
1291-
if headers:
1292-
br.addheaders = list(headers.items())
1293-
1294-
if username and password:
1295-
br.add_password(rebuilt_url, username, password)
1296-
1297-
resp = br.open(rebuilt_url)
1298-
shutil.copyfileobj(resp, httpfile)
1299-
1300-
# URLLib3
1301-
elif usehttp == "urllib3" and haveurllib3:
1302-
http = urllib3.PoolManager()
1303-
if username and password:
1304-
auth_headers = urllib3.make_headers(basic_auth="{}:{}".format(username, password))
1305-
headers.update(auth_headers)
1306-
# Request with preload_content=False to get a file-like object
1307-
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False)
1308-
shutil.copyfileobj(resp, httpfile)
1309-
resp.release_conn()
1310-
1311-
# urllib fallback
1275+
1276+
# Mechanize
1277+
elif usehttp == "mechanize" and havemechanize:
1278+
br = mechanize.Browser()
1279+
br.set_handle_robots(False)
1280+
if headers:
1281+
br.addheaders = list(headers.items())
1282+
if username and password:
1283+
br.add_password(rebuilt_url, username, password)
1284+
resp = br.open(rebuilt_url)
1285+
shutil.copyfileobj(resp, httpfile)
1286+
1287+
# URLLib3
1288+
elif usehttp == "urllib3" and haveurllib3:
1289+
http = urllib3.PoolManager()
1290+
if username and password:
1291+
auth_headers = urllib3.make_headers(basic_auth="{}:{}".format(username, password))
1292+
headers.update(auth_headers)
1293+
# Request with preload_content=False to get a file-like object
1294+
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False, decode_content=True)
1295+
shutil.copyfileobj(resp, httpfile)
1296+
resp.release_conn()
1297+
1298+
# urllib fallback
1299+
else:
1300+
req = Request(rebuilt_url, headers=headers)
1301+
if username and password:
1302+
mgr = HTTPPasswordMgrWithDefaultRealm()
1303+
mgr.add_password(None, rebuilt_url, username, password)
1304+
opener = build_opener(HTTPBasicAuthHandler(mgr))
13121305
else:
1313-
req = Request(rebuilt_url, headers=headers)
1314-
1315-
if username and password:
1316-
mgr = HTTPPasswordMgrWithDefaultRealm()
1317-
mgr.add_password(None, rebuilt_url, username, password)
1318-
opener = build_opener(HTTPBasicAuthHandler(mgr))
1319-
else:
1320-
opener = build_opener()
1321-
1322-
resp = opener.open(req, timeout=60)
1323-
shutil.copyfileobj(resp, httpfile)
1324-
1325-
try:
1326-
httpfile.seek(0, 0)
1327-
except Exception:
1328-
pass
1329-
1330-
return httpfile
1331-
1332-
except Exception as e:
1333-
_net_log(True, "HTTP download error: %s" % str(e))
1334-
try:
1335-
httpfile.close()
1336-
except Exception:
1337-
pass
1338-
return False
1306+
opener = build_opener()
1307+
resp = opener.open(req)
1308+
shutil.copyfileobj(resp, httpfile)
1309+
1310+
try:
1311+
httpfile.seek(0, 0)
1312+
except Exception:
1313+
pass
1314+
return httpfile
13391315

13401316
def download_file_from_http_string(url, headers=None, usehttp=__use_http_lib__):
1341-
"""Download from HTTP/HTTPS as string."""
13421317
fp = download_file_from_http_file(url, headers=headers, usehttp=usehttp)
13431318
return fp.read() if fp else False
13441319

‎PyWWW/pywwwget_merged.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1055,7 +1055,11 @@ def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10551055
auth = (username, password) if (username and password) else None
10561056
r = requests.get(rebuilt_url, headers=headers, auth=auth, stream=True, timeout=(5, 60))
10571057
r.raise_for_status()
1058-
shutil.copyfileobj(r.raw, httpfile)
1058+
r.raw.decode_content = True
1059+
#shutil.copyfileobj(r.raw, httpfile)
1060+
for chunk in r.iter_content(chunk_size=1024 * 1024):
1061+
if chunk:
1062+
httpfile.write(chunk)
10591063

10601064
# HTTPX
10611065
elif usehttp == "httpx" and havehttpx:
@@ -1085,7 +1089,7 @@ def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10851089
auth_headers = urllib3.make_headers(basic_auth="{}:{}".format(username, password))
10861090
headers.update(auth_headers)
10871091
# Request with preload_content=False to get a file-like object
1088-
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False)
1092+
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False, decode_content=True)
10891093
shutil.copyfileobj(resp, httpfile)
10901094
resp.release_conn()
10911095

‎PyWWW/pywwwget_nextver.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,7 +1069,11 @@ def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10691069
auth = (username, password) if (username and password) else None
10701070
r = requests.get(rebuilt_url, headers=headers, auth=auth, stream=True, timeout=(5, 60))
10711071
r.raise_for_status()
1072-
shutil.copyfileobj(r.raw, httpfile)
1072+
r.raw.decode_content = True
1073+
#shutil.copyfileobj(r.raw, httpfile)
1074+
for chunk in r.iter_content(chunk_size=1024 * 1024):
1075+
if chunk:
1076+
httpfile.write(chunk)
10731077

10741078
# HTTPX
10751079
elif usehttp == "httpx" and havehttpx:
@@ -1099,7 +1103,7 @@ def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10991103
auth_headers = urllib3.make_headers(basic_auth="{}:{}".format(username, password))
11001104
headers.update(auth_headers)
11011105
# Request with preload_content=False to get a file-like object
1102-
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False)
1106+
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False, decode_content=True)
11031107
shutil.copyfileobj(resp, httpfile)
11041108
resp.release_conn()
11051109

‎pywwwget_chatgpt.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1008,6 +1008,10 @@ def upload_file_to_pysftp_string(data, url):
10081008
# HTTP helpers (download only)
10091009
# --------------------------
10101010

1011+
# --------------------------
1012+
# HTTP helpers (download only)
1013+
# --------------------------
1014+
10111015
def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10121016
if headers is None:
10131017
headers = {}
@@ -1048,7 +1052,11 @@ def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10481052
auth = (username, password) if (username and password) else None
10491053
r = requests.get(rebuilt_url, headers=headers, auth=auth, stream=True, timeout=(5, 60))
10501054
r.raise_for_status()
1051-
shutil.copyfileobj(r.raw, httpfile)
1055+
r.raw.decode_content = True
1056+
#shutil.copyfileobj(r.raw, httpfile)
1057+
for chunk in r.iter_content(chunk_size=1024 * 1024):
1058+
if chunk:
1059+
httpfile.write(chunk)
10521060

10531061
# HTTPX
10541062
elif usehttp == "httpx" and havehttpx:
@@ -1078,7 +1086,7 @@ def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
10781086
auth_headers = urllib3.make_headers(basic_auth="{}:{}".format(username, password))
10791087
headers.update(auth_headers)
10801088
# Request with preload_content=False to get a file-like object
1081-
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False)
1089+
resp = http.request("GET", rebuilt_url, headers=headers, preload_content=False, decode_content=True)
10821090
shutil.copyfileobj(resp, httpfile)
10831091
resp.release_conn()
10841092

0 commit comments

Comments
 (0)