diff --git a/Dockerfile b/Dockerfile index 89019cd7f..e9990f557 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,13 @@ FROM python:3.6-alpine +# 设置标签 +LABEL name="proxy-pool" \ + version="1.0.0" \ + project="ci" \ + platform="linux/amd64,linux/arm64" \ + maintainer="andy" \ + description="Proxy Pool Service" + MAINTAINER jhao104 WORKDIR /app diff --git a/api/proxyApi.py b/api/proxyApi.py index bd2de57e2..a134c2623 100644 --- a/api/proxyApi.py +++ b/api/proxyApi.py @@ -46,8 +46,9 @@ def force_type(cls, response, environ=None): {"url": "/pop", "params": "", "desc": "get and delete a proxy"}, {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unable proxy"}, {"url": "/all", "params": "type: ''https'|''", "desc": "get all proxy from proxy pool"}, - {"url": "/count", "params": "", "desc": "return proxy count"} - # 'refresh': 'refresh proxy pool', + {"url": "/count", "params": "", "desc": "return proxy count"}, + {"url": "/get_anonymous", "params": "type: ''https'|''", "desc": "get an anonymous proxy"}, + {"url": "/all_anonymous", "params": "type: ''https'|''", "desc": "get all anonymous proxy"}, ] @@ -93,6 +94,7 @@ def delete(): @app.route('/count/') def getCount(): proxies = proxy_handler.getAll() + anonymous_proxies = proxy_handler.getAllAnonymous() http_type_dict = {} source_dict = {} for proxy in proxies: @@ -100,7 +102,28 @@ def getCount(): http_type_dict[http_type] = http_type_dict.get(http_type, 0) + 1 for source in proxy.source.split('/'): source_dict[source] = source_dict.get(source, 0) + 1 - return {"http_type": http_type_dict, "source": source_dict, "count": len(proxies)} + return { + "http_type": http_type_dict, + "source": source_dict, + "count": len(proxies), + "anonymous_count": len(anonymous_proxies) + } + + +@app.route('/get_anonymous/') +def get_anonymous(): + """获取一个匿名代理""" + https = request.args.get("type", "").lower() == 'https' + proxy = proxy_handler.getAnonymous(https) + return proxy.to_dict if proxy else {"code": 0, "src": "no anonymous proxy"} + + +@app.route('/all_anonymous/') +def get_all_anonymous(): + """获取所有匿名代理""" + https = request.args.get("type", "").lower() == 'https' + proxies = proxy_handler.getAllAnonymous(https) + return jsonify([_.to_dict for _ in proxies]) def runFlask(): @@ -118,7 +141,7 @@ def __init__(self, app, options=None): def load_config(self): _config = dict([(key, value) for key, value in iteritems(self.options) - if key in self.cfg.settings and value is not None]) + if key in self.cfg.settings and value is not None]) for key, value in iteritems(_config): self.cfg.set(key.lower(), value) diff --git a/db/dbClient.py b/db/dbClient.py index 4d9554b18..804eaa7fa 100644 --- a/db/dbClient.py +++ b/db/dbClient.py @@ -116,5 +116,23 @@ def changeTable(self, name): def getCount(self): return self.client.getCount() + def changeAnonymousTable(self, name): + self.client.changeAnonymousTable(name) + + def putAnonymous(self, key, **kwargs): + return self.client.putAnonymous(key, **kwargs) + + def getAnonymous(self, https, **kwargs): + return self.client.getAnonymous(https, **kwargs) + + def getAllAnonymous(self, https): + return self.client.getAllAnonymous(https) + + def deleteAnonymous(self, key, **kwargs): + return self.client.deleteAnonymous(key, **kwargs) + + def getAnonymousCount(self): + return self.client.getAnonymousCount() + def test(self): return self.client.test() diff --git a/db/redisClient.py b/db/redisClient.py index e66614d7e..4105cc32f 100644 --- a/db/redisClient.py +++ b/db/redisClient.py @@ -138,6 +138,73 @@ def changeTable(self, name): """ self.name = name + def changeAnonymousTable(self, name): + """ + 切换匿名代理表 + :param name: + :return: + """ + self.anonymous_name = name + + def putAnonymous(self, proxy_obj): + """ + 将匿名代理放入 anonymous hash + :param proxy_obj: Proxy obj + :return: + """ + if not hasattr(self, 'anonymous_name') or not self.anonymous_name: + return 0 + return self.__conn.hset(self.anonymous_name, proxy_obj.proxy, proxy_obj.to_json) + + def getAnonymous(self, https): + """ + 返回一个匿名代理 + :return: + """ + if not hasattr(self, 'anonymous_name') or not self.anonymous_name: + return None + if https: + items = self.__conn.hvals(self.anonymous_name) + proxies = list(filter(lambda x: json.loads(x).get("https"), items)) + return choice(proxies) if proxies else None + else: + proxies = self.__conn.hkeys(self.anonymous_name) + proxy = choice(proxies) if proxies else None + return self.__conn.hget(self.anonymous_name, proxy) if proxy else None + + def getAllAnonymous(self, https): + """ + 返回所有匿名代理 + :return: + """ + if not hasattr(self, 'anonymous_name') or not self.anonymous_name: + return [] + items = self.__conn.hvals(self.anonymous_name) + if https: + return list(filter(lambda x: json.loads(x).get("https"), items)) + else: + return items + + def deleteAnonymous(self, proxy_str): + """ + 删除匿名代理 + :param proxy_str: + :return: + """ + if not hasattr(self, 'anonymous_name') or not self.anonymous_name: + return 0 + return self.__conn.hdel(self.anonymous_name, proxy_str) + + def getAnonymousCount(self): + """ + 返回匿名代理数量 + :return: + """ + if not hasattr(self, 'anonymous_name') or not self.anonymous_name: + return {'total': 0, 'https': 0} + proxies = self.getAllAnonymous(https=False) + return {'total': len(proxies), 'https': len(list(filter(lambda x: json.loads(x).get("https"), proxies)))} + def test(self): log = LogHandler('redis_client') try: diff --git a/docker-compose.yml b/docker-compose.yml index 9d1a10ba4..85cdc5d7c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,10 @@ version: '2' services: proxy_pool: - build: . - container_name: proxy_pool + image: harbor.vocoor.com/ci/proxy-pool:latest + container_name: proxy-pool ports: - "5010:5010" - links: - - proxy_redis environment: - DB_CONN: "redis://@proxy_redis:6379/0" - proxy_redis: - image: "redis" - container_name: proxy_redis \ No newline at end of file + DB_CONN: "redis://:lilishop@192.168.2.130:6379/10" + restart: unless-stopped \ No newline at end of file diff --git a/fetcher/freeproxyAdapter.py b/fetcher/freeproxyAdapter.py new file mode 100644 index 000000000..3f8624fbb --- /dev/null +++ b/fetcher/freeproxyAdapter.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name: freeproxyAdapter.py + Description : 额外的免费代理源(移植自 freeproxy 项目) + Author : Antigravity + date: 2026/02/03 +------------------------------------------------- + 将 freeproxy 项目的代理源核心逻辑移植到此处 + 不依赖外部项目,可在 Docker 容器中独立运行 +------------------------------------------------- +""" +import re +import requests +from time import sleep + + +class FreeproxyAdapter: + """ + 额外的免费代理源 + + 从多个免费代理源获取代理,返回 host:port 格式 + """ + + DEFAULT_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + } + + def __init__(self, timeout=15): + self.timeout = timeout + self.session = requests.Session() + self.session.headers.update(self.DEFAULT_HEADERS) + + def fetch_thespeedx(self): + """ + TheSpeedX 代理列表 + 来源: https://github.com/TheSpeedX/SOCKS-List + """ + urls = [ + 'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt', + ] + + for url in urls: + try: + resp = self.session.get(url, timeout=self.timeout) + resp.raise_for_status() + for line in resp.text.strip().split('\n'): + line = line.strip() + if line and ':' in line: + # 验证格式 ip:port + parts = line.split(':') + if len(parts) == 2 and parts[1].isdigit(): + yield line + except Exception as e: + print(f"[TheSpeedX] 获取失败: {e}") + + def fetch_proxyscrape(self): + """ + ProxyScrape API + 来源: https://proxyscrape.com/free-proxy-list + """ + try: + url = "https://api.proxyscrape.com/v4/free-proxy-list/get?request=get_proxies&skip=0&proxy_format=protocolipport&format=json&limit=500" + resp = self.session.get(url, timeout=self.timeout) + resp.raise_for_status() + data = resp.json() + + for item in data.get('proxies', []): + if not item.get('alive'): + continue + ip = item.get('ip') + port = item.get('port') + if ip and port: + yield f"{ip}:{port}" + except Exception as e: + print(f"[ProxyScrape] 获取失败: {e}") + + def fetch_geonode(self): + """ + Geonode 代理列表 + 来源: https://geonode.com/free-proxy-list + """ + try: + url = "https://proxylist.geonode.com/api/proxy-list?limit=200&page=1&sort_by=lastChecked&sort_type=desc" + resp = self.session.get(url, timeout=self.timeout) + resp.raise_for_status() + data = resp.json() + + for item in data.get('data', []): + ip = item.get('ip') + port = item.get('port') + if ip and port: + yield f"{ip}:{port}" + except Exception as e: + print(f"[Geonode] 获取失败: {e}") + + def fetch_freeproxylist(self): + """ + Free Proxy List + 来源: https://free-proxy-list.net/ + """ + try: + url = "https://free-proxy-list.net/" + resp = self.session.get(url, timeout=self.timeout) + resp.raise_for_status() + + # 解析 HTML 中的代理 + pattern = r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(\d+)' + matches = re.findall(pattern, resp.text) + for ip, port in matches: + yield f"{ip}:{port}" + except Exception as e: + print(f"[FreeProxyList] 获取失败: {e}") + + def fetch_proxylistdownload(self): + """ + Proxy List Download + 来源: https://www.proxy-list.download/ + """ + urls = [ + 'https://www.proxy-list.download/api/v1/get?type=http', + 'https://www.proxy-list.download/api/v1/get?type=https', + ] + + for url in urls: + try: + resp = self.session.get(url, timeout=self.timeout) + resp.raise_for_status() + for line in resp.text.strip().split('\n'): + line = line.strip() + if line and ':' in line: + yield line + except Exception as e: + print(f"[ProxyListDownload] 获取失败: {e}") + + def fetch_sunny9577(self): + """ + Sunny9577 GitHub 代理列表 + 来源: https://github.com/sunny9577/proxy-scraper + """ + urls = [ + 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt', + ] + + for url in urls: + try: + resp = self.session.get(url, timeout=self.timeout) + resp.raise_for_status() + for line in resp.text.strip().split('\n'): + line = line.strip() + if line and ':' in line: + parts = line.split(':') + if len(parts) == 2 and parts[1].isdigit(): + yield line + except Exception as e: + print(f"[Sunny9577] 获取失败: {e}") + + def fetch_clarketm(self): + """ + Clarketm GitHub 代理列表 + 来源: https://github.com/clarketm/proxy-list + """ + try: + url = 'https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt' + resp = self.session.get(url, timeout=self.timeout) + resp.raise_for_status() + for line in resp.text.strip().split('\n'): + line = line.strip() + if line and ':' in line: + parts = line.split(':') + if len(parts) == 2 and parts[1].isdigit(): + yield line + except Exception as e: + print(f"[Clarketm] 获取失败: {e}") + + def fetch_all(self): + """ + 从所有代理源获取代理 + + Yields: + 代理地址 (host:port 格式) + """ + seen = set() + sources = [ + ('TheSpeedX', self.fetch_thespeedx), + ('ProxyScrape', self.fetch_proxyscrape), + ('Geonode', self.fetch_geonode), + ('FreeProxyList', self.fetch_freeproxylist), + ('ProxyListDownload', self.fetch_proxylistdownload), + ('Sunny9577', self.fetch_sunny9577), + ('Clarketm', self.fetch_clarketm), + ] + + for name, fetcher in sources: + try: + count = 0 + for proxy in fetcher(): + if proxy not in seen: + seen.add(proxy) + count += 1 + yield proxy + if count > 0: + print(f"[{name}] 获取到 {count} 个代理") + except Exception as e: + print(f"[{name}] 处理失败: {e}") + sleep(0.5) # 避免请求过快 + + +def get_freeproxy_proxies(): + """ + 便捷函数:获取所有外部代理源的代理 + + Yields: + 代理地址 (host:port 格式) + """ + adapter = FreeproxyAdapter() + for proxy in adapter.fetch_all(): + yield proxy + + +if __name__ == '__main__': + # 测试 + print("测试 FreeproxyAdapter...") + adapter = FreeproxyAdapter() + count = 0 + for proxy in adapter.fetch_all(): + print(f" {proxy}") + count += 1 + print(f"\n总共获取到 {count} 个代理") diff --git a/fetcher/proxyFetcher.py b/fetcher/proxyFetcher.py index cfc37f928..d40558e6d 100644 --- a/fetcher/proxyFetcher.py +++ b/fetcher/proxyFetcher.py @@ -170,6 +170,20 @@ def freeProxy11(): except Exception as e: print(e) + @staticmethod + def freeProxyExternal(): + """ + 外部免费代理源 + 包含多个 GitHub 和 API 代理源 + """ + try: + from fetcher.freeproxyAdapter import FreeproxyAdapter + adapter = FreeproxyAdapter() + for proxy in adapter.fetch_all(): + yield proxy + except Exception as e: + print(f"[freeProxyExternal] 获取代理失败: {e}") + # @staticmethod # def wallProxy01(): # """ diff --git a/handler/configHandler.py b/handler/configHandler.py index 29000bcc6..2abf31fa5 100644 --- a/handler/configHandler.py +++ b/handler/configHandler.py @@ -40,6 +40,14 @@ def dbConn(self): def tableName(self): return os.getenv("TABLE_NAME", setting.TABLE_NAME) + @LazyProperty + def anonymousTableName(self): + return os.getenv("ANONYMOUS_TABLE_NAME", setting.ANONYMOUS_TABLE_NAME) + + @LazyProperty + def anonymousCheckUrl(self): + return os.getenv("ANONYMOUS_CHECK_URL", setting.ANONYMOUS_CHECK_URL) + @property def fetchers(self): reload_six(setting) diff --git a/handler/proxyHandler.py b/handler/proxyHandler.py index 32e215e5d..15ea99435 100644 --- a/handler/proxyHandler.py +++ b/handler/proxyHandler.py @@ -25,6 +25,7 @@ def __init__(self): self.conf = ConfigHandler() self.db = DbClient(self.conf.dbConn) self.db.changeTable(self.conf.tableName) + self.db.changeAnonymousTable(self.conf.anonymousTableName) def get(self, https=False): """ @@ -84,3 +85,46 @@ def getCount(self): """ total_use_proxy = self.db.getCount() return {'count': total_use_proxy} + + # 匿名代理相关方法 + + def getAnonymous(self, https=False): + """ + return an anonymous proxy + Args: + https: True/False + Returns: + """ + proxy = self.db.getAnonymous(https) + return Proxy.createFromJson(proxy) if proxy else None + + def getAllAnonymous(self, https=False): + """ + get all anonymous proxy from pool as Proxy list + :return: + """ + proxies = self.db.getAllAnonymous(https) + return [Proxy.createFromJson(_) for _ in proxies] + + def putAnonymous(self, proxy): + """ + put proxy into anonymous proxy pool + :param proxy: + :return: + """ + self.db.putAnonymous(proxy) + + def deleteAnonymous(self, proxy): + """ + delete anonymous proxy + :param proxy: + :return: + """ + return self.db.deleteAnonymous(proxy.proxy) + + def getAnonymousCount(self): + """ + return anonymous proxy count + :return: + """ + return self.db.getAnonymousCount() diff --git a/helper/anonymousChecker.py b/helper/anonymousChecker.py new file mode 100644 index 000000000..4729138b0 --- /dev/null +++ b/helper/anonymousChecker.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name: anonymousChecker.py + Description : 匿名代理检测模块(增强版) + Author : Antigravity + date: 2026/02/04 +------------------------------------------------- + 主要改进: + 1. 使用多端点交叉验证(httpbin.org/ip + httpbin.org/headers) + 2. 检测 HTTP 头中是否暴露真实 IP + 3. 只有通过严格验证的代理才存入匿名库 +------------------------------------------------- +""" +import requests +from concurrent.futures import ThreadPoolExecutor, as_completed + +from handler.logHandler import LogHandler +from handler.proxyHandler import ProxyHandler +from handler.configHandler import ConfigHandler + + +# 常见的暴露真实 IP 的 HTTP 头 +PROXY_HEADERS = [ + 'X-Forwarded-For', + 'X-Real-Ip', + 'X-Forwarded', + 'Forwarded-For', + 'Forwarded', + 'Via', + 'X-Client-Ip', + 'Client-Ip', + 'True-Client-Ip', + 'Cf-Connecting-Ip', +] + + +class AnonymousChecker: + """匿名代理检测器(增强版)""" + + def __init__(self, timeout=10, max_workers=10): + self.log = LogHandler("anonymous_checker") + self.conf = ConfigHandler() + self.proxy_handler = ProxyHandler() + self.timeout = timeout + self.max_workers = max_workers + self.check_url = self.conf.anonymousCheckUrl + self.headers_url = self.check_url.replace('/ip', '/headers') + self._real_ip = None + + def _get_real_ip(self): + """获取本机真实出口 IP(不使用代理)""" + if self._real_ip: + return self._real_ip + try: + resp = requests.get(self.check_url, timeout=self.timeout) + resp.raise_for_status() + self._real_ip = resp.json().get("origin", "").split(',')[0].strip() + self.log.info(f"本机真实 IP: {self._real_ip}") + return self._real_ip + except Exception as e: + self.log.warning(f"获取本机真实 IP 失败: {e}") + return None + + def _check_anonymous(self, proxy, real_ip=None): + """ + 增强的匿名性检测 + + 严格检测条件: + 1. 请求能够成功 + 2. origin IP 只有代理 IP,不包含真实 IP + 3. HTTP 响应头中无暴露真实 IP 的信息 + + Args: + proxy: Proxy 对象 + real_ip: 本机真实 IP(用于对比验证) + + Returns: + (proxy, is_anonymous, origin_ip, reason) 元组 + """ + proxy_str = proxy.proxy # host:port + proxy_ip = proxy_str.split(':')[0] + + proxies = { + "http": f"http://{proxy_str}", + "https": f"http://{proxy_str}", + } + + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + } + + try: + # 步骤 1: 检测 origin IP + resp_ip = requests.get( + self.check_url, + proxies=proxies, + timeout=self.timeout, + headers=headers + ) + resp_ip.raise_for_status() + + origin = resp_ip.json().get("origin", "") + origin_ips = [ip.strip() for ip in origin.split(',')] + + # 检查 1: 如果包含真实 IP,直接判定为透明代理 + if real_ip and real_ip in origin_ips: + return proxy, False, origin, "透明代理(暴露真实IP)" + + # 检查 2: origin 中应该只有一个 IP 且是代理 IP + if len(origin_ips) != 1: + return proxy, False, origin, "普通匿名(多IP)" + + if origin_ips[0] != proxy_ip: + # 如果 origin 不是代理 IP,可能是代理的出口 IP 不同,仍算匿名 + # 但需要确保不是真实 IP + if real_ip and origin_ips[0] == real_ip: + return proxy, False, origin, "透明代理" + + # 步骤 2: 检测 HTTP 头是否暴露信息 + try: + resp_headers = requests.get( + self.headers_url, + proxies=proxies, + timeout=self.timeout, + headers=headers + ) + resp_headers.raise_for_status() + + response_headers = resp_headers.json().get("headers", {}) + + # 检查是否有暴露真实 IP 的头 + for header in PROXY_HEADERS: + header_value = response_headers.get(header, "") + if header_value and real_ip and real_ip in header_value: + return proxy, False, origin, f"头部暴露({header})" + + except Exception: + # headers 检测失败,降低信任度但仍可能是匿名代理 + self.log.debug(f"检测 {proxy_str} 的 headers 失败,仅通过 IP 检测") + + # 所有检查通过,为高匿代理 + return proxy, True, origin, "高匿代理" + + except requests.exceptions.Timeout: + return proxy, False, None, "超时" + except requests.exceptions.ProxyError: + return proxy, False, None, "代理连接失败" + except Exception as e: + self.log.debug(f"检测 {proxy_str} 匿名性失败: {e}") + return proxy, False, None, f"异常: {str(e)[:50]}" + + def run(self): + """ + 执行匿名性检测 + + 获取所有可用代理,使用增强检测其匿名性, + 只有真正高匿的代理才存入 anonymous_proxy 表 + """ + self.log.info("开始匿名性检测(增强版)...") + + # 获取本机真实 IP,用于严格验证 + real_ip = self._get_real_ip() + if not real_ip: + self.log.warning("未能获取本机 IP,将使用基础检测模式") + + # 获取所有可用代理 + all_proxies = self.proxy_handler.getAll() + if not all_proxies: + self.log.info("没有可用代理,跳过匿名性检测") + return {"anonymous": 0, "transparent": 0, "failed": 0} + + self.log.info(f"共有 {len(all_proxies)} 个代理需要检测") + + anonymous_count = 0 + transparent_count = 0 + failed_count = 0 + + # 并发检测 + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + futures = { + executor.submit(self._check_anonymous, proxy, real_ip): proxy + for proxy in all_proxies + } + + for future in as_completed(futures): + proxy, is_anonymous, origin, reason = future.result() + + if origin is None: + failed_count += 1 + continue + + if is_anonymous: + # 只有高匿代理才存入 + self.proxy_handler.putAnonymous(proxy) + anonymous_count += 1 + self.log.debug(f"高匿代理: {proxy.proxy} -> {origin}") + else: + transparent_count += 1 + self.log.debug(f"{reason}: {proxy.proxy} -> {origin}") + + self.log.info( + f"匿名性检测完成: " + f"高匿={anonymous_count}, 透明/普通={transparent_count}, 失败={failed_count}" + ) + + return { + "anonymous": anonymous_count, + "transparent": transparent_count, + "failed": failed_count + } + + +def runAnonymousCheck(): + """运行匿名性检测的便捷函数""" + checker = AnonymousChecker() + return checker.run() + + +if __name__ == '__main__': + # 测试 + print("=== 匿名性检测(增强版) ===") + result = runAnonymousCheck() + print(f"检测结果: {result}") diff --git a/helper/check.py b/helper/check.py index 937645c0f..12e2de8ce 100644 --- a/helper/check.py +++ b/helper/check.py @@ -78,12 +78,68 @@ def preValidator(cls, proxy): @classmethod def regionGetter(cls, proxy): + """ + 获取代理 IP 的地区信息 + 使用多个备用 API + """ + ip = proxy.proxy.split(':')[0] + + # API 列表(按可靠性排序) + apis = [ + # ip-api.com - 免费,稳定 + lambda: cls._get_region_ip_api(ip), + # ipinfo.io - 免费,限制较多 + lambda: cls._get_region_ipinfo(ip), + # ipwho.is - 免费,无限制 + lambda: cls._get_region_ipwho(ip), + ] + + for api_func in apis: + try: + result = api_func() + if result and result != 'error': + return result + except: + continue + return 'unknown' + + @classmethod + def _get_region_ip_api(cls, ip): + """ip-api.com""" + try: + url = f'http://ip-api.com/json/{ip}?fields=status,country,countryCode' + r = WebRequest().get(url=url, retry_time=1, timeout=3).json + if r.get('status') == 'success': + return f"{r.get('country', '')}({r.get('countryCode', '')})" + except: + pass + return None + + @classmethod + def _get_region_ipinfo(cls, ip): + """ipinfo.io""" + try: + url = f'https://ipinfo.io/{ip}/json' + r = WebRequest().get(url=url, retry_time=1, timeout=3).json + country = r.get('country', '') + city = r.get('city', '') + if country: + return f"{city},{country}" if city else country + except: + pass + return None + + @classmethod + def _get_region_ipwho(cls, ip): + """ipwho.is""" try: - url = 'https://searchplugin.csdn.net/api/v1/ip/get?ip=%s' % proxy.proxy.split(':')[0] - r = WebRequest().get(url=url, retry_time=1, timeout=2).json - return r['data']['address'] + url = f'https://ipwho.is/{ip}' + r = WebRequest().get(url=url, retry_time=1, timeout=3).json + if r.get('success'): + return f"{r.get('country', '')}({r.get('country_code', '')})" except: - return 'error' + pass + return None class _ThreadChecker(Thread): diff --git a/helper/scheduler.py b/helper/scheduler.py index cd91190a5..27e17da67 100644 --- a/helper/scheduler.py +++ b/helper/scheduler.py @@ -43,6 +43,16 @@ def __runProxyCheck(): proxy_queue.put(proxy) Checker("use", proxy_queue) + # 常规检测完成后执行匿名性检测 + __runAnonymousCheck() + + +def __runAnonymousCheck(): + """匿名性检测(增强版)""" + from helper.anonymousChecker import AnonymousChecker + checker = AnonymousChecker() + checker.run() + def runScheduler(): __runProxyFetch() diff --git a/setting.py b/setting.py index 9bab8475c..9391ef121 100644 --- a/setting.py +++ b/setting.py @@ -42,6 +42,12 @@ # proxy table name TABLE_NAME = 'use_proxy' +# anonymous proxy table name +ANONYMOUS_TABLE_NAME = 'anonymous_proxy' + +# anonymous check url +ANONYMOUS_CHECK_URL = 'http://httpbin.org/ip' + # ###### config the proxy fetch function ###### PROXY_FETCHER = [ @@ -55,7 +61,8 @@ "freeProxy08", "freeProxy09", "freeProxy10", - "freeProxy11" + "freeProxy11", + "freeProxyExternal", # freeproxy 项目代理源 ] # ############# proxy validator ################# diff --git a/test/testAnonymousCheck.py b/test/testAnonymousCheck.py new file mode 100644 index 000000000..9ed970682 --- /dev/null +++ b/test/testAnonymousCheck.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +""" +手动触发匿名性检测测试脚本 + +用于测试匿名代理检测功能,无需等待调度程序 +""" + +import requests +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + + +# 代理池服务地址 +PROXY_POOL_URL = "http://192.168.2.127:5010" + +# 匿名性检测 URL +ANONYMOUS_CHECK_URL = "http://httpbin.org/ip" + +# 超时时间 +TIMEOUT = 10 + + +def get_all_proxies(): + """获取所有代理""" + try: + resp = requests.get(f"{PROXY_POOL_URL}/all/", timeout=TIMEOUT) + return resp.json() + except Exception as e: + print(f"获取代理失败: {e}") + return [] + + +def check_anonymous(proxy_info): + """ + 检测代理是否为匿名代理 + + Returns: + (proxy_str, is_anonymous, origin_ip) + """ + proxy_str = proxy_info.get("proxy") + if not proxy_str: + return None, False, None + + proxy_ip = proxy_str.split(':')[0] + + proxies = { + "http": f"http://{proxy_str}", + "https": f"http://{proxy_str}", + } + + try: + resp = requests.get( + ANONYMOUS_CHECK_URL, + proxies=proxies, + timeout=TIMEOUT, + headers={"User-Agent": "Mozilla/5.0 (compatible; AnonymousChecker/1.0)"} + ) + resp.raise_for_status() + + origin = resp.json().get("origin", "") + origin_ips = [ip.strip() for ip in origin.split(',')] + + # 如果 origin 中只有代理 IP,则为匿名代理 + is_anonymous = (len(origin_ips) == 1 and origin_ips[0] == proxy_ip) + + return proxy_str, is_anonymous, origin + + except Exception as e: + return proxy_str, False, f"error: {str(e)[:30]}" + + +def main(): + print("=" * 60) + print("匿名代理检测测试") + print(f"代理池: {PROXY_POOL_URL}") + print("=" * 60) + + # 获取所有代理 + print("\n[1] 获取代理列表...") + proxies = get_all_proxies() + print(f" 共 {len(proxies)} 个代理") + + if not proxies: + print(" 没有代理可检测") + return + + # 并发检测 + print("\n[2] 检测匿名性...") + print("-" * 60) + + anonymous_list = [] + transparent_list = [] + failed_list = [] + + with ThreadPoolExecutor(max_workers=5) as executor: + futures = { + executor.submit(check_anonymous, p): p + for p in proxies + } + + for future in as_completed(futures): + proxy_str, is_anonymous, origin = future.result() + if proxy_str is None: + continue + + if "error" in str(origin): + failed_list.append(proxy_str) + print(f" [FAIL] {proxy_str:<25} {origin}") + elif is_anonymous: + anonymous_list.append(proxy_str) + print(f" [ANON] {proxy_str:<25} -> {origin}") + else: + transparent_list.append(proxy_str) + print(f" [TRAN] {proxy_str:<25} -> {origin}") + + # 结果统计 + print("-" * 60) + print(f"\n[3] 检测结果:") + print(f" 匿名代理: {len(anonymous_list)}") + print(f" 透明代理: {len(transparent_list)}") + print(f" 检测失败: {len(failed_list)}") + + if anonymous_list: + print(f"\n 匿名代理列表:") + for p in anonymous_list: + print(f" {p}") + + print("\n" + "=" * 60) + print("检测完成") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/test/testFullFlow.py b/test/testFullFlow.py new file mode 100644 index 000000000..a4e8d9c64 --- /dev/null +++ b/test/testFullFlow.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +""" +本地完整流程验证脚本 + +模拟整个流程:获取代理 -> 验证可用性 -> 匿名性检测 +""" +import requests +from concurrent.futures import ThreadPoolExecutor, as_completed + + +# 配置 +ANONYMOUS_CHECK_URL = 'http://httpbin.org/ip' +HTTP_CHECK_URL = 'http://httpbin.org/get' +TIMEOUT = 10 + + +def get_all_proxies_from_sources(): + """从所有代理源获取代理""" + proxies = set() + + # 从 freeproxyAdapter 获取 + print(" [1] 从 freeproxyAdapter 获取代理...") + try: + from fetcher.freeproxyAdapter import FreeproxyAdapter + adapter = FreeproxyAdapter() + for proxy in adapter.fetch_all(): + proxies.add(proxy) + except Exception as e: + print(f" 错误: {e}") + + print(f" 总计获取到 {len(proxies)} 个代理") + return list(proxies) + + +def verify_proxy(proxy_str): + """验证代理是否可用""" + proxies = { + "http": f"http://{proxy_str}", + "https": f"http://{proxy_str}", + } + try: + resp = requests.get( + HTTP_CHECK_URL, + proxies=proxies, + timeout=TIMEOUT, + headers={"User-Agent": "Mozilla/5.0"} + ) + resp.raise_for_status() + return proxy_str, True, None + except Exception as e: + return proxy_str, False, str(e)[:30] + + +def check_anonymous(proxy_str): + """检测代理是否匿名""" + proxy_ip = proxy_str.split(':')[0] + proxies = { + "http": f"http://{proxy_str}", + "https": f"http://{proxy_str}", + } + try: + resp = requests.get( + ANONYMOUS_CHECK_URL, + proxies=proxies, + timeout=TIMEOUT, + headers={"User-Agent": "Mozilla/5.0"} + ) + resp.raise_for_status() + + origin = resp.json().get("origin", "") + origin_ips = [ip.strip() for ip in origin.split(',')] + + # 只有代理 IP 返回时为匿名 + is_anonymous = (len(origin_ips) == 1 and origin_ips[0] == proxy_ip) + return proxy_str, is_anonymous, origin + except Exception as e: + return proxy_str, None, str(e)[:30] + + +def main(): + print("=" * 60) + print("本地完整流程验证") + print("=" * 60) + + # 阶段1:获取代理 + print("\n[阶段1] 获取代理") + all_proxies = get_all_proxies_from_sources() + + if not all_proxies: + print("没有获取到代理,退出") + return + + # 只测试前50个代理 + test_proxies = all_proxies[:50] + print(f"\n 将测试前 {len(test_proxies)} 个代理") + + # 阶段2:验证可用性 + print("\n[阶段2] 验证代理可用性") + print("-" * 60) + + valid_proxies = [] + invalid_count = 0 + + with ThreadPoolExecutor(max_workers=10) as executor: + futures = { + executor.submit(verify_proxy, p): p for p in test_proxies + } + for future in as_completed(futures): + proxy, is_valid, error = future.result() + if is_valid: + valid_proxies.append(proxy) + print(f" [OK] {proxy}") + else: + invalid_count += 1 + + print(f"\n 可用: {len(valid_proxies)}, 不可用: {invalid_count}") + + if not valid_proxies: + print("没有可用代理,退出") + return + + # 阶段3:匿名性检测 + print("\n[阶段3] 匿名性检测") + print("-" * 60) + + anonymous_proxies = [] + transparent_proxies = [] + failed_count = 0 + + with ThreadPoolExecutor(max_workers=5) as executor: + futures = { + executor.submit(check_anonymous, p): p for p in valid_proxies + } + for future in as_completed(futures): + proxy, is_anonymous, origin = future.result() + if is_anonymous is None: + failed_count += 1 + print(f" [FAIL] {proxy:<25} {origin}") + elif is_anonymous: + anonymous_proxies.append(proxy) + print(f" [ANON] {proxy:<25} -> {origin}") + else: + transparent_proxies.append(proxy) + print(f" [TRAN] {proxy:<25} -> {origin}") + + # 结果统计 + print("\n" + "=" * 60) + print("验证结果") + print("=" * 60) + print(f" 获取代理总数: {len(all_proxies)}") + print(f" 测试代理数量: {len(test_proxies)}") + print(f" 可用代理数量: {len(valid_proxies)}") + print(f" 匿名代理数量: {len(anonymous_proxies)}") + print(f" 透明代理数量: {len(transparent_proxies)}") + + if anonymous_proxies: + print(f"\n匿名代理列表:") + for p in anonymous_proxies: + print(f" {p}") + + +if __name__ == "__main__": + main() diff --git a/test/testProxyAvailability.py b/test/testProxyAvailability.py new file mode 100644 index 000000000..baeea3e9f --- /dev/null +++ b/test/testProxyAvailability.py @@ -0,0 +1,342 @@ +# -*- coding: utf-8 -*- +""" +代理池服务可用性测试脚本 + +测试部署在 http://192.168.2.127:5010/ 的代理池服务 +验证获取的代理是否可用 +""" + +import requests +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + + +# 代理池服务地址 +PROXY_POOL_URL = "http://192.168.2.127:5010" + +# 用于测试代理的目标网站 +TEST_URLS = [ + "http://httpbin.org/ip", + "http://www.baidu.com", + "https://www.qq.com", +] + +# 请求超时时间(秒) +TIMEOUT = 10 + + +class ProxyPoolTester: + """代理池测试器""" + + def __init__(self, pool_url: str): + self.pool_url = pool_url.rstrip("/") + + def get_service_info(self) -> dict: + """获取代理池服务信息""" + try: + resp = requests.get(f"{self.pool_url}/", timeout=TIMEOUT) + resp.raise_for_status() + return {"success": True, "data": resp.text} + except Exception as e: + return {"success": False, "error": str(e)} + + def get_proxy_count(self) -> dict: + """获取代理池中代理数量""" + try: + resp = requests.get(f"{self.pool_url}/count", timeout=TIMEOUT) + resp.raise_for_status() + return {"success": True, "data": resp.json()} + except Exception as e: + return {"success": False, "error": str(e)} + + def get_random_proxy(self, https: bool = False) -> dict: + """随机获取一个代理""" + try: + url = f"{self.pool_url}/get" + if https: + url += "?type=https" + resp = requests.get(url, timeout=TIMEOUT) + resp.raise_for_status() + return {"success": True, "data": resp.json()} + except Exception as e: + return {"success": False, "error": str(e)} + + def get_all_proxies(self, https: bool = False) -> dict: + """获取所有代理""" + try: + url = f"{self.pool_url}/all" + if https: + url += "?type=https" + resp = requests.get(url, timeout=TIMEOUT) + resp.raise_for_status() + return {"success": True, "data": resp.json()} + except Exception as e: + return {"success": False, "error": str(e)} + + def delete_proxy(self, proxy: str) -> dict: + """删除指定代理""" + try: + resp = requests.get( + f"{self.pool_url}/delete", + params={"proxy": proxy}, + timeout=TIMEOUT + ) + resp.raise_for_status() + return {"success": True, "data": resp.json()} + except Exception as e: + return {"success": False, "error": str(e)} + + +class ProxyValidator: + """代理验证器""" + + @staticmethod + def test_proxy(proxy: str, test_url: str, timeout: int = TIMEOUT) -> dict: + """ + 测试单个代理是否可用 + + Args: + proxy: 代理地址,格式为 host:port + test_url: 测试目标URL + timeout: 超时时间 + + Returns: + 测试结果字典 + """ + proxies = { + "http": f"http://{proxy}", + "https": f"http://{proxy}", + } + + start_time = time.time() + try: + resp = requests.get( + test_url, + proxies=proxies, + timeout=timeout, + headers={"User-Agent": "Mozilla/5.0 (compatible; ProxyTest/1.0)"} + ) + elapsed = time.time() - start_time + + # 尝试获取响应内容 + response_text = None + origin_ip = None + try: + response_text = resp.text + # 如果是 httpbin.org/ip,解析返回的 IP + if "httpbin" in test_url and "/ip" in test_url: + origin_ip = resp.json().get("origin") + except Exception: + pass + + return { + "proxy": proxy, + "test_url": test_url, + "success": True, + "status_code": resp.status_code, + "elapsed": round(elapsed, 3), + "response_text": response_text, + "origin_ip": origin_ip, + } + except requests.exceptions.Timeout: + return { + "proxy": proxy, + "test_url": test_url, + "success": False, + "error": "timeout", + "elapsed": round(time.time() - start_time, 3), + } + except requests.exceptions.ProxyError as e: + return { + "proxy": proxy, + "test_url": test_url, + "success": False, + "error": f"proxy_error: {str(e)[:50]}", + "elapsed": round(time.time() - start_time, 3), + } + except Exception as e: + return { + "proxy": proxy, + "test_url": test_url, + "success": False, + "error": str(e)[:100], + "elapsed": round(time.time() - start_time, 3), + } + + @staticmethod + def batch_test_proxy(proxy: str, test_urls: list, timeout: int = TIMEOUT) -> dict: + """ + 使用多个目标URL测试代理 + + Args: + proxy: 代理地址 + test_urls: 测试目标URL列表 + timeout: 超时时间 + + Returns: + 综合测试结果 + """ + results = [] + success_count = 0 + + for url in test_urls: + result = ProxyValidator.test_proxy(proxy, url, timeout) + results.append(result) + if result["success"]: + success_count += 1 + + return { + "proxy": proxy, + "total_tests": len(test_urls), + "success_count": success_count, + "success_rate": round(success_count / len(test_urls) * 100, 1), + "details": results, + } + + +def print_separator(char: str = "-", length: int = 60): + """打印分隔线""" + print(char * length) + + +def test_proxy_pool_service(): + """测试代理池服务的主函数""" + print("\n" + "=" * 60) + print("代理池服务可用性测试") + print(f"服务地址: {PROXY_POOL_URL}") + print("=" * 60) + + tester = ProxyPoolTester(PROXY_POOL_URL) + validator = ProxyValidator() + + # 1. 测试服务连通性 + print("\n[1] 测试服务连通性...") + service_info = tester.get_service_info() + if service_info["success"]: + print(f" 服务连接成功") + else: + print(f" 服务连接失败: {service_info['error']}") + print("\n请检查服务是否正常运行!") + return + + # 2. 获取代理数量 + print("\n[2] 获取代理池状态...") + count_result = tester.get_proxy_count() + if count_result["success"]: + count_data = count_result["data"] + print(f" 代理池状态: {count_data}") + total_count = count_data.get("count", 0) + if total_count == 0: + print("\n 代理池中没有可用代理,请等待调度程序采集代理后再试!") + return + else: + print(f" 获取失败: {count_result['error']}") + return + + # 3. 随机获取代理测试 + print("\n[3] 随机获取代理...") + proxy_result = tester.get_random_proxy() + if proxy_result["success"]: + proxy_data = proxy_result["data"] + proxy_addr = proxy_data.get("proxy") + print(f" 获取到代理: {proxy_addr}") + print(f" 代理详情: {proxy_data}") + else: + print(f" 获取失败: {proxy_result['error']}") + return + + # 4. 测试代理可用性 + print("\n[4] 测试代理可用性...") + print(f" 代理: {proxy_addr}") + print(f" 测试目标: {TEST_URLS}") + print_separator() + + batch_result = validator.batch_test_proxy(proxy_addr, TEST_URLS) + for detail in batch_result["details"]: + status = "OK" if detail["success"] else "FAIL" + elapsed = detail["elapsed"] + error_info = f" ({detail.get('error', '')})" if not detail["success"] else "" + print(f" [{status}] {detail['test_url']:<40} {elapsed}s{error_info}") + + print_separator() + print(f" 成功率: {batch_result['success_rate']}% " + f"({batch_result['success_count']}/{batch_result['total_tests']})") + + # 5. 批量测试多个代理 + print("\n[5] 批量测试多个代理...") + all_result = tester.get_all_proxies() + if not all_result["success"]: + print(f" 获取代理列表失败: {all_result['error']}") + return + + all_proxies = all_result["data"] + if isinstance(all_proxies, list): + proxy_list = [p.get("proxy") for p in all_proxies if p.get("proxy")] + else: + proxy_list = [] + + # 限制测试数量 + max_test_count = 10 + proxies_to_test = proxy_list[:max_test_count] + print(f" 共有 {len(proxy_list)} 个代理,测试前 {len(proxies_to_test)} 个") + + if not proxies_to_test: + print(" 没有代理可供测试") + return + + # 使用线程池并发测试 + test_url = "http://httpbin.org/ip" + print(f" 测试目标: {test_url}") + print_separator() + + success_proxies = [] + failed_proxies = [] + + with ThreadPoolExecutor(max_workers=5) as executor: + futures = { + executor.submit(validator.test_proxy, proxy, test_url): proxy + for proxy in proxies_to_test + } + + for future in as_completed(futures): + result = future.result() + proxy = result["proxy"] + if result["success"]: + success_proxies.append(proxy) + origin_ip = result.get("origin_ip", "") + origin_info = f" -> origin: {origin_ip}" if origin_ip else "" + print(f" [OK] {proxy:<25} {result['elapsed']}s{origin_info}") + else: + failed_proxies.append(proxy) + error = result.get("error", "unknown") + print(f" [FAIL] {proxy:<25} {result['elapsed']}s ({error[:30]})") + + print_separator() + total = len(proxies_to_test) + success = len(success_proxies) + print(f" 测试结果: 成功 {success}/{total} ({round(success/total*100, 1)}%)") + + # 6. 测试总结 + print("\n" + "=" * 60) + print("测试总结") + print("=" * 60) + print(f" 代理池服务地址: {PROXY_POOL_URL}") + print(f" 代理池总数: {len(proxy_list)}") + print(f" 测试代理数: {total}") + print(f" 可用代理数: {success}") + print(f" 可用率: {round(success/total*100, 1)}%") + + if success_proxies: + print("\n 可用代理列表:") + for p in success_proxies[:5]: + print(f" {p}") + if len(success_proxies) > 5: + print(f" ... 还有 {len(success_proxies) - 5} 个") + + print("\n" + "=" * 60) + print("测试完成") + print("=" * 60 + "\n") + + +if __name__ == "__main__": + test_proxy_pool_service()