修复默认ua的bug，以及浏览器渲染模式下，ua及代理优先级的问题

Author: Boris-code

feapder/network/downloader/_playwright.py

@@ -28,8 +28,22 @@ def _webdriver_pool(self):
         return self.__class__.webdriver_pool
 
     def download(self, request) -> Response:
-        proxy = request.get_proxy()
-        user_agent = request.get_user_agent()
+        # 代理优先级 自定义 > 配置文件 > 随机
+        if request.custom_proxies:
+            proxy = request.get_proxy()
+        elif setting.PLAYWRIGHT.get("proxy"):
+            proxy = setting.PLAYWRIGHT.get("proxy")
+        else:
+            proxy = request.get_proxy()
+
+        # user_agent优先级 自定义 > 配置文件 > 随机
+        if request.custom_ua:
+            user_agent = request.get_user_agent()
+        elif setting.PLAYWRIGHT.get("user_agent"):
+            user_agent = setting.PLAYWRIGHT.get("user_agent")
+        else:
+            user_agent = request.get_user_agent()
+
         cookies = request.get_cookies()
         url = request.url
         render_time = request.render_time or setting.PLAYWRIGHT.get("render_time")

feapder/network/downloader/_selenium.py

@@ -28,8 +28,22 @@ def _webdriver_pool(self):
         return self.__class__.webdriver_pool
 
     def download(self, request) -> Response:
-        proxy = request.get_proxy()
-        user_agent = request.get_user_agent()
+        # 代理优先级 自定义 > 配置文件 > 随机
+        if request.custom_proxies:
+            proxy = request.get_proxy()
+        elif setting.WEBDRIVER.get("proxy"):
+            proxy = setting.WEBDRIVER.get("proxy")
+        else:
+            proxy = request.get_proxy()
+
+        # user_agent优先级 自定义 > 配置文件 > 随机
+        if request.custom_ua:
+            user_agent = request.get_user_agent()
+        elif setting.WEBDRIVER.get("user_agent"):
+            user_agent = setting.WEBDRIVER.get("user_agent")
+        else:
+            user_agent = request.get_user_agent()
+
         cookies = request.get_cookies()
         url = request.url
         render_time = request.render_time or setting.WEBDRIVER.get("render_time")

feapder/network/request.py

@@ -60,7 +60,7 @@ class Request:
         "json",
     }
 
-    DEFAULT_KEY_VALUE = dict(
+    _DEFAULT_KEY_VALUE_ = dict(
         url="",
         method=None,
         retry_times=0,
@@ -79,6 +79,12 @@ class Request:
         make_absolute_links=None,
     )
 
+    _CUSTOM_PROPERTIES_ = {
+        "requests_kwargs",
+        "custom_ua",
+        "custom_proxies",
+    }
+
     def __init__(
         self,
         url="",
@@ -160,13 +166,17 @@ def __init__(
             else setting.MAKE_ABSOLUTE_LINKS
         )
 
+        # 自定义属性，不参与序列化
         self.requests_kwargs = {}
         for key, value in kwargs.items():
             if key in self.__class__.__REQUEST_ATTRS__:  # 取requests参数
                 self.requests_kwargs[key] = value
 
             self.__dict__[key] = value
 
+        self.custom_ua = False
+        self.custom_proxies = False
+
     def __repr__(self):
         try:
             return "<Request {}>".format(self.url)
@@ -246,9 +256,9 @@ def to_dict(self):
 
         for key, value in self.__dict__.items():
             if (
-                key in self.__class__.DEFAULT_KEY_VALUE
-                and self.__class__.DEFAULT_KEY_VALUE.get(key) == value
-                or key == "requests_kwargs"
+                key in self.__class__._DEFAULT_KEY_VALUE_
+                and self.__class__._DEFAULT_KEY_VALUE_.get(key) == value
+                or key in self.__class__._CUSTOM_PROPERTIES_
             ):
                 continue
 
@@ -301,23 +311,21 @@ def make_requests_kwargs(self):
                 method = "GET"
         self.method = method
 
-        # 随机user—agent
+        # 设置user—agent
         headers = self.requests_kwargs.get("headers", {})
         if "user-agent" not in headers and "User-Agent" not in headers:
-            if self.render:  # 如果是渲染默认，优先使用WEBDRIVER中配置的ua
-                ua = setting.WEBDRIVER.get(
-                    "user_agent"
-                ) or self.__class__.user_agent_pool.get(setting.USER_AGENT_TYPE)
-            else:
-                ua = self.__class__.user_agent_pool.get(setting.USER_AGENT_TYPE)
-
             if self.random_user_agent and setting.RANDOM_HEADERS:
+                # 随机user—agent
+                ua = self.__class__.user_agent_pool.get(setting.USER_AGENT_TYPE)
                 headers.update({"User-Agent": ua})
                 self.requests_kwargs.update(headers=headers)
+            else:
+                # 使用默认的user—agent
+                self.requests_kwargs.setdefault(
+                    "headers", {"User-Agent": setting.DEFAULT_USERAGENT}
+                )
         else:
-            self.requests_kwargs.setdefault(
-                "headers", {"User-Agent": setting.DEFAULT_USERAGENT}
-            )
+            self.custom_ua = True
 
         # 代理
         proxies = self.requests_kwargs.get("proxies", -1)
@@ -329,6 +337,8 @@ def make_requests_kwargs(self):
                     break
                 else:
                     log.debug("暂无可用代理 ...")
+        else:
+            self.custom_proxies = True
 
     def get_response(self, save_cached=False):
         """