Merge pull request #143 from raccoon-mh/master

Add GCP Proxy Feature

Author: raccoon-mh

pkg/pip_requirements.txt

@@ -4,4 +4,5 @@ MarkupSafe>=2.0.0rc2
 google-cloud-storage
 requests
 beautifulsoup4
-grpcio
+grpcio
+PySocks # for proxy support

src/setup.py

@@ -14,7 +14,7 @@
 #   limitations under the License.
 
 
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 with open("VERSION", "r") as f:
     VERSION = f.read().strip()
@@ -39,6 +39,7 @@
         "google-cloud-storage",
         "requests",
         "beautifulsoup4",
+        "PySocks",
     ],
     package_data={
         "spaceone": [

src/spaceone/inventory/connector/compute_engine/vm_instance.py

@@ -1,8 +1,5 @@
 import logging
 import os
-import google.oauth2.service_account
-import googleapiclient
-import googleapiclient.discovery
 
 from spaceone.inventory.libs.connector import GoogleCloudConnector
 
@@ -34,14 +31,6 @@ def get_connect(self, secret_data):
             - ...
         """
         self.project_id = secret_data.get("project_id")
-        credentials = (
-            google.oauth2.service_account.Credentials.from_service_account_info(
-                secret_data
-            )
-        )
-        self.client = googleapiclient.discovery.build(
-            "compute", "v1", credentials=credentials
-        )
 
     def list_regions(self):
         result = self.client.regions().list(project=self.project_id).execute()
@@ -342,7 +331,7 @@ def get_instance_in_group(self, key, value, instance_group, **query):
         )
         # NoneType error occurs sometimes. To prevent them insert default value.
         if response is None:
-            _LOGGER.debug(f"[get_instance_in_group] response is None")
+            _LOGGER.debug("[get_instance_in_group] response is None")
             response = {"items": []}
         else:
             _LOGGER.debug(f"[get_instance_in_group] response => {response}")

src/spaceone/inventory/libs/connector.py

@@ -1,7 +1,12 @@
+import logging
+import os
+
 import google.oauth2.service_account
 import googleapiclient
 import googleapiclient.discovery
-import logging
+import httplib2
+import socks
+from google_auth_httplib2 import AuthorizedHttp
 
 from spaceone.core.connector import BaseConnector
 
@@ -35,9 +40,26 @@ def __init__(self, *args, **kwargs):
                 secret_data
             )
         )
-        self.client = googleapiclient.discovery.build(
-            self.google_client_service, self.version, credentials=self.credentials
-        )
+        proxy_http = self._create_http_client()
+        if proxy_http:
+            self.client = googleapiclient.discovery.build(
+                self.google_client_service,
+                self.version,
+                http=AuthorizedHttp(
+                    self.credentials.with_scopes(
+                        [
+                            "https://www.googleapis.com/auth/cloud-platform"
+                        ]  # FOR PROXY SCOPE SUPPORT
+                    ),
+                    http=proxy_http,
+                ),
+            )
+        else:
+            self.client = googleapiclient.discovery.build(
+                self.google_client_service,
+                self.version,
+                credentials=self.credentials,
+            )
 
     def verify(self, **kwargs):
         if self.client is None:
@@ -55,3 +77,33 @@ def list_zones(self, **query):
         query = self.generate_query(**query)
         result = self.client.zones().list(**query).execute()
         return result.get("items", [])
+
+    def _create_http_client(self):
+        https_proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy")
+
+        if https_proxy:
+            # _LOGGER.info(
+            #     f"** Using proxy in environment variable HTTPS_PROXY/https_proxy: {https_proxy}"
+            # ) # TOO MANY LOGGING
+            try:
+                proxy_url = https_proxy.replace("http://", "").replace("https://", "")
+                if ":" in proxy_url:
+                    proxy_host, proxy_port = proxy_url.split(":", 1)
+                    proxy_port = int(proxy_port)
+
+                proxy_info = httplib2.ProxyInfo(
+                    proxy_host=proxy_host,
+                    proxy_port=proxy_port,
+                    proxy_type=socks.PROXY_TYPE_HTTP,
+                )
+
+                return httplib2.Http(
+                    proxy_info=proxy_info, disable_ssl_certificate_validation=True
+                )
+            except Exception as e:
+                _LOGGER.warning(
+                    f"Failed to configure proxy. Using direct connection.: {e}. "
+                )
+                return None
+        else:
+            return None

src/spaceone/inventory/service/collector_service.py

@@ -1,17 +1,18 @@
+import concurrent.futures
+import json
+import logging
 import os
 import time
-import logging
-import json
-import concurrent.futures
-from spaceone.inventory.connector.resource_manager.project import ProjectConnector
-from spaceone.inventory.libs.manager import GoogleCloudManager
+
 from spaceone.core import utils
 from spaceone.core.service import *
+from spaceone.inventory.conf.cloud_service_conf import *
+from spaceone.inventory.connector.resource_manager.project import ProjectConnector
+from spaceone.inventory.libs.manager import GoogleCloudManager
 from spaceone.inventory.libs.schema.cloud_service import (
-    ErrorResourceResponse,
     CloudServiceResponse,
+    ErrorResourceResponse,
 )
-from spaceone.inventory.conf.cloud_service_conf import *
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -97,7 +98,12 @@ def collect(self, params):
 
         start_time = time.time()
 
-        _LOGGER.debug(f"EXECUTOR START: Google Cloud Service")
+        _LOGGER.debug("EXECUTOR START: Google Cloud Service")
+        proxy_env = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy")
+        if proxy_env:
+            _LOGGER.debug(
+                f"** Using proxy in environment variable HTTPS_PROXY/https_proxy: {proxy_env}"
+            )  # src/spaceone/inventory/libs/connector.py _create_http_client
         # Get target manager to collect
         try:
             self.execute_managers = self._get_target_execute_manager(
@@ -214,7 +220,6 @@ def make_namespace_or_metric_response(
         namespace=None,
         resource_type: str = "inventory.Metric",
     ) -> dict:
-
         response = {
             "state": "SUCCESS",
             "resource_type": resource_type,