Add support for Python 3.8, 3.9 (#61)

thtrunck · web-flow · commit 4f2dd510f42b · 2023-05-16T10:10:44.000+02:00
* update a bunch of libraries

* more cleanup and removing now useless lib

* update plugin.json
diff --git a/Changelog.md b/Changelog.md
@@ -1,5 +1,10 @@
 # Changelog
 
+## Version 2.2.0 - Feature release
+
+ - Add more supported Python versions. This plugin can now use 2.7 (deprecated), 3.6, 3.7, 3.8, 3.9
+ - Fix vnet auto-attach managed identity when let AKS provision the control plane identity
+
 ## Version 2.1.0 - Feature release
 
 - Support GPU out-of-the-box
diff --git a/code-env/python/desc.json b/code-env/python/desc.json
@@ -1,6 +1,7 @@
 {
-  "acceptedPythonInterpreters": ["PYTHON27","PYTHON36","PYTHON37"],
+  "acceptedPythonInterpreters": ["PYTHON27","PYTHON36","PYTHON37","PYTHON38","PYTHON39"],
   "forceConda": false,
   "installCorePackages": true,
-  "installJupyterSupport": false
+  "installJupyterSupport": false,
+  "corePackagesSet": "AUTO"
 }
diff --git a/code-env/python/spec/Pipfile b/code-env/python/spec/Pipfile
diff --git a/code-env/python/spec/Pipfile.lock b/code-env/python/spec/Pipfile.lock
diff --git a/code-env/python/spec/requirements.txt b/code-env/python/spec/requirements.txt
@@ -1,13 +1,9 @@
-azure-common==1.1.27
-azure-identity==1.6.0
-azure-mgmt-authorization==1.0.0
-azure-mgmt-compute==21.0.0
-azure-mgmt-containerregistry==8.0.0
-azure-mgmt-containerservice==16.0.0
-azure-mgmt-core==1.2.2
-azure-mgmt-resource==18.0.0
-azure-mgmt-msi==1.0.0
+azure-identity==1.7.1
+azure-mgmt-authorization==2.0.0
+azure-mgmt-compute==23.1.0
+azure-mgmt-containerservice==16.4.0
+azure-mgmt-resource==20.0.0
+azure-mgmt-msi==6.0.0b1
 ipaddress==1.0.23
-msrest==0.6.21
 pyyaml==5.4.1
 uuid==1.30
diff --git a/code-env/python/spec/requirements_spec.txt b/code-env/python/spec/requirements_spec.txt
diff --git a/parameter-sets/cluster-identity/parameter-set.json b/parameter-sets/cluster-identity/parameter-set.json
@@ -74,7 +74,7 @@
             "label": "Kubelet user identity",
             "type": "STRING",
             "mandatory" : false,
-            "description": "User Assigned Managed Identity for the kubelets. Only available with Azure feature Microsoft.ContainerService/CustomKubeletIdentityPreview. See documentation.",
+            "description": "User Assigned Managed Identity for the kubelets.",
             "visibilityCondition": "model.identityType=='managed-identity' && !model.useAKSManagedKubeletIdentity && !model.inheritDSSIdentity"
         },
         {
diff --git a/plugin.json b/plugin.json
@@ -1,6 +1,6 @@
 {
     "id": "aks-clusters",
-    "version": "2.1.0",
+    "version": "2.2.0",
     "meta": {
         "label": "AKS clusters",
         "description": "Interact with or create Microsoft Azure Kubernetes Service clusters",
diff --git a/python-clusters/attach-aks-cluster/cluster.py b/python-clusters/attach-aks-cluster/cluster.py
@@ -1,11 +1,10 @@
-import os, json, logging, yaml
+import os, logging, yaml
 from dataiku.cluster import Cluster
 
 from azure.mgmt.containerservice import ContainerServiceClient
 from dku_utils.access import _is_none_or_blank
-from dku_utils.cluster import make_overrides, get_subscription_id
+from dku_utils.cluster import make_overrides
 from dku_azure.auth import get_credentials_from_connection_info, get_credentials_from_connection_infoV2
-from dku_azure.utils import run_and_process_cloud_error
 from dku_azure.utils import run_and_process_cloud_error, get_instance_metadata, get_subscription_id
 
 class MyCluster(Cluster):
diff --git a/python-clusters/create-aks-cluster/cluster.py b/python-clusters/create-aks-cluster/cluster.py
@@ -1,4 +1,4 @@
-import os, json, logging, yaml, time, uuid
+import os, logging, yaml, time, uuid
 from dataiku.cluster import Cluster
 
 from azure.mgmt.containerservice import ContainerServiceClient
@@ -7,15 +7,13 @@
 from azure.mgmt.authorization import AuthorizationManagementClient
 from azure.core.pipeline.policies import UserAgentPolicy
 from azure.core.exceptions import ResourceNotFoundError, HttpResponseError
-from msrestazure.azure_exceptions import CloudError
 
-from dku_utils.access import _is_none_or_blank, _has_not_blank_property
-from dku_utils.cluster import make_overrides, get_cluster_from_connection_info
+from dku_utils.access import _is_none_or_blank
+from dku_utils.cluster import make_overrides
 from dku_kube.nvidia_utils import add_gpu_driver_if_needed
 from dku_azure.auth import get_credentials_from_connection_info, get_credentials_from_connection_infoV2
 from dku_azure.clusters import ClusterBuilder
-from dku_azure.utils import run_and_process_cloud_error, get_subnet_id, get_instance_metadata, get_subscription_id
-from dku_azure.auth import AzureIdentityCredentialAdapter
+from dku_azure.utils import run_and_process_cloud_error, get_instance_metadata, get_subscription_id
 
 class MyCluster(Cluster):
     def __init__(self, cluster_id, cluster_name, config, plugin_config):
@@ -84,9 +82,7 @@ def start(self):
             existing = clusters_client.managed_clusters.get(resource_group, self.cluster_name)
             if existing is not None:
                 raise Exception("A cluster with name %s in resource group %s already exists" % (self.cluster_name, resource_group))
-        except CloudError as e:
-            logging.info("Cluster doesn't seem to exist yet")
-        except ResourceNotFoundError as e:
+        except ResourceNotFoundError:
             logging.info("Cluster doesn't seem to exist yet")
 
         cluster_builder = ClusterBuilder(clusters_client)
@@ -153,7 +149,7 @@ def start(self):
                     if not cluster_identity.get("useAKSManagedKubeletIdentity",True):
                         kubelet_mi = cluster_identity["kubeletUserAssignedIdentity"]
                         _,_,mi_subscription_id,_,mi_resource_group,_,_,_,mi_name = kubelet_mi.split("/")
-                        msiclient = ManagedServiceIdentityClient(AzureIdentityCredentialAdapter(credentials), mi_subscription_id)
+                        msiclient = ManagedServiceIdentityClient(credentials, mi_subscription_id)
                         mi = msiclient.user_assigned_identities.get(mi_resource_group, mi_name)
                         cluster_builder.with_kubelet_identity(kubelet_mi, mi.client_id, mi.principal_id)
                         logging.info("Configure kubelet identity with user assigned identity resourceId=\"{}\", clientId=\"{}\", objectId=\"{}\"".format(kubelet_mi, mi.client_id, mi.principal_id))
@@ -184,7 +180,7 @@ def start(self):
                     acr_scope = "/subscriptions/{acr_subscription_id}/resourceGroups/{acr_resource_group}/providers/Microsoft.ContainerRegistry/registries/{acr_name}".format(**locals())
                     try:
                         acr_roles = list(authorization_client.role_definitions.list(acr_scope,"roleName eq 'AcrPull'"))
-                    except ResourceNotFoundError as e:
+                    except ResourceNotFoundError:
                         raise Exception("ACR {} not found. Check it exists and you are Owner of it.".format(acr_scope))
                     if 0 == len(acr_roles):
                         raise Exception("Could not find the AcrPull role on the ACR {}. Check you are Owner of it.".format(acr_scope))
@@ -245,9 +241,9 @@ def start(self):
                 authorization_client = AuthorizationManagementClient(credentials, subscription_id)
                 try:
                     vnet_roles = list(authorization_client.role_definitions.list(vnet_id,"roleName eq 'Contributor'"))
-                except ResourceNotFoundError as e:
+                except ResourceNotFoundError:
                     raise Exception("Vnet {} not found. Check it exists and you are Owner of it.".format(vnet_id))
-                if 0 == len(acr_roles):
+                if 0 == len(vnet_roles):
                     raise Exception("Could not find the Contributor role on the vnet {}. Check you are Owner of it.".format(vnet_id))
                 else:
                     vnet_role_id = vnet_roles[0].id
@@ -424,7 +420,7 @@ def stop(self, data):
                     authorization_client = AuthorizationManagementClient(credentials, acr_attachment["subscription_id"])
                     try:
                         authorization_client.role_assignments.delete_by_id(acr_attachment["role_assignment"]["id"])
-                    except ResourceNotFoundError as e:
+                    except ResourceNotFoundError:
                         logging.warn("It looks that the ACR role assignment doesnt exist. Ignore this step")
         
         # Detach Vnet like ACR
@@ -436,13 +432,13 @@ def stop(self, data):
                 authorization_client = AuthorizationManagementClient(credentials, vnet_attachment["subscription_id"])
                 try:
                     authorization_client.role_assignments.delete_by_id(vnet_attachment["role_assignment"]["id"])
-                except ResourceNotFoundError as e:
+                except ResourceNotFoundError:
                     logging.warn("It looks that the Vnet role assignment doesnt exist. Ignore this step")
 
         def do_delete():
             future = clusters_client.managed_clusters.begin_delete(resource_group, cluster_name)
             return future.result()
-        delete_result = run_and_process_cloud_error(do_delete)
+        run_and_process_cloud_error(do_delete)
 
         # delete returns void, so we poll until the cluster is really gone
         gone = False
@@ -453,6 +449,6 @@ def do_delete():
                 if cluster.provisioning_state.lower() != 'deleting':
                     logging.info("Cluster is not deleting anymore, must be deleted now (state = %s)" % cluster.provisioning_state)
             # other exceptions should not be ignored
-            except ResourceNotFoundError as e:
+            except ResourceNotFoundError:
                 logging.info("Cluster doesn't seem to exist anymore, considering it deleted")
                 gone = True
diff --git a/python-lib/dku_azure/auth.py b/python-lib/dku_azure/auth.py
@@ -1,9 +1,4 @@
 from azure.identity import DefaultAzureCredential, ManagedIdentityCredential, ClientSecretCredential
-from azure.mgmt.msi import ManagedServiceIdentityClient
-from msrest.authentication import BasicTokenAuthentication
-from azure.core.pipeline.policies import BearerTokenCredentialPolicy
-from azure.core.pipeline import PipelineRequest, PipelineContext
-from azure.core.pipeline.transport import HttpRequest
 
 from dku_utils.access import _is_none_or_blank
 
@@ -20,7 +15,6 @@ def get_credentials_from_connection_info(connection_info, connection_info_secret
 
 def get_credentials_from_connection_infoV2(connection_infos):
     infos = connection_infos
-    user_managed_identity = infos.get('userManagedIdentity', None)
     identity_type = infos.get('identityType','default')
     managed_identity_id = None
     if identity_type == 'default':
@@ -40,50 +34,3 @@ def get_credentials_from_connection_infoV2(connection_infos):
         raise Exception("Identity type {} is unknown and cannot be used".format(identity_type))
 
     return credentials, managed_identity_id
-
-
-
-# ------------------------------------
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-# ------------------------------------
-# Adapt credentials from azure-identity to be compatible with SDK that needs msrestazure or azure.common.credentials
-# Need msrest >= 0.6.0
-# See also https://pypi.org/project/azure-identity/
-# See https://github.com/jongio/azidext/blob/master/python/azure_identity_credential_adapter.py
-class AzureIdentityCredentialAdapter(BasicTokenAuthentication):
-    def __init__(self, credential=None, resource_id="https://management.azure.com/.default", **kwargs):
-        """Adapt any azure-identity credential to work with SDK that needs azure.common.credentials or msrestazure.
-        Default resource is ARM (syntax of endpoint v2)
-        :param credential: Any azure-identity credential (DefaultAzureCredential by default)
-        :param str resource_id: The scope to use to get the token (default ARM)
-        """
-        super(AzureIdentityCredentialAdapter, self).__init__(None)
-        if credential is None:
-            credential = DefaultAzureCredential()
-        self._policy = BearerTokenCredentialPolicy(credential, resource_id, **kwargs)
-
-    def _make_request(self):
-        return PipelineRequest(
-            HttpRequest(
-                "AzureIdentityCredentialAdapter",
-                "https://fakeurl"
-            ),
-            PipelineContext(None)
-        )
-
-    def set_token(self):
-        """Ask the azure-core BearerTokenCredentialPolicy policy to get a token.
-        Using the policy gives us for free the caching system of azure-core.
-        We could make this code simpler by using private method, but by definition
-        I can't assure they will be there forever, so mocking a fake call to the policy
-        to extract the token, using 100% public API."""
-        request = self._make_request()
-        self._policy.on_request(request)
-        # Read Authorization, and get the second part after Bearer
-        token = request.http_request.headers["Authorization"].split(" ", 1)[1]
-        self.token = {"access_token": token}
-
-    def signed_session(self, session=None):
-        self.set_token()
-        return super(AzureIdentityCredentialAdapter, self).signed_session(session)
diff --git a/python-lib/dku_azure/clusters.py b/python-lib/dku_azure/clusters.py
@@ -1,9 +1,9 @@
-from dku_azure.utils import get_instance_metadata, get_vm_resource_id, get_host_network, get_subnet_id
+from dku_azure.utils import get_host_network, get_subnet_id
 from azure.mgmt.containerservice.models import ManagedClusterAgentPoolProfile, ManagedClusterAPIServerAccessProfile, ManagedClusterServicePrincipalProfile
-from azure.mgmt.containerservice.models import ContainerServiceNetworkProfile, ManagedCluster, ManagedClusterIdentity
+from azure.mgmt.containerservice.models import ContainerServiceNetworkProfile, ManagedCluster
 from dku_utils.access import _default_if_blank, _merge_objects, _print_as_json
 
-import logging, copy, json
+import logging, json
 
 
 class ClusterBuilder(object):
diff --git a/python-lib/dku_azure/utils.py b/python-lib/dku_azure/utils.py
@@ -1,8 +1,6 @@
 import requests
-import json
 import logging
 
-from msrestazure.azure_exceptions import CloudError
 from azure.mgmt.resource import ResourceManagementClient
 from dku_utils.access import _is_none_or_blank
 
@@ -12,8 +10,6 @@
 def run_and_process_cloud_error(fn):
     try:
         return fn()
-    except CloudError as e:
-        raise Exception('%s : %s' % (str(e), e.response.content))
     except Exception as e:
         raise e
         
@@ -23,7 +19,7 @@ def get_instance_metadata(api_version=INSTANCE_API_VERSION):
     Return VM metadata.
     """
     metadata_svc_endpoint = "{}/metadata/instance?api-version={}".format(AZURE_METADATA_SERVICE, api_version)
-    req = requests.get(metadata_svc_endpoint, headers={"metadata": "true"}, proxies={"http":None, "http":None})
+    req = requests.get(metadata_svc_endpoint, headers={"metadata": "true"}, proxies={"http":None})
     resp = req.json()
     return resp
 
diff --git a/python-lib/dku_kube/busybox_pod.py b/python-lib/dku_kube/busybox_pod.py
@@ -1,4 +1,4 @@
-import os, sys, json, yaml, logging, random, time
+import os, json, yaml, logging, random, time
 from .kubectl_command import run_with_timeout
 
 class BusyboxPod(object):
diff --git a/python-lib/dku_kube/kubeconfig.py b/python-lib/dku_kube/kubeconfig.py
@@ -1,5 +1,5 @@
-import os, sys, json, yaml, logging
-from dku_utils.access import _has_not_blank_property, _is_none_or_blank
+import os, json, yaml, logging
+from dku_utils.access import _has_not_blank_property
 
 def get_first_kube_config(kube_config_path=None):
     if kube_config_path is None:
diff --git a/python-lib/dku_kube/kubectl_command.py b/python-lib/dku_kube/kubectl_command.py
@@ -1,4 +1,4 @@
-import os, sys, json, yaml, logging, subprocess, time
+import subprocess, time
 
 class KubeCommandException(Exception):
     def __init__(self, message, out, err):
diff --git a/python-lib/dku_kube/role.py b/python-lib/dku_kube/role.py
@@ -1,6 +1,6 @@
-import os, sys, json, yaml, subprocess, logging
+import os, subprocess, logging
 from dku_google.gcloud import get_account
-from dku_utils.access import _has_not_blank_property, _is_none_or_blank
+from dku_utils.access import _is_none_or_blank
 
 def create_admin_binding(user_name=None, kube_config_path=None):
     if _is_none_or_blank(user_name):
diff --git a/python-lib/dku_utils/access.py b/python-lib/dku_utils/access.py
@@ -1,5 +1,8 @@
 from six import text_type
-from collections import Mapping, Iterable
+try:
+    from collections.abc import Mapping, Iterable # py3
+except ImportError:
+    from collections import Mapping, Iterable # py2
 import sys
 import json
 
@@ -36,7 +39,7 @@ def _default_if_blank(x, d):
         return x
     
 def _default_if_property_blank(d, k, v):
-    if not k in d:
+    if k not in d:
         return v
     x = d[k]
     return _default_if_blank(x, v)
diff --git a/python-lib/dku_utils/cluster.py b/python-lib/dku_utils/cluster.py
@@ -1,12 +1,10 @@
 import json, logging
-import requests
 import dataiku
 from azure.mgmt.containerservice import ContainerServiceClient
 from dataiku.core.intercom import backend_json_call
 from dku_azure.auth import get_credentials_from_connection_info, get_credentials_from_connection_infoV2
-from dku_azure.utils import get_instance_metadata, get_subscription_id
-from dku_utils.access import _default_if_blank, _default_if_property_blank
-from dku_utils.access import _has_not_blank_property, _is_none_or_blank
+from dku_azure.utils import get_subscription_id
+from dku_utils.access import _is_none_or_blank
 
 
 def make_overrides(config, kube_config, kube_config_path, acr_name=None):
diff --git a/python-runnables/inspect-node-pools/runnable.py b/python-runnables/inspect-node-pools/runnable.py
diff --git a/python-runnables/resize-node-pool/runnable.py b/python-runnables/resize-node-pool/runnable.py
diff --git a/python-runnables/test-network/runnable.py b/python-runnables/test-network/runnable.py

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`{`
`2`		`- "acceptedPythonInterpreters": ["PYTHON27","PYTHON36","PYTHON37"],`
	`2`	`+ "acceptedPythonInterpreters": ["PYTHON27","PYTHON36","PYTHON37","PYTHON38","PYTHON39"],`
`3`	`3`	`"forceConda": false,`
`4`	`4`	`"installCorePackages": true,`
`5`		`- "installJupyterSupport": false`
	`5`	`+ "installJupyterSupport": false,`
	`6`	`+ "corePackagesSet": "AUTO"`
`6`	`7`	`}`
Original file line number	Diff line number	Diff line change
`@@ -74,7 +74,7 @@`
`74`	`74`	`"label": "Kubelet user identity",`
`75`	`75`	`"type": "STRING",`
`76`	`76`	`"mandatory" : false,`
`77`		`- "description": "User Assigned Managed Identity for the kubelets. Only available with Azure feature Microsoft.ContainerService/CustomKubeletIdentityPreview. See documentation.",`
	`77`	`+ "description": "User Assigned Managed Identity for the kubelets.",`
`78`	`78`	`"visibilityCondition": "model.identityType=='managed-identity' && !model.useAKSManagedKubeletIdentity && !model.inheritDSSIdentity"`
`79`	`79`	`},`
`80`	`80`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"id": "aks-clusters",`
`3`		`- "version": "2.1.0",`
	`3`	`+ "version": "2.2.0",`
`4`	`4`	`"meta": {`
`5`	`5`	`"label": "AKS clusters",`
`6`	`6`	`"description": "Interact with or create Microsoft Azure Kubernetes Service clusters",`