[CoE] vCenter Storage Policies - moodule, state, and tests (#333)

* vCenter Storage Policies - moodule, state, and tests

* Add changelog

* Fix 3.7 3.8 python compatibility

* Fix dict merge

* Fix changes from PR review

* Fix vMomi create/update of policy. Fix drift report lib

* Pass present state (#337)

* create ntp config state

* Update src/saltext/vmware/states/esxi.py

Co-authored-by: Wayne Werner <waynejwerner@gmail.com>

* white space

* changes

* password present state

Co-authored-by: Wayne Werner <waynejwerner@gmail.com>

* Fixes for #333 (review)

Co-authored-by: Joey Gibson <56270734+joechainz@users.noreply.github.com>
Co-authored-by: Wayne Werner <waynejwerner@gmail.com>

Author: 3 people

changelog/333.added

@@ -0,0 +1,2 @@
+Added modules storage_policy.find and storage_policy.save.
+Added state storage_policy.config.

docs/ref/modules/all.rst

@@ -31,6 +31,7 @@ Execution Modules
     saltext.vmware.modules.nsxt_transport_zone
     saltext.vmware.modules.nsxt_uplink_profiles
     saltext.vmware.modules.ssl_adapter
+    saltext.vmware.modules.storage_policies
     saltext.vmware.modules.tag
     saltext.vmware.modules.vm
     saltext.vmware.modules.vmc_dhcp_profiles

docs/ref/modules/saltext.vmware.modules.storage_policies.rst

@@ -0,0 +1,6 @@
+
+saltext.vmware.modules.storage_policies
+=======================================
+
+.. automodule:: saltext.vmware.modules.storage_policies
+    :members:

docs/ref/states/all.rst

@@ -25,6 +25,7 @@ State Modules
     saltext.vmware.states.nsxt_transport_node_profiles
     saltext.vmware.states.nsxt_transport_zone
     saltext.vmware.states.nsxt_uplink_profiles
+    saltext.vmware.states.storage_policies
     saltext.vmware.states.tag
     saltext.vmware.states.vm
     saltext.vmware.states.vmc_dhcp_profiles

docs/ref/states/saltext.vmware.states.storage_policies.rst

@@ -0,0 +1,6 @@
+
+saltext.vmware.states.storage_policies
+======================================
+
+.. automodule:: saltext.vmware.states.storage_policies
+    :members:

src/saltext/vmware/modules/storage_policies.py

@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: Apache-2.0
+import json
+import logging
+
+import saltext.vmware.modules.storage_policies as policy_module
+import saltext.vmware.utils.common as utils_common
+import saltext.vmware.utils.connect as connect
+import saltext.vmware.utils.drift as drift
+
+log = logging.getLogger(__name__)
+
+try:
+    from pyVmomi import vim
+
+    HAS_PYVMOMI = True
+except ImportError:
+    HAS_PYVMOMI = False
+
+
+__virtualname__ = "storage_policy"
+__proxyenabled__ = ["storage_policy"]
+
+
+def __virtual__():
+    if not HAS_PYVMOMI:
+        return False, "Unable to import pyVmomi module."
+    return __virtualname__
+
+
+def config(name, config, service_instance=None, profile=None):
+    """
+    Get/Set storage policies configuration based on drift report.
+
+    name
+        Name of configuration. (required).
+
+    config
+        List of objects with configuration values. (required).
+
+    .. code-block:: yaml
+
+        storage_policies_drift:
+            storage_policy.config:
+                - profile: vcenter
+                - config:
+                - policyName: Performance - Thick
+                    constraints:
+                    - name: VSAN
+                        replicaPreference: 1 failures - RAID-1 (Mirroring)
+                        hostFailuresToTolerate: 2
+                        checksumDisabled: true
+                        stripeWidth: 2
+                        proportionalCapacity: 50
+                        cacheReservation: 0
+                - policyName: New VM Storage Policy
+                    constraints:
+                    - name: VSAN
+                        hostFailuresToTolerate: 2
+                        replicaPreference: 2 failures - RAID-1 (Mirroring)
+                        proportionalCapacity: 1
+
+    service_instance
+        Use this vCenter service connection instance instead of creating a new one. (optional).
+
+    profile
+        Profile to use (optional)
+    """
+
+    service_instance = service_instance or connect.get_service_instance(
+        config=__opts__, profile=profile
+    )
+    ret = {"name": name, "changes": {}, "result": None, "comment": ""}
+
+    # Clone config input to a Map.
+    # Can be used to transform input to internal objects and do validation if needed
+    new_config = {}
+    for policy_config in config:
+        # Create full representation of the object, default or empty values
+        new_policy_config = {"constraints": {}}
+        # Transform / Validate input vs object, e.g. constraints section
+        if "constraints" in policy_config:
+            for constraint in policy_config["constraints"]:
+                subProfileName = constraint["name"]
+                new_policy_config["constraints"][subProfileName] = {}
+                for key in constraint.keys():
+                    if key != "name":
+                        new_policy_config["constraints"][subProfileName][key] = constraint[key]
+        new_config[policy_config["policyName"]] = new_policy_config
+
+    log.debug("---------------NEW--------------")
+    log.debug(json.dumps(new_config, indent=2))
+    log.debug("---------------NEW--------------")
+
+    # Get all policies from vCenter, the objects are VMOMI objects
+    old_configs = policy_module.find(
+        policy_name=None, service_instance=service_instance, profile=profile
+    )
+
+    # make JSON representation of current policies
+    # old_configs holds only the rules that are in the scope of interest (provided in argument config_input)
+    old_config = {}
+    for policy in old_configs:
+        if policy["policyName"] not in new_config.keys():
+            continue
+
+        policy_json = {"constraints": {}}
+
+        for constraint in policy["constraints"]:
+            subProfileName = constraint["name"]
+            policy_json["constraints"][subProfileName] = {}
+            for key in constraint.keys():
+                if key != "name":
+                    policy_json["constraints"][subProfileName][key] = constraint[key]
+
+        old_config[policy["policyName"]] = policy_json
+
+    log.debug("--------------OLD---------------")
+    log.debug(json.dumps(old_config, indent=2))
+    log.debug("--------------OLD---------------")
+
+    # Find rules changes
+    changes = []
+    rule_diff = drift.drift_report(old_config, new_config, diff_level=0)
+    policies_diff = json.loads(json.dumps(rule_diff))  # clone object
+    if policies_diff is not None:
+        ret["changes"] = policies_diff
+
+        log.debug("==============DRIFT===============")
+        log.debug(json.dumps(rule_diff, indent=2))
+        log.debug("==============DRIFT===============")
+
+        # add changes for process if not dry-run
+        for policy_name in policies_diff:
+            new_policy = policies_diff[policy_name]["new"]
+            changes.append({**{"name": policy_name}, **new_policy})
+
+    # If it's not dry-run and has changes, then apply changes
+    if not __opts__["test"] and changes:
+        success = True
+
+        log.debug("===============CHANGES==============")
+        log.debug(json.dumps(changes, indent=2))
+        log.debug("===============CHANGES==============")
+
+        comments = {}
+        for change in changes:
+            try:
+                # convert change report to configuration object
+                new_constraints = []
+                for constr_name in change["constraints"].keys():
+                    constr_props = change["constraints"][constr_name].copy()
+                    constr_props["name"] = constr_name
+                    new_constraints.append(constr_props)
+                update = {"policyName": change["name"], "constraints": new_constraints}
+                # save policy
+                policy_module.save(update, service_instance, profile)
+                comments[change["name"]] = {
+                    "status": "SUCCESS",
+                    "message": f"Storage policy '{change['name']}' has been changed successfully.",
+                }
+            except Exception as err:
+                success = False
+                comments[change["name"]] = {
+                    "status": "FAILURE",
+                    "message": f"Error occured while save storage policy '{change['name']}': {err}",
+                }
+
+        ret["comment"] = comments  # it's more readable if passed as object
+        ret["result"] = success  # at least one success
+
+    return ret

src/saltext/vmware/states/storage_policies.py

@@ -0,0 +1,133 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Common functions used across modules
+"""
+import json
+
+
+def drift_report(obj1, obj2, diff_level=None):
+    """
+    Finds the drift between two objects/dictionaries
+
+    obj1
+        First dictionary
+    obj2
+        Second dictionary
+    diff_level
+        Defines on what tree level to show the drift.
+        If not specified the changes are shown in the last/leaf level.
+
+        Drift level None - all leaf nodes changes:
+        o
+        |_
+        | x
+        |__
+        |  |_
+        x  | x
+            |__
+            |  |_
+            x    x
+
+        Drift level 0 - changes in first level of config tree:
+        o
+        |
+        x...
+        |
+        x...
+
+        Drift level 1 - changes in second level of config tree:
+        o
+        |_
+        | x...
+        |_
+            x...
+
+        Drift level 2 - changes in third level of config tree:
+        o
+        |_
+        | |_
+        |_  x
+            |_
+            x
+    """
+    result_diffs = []
+    _drift_recurse_(obj1, obj2, result_diffs)
+
+    result_tree = {}
+    for tup in result_diffs:
+        _drift_tree_(tup, result_tree)
+
+    if diff_level is not None and isinstance(result_tree, dict):
+        result_subtree = {}
+        _drift_subtree_(result_tree, diff_level, result_subtree)
+        return result_subtree
+
+    return result_tree
+
+
+def _drift_tree_(diffs, result_tree):
+    branch = result_tree
+    for i in range(len(diffs)):
+        if i < (len(diffs) - 2):
+            if diffs[i] not in branch:
+                branch[diffs[i]] = {}
+            branch = branch[diffs[i]]
+        elif i == (len(diffs) - 2):
+            branch[diffs[i]] = diffs[i + 1]
+
+
+def _drift_subtree_(result_tree, diff_level, result_subtree, level=0, new_subtree=0):
+    for k in result_tree.keys():
+        if isinstance(result_tree[k], dict):
+            if level == diff_level:
+                result_subtree[k] = {"old": {}, "new": {}}
+                _drift_subtree_(
+                    result_tree[k], diff_level, result_subtree[k]["old"], level + 1, new_subtree=1
+                )
+                _drift_subtree_(
+                    result_tree[k], diff_level, result_subtree[k]["new"], level + 1, new_subtree=2
+                )
+            else:
+                if k not in result_subtree:
+                    result_subtree[k] = {}
+                _drift_subtree_(
+                    result_tree[k], diff_level, result_subtree[k], level + 1, new_subtree
+                )
+        else:
+            if new_subtree == 1:
+                result_subtree[k] = result_tree[k][0]
+            elif new_subtree == 2:
+                result_subtree[k] = result_tree[k][1]
+            else:
+                result_subtree[k] = {"old": result_tree[k][0], "new": result_tree[k][1]}
+
+
+def _drift_recurse_(obj1, obj2, result, keys=[]):
+    if isinstance(obj1, dict) and isinstance(obj2, dict):
+        if not keys:
+            # use symmetric difference for first level only
+            for k in set(obj1).symmetric_difference(obj2):
+                if k in obj1:
+                    # first level element value should be dict
+                    result.append(tuple(keys) + (k,) + ((obj1[k], {}),))
+                else:
+                    # first level element value should be dict
+                    result.append(tuple(keys) + (k,) + (({}, obj2[k]),))
+        else:
+            # otherwise make difference only for added elements in obj2
+            for k in set(obj1).symmetric_difference(obj2):
+                if k in obj2:
+                    # only for complex types - dict, list
+                    if isinstance(obj2[k], dict):
+                        result.append(tuple(keys) + (k,) + (({}, obj2[k]),))
+                    elif isinstance(obj2[k], (list, tuple)):
+                        result.append(tuple(keys) + (k,) + (([], obj2[k]),))
+        for k in set(obj1).intersection(obj2):
+            _drift_recurse_(obj1[k], obj2[k], result, keys=keys + [k])
+    else:
+        if isinstance(obj1, (list, tuple)) and isinstance(obj2, (list, tuple)):
+            if set(obj1) != set(obj2):
+                result.append(tuple(keys) + ((obj1, obj2),))
+        elif obj1 != obj2:
+            result.append(tuple(keys) + ((obj1, obj2),))