diff --git a/.ddev/config.toml b/.ddev/config.toml index 292b02c7825f0..c3583e24d1044 100644 --- a/.ddev/config.toml +++ b/.ddev/config.toml @@ -46,6 +46,7 @@ n8n = "n8n" hpe_aruba_edgeconnect = "HPE Aruba EdgeConnect" control_m = "Control-M" nifi = "Apache NiFi" +dell_powerflex = "Dell Powerflex" [overrides.metrics-prefix] krakend = "krakend.api." @@ -54,6 +55,7 @@ prefect = "prefect.server." n8n = "n8n." control_m = "control_m." nifi = "nifi." +dell_powerflex = "dell_powerflex." hpe_aruba_edgeconnect = "hpe_aruba_edgeconnect." [overrides.ci.ddev] @@ -204,7 +206,6 @@ kube_scheduler = "kube_scheduler" nifi = "nifi" nginx_ingress_controller = "nginx-ingress-controller" - [overrides.dep.updates] exclude = [ 'pyasn1', # https://github.com/pyasn1/pyasn1/issues/52 @@ -273,5 +274,6 @@ prefect = ["linux", "windows", "mac_os"] n8n = ["linux", "windows", "mac_os"] control_m = ["linux", "windows", "mac_os"] nifi = ["linux", "windows", "mac_os"] +dell_powerflex = ["linux", "windows", "mac_os"] hpe_aruba_edgeconnect = ["linux", "windows", "mac_os"] diff --git a/.github/chainguard/self.resolve-build-deps.push.sts.yaml b/.github/chainguard/self.resolve-build-deps.push.sts.yaml index e32d40011bf2d..347df46283f5a 100644 --- a/.github/chainguard/self.resolve-build-deps.push.sts.yaml +++ b/.github/chainguard/self.resolve-build-deps.push.sts.yaml @@ -1,20 +1,18 @@ # Policy for: .github/workflows/resolve-build-deps.yaml publish job in DataDog/integrations-core -# Triggered by push to master or release branches, or workflow_dispatch +# Triggered by push to feature branches that update dependencies # # Naming convention: # self: Only this repository (DataDog/integrations-core) can use this policy # resolve-build-deps: Specific workflow -# push: Primary trigger is push to protected branches +# push: Primary trigger is push to feature branches # # Security model: -# - Publish job runs on push or workflow_dispatch to protected branches (master and X.Y.x) +# - Publish job runs on push to any feature branch # - Workflow file must be committed to the same branch -# - Pull request events are excluded by the job's if condition +# - Access is scoped to this specific workflow file via job_workflow_ref # # Permissions granted: -# - contents: write - Push commits to branches -# - pull_requests: write - Create pull requests -# - workflows: write - Modify workflow files in generated commits +# - contents: write - Push commits (lockfiles) back to the feature branch # # Usage in workflows: # - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4 @@ -24,15 +22,12 @@ issuer: https://token.actions.githubusercontent.com -subject_pattern: repo:DataDog/integrations-core:ref:refs/heads/(master|\d+\.\d+\..*) +subject_pattern: repo:DataDog/integrations-core:ref:refs/heads/.* claim_pattern: - event_name: (push|workflow_dispatch) - job_workflow_ref: DataDog/integrations-core/\.github/workflows/resolve-build-deps\.yaml@refs/heads/(master|\d+\.\d+\..*) - ref: refs/heads/(master|\d+\.\d+\..*) + event_name: push + job_workflow_ref: DataDog/integrations-core/\.github/workflows/resolve-build-deps\.yaml@refs/heads/.* repository: DataDog/integrations-core permissions: contents: write - pull_requests: write - workflows: write diff --git a/.github/workflows/config/labeler.yml b/.github/workflows/config/labeler.yml index 127baf99c71be..1d44965d920cb 100644 --- a/.github/workflows/config/labeler.yml +++ b/.github/workflows/config/labeler.yml @@ -445,6 +445,10 @@ integration/delinea_secret_server: - changed-files: - any-glob-to-any-file: - delinea_secret_server/**/* +integration/dell_powerflex: +- changed-files: + - any-glob-to-any-file: + - dell_powerflex/**/* integration/directory: - changed-files: - any-glob-to-any-file: diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 5c9632f42192f..a0aedac7373c7 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -978,6 +978,26 @@ jobs: minimum-base-package: ${{ inputs.minimum-base-package }} pytest-args: ${{ inputs.pytest-args }} secrets: inherit + jc346754: + uses: ./.github/workflows/test-target.yml + with: + job-name: Dell Powerflex + target: dell_powerflex + platform: linux + runner: '["ubuntu-22.04"]' + repo: "${{ inputs.repo }}" + context: ${{ inputs.context }} + python-version: "${{ inputs.python-version }}" + latest: ${{ inputs.latest }} + agent-image: "${{ inputs.agent-image }}" + agent-image-py2: "${{ inputs.agent-image-py2 }}" + agent-image-windows: "${{ inputs.agent-image-windows }}" + agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}" + test-py2: ${{ inputs.test-py2 }} + test-py3: ${{ inputs.test-py3 }} + minimum-base-package: ${{ inputs.minimum-base-package }} + pytest-args: ${{ inputs.pytest-args }} + secrets: inherit jc8f84c3: uses: ./.github/workflows/test-target.yml with: diff --git a/code-coverage.datadog.yml b/code-coverage.datadog.yml index d670091a3135f..0273ce99805c4 100644 --- a/code-coverage.datadog.yml +++ b/code-coverage.datadog.yml @@ -226,6 +226,9 @@ services: - id: ddev paths: - ddev/src/ddev/ +- id: dell_powerflex + paths: + - dell_powerflex/datadog_checks/dell_powerflex/ - id: directory paths: - directory/datadog_checks/directory/ diff --git a/datadog_checks_base/changelog.d/23905.added b/datadog_checks_base/changelog.d/23905.added new file mode 100644 index 0000000000000..88807f95285bb --- /dev/null +++ b/datadog_checks_base/changelog.d/23905.added @@ -0,0 +1 @@ +Add ``AgentCheck.submit_generic_resource`` to submit resource snapshots on the ``genresources`` event-platform track with allow-list field selection. diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 6da57bc3b11a0..30ba1240db145 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -798,18 +798,170 @@ def database_monitoring_metadata(self, raw_event): aggregator.submit_event_platform_event(self, self.check_id, to_native_string(raw_event), "dbm-metadata") def event_platform_event(self, raw_event, event_track_type): - # type: (str, str) -> None + # type: (str | bytes, str) -> None """Send an event platform event. Parameters: - raw_event (str): - JSON formatted string representing the event to send + raw_event (str | bytes): + JSON formatted string representing the event to send, or + pre-encoded bytes for proto tracks such as ``genresources`` event_track_type (str): type of event ingested and processed by the event platform """ if raw_event is None: return - aggregator.submit_event_platform_event(self, self.check_id, to_native_string(raw_event), event_track_type) + if isinstance(raw_event, (bytearray, memoryview)): + raw_event = bytes(raw_event) + elif not isinstance(raw_event, bytes): + raw_event = to_native_string(raw_event) + aggregator.submit_event_platform_event(self, self.check_id, raw_event, event_track_type) + + def submit_generic_resource(self, *, type, key, fields, include, seen_at=None, expire_at=None): + # type: (str, str, dict | None, dict, int | None, int | None) -> None + """Ship a resource on the ``genresources`` event-platform track. + + ``fields`` is the resource body. ``include`` chooses what to keep from it: + ``{"paths": [...], "map_paths": [...], "annotation_keys": [...]}``. Evaluated against ``fields``, + ``paths`` select individual values, ``map_paths`` select whole flat maps (e.g. + ``metadata.labels``), and ``annotation_keys`` glob ``metadata.annotations`` keys. A path that + resolves to a structured object is dropped. Pass ``include=INCLUDE_ALL`` to ship ``fields`` + as-is — only safe when your code constructed every value, never for a raw upstream object. + ``seen_at`` / ``expire_at`` are optional ``int`` unix-seconds. + """ + if fields is None: + return + + # stdlib json on purpose: module-level json is the orjson wrapper, which coerces datetime instead of failing. + import json as _json + + # Lazy import: avoids loading the protobuf runtime for every check that imports base.py. + from datadog_checks.base.utils.genresources import ( + GENRESOURCES_TRACK, + INCLUDE_ALL, + INTEGRATIONS_CORE_SOURCE, + MAX_FIELDS_JSON_BYTES, + GenericResource, + GenericResourceEvent, + apply_allow_list, + find_invalid_include, + ) + + integration = self.name + + def _emit_dropped(count=1): + datadog_agent.emit_agent_telemetry(integration, "datadog.agent.check.genresources.dropped", count, "count") + + if not key: + self.log.warning("genresources: dropping resource with empty key for type=%s", type) + _emit_dropped() + return + + if not type: + self.log.warning("genresources: dropping resource with empty type for key=%s", key) + _emit_dropped() + return + + if not isinstance(fields, dict): + self.log.warning( + "genresources: dropping resource with non-dict fields type=%s key=%s actual_type=%s", + type, + key, + fields.__class__.__name__, + ) + _emit_dropped() + return + + if include is INCLUDE_ALL: + # Caller built `fields` in code and owns its contents; ship as-is, no allow-list. + included = fields + else: + if not isinstance(include, dict): + self.log.warning( + "genresources: dropping resource with non-dict include type=%s key=%s actual_type=%s", + type, + key, + include.__class__.__name__, + ) + _emit_dropped() + return + + paths = include.get("paths", []) + map_paths = include.get("map_paths", []) + annotation_keys = include.get("annotation_keys", []) + + def _is_str_list(value): + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + if not (_is_str_list(paths) and _is_str_list(map_paths) and _is_str_list(annotation_keys)): + self.log.warning("genresources: dropping resource with malformed include type=%s key=%s", type, key) + _emit_dropped() + return + + if any(not pattern.strip("*?") for pattern in annotation_keys): + self.log.warning( + "genresources: dropping resource with catch-all annotation pattern type=%s key=%s", type, key + ) + _emit_dropped() + return + + invalid = find_invalid_include(fields, paths, map_paths) + if invalid is not None: + offending_path, reason = invalid + self.log.warning( + "genresources: dropping resource (%s) path=%s type=%s key=%s", reason, offending_path, type, key + ) + _emit_dropped() + return + + included = apply_allow_list(fields, paths=paths, map_paths=map_paths, annotation_keys=annotation_keys) + + if not included: + self.log.warning("genresources: dropping resource with empty inclusion type=%s key=%s", type, key) + _emit_dropped() + return + + try: + fields_json = _json.dumps(included, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") + except (TypeError, ValueError): + self.log.exception("genresources: failed to encode fields for type=%s key=%s", type, key) + _emit_dropped() + return + + if len(fields_json) > MAX_FIELDS_JSON_BYTES: + self.log.warning( + "genresources: dropping oversize resource type=%s key=%s size=%d", + type, + key, + len(fields_json), + ) + _emit_dropped() + return + + resource = GenericResource(type=type, key=key, fields_json=fields_json) + + def _set_seconds(ts, value, label): + if value is None: + return + if isinstance(value, int) and not isinstance(value, bool): + ts.seconds = value + else: + self.log.warning( + "genresources: ignoring non-int %s for type=%s key=%s value=%r", label, type, key, value + ) + + _set_seconds(resource.seen_at, seen_at, "seen_at") + _set_seconds(resource.expire_at, expire_at, "expire_at") + + event = GenericResourceEvent(source=INTEGRATIONS_CORE_SOURCE, resource=resource) + try: + payload = event.SerializeToString() + except Exception: + self.log.exception("genresources: failed to serialize type=%s key=%s", type, key) + _emit_dropped() + return + + self.event_platform_event(payload, GENRESOURCES_TRACK) + datadog_agent.emit_agent_telemetry(integration, "datadog.agent.check.genresources.emitted", 1, "count") def should_send_metric(self, metric_name): return not self._metric_excluded(metric_name) and self._metric_included(metric_name) diff --git a/datadog_checks_base/datadog_checks/base/utils/genresources/__init__.py b/datadog_checks_base/datadog_checks/base/utils/genresources/__init__.py new file mode 100644 index 0000000000000..187ccb6636f80 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/genresources/__init__.py @@ -0,0 +1,21 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from .inclusion import INCLUDE_ALL, apply_allow_list, find_invalid_include +from .proto.genericresource_pb2 import GenericResource, GenericResourceEvent + +GENRESOURCES_TRACK = "genresources" +INTEGRATIONS_CORE_SOURCE = "integrations-core" +MAX_FIELDS_JSON_BYTES = 1_000_000 + +__all__ = [ + "GENRESOURCES_TRACK", + "INCLUDE_ALL", + "INTEGRATIONS_CORE_SOURCE", + "MAX_FIELDS_JSON_BYTES", + "GenericResource", + "GenericResourceEvent", + "apply_allow_list", + "find_invalid_include", +] diff --git a/datadog_checks_base/datadog_checks/base/utils/genresources/inclusion.py b/datadog_checks_base/datadog_checks/base/utils/genresources/inclusion.py new file mode 100644 index 0000000000000..e2cae1a5ab870 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/genresources/inclusion.py @@ -0,0 +1,208 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +"""Allow-list field inclusion for genresources event payloads.""" + +from __future__ import annotations + +import copy +import fnmatch +from collections.abc import Iterator, Mapping, Sequence + + +class _IncludeAll: + """Sentinel for ``submit_generic_resource(include=INCLUDE_ALL)``: ship the whole dict as-is.""" + + __slots__ = () + + def __repr__(self) -> str: + return "INCLUDE_ALL" + + +#: Pass as ``include`` to ship a caller-constructed dict without an allow-list. Only safe when the +#: integration built every value itself; never use it on a raw upstream object, or it re-opens the leak. +INCLUDE_ALL = _IncludeAll() + + +def apply_allow_list( + fields: Mapping[str, object], + *, + paths: Sequence[str], + map_paths: Sequence[str], + annotation_keys: Sequence[str], +) -> dict: + """Return a new dict with only the allow-listed parts of ``fields``. + + ``paths`` select plain values (dotted segments; ``[*]`` matches every array element). + ``map_paths`` select whole flat maps wholesale (e.g. ``metadata.labels``). ``annotation_keys`` + apply fnmatch globs to ``metadata.annotations`` keys; annotations never come in through + ``paths``/``map_paths``. The input is never mutated. + """ + src = copy.deepcopy(dict(fields)) + result: dict = {} + for path in (*paths, *map_paths): + segments = path.split(".") + if _targets_annotations(segments): + continue + _carve(src, result, segments) + + metadata = result.get("metadata") + if isinstance(metadata, dict): + metadata.pop("annotations", None) + _carve_annotations(src, result, annotation_keys) + _prune_empty(result) + return result + + +def find_invalid_include( + fields: Mapping[str, object], + paths: Sequence[str], + map_paths: Sequence[str], +) -> tuple[str, str] | None: + """Return ``(path, reason)`` for the first invalid include, else None. + + ``paths`` must resolve to values or lists of values; ``map_paths`` must resolve to flat maps. + """ + for path in paths: + segments = path.split(".") + if _targets_annotations(segments): + continue + for value in _resolve(fields, segments): + if not _is_plain_value(value): + return path, "nested include value" + + for path in map_paths: + segments = path.split(".") + if _targets_annotations(segments): + continue + for value in _resolve(fields, segments): + if not _is_flat_map(value): + return path, "non-flat map_path" + + return None + + +def _resolve(node: object, segments: list[str]) -> Iterator[object]: + """Yield each value the path resolves to (one per ``[*]`` element); missing paths yield nothing.""" + if not segments: + yield node + return + head, *rest = segments + star = head.endswith("[*]") + key = head[:-3] if star else head + if not isinstance(node, Mapping) or key not in node: + return + value = node[key] + if star: + if not isinstance(value, list): + return + if not rest: + yield value + else: + for item in value: + yield from _resolve(item, rest) + elif rest: + yield from _resolve(value, rest) + else: + yield value + + +def _is_plain_value(value: object) -> bool: + """True for a value (str/int/float/bool), ``None``, or a list of those; a map is never a plain value.""" + if isinstance(value, Mapping): + return False + if isinstance(value, list): + return all(not isinstance(item, (Mapping, list)) for item in value) + return True + + +def _is_flat_map(value: object) -> bool: + """True for a map whose values are all plain (no nested objects or lists).""" + return isinstance(value, Mapping) and all(not isinstance(v, (Mapping, list)) for v in value.values()) + + +def _targets_annotations(segments: list[str]) -> bool: + """True if the path touches an ``annotations`` map at any depth.""" + return any(segment.removesuffix("[*]") == "annotations" for segment in segments) + + +def _carve(src_node: object, dst_node: dict, segments: list[str]) -> bool: + """Copy the value at ``segments`` into ``dst_node``; return True iff a value was copied. + + A container is attached only once a descendant copies data, so a missing path leaves no shell. + Existing containers are reused, so paths sharing a prefix merge (per element for ``[*]``). + """ + head, *rest = segments + star = head.endswith("[*]") + key = head[:-3] if star else head + + if not isinstance(src_node, Mapping) or key not in src_node: + return False + src_value = src_node[key] + + if not star: + if not rest: + dst_node[key] = copy.deepcopy(src_value) + return True + if not isinstance(src_value, Mapping): + return False + child = dst_node.get(key) + created = not isinstance(child, dict) + if created: + child = {} + copied = _carve(src_value, child, rest) + if copied and created: + dst_node[key] = child + return copied + + if not isinstance(src_value, list): + return False + dst_list = dst_node.get(key) + created = not isinstance(dst_list, list) + if created: + dst_list = [] + any_copied = False + for index, item in enumerate(src_value): + while len(dst_list) <= index: + dst_list.append({}) + if not rest: + dst_list[index] = copy.deepcopy(item) + any_copied = True + elif isinstance(item, Mapping) and isinstance(dst_list[index], dict): + if _carve(item, dst_list[index], rest): + any_copied = True + if any_copied and created: + dst_node[key] = dst_list + return any_copied + + +def _carve_annotations(src: Mapping[str, object], dst: dict, annotation_keys: Sequence[str]) -> None: + """Add the allow-listed ``metadata.annotations`` (plain values only) to ``dst``.""" + metadata = src.get("metadata") + annotations = metadata.get("annotations") if isinstance(metadata, Mapping) else None + if not isinstance(annotations, Mapping): + return + kept = {} + for pattern in annotation_keys: + for key, value in annotations.items(): + if fnmatch.fnmatchcase(key, pattern) and not isinstance(value, (Mapping, list)): + kept[key] = copy.deepcopy(value) + if kept: + dst.setdefault("metadata", {}) + if isinstance(dst["metadata"], dict): + dst["metadata"]["annotations"] = kept + + +def _prune_empty(node: object) -> None: + """Remove empty dicts and empty lists in-place, bottom-up; values and ``None`` are kept.""" + if isinstance(node, dict): + for key in list(node): + value = node[key] + _prune_empty(value) + if value == {} or value == []: + del node[key] + elif isinstance(node, list): + for item in node: + _prune_empty(item) + node[:] = [item for item in node if item != {} and item != []] diff --git a/datadog_checks_base/datadog_checks/base/utils/genresources/proto/__init__.py b/datadog_checks_base/datadog_checks/base/utils/genresources/proto/__init__.py new file mode 100644 index 0000000000000..9f7c793aaa660 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/genresources/proto/__init__.py @@ -0,0 +1,7 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +# +# Regenerate genericresource_pb2.py with: +# cd datadog_checks_base/datadog_checks/base/utils/genresources/proto +# protoc -I . --python_out=. genericresource.proto diff --git a/datadog_checks_base/datadog_checks/base/utils/genresources/proto/genericresource.proto b/datadog_checks_base/datadog_checks/base/utils/genresources/proto/genericresource.proto new file mode 100644 index 0000000000000..78a4a53a6247a --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/genresources/proto/genericresource.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +// Wire envelope for the `genresources` event-platform track. +// Field numbers are preserved from the upstream schema so payloads remain +// wire-compatible with the downstream resource intake. +// Do not add fields without coordinating with the resource intake owners. + +package datadog.genresources.v1; + +option go_package = "github.com/DataDog/datadog-agent/pkg/proto/pbgo/genresources"; + +import "google/protobuf/timestamp.proto"; + +message GenericResource { + reserved 1, 3, 4, 5, 8, 9, 10, 11, 14; + string type = 2; + google.protobuf.Timestamp seen_at = 6; + google.protobuf.Timestamp expire_at = 7; + bytes fields_json = 12; + string key = 13; +} + +message GenericResourceEvent { + string source = 1; + reserved 2; + GenericResource resource = 3; +} diff --git a/datadog_checks_base/datadog_checks/base/utils/genresources/proto/genericresource_pb2.py b/datadog_checks_base/datadog_checks/base/utils/genresources/proto/genericresource_pb2.py new file mode 100644 index 0000000000000..eaa67905390d5 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/genresources/proto/genericresource_pb2.py @@ -0,0 +1,38 @@ +# ruff: noqa +# pylint: skip-file +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: genericresource.proto +# Protobuf Python Version: 7.34.0 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion(_runtime_version.Domain.PUBLIC, 7, 34, 0, '', 'genericresource.proto') +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x15genericresource.proto\x12\x17\x64\x61tadog.genresources.v1\x1a\x1fgoogle/protobuf/timestamp.proto"\xd3\x01\n\x0fGenericResource\x12\x0c\n\x04type\x18\x02 \x01(\t\x12+\n\x07seen_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12-\n\texpire_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x13\n\x0b\x66ields_json\x18\x0c \x01(\x0c\x12\x0b\n\x03key\x18\r \x01(\tJ\x04\x08\x01\x10\x02J\x04\x08\x03\x10\x04J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\n\x10\x0bJ\x04\x08\x0b\x10\x0cJ\x04\x08\x0e\x10\x0f"h\n\x14GenericResourceEvent\x12\x0e\n\x06source\x18\x01 \x01(\t\x12:\n\x08resource\x18\x03 \x01(\x0b\x32(.datadog.genresources.v1.GenericResourceJ\x04\x08\x02\x10\x03\x42>Z GenericResourceEvent: + event = GenericResourceEvent() + event.ParseFromString(payload) + return event + + +def test_submit_generic_resource_emits_expected_event(aggregator, check): + check.submit_generic_resource( + type="argocd_application", + key="cluster:argocd:guestbook", + fields={ + "metadata": { + "name": "guestbook", + "namespace": "argocd", + "labels": {"team": "platform", "env": "prod"}, + "annotations": {"owner": "team-a", "kubectl.kubernetes.io/last-applied-configuration": "SECRET"}, + }, + "spec": { + "project": "default", + "source": {"repoURL": "https://repo", "helm": {"valuesObject": "SECRET"}}, + "sources": [ + {"repoURL": "https://a", "path": "p1", "ref": "x"}, + {"repoURL": "https://b", "path": "p2"}, + ], + }, + "status": {"health": {"status": "Healthy"}, "sync": {"status": "Synced"}}, + }, + include={ + "paths": [ + "metadata.name", + "metadata.namespace", + "spec.project", + "spec.sources[*].repoURL", + "spec.sources[*].path", + "status.health.status", + ], + "map_paths": ["metadata.labels"], + "annotation_keys": ["owner"], + }, + seen_at=1700000000, + expire_at=1700021600, + ) + + [payload] = aggregator.get_event_platform_events("genresources", parse_json=False) + event = _decode(payload) + assert event.source == "integrations-core" + assert event.resource.type == "argocd_application" + assert event.resource.key == "cluster:argocd:guestbook" + assert event.resource.seen_at.seconds == 1700000000 + assert event.resource.expire_at.seconds == 1700021600 + assert json.loads(event.resource.fields_json) == { + "metadata": { + "name": "guestbook", + "namespace": "argocd", + "labels": {"team": "platform", "env": "prod"}, + "annotations": {"owner": "team-a"}, + }, + "spec": { + "project": "default", + "sources": [{"repoURL": "https://a", "path": "p1"}, {"repoURL": "https://b", "path": "p2"}], + }, + "status": {"health": {"status": "Healthy"}}, + } + datadog_agent.assert_telemetry("argocd", "datadog.agent.check.genresources.emitted", "count", 1) + + +def test_submit_generic_resource_given_include_all_ships_constructed_fields(aggregator, check): + check.submit_generic_resource( + type="sonarqube_project", + key="my-proj", + fields={"name": "my-proj", "quality_gate": "OK", "ncloc": 1234}, + include=INCLUDE_ALL, + ) + + [payload] = aggregator.get_event_platform_events("genresources", parse_json=False) + event = _decode(payload) + assert event.resource.type == "sonarqube_project" + assert json.loads(event.resource.fields_json) == {"name": "my-proj", "quality_gate": "OK", "ncloc": 1234} + + +@pytest.mark.parametrize( + "kwargs, expected_log", + [ + ( + {"type": "t", "key": "", "fields": {"metadata": {"name": "x"}}, "include": VALUE_INCLUDE}, + "empty key", + ), + ( + {"type": "", "key": "k", "fields": {"metadata": {"name": "x"}}, "include": VALUE_INCLUDE}, + "empty type", + ), + ( + {"type": "t", "key": "k", "fields": ["not", "a", "dict"], "include": VALUE_INCLUDE}, + "non-dict fields", + ), + ( + {"type": "t", "key": "k", "fields": {"metadata": {"name": "x"}}, "include": None}, + "non-dict include", + ), + ( + { + "type": "t", + "key": "k", + "fields": {"metadata": {"name": "x"}}, + "include": {"paths": "metadata.name", "map_paths": [], "annotation_keys": []}, + }, + "malformed include", + ), + ( + { + "type": "t", + "key": "k", + "fields": {"metadata": {"name": "x"}}, + "include": {"paths": [], "map_paths": [], "annotation_keys": ["*"]}, + }, + "catch-all annotation pattern", + ), + ( + { + "type": "t", + "key": "k", + "fields": {"spec": {"project": "p"}}, + "include": {"paths": ["spec"], "map_paths": [], "annotation_keys": []}, + }, + "nested include value", + ), + ( + { + "type": "t", + "key": "k", + "fields": {"spec": {"source": {"repoURL": "r", "helm": {"x": 1}}}}, + "include": {"paths": [], "map_paths": ["spec.source"], "annotation_keys": []}, + }, + "non-flat map_path", + ), + ( + { + "type": "t", + "key": "k", + "fields": {"metadata": {"name": "x"}}, + "include": {"paths": ["does.not.exist"], "map_paths": [], "annotation_keys": []}, + }, + "empty inclusion", + ), + ( + { + "type": "t", + "key": "k", + "fields": {"spec": {"ratio": float("nan")}}, + "include": {"paths": ["spec.ratio"], "map_paths": [], "annotation_keys": []}, + }, + "failed to encode fields", + ), + ( + {"type": "t", "key": "k", "fields": {"metadata": {"name": "x" * 2_000_000}}, "include": VALUE_INCLUDE}, + "oversize resource", + ), + ], + ids=[ + "empty_key", + "empty_type", + "non_dict_fields", + "non_dict_include", + "malformed_include", + "catch_all_annotation", + "nested_object_include", + "invalid_map_path", + "empty_inclusion", + "nan_value", + "oversize_payload", + ], +) +def test_submit_generic_resource_drops_and_counts_invalid_input(aggregator, check, caplog, kwargs, expected_log): + check.submit_generic_resource(**kwargs) + assert aggregator.get_event_platform_events("genresources", parse_json=False) == [] + datadog_agent.assert_telemetry("argocd", "datadog.agent.check.genresources.dropped", "count", 1) + assert any(expected_log in record.getMessage() for record in caplog.records) + + +def test_submit_generic_resource_ignores_non_int_timestamps(aggregator, check): + check.submit_generic_resource( + type="argocd_application", + key="k", + fields={"metadata": {"name": "x"}}, + include=VALUE_INCLUDE, + seen_at=1700000000.9, + expire_at=1700000000.5, + ) + + [payload] = aggregator.get_event_platform_events("genresources", parse_json=False) + event = _decode(payload) + assert event.resource.seen_at.seconds == 0 + assert event.resource.expire_at.seconds == 0 diff --git a/datadog_checks_base/tests/base/utils/test_genresources.py b/datadog_checks_base/tests/base/utils/test_genresources.py new file mode 100644 index 0000000000000..d48ce80d7ce9e --- /dev/null +++ b/datadog_checks_base/tests/base/utils/test_genresources.py @@ -0,0 +1,104 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from __future__ import annotations + +import pytest + +from datadog_checks.base.utils.genresources.inclusion import apply_allow_list, find_invalid_include + + +def _fields(): + return { + "metadata": { + "name": "guestbook", + "namespace": "argocd", + "labels": {"team": "platform", "env": "prod"}, + "annotations": {"owner": "team-a", "kubectl.kubernetes.io/last-applied-configuration": "SECRET"}, + }, + "spec": { + "project": "default", + "source": {"repoURL": "https://repo", "helm": {"valuesObject": "SECRET"}}, + "sources": [ + {"repoURL": "https://a", "path": "p1", "ref": "x"}, + {"repoURL": "https://b", "path": "p2"}, + ], + }, + "status": {"health": {"status": "Healthy"}}, + "operation": None, + } + + +def test_apply_allow_list_given_value_map_and_annotation_includes_returns_only_those(): + result = apply_allow_list( + _fields(), + paths=["metadata.name", "spec.project", "status.health.status", "operation", "spec.missing"], + map_paths=["metadata.labels"], + annotation_keys=["owner"], + ) + assert result == { + "metadata": { + "name": "guestbook", + "labels": {"team": "platform", "env": "prod"}, + "annotations": {"owner": "team-a"}, + }, + "spec": {"project": "default"}, + "status": {"health": {"status": "Healthy"}}, + "operation": None, + } + + +def test_apply_allow_list_given_two_wildcard_paths_returns_both_fields_per_element(): + result = apply_allow_list( + _fields(), + paths=["spec.sources[*].repoURL", "spec.sources[*].path"], + map_paths=[], + annotation_keys=[], + ) + assert result == { + "spec": {"sources": [{"repoURL": "https://a", "path": "p1"}, {"repoURL": "https://b", "path": "p2"}]} + } + + +def test_apply_allow_list_given_partial_wildcard_match_drops_empty_elements(): + fields = {"spec": {"sources": [{"repoURL": "a"}, {"other": "b"}]}} + result = apply_allow_list(fields, paths=["spec.sources[*].repoURL"], map_paths=[], annotation_keys=[]) + assert result == {"spec": {"sources": [{"repoURL": "a"}]}} + + +def test_apply_allow_list_given_object_annotation_value_skips_it(): + fields = {"metadata": {"annotations": {"flat": "ok", "obj": {"x": 1}, "arr": [1]}}} + result = apply_allow_list(fields, paths=[], map_paths=[], annotation_keys=["flat", "obj", "arr"]) + assert result == {"metadata": {"annotations": {"flat": "ok"}}} + + +def test_apply_allow_list_does_not_mutate_input(): + fields = _fields() + apply_allow_list(fields, paths=["metadata.name"], map_paths=["metadata.labels"], annotation_keys=["owner"]) + assert fields == _fields() + + +@pytest.mark.parametrize( + "paths, map_paths, expected", + [ + (["metadata.name", "spec.sources[*].repoURL"], [], None), + (["metadata"], [], ("metadata", "nested include value")), + (["spec.source"], [], ("spec.source", "nested include value")), + (["spec.sources[*]"], [], ("spec.sources[*]", "nested include value")), + ([], ["metadata.labels"], None), + ([], ["spec.source"], ("spec.source", "non-flat map_path")), + (["metadata.annotations"], [], None), + ], + ids=[ + "value_paths_ok", + "flat_object_in_paths_rejected", + "nested_object_in_paths_rejected", + "array_of_objects_rejected", + "flat_map_path_ok", + "nested_map_path_rejected", + "annotations_path_skipped", + ], +) +def test_find_invalid_include_enforces_value_and_map_contract(paths, map_paths, expected): + assert find_invalid_include(_fields(), paths, map_paths) == expected diff --git a/dell_powerflex/CHANGELOG.md b/dell_powerflex/CHANGELOG.md new file mode 100644 index 0000000000000..2e9d588d488d2 --- /dev/null +++ b/dell_powerflex/CHANGELOG.md @@ -0,0 +1,3 @@ +# CHANGELOG - Dell PowerFlex + + \ No newline at end of file diff --git a/dell_powerflex/README.md b/dell_powerflex/README.md new file mode 100644 index 0000000000000..4da7d0080bb70 --- /dev/null +++ b/dell_powerflex/README.md @@ -0,0 +1,144 @@ +# Agent Check: Dell PowerFlex + +## Overview + +This check monitors [Dell PowerFlex][1] software-defined storage environments through the Datadog Agent. It collects metrics, events, and alerts from the PowerFlex Gateway REST API across the following resource types: + +- **Systems**: MDM cluster state, capacity, and I/O statistics +- **Protection Domains**: capacity, rebuild, rebalance, and I/O metrics +- **Storage Pools**: capacity utilization, usage ratios, and throughput +- **Volumes**: per-volume I/O and SDC mappings +- **SDS (Storage Data Servers)**: device counts, capacity, cache, and I/O +- **SDC (Storage Data Clients)**: mapped volumes and user data I/O +- **Devices**: read/write latency, capacity, and I/O bandwidth + +## Setup + +### Installation + +The Dell PowerFlex check is included in the [Datadog Agent][2] package. No additional installation is needed on your server. + +### Configuration + +1. Edit the `dell_powerflex.d/conf.yaml` file in the `conf.d/` folder at the root of your [Agent's configuration directory][3] to start collecting your Dell PowerFlex metrics. See the [sample dell_powerflex.d/conf.yaml][4] for all available configuration options. + + ```yaml + instances: + - powerflex_gateway_url: https://:443 + powerflex_username: + powerflex_password: + ``` + +2. [Restart the Agent][5]. + +#### Optional: Event and alert collection + +To collect events and alerts from the PowerFlex Gateway, enable them in your configuration: + +```yaml +instances: + - powerflex_gateway_url: https://:443 + powerflex_username: + powerflex_password: + collect_events: true + collect_alerts: true +``` + +#### Optional: Resource filtering + +Use `resource_filters` to control which resources are collected and whether per-resource statistics API calls are made. This is useful for large environments where you want to limit the number of API calls. Exclude filters take precedence over include filters. By default, all resources are collected with statistics enabled, except for devices which have statistics disabled by default. + +```yaml +instances: + - powerflex_gateway_url: https://:443 + powerflex_username: + powerflex_password: + resource_filters: + - resource: storage_pool + property: name + patterns: + - "^prod-" + - resource: sds + property: name + type: exclude + patterns: + - "^standby-" + - resource: device + property: name + collect_statistics: false +``` + +#### Log collection + +The Dell PowerFlex integration can collect logs from multiple PowerFlex components. + +1. Collecting logs is disabled by default in the Datadog Agent. Enable it in your `datadog.yaml` file: + + ```yaml + logs_enabled: true + ``` + +2. Add this configuration block to your `dell_powerflex.d/conf.yaml` file to start collecting your Dell PowerFlex logs. Adjust the `path` and `service` values to match your environment: + + ```yaml + logs: + - type: file + path: /opt/emc/scaleio/mdm/logs/eventLogger.log + source: dell_powerflex + service: + + - type: file + path: /opt/emc/scaleio/mdm/logs/trc.0 + source: dell_powerflex + service: + + - type: file + path: /opt/emc/scaleio/sds/logs/trc.0 + source: dell_powerflex + service: + + - type: file + path: /opt/emc/scaleio/lia/logs/trc.0 + source: dell_powerflex + service: + + - type: file + path: /opt/emc/scaleio/activemq/data/activemq.log + source: dell_powerflex + service: + ``` + + See the [sample dell_powerflex.d/conf.yaml][4] for all available configuration options. + +3. [Restart the Agent][5]. + +### Validation + +[Run the Agent's status subcommand][6] and look for `dell_powerflex` under the Checks section. + +## Data Collected + +### Metrics + +See [metadata.csv][7] for a list of metrics provided by this check. + +### Events + +When `collect_events` is enabled, the Dell PowerFlex integration collects CRITICAL and MAJOR severity events from the PowerFlex Gateway. When `collect_alerts` is enabled, it collects alerts. Both are forwarded as Datadog events. + +### Service Checks + +The Dell PowerFlex integration does not include any service checks. + +## Troubleshooting + +Need help? Contact [Datadog support][8]. + +[1]: https://www.dell.com/en-us/dt/storage/powerflex.htm +[2]: https://app.datadoghq.com/account/settings/agent/latest +[3]: https://docs.datadoghq.com/agent/guide/agent-configuration-files/#agent-configuration-directory +[4]: https://github.com/DataDog/integrations-core/blob/master/dell_powerflex/datadog_checks/dell_powerflex/data/conf.yaml.example +[5]: https://docs.datadoghq.com/agent/guide/agent-commands/#start-stop-and-restart-the-agent +[6]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[7]: https://github.com/DataDog/integrations-core/blob/master/dell_powerflex/metadata.csv +[8]: https://docs.datadoghq.com/help/ diff --git a/dell_powerflex/assets/configuration/spec.yaml b/dell_powerflex/assets/configuration/spec.yaml new file mode 100644 index 0000000000000..e3f372c585562 --- /dev/null +++ b/dell_powerflex/assets/configuration/spec.yaml @@ -0,0 +1,134 @@ +name: Dell PowerFlex +files: +- name: dell_powerflex.yaml + options: + - template: init_config + options: + - template: init_config/default + - template: instances + options: + - name: powerflex_gateway_url + required: true + display_priority: 3 + description: URL of the PowerFlex Gateway REST API. + value: + example: https://localhost:443 + type: string + - name: powerflex_username + display_priority: 2 + description: Username for PowerFlex Gateway authentication via Keycloak. + value: + type: string + - name: powerflex_password + secret: true + display_priority: 1 + description: Password for PowerFlex Gateway authentication via Keycloak. + value: + type: string + example: + - name: powerflex_client_id + description: OAuth2 client ID for Keycloak authentication. + value: + type: string + example: powerflexUI + default: powerflexUI + - name: collect_events + description: Enable collection of CRITICAL and MAJOR severity events from the PowerFlex Gateway. + value: + type: boolean + example: false + default: false + - name: collect_alerts + description: Enable collection of alerts from the PowerFlex Gateway. + value: + type: boolean + example: false + default: false + - name: max_workers + description: | + Maximum number of threads used to fetch per-resource statistics in parallel. + Increase this value for large environments with many resources. + value: + type: integer + example: 4 + default: 4 + - name: resource_filters + description: | + Filter resources by property regex patterns and control statistics + collection per resource type. Exclude takes precedence over include. + If no filter is configured for a resource type, all resources of + that type are collected with statistics enabled, except for devices + which have statistics disabled by default. + + Each filter entry has the following fields: + resource: The resource type to filter. + property: The API property name to match against. + patterns: List of regex patterns to match against the property value. + type: Either 'include' (default) or 'exclude'. If a resource matches + both an include and exclude filter, it is excluded. + collect_statistics: When false, skip per-resource statistics API + calls to reduce load. Defaults to true. + + Supported resource types and common filterable properties: + volume: name, id, volumeType, storagePoolId, ancestorVolumeId + storage_pool: name, id, mediaType, protectionDomainId + protection_domain: name, id, protectionDomainState + sds: name, id, protectionDomainId, sdsState, faultSetId + sdc: id, sdcGuid, sdcType, sdcIp + device: name, id, storagePoolId, sdsId, deviceCurrentPathName + + Note: the system resource type is not filterable and will be ignored if specified. + value: + type: array + items: + type: object + example: + - resource: volume + property: name + patterns: + - ".*" + - resource: storage_pool + property: name + collect_statistics: true + patterns: + - "^prod-" + - resource: protection_domain + property: name + patterns: + - ".*" + - resource: sds + property: name + type: exclude + patterns: + - "^standby-" + - resource: sdc + property: sdcType + patterns: + - "^AppSdc$" + - resource: device + property: name + collect_statistics: false + - template: instances/http + - template: instances/default + - template: logs + example: + - type: file + path: /opt/emc/scaleio/mdm/logs/eventLogger.log + source: dell_powerflex + service: + - type: file + path: /opt/emc/scaleio/mdm/logs/trc.0 + source: dell_powerflex + service: + - type: file + path: /opt/emc/scaleio/sds/logs/trc.0 + source: dell_powerflex + service: + - type: file + path: /opt/emc/scaleio/lia/logs/trc.0 + source: dell_powerflex + service: + - type: file + path: /opt/emc/scaleio/activemq/data/activemq.log + source: dell_powerflex + service: diff --git a/dell_powerflex/changelog.d/23183.added b/dell_powerflex/changelog.d/23183.added new file mode 100644 index 0000000000000..43da9447577c7 --- /dev/null +++ b/dell_powerflex/changelog.d/23183.added @@ -0,0 +1 @@ +Initial Release. \ No newline at end of file diff --git a/dell_powerflex/datadog_checks/__init__.py b/dell_powerflex/datadog_checks/__init__.py new file mode 100644 index 0000000000000..becee4feb84ae --- /dev/null +++ b/dell_powerflex/datadog_checks/__init__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/dell_powerflex/datadog_checks/dell_powerflex/__about__.py b/dell_powerflex/datadog_checks/dell_powerflex/__about__.py new file mode 100644 index 0000000000000..e50f43adfb9b1 --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/__about__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +__version__ = '0.0.1' diff --git a/dell_powerflex/datadog_checks/dell_powerflex/__init__.py b/dell_powerflex/datadog_checks/dell_powerflex/__init__.py new file mode 100644 index 0000000000000..8f0a4ef16a487 --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/__init__.py @@ -0,0 +1,7 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .__about__ import __version__ +from .check import DellPowerflexCheck + +__all__ = ['__version__', 'DellPowerflexCheck'] diff --git a/dell_powerflex/datadog_checks/dell_powerflex/api.py b/dell_powerflex/datadog_checks/dell_powerflex/api.py new file mode 100644 index 0000000000000..5fb81f04041e9 --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/api.py @@ -0,0 +1,127 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from time import time +from typing import Any + +from datadog_checks.base.log import CheckLoggingAdapter +from datadog_checks.base.utils.http import RequestsWrapper + +TOKEN_PATH = '/auth/realms/powerflex/protocol/openid-connect/token' + + +class PowerFlexAPI: + def __init__( + self, + http: RequestsWrapper, + gateway_url: str, + logger: CheckLoggingAdapter, + username: str | None = None, + password: str | None = None, + client_id: str = 'powerflexUI', + min_collection_interval: float = 15, + ) -> None: + self._http = http + self._gateway_url = gateway_url + self._username = username + self._password = password + self._client_id = client_id + self._log = logger + self._min_collection_interval = min_collection_interval + self._token: str | None = None + self._token_expiry: float = 0.0 + + def ensure_authenticated(self) -> None: + if not self._username: + self._log.debug('No username configured, skipping authentication') + return + if self._token and time() < (self._token_expiry - self._min_collection_interval): + return + self._authenticate() + + def _authenticate(self) -> None: + url = f'{self._gateway_url}{TOKEN_PATH}' + response = self._http.post( + url, + data={ + 'grant_type': 'password', + 'client_id': self._client_id, + 'username': self._username, + 'password': self._password, + }, + ) + response.raise_for_status() + data = response.json() + self._token = data.get('access_token') + if not self._token: + raise ValueError(f"Auth response missing access_token: {data}") + expires_in = data.get('expires_in', 300) + self._token_expiry = time() + expires_in + self._http.options['headers']['Authorization'] = f'Bearer {self._token}' + self._log.debug('Refreshed PowerFlex auth token, expires in %ds', expires_in) + + def _get(self, path: str) -> Any: + response = self._http.get(f"{self._gateway_url}{path}") + response.raise_for_status() + return response.json() + + def get_version(self) -> str: + return self._get('/api/version') + + def get_systems(self) -> list[dict]: + return self._get('/api/types/System/instances') + + def get_system_statistics(self, system_id: str) -> dict: + return self._get(f'/api/instances/System::{system_id}/relationships/Statistics') + + def get_volumes(self) -> list[dict]: + return self._get('/api/types/Volume/instances') + + def get_volume_statistics(self, volume_id: str) -> dict: + return self._get(f'/api/instances/Volume::{volume_id}/relationships/Statistics') + + def get_storage_pools(self) -> list[dict]: + return self._get('/api/types/StoragePool/instances') + + def get_storage_pool_statistics(self, pool_id: str) -> dict: + return self._get(f'/api/instances/StoragePool::{pool_id}/relationships/Statistics') + + def get_sdc_list(self) -> list[dict]: + return self._get('/api/types/Sdc/instances') + + def get_sdc_statistics(self, sdc_id: str) -> dict: + return self._get(f'/api/instances/Sdc::{sdc_id}/relationships/Statistics') + + def get_sds_list(self) -> list[dict]: + return self._get('/api/types/Sds/instances') + + def get_sds_statistics(self, sds_id: str) -> dict: + return self._get(f'/api/instances/Sds::{sds_id}/relationships/Statistics') + + def get_devices(self) -> list[dict]: + return self._get('/api/types/Device/instances') + + def get_device_statistics(self, device_id: str) -> dict: + return self._get(f'/api/instances/Device::{device_id}/relationships/Statistics') + + def get_protection_domains(self) -> list[dict]: + return self._get('/api/types/ProtectionDomain/instances') + + def get_protection_domain_statistics(self, pd_id: str) -> dict: + return self._get(f'/api/instances/ProtectionDomain::{pd_id}/relationships/Statistics') + + def get_alerts(self, since: str) -> list[dict]: + # All severities are collected as alerts are less noisy than events. + query = f'/rest/v1/alerts?filter=last_updated ge {since}' + response = self._get(query) + results = response.get('results', []) if isinstance(response, dict) else [] + self._log.debug('Collected %d alerts', len(results)) + return results + + def get_events(self, since: str) -> list[dict]: + query = f'/rest/v1/events?filter=timestamp ge {since}' + response = self._get(query) + results = response.get('results', []) if isinstance(response, dict) else [] + events = [e for e in results if e.get('severity') in ('CRITICAL', 'MAJOR')] + self._log.debug('Collected %d events', len(events)) + return events diff --git a/dell_powerflex/datadog_checks/dell_powerflex/check.py b/dell_powerflex/datadog_checks/dell_powerflex/check.py new file mode 100644 index 0000000000000..8b4c1e3648ba7 --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/check.py @@ -0,0 +1,411 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone +from typing import Any + +from datadog_checks.base import AgentCheck + +from .api import PowerFlexAPI +from .config_models import ConfigMixin +from .constants import ( + BWC_SUB_FIELDS, + DEVICE_RESOURCE_TYPE, + DEVICE_STATS_BWC_METRICS, + DEVICE_STATS_SIMPLE_METRICS, + PROTECTION_DOMAIN_RESOURCE_TYPE, + PROTECTION_DOMAIN_STATS_BWC_METRICS, + PROTECTION_DOMAIN_STATS_SIMPLE_METRICS, + SDC_RESOURCE_TYPE, + SDC_STATS_BWC_METRICS, + SDC_STATS_SIMPLE_METRICS, + SDS_RESOURCE_TYPE, + SDS_STATS_BWC_METRICS, + SDS_STATS_SIMPLE_METRICS, + SEVERITY_TO_ALERT_TYPE, + STORAGE_POOL_RESOURCE_TYPE, + STORAGE_POOL_STATS_BWC_METRICS, + STORAGE_POOL_STATS_SIMPLE_METRICS, + SYSTEM_MDM_CLUSTER_SIMPLE_METRICS, + SYSTEM_MDM_CLUSTER_STATE_METRICS, + SYSTEM_RESOURCE_TYPE, + SYSTEM_STATS_BWC_METRICS, + SYSTEM_STATS_SIMPLE_METRICS, + VOLUME_RESOURCE_TYPE, + VOLUME_STATS_BWC_METRICS, + VOLUME_STATS_SIMPLE_METRICS, +) +from .resource_filters import ResourceFilter, parse_resource_filters, should_collect_resource, should_collect_statistics + + +class DellPowerflexCheck(AgentCheck, ConfigMixin): + __NAMESPACE__ = 'dell_powerflex' + + def __init__(self, name: str, init_config: dict, instances: list) -> None: + super().__init__(name, init_config, instances) + self._base_tags: list[str] = [] + self._api: PowerFlexAPI + self._resource_filters: list[ResourceFilter] = [] + self.check_initializations.append(self._parse_config) + + def _parse_config(self) -> None: + self._base_tags = [f'powerflex_gateway_url:{self.config.powerflex_gateway_url}'] + list(self.config.tags or ()) + self._api = PowerFlexAPI( + self.http, + self.config.powerflex_gateway_url, + username=self.config.powerflex_username, + password=self.config.powerflex_password, + client_id=self.config.powerflex_client_id, # type: ignore[arg-type] + logger=self.log, + min_collection_interval=self.config.min_collection_interval, # type: ignore[arg-type] + ) + self._resource_filters = parse_resource_filters(self.config.resource_filters, self.log) + + def check(self, _: Any) -> None: + try: + self._api.ensure_authenticated() + self._api.get_version() + self.gauge('api.can_connect', 1, tags=self._base_tags) + except Exception as e: + self.log.warning('Could not connect to PowerFlex Gateway, skipping metric collection: %s', e) + self.gauge('api.can_connect', 0, tags=self._base_tags) + return + for collector in ( + self._collect_systems, + self._collect_volumes, + self._collect_storage_pools, + self._collect_protection_domains, + self._collect_sds_list, + self._collect_sdc_list, + self._collect_devices, + self._collect_events_and_alerts, + ): + try: + collector() + except Exception as e: + self.log.warning('Failed during %s collection: %s', collector.__name__, e) + self.log.debug('Failed during %s collection', collector.__name__, exc_info=True) + + def _collect_statistics( + self, + work: list[tuple[str, list[str]]], + stats_api: Callable[[str], dict], + simple_metrics: list[tuple[str, str]], + bwc_metrics: list[tuple[str, str]], + ) -> None: + with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor: + future_to_resource = { + executor.submit(stats_api, resource_id): (resource_id, tags) for resource_id, tags in work + } + for future in as_completed(future_to_resource): + resource_id, tags = future_to_resource[future] + try: + stats = future.result() + except Exception as e: + self.log.warning('Failed to collect statistics for %s: %s', resource_id, e) + self.log.debug('Failed to collect statistics for %s', resource_id, exc_info=True) + continue + for api_field, metric_suffix in simple_metrics: + self.gauge(metric_suffix, stats.get(api_field), tags=tags) + self._collect_bwc_metrics(stats, bwc_metrics, tags) + + def _collect_systems(self) -> None: + systems = self._api.get_systems() + self.log.debug('Collected %d systems', len(systems)) + stats_work: list[tuple[str, list[str]]] = [] + for system in systems: + try: + tags = self._collect_system(system) + stats_work.append((system.get('id', ''), tags)) + except Exception as e: + self.log.warning('Failed to collect metrics for system %s: %s', system.get('id'), e) + if stats_work: + self._collect_statistics( + stats_work, self._api.get_system_statistics, SYSTEM_STATS_SIMPLE_METRICS, SYSTEM_STATS_BWC_METRICS + ) + + def _collect_system(self, system: dict) -> list[str]: + tags = self._base_tags + [f"system_id:{system.get('id', '')}", f"dell_type:{SYSTEM_RESOURCE_TYPE}"] + if system.get('name'): + tags = tags + [f"system_name:{system.get('name', '')}"] + self.gauge('system.count', 1, tags=tags) + mdm_cluster = system.get('mdmCluster', {}) + for api_field, metric_suffix in SYSTEM_MDM_CLUSTER_SIMPLE_METRICS: + self.gauge(metric_suffix, mdm_cluster.get(api_field), tags=tags) + for api_field, metric_suffix, tag_key in SYSTEM_MDM_CLUSTER_STATE_METRICS: + self.gauge( + metric_suffix, + 1, + tags=tags + [f"{tag_key}:{mdm_cluster.get(api_field, '')}"], + ) + return tags + + def _collect_volumes(self) -> None: + volumes = self._api.get_volumes() + self.log.debug('Collected %d volumes', len(volumes)) + collect_stats = should_collect_statistics(VOLUME_RESOURCE_TYPE, self._resource_filters) + stats_work: list[tuple[str, list[str]]] = [] + for volume in volumes: + try: + if not should_collect_resource(VOLUME_RESOURCE_TYPE, volume, self._resource_filters, self.log): + continue + tags = self._collect_volume(volume) + if collect_stats: + stats_work.append((volume.get('id', ''), tags)) + except Exception as e: + self.log.warning('Failed to collect metrics for volume %s: %s', volume.get('id'), e) + if stats_work: + self._collect_statistics( + stats_work, self._api.get_volume_statistics, VOLUME_STATS_SIMPLE_METRICS, VOLUME_STATS_BWC_METRICS + ) + + def _collect_volume(self, volume: dict) -> list[str]: + tags = self._base_tags + [ + f"volume_id:{volume.get('id', '')}", + f"volume_name:{volume.get('name', '')}", + f"volume_type:{volume.get('volumeType', '')}", + f"storage_pool_id:{volume.get('storagePoolId', '')}", + f"dell_type:{VOLUME_RESOURCE_TYPE}", + ] + if volume.get('ancestorVolumeId'): + tags = tags + [f"ancestor_volume_id:{volume.get('ancestorVolumeId', '')}"] + self.gauge('volume.count', 1, tags=tags) + for sdc in volume.get('mappedSdcInfo') or []: + mapping_tags = tags + [f"sdc_id:{sdc.get('sdcId', '')}"] + self.gauge('volume.sdc_mapping', 1, tags=mapping_tags) + return tags + + def _collect_storage_pools(self) -> None: + storage_pools = self._api.get_storage_pools() + self.log.debug('Collected %d storage pools', len(storage_pools)) + collect_stats = should_collect_statistics(STORAGE_POOL_RESOURCE_TYPE, self._resource_filters) + stats_work: list[tuple[str, list[str]]] = [] + for pool in storage_pools: + try: + if not should_collect_resource(STORAGE_POOL_RESOURCE_TYPE, pool, self._resource_filters, self.log): + continue + tags = self._collect_storage_pool(pool) + if collect_stats: + stats_work.append((pool.get('id', ''), tags)) + except Exception as e: + self.log.warning('Failed to collect metrics for storage pool %s: %s', pool.get('id'), e) + if stats_work: + self._collect_statistics( + stats_work, + self._api.get_storage_pool_statistics, + STORAGE_POOL_STATS_SIMPLE_METRICS, + STORAGE_POOL_STATS_BWC_METRICS, + ) + + def _collect_storage_pool(self, pool: dict) -> list[str]: + tags = self._base_tags + [ + f"storage_pool_id:{pool.get('id', '')}", + f"storage_pool_name:{pool.get('name', '')}", + f"protection_domain_id:{pool.get('protectionDomainId', '')}", + f"dell_type:{STORAGE_POOL_RESOURCE_TYPE}", + ] + self.gauge('storage_pool.count', 1, tags=tags) + return tags + + def _collect_protection_domains(self) -> None: + protection_domains = self._api.get_protection_domains() + self.log.debug('Collected %d protection domains', len(protection_domains)) + collect_stats = should_collect_statistics(PROTECTION_DOMAIN_RESOURCE_TYPE, self._resource_filters) + stats_work: list[tuple[str, list[str]]] = [] + for pd in protection_domains: + try: + if not should_collect_resource(PROTECTION_DOMAIN_RESOURCE_TYPE, pd, self._resource_filters, self.log): + continue + tags = self._collect_protection_domain(pd) + if collect_stats: + stats_work.append((pd.get('id', ''), tags)) + except Exception as e: + self.log.warning('Failed to collect metrics for protection domain %s: %s', pd.get('id'), e) + if stats_work: + self._collect_statistics( + stats_work, + self._api.get_protection_domain_statistics, + PROTECTION_DOMAIN_STATS_SIMPLE_METRICS, + PROTECTION_DOMAIN_STATS_BWC_METRICS, + ) + + def _collect_protection_domain(self, pd: dict) -> list[str]: + tags = self._base_tags + [ + f"protection_domain_id:{pd.get('id', '')}", + f"protection_domain_name:{pd.get('name', '')}", + f"system_id:{pd.get('systemId', '')}", + f"dell_type:{PROTECTION_DOMAIN_RESOURCE_TYPE}", + ] + self.gauge('protection_domain.count', 1, tags=tags) + return tags + + def _collect_sds_list(self) -> None: + sds_list = self._api.get_sds_list() + self.log.debug('Collected %d SDSs', len(sds_list)) + collect_stats = should_collect_statistics(SDS_RESOURCE_TYPE, self._resource_filters) + stats_work: list[tuple[str, list[str]]] = [] + for sds in sds_list: + try: + if not should_collect_resource(SDS_RESOURCE_TYPE, sds, self._resource_filters, self.log): + continue + tags = self._collect_sds(sds) + if collect_stats: + stats_work.append((sds.get('id', ''), tags)) + except Exception as e: + self.log.warning('Failed to collect metrics for SDS %s: %s', sds.get('id'), e) + if stats_work: + self._collect_statistics( + stats_work, self._api.get_sds_statistics, SDS_STATS_SIMPLE_METRICS, SDS_STATS_BWC_METRICS + ) + + def _collect_sds(self, sds: dict) -> list[str]: + tags = self._base_tags + [ + f"sds_id:{sds.get('id', '')}", + f"sds_name:{sds.get('name', '')}", + f"protection_domain_id:{sds.get('protectionDomainId', '')}", + f"dell_type:{SDS_RESOURCE_TYPE}", + ] + if sds.get('faultSetId'): + tags = tags + [f"fault_set_id:{sds.get('faultSetId', '')}"] + self.gauge('sds.count', 1, tags=tags) + return tags + + def _collect_sdc_list(self) -> None: + sdc_list = self._api.get_sdc_list() + self.log.debug('Collected %d SDCs', len(sdc_list)) + collect_stats = should_collect_statistics(SDC_RESOURCE_TYPE, self._resource_filters) + stats_work: list[tuple[str, list[str]]] = [] + for sdc in sdc_list: + try: + if not should_collect_resource(SDC_RESOURCE_TYPE, sdc, self._resource_filters, self.log): + continue + tags = self._collect_sdc(sdc) + if collect_stats: + stats_work.append((sdc.get('id', ''), tags)) + except Exception as e: + self.log.warning('Failed to collect metrics for SDC %s: %s', sdc.get('id'), e) + if stats_work: + self._collect_statistics( + stats_work, self._api.get_sdc_statistics, SDC_STATS_SIMPLE_METRICS, SDC_STATS_BWC_METRICS + ) + + def _collect_sdc(self, sdc: dict) -> list[str]: + tags = self._base_tags + [ + f"sdc_id:{sdc.get('id', '')}", + f"sdc_guid:{sdc.get('sdcGuid', '')}", + f"sdc_type:{sdc.get('sdcType', '')}", + f"sdc_ip:{sdc.get('sdcIp', '')}", + f"dell_type:{SDC_RESOURCE_TYPE}", + ] + if sdc.get('peerMdmId'): + tags = tags + [f"peer_mdm_id:{sdc.get('peerMdmId', '')}"] + self.gauge('sdc.count', 1, tags=tags) + return tags + + def _collect_devices(self) -> None: + devices = self._api.get_devices() + self.log.debug('Collected %d devices', len(devices)) + collect_stats = should_collect_statistics(DEVICE_RESOURCE_TYPE, self._resource_filters) + stats_work: list[tuple[str, list[str]]] = [] + for device in devices: + try: + if not should_collect_resource(DEVICE_RESOURCE_TYPE, device, self._resource_filters, self.log): + continue + tags = self._collect_device(device) + if collect_stats: + stats_work.append((device.get('id', ''), tags)) + except Exception as e: + self.log.warning('Failed to collect metrics for device %s: %s', device.get('id'), e) + if stats_work: + self._collect_statistics( + stats_work, self._api.get_device_statistics, DEVICE_STATS_SIMPLE_METRICS, DEVICE_STATS_BWC_METRICS + ) + + def _collect_device(self, device: dict) -> list[str]: + tags = self._base_tags + [ + f"device_id:{device.get('id', '')}", + f"device_name:{device.get('name', '')}", + f"current_path_name:{device.get('deviceCurrentPathName', '')}", + f"storage_pool_id:{device.get('storagePoolId', '')}", + f"sds_id:{device.get('sdsId', '')}", + f"dell_type:{DEVICE_RESOURCE_TYPE}", + ] + self.gauge('device.count', 1, tags=tags) + return tags + + def _collect_events_and_alerts(self) -> None: + now = datetime.now(tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ') + if self.config.collect_events: + last_event_ts = self.read_persistent_cache('last_event_timestamp') or now + if self._collect_events(last_event_ts): + self.write_persistent_cache('last_event_timestamp', now) + if self.config.collect_alerts: + last_alert_ts = self.read_persistent_cache('last_alert_timestamp') or now + if self._collect_alerts(last_alert_ts): + self.write_persistent_cache('last_alert_timestamp', now) + + def _collect_events(self, since: str) -> bool: + try: + events = self._api.get_events(since=since) + except Exception as e: + self.log.warning('Failed to collect events: %s', e) + return False + for event in events: + try: + self.event(self._build_dd_event(event, 'powerflex_event_name', 'service_name', 'dell_powerflex.event')) + except Exception as e: + self.log.warning('Skipping malformed event %s: %s', event.get('name'), e) + return True + + def _collect_alerts(self, since: str) -> bool: + try: + alerts = self._api.get_alerts(since=since) + except Exception as e: + self.log.warning('Failed to collect alerts: %s', e) + return False + for alert in alerts: + try: + self.event(self._build_dd_event(alert, 'powerflex_alert_name', 'service', 'dell_powerflex.alert')) + except Exception as e: + self.log.warning('Skipping malformed alert %s: %s', alert.get('name'), e) + return True + + def _build_dd_event(self, raw: dict, name_tag_key: str, service_key: str, event_type: str) -> dict[str, Any]: + raw_ts = raw.get('timestamp') + timestamp = datetime.fromisoformat(raw_ts).timestamp() if raw_ts else datetime.now(tz=timezone.utc).timestamp() + + severity = raw.get('severity', '') + alert_type = SEVERITY_TO_ALERT_TYPE.get(str(severity).upper(), 'info') if severity else 'info' + + tags = list(self._base_tags) + tags.append(f"{name_tag_key}:{raw.get('name', '')}") + tags.append(f"severity:{severity}") + tags.append(f"category:{raw.get('category', '')}") + tags.append(f"domain:{raw.get('domain', '')}") + tags.append(f"dell_type:{raw.get('resource_type', '')}") + tags.append(f"resource_name:{raw.get('resource_name', '')}") + tags.append(f"service_name:{raw.get(service_key, '')}") + + name = raw.get('name', '') + title = name.replace('_', ' ').title() + resource_name = raw.get('resource_name', '') + description = raw.get('description', '') + msg_text = f"{resource_name}: {description}" if resource_name else description + + return { + 'timestamp': timestamp, + 'event_type': event_type, + 'msg_title': title, + 'msg_text': msg_text, + 'alert_type': alert_type, + 'source_type_name': 'dell-powerflex', + 'tags': tags, + } + + def _collect_bwc_metrics(self, stats: dict, bwc_metrics: list[tuple[str, str]], tags: list[str]) -> None: + for api_field, metric_suffix in bwc_metrics: + bwc = stats.get(api_field) or {} + for bwc_field, bwc_suffix in BWC_SUB_FIELDS: + self.gauge(f'{metric_suffix}.{bwc_suffix}', bwc.get(bwc_field), tags=tags) diff --git a/dell_powerflex/datadog_checks/dell_powerflex/config_models/__init__.py b/dell_powerflex/datadog_checks/dell_powerflex/config_models/__init__.py new file mode 100644 index 0000000000000..f678b7e73d91a --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/config_models/__init__.py @@ -0,0 +1,24 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from .instance import InstanceConfig +from .shared import SharedConfig + + +class ConfigMixin: + _config_model_instance: InstanceConfig + _config_model_shared: SharedConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared diff --git a/dell_powerflex/datadog_checks/dell_powerflex/config_models/defaults.py b/dell_powerflex/datadog_checks/dell_powerflex/config_models/defaults.py new file mode 100644 index 0000000000000..6f07ffd8c320c --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/config_models/defaults.py @@ -0,0 +1,96 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + + +def instance_allow_redirects(): + return True + + +def instance_auth_type(): + return 'basic' + + +def instance_collect_alerts(): + return False + + +def instance_collect_events(): + return False + + +def instance_disable_generic_tags(): + return False + + +def instance_empty_default_hostname(): + return False + + +def instance_enable_legacy_tags_normalization(): + return True + + +def instance_kerberos_auth(): + return 'disabled' + + +def instance_kerberos_delegate(): + return False + + +def instance_kerberos_force_initiate(): + return False + + +def instance_log_requests(): + return False + + +def instance_max_workers(): + return 4 + + +def instance_min_collection_interval(): + return 15 + + +def instance_persist_connections(): + return False + + +def instance_powerflex_client_id(): + return 'powerflexUI' + + +def instance_request_size(): + return 16 + + +def instance_skip_proxy(): + return False + + +def instance_timeout(): + return 10 + + +def instance_tls_ignore_warning(): + return False + + +def instance_tls_use_host_header(): + return False + + +def instance_tls_verify(): + return True + + +def instance_use_legacy_auth_encoding(): + return True diff --git a/dell_powerflex/datadog_checks/dell_powerflex/config_models/instance.py b/dell_powerflex/datadog_checks/dell_powerflex/config_models/instance.py new file mode 100644 index 0000000000000..2457e0a1d7a7a --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/config_models/instance.py @@ -0,0 +1,136 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from types import MappingProxyType +from typing import Any, Optional + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from typing_extensions import Literal + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +SECURE_FIELD_NAMES = frozenset( + ['auth_token', 'kerberos_cache', 'kerberos_keytab', 'tls_ca_cert', 'tls_cert', 'tls_private_key'] +) + + +class AuthToken(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + reader: Optional[MappingProxyType[str, Any]] = None + writer: Optional[MappingProxyType[str, Any]] = None + + +class MetricPatterns(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + exclude: Optional[tuple[str, ...]] = None + include: Optional[tuple[str, ...]] = None + + +class Proxy(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + http: Optional[str] = None + https: Optional[str] = None + no_proxy: Optional[tuple[str, ...]] = None + + +class InstanceConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + allow_redirects: Optional[bool] = None + auth_token: Optional[AuthToken] = None + auth_type: Optional[str] = None + aws_host: Optional[str] = None + aws_region: Optional[str] = None + aws_service: Optional[str] = None + collect_alerts: Optional[bool] = None + collect_events: Optional[bool] = None + connect_timeout: Optional[float] = None + disable_generic_tags: Optional[bool] = None + empty_default_hostname: Optional[bool] = None + enable_legacy_tags_normalization: Optional[bool] = None + extra_headers: Optional[MappingProxyType[str, Any]] = None + headers: Optional[MappingProxyType[str, Any]] = None + kerberos_auth: Optional[Literal['required', 'optional', 'disabled']] = None + kerberos_cache: Optional[str] = None + kerberos_delegate: Optional[bool] = None + kerberos_force_initiate: Optional[bool] = None + kerberos_hostname: Optional[str] = None + kerberos_keytab: Optional[str] = None + kerberos_principal: Optional[str] = None + log_requests: Optional[bool] = None + max_workers: Optional[int] = None + metric_patterns: Optional[MetricPatterns] = None + min_collection_interval: Optional[float] = None + ntlm_domain: Optional[str] = None + password: Optional[str] = None + persist_connections: Optional[bool] = None + powerflex_client_id: Optional[str] = None + powerflex_gateway_url: str + powerflex_password: Optional[str] = None + powerflex_username: Optional[str] = None + proxy: Optional[Proxy] = None + read_timeout: Optional[float] = None + request_size: Optional[float] = None + resource_filters: Optional[tuple[MappingProxyType[str, Any], ...]] = None + service: Optional[str] = None + skip_proxy: Optional[bool] = None + tags: Optional[tuple[str, ...]] = None + timeout: Optional[float] = None + tls_ca_cert: Optional[str] = None + tls_cert: Optional[str] = None + tls_ciphers: Optional[tuple[str, ...]] = None + tls_ignore_warning: Optional[bool] = None + tls_private_key: Optional[str] = None + tls_protocols_allowed: Optional[tuple[str, ...]] = None + tls_use_host_header: Optional[bool] = None + tls_verify: Optional[bool] = None + use_legacy_auth_encoding: Optional[bool] = None + username: Optional[str] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'instance_{info.field_name}', identity)(value, field=field) + + if info.field_name in SECURE_FIELD_NAMES: + validation.security.check_field_trusted_provider( + info.field_name, value, info.context.get('security_config') + ) + else: + value = getattr(defaults, f'instance_{info.field_name}', lambda: value)() + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_instance', identity)(model)) diff --git a/dell_powerflex/datadog_checks/dell_powerflex/config_models/shared.py b/dell_powerflex/datadog_checks/dell_powerflex/config_models/shared.py new file mode 100644 index 0000000000000..10cab800f6c1e --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/config_models/shared.py @@ -0,0 +1,45 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import validators + + +class SharedConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + service: Optional[str] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'shared_{info.field_name}', identity)(value, field=field) + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_shared', identity)(model)) diff --git a/dell_powerflex/datadog_checks/dell_powerflex/config_models/validators.py b/dell_powerflex/datadog_checks/dell_powerflex/config_models/validators.py new file mode 100644 index 0000000000000..5e48f02a73da4 --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/config_models/validators.py @@ -0,0 +1,13 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# Here you can include additional config validators or transformers +# +# def initialize_instance(values, **kwargs): +# if 'my_option' not in values and 'my_legacy_option' in values: +# values['my_option'] = values['my_legacy_option'] +# if values.get('my_number') > 10: +# raise ValueError('my_number max value is 10, got %s' % str(values.get('my_number'))) +# +# return values diff --git a/dell_powerflex/datadog_checks/dell_powerflex/constants.py b/dell_powerflex/datadog_checks/dell_powerflex/constants.py new file mode 100644 index 0000000000000..262fd8281ad1f --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/constants.py @@ -0,0 +1,232 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +SYSTEM_RESOURCE_TYPE = 'system' + +SYSTEM_MDM_CLUSTER_SIMPLE_METRICS = [ + ('goodNodesNum', 'mdm_cluster.good_nodes'), + ('goodReplicasNum', 'mdm_cluster.good_replicas'), +] + +SYSTEM_MDM_CLUSTER_STATE_METRICS = [ + ('clusterState', 'mdm_cluster.cluster_state', 'cluster_state'), + ('clusterMode', 'mdm_mode', 'mdm_mode'), +] + +SYSTEM_STATS_SIMPLE_METRICS = [ + ('capacityInUseInKb', 'capacity.in_use_in_kb'), + ('maxCapacityInKb', 'max_capacity.in_kb'), + ('thickCapacityInUseInKb', 'thick_capacity.in_use_in_kb'), + ('thinCapacityInUseInKb', 'thin_capacity.in_use_in_kb'), + ('snapCapacityInUseInKb', 'snap_capacity.in_use_in_kb'), + ('unusedCapacityInKb', 'unused_capacity.in_kb'), + ('spareCapacityInKb', 'spare_capacity.in_kb'), + ('backgroundScanFixedReadErrorCount', 'fixed_read_error_count'), + ('rmcacheSizeInKb', 'rmcache.size_in_kb'), + ('rmcacheSizeInUseInKb', 'rmcache.size_in_use_in_kb'), + ('numOfUnmappedVolumes', 'num_of_unmapped_volumes'), + ('numOfMappedToAllVolumes', 'num_of_mapped_to_all_volumes'), + ('numOfSnapshots', 'num_of_snapshots'), + ('rfcacheReadsReceived', 'rfcache.reads_received'), + ('rfcacheWritesReceived', 'rfcache.writes_received'), + ('rfacheReadHit', 'rfcache.read_hit'), + ('rfcacheReadMiss', 'rfcache.read_miss'), + ('rfacheWriteHit', 'rfcache.write_hit'), + ('rfcacheWriteMiss', 'rfcache.write_miss'), + ('userDataCapacityInKb', 'user_data.capacity_in_kb'), + ('snapshotCapacityInKb', 'snapshot.capacity_in_kb'), + ('overallUsageRatio', 'overall_usage_ratio'), + ('numSdsReconnections', 'num_sds_reconnections'), + ('numDevErrors', 'num_dev_errors'), + ('numSdsSdrDisconnections', 'num_sds_sdr_disconnections'), + ('numSdrSdcDisconnections', 'num_sdr_sdc_disconnections'), +] + +BWC_SUB_FIELDS = [ + ('numSeconds', 'num_seconds'), + ('totalWeightInKb', 'total_weight_in_kb'), + ('numOccured', 'num_occured'), +] + +# Shared simple metrics across storage pool, protection domain, SDS, and device +COMMON_SIMPLE_METRICS = [ + ('capacityLimitInKb', 'capacity_limit.in_kb'), + ('maxCapacityInKb', 'max_capacity.in_kb'), + ('capacityInUseInKb', 'capacity.in_use_in_kb'), + ('thickCapacityInUseInKb', 'thick_capacity.in_use_in_kb'), + ('thinCapacityInUseInKb', 'thin_capacity.in_use_in_kb'), + ('snapCapacityInUseInKb', 'snap_capacity.in_use_in_kb'), + ('inUseVacInKb', 'in_use_vac.in_kb'), + ('backgroundScanFixedReadErrorCount', 'fixed_read_error_count'), + ('rfcacheReadsReceived', 'rfcache.reads_received'), + ('rfcacheWritesReceived', 'rfcache.writes_received'), + ('rfacheReadHit', 'rfcache.read_hit'), + ('rfcacheReadMiss', 'rfcache.read_miss'), + ('rfacheWriteHit', 'rfcache.write_hit'), + ('rfcacheWriteMiss', 'rfcache.write_miss'), + ('userDataCapacityInKb', 'user_data.capacity_in_kb'), + ('snapshotCapacityInKb', 'snapshot.capacity_in_kb'), +] + +# Shared userData BWC metrics across system, volume, storage pool, protection domain, and SDC +COMMON_BWC_METRICS = [ + ('userDataReadBwc', 'user_data_read_bwc'), + ('userDataWriteBwc', 'user_data_write_bwc'), + ('userDataTrimBwc', 'user_data_trim_bwc'), + ('userDataSdcReadLatency', 'user_data_sdc_read_latency'), + ('userDataSdcWriteLatency', 'user_data_sdc_write_latency'), + ('userDataSdcTrimLatency', 'user_data_sdc_trim_latency'), +] + +# Shared I/O BWC metrics across storage pool, protection domain, SDS, and device +COMMON_IO_BWC_METRICS = [ + ('primaryReadBwc', 'primary_read_bwc'), + ('primaryWriteBwc', 'primary_write_bwc'), + ('secondaryReadBwc', 'secondary_read_bwc'), + ('secondaryWriteBwc', 'secondary_write_bwc'), + ('totalReadBwc', 'total_read_bwc'), + ('totalWriteBwc', 'total_write_bwc'), + ('targetReadLatency', 'target_read_latency'), + ('targetWriteLatency', 'target_write_latency'), +] + +SYSTEM_STATS_BWC_METRICS = ( + COMMON_BWC_METRICS + + COMMON_IO_BWC_METRICS + + [ + ('journalerReadLatency', 'journaler_read_latency'), + ('journalerWriteLatency', 'journaler_write_latency'), + ] +) + +VOLUME_RESOURCE_TYPE = 'volume' + +VOLUME_STATS_SIMPLE_METRICS = [ + ('numOfChildVolumes', 'num_of_child_volumes'), + ('numOfMappedSdcs', 'num_of_mapped_sdcs'), + ('rplTotalJournalCap', 'rpl_total_journal_cap'), + ('rplUsedJournalCap', 'rpl_used_journal_cap'), +] + +VOLUME_STATS_BWC_METRICS = list(COMMON_BWC_METRICS) + +SDC_RESOURCE_TYPE = 'sdc' + +SDC_STATS_SIMPLE_METRICS = [ + ('numOfMappedVolumes', 'num_of_mapped_volumes'), +] + +SDC_STATS_BWC_METRICS = list(COMMON_BWC_METRICS) + +STORAGE_POOL_RESOURCE_TYPE = 'storage_pool' + +STORAGE_POOL_STATS_SIMPLE_METRICS = COMMON_SIMPLE_METRICS + [ + ('unreachableUnusedCapacityInKb', 'unreachable_unused_capacity.in_kb'), + ('unusedCapacityInKb', 'unused_capacity.in_kb'), + ('spareCapacityInKb', 'spare_capacity.in_kb'), + ('capacityAvailableForVolumeAllocationInKb', 'capacity_available_for_volume_allocation.in_kb'), + ('protectedCapacityInKb', 'protected_capacity.in_kb'), + ('failedCapacityInKb', 'failed_capacity.in_kb'), + ('numOfUnmappedVolumes', 'num_of_unmapped_volumes'), + ('numOfSnapshots', 'num_of_snapshots'), + ('numOfVolumes', 'num_of_volumes'), + ('overallUsageRatio', 'overall_usage_ratio'), + ('exposedCapacityInKb', 'exposed_capacity.in_kb'), + ('ActualNetCapacityInUseInKb', 'actual_net_capacity.in_use_in_kb'), +] + +COMMON_REBUILD_BWC_METRICS = [ + ('rebalanceReadBwc', 'rebalance_read_bwc'), + ('rebalanceWriteBwc', 'rebalance_write_bwc'), + ('fwdRebuildReadBwc', 'fwd_rebuild_read_bwc'), + ('fwdRebuildWriteBwc', 'fwd_rebuild_write_bwc'), + ('bckRebuildReadBwc', 'bck_rebuild_read_bwc'), + ('bckRebuildWriteBwc', 'bck_rebuild_write_bwc'), + ('normRebuildReadBwc', 'norm_rebuild_read_bwc'), + ('normRebuildWriteBwc', 'norm_rebuild_write_bwc'), +] + +STORAGE_POOL_STATS_BWC_METRICS = COMMON_BWC_METRICS + COMMON_IO_BWC_METRICS + COMMON_REBUILD_BWC_METRICS + +SDS_RESOURCE_TYPE = 'sds' + +SDS_STATS_SIMPLE_METRICS = COMMON_SIMPLE_METRICS + [ + ('unreachableUnusedCapacityInKb', 'unreachable_unused_capacity.in_kb'), + ('unusedCapacityInKb', 'unused_capacity.in_kb'), + ('failedVacInKb', 'failed_vac.in_kb'), + ('numOfDevices', 'num_of_devices'), + ('compressionRatio', 'compression_ratio'), + ('rfcacheReadsPending', 'rfcache.reads_pending'), + ('rfcacheIoErrors', 'rfcache.io_errors'), + ('rmcacheSizeInKb', 'rmcache.size_in_kb'), + ('rmcacheSizeInUseInKb', 'rmcache.size_in_use_in_kb'), +] + +SDS_STATS_BWC_METRICS = COMMON_IO_BWC_METRICS + [ + ('volMigrationReadBwc', 'vol_migration_read_bwc'), + ('volMigrationWriteBwc', 'vol_migration_write_bwc'), + ('userDataReadBwc', 'user_data_read_bwc'), + ('userDataWriteBwc', 'user_data_write_bwc'), + ('userDataSdcReadLatency', 'user_data_sdc_read_latency'), + ('userDataSdcWriteLatency', 'user_data_sdc_write_latency'), +] + +DEVICE_RESOURCE_TYPE = 'device' + +DEVICE_STATS_SIMPLE_METRICS = COMMON_SIMPLE_METRICS + [ + ('avgReadSizeInBytes', 'avg_read_size_in_bytes'), + ('avgWriteSizeInBytes', 'avg_write_size_in_bytes'), + ('avgReadLatencyInMicrosec', 'avg_read_latency_in_microsec'), + ('avgWriteLatencyInMicrosec', 'avg_write_latency_in_microsec'), + ('failedVacInKb', 'failed_vac.in_kb'), + ('compressionRatio', 'compression_ratio'), + ('inaccessibleCapacityInKb', 'inaccessible_capacity.in_kb'), +] + +DEVICE_STATS_BWC_METRICS = list(COMMON_IO_BWC_METRICS) + +PROTECTION_DOMAIN_RESOURCE_TYPE = 'protection_domain' + +PROTECTION_DOMAIN_STATS_SIMPLE_METRICS = COMMON_SIMPLE_METRICS + [ + ('exposedCapacityInKb', 'exposed_capacity.in_kb'), + ('ActualNetCapacityInUseInKb', 'actual_net_capacity.in_use_in_kb'), + ('unreachableUnusedCapacityInKb', 'unreachable_unused_capacity.in_kb'), + ('unusedCapacityInKb', 'unused_capacity.in_kb'), + ('spareCapacityInKb', 'spare_capacity.in_kb'), + ('capacityAvailableForVolumeAllocationInKb', 'capacity_available_for_volume_allocation.in_kb'), + ('volumeAllocationLimitInKb', 'volume_allocation_limit.in_kb'), + ('protectedCapacityInKb', 'protected_capacity.in_kb'), + ('failedCapacityInKb', 'failed_capacity.in_kb'), + ('numOfUnmappedVolumes', 'num_of_unmapped_volumes'), + ('numOfSnapshots', 'num_of_snapshots'), + ('overallUsageRatio', 'overall_usage_ratio'), + ('netUserDataCapacityInKb', 'net_user_data_capacity.in_kb'), + ('netCapacityInUseInKb', 'net_capacity.in_use_in_kb'), + ('rebuildWaitSendQLength', 'rebuild_wait_send_q_length'), + ('rebalanceWaitSendQLength', 'rebalance_wait_send_q_length'), + ('rmcacheSizeInKb', 'rmcache.size_in_kb'), + ('rmcacheSizeInUseInKb', 'rmcache.size_in_use_in_kb'), + ('numOfThickBaseVolumes', 'num_of_thick_base_volumes'), + ('numOfThinBaseVolumes', 'num_of_thin_base_volumes'), + ('numOfSds', 'num_of_sds'), + ('numOfStoragePools', 'num_of_storage_pools'), + ('numOfFaultSets', 'num_of_fault_sets'), +] + +PROTECTION_DOMAIN_STATS_BWC_METRICS = ( + COMMON_BWC_METRICS + + COMMON_IO_BWC_METRICS + + COMMON_REBUILD_BWC_METRICS + + [ + ('volMigrationReadBwc', 'vol_migration_read_bwc'), + ('volMigrationWriteBwc', 'vol_migration_write_bwc'), + ] +) + +SEVERITY_TO_ALERT_TYPE = { + 'CRITICAL': 'error', + 'MAJOR': 'error', + 'MINOR': 'warning', + 'INFORMATION': 'info', +} diff --git a/dell_powerflex/datadog_checks/dell_powerflex/data/conf.yaml.example b/dell_powerflex/datadog_checks/dell_powerflex/data/conf.yaml.example new file mode 100644 index 0000000000000..bc4ee863ebed7 --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/data/conf.yaml.example @@ -0,0 +1,501 @@ +## All options defined here are available to all instances. +# +init_config: + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Additionally, this sets the default `service` for every log source. + # + # service: + +## Every instance is scheduled independently of the others. +# +instances: + + ## @param powerflex_gateway_url - string - required + ## URL of the PowerFlex Gateway REST API. + # + - powerflex_gateway_url: https://localhost:443 + + ## @param powerflex_username - string - optional + ## Username for PowerFlex Gateway authentication via Keycloak. + # + # powerflex_username: + + ## @param powerflex_password - string - optional + ## Password for PowerFlex Gateway authentication via Keycloak. + # + # powerflex_password: + + ## @param powerflex_client_id - string - optional - default: powerflexUI + ## OAuth2 client ID for Keycloak authentication. + # + # powerflex_client_id: powerflexUI + + ## @param collect_events - boolean - optional - default: false + ## Enable collection of CRITICAL and MAJOR severity events from the PowerFlex Gateway. + # + # collect_events: false + + ## @param collect_alerts - boolean - optional - default: false + ## Enable collection of alerts from the PowerFlex Gateway. + # + # collect_alerts: false + + ## @param max_workers - integer - optional - default: 4 + ## Maximum number of threads used to fetch per-resource statistics in parallel. + ## Increase this value for large environments with many resources. + # + # max_workers: 4 + + ## @param resource_filters - list of mappings - optional + ## Filter resources by property regex patterns and control statistics + ## collection per resource type. Exclude takes precedence over include. + ## If no filter is configured for a resource type, all resources of + ## that type are collected with statistics enabled, except for devices + ## which have statistics disabled by default. + ## + ## Each filter entry has the following fields: + ## resource: The resource type to filter. + ## property: The API property name to match against. + ## patterns: List of regex patterns to match against the property value. + ## type: Either 'include' (default) or 'exclude'. If a resource matches + ## both an include and exclude filter, it is excluded. + ## collect_statistics: When false, skip per-resource statistics API + ## calls to reduce load. Defaults to true. + ## + ## Supported resource types and common filterable properties: + ## volume: name, id, volumeType, storagePoolId, ancestorVolumeId + ## storage_pool: name, id, mediaType, protectionDomainId + ## protection_domain: name, id, protectionDomainState + ## sds: name, id, protectionDomainId, sdsState, faultSetId + ## sdc: id, sdcGuid, sdcType, sdcIp + ## device: name, id, storagePoolId, sdsId, deviceCurrentPathName + ## + ## Note: the system resource type is not filterable and will be ignored if specified. + # + # resource_filters: + # - resource: volume + # property: name + # patterns: + # - .* + # - resource: storage_pool + # property: name + # collect_statistics: true + # patterns: + # - ^prod- + # - resource: protection_domain + # property: name + # patterns: + # - .* + # - resource: sds + # property: name + # type: exclude + # patterns: + # - ^standby- + # - resource: sdc + # property: sdcType + # patterns: + # - ^AppSdc$ + # - resource: device + # property: name + # collect_statistics: false + + ## @param proxy - mapping - optional + ## This overrides the `proxy` setting in `init_config`. + ## + ## Set HTTP or HTTPS proxies for this instance. Use the `no_proxy` list + ## to specify hosts that must bypass proxies. + ## + ## The SOCKS protocol is also supported, for example: + ## + ## socks5://user:pass@host:port + ## + ## Using the scheme `socks5` causes the DNS resolution to happen on the + ## client, rather than on the proxy server. This is in line with `curl`, + ## which uses the scheme to decide whether to do the DNS resolution on + ## the client or proxy. If you want to resolve the domains on the proxy + ## server, use `socks5h` as the scheme. + # + # proxy: + # http: http://: + # https: https://: + # no_proxy: + # - + # - + + ## @param skip_proxy - boolean - optional - default: false + ## This overrides the `skip_proxy` setting in `init_config`. + ## + ## If set to `true`, this makes the check bypass any proxy + ## settings enabled and attempt to reach services directly. + # + # skip_proxy: false + + ## @param auth_type - string - optional - default: basic + ## The type of authentication to use. The available types (and related options) are: + ## ``` + ## - basic + ## |__ username + ## |__ password + ## |__ use_legacy_auth_encoding + ## - digest + ## |__ username + ## |__ password + ## - ntlm + ## |__ ntlm_domain + ## |__ password + ## - kerberos + ## |__ kerberos_auth + ## |__ kerberos_cache + ## |__ kerberos_delegate + ## |__ kerberos_force_initiate + ## |__ kerberos_hostname + ## |__ kerberos_keytab + ## |__ kerberos_principal + ## - aws + ## |__ aws_region + ## |__ aws_host + ## |__ aws_service + ## ``` + ## The `aws` auth type relies on boto3 to automatically gather AWS credentials, for example: from `.aws/credentials`. + ## Details: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#configuring-credentials + # + # auth_type: basic + + ## @param use_legacy_auth_encoding - boolean - optional - default: true + ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`. + # + # use_legacy_auth_encoding: true + + ## @param username - string - optional + ## The username to use if services are behind basic or digest auth. + # + # username: + + ## @param password - string - optional + ## The password to use if services are behind basic or NTLM auth. + # + # password: + + ## @param ntlm_domain - string - optional + ## If your services use NTLM authentication, specify + ## the domain used in the check. For NTLM Auth, append + ## the username to domain, not as the `username` parameter. + # + # ntlm_domain: \ + + ## @param kerberos_auth - string - optional - default: disabled + ## If your services use Kerberos authentication, you can specify the Kerberos + ## strategy to use between: + ## + ## - required + ## - optional + ## - disabled + ## + ## See https://github.com/requests/requests-kerberos#mutual-authentication + # + # kerberos_auth: disabled + + ## @param kerberos_cache - string - optional + ## Sets the KRB5CCNAME environment variable. + ## It should point to a credential cache with a valid TGT. + # + # kerberos_cache: + + ## @param kerberos_delegate - boolean - optional - default: false + ## Set to `true` to enable Kerberos delegation of credentials to a server that requests delegation. + ## + ## See https://github.com/requests/requests-kerberos#delegation + # + # kerberos_delegate: false + + ## @param kerberos_force_initiate - boolean - optional - default: false + ## Set to `true` to preemptively initiate the Kerberos GSS exchange and + ## present a Kerberos ticket on the initial request (and all subsequent). + ## + ## See https://github.com/requests/requests-kerberos#preemptive-authentication + # + # kerberos_force_initiate: false + + ## @param kerberos_hostname - string - optional + ## Override the hostname used for the Kerberos GSS exchange if its DNS name doesn't + ## match its Kerberos hostname, for example: behind a content switch or load balancer. + ## + ## See https://github.com/requests/requests-kerberos#hostname-override + # + # kerberos_hostname: + + ## @param kerberos_principal - string - optional + ## Set an explicit principal, to force Kerberos to look for a + ## matching credential cache for the named user. + ## + ## See https://github.com/requests/requests-kerberos#explicit-principal + # + # kerberos_principal: + + ## @param kerberos_keytab - string - optional + ## Set the path to your Kerberos key tab file. + # + # kerberos_keytab: + + ## @param auth_token - mapping - optional + ## This allows for the use of authentication information from dynamic sources. + ## Both a reader and writer must be configured. + ## + ## The available readers are: + ## + ## - type: file + ## path (required): The absolute path for the file to read from. + ## pattern: A regular expression pattern with a single capture group used to find the + ## token rather than using the entire file, for example: Your secret is (.+) + ## - type: oauth + ## url (required): The token endpoint. + ## client_id (required): The client identifier. + ## client_secret (required): The client secret. + ## basic_auth: Whether the provider expects credentials to be transmitted in + ## an HTTP Basic Auth header. The default is: false + ## options: Mapping of additional options to pass to the provider, such as the audience + ## or the scope. For example: + ## options: + ## audience: https://example.com + ## scope: read:example + ## + ## The available writers are: + ## + ## - type: header + ## name (required): The name of the field, for example: Authorization + ## value: The template value, for example `Bearer `. The default is: + ## placeholder: The substring in `value` to replace with the token, defaults to: + # + # auth_token: + # reader: + # type: + # : + # : + # writer: + # type: + # : + # : + + ## @param aws_region - string - optional + ## If your services require AWS Signature Version 4 signing, set the region. + ## + ## See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html + # + # aws_region: + + ## @param aws_host - string - optional + ## If your services require AWS Signature Version 4 signing, set the host. + ## This only needs the hostname and does not require the protocol (HTTP, HTTPS, and more). + ## For example, if connecting to https://us-east-1.amazonaws.com/, set `aws_host` to `us-east-1.amazonaws.com`. + ## + ## Note: This setting is not necessary for official integrations. + ## + ## See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html + # + # aws_host: + + ## @param aws_service - string - optional + ## If your services require AWS Signature Version 4 signing, set the service code. For a list + ## of available service codes, see https://docs.aws.amazon.com/general/latest/gr/rande.html + ## + ## Note: This setting is not necessary for official integrations. + ## + ## See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html + # + # aws_service: + + ## @param tls_verify - boolean - optional - default: true + ## Instructs the check to validate the TLS certificate of services. + # + # tls_verify: true + + ## @param tls_use_host_header - boolean - optional - default: false + ## If a `Host` header is set, this enables its use for SNI (matching against the TLS certificate CN or SAN). + # + # tls_use_host_header: false + + ## @param tls_ignore_warning - boolean - optional - default: false + ## If `tls_verify` is disabled, security warnings are logged by the check. + ## Disable those by setting `tls_ignore_warning` to true. + # + # tls_ignore_warning: false + + ## @param tls_cert - string - optional + ## The path to a single file in PEM format containing a certificate as well as any + ## number of CA certificates needed to establish the certificate's authenticity for + ## use when connecting to services. It may also contain an unencrypted private key to use. + # + # tls_cert: + + ## @param tls_private_key - string - optional + ## The unencrypted private key to use for `tls_cert` when connecting to services. This is + ## required if `tls_cert` is set and it does not already contain a private key. + # + # tls_private_key: + + ## @param tls_ca_cert - string - optional + ## The path to a file of concatenated CA certificates in PEM format or a directory + ## containing several CA certificates in PEM format. If a directory, the directory + ## must have been processed using the `openssl rehash` command. See: + ## https://www.openssl.org/docs/man3.2/man1/c_rehash.html + # + # tls_ca_cert: + + ## @param tls_protocols_allowed - list of strings - optional + ## The expected versions of TLS/SSL when fetching intermediate certificates. + ## Only `SSLv3`, `TLSv1.2`, `TLSv1.3` are allowed by default. The possible values are: + ## SSLv3 + ## TLSv1 + ## TLSv1.1 + ## TLSv1.2 + ## TLSv1.3 + # + # tls_protocols_allowed: + # - SSLv3 + # - TLSv1.2 + # - TLSv1.3 + + ## @param tls_ciphers - list of strings - optional + ## The list of ciphers suites to use when connecting to an endpoint. If not specified, + ## `ALL` ciphers are used. For list of ciphers see: + ## https://www.openssl.org/docs/man1.0.2/man1/ciphers.html + # + # tls_ciphers: + # - TLS_AES_256_GCM_SHA384 + # - TLS_CHACHA20_POLY1305_SHA256 + # - TLS_AES_128_GCM_SHA256 + + ## @param headers - mapping - optional + ## The headers parameter allows you to send specific headers with every request. + ## You can use it for explicitly specifying the host header or adding headers for + ## authorization purposes. + ## + ## This overrides any default headers. + # + # headers: + # Host: + # X-Auth-Token: + + ## @param extra_headers - mapping - optional + ## Additional headers to send with every request. + # + # extra_headers: + # Host: + # X-Auth-Token: + + ## @param timeout - number - optional - default: 10 + ## The timeout for accessing services. + ## + ## This overrides the `timeout` setting in `init_config`. + # + # timeout: 10 + + ## @param connect_timeout - number - optional + ## The connect timeout for accessing services. Defaults to `timeout`. + # + # connect_timeout: + + ## @param read_timeout - number - optional + ## The read timeout for accessing services. Defaults to `timeout`. + # + # read_timeout: + + ## @param request_size - number - optional - default: 16 + ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time. + # + # request_size: 16 + + ## @param log_requests - boolean - optional - default: false + ## Whether or not to debug log the HTTP(S) requests made, including the method and URL. + # + # log_requests: false + + ## @param persist_connections - boolean - optional - default: false + ## Whether or not to persist cookies and use connection pooling for improved performance. + # + # persist_connections: false + + ## @param allow_redirects - boolean - optional - default: true + ## Whether or not to allow URL redirection. + # + # allow_redirects: true + + ## @param tags - list of strings - optional + ## A list of tags to attach to every metric and service check emitted by this instance. + ## + ## Learn more about tagging at https://docs.datadoghq.com/tagging + # + # tags: + # - : + # - : + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Overrides any `service` defined in the `init_config` section. + # + # service: + + ## @param min_collection_interval - number - optional - default: 15 + ## This changes the collection interval of the check. For more information, see: + ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval + # + # min_collection_interval: 15 + + ## @param empty_default_hostname - boolean - optional - default: false + ## This forces the check to send metrics with no hostname. + ## + ## This is useful for cluster-level checks. + # + # empty_default_hostname: false + + ## @param metric_patterns - mapping - optional + ## A mapping of metrics to include or exclude, with each entry being a regular expression. + ## + ## Metrics defined in `exclude` will take precedence in case of overlap. + # + # metric_patterns: + # include: + # - + # exclude: + # - + +## Log Section +## +## type - required - Type of log input source (tcp / udp / file / windows_event). +## port / path / channel_path - required - Set port if type is tcp or udp. +## Set path if type is file. +## Set channel_path if type is windows_event. +## source - required - Attribute that defines which integration sent the logs. +## encoding - optional - For file specifies the file encoding. Default is utf-8. Other +## possible values are utf-16-le and utf-16-be. +## service - optional - The name of the service that generates the log. +## Overrides any `service` defined in the `init_config` section. +## tags - optional - Add tags to the collected logs. +## +## Discover Datadog log collection: https://docs.datadoghq.com/logs/log_collection/ +# +# logs: +# - type: file +# path: /opt/emc/scaleio/mdm/logs/eventLogger.log +# source: dell_powerflex +# service: +# - type: file +# path: /opt/emc/scaleio/mdm/logs/trc.0 +# source: dell_powerflex +# service: +# - type: file +# path: /opt/emc/scaleio/sds/logs/trc.0 +# source: dell_powerflex +# service: +# - type: file +# path: /opt/emc/scaleio/lia/logs/trc.0 +# source: dell_powerflex +# service: +# - type: file +# path: /opt/emc/scaleio/activemq/data/activemq.log +# source: dell_powerflex +# service: diff --git a/dell_powerflex/datadog_checks/dell_powerflex/resource_filters.py b/dell_powerflex/datadog_checks/dell_powerflex/resource_filters.py new file mode 100644 index 0000000000000..2144e61a707bb --- /dev/null +++ b/dell_powerflex/datadog_checks/dell_powerflex/resource_filters.py @@ -0,0 +1,157 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from datadog_checks.base.log import CheckLoggingAdapter + +from .constants import ( + DEVICE_RESOURCE_TYPE, + PROTECTION_DOMAIN_RESOURCE_TYPE, + SDC_RESOURCE_TYPE, + SDS_RESOURCE_TYPE, + STORAGE_POOL_RESOURCE_TYPE, + VOLUME_RESOURCE_TYPE, +) + +FILTERABLE_RESOURCE_TYPES = frozenset( + { + VOLUME_RESOURCE_TYPE, + STORAGE_POOL_RESOURCE_TYPE, + PROTECTION_DOMAIN_RESOURCE_TYPE, + SDS_RESOURCE_TYPE, + SDC_RESOURCE_TYPE, + DEVICE_RESOURCE_TYPE, + } +) + +FILTER_TYPES = frozenset({'include', 'exclude'}) + + +@dataclass(frozen=True) +class ResourceFilter: + resource: str + property_name: str + patterns: tuple[re.Pattern[str], ...] = () + filter_type: str = 'include' + collect_statistics: bool = True + + +def parse_resource_filters( + raw_filters: Sequence[Mapping[str, Any]] | None, + logger: CheckLoggingAdapter, +) -> list[ResourceFilter]: + """Parse raw filter configs into a list of validated filters.""" + if not raw_filters: + return [] + + result: list[ResourceFilter] = [] + for f in raw_filters: + resource = f.get('resource', '') + if not isinstance(resource, str) or resource not in FILTERABLE_RESOURCE_TYPES: + logger.warning('Invalid resource type in resource_filters: %s', resource) + continue + + prop = f.get('property', '') + if not isinstance(prop, str) or not prop: + logger.warning('Missing or invalid property in resource_filters for %s', resource) + continue + + filter_type = str(f.get('type', 'include')).lower() + if filter_type not in FILTER_TYPES: + logger.warning('Invalid filter type in resource_filters for %s: %s', resource, filter_type) + continue + + compiled = _compile_patterns(f.get('patterns', []), resource, logger) + collect_statistics = f.get('collect_statistics', True) + + if not compiled and collect_statistics: + logger.warning('No valid patterns in resource_filters for %s', resource) + continue + + result.append( + ResourceFilter( + resource=resource, + property_name=prop, + patterns=tuple(compiled), + filter_type=filter_type, + collect_statistics=collect_statistics, + ) + ) + + return result + + +def should_collect_resource( + resource_type: str, + entity: dict[str, Any], + filters: list[ResourceFilter], + logger: CheckLoggingAdapter, +) -> bool: + """Return True if the entity passes all filters for its resource type. + + Exclude filters take precedence over include filters. + """ + relevant = [rf for rf in filters if rf.resource == resource_type] + if not relevant: + return True + + excludes = [rf for rf in relevant if rf.filter_type == 'exclude'] + for rf in excludes: + value = entity.get(rf.property_name) + if value is None: + continue + value_str = str(value) + if any(pattern.search(value_str) for pattern in rf.patterns): + logger.debug('Skipping %s %s: matched exclude pattern on %s', resource_type, value_str, rf.property_name) + return False + + includes = [rf for rf in relevant if rf.filter_type == 'include' and rf.patterns] + if not includes: + return True + + for rf in includes: + value = entity.get(rf.property_name) + if value is None: + logger.debug('Skipping %s: property %s not found', resource_type, rf.property_name) + return False + value_str = str(value) + if not any(pattern.search(value_str) for pattern in rf.patterns): + logger.debug('Skipping %s %s: did not match any include pattern', resource_type, value_str) + return False + + return True + + +STATISTICS_DISABLED_BY_DEFAULT = frozenset({DEVICE_RESOURCE_TYPE}) + + +def should_collect_statistics( + resource_type: str, + filters: list[ResourceFilter], +) -> bool: + """Return True if statistics should be collected for this resource type.""" + relevant = [rf for rf in filters if rf.resource == resource_type] + if not relevant: + return resource_type not in STATISTICS_DISABLED_BY_DEFAULT + return all(rf.collect_statistics for rf in relevant) + + +def _compile_patterns( + raw_patterns: list[str] | None, + resource: str, + logger: CheckLoggingAdapter, +) -> list[re.Pattern[str]]: + """Compile a list of regex pattern strings.""" + compiled = [] + for p in raw_patterns or []: + if not isinstance(p, str): + continue + try: + compiled.append(re.compile(p)) + except re.error as e: + logger.warning('Invalid regex pattern in resource_filters for %s: %s (%s)', resource, p, e) + return compiled diff --git a/dell_powerflex/hatch.toml b/dell_powerflex/hatch.toml new file mode 100644 index 0000000000000..482be99e761df --- /dev/null +++ b/dell_powerflex/hatch.toml @@ -0,0 +1,23 @@ +[env.collectors.datadog-checks] +check-types = true + +[[envs.default.matrix]] +python = ["3.13"] +setup = ["caddy"] + +[[envs.default.matrix]] +python = ["3.13"] +setup = ["lab"] + +[envs.default.overrides] +name."caddy".e2e-env = { value = true } +matrix.setup.e2e-env = { value = true, if = ["lab"], env = ["POWERFLEX_GATEWAY_URL", "POWERFLEX_USERNAME", "POWERFLEX_PASSWORD"] } +matrix.setup.env-vars = [ + { key = "USE_POWERFLEX_LAB", value = "True", if = ["lab"] }, +] + +[envs.default] +e2e-env = false + +[envs.default.env-vars] +DDEV_SKIP_GENERIC_TAGS_CHECK = "true" diff --git a/dell_powerflex/metadata.csv b/dell_powerflex/metadata.csv new file mode 100644 index 0000000000000..8fa60e51a16e5 --- /dev/null +++ b/dell_powerflex/metadata.csv @@ -0,0 +1,152 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags +dell_powerflex.actual_net_capacity.in_use_in_kb,gauge,,kibibyte,,Actual net capacity in use.,0,dell_powerflex,actual net capacity in use,, +dell_powerflex.api.can_connect,gauge,,,,Whether or not the PowerFlex Gateway API is reachable.,0,dell_powerflex,api can connect,, +dell_powerflex.avg_read_latency_in_microsec,gauge,,microsecond,,Average read latency in microseconds.,0,dell_powerflex,avg read latency,, +dell_powerflex.avg_read_size_in_bytes,gauge,,byte,,Average read size in bytes.,0,dell_powerflex,avg read size,, +dell_powerflex.avg_write_latency_in_microsec,gauge,,microsecond,,Average write latency in microseconds.,0,dell_powerflex,avg write latency,, +dell_powerflex.avg_write_size_in_bytes,gauge,,byte,,Average write size in bytes.,0,dell_powerflex,avg write size,, +dell_powerflex.bck_rebuild_read_bwc.num_occured,gauge,,,,Number of backward rebuild read I/O operations.,0,dell_powerflex,bck rebuild read occurrences,, +dell_powerflex.bck_rebuild_read_bwc.num_seconds,gauge,,second,,Duration of the backward rebuild read sampling window.,0,dell_powerflex,bck rebuild read seconds,, +dell_powerflex.bck_rebuild_read_bwc.total_weight_in_kb,gauge,,kibibyte,,Total backward rebuild read bandwidth.,0,dell_powerflex,bck rebuild read bandwidth,, +dell_powerflex.bck_rebuild_write_bwc.num_occured,gauge,,,,Number of backward rebuild write I/O operations.,0,dell_powerflex,bck rebuild write occurrences,, +dell_powerflex.bck_rebuild_write_bwc.num_seconds,gauge,,second,,Duration of the backward rebuild write sampling window.,0,dell_powerflex,bck rebuild write seconds,, +dell_powerflex.bck_rebuild_write_bwc.total_weight_in_kb,gauge,,kibibyte,,Total backward rebuild write bandwidth.,0,dell_powerflex,bck rebuild write bandwidth,, +dell_powerflex.capacity.in_use_in_kb,gauge,,kibibyte,,Total capacity in use.,0,dell_powerflex,capacity in use,, +dell_powerflex.capacity_available_for_volume_allocation.in_kb,gauge,,kibibyte,,Capacity available for volume allocation.,0,dell_powerflex,available capacity,, +dell_powerflex.capacity_limit.in_kb,gauge,,kibibyte,,Capacity limit.,0,dell_powerflex,capacity limit,, +dell_powerflex.compression_ratio,gauge,,,,Compression ratio.,0,dell_powerflex,compression ratio,, +dell_powerflex.device.count,gauge,,,,Timeseries with value 1 for each device. Use 'sum by ' queries to count all devices with the tag X.,0,dell_powerflex,device count,, +dell_powerflex.exposed_capacity.in_kb,gauge,,kibibyte,,Exposed capacity.,0,dell_powerflex,exposed capacity,, +dell_powerflex.failed_capacity.in_kb,gauge,,kibibyte,,Failed capacity.,0,dell_powerflex,failed capacity,, +dell_powerflex.failed_vac.in_kb,gauge,,kibibyte,,Failed volume allocation capacity.,0,dell_powerflex,failed vac,, +dell_powerflex.fixed_read_error_count,gauge,,,,Number of fixed read errors found during background scan.,0,dell_powerflex,fixed read error count,, +dell_powerflex.fwd_rebuild_read_bwc.num_occured,gauge,,,,Number of forward rebuild read I/O operations.,0,dell_powerflex,fwd rebuild read occurrences,, +dell_powerflex.fwd_rebuild_read_bwc.num_seconds,gauge,,second,,Duration of the forward rebuild read sampling window.,0,dell_powerflex,fwd rebuild read seconds,, +dell_powerflex.fwd_rebuild_read_bwc.total_weight_in_kb,gauge,,kibibyte,,Total forward rebuild read bandwidth.,0,dell_powerflex,fwd rebuild read bandwidth,, +dell_powerflex.fwd_rebuild_write_bwc.num_occured,gauge,,,,Number of forward rebuild write I/O operations.,0,dell_powerflex,fwd rebuild write occurrences,, +dell_powerflex.fwd_rebuild_write_bwc.num_seconds,gauge,,second,,Duration of the forward rebuild write sampling window.,0,dell_powerflex,fwd rebuild write seconds,, +dell_powerflex.fwd_rebuild_write_bwc.total_weight_in_kb,gauge,,kibibyte,,Total forward rebuild write bandwidth.,0,dell_powerflex,fwd rebuild write bandwidth,, +dell_powerflex.in_use_vac.in_kb,gauge,,kibibyte,,Volume allocation capacity in use.,0,dell_powerflex,in use vac,, +dell_powerflex.inaccessible_capacity.in_kb,gauge,,kibibyte,,Inaccessible capacity.,0,dell_powerflex,inaccessible capacity,, +dell_powerflex.journaler_read_latency.num_occured,gauge,,,,Number of journaler read I/O operations in the sampling window.,0,dell_powerflex,journaler read latency occurrences,, +dell_powerflex.journaler_read_latency.num_seconds,gauge,,second,,Duration of the journaler read latency sampling window.,0,dell_powerflex,journaler read latency seconds,, +dell_powerflex.journaler_read_latency.total_weight_in_kb,gauge,,microsecond,,Total accumulated journaler read latency in the sampling window.,0,dell_powerflex,journaler read latency total weight,, +dell_powerflex.journaler_write_latency.num_occured,gauge,,,,Number of journaler write I/O operations in the sampling window.,0,dell_powerflex,journaler write latency occurrences,, +dell_powerflex.journaler_write_latency.num_seconds,gauge,,second,,Duration of the journaler write latency sampling window.,0,dell_powerflex,journaler write latency seconds,, +dell_powerflex.journaler_write_latency.total_weight_in_kb,gauge,,microsecond,,Total accumulated journaler write latency in the sampling window.,0,dell_powerflex,journaler write latency total weight,, +dell_powerflex.max_capacity.in_kb,gauge,,kibibyte,,Maximum usable capacity.,0,dell_powerflex,max capacity,, +dell_powerflex.mdm_cluster.cluster_state,gauge,,,,State of the MDM cluster. Emitted as 1 with a cluster_state tag.,0,dell_powerflex,mdm cluster state,, +dell_powerflex.mdm_cluster.good_nodes,gauge,,,,Number of MDM nodes in good health.,0,dell_powerflex,mdm good nodes,, +dell_powerflex.mdm_cluster.good_replicas,gauge,,,,Number of MDM replicas in good health.,0,dell_powerflex,mdm good replicas,, +dell_powerflex.mdm_mode,gauge,,,,MDM cluster mode. Emitted as 1 with an mdm_mode tag.,0,dell_powerflex,mdm mode,, +dell_powerflex.net_capacity.in_use_in_kb,gauge,,kibibyte,,Net capacity in use.,0,dell_powerflex,net capacity in use,, +dell_powerflex.net_user_data_capacity.in_kb,gauge,,kibibyte,,Net user data capacity.,0,dell_powerflex,net user data capacity,, +dell_powerflex.norm_rebuild_read_bwc.num_occured,gauge,,,,Number of normal rebuild read I/O operations.,0,dell_powerflex,norm rebuild read occurrences,, +dell_powerflex.norm_rebuild_read_bwc.num_seconds,gauge,,second,,Duration of the normal rebuild read sampling window.,0,dell_powerflex,norm rebuild read seconds,, +dell_powerflex.norm_rebuild_read_bwc.total_weight_in_kb,gauge,,kibibyte,,Total normal rebuild read bandwidth.,0,dell_powerflex,norm rebuild read bandwidth,, +dell_powerflex.norm_rebuild_write_bwc.num_occured,gauge,,,,Number of normal rebuild write I/O operations.,0,dell_powerflex,norm rebuild write occurrences,, +dell_powerflex.norm_rebuild_write_bwc.num_seconds,gauge,,second,,Duration of the normal rebuild write sampling window.,0,dell_powerflex,norm rebuild write seconds,, +dell_powerflex.norm_rebuild_write_bwc.total_weight_in_kb,gauge,,kibibyte,,Total normal rebuild write bandwidth.,0,dell_powerflex,norm rebuild write bandwidth,, +dell_powerflex.num_dev_errors,gauge,,,,Number of device errors.,0,dell_powerflex,device errors,, +dell_powerflex.num_of_child_volumes,gauge,,,,Number of child volumes (snapshots) of this volume.,0,dell_powerflex,child volumes,, +dell_powerflex.num_of_devices,gauge,,,,Number of devices.,0,dell_powerflex,num of devices,, +dell_powerflex.num_of_fault_sets,gauge,,,,Number of fault sets.,0,dell_powerflex,fault sets,, +dell_powerflex.num_of_mapped_sdcs,gauge,,,,Number of SDCs mapped to this volume.,0,dell_powerflex,mapped sdcs,, +dell_powerflex.num_of_mapped_to_all_volumes,gauge,,,,Number of volumes mapped to all SDCs.,0,dell_powerflex,volumes mapped to all,, +dell_powerflex.num_of_mapped_volumes,gauge,,,,Number of volumes mapped to this SDC.,0,dell_powerflex,mapped volumes,, +dell_powerflex.num_of_sds,gauge,,,,Number of SDS nodes.,0,dell_powerflex,sds count,, +dell_powerflex.num_of_snapshots,gauge,,,,Number of snapshots.,0,dell_powerflex,snapshots,, +dell_powerflex.num_of_storage_pools,gauge,,,,Number of storage pools.,0,dell_powerflex,storage pools,, +dell_powerflex.num_of_thick_base_volumes,gauge,,,,Number of thick base volumes.,0,dell_powerflex,thick base volumes,, +dell_powerflex.num_of_thin_base_volumes,gauge,,,,Number of thin base volumes.,0,dell_powerflex,thin base volumes,, +dell_powerflex.num_of_unmapped_volumes,gauge,,,,Number of unmapped volumes.,0,dell_powerflex,unmapped volumes,, +dell_powerflex.num_of_volumes,gauge,,,,Number of volumes.,0,dell_powerflex,volumes,, +dell_powerflex.num_sdr_sdc_disconnections,gauge,,,,Number of SDR-SDC disconnections.,0,dell_powerflex,sdr sdc disconnections,, +dell_powerflex.num_sds_reconnections,gauge,,,,Number of SDS reconnections.,0,dell_powerflex,sds reconnections,, +dell_powerflex.num_sds_sdr_disconnections,gauge,,,,Number of SDS-SDR disconnections.,0,dell_powerflex,sds sdr disconnections,, +dell_powerflex.overall_usage_ratio,gauge,,,,Overall capacity usage ratio.,0,dell_powerflex,overall usage ratio,, +dell_powerflex.primary_read_bwc.num_occured,gauge,,,,Number of primary read I/O operations.,0,dell_powerflex,primary read occurrences,, +dell_powerflex.primary_read_bwc.num_seconds,gauge,,second,,Duration of the primary read sampling window.,0,dell_powerflex,primary read seconds,, +dell_powerflex.primary_read_bwc.total_weight_in_kb,gauge,,kibibyte,,Total primary read bandwidth.,0,dell_powerflex,primary read bandwidth,, +dell_powerflex.primary_write_bwc.num_occured,gauge,,,,Number of primary write I/O operations.,0,dell_powerflex,primary write occurrences,, +dell_powerflex.primary_write_bwc.num_seconds,gauge,,second,,Duration of the primary write sampling window.,0,dell_powerflex,primary write seconds,, +dell_powerflex.primary_write_bwc.total_weight_in_kb,gauge,,kibibyte,,Total primary write bandwidth.,0,dell_powerflex,primary write bandwidth,, +dell_powerflex.protected_capacity.in_kb,gauge,,kibibyte,,Protected capacity.,0,dell_powerflex,protected capacity,, +dell_powerflex.protection_domain.count,gauge,,,,Timeseries with value 1 for each protection domain. Use 'sum by ' queries to count all protection domains with the tag X.,0,dell_powerflex,protection domain count,, +dell_powerflex.rebalance_read_bwc.num_occured,gauge,,,,Number of rebalance read I/O operations.,0,dell_powerflex,rebalance read occurrences,, +dell_powerflex.rebalance_read_bwc.num_seconds,gauge,,second,,Duration of the rebalance read sampling window.,0,dell_powerflex,rebalance read seconds,, +dell_powerflex.rebalance_read_bwc.total_weight_in_kb,gauge,,kibibyte,,Total rebalance read bandwidth.,0,dell_powerflex,rebalance read bandwidth,, +dell_powerflex.rebalance_wait_send_q_length,gauge,,,,Rebalance wait send queue length.,0,dell_powerflex,rebalance wait send q length,, +dell_powerflex.rebalance_write_bwc.num_occured,gauge,,,,Number of rebalance write I/O operations.,0,dell_powerflex,rebalance write occurrences,, +dell_powerflex.rebalance_write_bwc.num_seconds,gauge,,second,,Duration of the rebalance write sampling window.,0,dell_powerflex,rebalance write seconds,, +dell_powerflex.rebalance_write_bwc.total_weight_in_kb,gauge,,kibibyte,,Total rebalance write bandwidth.,0,dell_powerflex,rebalance write bandwidth,, +dell_powerflex.rebuild_wait_send_q_length,gauge,,,,Rebuild wait send queue length.,0,dell_powerflex,rebuild wait send q length,, +dell_powerflex.rfcache.io_errors,gauge,,,,Number of RF cache I/O errors.,0,dell_powerflex,rfcache io errors,, +dell_powerflex.rfcache.read_hit,gauge,,,,Number of RF cache read hits.,0,dell_powerflex,rfcache read hits,, +dell_powerflex.rfcache.read_miss,gauge,,,,Number of RF cache read misses.,0,dell_powerflex,rfcache read misses,, +dell_powerflex.rfcache.reads_pending,gauge,,,,Number of RF cache reads pending.,0,dell_powerflex,rfcache reads pending,, +dell_powerflex.rfcache.reads_received,gauge,,,,Number of read requests received by the RF cache.,0,dell_powerflex,rfcache reads received,, +dell_powerflex.rfcache.write_hit,gauge,,,,Number of RF cache write hits.,0,dell_powerflex,rfcache write hits,, +dell_powerflex.rfcache.write_miss,gauge,,,,Number of RF cache write misses.,0,dell_powerflex,rfcache write misses,, +dell_powerflex.rfcache.writes_received,gauge,,,,Number of write requests received by the RF cache.,0,dell_powerflex,rfcache writes received,, +dell_powerflex.rmcache.size_in_kb,gauge,,kibibyte,,Total size of the RAM read cache.,0,dell_powerflex,rmcache size,, +dell_powerflex.rmcache.size_in_use_in_kb,gauge,,kibibyte,,RAM read cache capacity currently in use.,0,dell_powerflex,rmcache size in use,, +dell_powerflex.rpl_total_journal_cap,gauge,,kibibyte,,Total replication journal capacity.,0,dell_powerflex,rpl total journal cap,, +dell_powerflex.rpl_used_journal_cap,gauge,,kibibyte,,Used replication journal capacity.,0,dell_powerflex,rpl used journal cap,, +dell_powerflex.sdc.count,gauge,,,,Timeseries with value 1 for each SDC. Use 'sum by ' queries to count all SDCs with the tag X.,0,dell_powerflex,sdc count,, +dell_powerflex.sds.count,gauge,,,,Timeseries with value 1 for each SDS. Use 'sum by ' queries to count all SDSs with the tag X.,0,dell_powerflex,sds count,, +dell_powerflex.secondary_read_bwc.num_occured,gauge,,,,Number of secondary read I/O operations.,0,dell_powerflex,secondary read occurrences,, +dell_powerflex.secondary_read_bwc.num_seconds,gauge,,second,,Duration of the secondary read sampling window.,0,dell_powerflex,secondary read seconds,, +dell_powerflex.secondary_read_bwc.total_weight_in_kb,gauge,,kibibyte,,Total secondary read bandwidth.,0,dell_powerflex,secondary read bandwidth,, +dell_powerflex.secondary_write_bwc.num_occured,gauge,,,,Number of secondary write I/O operations.,0,dell_powerflex,secondary write occurrences,, +dell_powerflex.secondary_write_bwc.num_seconds,gauge,,second,,Duration of the secondary write sampling window.,0,dell_powerflex,secondary write seconds,, +dell_powerflex.secondary_write_bwc.total_weight_in_kb,gauge,,kibibyte,,Total secondary write bandwidth.,0,dell_powerflex,secondary write bandwidth,, +dell_powerflex.snap_capacity.in_use_in_kb,gauge,,kibibyte,,Snapshot capacity in use.,0,dell_powerflex,snap capacity in use,, +dell_powerflex.snapshot.capacity_in_kb,gauge,,kibibyte,,Total capacity allocated for snapshots.,0,dell_powerflex,snapshot capacity,, +dell_powerflex.spare_capacity.in_kb,gauge,,kibibyte,,Spare capacity reserved.,0,dell_powerflex,spare capacity,, +dell_powerflex.storage_pool.count,gauge,,,,Timeseries with value 1 for each storage pool. Use 'sum by ' queries to count all storage pools with the tag X.,0,dell_powerflex,storage pool count,, +dell_powerflex.system.count,gauge,,,,Timeseries with value 1 for each system. Use 'sum by ' queries to count all systems with the tag X.,0,dell_powerflex,system count,, +dell_powerflex.target_read_latency.num_occured,gauge,,,,Number of target read I/O operations.,0,dell_powerflex,target read latency occurrences,, +dell_powerflex.target_read_latency.num_seconds,gauge,,second,,Duration of the target read latency sampling window.,0,dell_powerflex,target read latency seconds,, +dell_powerflex.target_read_latency.total_weight_in_kb,gauge,,microsecond,,Total accumulated target read latency in the sampling window.,0,dell_powerflex,target read latency total weight,, +dell_powerflex.target_write_latency.num_occured,gauge,,,,Number of target write I/O operations.,0,dell_powerflex,target write latency occurrences,, +dell_powerflex.target_write_latency.num_seconds,gauge,,second,,Duration of the target write latency sampling window.,0,dell_powerflex,target write latency seconds,, +dell_powerflex.target_write_latency.total_weight_in_kb,gauge,,microsecond,,Total accumulated target write latency in the sampling window.,0,dell_powerflex,target write latency total weight,, +dell_powerflex.thick_capacity.in_use_in_kb,gauge,,kibibyte,,Thick-provisioned capacity in use.,0,dell_powerflex,thick capacity in use,, +dell_powerflex.thin_capacity.in_use_in_kb,gauge,,kibibyte,,Thin-provisioned capacity in use.,0,dell_powerflex,thin capacity in use,, +dell_powerflex.total_read_bwc.num_occured,gauge,,,,Number of total read I/O operations.,0,dell_powerflex,total read occurrences,, +dell_powerflex.total_read_bwc.num_seconds,gauge,,second,,Duration of the total read sampling window.,0,dell_powerflex,total read seconds,, +dell_powerflex.total_read_bwc.total_weight_in_kb,gauge,,kibibyte,,Total read bandwidth.,0,dell_powerflex,total read bandwidth,, +dell_powerflex.total_write_bwc.num_occured,gauge,,,,Number of total write I/O operations.,0,dell_powerflex,total write occurrences,, +dell_powerflex.total_write_bwc.num_seconds,gauge,,second,,Duration of the total write sampling window.,0,dell_powerflex,total write seconds,, +dell_powerflex.total_write_bwc.total_weight_in_kb,gauge,,kibibyte,,Total write bandwidth.,0,dell_powerflex,total write bandwidth,, +dell_powerflex.unreachable_unused_capacity.in_kb,gauge,,kibibyte,,Unreachable unused capacity.,0,dell_powerflex,unreachable unused capacity,, +dell_powerflex.unused_capacity.in_kb,gauge,,kibibyte,,Unused capacity available.,0,dell_powerflex,unused capacity,, +dell_powerflex.user_data.capacity_in_kb,gauge,,kibibyte,,Capacity consumed by user data.,0,dell_powerflex,user data capacity,, +dell_powerflex.user_data_read_bwc.num_occured,gauge,,,,Number of user data read I/O operations in the sampling window.,0,dell_powerflex,user data read occurrences,, +dell_powerflex.user_data_read_bwc.num_seconds,gauge,,second,,Duration of the user data read sampling window.,0,dell_powerflex,user data read seconds,, +dell_powerflex.user_data_read_bwc.total_weight_in_kb,gauge,,kibibyte,,Total user data read bandwidth in the sampling window.,0,dell_powerflex,user data read bandwidth,, +dell_powerflex.user_data_sdc_read_latency.num_occured,gauge,,,,Number of SDC read I/O operations in the sampling window.,0,dell_powerflex,sdc read latency occurrences,, +dell_powerflex.user_data_sdc_read_latency.num_seconds,gauge,,second,,Duration of the SDC read latency sampling window.,0,dell_powerflex,sdc read latency seconds,, +dell_powerflex.user_data_sdc_read_latency.total_weight_in_kb,gauge,,microsecond,,Total accumulated SDC read latency in the sampling window.,0,dell_powerflex,sdc read latency total weight,, +dell_powerflex.user_data_sdc_trim_latency.num_occured,gauge,,,,Number of SDC trim I/O operations in the sampling window.,0,dell_powerflex,sdc trim latency occurrences,, +dell_powerflex.user_data_sdc_trim_latency.num_seconds,gauge,,second,,Duration of the SDC trim latency sampling window.,0,dell_powerflex,sdc trim latency seconds,, +dell_powerflex.user_data_sdc_trim_latency.total_weight_in_kb,gauge,,microsecond,,Total accumulated SDC trim latency in the sampling window.,0,dell_powerflex,sdc trim latency total weight,, +dell_powerflex.user_data_sdc_write_latency.num_occured,gauge,,,,Number of SDC write I/O operations in the sampling window.,0,dell_powerflex,sdc write latency occurrences,, +dell_powerflex.user_data_sdc_write_latency.num_seconds,gauge,,second,,Duration of the SDC write latency sampling window.,0,dell_powerflex,sdc write latency seconds,, +dell_powerflex.user_data_sdc_write_latency.total_weight_in_kb,gauge,,microsecond,,Total accumulated SDC write latency in the sampling window.,0,dell_powerflex,sdc write latency total weight,, +dell_powerflex.user_data_trim_bwc.num_occured,gauge,,,,Number of user data trim I/O operations in the sampling window.,0,dell_powerflex,user data trim occurrences,, +dell_powerflex.user_data_trim_bwc.num_seconds,gauge,,second,,Duration of the user data trim sampling window.,0,dell_powerflex,user data trim seconds,, +dell_powerflex.user_data_trim_bwc.total_weight_in_kb,gauge,,kibibyte,,Total user data trim bandwidth in the sampling window.,0,dell_powerflex,user data trim bandwidth,, +dell_powerflex.user_data_write_bwc.num_occured,gauge,,,,Number of user data write I/O operations in the sampling window.,0,dell_powerflex,user data write occurrences,, +dell_powerflex.user_data_write_bwc.num_seconds,gauge,,second,,Duration of the user data write sampling window.,0,dell_powerflex,user data write seconds,, +dell_powerflex.user_data_write_bwc.total_weight_in_kb,gauge,,kibibyte,,Total user data write bandwidth in the sampling window.,0,dell_powerflex,user data write bandwidth,, +dell_powerflex.vol_migration_read_bwc.num_occured,gauge,,,,Number of volume migration read I/O operations.,0,dell_powerflex,vol migration read occurrences,, +dell_powerflex.vol_migration_read_bwc.num_seconds,gauge,,second,,Duration of the volume migration read sampling window.,0,dell_powerflex,vol migration read seconds,, +dell_powerflex.vol_migration_read_bwc.total_weight_in_kb,gauge,,kibibyte,,Total volume migration read bandwidth.,0,dell_powerflex,vol migration read bandwidth,, +dell_powerflex.vol_migration_write_bwc.num_occured,gauge,,,,Number of volume migration write I/O operations.,0,dell_powerflex,vol migration write occurrences,, +dell_powerflex.vol_migration_write_bwc.num_seconds,gauge,,second,,Duration of the volume migration write sampling window.,0,dell_powerflex,vol migration write seconds,, +dell_powerflex.vol_migration_write_bwc.total_weight_in_kb,gauge,,kibibyte,,Total volume migration write bandwidth.,0,dell_powerflex,vol migration write bandwidth,, +dell_powerflex.volume.count,gauge,,,,Timeseries with value 1 for each volume. Use 'sum by ' queries to count all volumes with the tag X.,0,dell_powerflex,volume count,, +dell_powerflex.volume.sdc_mapping,gauge,,,,Indicates a volume-to-SDC mapping relationship. A value of 1 is emitted for each SDC mapped to a volume.,0,dell_powerflex,volume sdc mapping,, +dell_powerflex.volume_allocation_limit.in_kb,gauge,,kibibyte,,Volume allocation limit.,0,dell_powerflex,volume allocation limit,, diff --git a/dell_powerflex/pyproject.toml b/dell_powerflex/pyproject.toml new file mode 100644 index 0000000000000..d799f4da3ecd7 --- /dev/null +++ b/dell_powerflex/pyproject.toml @@ -0,0 +1,60 @@ +[build-system] +requires = [ + "hatchling>=0.13.0", +] +build-backend = "hatchling.build" + +[project] +name = "datadog-dell-powerflex" +description = "The Dell PowerFlex check" +readme = "README.md" +license = "BSD-3-Clause" +requires-python = ">=3.13" +keywords = [ + "datadog", + "datadog agent", + "datadog check", + "dell_powerflex", +] +authors = [ + { name = "Datadog", email = "packages@datadoghq.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Private :: Do Not Upload", + "Programming Language :: Python :: 3.13", + "Topic :: System :: Monitoring", +] +dependencies = [ + "datadog-checks-base>=37.24.0", +] +dynamic = [ + "version", +] + +[project.optional-dependencies] +deps = [] + +[project.urls] +Source = "https://github.com/DataDog/integrations-core" + +[tool.hatch.version] +path = "datadog_checks/dell_powerflex/__about__.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/datadog_checks", + "/tests", + "/manifest.json", +] + +[tool.hatch.build.targets.wheel] +include = [ + "/datadog_checks/dell_powerflex", +] +dev-mode-dirs = [ + ".", +] diff --git a/dell_powerflex/tests/__init__.py b/dell_powerflex/tests/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/dell_powerflex/tests/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/dell_powerflex/tests/common.py b/dell_powerflex/tests/common.py new file mode 100644 index 0000000000000..72873d26b7905 --- /dev/null +++ b/dell_powerflex/tests/common.py @@ -0,0 +1,534 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Any + +SYSTEM_MDM_CLUSTER_METRICS: list[dict[str, Any]] = [ + {'name': 'dell_powerflex.mdm_cluster.good_nodes', 'value': 3}, + {'name': 'dell_powerflex.mdm_cluster.good_replicas', 'value': 2}, + { + 'name': 'dell_powerflex.mdm_cluster.cluster_state', + 'value': 1, + 'extra_tags': ['cluster_state:ClusteredNormal'], + }, + {'name': 'dell_powerflex.mdm_mode', 'value': 1, 'extra_tags': ['mdm_mode:ThreeNodes']}, +] + +SYSTEM_STATS_SIMPLE_METRICS = [ + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 1048576}, + {'name': 'dell_powerflex.max_capacity.in_kb', 'value': 311270400}, + {'name': 'dell_powerflex.thick_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.thin_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.snap_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.unused_capacity.in_kb', 'value': 179488768}, + {'name': 'dell_powerflex.spare_capacity.in_kb', 'value': 130733056}, + {'name': 'dell_powerflex.fixed_read_error_count', 'value': 0}, + {'name': 'dell_powerflex.rmcache.size_in_kb', 'value': 393216}, + {'name': 'dell_powerflex.rmcache.size_in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.num_of_unmapped_volumes', 'value': 2}, + {'name': 'dell_powerflex.num_of_mapped_to_all_volumes', 'value': 0}, + {'name': 'dell_powerflex.num_of_snapshots', 'value': 2}, + {'name': 'dell_powerflex.rfcache.reads_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.writes_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_miss', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_miss', 'value': 0}, + {'name': 'dell_powerflex.user_data.capacity_in_kb', 'value': 1048576}, + {'name': 'dell_powerflex.snapshot.capacity_in_kb', 'value': 0}, + {'name': 'dell_powerflex.overall_usage_ratio', 'value': 96.0}, + {'name': 'dell_powerflex.num_sds_reconnections', 'value': 8}, + {'name': 'dell_powerflex.num_dev_errors', 'value': 0}, + {'name': 'dell_powerflex.num_sds_sdr_disconnections', 'value': 0}, + {'name': 'dell_powerflex.num_sdr_sdc_disconnections', 'value': 0}, +] + +VOLUME_STATS_SIMPLE_METRICS = [ + {'name': 'dell_powerflex.num_of_child_volumes', 'value': 1}, + {'name': 'dell_powerflex.num_of_mapped_sdcs', 'value': 1}, + {'name': 'dell_powerflex.rpl_total_journal_cap', 'value': 0}, + {'name': 'dell_powerflex.rpl_used_journal_cap', 'value': 0}, +] + +VOLUME_STATS_BWC_METRICS = [ + 'dell_powerflex.user_data_read_bwc', + 'dell_powerflex.user_data_write_bwc', + 'dell_powerflex.user_data_trim_bwc', + 'dell_powerflex.user_data_sdc_read_latency', + 'dell_powerflex.user_data_sdc_write_latency', + 'dell_powerflex.user_data_sdc_trim_latency', +] + +STORAGE_POOL_STATS_SIMPLE_METRICS = [ + {'name': 'dell_powerflex.capacity_limit.in_kb', 'value': 311270400}, + {'name': 'dell_powerflex.max_capacity.in_kb', 'value': 311270400}, + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 1048576}, + {'name': 'dell_powerflex.thick_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.thin_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.snap_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.unreachable_unused_capacity.in_kb', 'value': 0}, + {'name': 'dell_powerflex.unused_capacity.in_kb', 'value': 179488768}, + {'name': 'dell_powerflex.spare_capacity.in_kb', 'value': 130733056}, + {'name': 'dell_powerflex.capacity_available_for_volume_allocation.in_kb', 'value': 75497472}, + {'name': 'dell_powerflex.protected_capacity.in_kb', 'value': 1048576}, + {'name': 'dell_powerflex.failed_capacity.in_kb', 'value': 0}, + {'name': 'dell_powerflex.in_use_vac.in_kb', 'value': 67108864}, + {'name': 'dell_powerflex.fixed_read_error_count', 'value': 0}, + {'name': 'dell_powerflex.num_of_unmapped_volumes', 'value': 2}, + {'name': 'dell_powerflex.num_of_snapshots', 'value': 2}, + {'name': 'dell_powerflex.num_of_volumes', 'value': 4}, + {'name': 'dell_powerflex.rfcache.reads_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.writes_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_miss', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_miss', 'value': 0}, + {'name': 'dell_powerflex.user_data.capacity_in_kb', 'value': 1048576}, + {'name': 'dell_powerflex.snapshot.capacity_in_kb', 'value': 0}, + {'name': 'dell_powerflex.overall_usage_ratio', 'value': 96.0}, + {'name': 'dell_powerflex.exposed_capacity.in_kb', 'value': 0}, + {'name': 'dell_powerflex.actual_net_capacity.in_use_in_kb', 'value': 0}, +] + +STORAGE_POOL_STATS_BWC_METRICS = [ + 'dell_powerflex.user_data_read_bwc', + 'dell_powerflex.user_data_write_bwc', + 'dell_powerflex.user_data_trim_bwc', + 'dell_powerflex.user_data_sdc_read_latency', + 'dell_powerflex.user_data_sdc_write_latency', + 'dell_powerflex.user_data_sdc_trim_latency', + 'dell_powerflex.primary_read_bwc', + 'dell_powerflex.primary_write_bwc', + 'dell_powerflex.secondary_read_bwc', + 'dell_powerflex.secondary_write_bwc', + 'dell_powerflex.rebalance_read_bwc', + 'dell_powerflex.rebalance_write_bwc', + 'dell_powerflex.total_read_bwc', + 'dell_powerflex.total_write_bwc', + 'dell_powerflex.target_read_latency', + 'dell_powerflex.target_write_latency', + 'dell_powerflex.fwd_rebuild_read_bwc', + 'dell_powerflex.fwd_rebuild_write_bwc', + 'dell_powerflex.bck_rebuild_read_bwc', + 'dell_powerflex.bck_rebuild_write_bwc', + 'dell_powerflex.norm_rebuild_read_bwc', + 'dell_powerflex.norm_rebuild_write_bwc', +] + +PROTECTION_DOMAIN_STATS_SIMPLE_METRICS = [ + {'name': 'dell_powerflex.capacity_limit.in_kb', 'value': 311270400}, + {'name': 'dell_powerflex.max_capacity.in_kb', 'value': 311270400}, + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 1048576}, + {'name': 'dell_powerflex.thick_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.thin_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.snap_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.unreachable_unused_capacity.in_kb', 'value': 0}, + {'name': 'dell_powerflex.unused_capacity.in_kb', 'value': 179488768}, + {'name': 'dell_powerflex.spare_capacity.in_kb', 'value': 130733056}, + {'name': 'dell_powerflex.capacity_available_for_volume_allocation.in_kb', 'value': 75497472}, + {'name': 'dell_powerflex.volume_allocation_limit.in_kb', 'value': 864026624}, + {'name': 'dell_powerflex.protected_capacity.in_kb', 'value': 1048576}, + {'name': 'dell_powerflex.failed_capacity.in_kb', 'value': 0}, + {'name': 'dell_powerflex.in_use_vac.in_kb', 'value': 67108864}, + {'name': 'dell_powerflex.fixed_read_error_count', 'value': 0}, + {'name': 'dell_powerflex.num_of_unmapped_volumes', 'value': 2}, + {'name': 'dell_powerflex.num_of_snapshots', 'value': 2}, + {'name': 'dell_powerflex.rfcache.reads_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.writes_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_miss', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_miss', 'value': 0}, + {'name': 'dell_powerflex.net_user_data_capacity.in_kb', 'value': 524288}, + {'name': 'dell_powerflex.user_data.capacity_in_kb', 'value': 1048576}, + {'name': 'dell_powerflex.snapshot.capacity_in_kb', 'value': 0}, + {'name': 'dell_powerflex.overall_usage_ratio', 'value': 96.0}, + {'name': 'dell_powerflex.net_capacity.in_use_in_kb', 'value': 524288}, + {'name': 'dell_powerflex.rebuild_wait_send_q_length', 'value': 0}, + {'name': 'dell_powerflex.rebalance_wait_send_q_length', 'value': 0}, + {'name': 'dell_powerflex.rmcache.size_in_kb', 'value': 393216}, + {'name': 'dell_powerflex.rmcache.size_in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.num_of_thick_base_volumes', 'value': 0}, + {'name': 'dell_powerflex.num_of_thin_base_volumes', 'value': 2}, + {'name': 'dell_powerflex.num_of_sds', 'value': 3}, + {'name': 'dell_powerflex.num_of_storage_pools', 'value': 2}, + {'name': 'dell_powerflex.num_of_fault_sets', 'value': 0}, + {'name': 'dell_powerflex.exposed_capacity.in_kb', 'value': 0}, + {'name': 'dell_powerflex.actual_net_capacity.in_use_in_kb', 'value': 0}, +] + +PROTECTION_DOMAIN_STATS_BWC_METRICS = [ + 'dell_powerflex.user_data_read_bwc', + 'dell_powerflex.user_data_write_bwc', + 'dell_powerflex.user_data_trim_bwc', + 'dell_powerflex.user_data_sdc_read_latency', + 'dell_powerflex.user_data_sdc_write_latency', + 'dell_powerflex.user_data_sdc_trim_latency', + 'dell_powerflex.primary_read_bwc', + 'dell_powerflex.primary_write_bwc', + 'dell_powerflex.secondary_read_bwc', + 'dell_powerflex.secondary_write_bwc', + 'dell_powerflex.rebalance_read_bwc', + 'dell_powerflex.rebalance_write_bwc', + 'dell_powerflex.total_read_bwc', + 'dell_powerflex.total_write_bwc', + 'dell_powerflex.target_read_latency', + 'dell_powerflex.target_write_latency', + 'dell_powerflex.fwd_rebuild_read_bwc', + 'dell_powerflex.fwd_rebuild_write_bwc', + 'dell_powerflex.bck_rebuild_read_bwc', + 'dell_powerflex.bck_rebuild_write_bwc', + 'dell_powerflex.norm_rebuild_read_bwc', + 'dell_powerflex.norm_rebuild_write_bwc', + 'dell_powerflex.vol_migration_read_bwc', + 'dell_powerflex.vol_migration_write_bwc', +] + +SDS_STATS_SIMPLE_METRICS = [ + {'name': 'dell_powerflex.capacity_limit.in_kb', 'value': 103756800}, + {'name': 'dell_powerflex.max_capacity.in_kb', 'value': 103756800}, + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 349184}, + {'name': 'dell_powerflex.thick_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.thin_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.snap_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.unreachable_unused_capacity.in_kb', 'value': 0}, + {'name': 'dell_powerflex.unused_capacity.in_kb', 'value': 103407616}, + {'name': 'dell_powerflex.failed_vac.in_kb', 'value': 0}, + {'name': 'dell_powerflex.in_use_vac.in_kb', 'value': 22380544}, + {'name': 'dell_powerflex.fixed_read_error_count', 'value': 0}, + {'name': 'dell_powerflex.num_of_devices', 'value': 1}, + {'name': 'dell_powerflex.compression_ratio', 'value': 1.0}, + {'name': 'dell_powerflex.rfcache.reads_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.writes_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_miss', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_miss', 'value': 0}, + {'name': 'dell_powerflex.rfcache.reads_pending', 'value': 0}, + {'name': 'dell_powerflex.rfcache.io_errors', 'value': 0}, + {'name': 'dell_powerflex.user_data.capacity_in_kb', 'value': 349184}, + {'name': 'dell_powerflex.snapshot.capacity_in_kb', 'value': 0}, + {'name': 'dell_powerflex.rmcache.size_in_kb', 'value': 131072}, + {'name': 'dell_powerflex.rmcache.size_in_use_in_kb', 'value': 0}, +] + +SDS_STATS_BWC_METRICS = [ + 'dell_powerflex.primary_read_bwc', + 'dell_powerflex.primary_write_bwc', + 'dell_powerflex.secondary_read_bwc', + 'dell_powerflex.secondary_write_bwc', + 'dell_powerflex.total_read_bwc', + 'dell_powerflex.total_write_bwc', + 'dell_powerflex.vol_migration_read_bwc', + 'dell_powerflex.vol_migration_write_bwc', + 'dell_powerflex.target_read_latency', + 'dell_powerflex.target_write_latency', + 'dell_powerflex.user_data_read_bwc', + 'dell_powerflex.user_data_write_bwc', + 'dell_powerflex.user_data_sdc_read_latency', + 'dell_powerflex.user_data_sdc_write_latency', +] + +DEVICE_STATS_SIMPLE_METRICS = [ + {'name': 'dell_powerflex.fixed_read_error_count', 'value': 0}, + {'name': 'dell_powerflex.avg_read_size_in_bytes', 'value': 353621}, + {'name': 'dell_powerflex.avg_write_size_in_bytes', 'value': 0}, + {'name': 'dell_powerflex.avg_read_latency_in_microsec', 'value': 9596}, + {'name': 'dell_powerflex.avg_write_latency_in_microsec', 'value': 0}, + {'name': 'dell_powerflex.capacity_limit.in_kb', 'value': 103756800}, + {'name': 'dell_powerflex.max_capacity.in_kb', 'value': 103756800}, + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 349184}, + {'name': 'dell_powerflex.thick_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.thin_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.snap_capacity.in_use_in_kb', 'value': 0}, + {'name': 'dell_powerflex.failed_vac.in_kb', 'value': 0}, + {'name': 'dell_powerflex.in_use_vac.in_kb', 'value': 22380544}, + {'name': 'dell_powerflex.rfcache.reads_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.writes_received', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.read_miss', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_hit', 'value': 0}, + {'name': 'dell_powerflex.rfcache.write_miss', 'value': 0}, + {'name': 'dell_powerflex.user_data.capacity_in_kb', 'value': 349184}, + {'name': 'dell_powerflex.snapshot.capacity_in_kb', 'value': 0}, + {'name': 'dell_powerflex.compression_ratio', 'value': 1.0}, + {'name': 'dell_powerflex.inaccessible_capacity.in_kb', 'value': 0}, +] + +DEVICE_ONLY_METRICS = [ + 'dell_powerflex.avg_read_size_in_bytes', + 'dell_powerflex.avg_write_size_in_bytes', + 'dell_powerflex.avg_read_latency_in_microsec', + 'dell_powerflex.avg_write_latency_in_microsec', + 'dell_powerflex.inaccessible_capacity.in_kb', +] + +DEVICE_STATS_BWC_METRICS = [ + 'dell_powerflex.primary_read_bwc', + 'dell_powerflex.primary_write_bwc', + 'dell_powerflex.secondary_read_bwc', + 'dell_powerflex.secondary_write_bwc', + 'dell_powerflex.total_read_bwc', + 'dell_powerflex.total_write_bwc', + 'dell_powerflex.target_read_latency', + 'dell_powerflex.target_write_latency', +] + +SDC_STATS_SIMPLE_METRICS = [ + {'name': 'dell_powerflex.num_of_mapped_volumes', 'value': 2}, +] + +SDC_STATS_BWC_METRICS = [ + 'dell_powerflex.user_data_read_bwc', + 'dell_powerflex.user_data_write_bwc', + 'dell_powerflex.user_data_trim_bwc', + 'dell_powerflex.user_data_sdc_read_latency', + 'dell_powerflex.user_data_sdc_write_latency', + 'dell_powerflex.user_data_sdc_trim_latency', +] + +SYSTEM_STATS_BWC_METRICS = [ + 'dell_powerflex.user_data_read_bwc', + 'dell_powerflex.user_data_write_bwc', + 'dell_powerflex.user_data_trim_bwc', + 'dell_powerflex.user_data_sdc_read_latency', + 'dell_powerflex.user_data_sdc_write_latency', + 'dell_powerflex.user_data_sdc_trim_latency', + 'dell_powerflex.primary_read_bwc', + 'dell_powerflex.primary_write_bwc', + 'dell_powerflex.secondary_read_bwc', + 'dell_powerflex.secondary_write_bwc', + 'dell_powerflex.total_read_bwc', + 'dell_powerflex.total_write_bwc', + 'dell_powerflex.target_read_latency', + 'dell_powerflex.target_write_latency', + 'dell_powerflex.journaler_read_latency', + 'dell_powerflex.journaler_write_latency', +] + +BWC_SUFFIXES = ['num_seconds', 'total_weight_in_kb', 'num_occured'] + + +DEFAULT_GATEWAY_URL = 'https://localhost:443' +BASE_TAGS = [f'powerflex_gateway_url:{DEFAULT_GATEWAY_URL}'] + +# E2E expected metric points (excludes the dynamic powerflex_gateway_url tag). +# The test prepends that tag before asserting. + +SYSTEM_TAGS = ['system_id:1fcf40fc60c6520f', 'dell_type:system'] +POOL1_TAGS = [ + 'storage_pool_id:25155ba600000000', + 'storage_pool_name:pool1', + 'protection_domain_id:68c139ee00000000', + 'dell_type:storage_pool', +] +POOL2_TAGS = [ + 'storage_pool_id:2515d0d600000001', + 'storage_pool_name:storagepool2', + 'protection_domain_id:68c139ee00000000', + 'dell_type:storage_pool', +] +PD_TAGS = [ + 'protection_domain_id:68c139ee00000000', + 'protection_domain_name:domain1', + 'system_id:1fcf40fc60c6520f', + 'dell_type:protection_domain', +] +SDS3_TAGS = [ + 'sds_id:d1c062b700000000', + 'sds_name:SDS3', + 'protection_domain_id:68c139ee00000000', + 'fault_set_id:faultset00000001', + 'dell_type:sds', +] +SDS2_TAGS = ['sds_id:d1c062b800000001', 'sds_name:SDS2', 'protection_domain_id:68c139ee00000000', 'dell_type:sds'] +SDS1_TAGS = ['sds_id:d1c062b900000002', 'sds_name:SDS1', 'protection_domain_id:68c139ee00000000', 'dell_type:sds'] +SDC1_TAGS = [ + 'sdc_id:1b8659fd00000001', + 'sdc_guid:33FC0AF2-5180-45D8-9BDC-8E2F78CD60BF', + 'sdc_type:AppSdc', + 'sdc_ip:10.0.1.250', + 'peer_mdm_id:mdm00000001', + 'dell_type:sdc', +] +SDC2_TAGS = [ + 'sdc_id:1b8659fc00000000', + 'sdc_guid:BE3BC972-269A-4931-96B8-286BFA45C004', + 'sdc_type:AppSdc', + 'sdc_ip:10.0.1.223', + 'dell_type:sdc', +] +SDC3_TAGS = [ + 'sdc_id:1b8659fe00000002', + 'sdc_guid:46EE0B53-B823-4E68-B0B4-41A2DEC5A425', + 'sdc_type:AppSdc', + 'sdc_ip:10.0.1.228', + 'dell_type:sdc', +] +VOL_VOLUMEE_TAGS = [ + 'volume_id:c58b06e700000000', + 'volume_name:volumee', + 'volume_type:ThinProvisioned', + 'storage_pool_id:25155ba600000000', + 'dell_type:volume', +] +VOL_BIGVOLUME_TAGS = [ + 'volume_id:c58b06e800000001', + 'volume_name:bigvolume', + 'volume_type:ThinProvisioned', + 'storage_pool_id:25155ba600000000', + 'dell_type:volume', +] +VOL_SNAP1_TAGS = [ + 'volume_id:c58b06e900000002', + 'volume_name:volumee-snap-01', + 'volume_type:Snapshot', + 'storage_pool_id:25155ba600000000', + 'ancestor_volume_id:c58b06e700000000', + 'dell_type:volume', +] +VOL_SNAP2_TAGS = [ + 'volume_id:c58b06ea00000003', + 'volume_name:volumee-snap-02', + 'volume_type:Snapshot', + 'storage_pool_id:25155ba600000000', + 'ancestor_volume_id:c58b06e900000002', + 'dell_type:volume', +] +DEV1_TAGS = [ + 'device_id:f7fd7d0b00020000', + 'device_name:sds1-dev1', + 'current_path_name:/dev/sdb', + 'storage_pool_id:25155ba600000000', + 'sds_id:d1c062b900000002', + 'dell_type:device', +] +DEV2_TAGS = [ + 'device_id:f7fd7d0a00010000', + 'device_name:sds2-dev1', + 'current_path_name:/dev/sdb', + 'storage_pool_id:25155ba600000000', + 'sds_id:d1c062b800000001', + 'dell_type:device', +] +DEV3_TAGS = [ + 'device_id:f7f77d0900000000', + 'device_name:sds3-dev1', + 'current_path_name:/dev/sdb', + 'storage_pool_id:25155ba600000000', + 'sds_id:d1c062b700000000', + 'dell_type:device', +] + +ALL_EXPECTED_METRICS: list[dict] = [ + # ---- system ---- + {'name': 'dell_powerflex.system.count', 'value': 1, 'tags': SYSTEM_TAGS}, + *[ + {'name': m['name'], 'value': m['value'], 'tags': SYSTEM_TAGS + m.get('extra_tags', [])} + for m in SYSTEM_MDM_CLUSTER_METRICS + SYSTEM_STATS_SIMPLE_METRICS + ], + *[ + { + 'name': f'{p}.{s}', + 'value': 42 if p == 'dell_powerflex.user_data_read_bwc' and s == 'num_occured' else 0, + 'tags': SYSTEM_TAGS, + } + for p in SYSTEM_STATS_BWC_METRICS + for s in BWC_SUFFIXES + ], + # ---- storage_pool: pool1 ---- + {'name': 'dell_powerflex.storage_pool.count', 'value': 1, 'tags': POOL1_TAGS}, + *[{'name': m['name'], 'value': m['value'], 'tags': POOL1_TAGS} for m in STORAGE_POOL_STATS_SIMPLE_METRICS], + *[ + {'name': f'{p}.{s}', 'value': 0, 'tags': POOL1_TAGS} + for p in STORAGE_POOL_STATS_BWC_METRICS + for s in BWC_SUFFIXES + ], + # ---- storage_pool: storagepool2 ---- + {'name': 'dell_powerflex.storage_pool.count', 'value': 1, 'tags': POOL2_TAGS}, + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 0, 'tags': POOL2_TAGS}, + {'name': 'dell_powerflex.max_capacity.in_kb', 'value': 0, 'tags': POOL2_TAGS}, + {'name': 'dell_powerflex.num_of_volumes', 'value': 0, 'tags': POOL2_TAGS}, + *[ + {'name': f'{p}.{s}', 'value': 0, 'tags': POOL2_TAGS} + for p in STORAGE_POOL_STATS_BWC_METRICS + for s in BWC_SUFFIXES + ], + # ---- protection_domain: domain1 ---- + {'name': 'dell_powerflex.protection_domain.count', 'value': 1, 'tags': PD_TAGS}, + *[{'name': m['name'], 'value': m['value'], 'tags': PD_TAGS} for m in PROTECTION_DOMAIN_STATS_SIMPLE_METRICS], + *[ + {'name': f'{p}.{s}', 'value': 0, 'tags': PD_TAGS} + for p in PROTECTION_DOMAIN_STATS_BWC_METRICS + for s in BWC_SUFFIXES + ], + # ---- sds: SDS3 ---- + {'name': 'dell_powerflex.sds.count', 'value': 1, 'tags': SDS3_TAGS}, + *[{'name': m['name'], 'value': m['value'], 'tags': SDS3_TAGS} for m in SDS_STATS_SIMPLE_METRICS], + *[{'name': f'{p}.{s}', 'value': 0, 'tags': SDS3_TAGS} for p in SDS_STATS_BWC_METRICS for s in BWC_SUFFIXES], + # ---- sds: SDS2 ---- + {'name': 'dell_powerflex.sds.count', 'value': 1, 'tags': SDS2_TAGS}, + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 350208, 'tags': SDS2_TAGS}, + {'name': 'dell_powerflex.unused_capacity.in_kb', 'value': 103406592, 'tags': SDS2_TAGS}, + {'name': 'dell_powerflex.num_of_devices', 'value': 1, 'tags': SDS2_TAGS}, + *[{'name': f'{p}.{s}', 'value': 0, 'tags': SDS2_TAGS} for p in SDS_STATS_BWC_METRICS for s in BWC_SUFFIXES], + # ---- sds: SDS1 ---- + {'name': 'dell_powerflex.sds.count', 'value': 1, 'tags': SDS1_TAGS}, + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 349184, 'tags': SDS1_TAGS}, + {'name': 'dell_powerflex.unused_capacity.in_kb', 'value': 103407616, 'tags': SDS1_TAGS}, + {'name': 'dell_powerflex.num_of_devices', 'value': 1, 'tags': SDS1_TAGS}, + *[{'name': f'{p}.{s}', 'value': 0, 'tags': SDS1_TAGS} for p in SDS_STATS_BWC_METRICS for s in BWC_SUFFIXES], + # ---- sdc: SDC1 ---- + {'name': 'dell_powerflex.sdc.count', 'value': 1, 'tags': SDC1_TAGS}, + *[{'name': m['name'], 'value': m['value'], 'tags': SDC1_TAGS} for m in SDC_STATS_SIMPLE_METRICS], + *[{'name': f'{p}.{s}', 'value': 0, 'tags': SDC1_TAGS} for p in SDC_STATS_BWC_METRICS for s in BWC_SUFFIXES], + # ---- sdc: SDC2 ---- + {'name': 'dell_powerflex.sdc.count', 'value': 1, 'tags': SDC2_TAGS}, + {'name': 'dell_powerflex.num_of_mapped_volumes', 'value': 0, 'tags': SDC2_TAGS}, + *[{'name': f'{p}.{s}', 'value': 0, 'tags': SDC2_TAGS} for p in SDC_STATS_BWC_METRICS for s in BWC_SUFFIXES], + # ---- sdc: SDC3 ---- + {'name': 'dell_powerflex.sdc.count', 'value': 1, 'tags': SDC3_TAGS}, + {'name': 'dell_powerflex.num_of_mapped_volumes', 'value': 0, 'tags': SDC3_TAGS}, + *[{'name': f'{p}.{s}', 'value': 0, 'tags': SDC3_TAGS} for p in SDC_STATS_BWC_METRICS for s in BWC_SUFFIXES], + # ---- volume: volumee ---- + {'name': 'dell_powerflex.volume.count', 'value': 1, 'tags': VOL_VOLUMEE_TAGS}, + *[{'name': m['name'], 'value': m['value'], 'tags': VOL_VOLUMEE_TAGS} for m in VOLUME_STATS_SIMPLE_METRICS], + *[ + {'name': f'{p}.{s}', 'value': 0, 'tags': VOL_VOLUMEE_TAGS} + for p in VOLUME_STATS_BWC_METRICS + for s in BWC_SUFFIXES + ], + {'name': 'dell_powerflex.volume.sdc_mapping', 'value': 1, 'tags': VOL_VOLUMEE_TAGS + ['sdc_id:1b8659fd00000001']}, + # ---- volume: bigvolume ---- + {'name': 'dell_powerflex.volume.count', 'value': 1, 'tags': VOL_BIGVOLUME_TAGS}, + {'name': 'dell_powerflex.num_of_child_volumes', 'value': 0, 'tags': VOL_BIGVOLUME_TAGS}, + {'name': 'dell_powerflex.num_of_mapped_sdcs', 'value': 1, 'tags': VOL_BIGVOLUME_TAGS}, + *[ + {'name': f'{p}.{s}', 'value': 0, 'tags': VOL_BIGVOLUME_TAGS} + for p in VOLUME_STATS_BWC_METRICS + for s in BWC_SUFFIXES + ], + {'name': 'dell_powerflex.volume.sdc_mapping', 'value': 1, 'tags': VOL_BIGVOLUME_TAGS + ['sdc_id:1b8659fd00000001']}, + # ---- volume: volumee-snap-01 ---- + {'name': 'dell_powerflex.volume.count', 'value': 1, 'tags': VOL_SNAP1_TAGS}, + {'name': 'dell_powerflex.num_of_child_volumes', 'value': 1, 'tags': VOL_SNAP1_TAGS}, + {'name': 'dell_powerflex.num_of_mapped_sdcs', 'value': 0, 'tags': VOL_SNAP1_TAGS}, + # ---- volume: volumee-snap-02 ---- + {'name': 'dell_powerflex.volume.count', 'value': 1, 'tags': VOL_SNAP2_TAGS}, + {'name': 'dell_powerflex.num_of_child_volumes', 'value': 0, 'tags': VOL_SNAP2_TAGS}, + {'name': 'dell_powerflex.num_of_mapped_sdcs', 'value': 0, 'tags': VOL_SNAP2_TAGS}, + # ---- device: sds1-dev1 ---- + {'name': 'dell_powerflex.device.count', 'value': 1, 'tags': DEV1_TAGS}, + *[{'name': m['name'], 'value': m['value'], 'tags': DEV1_TAGS} for m in DEVICE_STATS_SIMPLE_METRICS], + *[{'name': f'{p}.{s}', 'value': 0, 'tags': DEV1_TAGS} for p in DEVICE_STATS_BWC_METRICS for s in BWC_SUFFIXES], + # ---- device: sds2-dev1 ---- + {'name': 'dell_powerflex.device.count', 'value': 1, 'tags': DEV2_TAGS}, + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 350208, 'tags': DEV2_TAGS}, + {'name': 'dell_powerflex.avg_read_latency_in_microsec', 'value': 12793, 'tags': DEV2_TAGS}, + *[{'name': f'{p}.{s}', 'value': 0, 'tags': DEV2_TAGS} for p in DEVICE_STATS_BWC_METRICS for s in BWC_SUFFIXES], + # ---- device: sds3-dev1 ---- + {'name': 'dell_powerflex.device.count', 'value': 1, 'tags': DEV3_TAGS}, + {'name': 'dell_powerflex.capacity.in_use_in_kb', 'value': 349184, 'tags': DEV3_TAGS}, + {'name': 'dell_powerflex.avg_read_latency_in_microsec', 'value': 10023, 'tags': DEV3_TAGS}, + *[{'name': f'{p}.{s}', 'value': 0, 'tags': DEV3_TAGS} for p in DEVICE_STATS_BWC_METRICS for s in BWC_SUFFIXES], +] diff --git a/dell_powerflex/tests/conftest.py b/dell_powerflex/tests/conftest.py new file mode 100644 index 0000000000000..493c0bbf385d1 --- /dev/null +++ b/dell_powerflex/tests/conftest.py @@ -0,0 +1,130 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import json +import os +from pathlib import Path +from urllib.parse import urlparse + +import mock +import pytest +import requests + +from datadog_checks.dev import docker_run +from datadog_checks.dev.conditions import CheckDockerLogs, CheckEndpoints +from datadog_checks.dev.docker import get_docker_hostname +from datadog_checks.dev.fs import get_here +from datadog_checks.dev.utils import find_free_port + +from .common import DEFAULT_GATEWAY_URL + +USE_POWERFLEX_LAB = os.environ.get('USE_POWERFLEX_LAB') +POWERFLEX_GATEWAY_URL = os.environ.get('POWERFLEX_GATEWAY_URL') +POWERFLEX_USERNAME = os.environ.get('POWERFLEX_USERNAME') +POWERFLEX_PASSWORD = os.environ.get('POWERFLEX_PASSWORD') + +COMPOSE_FILE = os.path.join(get_here(), 'docker', 'docker-compose.yaml') + +LAB_INSTANCE = { + 'powerflex_gateway_url': POWERFLEX_GATEWAY_URL, + 'powerflex_username': POWERFLEX_USERNAME, + 'powerflex_password': POWERFLEX_PASSWORD, + 'collect_events': True, + 'collect_alerts': True, + 'resource_filters': [ + {'resource': 'device', 'property': 'name', 'patterns': ['.*'], 'collect_statistics': True}, + ], +} + + +@pytest.fixture(scope='session') +def dd_environment(): + if USE_POWERFLEX_LAB: + yield LAB_INSTANCE + else: + port = find_free_port(get_docker_hostname()) + caddy_instance = { + 'powerflex_gateway_url': f'http://{get_docker_hostname()}:{port}', + 'powerflex_username': 'admin', + 'powerflex_password': 'password', + 'collect_events': True, + 'collect_alerts': True, + 'resource_filters': [ + {'resource': 'device', 'property': 'name', 'patterns': ['.*'], 'collect_statistics': True}, + ], + } + conditions = [ + CheckDockerLogs(identifier='powerflex-api', patterns=['server running']), + CheckEndpoints(f'http://{get_docker_hostname()}:{port}/api/version'), + ] + with docker_run(COMPOSE_FILE, conditions=conditions, env_vars={'POWERFLEX_PORT': str(port)}): + yield caddy_instance + + +@pytest.fixture +def instance(): + return { + 'powerflex_gateway_url': DEFAULT_GATEWAY_URL, + 'powerflex_username': 'admin', + 'powerflex_password': 'password', + } + + +def _get_url_path(url): + parsed = urlparse(url) + return parsed.path.replace('::', '__') + + +@pytest.fixture(scope='function') +def mock_responses(): + responses_map = {} + + def load_fixtures(): + root = os.path.join(get_here(), 'fixtures', 'GET') + for file in Path(root).rglob('*'): + if file.is_file(): + relative = file.relative_to(root) + path = '/' + str(relative.parent) if str(relative.parent) != '.' else '/' + responses_map.setdefault(path, {})[file.stem] = json.loads(file.read_text()) + + def get(url, file='response', **kwargs): + return responses_map.get(_get_url_path(url), {}).get(file) + + load_fixtures() + yield get + + +@pytest.fixture +def mock_http_call(mock_responses): + def call(url, file='response', **kwargs): + data = mock_responses(url, file=file, **kwargs) + if data is not None: + return data + resp = requests.models.Response() + resp.status_code = 404 + resp.reason = "Not Found" + resp.url = url + raise requests.exceptions.HTTPError(response=resp) + + yield call + + +@pytest.fixture +def mock_auth(monkeypatch): + def post(url, *args, **kwargs): + token_response = {'access_token': 'fake-token', 'expires_in': 300} + mock_json = mock.MagicMock(return_value=token_response) + return mock.MagicMock(json=mock_json, status_code=200) + + monkeypatch.setattr('requests.Session.post', mock.MagicMock(side_effect=post)) + + +@pytest.fixture +def mock_http_get(monkeypatch, mock_http_call, mock_auth): + def get(url, *args, **kwargs): + mock_json = mock.MagicMock(return_value=mock_http_call(url, **kwargs)) + return mock.MagicMock(json=mock_json, status_code=200) + + mock_get = mock.MagicMock(side_effect=get) + monkeypatch.setattr('requests.Session.get', mock_get) + return mock_get diff --git a/dell_powerflex/tests/docker/Caddyfile b/dell_powerflex/tests/docker/Caddyfile new file mode 100644 index 0000000000000..18dbc686dc78c --- /dev/null +++ b/dell_powerflex/tests/docker/Caddyfile @@ -0,0 +1,31 @@ +{ + debug + admin :2019 +} +:8080 { + root * /usr/share/caddy/ + + # Auth endpoint - return mock OAuth2 token + @post_token { + method POST + path /auth/realms/powerflex/protocol/openid-connect/token + } + handle @post_token { + header Content-Type application/json + respond `{"access_token":"fake-token","expires_in":300}` 200 + } + + # All GET requests - serve fixture files + # Converts :: to __ in paths (stats endpoints use Type::id format) + # and maps to GET/{path}/response.json fixture files + @get_request { + method GET + } + handle @get_request { + route { + uri replace :: __ + rewrite * /GET{path}/response.json + file_server + } + } +} diff --git a/dell_powerflex/tests/docker/docker-compose.yaml b/dell_powerflex/tests/docker/docker-compose.yaml new file mode 100644 index 0000000000000..da235a6c9abf5 --- /dev/null +++ b/dell_powerflex/tests/docker/docker-compose.yaml @@ -0,0 +1,9 @@ +services: + powerflex-api: + image: caddy:2.7 + container_name: powerflex-api + ports: + - "${POWERFLEX_PORT:-8080}:8080" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile + - ../fixtures/:/usr/share/caddy diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Device__f7f77d0900000000/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Device__f7f77d0900000000/relationships/Statistics/response.json new file mode 100644 index 0000000000000..9525510f6964f --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Device__f7f77d0900000000/relationships/Statistics/response.json @@ -0,0 +1,52 @@ +{ + "backgroundScanFixedReadErrorCount": 0, + "avgReadSizeInBytes": 353621, + "avgWriteSizeInBytes": 0, + "avgReadLatencyInMicrosec": 10023, + "avgWriteLatencyInMicrosec": 0, + "primaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "primaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "capacityLimitInKb": 103756800, + "maxCapacityInKb": 103756800, + "capacityInUseInKb": 349184, + "thickCapacityInUseInKb": 0, + "thinCapacityInUseInKb": 0, + "snapCapacityInUseInKb": 0, + "unusedCapacityInKb": 103407616, + "failedVacInKb": 0, + "inUseVacInKb": 22347776, + "rfcacheReadsReceived": 0, + "rfcacheWritesReceived": 0, + "rfacheReadHit": 0, + "rfcacheReadMiss": 0, + "rfacheWriteHit": 0, + "rfcacheWriteMiss": 0, + "userDataCapacityInKb": 349184, + "snapshotCapacityInKb": 0, + "compressionRatio": 1.0, + "inaccessibleCapacityInKb": 0, + "BackgroundScannedInMB": 379, + "backgroundScanFixedCompareErrorCount": 0, + "pendingMovingOutBckRebuildJobs": 0, + "activeMovingOutBckRebuildJobs": 0, + "pendingMovingOutFwdRebuildJobs": 0, + "activeMovingOutFwdRebuildJobs": 0, + "pendingMovingInRebalanceJobs": 0, + "activeMovingInRebalanceJobs": 0, + "pendingMovingOutNormRebuildJobs": 0, + "activeMovingOutNormRebuildJobs": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingInNormRebuildJobs": 0, + "pendingMovingInBckRebuildJobs": 0, + "activeMovingInBckRebuildJobs": 0, + "pendingMovingInFwdRebuildJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "pendingMovingOutRebalanceJobs": 0, + "activeMovingOutRebalanceJobs": 0 +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Device__f7fd7d0a00010000/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Device__f7fd7d0a00010000/relationships/Statistics/response.json new file mode 100644 index 0000000000000..c9ec6466ea250 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Device__f7fd7d0a00010000/relationships/Statistics/response.json @@ -0,0 +1,52 @@ +{ + "backgroundScanFixedReadErrorCount": 0, + "avgReadSizeInBytes": 528384, + "avgWriteSizeInBytes": 0, + "avgReadLatencyInMicrosec": 12793, + "avgWriteLatencyInMicrosec": 0, + "primaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "primaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "capacityLimitInKb": 103756800, + "maxCapacityInKb": 103756800, + "capacityInUseInKb": 350208, + "thickCapacityInUseInKb": 0, + "thinCapacityInUseInKb": 0, + "snapCapacityInUseInKb": 0, + "unusedCapacityInKb": 103406592, + "failedVacInKb": 0, + "inUseVacInKb": 22380544, + "rfcacheReadsReceived": 0, + "rfcacheWritesReceived": 0, + "rfacheReadHit": 0, + "rfcacheReadMiss": 0, + "rfacheWriteHit": 0, + "rfcacheWriteMiss": 0, + "userDataCapacityInKb": 350208, + "snapshotCapacityInKb": 0, + "compressionRatio": 1.0, + "inaccessibleCapacityInKb": 0, + "BackgroundScannedInMB": 379, + "backgroundScanFixedCompareErrorCount": 0, + "pendingMovingOutBckRebuildJobs": 0, + "activeMovingOutBckRebuildJobs": 0, + "pendingMovingOutFwdRebuildJobs": 0, + "activeMovingOutFwdRebuildJobs": 0, + "pendingMovingInRebalanceJobs": 0, + "activeMovingInRebalanceJobs": 0, + "pendingMovingOutNormRebuildJobs": 0, + "activeMovingOutNormRebuildJobs": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingInNormRebuildJobs": 0, + "pendingMovingInBckRebuildJobs": 0, + "activeMovingInBckRebuildJobs": 0, + "pendingMovingInFwdRebuildJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "pendingMovingOutRebalanceJobs": 0, + "activeMovingOutRebalanceJobs": 0 +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Device__f7fd7d0b00020000/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Device__f7fd7d0b00020000/relationships/Statistics/response.json new file mode 100644 index 0000000000000..c150644dad5cc --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Device__f7fd7d0b00020000/relationships/Statistics/response.json @@ -0,0 +1,52 @@ +{ + "backgroundScanFixedReadErrorCount": 0, + "avgReadSizeInBytes": 353621, + "avgWriteSizeInBytes": 0, + "avgReadLatencyInMicrosec": 9596, + "avgWriteLatencyInMicrosec": 0, + "primaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "primaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "capacityLimitInKb": 103756800, + "maxCapacityInKb": 103756800, + "capacityInUseInKb": 349184, + "thickCapacityInUseInKb": 0, + "thinCapacityInUseInKb": 0, + "snapCapacityInUseInKb": 0, + "unusedCapacityInKb": 103407616, + "failedVacInKb": 0, + "inUseVacInKb": 22380544, + "rfcacheReadsReceived": 0, + "rfcacheWritesReceived": 0, + "rfacheReadHit": 0, + "rfcacheReadMiss": 0, + "rfacheWriteHit": 0, + "rfcacheWriteMiss": 0, + "userDataCapacityInKb": 349184, + "snapshotCapacityInKb": 0, + "compressionRatio": 1.0, + "inaccessibleCapacityInKb": 0, + "BackgroundScannedInMB": 379, + "backgroundScanFixedCompareErrorCount": 0, + "pendingMovingOutBckRebuildJobs": 0, + "activeMovingOutBckRebuildJobs": 0, + "pendingMovingOutFwdRebuildJobs": 0, + "activeMovingOutFwdRebuildJobs": 0, + "pendingMovingInRebalanceJobs": 0, + "activeMovingInRebalanceJobs": 0, + "pendingMovingOutNormRebuildJobs": 0, + "activeMovingOutNormRebuildJobs": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingInNormRebuildJobs": 0, + "pendingMovingInBckRebuildJobs": 0, + "activeMovingInBckRebuildJobs": 0, + "pendingMovingInFwdRebuildJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "pendingMovingOutRebalanceJobs": 0, + "activeMovingOutRebalanceJobs": 0 +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/ProtectionDomain__68c139ee00000000/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/ProtectionDomain__68c139ee00000000/relationships/Statistics/response.json new file mode 100644 index 0000000000000..6f2d4505fdf8a --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/ProtectionDomain__68c139ee00000000/relationships/Statistics/response.json @@ -0,0 +1,330 @@ +{ + "backgroundScanFixedReadErrorCount": 0, + "pendingMovingOutBckRebuildJobs": 0, + "rfcachePoolWritePending": 0, + "degradedHealthyCapacityInKb": 0, + "rplCgRpoCompliance": 100, + "activeMovingOutFwdRebuildJobs": 0, + "numSdsSdrDisconnections": 0, + "rfcachePoolWritePendingG1Sec": 0, + "bckRebuildWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "netFglUncompressedDataSizeInKb": 0, + "primaryReadFromDevBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "BackgroundScannedInMB": 1139, + "protectedMaintenanceModeWaitSendQLength": 0, + "maxUserDataCapacityInKb": 180537344, + "rplMaxDstCapacity": 0, + "rfcacheReadsSkippedAlignedSizeTooLarge": 0, + "rfcachePoolSize": 0, + "pendingMovingInRebalanceJobs": 0, + "fglMetadataCacheHitrate": 0.0, + "rfcacheWritesSkippedHeavyLoad": 0, + "rfcachePoolPagesInuse": 0, + "unusedCapacityInKb": 179488768, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheFdAvgWriteTime": 0, + "rmcacheEntryEvictionCount": 0, + "totalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "journalerReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "sdtDataReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "persistentChecksumCapacityInKb": 153600, + "rmPendingAllocatedInKb": 0, + "sdtHostReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheIosOutstanding": 0, + "sdtDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rmcacheBigBlockEvictionSizeCountInKb": 0, + "numRpoViolatingRplCgsSrc": 0, + "numOfMappedToAllVolumes": 0, + "capacityAvailableForVolumeAllocationInKb": 75497472, + "netThinUserDataCapacityInKb": 524288, + "backgroundScanFixedCompareErrorCount": 0, + "volMigrationWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "thinAndSnapshotRatio": 96.0, + "rebuildPerReceiveJobNetThrottlingInKbps": 0, + "fglUserDataCapacityInKb": 0, + "pendingMovingInEnterProtectedMaintenanceModeJobs": 0, + "rcgRemoteWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rmcache32kbEntryCount": 0, + "rfcachePoolEvictions": 0, + "rfcachePoolNumCacheDevs": 0, + "activeMovingInNormRebuildJobs": 0, + "rplUsedJournalCapacityDst": {"userDataSizeInKB": 0, "userDataSizeFoldedInKB": null}, + "journalerWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "aggregateCompressionLevel": "Uncompressed", + "rfcacheFdWriteTimeGreater500Millis": 0, + "sdtDataWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetOtherLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "journalerReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "netUserDataCapacityInKb": 524288, + "rmcacheSkipCountCacheAllBusy": 0, + "rfcachePoolNumSrcDevs": 0, + "rfcacheFdMonitorErrorStuckIo": 0, + "rplRemoteUserBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "pendingMovingOutExitProtectedMaintenanceModeJobs": 0, + "rplTransmitLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "overallUsageRatio": 96.0, + "volMigrationReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheReadsSkippedInternalError": 0, + "rfcachePoolWritePendingG500Micro": 0, + "pendingMovingInBckRebuildJobs": 0, + "netCapacityInUseNoOverheadInKb": 524288, + "sdtIds": [], + "activeBckRebuildCapacityInKb": 0, + "rebalanceCapacityInKb": 0, + "totalVolumeAllocationLimitInKb": 897581056, + "pendingMovingInExitProtectedMaintenanceModeJobs": 0, + "rfcachePoolInLowMemoryCondition": 0, + "rfcacheReadsSkippedLowResources": 0, + "rplReceiveLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rplJournalCapAllowed": 0, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "thinCapacityInUseInKb": 0, + "rplReceiveBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcachePoolLowResourcesInitiatedPassthroughMode": 0, + "rfcachePoolWritePendingG10Millis": 0, + "activeMovingInEnterProtectedMaintenanceModeJobs": 0, + "rfcacheWritesSkippedInternalError": 0, + "rcgRemoteReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcachePoolWriteHit": 0, + "rmcache128kbEntryCount": 0, + "rfcacheWritesSkippedCacheMiss": 0, + "netUserDataCapacityNoTrimInKb": 524288, + "rfcacheFdReadTimeGreater5Sec": 0, + "numOfFaultSets": 0, + "degradedFailedCapacityInKb": 0, + "activeNormRebuildCapacityInKb": 0, + "fglSparesInKb": 0, + "snapCapacityInUseInKb": 0, + "compressionRatio": 1.0, + "rfcacheFdIoErrors": 0, + "rfcacheWriteMiss": 0, + "primaryReadFromRmcacheBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numRpoViolatingRplCgsDest": 0, + "userDataCapacityNoTrimInKb": 1048576, + "rfacheReadHit": 0, + "rfcachePooIosOutstanding": 0, + "storagePoolIds": ["25155ba600000000", "2515d0d600000001"], + "compressedDataCompressionRatio": 0.0, + "rplUsedJournalCap": 0, + "rcgLocalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "pendingMovingCapacityInKb": 0, + "numOfSnapshots": 2, + "rcgLocalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rmcacheBigBlockEvictionCount": 0, + "pendingFwdRebuildCapacityInKb": 0, + "rmcacheNoEvictionCount": 0, + "rmcacheCurrNumOf128kbEntries": 0, + "tempCapacityInKb": 0, + "normRebuildCapacityInKb": 0, + "rfcachePoolReadPendingG1Millis": 0, + "numOfAccelerationPools": 1, + "logWrittenBlocksInKb": 0, + "rmcacheSizeInUseInKb": 0, + "primaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfThickBaseVolumes": 0, + "rfcachePoolReadPendingG10Millis": 0, + "enterProtectedMaintenanceModeReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "activeRebalanceCapacityInKb": 0, + "numOfReplicationJournalVolumes": 0, + "rfcacheReadsSkippedLockIos": 0, + "unreachableUnusedCapacityInKb": 0, + "netProvisionedAddressesInKb": 524288, + "rmcache8kbEntryCount": 0, + "rfcachePoolReadPendingG500Micro": 0, + "rplMaxDstCapacityRatio": 50, + "sdsIds": ["d1c062b900000002", "d1c062b800000001", "d1c062b700000000"], + "trimmedUserDataCapacityInKb": 0, + "rplCapacityAlertLevel": "invalid", + "provisionedAddressesInKb": 1048576, + "numOfVolumesInDeletion": 0, + "pendingMovingOutFwdRebuildJobs": 0, + "maxCapacityInKb": 311270400, + "rmcacheSkipCountLargeIo": 0, + "rmPendingThickInKb": 0, + "protectedCapacityInKb": 1048576, + "secondaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "normRebuildReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "thinCapacityAllocatedInKb": 67108864, + "netFglUserDataCapacityInKb": 0, + "metadataOverheadInKb": 0, + "thinCapacityAllocatedInKm": 67108864, + "rebalanceWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rmcacheCurrNumOf8kbEntries": 0, + "primaryVacInKb": 33554432, + "sdtHostWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryVacInKb": 33554432, + "netSnapshotCapacityInKb": 0, + "rplTotalJournalCap": 0, + "rfcachePoolWriteMiss": 0, + "rfcachePoolReadPendingG1Sec": 0, + "failedCapacityInKb": 0, + "netMetadataOverheadInKb": 0, + "rmcache4kbEntryCount": 0, + "rfcachePoolWritePendingG1Millis": 0, + "rebalanceWaitSendQLength": 0, + "rfcacheFdReadTimeGreater1Min": 0, + "rebalancePerReceiveJobNetThrottlingInKbps": 25600, + "activeMovingOutBckRebuildJobs": 0, + "rfcacheReadsFromCache": 0, + "rfcacheFdReadTimeGreater1Sec": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingOutEnterProtectedMaintenanceModeJobs": 0, + "rmcache64kbEntryCount": 0, + "enterProtectedMaintenanceModeCapacityInKb": 0, + "failedVacInKb": 0, + "primaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "fglUncompressedDataSizeInKb": 0, + "fglCompressedDataSizeInKb": 0, + "pendingRebalanceCapacityInKb": 0, + "rfcacheAvgReadTime": 0, + "semiProtectedCapacityInKb": 0, + "pendingMovingOutEnterProtectedMaintenanceModeJobs": 0, + "rfcachePoolSourceIdMismatch": 0, + "mgUserDdataCcapacityInKb": 1048576, + "netMgUserDataCapacityInKb": 524288, + "snapshotCapacityInKb": 0, + "rfcacheFdAvgReadTime": 0, + "fwdRebuildReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheWritesReceived": 0, + "rplSlimModeThreshold": 80, + "netUnusedCapacityInKb": 89744384, + "rfcachePoolSuspendedIos": 0, + "thinUserDataCapacityInKb": 1048576, + "protectedVacInKb": 67108864, + "activeMovingRebalanceJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "bckRebuildCapacityInKb": 0, + "sdrIds": [], + "netTrimmedUserDataCapacityInKb": 0, + "pendingMovingRebalanceJobs": 0, + "numOfMarkedVolumesForReplication": 0, + "faultSetIds": [], + "degradedHealthyVacInKb": 0, + "rplCgIds": [], + "rfcachePoolLockTimeGreater1Sec": 0, + "rplRemoteApplyBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "semiProtectedVacInKb": 0, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "pendingBckRebuildCapacityInKb": 0, + "rmcacheCurrNumOf4kbEntries": 0, + "rplLocalApplyBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "capacityLimitInKb": 311270400, + "activeMovingCapacityInKb": 0, + "targetWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "pendingExitProtectedMaintenanceModeCapacityInKb": 0, + "numOfRplCgs": 0, + "rfcacheIosSkipped": 0, + "rfcacheFdWriteTimeGreater5Sec": 0, + "exitProtectedMaintenanceModeReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inMaintenanceVacInKb": 0, + "netFglSparesInKb": 0, + "rfcacheReadsSkipped": 0, + "rfcachePoolReadHit": 0, + "rebuildWaitSendQLength": 0, + "activeExitProtectedMaintenanceModeCapacityInKb": 0, + "numOfUnmappedVolumes": 2, + "activeMovingOutExitProtectedMaintenanceModeJobs": 0, + "rmcacheCurrNumOf64kbEntries": 0, + "volumeAddressSpaceInKb": 50331648, + "tempCapacityVacInKb": 0, + "rfcacheWritesSkippedMaxIoSize": 0, + "netMaxUserDataCapacityInKb": 90268672, + "rfacheWriteHit": 0, + "atRestCapacityInKb": 524288, + "bckRebuildReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheSourceDeviceWrites": 0, + "rfcacheFdInlightReads": 0, + "spareCapacityInKb": 130733056, + "numOfSdt": 0, + "enterProtectedMaintenanceModeWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheIoErrors": 0, + "numOfSds": 3, + "normRebuildWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfSdr": 0, + "capacityInUseInKb": 1048576, + "rebalanceReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rmcacheSkipCountUnaligned4kbIo": 0, + "rfcacheReadsSkippedMaxIoSize": 0, + "activeMovingInExitProtectedMaintenanceModeJobs": 0, + "secondaryReadFromDevBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcachePoolSuspendedPequestsRedundantSearchs": 0, + "secondaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryReadFromRmcacheBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheWritesSkippedStuckIo": 0, + "numOfStoragePools": 2, + "rfcachePoolCachePages": 0, + "accelerationPoolIds": ["af4a46c300000000"], + "inMaintenanceCapacityInKb": 0, + "sdtDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "sdtDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rplSasBarriersBacklogSize": 0, + "netFglCompressedDataSizeInKb": 0, + "vtreeMigrationWaitSendQLength": 0, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inUseVacInKb": 67108864, + "fwdRebuildCapacityInKb": 0, + "thickCapacityInUseInKb": 0, + "sdtHostTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "activeMovingInRebalanceJobs": 0, + "backgroundScanReadErrorCount": 0, + "rmcacheCurrNumOf32kbEntries": 0, + "sdtDataTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheWritesSkippedLowResources": 0, + "rplApplyLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "capacityInUseNoOverheadInKb": 1048576, + "rfcacheFdCacheOverloaded": 0, + "rplLocalUserBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rmcache16kbEntryCount": 0, + "exitProtectedMaintenanceModeWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rmcacheEntryEvictionSizeCountInKb": 0, + "rfcacheSkippedUnlinedWrite": 0, + "netCapacityInUseInKb": 524288, + "rfcacheAvgWriteTime": 0, + "journalerWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "pendingNormRebuildCapacityInKb": 0, + "vtreeMigrationPerReceiveJobNetThrottlingInKbps": 30720, + "rfcacheFdReadTimeGreater500Millis": 0, + "pendingMovingOutNormrebuildJobs": 0, + "rfcacheSourceDeviceReads": 0, + "rmcacheCurrNumOf16kbEntries": 0, + "initialCopyProgress": 0.0, + "rfcacheReadsPending": 0, + "volumeAllocationLimitInKb": 864026624, + "rfcacheReadsSkippedHeavyLoad": 0, + "fwdRebuildWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheFdInlightWrites": 0, + "protectedMaintenanceModePerReceiveJobNetThrottlingInKbps": 0, + "rfcacheReadMiss": 0, + "targetReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataCapacityInKb": 1048576, + "rfcacheFdReadsReceived": 0, + "activeMovingInBckRebuildJobs": 0, + "movingCapacityInKb": 0, + "activeEnterProtectedMaintenanceModeCapacityInKb": 0, + "backgroundScanCompareErrorCount": 0, + "pendingMovingInFwdRebuildJobs": 0, + "rfcacheReadsReceived": 0, + "rfcachePoolReadsPending": 0, + "pendingEnterProtectedMaintenanceModeCapacityInKb": 0, + "snapCapacityInUseOccupiedInKb": 0, + "vtreeAddresSpaceInKb": 33554432, + "activeFwdRebuildCapacityInKb": 0, + "rfcacheReadsSkippedStuckIo": 0, + "activeMovingOutNormRebuildJobs": 0, + "numOfVtreeMigrationsInPd": 0, + "rfcacheFdWritesReceived": 0, + "rplTransmitBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rmcacheSizeInKb": 393216, + "rfcacheFdWriteTimeGreater1Min": 0, + "rfcacheFdWriteTimeGreater1Sec": 0, + "rfcacheWritePending": 0, + "numOfThinBaseVolumes": 2, + "degradedFailedVacInKb": 0, + "rfcachePoolIoTimeGreater1Min": 0, + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcachePoolReadMiss": 0, + "exposedCapacityInKb": 0, + "ActualNetCapacityInUseInKb": 0 +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Sdc__1b8659fc00000000/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Sdc__1b8659fc00000000/relationships/Statistics/response.json new file mode 100644 index 0000000000000..95f13bc8adbe3 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Sdc__1b8659fc00000000/relationships/Statistics/response.json @@ -0,0 +1,10 @@ +{ + "numOfMappedVolumes": 0, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "volumeIds": [], + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0} +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Sdc__1b8659fd00000001/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Sdc__1b8659fd00000001/relationships/Statistics/response.json new file mode 100644 index 0000000000000..e3bfd983b3e49 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Sdc__1b8659fd00000001/relationships/Statistics/response.json @@ -0,0 +1,10 @@ +{ + "numOfMappedVolumes": 2, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "volumeIds": ["c58b06e800000001", "c58b06e700000000"], + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0} +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Sdc__1b8659fe00000002/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Sdc__1b8659fe00000002/relationships/Statistics/response.json new file mode 100644 index 0000000000000..95f13bc8adbe3 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Sdc__1b8659fe00000002/relationships/Statistics/response.json @@ -0,0 +1,10 @@ +{ + "numOfMappedVolumes": 0, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "volumeIds": [], + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0} +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Sds__d1c062b700000000/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Sds__d1c062b700000000/relationships/Statistics/response.json new file mode 100644 index 0000000000000..1ad36e8cb0948 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Sds__d1c062b700000000/relationships/Statistics/response.json @@ -0,0 +1,72 @@ +{ + "backgroundScanFixedReadErrorCount": 0, + "rfcachePoolWritePending": 0, + "rfcachePoolWritePendingG1Sec": 0, + "primaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "primaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "volMigrationReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "volMigrationWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "capacityLimitInKb": 103756800, + "maxCapacityInKb": 103756800, + "capacityInUseInKb": 349184, + "thickCapacityInUseInKb": 0, + "thinCapacityInUseInKb": 0, + "snapCapacityInUseInKb": 0, + "unreachableUnusedCapacityInKb": 0, + "unusedCapacityInKb": 103407616, + "failedVacInKb": 0, + "inUseVacInKb": 22380544, + "numOfDevices": 1, + "compressionRatio": 1.0, + "rfcacheReadsReceived": 0, + "rfcacheWritesReceived": 0, + "rfacheReadHit": 0, + "rfcacheReadMiss": 0, + "rfacheWriteHit": 0, + "rfcacheWriteMiss": 0, + "rfcacheReadsPending": 0, + "rfcacheIoErrors": 0, + "rfcacheIosOutstanding": 0, + "rfcacheIosSkipped": 0, + "userDataCapacityInKb": 349184, + "snapshotCapacityInKb": 0, + "rmcacheSizeInKb": 131072, + "rmcacheSizeInUseInKb": 0, + "rmcacheEntryEvictionCount": 0, + "rmcacheBigBlockEvictionSizeCountInKb": 0, + "rmcacheNoEvictionCount": 0, + "rmcacheCurrNumOf4kbEntries": 0, + "rmcacheCurrNumOf8kbEntries": 0, + "rmcacheCurrNumOf16kbEntries": 0, + "rmcacheCurrNumOf32kbEntries": 0, + "rmcacheCurrNumOf64kbEntries": 0, + "rmcacheCurrNumOf128kbEntries": 0, + "BackgroundScannedInMB": 379, + "backgroundScanFixedCompareErrorCount": 0, + "pendingMovingOutBckRebuildJobs": 0, + "activeMovingOutBckRebuildJobs": 0, + "pendingMovingOutFwdRebuildJobs": 0, + "activeMovingOutFwdRebuildJobs": 0, + "pendingMovingInRebalanceJobs": 0, + "activeMovingInRebalanceJobs": 0, + "pendingMovingOutNormRebuildJobs": 0, + "activeMovingOutNormRebuildJobs": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingInNormRebuildJobs": 0, + "pendingMovingInBckRebuildJobs": 0, + "activeMovingInBckRebuildJobs": 0, + "pendingMovingInFwdRebuildJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "pendingMovingOutRebalanceJobs": 0, + "activeMovingOutRebalanceJobs": 0 +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Sds__d1c062b800000001/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Sds__d1c062b800000001/relationships/Statistics/response.json new file mode 100644 index 0000000000000..4436c97b63bea --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Sds__d1c062b800000001/relationships/Statistics/response.json @@ -0,0 +1,72 @@ +{ + "backgroundScanFixedReadErrorCount": 0, + "rfcachePoolWritePending": 0, + "rfcachePoolWritePendingG1Sec": 0, + "primaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "primaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "volMigrationReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "volMigrationWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "capacityLimitInKb": 103756800, + "maxCapacityInKb": 103756800, + "capacityInUseInKb": 350208, + "thickCapacityInUseInKb": 0, + "thinCapacityInUseInKb": 0, + "snapCapacityInUseInKb": 0, + "unreachableUnusedCapacityInKb": 0, + "unusedCapacityInKb": 103406592, + "failedVacInKb": 0, + "inUseVacInKb": 22380544, + "numOfDevices": 1, + "compressionRatio": 1.0, + "rfcacheReadsReceived": 0, + "rfcacheWritesReceived": 0, + "rfacheReadHit": 0, + "rfcacheReadMiss": 0, + "rfacheWriteHit": 0, + "rfcacheWriteMiss": 0, + "rfcacheReadsPending": 0, + "rfcacheIoErrors": 0, + "rfcacheIosOutstanding": 0, + "rfcacheIosSkipped": 0, + "userDataCapacityInKb": 350208, + "snapshotCapacityInKb": 0, + "rmcacheSizeInKb": 131072, + "rmcacheSizeInUseInKb": 0, + "rmcacheEntryEvictionCount": 0, + "rmcacheBigBlockEvictionSizeCountInKb": 0, + "rmcacheNoEvictionCount": 0, + "rmcacheCurrNumOf4kbEntries": 0, + "rmcacheCurrNumOf8kbEntries": 0, + "rmcacheCurrNumOf16kbEntries": 0, + "rmcacheCurrNumOf32kbEntries": 0, + "rmcacheCurrNumOf64kbEntries": 0, + "rmcacheCurrNumOf128kbEntries": 0, + "BackgroundScannedInMB": 379, + "backgroundScanFixedCompareErrorCount": 0, + "pendingMovingOutBckRebuildJobs": 0, + "activeMovingOutBckRebuildJobs": 0, + "pendingMovingOutFwdRebuildJobs": 0, + "activeMovingOutFwdRebuildJobs": 0, + "pendingMovingInRebalanceJobs": 0, + "activeMovingInRebalanceJobs": 0, + "pendingMovingOutNormRebuildJobs": 0, + "activeMovingOutNormRebuildJobs": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingInNormRebuildJobs": 0, + "pendingMovingInBckRebuildJobs": 0, + "activeMovingInBckRebuildJobs": 0, + "pendingMovingInFwdRebuildJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "pendingMovingOutRebalanceJobs": 0, + "activeMovingOutRebalanceJobs": 0 +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Sds__d1c062b900000002/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Sds__d1c062b900000002/relationships/Statistics/response.json new file mode 100644 index 0000000000000..1ad36e8cb0948 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Sds__d1c062b900000002/relationships/Statistics/response.json @@ -0,0 +1,72 @@ +{ + "backgroundScanFixedReadErrorCount": 0, + "rfcachePoolWritePending": 0, + "rfcachePoolWritePendingG1Sec": 0, + "primaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "primaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "volMigrationReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "volMigrationWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "targetWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "capacityLimitInKb": 103756800, + "maxCapacityInKb": 103756800, + "capacityInUseInKb": 349184, + "thickCapacityInUseInKb": 0, + "thinCapacityInUseInKb": 0, + "snapCapacityInUseInKb": 0, + "unreachableUnusedCapacityInKb": 0, + "unusedCapacityInKb": 103407616, + "failedVacInKb": 0, + "inUseVacInKb": 22380544, + "numOfDevices": 1, + "compressionRatio": 1.0, + "rfcacheReadsReceived": 0, + "rfcacheWritesReceived": 0, + "rfacheReadHit": 0, + "rfcacheReadMiss": 0, + "rfacheWriteHit": 0, + "rfcacheWriteMiss": 0, + "rfcacheReadsPending": 0, + "rfcacheIoErrors": 0, + "rfcacheIosOutstanding": 0, + "rfcacheIosSkipped": 0, + "userDataCapacityInKb": 349184, + "snapshotCapacityInKb": 0, + "rmcacheSizeInKb": 131072, + "rmcacheSizeInUseInKb": 0, + "rmcacheEntryEvictionCount": 0, + "rmcacheBigBlockEvictionSizeCountInKb": 0, + "rmcacheNoEvictionCount": 0, + "rmcacheCurrNumOf4kbEntries": 0, + "rmcacheCurrNumOf8kbEntries": 0, + "rmcacheCurrNumOf16kbEntries": 0, + "rmcacheCurrNumOf32kbEntries": 0, + "rmcacheCurrNumOf64kbEntries": 0, + "rmcacheCurrNumOf128kbEntries": 0, + "BackgroundScannedInMB": 379, + "backgroundScanFixedCompareErrorCount": 0, + "pendingMovingOutBckRebuildJobs": 0, + "activeMovingOutBckRebuildJobs": 0, + "pendingMovingOutFwdRebuildJobs": 0, + "activeMovingOutFwdRebuildJobs": 0, + "pendingMovingInRebalanceJobs": 0, + "activeMovingInRebalanceJobs": 0, + "pendingMovingOutNormRebuildJobs": 0, + "activeMovingOutNormRebuildJobs": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingInNormRebuildJobs": 0, + "pendingMovingInBckRebuildJobs": 0, + "activeMovingInBckRebuildJobs": 0, + "pendingMovingInFwdRebuildJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "pendingMovingOutRebalanceJobs": 0, + "activeMovingOutRebalanceJobs": 0 +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/StoragePool__25155ba600000000/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/StoragePool__25155ba600000000/relationships/Statistics/response.json new file mode 100644 index 0000000000000..1169b85b953fd --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/StoragePool__25155ba600000000/relationships/Statistics/response.json @@ -0,0 +1,220 @@ +{ + "backgroundScanFixedReadErrorCount": 0, + "pendingMovingOutBckRebuildJobs": 0, + "degradedHealthyCapacityInKb": 0, + "activeMovingOutFwdRebuildJobs": 0, + "bckRebuildWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "netFglUncompressedDataSizeInKb": 0, + "primaryReadFromDevBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "BackgroundScannedInMB": 1139, + "volumeIds": ["c58b06e700000000", "c58b06e900000002", "c58b06ea00000003", "c58b06e800000001"], + "maxUserDataCapacityInKb": 180537344, + "persistentChecksumBuilderProgress": 100.0, + "rfcacheReadsSkippedAlignedSizeTooLarge": 0, + "pendingMovingInRebalanceJobs": 0, + "rfcacheWritesSkippedHeavyLoad": 0, + "unusedCapacityInKb": 179488768, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfDeviceAtFaultRebuilds": 0, + "totalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "persistentChecksumCapacityInKb": 153600, + "rmPendingAllocatedInKb": 0, + "numOfVolumes": 4, + "rfcacheIosOutstanding": 0, + "numOfMappedToAllVolumes": 0, + "capacityAvailableForVolumeAllocationInKb": 75497472, + "netThinUserDataCapacityInKb": 524288, + "backgroundScanFixedCompareErrorCount": 0, + "volMigrationWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "thinAndSnapshotRatio": 96.0, + "pendingMovingInEnterProtectedMaintenanceModeJobs": 0, + "fglUserDataCapacityInKb": 0, + "activeMovingInNormRebuildJobs": 0, + "aggregateCompressionLevel": "Uncompressed", + "targetOtherLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "netUserDataCapacityInKb": 524288, + "pendingMovingOutExitProtectedMaintenanceModeJobs": 0, + "overallUsageRatio": 96.0, + "volMigrationReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "pendingMovingInBckRebuildJobs": 0, + "netCapacityInUseNoOverheadInKb": 524288, + "rfcacheReadsSkippedInternalError": 0, + "activeBckRebuildCapacityInKb": 0, + "rebalanceCapacityInKb": 0, + "totalVolumeAllocationLimitInKb": 897581056, + "pendingMovingInExitProtectedMaintenanceModeJobs": 0, + "rfcacheReadsSkippedLowResources": 0, + "rplJournalCapAllowed": 0, + "thinCapacityInUseInKb": 0, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "activeMovingInEnterProtectedMaintenanceModeJobs": 0, + "rfcacheWritesSkippedInternalError": 0, + "netUserDataCapacityNoTrimInKb": 524288, + "rfcacheWritesSkippedCacheMiss": 0, + "degradedFailedCapacityInKb": 0, + "activeNormRebuildCapacityInKb": 0, + "numOfMigratingVolumes": 0, + "snapCapacityInUseInKb": 0, + "fglSparesInKb": 0, + "compressionRatio": 1.0, + "rfcacheWriteMiss": 0, + "primaryReadFromRmcacheBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "migratingVtreeIds": [], + "numOfVtrees": 2, + "userDataCapacityNoTrimInKb": 1048576, + "rfacheReadHit": 0, + "compressedDataCompressionRatio": 0.0, + "rplUsedJournalCap": 0, + "pendingMovingCapacityInKb": 0, + "numOfSnapshots": 2, + "pendingFwdRebuildCapacityInKb": 0, + "tempCapacityInKb": 0, + "totalFglMigrationSizeInKb": 0, + "normRebuildCapacityInKb": 0, + "logWrittenBlocksInKb": 0, + "numOfThickBaseVolumes": 0, + "primaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "enterProtectedMaintenanceModeReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "activeRebalanceCapacityInKb": 0, + "numOfReplicationJournalVolumes": 0, + "rfcacheReadsSkippedLockIos": 0, + "unreachableUnusedCapacityInKb": 0, + "netProvisionedAddressesInKb": 524288, + "trimmedUserDataCapacityInKb": 0, + "provisionedAddressesInKb": 1048576, + "numOfVolumesInDeletion": 0, + "maxCapacityInKb": 311270400, + "pendingMovingOutFwdRebuildJobs": 0, + "rmPendingThickInKb": 0, + "protectedCapacityInKb": 1048576, + "secondaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "normRebuildReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "thinCapacityAllocatedInKb": 67108864, + "netFglUserDataCapacityInKb": 0, + "metadataOverheadInKb": 0, + "thinCapacityAllocatedInKm": 67108864, + "rebalanceWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "primaryVacInKb": 33554432, + "deviceIds": ["f7f77d0900000000", "f7fd7d0a00010000", "f7fd7d0b00020000"], + "secondaryVacInKb": 33554432, + "netSnapshotCapacityInKb": 0, + "numOfDevices": 3, + "rplTotalJournalCap": 0, + "failedCapacityInKb": 0, + "netMetadataOverheadInKb": 0, + "rfcacheReadsFromCache": 0, + "activeMovingOutBckRebuildJobs": 0, + "enterProtectedMaintenanceModeCapacityInKb": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingOutEnterProtectedMaintenanceModeJobs": 0, + "failedVacInKb": 0, + "primaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "fglCompressedDataSizeInKb": 0, + "fglUncompressedDataSizeInKb": 0, + "pendingRebalanceCapacityInKb": 0, + "rfcacheAvgReadTime": 0, + "semiProtectedCapacityInKb": 0, + "pendingMovingOutEnterProtectedMaintenanceModeJobs": 0, + "mgUserDdataCcapacityInKb": 1048576, + "snapshotCapacityInKb": 0, + "netMgUserDataCapacityInKb": 524288, + "fwdRebuildReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheWritesReceived": 0, + "netUnusedCapacityInKb": 89744384, + "thinUserDataCapacityInKb": 1048576, + "protectedVacInKb": 67108864, + "activeMovingRebalanceJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "bckRebuildCapacityInKb": 0, + "netTrimmedUserDataCapacityInKb": 0, + "pendingMovingRebalanceJobs": 0, + "numOfMarkedVolumesForReplication": 0, + "degradedHealthyVacInKb": 0, + "semiProtectedVacInKb": 0, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "pendingBckRebuildCapacityInKb": 0, + "capacityLimitInKb": 311270400, + "vtreeIds": ["ca7e2cd900000000", "ca7e2cda00000001"], + "activeMovingCapacityInKb": 0, + "pendingExitProtectedMaintenanceModeCapacityInKb": 0, + "targetWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheIosSkipped": 0, + "exitProtectedMaintenanceModeReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inMaintenanceVacInKb": 0, + "netFglSparesInKb": 0, + "rfcacheReadsSkipped": 0, + "activeExitProtectedMaintenanceModeCapacityInKb": 0, + "numOfUnmappedVolumes": 2, + "activeMovingOutExitProtectedMaintenanceModeJobs": 0, + "tempCapacityVacInKb": 0, + "volumeAddressSpaceInKb": 50331648, + "currentFglMigrationSizeInKb": 0, + "rfcacheWritesSkippedMaxIoSize": 0, + "netMaxUserDataCapacityInKb": 90268672, + "numOfMigratingVtrees": 0, + "rfacheWriteHit": 0, + "atRestCapacityInKb": 524288, + "bckRebuildReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheSourceDeviceWrites": 0, + "spareCapacityInKb": 130733056, + "enterProtectedMaintenanceModeWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inaccessibleCapacityInKb": 0, + "normRebuildWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheIoErrors": 0, + "capacityInUseInKb": 1048576, + "rebalanceReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheReadsSkippedMaxIoSize": 0, + "activeMovingInExitProtectedMaintenanceModeJobs": 0, + "secondaryReadFromDevBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryReadFromRmcacheBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheWritesSkippedStuckIo": 0, + "secondaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inMaintenanceCapacityInKb": 0, + "exposedCapacityInKb": 0, + "netFglCompressedDataSizeInKb": 0, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inUseVacInKb": 67108864, + "fwdRebuildCapacityInKb": 0, + "thickCapacityInUseInKb": 0, + "activeMovingInRebalanceJobs": 0, + "backgroundScanReadErrorCount": 0, + "migratingVolumeIds": [], + "rfcacheWritesSkippedLowResources": 0, + "capacityInUseNoOverheadInKb": 1048576, + "exitProtectedMaintenanceModeWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheSkippedUnlinedWrite": 0, + "netCapacityInUseInKb": 524288, + "numOfOutgoingMigrations": 0, + "rfcacheAvgWriteTime": 0, + "pendingNormRebuildCapacityInKb": 0, + "pendingMovingOutNormrebuildJobs": 0, + "rfcacheSourceDeviceReads": 0, + "rfcacheReadsPending": 0, + "volumeAllocationLimitInKb": 864026624, + "fwdRebuildWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheReadsSkippedHeavyLoad": 0, + "rfcacheReadMiss": 0, + "targetReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataCapacityInKb": 1048576, + "activeMovingInBckRebuildJobs": 0, + "movingCapacityInKb": 0, + "activeEnterProtectedMaintenanceModeCapacityInKb": 0, + "backgroundScanCompareErrorCount": 0, + "pendingMovingInFwdRebuildJobs": 0, + "rfcacheReadsReceived": 0, + "spSdsIds": ["f7ffcbad00000000", "f7facbae00010000", "f7fbcbaf00020000"], + "pendingEnterProtectedMaintenanceModeCapacityInKb": 0, + "vtreeAddresSpaceInKb": 33554432, + "snapCapacityInUseOccupiedInKb": 0, + "activeFwdRebuildCapacityInKb": 0, + "rfcacheReadsSkippedStuckIo": 0, + "activeMovingOutNormRebuildJobs": 0, + "rfcacheWritePending": 0, + "numOfThinBaseVolumes": 2, + "degradedFailedVacInKb": 0, + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfIncomingVtreeMigrations": 0, + "ActualNetCapacityInUseInKb": 0 +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/StoragePool__2515d0d600000001/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/StoragePool__2515d0d600000001/relationships/Statistics/response.json new file mode 100644 index 0000000000000..506a5ffc1aa5f --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/StoragePool__2515d0d600000001/relationships/Statistics/response.json @@ -0,0 +1,219 @@ +{ + "backgroundScanFixedReadErrorCount": 0, + "pendingMovingOutBckRebuildJobs": 0, + "degradedHealthyCapacityInKb": 0, + "activeMovingOutFwdRebuildJobs": 0, + "bckRebuildWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "netFglUncompressedDataSizeInKb": 0, + "primaryReadFromDevBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "BackgroundScannedInMB": 0, + "volumeIds": [], + "maxUserDataCapacityInKb": 0, + "persistentChecksumBuilderProgress": 100.0, + "rfcacheReadsSkippedAlignedSizeTooLarge": 0, + "pendingMovingInRebalanceJobs": 0, + "rfcacheWritesSkippedHeavyLoad": 0, + "unusedCapacityInKb": 0, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "totalReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfDeviceAtFaultRebuilds": 0, + "totalWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "persistentChecksumCapacityInKb": 0, + "rmPendingAllocatedInKb": 0, + "numOfVolumes": 0, + "rfcacheIosOutstanding": 0, + "numOfMappedToAllVolumes": 0, + "capacityAvailableForVolumeAllocationInKb": 0, + "netThinUserDataCapacityInKb": 0, + "backgroundScanFixedCompareErrorCount": 0, + "volMigrationWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "thinAndSnapshotRatio": 0.0, + "pendingMovingInEnterProtectedMaintenanceModeJobs": 0, + "fglUserDataCapacityInKb": 0, + "activeMovingInNormRebuildJobs": 0, + "aggregateCompressionLevel": "Uncompressed", + "targetOtherLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "netUserDataCapacityInKb": 0, + "pendingMovingOutExitProtectedMaintenanceModeJobs": 0, + "overallUsageRatio": 0.0, + "volMigrationReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "pendingMovingInBckRebuildJobs": 0, + "netCapacityInUseNoOverheadInKb": 0, + "rfcacheReadsSkippedInternalError": 0, + "activeBckRebuildCapacityInKb": 0, + "rebalanceCapacityInKb": 0, + "totalVolumeAllocationLimitInKb": 0, + "pendingMovingInExitProtectedMaintenanceModeJobs": 0, + "rfcacheReadsSkippedLowResources": 0, + "rplJournalCapAllowed": 0, + "thinCapacityInUseInKb": 0, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "activeMovingInEnterProtectedMaintenanceModeJobs": 0, + "rfcacheWritesSkippedInternalError": 0, + "netUserDataCapacityNoTrimInKb": 0, + "rfcacheWritesSkippedCacheMiss": 0, + "degradedFailedCapacityInKb": 0, + "activeNormRebuildCapacityInKb": 0, + "numOfMigratingVolumes": 0, + "snapCapacityInUseInKb": 0, + "fglSparesInKb": 0, + "compressionRatio": 0.0, + "rfcacheWriteMiss": 0, + "primaryReadFromRmcacheBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "migratingVtreeIds": [], + "numOfVtrees": 0, + "userDataCapacityNoTrimInKb": 0, + "rfacheReadHit": 0, + "compressedDataCompressionRatio": 0.0, + "rplUsedJournalCap": 0, + "pendingMovingCapacityInKb": 0, + "numOfSnapshots": 0, + "pendingFwdRebuildCapacityInKb": 0, + "tempCapacityInKb": 0, + "totalFglMigrationSizeInKb": 0, + "normRebuildCapacityInKb": 0, + "logWrittenBlocksInKb": 0, + "numOfThickBaseVolumes": 0, + "primaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "enterProtectedMaintenanceModeReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "activeRebalanceCapacityInKb": 0, + "numOfReplicationJournalVolumes": 0, + "rfcacheReadsSkippedLockIos": 0, + "unreachableUnusedCapacityInKb": 0, + "netProvisionedAddressesInKb": 0, + "trimmedUserDataCapacityInKb": 0, + "provisionedAddressesInKb": 0, + "numOfVolumesInDeletion": 0, + "maxCapacityInKb": 0, + "pendingMovingOutFwdRebuildJobs": 0, + "rmPendingThickInKb": 0, + "protectedCapacityInKb": 0, + "secondaryWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "normRebuildReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "thinCapacityAllocatedInKb": 0, + "netFglUserDataCapacityInKb": 0, + "metadataOverheadInKb": 0, + "thinCapacityAllocatedInKm": 0, + "rebalanceWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "primaryVacInKb": 0, + "deviceIds": [], + "secondaryVacInKb": 0, + "netSnapshotCapacityInKb": 0, + "numOfDevices": 0, + "rplTotalJournalCap": 0, + "failedCapacityInKb": 0, + "netMetadataOverheadInKb": 0, + "rfcacheReadsFromCache": 0, + "activeMovingOutBckRebuildJobs": 0, + "enterProtectedMaintenanceModeCapacityInKb": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingOutEnterProtectedMaintenanceModeJobs": 0, + "failedVacInKb": 0, + "primaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "fglCompressedDataSizeInKb": 0, + "fglUncompressedDataSizeInKb": 0, + "pendingRebalanceCapacityInKb": 0, + "rfcacheAvgReadTime": 0, + "semiProtectedCapacityInKb": 0, + "pendingMovingOutEnterProtectedMaintenanceModeJobs": 0, + "mgUserDdataCcapacityInKb": 0, + "snapshotCapacityInKb": 0, + "netMgUserDataCapacityInKb": 0, + "fwdRebuildReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheWritesReceived": 0, + "netUnusedCapacityInKb": 0, + "thinUserDataCapacityInKb": 0, + "protectedVacInKb": 0, + "activeMovingRebalanceJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "bckRebuildCapacityInKb": 0, + "netTrimmedUserDataCapacityInKb": 0, + "pendingMovingRebalanceJobs": 0, + "numOfMarkedVolumesForReplication": 0, + "degradedHealthyVacInKb": 0, + "semiProtectedVacInKb": 0, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "pendingBckRebuildCapacityInKb": 0, + "capacityLimitInKb": 0, + "vtreeIds": [], + "activeMovingCapacityInKb": 0, + "pendingExitProtectedMaintenanceModeCapacityInKb": 0, + "targetWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheIosSkipped": 0, + "exitProtectedMaintenanceModeReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inMaintenanceVacInKb": 0, + "netFglSparesInKb": 0, + "rfcacheReadsSkipped": 0, + "activeExitProtectedMaintenanceModeCapacityInKb": 0, + "numOfUnmappedVolumes": 0, + "activeMovingOutExitProtectedMaintenanceModeJobs": 0, + "tempCapacityVacInKb": 0, + "volumeAddressSpaceInKb": 0, + "currentFglMigrationSizeInKb": 0, + "rfcacheWritesSkippedMaxIoSize": 0, + "netMaxUserDataCapacityInKb": 0, + "numOfMigratingVtrees": 0, + "rfacheWriteHit": 0, + "atRestCapacityInKb": 0, + "bckRebuildReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheSourceDeviceWrites": 0, + "spareCapacityInKb": 0, + "enterProtectedMaintenanceModeWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inaccessibleCapacityInKb": 0, + "normRebuildWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheIoErrors": 0, + "capacityInUseInKb": 0, + "rebalanceReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheReadsSkippedMaxIoSize": 0, + "activeMovingInExitProtectedMaintenanceModeJobs": 0, + "secondaryReadFromDevBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "secondaryReadFromRmcacheBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheWritesSkippedStuckIo": 0, + "secondaryReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inMaintenanceCapacityInKb": 0, + "exposedCapacityInKb": 0, + "netFglCompressedDataSizeInKb": 0, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "inUseVacInKb": 0, + "fwdRebuildCapacityInKb": 0, + "thickCapacityInUseInKb": 0, + "activeMovingInRebalanceJobs": 0, + "backgroundScanReadErrorCount": 0, + "migratingVolumeIds": [], + "rfcacheWritesSkippedLowResources": 0, + "capacityInUseNoOverheadInKb": 0, + "exitProtectedMaintenanceModeWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheSkippedUnlinedWrite": 0, + "netCapacityInUseInKb": 0, + "numOfOutgoingMigrations": 0, + "rfcacheAvgWriteTime": 0, + "pendingNormRebuildCapacityInKb": 0, + "pendingMovingOutNormrebuildJobs": 0, + "rfcacheSourceDeviceReads": 0, + "rfcacheReadsPending": 0, + "volumeAllocationLimitInKb": 0, + "fwdRebuildWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rfcacheReadsSkippedHeavyLoad": 0, + "rfcacheReadMiss": 0, + "targetReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataCapacityInKb": 0, + "activeMovingInBckRebuildJobs": 0, + "movingCapacityInKb": 0, + "activeEnterProtectedMaintenanceModeCapacityInKb": 0, + "backgroundScanCompareErrorCount": 0, + "pendingMovingInFwdRebuildJobs": 0, + "rfcacheReadsReceived": 0, + "spSdsIds": [], + "pendingEnterProtectedMaintenanceModeCapacityInKb": 0, + "vtreeAddresSpaceInKb": 0, + "snapCapacityInUseOccupiedInKb": 0, + "activeFwdRebuildCapacityInKb": 0, + "rfcacheReadsSkippedStuckIo": 0, + "activeMovingOutNormRebuildJobs": 0, + "rfcacheWritePending": 0, + "numOfThinBaseVolumes": 0, + "degradedFailedVacInKb": 0, + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfIncomingVtreeMigrations": 0 +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/System__1fcf40fc60c6520f/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/System__1fcf40fc60c6520f/relationships/Statistics/response.json new file mode 100644 index 0000000000000..cb592cb195ead --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/System__1fcf40fc60c6520f/relationships/Statistics/response.json @@ -0,0 +1,601 @@ +{ + "numOfVtreeMigrationsInSystem": 0, + "backgroundScanFixedReadErrorCount": 0, + "numOfCrashPointConfigs": null, + "pendingMovingOutBckRebuildJobs": 0, + "rfcachePoolWritePending": 0, + "degradedHealthyCapacityInKb": 0, + "rplCgRpoCompliance": 100, + "numSdsSdrDisconnections": 0, + "activeMovingOutFwdRebuildJobs": 0, + "rfcachePoolWritePendingG1Sec": 0, + "bckRebuildWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "netFglUncompressedDataSizeInKb": 0, + "primaryReadFromDevBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "numOfSnapPolicies": 0, + "BackgroundScannedInMB": 1139, + "protectedMaintenanceModeWaitSendQLength": 0, + "maxUserDataCapacityInKb": 180537344, + "rfcacheReadsSkippedAlignedSizeTooLarge": 0, + "rfcachePoolSize": 0, + "pendingMovingInRebalanceJobs": 0, + "fglMetadataCacheHitrate": 0.0, + "rfcacheWritesSkippedHeavyLoad": 0, + "rfcachePoolPagesInuse": 0, + "unusedCapacityInKb": 179488768, + "userDataSdcReadLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheFdAvgWriteTime": 0, + "rmcacheEntryEvictionCount": 0, + "totalReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "totalWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "sdtDataReadLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "journalerReadLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "persistentChecksumCapacityInKb": 153600, + "rmPendingAllocatedInKb": 0, + "sdtHostReadLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "numOfVolumes": 4, + "rfcacheIosOutstanding": 0, + "sdtDataReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "numRpoViolatingRplCgsSrc": 0, + "rmcacheBigBlockEvictionSizeCountInKb": 0, + "numOfMappedToAllVolumes": 0, + "capacityAvailableForVolumeAllocationInKb": 75497472, + "currentTickerValue": 136238, + "netThinUserDataCapacityInKb": 524288, + "backgroundScanFixedCompareErrorCount": 0, + "volMigrationWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "thinAndSnapshotRatio": 96.0, + "rebuildPerReceiveJobNetThrottlingInKbps": 0, + "pendingMovingInEnterProtectedMaintenanceModeJobs": 0, + "fglUserDataCapacityInKb": 0, + "rcgRemoteWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rmcache32kbEntryCount": 0, + "rfcachePoolEvictions": 0, + "rfcachePoolNumCacheDevs": 0, + "activeMovingInNormRebuildJobs": 0, + "journalerWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "aggregateCompressionLevel": "Uncompressed", + "rfcacheFdWriteTimeGreater500Millis": 0, + "sdtDataWriteLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "targetOtherLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "journalerReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "netUserDataCapacityInKb": 524288, + "rmcacheSkipCountCacheAllBusy": 0, + "rfcachePoolNumSrcDevs": 0, + "numOfSdc": 3, + "rfcacheFdMonitorErrorStuckIo": 0, + "rplRemoteUserBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rplTransmitLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "pendingMovingOutExitProtectedMaintenanceModeJobs": 0, + "overallUsageRatio": 96.0, + "volMigrationReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "pendingMovingInBckRebuildJobs": 0, + "netCapacityInUseNoOverheadInKb": 524288, + "rfcachePoolWritePendingG500Micro": 0, + "rfcacheReadsSkippedInternalError": 0, + "activeBckRebuildCapacityInKb": 0, + "sdtIds": [], + "snapPolicyIds": [], + "rebalanceCapacityInKb": 0, + "totalVolumeAllocationLimitInKb": 897581056, + "pendingMovingInExitProtectedMaintenanceModeJobs": 0, + "numDevErrors": 0, + "rfcacheReadsSkippedLowResources": 0, + "rfcachePoolInLowMemoryCondition": 0, + "rplReceiveLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rplJournalCapAllowed": 0, + "numSnapshotsTaken": 2, + "thinCapacityInUseInKb": 0, + "userDataSdcTrimLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rplReceiveBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcachePoolLowResourcesInitiatedPassthroughMode": 0, + "activeMovingInEnterProtectedMaintenanceModeJobs": 0, + "rfcachePoolWritePendingG10Millis": 0, + "rfcacheWritesSkippedInternalError": 0, + "rfcachePoolWriteHit": 0, + "rcgRemoteReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rmcache128kbEntryCount": 0, + "netUserDataCapacityNoTrimInKb": 524288, + "rfcacheFdReadTimeGreater5Sec": 0, + "rfcacheWritesSkippedCacheMiss": 0, + "numOscillationCountersPassedThreshold": 0, + "numOfFaultSets": 0, + "degradedFailedCapacityInKb": 0, + "activeNormRebuildCapacityInKb": 0, + "snapCapacityInUseInKb": 0, + "fglSparesInKb": 0, + "compressionRatio": 1.0, + "rfcacheWriteMiss": 0, + "rfcacheFdIoErrors": 0, + "primaryReadFromRmcacheBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "numOfVtrees": 2, + "userDataCapacityNoTrimInKb": 1048576, + "numRpoViolatingRplCgsDest": 0, + "rfacheReadHit": 0, + "numSmartAttributesPassedThreshold": 0, + "rfcachePooIosOutstanding": 0, + "compressedDataCompressionRatio": 0.0, + "rplUsedJournalCap": 0, + "rcgLocalWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "pendingMovingCapacityInKb": 0, + "numOfSnapshots": 2, + "rcgLocalReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "sdcIds": [ + "1b8659fc00000000", + "1b8659fd00000001", + "1b8659fe00000002" + ], + "pendingFwdRebuildCapacityInKb": 0, + "rmcacheNoEvictionCount": 0, + "rmcacheBigBlockEvictionCount": 0, + "tempCapacityInKb": 0, + "rmcacheCurrNumOf128kbEntries": 0, + "normRebuildCapacityInKb": 0, + "rfcachePoolReadPendingG1Millis": 0, + "numOfAccelerationPools": 1, + "logWrittenBlocksInKb": 0, + "rmcacheSizeInUseInKb": 0, + "numOfThickBaseVolumes": 0, + "primaryWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcachePoolReadPendingG10Millis": 0, + "enterProtectedMaintenanceModeReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "activeRebalanceCapacityInKb": 0, + "numOfReplicationJournalVolumes": 0, + "rfcacheReadsSkippedLockIos": 0, + "unreachableUnusedCapacityInKb": 0, + "netProvisionedAddressesInKb": 524288, + "rfcachePoolReadPendingG500Micro": 0, + "rmcache8kbEntryCount": 0, + "trimmedUserDataCapacityInKb": 0, + "provisionedAddressesInKb": 1048576, + "numOfVolumesInDeletion": 0, + "maxCapacityInKb": 311270400, + "pendingMovingOutFwdRebuildJobs": 0, + "rmcacheSkipCountLargeIo": 0, + "rmPendingThickInKb": 0, + "protectedCapacityInKb": 1048576, + "secondaryWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "normRebuildReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "thinCapacityAllocatedInKb": 67108864, + "netFglUserDataCapacityInKb": 0, + "metadataOverheadInKb": 0, + "thinCapacityAllocatedInKm": 67108864, + "rebalanceWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rmcacheCurrNumOf8kbEntries": 0, + "primaryVacInKb": 33554432, + "sdtHostWriteLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "crashPointConfigIds": null, + "numVolumeMigrationsPerformed": 0, + "secondaryVacInKb": 33554432, + "netSnapshotCapacityInKb": 0, + "numOfDevices": 3, + "numCmatrixPolicyChanges": 6, + "rplTotalJournalCap": 0, + "rfcachePoolWriteMiss": 0, + "rfcachePoolReadPendingG1Sec": 0, + "failedCapacityInKb": 0, + "netMetadataOverheadInKb": 0, + "rfcachePoolWritePendingG1Millis": 0, + "rfcacheFdReadTimeGreater1Min": 0, + "rebalanceWaitSendQLength": 0, + "rmcache4kbEntryCount": 0, + "rebalancePerReceiveJobNetThrottlingInKbps": 25600, + "rfcacheReadsFromCache": 0, + "rfcacheFdReadTimeGreater1Sec": 0, + "activeMovingOutBckRebuildJobs": 0, + "rmcache64kbEntryCount": 0, + "enterProtectedMaintenanceModeCapacityInKb": 0, + "pendingMovingInNormRebuildJobs": 0, + "activeMovingOutEnterProtectedMaintenanceModeJobs": 0, + "failedVacInKb": 0, + "primaryReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "fglCompressedDataSizeInKb": 0, + "fglUncompressedDataSizeInKb": 0, + "pendingRebalanceCapacityInKb": 0, + "rfcacheAvgReadTime": 0, + "semiProtectedCapacityInKb": 0, + "pendingMovingOutEnterProtectedMaintenanceModeJobs": 0, + "numOfRplPairs": 0, + "rfcachePoolSourceIdMismatch": 0, + "mgUserDdataCcapacityInKb": 1048576, + "snapshotCapacityInKb": 0, + "rfcacheFdAvgReadTime": 0, + "netMgUserDataCapacityInKb": 524288, + "numOfPeerMdm": 0, + "fwdRebuildReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheWritesReceived": 0, + "netUnusedCapacityInKb": 89744384, + "rfcachePoolSuspendedIos": 0, + "numOfLibsdc": 0, + "thinUserDataCapacityInKb": 1048576, + "protectedVacInKb": 67108864, + "rplTopPerformingRcgs": [], + "activeMovingRebalanceJobs": 0, + "activeMovingInFwdRebuildJobs": 0, + "bckRebuildCapacityInKb": 0, + "sdrIds": [], + "netTrimmedUserDataCapacityInKb": 0, + "pendingMovingRebalanceJobs": 0, + "numOfMarkedVolumesForReplication": 0, + "degradedHealthyVacInKb": 0, + "rfcachePoolLockTimeGreater1Sec": 0, + "rplRemoteApplyBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "semiProtectedVacInKb": 0, + "rplProtectedCapacityDest": 0, + "userDataReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 42 + }, + "pendingBckRebuildCapacityInKb": 0, + "rmcacheCurrNumOf4kbEntries": 0, + "capacityLimitInKb": 311270400, + "rplLocalApplyBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "numOfProtectionDomains": 1, + "activeMovingCapacityInKb": 0, + "pendingExitProtectedMaintenanceModeCapacityInKb": 0, + "targetWriteLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "numOfRplCgs": 0, + "rfcacheIosSkipped": 0, + "rfcacheFdWriteTimeGreater5Sec": 0, + "exitProtectedMaintenanceModeReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "userDataWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "inMaintenanceVacInKb": 0, + "netFglSparesInKb": 0, + "rfcacheReadsSkipped": 0, + "rfcachePoolReadHit": 0, + "activeExitProtectedMaintenanceModeCapacityInKb": 0, + "rebuildWaitSendQLength": 0, + "numOfUnmappedVolumes": 2, + "activeMovingOutExitProtectedMaintenanceModeJobs": 0, + "rmcacheCurrNumOf64kbEntries": 0, + "tempCapacityVacInKb": 0, + "volumeAddressSpaceInKb": 50331648, + "rfcacheWritesSkippedMaxIoSize": 0, + "numSdsReconnections": 8, + "netMaxUserDataCapacityInKb": 90268672, + "rfacheWriteHit": 0, + "atRestCapacityInKb": 524288, + "bckRebuildReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheSourceDeviceWrites": 0, + "rfcacheFdInlightReads": 0, + "spareCapacityInKb": 130733056, + "enterProtectedMaintenanceModeWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "numOfSdt": 0, + "numOfSds": 3, + "normRebuildWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheIoErrors": 0, + "capacityInUseInKb": 1048576, + "numOfSdr": 0, + "numSdSdcDisconnections": 9, + "rebalanceReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rmcacheSkipCountUnaligned4kbIo": 0, + "rfcacheReadsSkippedMaxIoSize": 0, + "numSdrSdcDisconnections": 0, + "activeMovingInExitProtectedMaintenanceModeJobs": 0, + "secondaryReadFromDevBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcachePoolSuspendedPequestsRedundantSearchs": 0, + "secondaryReadFromRmcacheBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheWritesSkippedStuckIo": 0, + "numOfStoragePools": 2, + "secondaryReadBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcachePoolCachePages": 0, + "inMaintenanceCapacityInKb": 0, + "sdtDataWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "protectionDomainIds": [ + "68c139ee00000000" + ], + "netFglCompressedDataSizeInKb": 0, + "sdtDataTrimBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "vtreeMigrationWaitSendQLength": 0, + "userDataSdcWriteLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "inUseVacInKb": 67108864, + "fwdRebuildCapacityInKb": 0, + "libsdcIds": [], + "thickCapacityInUseInKb": 0, + "sdtHostTrimLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "activeMovingInRebalanceJobs": 0, + "backgroundScanReadErrorCount": 0, + "rmcacheCurrNumOf32kbEntries": 0, + "sdtDataTrimLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheWritesSkippedLowResources": 0, + "rplApplyLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "peerMdmIds": [], + "capacityInUseNoOverheadInKb": 1048576, + "rplLocalUserBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheFdCacheOverloaded": 0, + "rmcache16kbEntryCount": 0, + "exitProtectedMaintenanceModeWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rmcacheEntryEvictionSizeCountInKb": 0, + "rfcacheSkippedUnlinedWrite": 0, + "netCapacityInUseInKb": 524288, + "rplProtectedCapacitySrc": 0, + "rfcacheAvgWriteTime": 0, + "journalerWriteLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "pendingNormRebuildCapacityInKb": 0, + "vtreeMigrationPerReceiveJobNetThrottlingInKbps": 30720, + "rfcacheFdReadTimeGreater500Millis": 0, + "pendingMovingOutNormrebuildJobs": 0, + "rfcacheSourceDeviceReads": 0, + "rmcacheCurrNumOf16kbEntries": 0, + "initialCopyProgress": 0.0, + "rfcacheReadsPending": 0, + "volumeAllocationLimitInKb": 864026624, + "fwdRebuildWriteBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheReadsSkippedHeavyLoad": 0, + "rfcacheReadMiss": 0, + "protectedMaintenanceModePerReceiveJobNetThrottlingInKbps": 0, + "rfcacheFdInlightWrites": 0, + "targetReadLatency": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "userDataCapacityInKb": 1048576, + "rfcacheFdReadsReceived": 0, + "activeMovingInBckRebuildJobs": 0, + "movingCapacityInKb": 0, + "activeEnterProtectedMaintenanceModeCapacityInKb": 0, + "backgroundScanCompareErrorCount": 0, + "pendingMovingInFwdRebuildJobs": 0, + "rfcacheReadsReceived": 0, + "rfcachePoolReadsPending": 0, + "pendingEnterProtectedMaintenanceModeCapacityInKb": 0, + "vtreeAddresSpaceInKb": 33554432, + "snapCapacityInUseOccupiedInKb": 0, + "activeFwdRebuildCapacityInKb": 0, + "rfcacheReadsSkippedStuckIo": 0, + "activeMovingOutNormRebuildJobs": 0, + "rplTransmitBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheFdWritesReceived": 0, + "rmcacheSizeInKb": 393216, + "rfcacheFdWriteTimeGreater1Min": 0, + "rfcacheWritePending": 0, + "rfcacheFdWriteTimeGreater1Sec": 0, + "numOfThinBaseVolumes": 2, + "degradedFailedVacInKb": 0, + "numOfRfcacheDevices": 0, + "movePairsIds": null, + "rfcachePoolIoTimeGreater1Min": 0, + "userDataTrimBwc": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcachePoolReadMiss": 0, + "rebalanceReadBwc2": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + }, + "rfcacheReadsSkippedHeavyLoad2": 0, + "userDataSdcReadLatency2": { + "numSeconds": 0, + "totalWeightInKb": 0, + "numOccured": 0 + } +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06e700000000/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06e700000000/relationships/Statistics/response.json new file mode 100644 index 0000000000000..9e434472ce507 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06e700000000/relationships/Statistics/response.json @@ -0,0 +1,12 @@ +{ + "rplUsedJournalCap": 0, + "numOfChildVolumes": 1, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rplTotalJournalCap": 0, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfMappedSdcs": 1, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0} +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06e800000001/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06e800000001/relationships/Statistics/response.json new file mode 100644 index 0000000000000..0035eecaee123 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06e800000001/relationships/Statistics/response.json @@ -0,0 +1,22 @@ +{ + "rplUsedJournalCap": 0, + "replicationState": "UnmarkedForReplication", + "numOfChildVolumes": 0, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rplTotalJournalCap": 0, + "initiatorSdcId": null, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "mappedSdcIds": ["1b8659fd00000001"], + "registrationKey": null, + "registrationKeys": [], + "descendantVolumeIds": [], + "numOfMappedSdcs": 1, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfDescendantVolumes": 0, + "reservationType": "NotReserved", + "replicationJournalVolume": false, + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "childVolumeIds": [], + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0} +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06e900000002/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06e900000002/relationships/Statistics/response.json new file mode 100644 index 0000000000000..874745df8531c --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06e900000002/relationships/Statistics/response.json @@ -0,0 +1,22 @@ +{ + "rplUsedJournalCap": 0, + "replicationState": "UnmarkedForReplication", + "numOfChildVolumes": 1, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rplTotalJournalCap": 0, + "initiatorSdcId": null, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "mappedSdcIds": [], + "registrationKey": null, + "registrationKeys": [], + "descendantVolumeIds": ["c58b06ea00000003"], + "numOfMappedSdcs": 0, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfDescendantVolumes": 1, + "reservationType": "NotReserved", + "replicationJournalVolume": false, + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "childVolumeIds": ["c58b06ea00000003"], + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0} +} diff --git a/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06ea00000003/relationships/Statistics/response.json b/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06ea00000003/relationships/Statistics/response.json new file mode 100644 index 0000000000000..e5b8c18df4ba4 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/instances/Volume__c58b06ea00000003/relationships/Statistics/response.json @@ -0,0 +1,22 @@ +{ + "rplUsedJournalCap": 0, + "replicationState": "UnmarkedForReplication", + "numOfChildVolumes": 0, + "userDataWriteBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "rplTotalJournalCap": 0, + "initiatorSdcId": null, + "userDataSdcReadLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "userDataSdcTrimLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "mappedSdcIds": [], + "registrationKey": null, + "registrationKeys": [], + "descendantVolumeIds": [], + "numOfMappedSdcs": 0, + "userDataReadBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "numOfDescendantVolumes": 0, + "reservationType": "NotReserved", + "replicationJournalVolume": false, + "userDataTrimBwc": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0}, + "childVolumeIds": [], + "userDataSdcWriteLatency": {"numSeconds": 0, "totalWeightInKb": 0, "numOccured": 0} +} diff --git a/dell_powerflex/tests/fixtures/GET/api/types/Device/instances/response.json b/dell_powerflex/tests/fixtures/GET/api/types/Device/instances/response.json new file mode 100644 index 0000000000000..e88ca350567f1 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/types/Device/instances/response.json @@ -0,0 +1,221 @@ +[ + { + "fglNvdimmWriteCacheState": "Disabled", + "mediaType": "SSD", + "deviceType": "Unknown", + "serialNumber": "VBOX_HARDDISK_VBa1b2c3d4-00000001", + "vendorName": "ATA", + "modelName": "VBOX HARDDISK", + "firmwareVersion": "1.0", + "deviceState": "Normal", + "errorState": "None", + "rdmaBaseDeviceName": null, + "rdmaBaseDeviceMaxSize": 0, + "rdmaBaseDeviceDeletedSize": 0, + "capacityLimitInKb": 103756800, + "maxCapacityInKb": 103756800, + "logicalCapacityInKb": 0, + "persistentChecksumState": "Protected", + "autoDetectMediaType": "SSD", + "externalAccelerationType": "None", + "rfcacheProps": null, + "storagePoolId": "25155ba600000000", + "sdsId": "d1c062b900000002", + "name": "sds1-dev1", + "id": "f7fd7d0b00020000", + "deviceCurrentPathName": "/dev/sdb", + "deviceOriginalPathName": "/dev/sdb", + "temperatureState": "NeverFailed", + "aggregatedState": "NeverFailed", + "ssdEndOfLifeState": "NeverFailed", + "longSuccessfulIos": { + "shortWindow": { + "threshold": 10000, + "windowSizeInSec": 60, + "lastOscillationCount": 0, + "lastOscillationTime": 0, + "maxFailuresCount": 0 + }, + "mediumWindow": { + "threshold": 100000, + "windowSizeInSec": 3600, + "lastOscillationCount": 0, + "lastOscillationTime": 0, + "maxFailuresCount": 0 + }, + "longWindow": { + "threshold": 1000000, + "windowSizeInSec": 86400, + "lastOscillationCount": 0, + "lastOscillationTime": 0, + "maxFailuresCount": 0 + } + }, + "ataSecurityActive": false, + "fglNvdimmSizeMb": 0, + "links": [ + { + "rel": "self", + "href": "/api/instances/Device::f7fd7d0b00020000" + }, + { + "rel": "/api/Device/relationship/Statistics", + "href": "/api/instances/Device::f7fd7d0b00020000/relationships/Statistics" + }, + { + "rel": "/api/parent/relationship/sdsId", + "href": "/api/instances/Sds::d1c062b900000002" + }, + { + "rel": "/api/parent/relationship/storagePoolId", + "href": "/api/instances/StoragePool::25155ba600000000" + } + ] + }, + { + "fglNvdimmWriteCacheState": "Disabled", + "mediaType": "SSD", + "deviceType": "Unknown", + "serialNumber": "VBOX_HARDDISK_VBa1b2c3d4-00000002", + "vendorName": "ATA", + "modelName": "VBOX HARDDISK", + "firmwareVersion": "1.0", + "deviceState": "Normal", + "errorState": "None", + "rdmaBaseDeviceName": null, + "rdmaBaseDeviceMaxSize": 0, + "rdmaBaseDeviceDeletedSize": 0, + "capacityLimitInKb": 103756800, + "maxCapacityInKb": 103756800, + "logicalCapacityInKb": 0, + "persistentChecksumState": "Protected", + "autoDetectMediaType": "SSD", + "externalAccelerationType": "None", + "rfcacheProps": null, + "storagePoolId": "25155ba600000000", + "sdsId": "d1c062b800000001", + "name": "sds2-dev1", + "id": "f7fd7d0a00010000", + "deviceCurrentPathName": "/dev/sdb", + "deviceOriginalPathName": "/dev/sdb", + "temperatureState": "NeverFailed", + "aggregatedState": "NeverFailed", + "ssdEndOfLifeState": "NeverFailed", + "longSuccessfulIos": { + "shortWindow": { + "threshold": 10000, + "windowSizeInSec": 60, + "lastOscillationCount": 0, + "lastOscillationTime": 0, + "maxFailuresCount": 0 + }, + "mediumWindow": { + "threshold": 100000, + "windowSizeInSec": 3600, + "lastOscillationCount": 0, + "lastOscillationTime": 0, + "maxFailuresCount": 0 + }, + "longWindow": { + "threshold": 1000000, + "windowSizeInSec": 86400, + "lastOscillationCount": 0, + "lastOscillationTime": 0, + "maxFailuresCount": 0 + } + }, + "ataSecurityActive": false, + "fglNvdimmSizeMb": 0, + "links": [ + { + "rel": "self", + "href": "/api/instances/Device::f7fd7d0a00010000" + }, + { + "rel": "/api/Device/relationship/Statistics", + "href": "/api/instances/Device::f7fd7d0a00010000/relationships/Statistics" + }, + { + "rel": "/api/parent/relationship/sdsId", + "href": "/api/instances/Sds::d1c062b800000001" + }, + { + "rel": "/api/parent/relationship/storagePoolId", + "href": "/api/instances/StoragePool::25155ba600000000" + } + ] + }, + { + "fglNvdimmWriteCacheState": "Disabled", + "mediaType": "SSD", + "deviceType": "Unknown", + "serialNumber": "VBOX_HARDDISK_VBa1b2c3d4-00000003", + "vendorName": "ATA", + "modelName": "VBOX HARDDISK", + "firmwareVersion": "1.0", + "deviceState": "Normal", + "errorState": "None", + "rdmaBaseDeviceName": null, + "rdmaBaseDeviceMaxSize": 0, + "rdmaBaseDeviceDeletedSize": 0, + "capacityLimitInKb": 103756800, + "maxCapacityInKb": 103756800, + "logicalCapacityInKb": 0, + "persistentChecksumState": "Protected", + "autoDetectMediaType": "SSD", + "externalAccelerationType": "None", + "rfcacheProps": null, + "storagePoolId": "25155ba600000000", + "sdsId": "d1c062b700000000", + "name": "sds3-dev1", + "id": "f7f77d0900000000", + "deviceCurrentPathName": "/dev/sdb", + "deviceOriginalPathName": "/dev/sdb", + "temperatureState": "NeverFailed", + "aggregatedState": "NeverFailed", + "ssdEndOfLifeState": "NeverFailed", + "longSuccessfulIos": { + "shortWindow": { + "threshold": 10000, + "windowSizeInSec": 60, + "lastOscillationCount": 0, + "lastOscillationTime": 0, + "maxFailuresCount": 0 + }, + "mediumWindow": { + "threshold": 100000, + "windowSizeInSec": 3600, + "lastOscillationCount": 0, + "lastOscillationTime": 0, + "maxFailuresCount": 0 + }, + "longWindow": { + "threshold": 1000000, + "windowSizeInSec": 86400, + "lastOscillationCount": 0, + "lastOscillationTime": 0, + "maxFailuresCount": 0 + } + }, + "ataSecurityActive": false, + "fglNvdimmSizeMb": 0, + "links": [ + { + "rel": "self", + "href": "/api/instances/Device::f7f77d0900000000" + }, + { + "rel": "/api/Device/relationship/Statistics", + "href": "/api/instances/Device::f7f77d0900000000/relationships/Statistics" + }, + { + "rel": "/api/parent/relationship/sdsId", + "href": "/api/instances/Sds::d1c062b700000000" + }, + { + "rel": "/api/parent/relationship/storagePoolId", + "href": "/api/instances/StoragePool::25155ba600000000" + } + ] + } +] diff --git a/dell_powerflex/tests/fixtures/GET/api/types/ProtectionDomain/instances/response.json b/dell_powerflex/tests/fixtures/GET/api/types/ProtectionDomain/instances/response.json new file mode 100644 index 0000000000000..5aec79c277c8d --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/types/ProtectionDomain/instances/response.json @@ -0,0 +1,80 @@ +[ + { + "rfcacheEnabled": true, + "rebuildNetworkThrottlingEnabled": false, + "vtreeMigrationNetworkThrottlingEnabled": false, + "overallIoNetworkThrottlingEnabled": false, + "rfcacheAccpId": null, + "rebalanceNetworkThrottlingEnabled": false, + "systemId": "1fcf40fc60c6520f", + "name": "domain1", + "sdrSdsConnectivityInfo": { + "clientServerConnStatus": "CLIENT_SERVER_CONN_STATUS_ALL_CONNECTED", + "disconnectedClientId": null, + "disconnectedClientName": null, + "disconnectedServerId": null, + "disconnectedServerName": null, + "disconnectedServerIp": null + }, + "protectionDomainState": "Active", + "rplCapAlertLevel": "invalid", + "rebuildNetworkThrottlingInKbps": null, + "rebalanceNetworkThrottlingInKbps": null, + "overallIoNetworkThrottlingInKbps": null, + "vtreeMigrationNetworkThrottlingInKbps": null, + "sdsDecoupledCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 300}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 500}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 700} + }, + "sdsConfigurationFailureCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 300}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 500}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 700} + }, + "mdmSdsNetworkDisconnectionsCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 300}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 500}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 700} + }, + "sdsSdsNetworkDisconnectionsCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 300}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 500}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 700} + }, + "rfcacheOpertionalMode": "WriteMiss", + "rfcachePageSizeKb": 64, + "rfcacheMaxIoSizeKb": 128, + "sdsReceiveBufferAllocationFailuresCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 20000}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 200000}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 2000000} + }, + "fglDefaultNumConcurrentWrites": 1000, + "fglMetadataCacheEnabled": false, + "fglDefaultMetadataCacheSize": 0, + "protectedMaintenanceModeNetworkThrottlingEnabled": false, + "protectedMaintenanceModeNetworkThrottlingInKbps": null, + "sdtSdsConnectivityInfo": { + "clientServerConnStatus": "CLIENT_SERVER_CONN_STATUS_ALL_CONNECTED", + "disconnectedClientId": null, + "disconnectedClientName": null, + "disconnectedServerId": null, + "disconnectedServerName": null, + "disconnectedServerIp": null + }, + "id": "68c139ee00000000", + "links": [ + {"rel": "self", "href": "/api/instances/ProtectionDomain::68c139ee00000000"}, + {"rel": "/api/ProtectionDomain/relationship/Statistics", "href": "/api/instances/ProtectionDomain::68c139ee00000000/relationships/Statistics"}, + {"rel": "/api/ProtectionDomain/relationship/Sdr", "href": "/api/instances/ProtectionDomain::68c139ee00000000/relationships/Sdr"}, + {"rel": "/api/ProtectionDomain/relationship/AccelerationPool", "href": "/api/instances/ProtectionDomain::68c139ee00000000/relationships/AccelerationPool"}, + {"rel": "/api/ProtectionDomain/relationship/Sdt", "href": "/api/instances/ProtectionDomain::68c139ee00000000/relationships/Sdt"}, + {"rel": "/api/ProtectionDomain/relationship/StoragePool", "href": "/api/instances/ProtectionDomain::68c139ee00000000/relationships/StoragePool"}, + {"rel": "/api/ProtectionDomain/relationship/Sds", "href": "/api/instances/ProtectionDomain::68c139ee00000000/relationships/Sds"}, + {"rel": "/api/ProtectionDomain/relationship/ReplicationConsistencyGroup", "href": "/api/instances/ProtectionDomain::68c139ee00000000/relationships/ReplicationConsistencyGroup"}, + {"rel": "/api/ProtectionDomain/relationship/FaultSet", "href": "/api/instances/ProtectionDomain::68c139ee00000000/relationships/FaultSet"}, + {"rel": "/api/parent/relationship/systemId", "href": "/api/instances/System::1fcf40fc60c6520f"} + ] + } +] diff --git a/dell_powerflex/tests/fixtures/GET/api/types/Sdc/instances/response.json b/dell_powerflex/tests/fixtures/GET/api/types/Sdc/instances/response.json new file mode 100644 index 0000000000000..c7788f757b053 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/types/Sdc/instances/response.json @@ -0,0 +1,140 @@ +[ + { + "id": "1b8659fd00000001", + "name": null, + "sdcIp": "10.0.1.250", + "sdcGuid": "33FC0AF2-5180-45D8-9BDC-8E2F78CD60BF", + "sdcType": "AppSdc", + "peerMdmId": "mdm00000001", + "sdcApproved": true, + "sdcApprovedIps": null, + "sdcAgentActive": true, + "mdmConnectionState": "Connected", + "hostOsFullType": "Linux 5.15.0-91-generic", + "systemId": "1fcf40fc60c6520f", + "softwareVersionInfo": "R4_5.2100.0", + "installedSoftwareVersionInfo": "R4_5.2100.0", + "kernelVersion": "5.15.0-91-generic", + "kernelBuildNumber": null, + "perfProfile": "Default", + "versionInfo": "R4_5.2100.0", + "sdcDefaultInitiatorConnectivityState": "Connected", + "certificateInfo": { + "thumbprint": "AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99", + "subject": "CN=SDC1, O=Dell, C=US", + "issuer": "CN=PowerFlex Root CA, O=Dell, C=US", + "validFrom": "2025-01-01T00:00:00Z", + "validTo": "2026-01-01T00:00:00Z" + }, + "links": [ + { + "rel": "self", + "href": "/api/instances/Sdc::1b8659fd00000001" + }, + { + "rel": "/api/Sdc/relationship/Statistics", + "href": "/api/instances/Sdc::1b8659fd00000001/relationships/Statistics" + }, + { + "rel": "/api/Sdc/relationship/Volume", + "href": "/api/instances/Sdc::1b8659fd00000001/relationships/Volume" + }, + { + "rel": "/api/parent/relationship/systemId", + "href": "/api/instances/System::1fcf40fc60c6520f" + } + ] + }, + { + "id": "1b8659fc00000000", + "name": null, + "sdcIp": "10.0.1.223", + "sdcGuid": "BE3BC972-269A-4931-96B8-286BFA45C004", + "sdcType": "AppSdc", + "peerMdmId": null, + "sdcApproved": true, + "sdcApprovedIps": null, + "sdcAgentActive": true, + "mdmConnectionState": "Connected", + "hostOsFullType": "Linux 5.15.0-91-generic", + "systemId": "1fcf40fc60c6520f", + "softwareVersionInfo": "R4_5.2100.0", + "installedSoftwareVersionInfo": "R4_5.2100.0", + "kernelVersion": "5.15.0-91-generic", + "kernelBuildNumber": null, + "perfProfile": "Default", + "versionInfo": "R4_5.2100.0", + "sdcDefaultInitiatorConnectivityState": "Connected", + "certificateInfo": { + "thumbprint": "11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00", + "subject": "CN=SDC2, O=Dell, C=US", + "issuer": "CN=PowerFlex Root CA, O=Dell, C=US", + "validFrom": "2025-01-01T00:00:00Z", + "validTo": "2026-01-01T00:00:00Z" + }, + "links": [ + { + "rel": "self", + "href": "/api/instances/Sdc::1b8659fc00000000" + }, + { + "rel": "/api/Sdc/relationship/Statistics", + "href": "/api/instances/Sdc::1b8659fc00000000/relationships/Statistics" + }, + { + "rel": "/api/Sdc/relationship/Volume", + "href": "/api/instances/Sdc::1b8659fc00000000/relationships/Volume" + }, + { + "rel": "/api/parent/relationship/systemId", + "href": "/api/instances/System::1fcf40fc60c6520f" + } + ] + }, + { + "id": "1b8659fe00000002", + "name": null, + "sdcIp": "10.0.1.228", + "sdcGuid": "46EE0B53-B823-4E68-B0B4-41A2DEC5A425", + "sdcType": "AppSdc", + "peerMdmId": null, + "sdcApproved": true, + "sdcApprovedIps": null, + "sdcAgentActive": true, + "mdmConnectionState": "Connected", + "hostOsFullType": "Linux 5.15.0-91-generic", + "systemId": "1fcf40fc60c6520f", + "softwareVersionInfo": "R4_5.2100.0", + "installedSoftwareVersionInfo": "R4_5.2100.0", + "kernelVersion": "5.15.0-91-generic", + "kernelBuildNumber": null, + "perfProfile": "Default", + "versionInfo": "R4_5.2100.0", + "sdcDefaultInitiatorConnectivityState": "Connected", + "certificateInfo": { + "thumbprint": "FF:EE:DD:CC:BB:AA:99:88:77:66:55:44:33:22:11:00:FF:EE:DD:CC:BB:AA:99:88:77:66:55:44:33:22:11:00", + "subject": "CN=SDC3, O=Dell, C=US", + "issuer": "CN=PowerFlex Root CA, O=Dell, C=US", + "validFrom": "2025-01-01T00:00:00Z", + "validTo": "2026-01-01T00:00:00Z" + }, + "links": [ + { + "rel": "self", + "href": "/api/instances/Sdc::1b8659fe00000002" + }, + { + "rel": "/api/Sdc/relationship/Statistics", + "href": "/api/instances/Sdc::1b8659fe00000002/relationships/Statistics" + }, + { + "rel": "/api/Sdc/relationship/Volume", + "href": "/api/instances/Sdc::1b8659fe00000002/relationships/Volume" + }, + { + "rel": "/api/parent/relationship/systemId", + "href": "/api/instances/System::1fcf40fc60c6520f" + } + ] + } +] diff --git a/dell_powerflex/tests/fixtures/GET/api/types/Sds/instances/response.json b/dell_powerflex/tests/fixtures/GET/api/types/Sds/instances/response.json new file mode 100644 index 0000000000000..da6b2d738629f --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/types/Sds/instances/response.json @@ -0,0 +1,233 @@ +[ + { + "ipList": [ + { + "ip": "10.0.0.1", + "role": "sdsSdsOnly" + }, + { + "ip": "10.0.1.1", + "role": "sdsOnly" + } + ], + "onVmWare": true, + "certificateInfo": { + "thumbprint": "AB:CD:EF:01:23:45:67:89:AB:CD:EF:01:23:45:67:89:AB:CD:EF:01:23:45:67:89:AB:CD:EF:01:23:45:67:89", + "subject": "CN=SDS3, O=Dell, C=US", + "issuer": "CN=PowerFlex Root CA, O=Dell, C=US", + "validFrom": "2025-01-01T00:00:00Z", + "validTo": "2026-01-01T00:00:00Z" + }, + "port": 7072, + "sdsState": "Normal", + "membershipState": "Joined", + "mdmConnectionState": "Connected", + "drlMode": "Volatile", + "rmcacheEnabled": true, + "rmcacheSizeInKb": 131072, + "rmcacheFrozen": false, + "rmcacheMemoryAllocationState": "AllocationPending", + "rfcacheEnabled": false, + "maintenanceState": "NoMaintenance", + "maintenanceType": "NoMaintenance", + "rfcacheErrorDeviceDoesNotExist": false, + "rfcacheErrorLowResources": false, + "rfcacheErrorApiVersionMismatch": false, + "rfcacheErrorInconsistentCacheConfiguration": false, + "rfcacheErrorInconsistentSourceConfiguration": false, + "rfcacheErrorInvalidDriverPath": false, + "authenticationError": "None", + "fglNumConcurrentWrites": 1000, + "fglMetadataCacheState": "Disabled", + "fglMetadataCacheSize": 0, + "numIoBuffers": null, + "faultSetId": "faultset00000001", + "numRestarts": 0, + "lastUpgradeTime": 0, + "sdsDecoupled": null, + "sdsConfigurationFailure": null, + "sdsReceiveBufferAllocationFailures": null, + "softwareVersionInfo": "R4_5.2100.0", + "configuredDrlMode": "Volatile", + "protectionDomainId": "68c139ee00000000", + "protectionDomainName": "domain1", + "name": "SDS3", + "id": "d1c062b700000000", + "links": [ + { + "rel": "self", + "href": "/api/instances/Sds::d1c062b700000000" + }, + { + "rel": "/api/Sds/relationship/Statistics", + "href": "/api/instances/Sds::d1c062b700000000/relationships/Statistics" + }, + { + "rel": "/api/Sds/relationship/Device", + "href": "/api/instances/Sds::d1c062b700000000/relationships/Device" + }, + { + "rel": "/api/Sds/relationship/RfcacheDevice", + "href": "/api/instances/Sds::d1c062b700000000/relationships/RfcacheDevice" + }, + { + "rel": "/api/parent/relationship/protectionDomainId", + "href": "/api/instances/ProtectionDomain::68c139ee00000000" + } + ] + }, + { + "ipList": [ + { + "ip": "10.0.0.2", + "role": "sdsSdsOnly" + }, + { + "ip": "10.0.1.2", + "role": "sdsOnly" + } + ], + "onVmWare": true, + "certificateInfo": { + "thumbprint": "12:34:56:78:9A:BC:DE:F0:12:34:56:78:9A:BC:DE:F0:12:34:56:78:9A:BC:DE:F0:12:34:56:78:9A:BC:DE:F0", + "subject": "CN=SDS2, O=Dell, C=US", + "issuer": "CN=PowerFlex Root CA, O=Dell, C=US", + "validFrom": "2025-01-01T00:00:00Z", + "validTo": "2026-01-01T00:00:00Z" + }, + "port": 7072, + "sdsState": "Normal", + "membershipState": "Joined", + "mdmConnectionState": "Connected", + "drlMode": "Volatile", + "rmcacheEnabled": true, + "rmcacheSizeInKb": 131072, + "rmcacheFrozen": false, + "rmcacheMemoryAllocationState": "AllocationPending", + "rfcacheEnabled": false, + "maintenanceState": "NoMaintenance", + "maintenanceType": "NoMaintenance", + "rfcacheErrorDeviceDoesNotExist": false, + "rfcacheErrorLowResources": false, + "rfcacheErrorApiVersionMismatch": false, + "rfcacheErrorInconsistentCacheConfiguration": false, + "rfcacheErrorInconsistentSourceConfiguration": false, + "rfcacheErrorInvalidDriverPath": false, + "authenticationError": "None", + "fglNumConcurrentWrites": 1000, + "fglMetadataCacheState": "Disabled", + "fglMetadataCacheSize": 0, + "numIoBuffers": null, + "faultSetId": null, + "numRestarts": 0, + "lastUpgradeTime": 0, + "sdsDecoupled": null, + "sdsConfigurationFailure": null, + "sdsReceiveBufferAllocationFailures": null, + "softwareVersionInfo": "R4_5.2100.0", + "configuredDrlMode": "Volatile", + "protectionDomainId": "68c139ee00000000", + "protectionDomainName": "domain1", + "name": "SDS2", + "id": "d1c062b800000001", + "links": [ + { + "rel": "self", + "href": "/api/instances/Sds::d1c062b800000001" + }, + { + "rel": "/api/Sds/relationship/Statistics", + "href": "/api/instances/Sds::d1c062b800000001/relationships/Statistics" + }, + { + "rel": "/api/Sds/relationship/Device", + "href": "/api/instances/Sds::d1c062b800000001/relationships/Device" + }, + { + "rel": "/api/Sds/relationship/RfcacheDevice", + "href": "/api/instances/Sds::d1c062b800000001/relationships/RfcacheDevice" + }, + { + "rel": "/api/parent/relationship/protectionDomainId", + "href": "/api/instances/ProtectionDomain::68c139ee00000000" + } + ] + }, + { + "ipList": [ + { + "ip": "10.0.0.3", + "role": "sdsSdsOnly" + }, + { + "ip": "10.0.1.3", + "role": "sdsOnly" + } + ], + "onVmWare": true, + "certificateInfo": { + "thumbprint": "FE:DC:BA:98:76:54:32:10:FE:DC:BA:98:76:54:32:10:FE:DC:BA:98:76:54:32:10:FE:DC:BA:98:76:54:32:10", + "subject": "CN=SDS1, O=Dell, C=US", + "issuer": "CN=PowerFlex Root CA, O=Dell, C=US", + "validFrom": "2025-01-01T00:00:00Z", + "validTo": "2026-01-01T00:00:00Z" + }, + "port": 7072, + "sdsState": "Normal", + "membershipState": "Joined", + "mdmConnectionState": "Connected", + "drlMode": "Volatile", + "rmcacheEnabled": true, + "rmcacheSizeInKb": 131072, + "rmcacheFrozen": false, + "rmcacheMemoryAllocationState": "AllocationPending", + "rfcacheEnabled": false, + "maintenanceState": "NoMaintenance", + "maintenanceType": "NoMaintenance", + "rfcacheErrorDeviceDoesNotExist": false, + "rfcacheErrorLowResources": false, + "rfcacheErrorApiVersionMismatch": false, + "rfcacheErrorInconsistentCacheConfiguration": false, + "rfcacheErrorInconsistentSourceConfiguration": false, + "rfcacheErrorInvalidDriverPath": false, + "authenticationError": "None", + "fglNumConcurrentWrites": 1000, + "fglMetadataCacheState": "Disabled", + "fglMetadataCacheSize": 0, + "numIoBuffers": null, + "faultSetId": null, + "numRestarts": 0, + "lastUpgradeTime": 0, + "sdsDecoupled": null, + "sdsConfigurationFailure": null, + "sdsReceiveBufferAllocationFailures": null, + "softwareVersionInfo": "R4_5.2100.0", + "configuredDrlMode": "Volatile", + "protectionDomainId": "68c139ee00000000", + "protectionDomainName": "domain1", + "name": "SDS1", + "id": "d1c062b900000002", + "links": [ + { + "rel": "self", + "href": "/api/instances/Sds::d1c062b900000002" + }, + { + "rel": "/api/Sds/relationship/Statistics", + "href": "/api/instances/Sds::d1c062b900000002/relationships/Statistics" + }, + { + "rel": "/api/Sds/relationship/Device", + "href": "/api/instances/Sds::d1c062b900000002/relationships/Device" + }, + { + "rel": "/api/Sds/relationship/RfcacheDevice", + "href": "/api/instances/Sds::d1c062b900000002/relationships/RfcacheDevice" + }, + { + "rel": "/api/parent/relationship/protectionDomainId", + "href": "/api/instances/ProtectionDomain::68c139ee00000000" + } + ] + } +] diff --git a/dell_powerflex/tests/fixtures/GET/api/types/StoragePool/instances/response.json b/dell_powerflex/tests/fixtures/GET/api/types/StoragePool/instances/response.json new file mode 100644 index 0000000000000..406ca5534dc3c --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/types/StoragePool/instances/response.json @@ -0,0 +1,14 @@ +[ + { + "mediaType": "HDD", + "protectionDomainId": "68c139ee00000000", + "name": "storagepool2", + "id": "2515d0d600000001" + }, + { + "mediaType": "SSD", + "protectionDomainId": "68c139ee00000000", + "name": "pool1", + "id": "25155ba600000000" + } +] diff --git a/dell_powerflex/tests/fixtures/GET/api/types/System/instances/response.json b/dell_powerflex/tests/fixtures/GET/api/types/System/instances/response.json new file mode 100644 index 0000000000000..b843002c98a35 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/types/System/instances/response.json @@ -0,0 +1,164 @@ +[ + { + "lastRealignNetPathTime": 0, + "realignNetPathInterval": 0, + "restrictedSdcMode": "None", + "swid": "", + "daysInstalled": 0, + "maxCapacityInGb": "Unlimited", + "enterpriseFeaturesEnabled": true, + "restrictedSdcModeEnabled": false, + "systemVersionName": "DellEMC PowerFlex Version: R4_5.4000.111", + "perfProfile": "HighPerformance", + "authenticationMethod": "Mno", + "installId": "aaabbbcccdddeeef", + "mdmCluster": { + "tieBreakers": [ + { + "opensslVersion": "N/A", + "status": "Normal", + "role": "TieBreaker", + "managementIPs": [ + "10.0.1.228" + ], + "ips": [ + "10.0.1.228" + ], + "versionInfo": "R4_5.4000.0", + "name": "TB1", + "id": "06edf25b7acf5602", + "port": 9011 + } + ], + "slaves": [ + { + "virtualInterfaces": [], + "opensslVersion": "N/A", + "status": "Normal", + "role": "Manager", + "managementIPs": [ + "10.0.1.250" + ], + "ips": [ + "10.0.1.250" + ], + "versionInfo": "R4_5.4000.0", + "name": "MDM1", + "id": "0710bf1106383a00", + "port": 9011 + } + ], + "goodReplicasNum": 2, + "clusterState": "ClusteredNormal", + "clusterMode": "ThreeNodes", + "goodNodesNum": 3, + "master": { + "virtualInterfaces": [], + "opensslVersion": "N/A", + "status": "Normal", + "role": "Manager", + "managementIPs": [ + "10.0.1.223" + ], + "ips": [ + "10.0.1.223" + ], + "versionInfo": "R4_5.4000.0", + "name": "MDM2", + "id": "5ec3c8b17ce02e01", + "port": 9011 + }, + "id": "2292122188054417935" + }, + "isInitialLicense": true, + "capacityTimeLeftInDays": "90", + "sdcSdsConnectivityInfo": { + "clientServerConnectivityStatus": "AllConnected", + "disconnectedClientId": null, + "disconnectedClientName": null, + "disconnectedServerId": null, + "disconnectedServerName": null, + "disconnectedServerIp": null + }, + "sdrSdsConnectivityInfo": { + "clientServerConnectivityStatus": "AllConnected", + "disconnectedClientId": null, + "disconnectedClientName": null, + "disconnectedServerId": null, + "disconnectedServerName": null, + "disconnectedServerIp": null + }, + "sdcSdrConnectivityInfo": { + "clientServerConnectivityStatus": "AllConnected", + "disconnectedClientId": null, + "disconnectedClientName": null, + "disconnectedServerId": null, + "disconnectedServerName": null, + "disconnectedServerIp": null + }, + "addressSpaceUsage": "Normal", + "upgradeState": "NoUpgrade", + "capacityAlertHighThresholdPercent": 80, + "capacityAlertCriticalThresholdPercent": 90, + "mdmSecurityPolicy": "Authentication", + "sdtSdsConnectivityInfo": { + "clientServerConnectivityStatus": "AllConnected", + "disconnectedClientId": null, + "disconnectedClientName": null, + "disconnectedServerId": null, + "disconnectedServerName": null, + "disconnectedServerIp": null + }, + "systemNqn": "nqn.1988-11.com.dell:powerflex:00:aaabbbcccdddeee0", + "remoteReadOnlyLimitState": false, + "mdmManagementPort": 8611, + "mdmExternalPort": 7611, + "sdcMdmNetworkDisconnectionsCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 300}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 500}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 700} + }, + "sdcSdsNetworkDisconnectionsCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 800}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 4000}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 20000} + }, + "sdcMemoryAllocationFailuresCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 300}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 500}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 700} + }, + "sdcSocketAllocationFailuresCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 300}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 500}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 700} + }, + "sdcLongOperationsCounterParameters": { + "shortWindow": {"windowSizeInSec": 60, "threshold": 10000}, + "mediumWindow": {"windowSizeInSec": 3600, "threshold": 100000}, + "longWindow": {"windowSizeInSec": 86400, "threshold": 1000000} + }, + "tlsVersion": "TLSv1.2", + "lastUpgradeTime": 0, + "cliPasswordAllowed": true, + "managementClientSecureCommunicationEnabled": true, + "showGuid": true, + "defragmentationEnabled": true, + "id": "1fcf40fc60c6520f", + "links": [ + {"rel": "self", "href": "/api/instances/System::1fcf40fc60c6520f"}, + {"rel": "/api/System/relationship/Statistics", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/Statistics"}, + {"rel": "/api/System/relationship/Sdr", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/Sdr"}, + {"rel": "/api/System/relationship/ProtectionDomain", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/ProtectionDomain"}, + {"rel": "/api/System/relationship/Sdt", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/Sdt"}, + {"rel": "/api/System/relationship/User", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/User"}, + {"rel": "/api/System/relationship/PeerMdm", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/PeerMdm"}, + {"rel": "/api/System/relationship/RemoteSystem", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/RemoteSystem"}, + {"rel": "/api/System/relationship/HostGroup", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/HostGroup"}, + {"rel": "/api/System/relationship/SnapshotCopyTask", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/SnapshotCopyTask"}, + {"rel": "/api/System/relationship/SystemNetwork", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/SystemNetwork"}, + {"rel": "/api/System/relationship/Sdc", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/Sdc"}, + {"rel": "/api/System/relationship/SnapshotPolicy", "href": "/api/instances/System::1fcf40fc60c6520f/relationships/SnapshotPolicy"} + ] + } +] diff --git a/dell_powerflex/tests/fixtures/GET/api/types/Volume/instances/response.json b/dell_powerflex/tests/fixtures/GET/api/types/Volume/instances/response.json new file mode 100644 index 0000000000000..edce4f809ab01 --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/types/Volume/instances/response.json @@ -0,0 +1,46 @@ +[ + { + "mappedSdcInfo": null, + "name": "volumee-snap-01", + "storagePoolId": "25155ba600000000", + "volumeType": "Snapshot", + "ancestorVolumeId": "c58b06e700000000", + "id": "c58b06e900000002" + }, + { + "mappedSdcInfo": null, + "name": "volumee-snap-02", + "storagePoolId": "25155ba600000000", + "volumeType": "Snapshot", + "ancestorVolumeId": "c58b06e900000002", + "id": "c58b06ea00000003" + }, + { + "mappedSdcInfo": [ + { + "sdcId": "1b8659fd00000001", + "sdcIp": "10.0.1.250", + "accessMode": "ReadWrite" + } + ], + "name": "bigvolume", + "storagePoolId": "25155ba600000000", + "volumeType": "ThinProvisioned", + "ancestorVolumeId": null, + "id": "c58b06e800000001" + }, + { + "mappedSdcInfo": [ + { + "sdcId": "1b8659fd00000001", + "sdcIp": "10.0.1.250", + "accessMode": "ReadWrite" + } + ], + "name": "volumee", + "storagePoolId": "25155ba600000000", + "volumeType": "ThinProvisioned", + "ancestorVolumeId": null, + "id": "c58b06e700000000" + } +] diff --git a/dell_powerflex/tests/fixtures/GET/api/version/response.json b/dell_powerflex/tests/fixtures/GET/api/version/response.json new file mode 100644 index 0000000000000..589fbb1ed6a6b --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/api/version/response.json @@ -0,0 +1 @@ +"4.5" diff --git a/dell_powerflex/tests/fixtures/GET/rest/v1/alerts/response.json b/dell_powerflex/tests/fixtures/GET/rest/v1/alerts/response.json new file mode 100644 index 0000000000000..b572ea2af24fc --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/rest/v1/alerts/response.json @@ -0,0 +1,75 @@ +{ + "results": [ + { + "id": "0000000000000026", + "code": "20100100", + "name": "UNABLE_TO_RECEIVE_MDM_EVENTS", + "description": "Unable to receive mdm events from [10.0.1.250] .", + "severity": "MAJOR", + "original_severity": "MAJOR", + "timestamp": "2026-03-24T08:01:09.588Z", + "resource_type": "mdms", + "domain": "BLOCK", + "resource_name": "MDM1", + "resource_id": "0710bf1106383a00", + "service": "block-events-gw", + "details": { + "ip": "[10.0.1.250]" + }, + "last_updated": "2026-03-24T08:01:09.588Z", + "cleared_timestamp": "0001-01-01T00:00:00Z", + "ack_timestamp": "0001-01-01T00:00:00Z", + "system_impact": "PowerFlex Block events will not appear in the management therefor risking missing problematic states. ", + "repair_flow": "Check the network connection between MDM and M&O, make sure ActiveMQ is running properly on the MDM", + "related_events": [ + { + "id": "84d2a76d0c46f9e9", + "timestamp": "2026-03-24T08:01:09.588Z", + "resulting_severity": "INFORMATION", + "lifecycle_change": "RAISED", + "description": "Unable to connect to MDM Event bus with IP [10.0.1.250]" + } + ], + "category": "MAINTENANCE", + "send_to_srs": true, + "sent_to_srs_timestamp": "2026-03-24T08:01:09.588Z", + "srs_symptom_id": "PFX00005.0000001", + "snmp_code": "block-gateway-config.no-events-from-mdm" + }, + { + "id": "0000000000000024", + "code": "20300003", + "name": "TRIAL_LICENSE_USED", + "description": "PowerFlex is using a trial license", + "severity": "MINOR", + "original_severity": "MINOR", + "timestamp": "2026-03-23T21:14:21.64Z", + "resource_type": "block-config", + "domain": "BLOCK", + "resource_name": "System:1fcf40fc60c6520f", + "resource_id": "1fcf40fc60c6520f", + "service": "block-alerts-server", + "details": { + "system-id": "1fcf40fc60c6520f" + }, + "last_updated": "2026-03-23T21:14:21.64Z", + "cleared_timestamp": "0001-01-01T00:00:00Z", + "ack_timestamp": "0001-01-01T00:00:00Z", + "repair_flow": "Purchase a license and install it.", + "related_events": [ + { + "id": "afb0a6e3e05add68", + "timestamp": "2026-03-23T21:14:21.64Z", + "resulting_severity": "MINOR", + "lifecycle_change": "RAISED", + "description": "PowerFlex is using a trial license" + } + ], + "category": "SOFTWARE", + "send_to_srs": true, + "sent_to_srs_timestamp": "2026-03-23T21:14:21.64Z", + "srs_symptom_id": "SIO01.02.0000003", + "snmp_code": "System.License.Trial_License_Used" + } + ] +} diff --git a/dell_powerflex/tests/fixtures/GET/rest/v1/events/response.json b/dell_powerflex/tests/fixtures/GET/rest/v1/events/response.json new file mode 100644 index 0000000000000..0ecf38978e44d --- /dev/null +++ b/dell_powerflex/tests/fixtures/GET/rest/v1/events/response.json @@ -0,0 +1,181 @@ +{ + "paging": { + "total_instances": 5, + "first": 1, + "last": 5, + "next": "", + "prev": "" + }, + "results": [ + { + "code": "70020024", + "name": "HEALTH_CHECK_FAILED", + "description": "Health check failed: SDNAS Gateway pod failed to response.", + "severity": "CRITICAL", + "category": "AUDIT", + "details": { + "msgcode": "VXFM00801", + "message": "SDNAS Gateway pod failed to response" + }, + "domain": "MANAGEMENT", + "id": "05133dcbf3eb8d48", + "timestamp": "2026-03-18T03:40:16.253Z", + "resource_type": "basic-system-config", + "resource_name": "pfm-asmmanager", + "resource_id": "70020001", + "service_name": "pfm-asmmanager", + "service_version": "1.0", + "service_instance_id": "e1ee87c1-4013-4525-8c2e-5908680f422f", + "originating_application_name": null, + "request_id": null, + "related_events": null, + "job_id": null, + "trace_parent": null, + "trace_state": null, + "source_ip": null, + "user_id": null, + "is_internal": false + }, + { + "code": "60010020", + "name": "UNKNOWN_SNMP_TRAP", + "description": "MDS has sent a severity level CRITICAL trap alert type:MDM.MDM_Cluster.MDM_CONNECTION_LOST.", + "severity": "CRITICAL", + "category": "SOFTWARE", + "details": { + "sysUpTime.0": "4032047", + "snmpTrapOID.0": "SCALEIO-MIB::scaleioAEAlert", + "scaleioAlertSeverity": "5", + "scaleioAlertType": "MDM.MDM_Cluster.MDM_CONNECTION_LOST", + "scaleioAlertSourceObjectId": "0", + "scaleioAlertActionCode": "SIO02.01.0000008", + "installationID": "N/A", + "systemName": "N/A", + "device_type": "Unknown", + "symptomCode": null, + "srs_symptom_id": "SIO02.01.0000008", + "snmp_code": "MDM.MDM_Cluster.MDM_CONNECTION_LOST", + "instance": "MDS", + "alert_severity": "5", + "trap": "{\"1.3.6.1.2.1.1.3.0\": \"4032047\", \"1.3.6.1.6.3.1.1.4.1.0\": \"1.3.6.1.4.1.1139.101.2\", \"1.3.6.1.4.1.1139.101.1.1\": \"5\", \"1.3.6.1.4.1.1139.101.1.2\": \"MDM.MDM_Cluster.MDM_CONNECTION_LOST\", \"1.3.6.1.4.1.1139.101.1.3\": \"0\", \"1.3.6.1.4.1.1139.101.1.4\": \"SIO02.01.0000008\", \"1.3.6.1.4.1.1139.101.1.6\": \"N/A\", \"1.3.6.1.4.1.1139.101.1.7\": \"N/A\"}", + "source_ip": "10.42.1.14" + }, + "domain": "MANAGEMENT", + "id": "8b96af0117be8860", + "timestamp": "2026-03-19T02:31:25.057Z", + "resource_type": "block-gateway-config", + "resource_name": "10.42.1.14", + "resource_id": "0", + "service_name": "pfx-notifications", + "service_version": null, + "service_instance_id": "0xbd1f8f456d5f414d", + "originating_application_name": "10.42.1.14", + "request_id": null, + "related_events": null, + "job_id": null, + "trace_parent": null, + "trace_state": null, + "source_ip": null, + "user_id": null, + "is_internal": false + }, + { + "code": "70400009", + "name": "POSTGRES_INSTANCE_DIFFERENT_TIMELINE", + "description": "Postgres instance postgres-ha-cmo1-m2c2-0 is not on same timeline as leader. postgres-ha-cmo1-m2c2-0 is on timeline undefined while leader is on timeline 4.", + "severity": "MAJOR", + "category": "SOFTWARE", + "details": null, + "domain": "MANAGEMENT", + "id": "762c18de36d39756", + "timestamp": "2026-03-18T15:19:52.181Z", + "resource_type": "power-flex-config", + "resource_name": "postgres-ha-cmo1-m2c2-0", + "resource_id": "0", + "service_name": "pfx-monitor", + "service_version": "1.0", + "service_instance_id": "d937f414-28e8-4c22-aba0-be63fa78cc33", + "originating_application_name": "pfx-monitor", + "request_id": null, + "related_events": null, + "job_id": null, + "trace_parent": null, + "trace_state": null, + "source_ip": null, + "user_id": null, + "is_internal": false + }, + { + "code": "70020024", + "name": "HEALTH_CHECK_FAILED", + "description": "Health check failed: I/O error on GET request for \"https://thin-deployer.powerflex.svc:9433/asm/status\": Connection refused (Connection refused); nested exception is java.net.ConnectException: Connection refused (Connection refused).", + "severity": "MINOR", + "category": "AUDIT", + "details": { + "msgcode": "VXFM00801", + "message": "I/O error on GET request for \"https://thin-deployer.powerflex.svc:9433/asm/status\": Connection refused (Connection refused); nested exception is java.net.ConnectException: Connection refused (Connection refused)" + }, + "domain": "MANAGEMENT", + "id": "2e43880ad98c4806", + "timestamp": "2026-03-18T15:20:52.067Z", + "resource_type": "basic-system-config", + "resource_name": "pfm-asmmanager", + "resource_id": "70020001", + "service_name": "pfm-asmmanager", + "service_version": "1.0", + "service_instance_id": "01e786db-5494-4357-8f4e-d8070d1032da", + "originating_application_name": null, + "request_id": null, + "related_events": null, + "job_id": null, + "trace_parent": null, + "trace_state": null, + "source_ip": null, + "user_id": null, + "is_internal": false + }, + { + "code": "60010020", + "name": "UNKNOWN_SNMP_TRAP", + "description": "MDS has sent a severity level CRITICAL trap alert type:MDM.MDM_Cluster.MDM_CONNECTION_LOST.", + "severity": "CRITICAL", + "category": "SOFTWARE", + "details": { + "sysUpTime.0": "3630057", + "snmpTrapOID.0": "SCALEIO-MIB::scaleioAEAlert", + "scaleioAlertSeverity": "5", + "scaleioAlertType": "MDM.MDM_Cluster.MDM_CONNECTION_LOST", + "scaleioAlertSourceObjectId": "0", + "scaleioAlertActionCode": "SIO02.01.0000008", + "installationID": "N/A", + "systemName": "N/A", + "device_type": "Unknown", + "symptomCode": null, + "srs_symptom_id": "SIO02.01.0000008", + "snmp_code": "MDM.MDM_Cluster.MDM_CONNECTION_LOST", + "instance": "MDS", + "alert_severity": "5", + "trap": "{\"1.3.6.1.2.1.1.3.0\": \"3630057\", \"1.3.6.1.6.3.1.1.4.1.0\": \"1.3.6.1.4.1.1139.101.2\", \"1.3.6.1.4.1.1139.101.1.1\": \"5\", \"1.3.6.1.4.1.1139.101.1.2\": \"MDM.MDM_Cluster.MDM_CONNECTION_LOST\", \"1.3.6.1.4.1.1139.101.1.3\": \"0\", \"1.3.6.1.4.1.1139.101.1.4\": \"SIO02.01.0000008\", \"1.3.6.1.4.1.1139.101.1.6\": \"N/A\", \"1.3.6.1.4.1.1139.101.1.7\": \"N/A\"}", + "source_ip": "10.42.1.40" + }, + "domain": "MANAGEMENT", + "id": "bbf248e092c16f1e", + "timestamp": "2026-03-18T14:05:13.966Z", + "resource_type": "block-gateway-config", + "resource_name": "10.42.1.40", + "resource_id": "0", + "service_name": "pfx-notifications", + "service_version": null, + "service_instance_id": "0xaf50474e0c44f164", + "originating_application_name": "10.42.1.40", + "request_id": null, + "related_events": null, + "job_id": null, + "trace_parent": null, + "trace_state": null, + "source_ip": null, + "user_id": null, + "is_internal": false + } + ] +} \ No newline at end of file diff --git a/dell_powerflex/tests/test_e2e.py b/dell_powerflex/tests/test_e2e.py new file mode 100644 index 0000000000000..9bf028a7bb73c --- /dev/null +++ b/dell_powerflex/tests/test_e2e.py @@ -0,0 +1,23 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from datadog_checks.dev.utils import get_metadata_metrics + +from .common import ALL_EXPECTED_METRICS + + +@pytest.mark.e2e +def test_e2e(dd_agent_check): + aggregator = dd_agent_check() + + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + + for metric in ALL_EXPECTED_METRICS: + aggregator.assert_metric(metric['name'], value=metric['value']) + for tag in metric['tags']: + aggregator.assert_metric_has_tag(metric['name'], tag) + + aggregator.assert_all_metrics_covered() + aggregator.assert_metrics_using_metadata(get_metadata_metrics()) diff --git a/dell_powerflex/tests/test_unit.py b/dell_powerflex/tests/test_unit.py new file mode 100644 index 0000000000000..17a7f8ddeffcd --- /dev/null +++ b/dell_powerflex/tests/test_unit.py @@ -0,0 +1,797 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import logging +from unittest.mock import MagicMock + +import pytest +from requests.exceptions import ConnectionError, HTTPError + +from datadog_checks.dell_powerflex import DellPowerflexCheck +from datadog_checks.dev.utils import get_metadata_metrics + +from .common import ( + ALL_EXPECTED_METRICS, + BASE_TAGS, + DEFAULT_GATEWAY_URL, + DEV1_TAGS, + DEV2_TAGS, + DEV3_TAGS, + DEVICE_ONLY_METRICS, + DEVICE_STATS_BWC_METRICS, + DEVICE_STATS_SIMPLE_METRICS, + PD_TAGS, + POOL1_TAGS, + POOL2_TAGS, + PROTECTION_DOMAIN_STATS_BWC_METRICS, + PROTECTION_DOMAIN_STATS_SIMPLE_METRICS, + SDC1_TAGS, + SDC2_TAGS, + SDC3_TAGS, + SDC_STATS_BWC_METRICS, + SDC_STATS_SIMPLE_METRICS, + SDS1_TAGS, + SDS2_TAGS, + SDS3_TAGS, + SDS_STATS_BWC_METRICS, + SDS_STATS_SIMPLE_METRICS, + STORAGE_POOL_STATS_BWC_METRICS, + STORAGE_POOL_STATS_SIMPLE_METRICS, + SYSTEM_MDM_CLUSTER_METRICS, + SYSTEM_STATS_BWC_METRICS, + SYSTEM_STATS_SIMPLE_METRICS, + SYSTEM_TAGS, + VOL_BIGVOLUME_TAGS, + VOL_SNAP1_TAGS, + VOL_SNAP2_TAGS, + VOL_VOLUMEE_TAGS, + VOLUME_STATS_BWC_METRICS, + VOLUME_STATS_SIMPLE_METRICS, +) + +pytestmark = [pytest.mark.unit] + + +def assert_bwc_metrics(aggregator, bwc_metrics, tags, value=0): + for metric_prefix in bwc_metrics: + aggregator.assert_metric(f'{metric_prefix}.num_seconds', value=value, tags=tags) + aggregator.assert_metric(f'{metric_prefix}.total_weight_in_kb', value=value, tags=tags) + aggregator.assert_metric(f'{metric_prefix}.num_occured', value=value, tags=tags) + + +def test_can_connect_down(dd_run_check, aggregator, instance, mocker): + mocker.patch('requests.Session.post', side_effect=ConnectionError('connection refused')) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.api.can_connect', value=0, tags=BASE_TAGS) + + +def test_auth_response_missing_access_token(dd_run_check, aggregator, instance, mocker, caplog): + mocker.patch( + 'requests.Session.post', + return_value=MagicMock( + raise_for_status=MagicMock(), + json=MagicMock(return_value={'error': 'unauthorized_client'}), + status_code=200, + ), + ) + caplog.set_level(logging.WARNING) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.api.can_connect', value=0, tags=BASE_TAGS) + assert 'Auth response missing access_token' in caplog.text + + +def test_version_failure(dd_run_check, aggregator, instance, mock_auth, mocker, caplog): + mocker.patch('requests.Session.get', side_effect=HTTPError(response=MagicMock(status_code=500))) + + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + caplog.set_level(logging.WARNING) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.api.can_connect', value=0, tags=BASE_TAGS) + assert 'Could not connect to PowerFlex Gateway' in caplog.text + + +def test_can_connect_up(dd_run_check, aggregator, instance, mock_auth, mocker): + mocker.patch('requests.Session.get', return_value=MagicMock(raise_for_status=MagicMock())) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1, tags=BASE_TAGS) + + +def test_unauthenticated_mode(dd_run_check, aggregator, mock_http_call, mocker): + instance = {'powerflex_gateway_url': DEFAULT_GATEWAY_URL} + mocker.patch( + 'requests.Session.get', + side_effect=lambda url, *args, **kwargs: MagicMock( + json=MagicMock(return_value=mock_http_call(url)), status_code=200 + ), + ) + mock_post = mocker.patch('requests.Session.post') + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + mock_post.assert_not_called() + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', at_least=1) + aggregator.assert_metric('dell_powerflex.system.count', at_least=1) + aggregator.assert_metric('dell_powerflex.storage_pool.count', at_least=1) + aggregator.assert_metric('dell_powerflex.volume.count', at_least=1) + + +def test_token_refresh_uses_min_collection_interval(dd_run_check, instance, mock_http_get, mocker): + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + + dd_run_check(check) + spy = mocker.spy(check._api, '_authenticate') + + # Token still valid — no re-auth + dd_run_check(check) + assert spy.call_count == 0 + + # Simulate token nearing expiry + mocker.patch('datadog_checks.dell_powerflex.api.time', return_value=check._api._token_expiry - 10) + dd_run_check(check) + assert spy.call_count == 1 + + +def test_collect_system(dd_run_check, aggregator, instance, mock_http_get): + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + system_tags = BASE_TAGS + SYSTEM_TAGS + + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1, tags=BASE_TAGS) + aggregator.assert_metric('dell_powerflex.system.count', value=1, tags=system_tags) + + for metric in SYSTEM_MDM_CLUSTER_METRICS: + aggregator.assert_metric( + metric['name'], + value=metric['value'], + tags=system_tags + metric.get('extra_tags', []), + ) + + for metric in SYSTEM_STATS_SIMPLE_METRICS: + aggregator.assert_metric(metric['name'], value=metric['value'], tags=system_tags) + + assert_bwc_metrics( + aggregator, [m for m in SYSTEM_STATS_BWC_METRICS if m != 'dell_powerflex.user_data_read_bwc'], system_tags + ) + # userDataReadBwc fixture has numOccured=42 + aggregator.assert_metric('dell_powerflex.user_data_read_bwc.num_seconds', value=0, tags=system_tags) + aggregator.assert_metric('dell_powerflex.user_data_read_bwc.total_weight_in_kb', value=0, tags=system_tags) + aggregator.assert_metric('dell_powerflex.user_data_read_bwc.num_occured', value=42, tags=system_tags) + + +def test_assert_all_metrics(dd_run_check, aggregator, instance, mock_http_get): + instance['resource_filters'] = [ + {'resource': 'device', 'property': 'name', 'patterns': ['.*'], 'collect_statistics': True}, + ] + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + for metric in ALL_EXPECTED_METRICS: + aggregator.assert_metric(metric['name'], value=metric['value'], tags=BASE_TAGS + metric['tags']) + + aggregator.assert_metrics_using_metadata( + get_metadata_metrics(), check_symmetric_inclusion=True, check_submission_type=True + ) + + +def test_device_statistics_disabled_by_default(dd_run_check, aggregator, instance, mock_http_get): + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + for metric in DEVICE_ONLY_METRICS: + aggregator.assert_metric(metric, count=0) + + +def test_collect_volumes(dd_run_check, aggregator, instance, mock_http_get): + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + # volumee: ThinProvisioned, mapped to one SDC, no ancestor + volume_tags = BASE_TAGS + VOL_VOLUMEE_TAGS + aggregator.assert_metric('dell_powerflex.volume.count', value=1, tags=volume_tags) + for metric in VOLUME_STATS_SIMPLE_METRICS: + aggregator.assert_metric(metric['name'], value=metric['value'], tags=volume_tags) + assert_bwc_metrics(aggregator, VOLUME_STATS_BWC_METRICS, volume_tags) + aggregator.assert_metric( + 'dell_powerflex.volume.sdc_mapping', value=1, tags=volume_tags + ['sdc_id:1b8659fd00000001'] + ) + + # bigvolume: ThinProvisioned, mapped to one SDC, no children + bigvolume_tags = BASE_TAGS + VOL_BIGVOLUME_TAGS + aggregator.assert_metric('dell_powerflex.volume.count', value=1, tags=bigvolume_tags) + aggregator.assert_metric('dell_powerflex.num_of_child_volumes', value=0, tags=bigvolume_tags) + aggregator.assert_metric('dell_powerflex.num_of_mapped_sdcs', value=1, tags=bigvolume_tags) + aggregator.assert_metric( + 'dell_powerflex.volume.sdc_mapping', value=1, tags=bigvolume_tags + ['sdc_id:1b8659fd00000001'] + ) + + for snap_resource_tags, children in [(VOL_SNAP1_TAGS, 1), (VOL_SNAP2_TAGS, 0)]: + snap_tags = BASE_TAGS + snap_resource_tags + aggregator.assert_metric('dell_powerflex.volume.count', value=1, tags=snap_tags) + aggregator.assert_metric('dell_powerflex.num_of_child_volumes', value=children, tags=snap_tags) + aggregator.assert_metric('dell_powerflex.num_of_mapped_sdcs', value=0, tags=snap_tags) + + +def test_collect_storage_pools(dd_run_check, aggregator, instance, mock_http_get): + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + pool_tags = BASE_TAGS + POOL1_TAGS + aggregator.assert_metric('dell_powerflex.storage_pool.count', value=1, tags=pool_tags) + for metric in STORAGE_POOL_STATS_SIMPLE_METRICS: + aggregator.assert_metric(metric['name'], value=metric['value'], tags=pool_tags) + assert_bwc_metrics(aggregator, STORAGE_POOL_STATS_BWC_METRICS, pool_tags) + + # storagepool2: HDD, empty pool, no ActualNetCapacityInUseInKb + pool2_tags = BASE_TAGS + POOL2_TAGS + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', value=0, tags=pool2_tags) + aggregator.assert_metric('dell_powerflex.max_capacity.in_kb', value=0, tags=pool2_tags) + aggregator.assert_metric('dell_powerflex.num_of_volumes', value=0, tags=pool2_tags) + assert_bwc_metrics(aggregator, STORAGE_POOL_STATS_BWC_METRICS, pool2_tags) + + +def test_collect_protection_domains(dd_run_check, aggregator, instance, mock_http_get): + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + pd_tags = BASE_TAGS + PD_TAGS + aggregator.assert_metric('dell_powerflex.protection_domain.count', value=1, tags=pd_tags) + for metric in PROTECTION_DOMAIN_STATS_SIMPLE_METRICS: + aggregator.assert_metric(metric['name'], value=metric['value'], tags=pd_tags) + assert_bwc_metrics(aggregator, PROTECTION_DOMAIN_STATS_BWC_METRICS, pd_tags) + + +@pytest.mark.parametrize( + 'method, log_message', + [ + ('_collect_system', 'Failed to collect metrics for system'), + ('_collect_volume', 'Failed to collect metrics for volume'), + ('_collect_storage_pool', 'Failed to collect metrics for storage pool'), + ('_collect_protection_domain', 'Failed to collect metrics for protection domain'), + ('_collect_sds', 'Failed to collect metrics for SDS'), + ('_collect_sdc', 'Failed to collect metrics for SDC'), + ('_collect_device', 'Failed to collect metrics for device'), + ], +) +def test_resource_collect_failure( + dd_run_check, aggregator, instance, mock_http_get, mocker, caplog, method, log_message +): + mocker.patch( + f'datadog_checks.dell_powerflex.check.DellPowerflexCheck.{method}', + side_effect=Exception(), + ) + caplog.set_level(logging.WARNING) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + assert log_message in caplog.text + + +def test_collector_failure_does_not_stop_next_collectors( + dd_run_check, aggregator, instance, mock_http_get, mocker, caplog +): + mocker.patch( + 'datadog_checks.dell_powerflex.api.PowerFlexAPI.get_sds_list', + side_effect=Exception('API error'), + ) + caplog.set_level(logging.WARNING) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + assert 'Failed during _collect_sds_list collection' in caplog.text + aggregator.assert_metric('dell_powerflex.storage_pool.count', value=1, tags=BASE_TAGS + POOL1_TAGS) + + +def test_collect_sds(dd_run_check, aggregator, instance, mock_http_get): + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + # SDS3: d1c062b700000000, has fault_set_id + sds3_tags = BASE_TAGS + SDS3_TAGS + aggregator.assert_metric('dell_powerflex.sds.count', value=1, tags=sds3_tags) + for metric in SDS_STATS_SIMPLE_METRICS: + aggregator.assert_metric(metric['name'], value=metric['value'], tags=sds3_tags) + assert_bwc_metrics(aggregator, SDS_STATS_BWC_METRICS, sds3_tags) + + for sds_resource_tags, cap, unused in [ + (SDS2_TAGS, 350208, 103406592), + (SDS1_TAGS, 349184, 103407616), + ]: + tags = BASE_TAGS + sds_resource_tags + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', value=cap, tags=tags) + aggregator.assert_metric('dell_powerflex.unused_capacity.in_kb', value=unused, tags=tags) + aggregator.assert_metric('dell_powerflex.num_of_devices', value=1, tags=tags) + assert_bwc_metrics(aggregator, SDS_STATS_BWC_METRICS, tags) + + +def test_collect_sdc(dd_run_check, aggregator, instance, mock_http_get): + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + # SDC1: 1b8659fd00000001, numOfMappedVolumes=2 + sdc1_tags = BASE_TAGS + SDC1_TAGS + aggregator.assert_metric('dell_powerflex.sdc.count', value=1, tags=sdc1_tags) + for metric in SDC_STATS_SIMPLE_METRICS: + aggregator.assert_metric(metric['name'], value=metric['value'], tags=sdc1_tags) + assert_bwc_metrics(aggregator, SDC_STATS_BWC_METRICS, sdc1_tags) + + for sdc_resource_tags in [SDC2_TAGS, SDC3_TAGS]: + tags = BASE_TAGS + sdc_resource_tags + aggregator.assert_metric('dell_powerflex.num_of_mapped_volumes', value=0, tags=tags) + assert_bwc_metrics(aggregator, SDC_STATS_BWC_METRICS, tags) + + +def test_collect_devices(dd_run_check, aggregator, instance, mock_http_get): + instance['resource_filters'] = [ + {'resource': 'device', 'property': 'name', 'patterns': ['.*'], 'collect_statistics': True}, + ] + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + # Device1: f7fd7d0b00020000, sds1-dev1 - full assertions + dev1_tags = BASE_TAGS + DEV1_TAGS + aggregator.assert_metric('dell_powerflex.device.count', value=1, tags=dev1_tags) + for metric in DEVICE_STATS_SIMPLE_METRICS: + aggregator.assert_metric(metric['name'], value=metric['value'], tags=dev1_tags) + assert_bwc_metrics(aggregator, DEVICE_STATS_BWC_METRICS, dev1_tags) + + for dev_resource_tags, cap, latency in [ + (DEV2_TAGS, 350208, 12793), + (DEV3_TAGS, 349184, 10023), + ]: + tags = BASE_TAGS + dev_resource_tags + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', value=cap, tags=tags) + aggregator.assert_metric('dell_powerflex.avg_read_latency_in_microsec', value=latency, tags=tags) + assert_bwc_metrics(aggregator, DEVICE_STATS_BWC_METRICS, tags) + + +def test_collect_system_with_name(dd_run_check, aggregator, instance, mock_http_get, mock_responses): + mock_responses(f'{DEFAULT_GATEWAY_URL}/api/types/System/instances')[0]['name'] = 'my-powerflex' + + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + system_tags = BASE_TAGS + SYSTEM_TAGS + ['system_name:my-powerflex'] + aggregator.assert_metric('dell_powerflex.mdm_cluster.good_nodes', value=3, tags=system_tags) + aggregator.assert_metric('dell_powerflex.mdm_cluster.good_replicas', value=2, tags=system_tags) + + +def test_include_filter_by_name(dd_run_check, aggregator, instance, mock_http_get, caplog): + instance['resource_filters'] = [ + {'resource': 'storage_pool', 'property': 'name', 'patterns': ['^pool1$']}, + ] + caplog.set_level(logging.DEBUG) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=1, tags=BASE_TAGS + POOL1_TAGS) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=0, tags=BASE_TAGS + POOL2_TAGS) + assert 'Skipping storage_pool storagepool2: did not match any include pattern' in caplog.text + + +def test_exclude_filter_by_name(dd_run_check, aggregator, instance, mock_http_get, caplog): + instance['resource_filters'] = [ + {'resource': 'sds', 'property': 'name', 'type': 'exclude', 'patterns': ['^SDS3$']}, + ] + caplog.set_level(logging.DEBUG) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=0, tags=BASE_TAGS + SDS3_TAGS) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=1, tags=BASE_TAGS + SDS2_TAGS) + assert 'Skipping sds SDS3: matched exclude pattern' in caplog.text + + +def test_exclude_takes_precedence_over_include(dd_run_check, aggregator, instance, mock_http_get): + instance['resource_filters'] = [ + {'resource': 'storage_pool', 'property': 'name', 'patterns': ['.*']}, + {'resource': 'storage_pool', 'property': 'name', 'type': 'exclude', 'patterns': ['^pool1$']}, + ] + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=0, tags=BASE_TAGS + POOL1_TAGS) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', tags=BASE_TAGS + POOL2_TAGS) + + +@pytest.mark.parametrize( + 'resource_filters', + [ + pytest.param( + [{'resource': 'sds', 'property': 'name', 'patterns': ['.*'], 'collect_statistics': False}], + id='with_patterns', + ), + pytest.param( + [{'resource': 'sds', 'property': 'name', 'collect_statistics': False}], + id='without_patterns', + ), + ], +) +def test_collect_statistics_false(dd_run_check, aggregator, instance, mock_http_get, resource_filters): + instance['resource_filters'] = resource_filters + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + sds3_tags = BASE_TAGS + SDS3_TAGS + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + aggregator.assert_metric('dell_powerflex.sds.count', value=1, tags=sds3_tags) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=0, tags=sds3_tags) + + +def test_filter_by_volume_type(dd_run_check, aggregator, instance, mock_http_get): + instance['resource_filters'] = [ + {'resource': 'volume', 'property': 'volumeType', 'patterns': ['^ThinProvisioned$']}, + ] + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.num_of_child_volumes', tags=BASE_TAGS + VOL_VOLUMEE_TAGS) + aggregator.assert_metric('dell_powerflex.num_of_child_volumes', count=0, tags=BASE_TAGS + VOL_SNAP1_TAGS) + + +def test_unfiltered_resources_not_affected(dd_run_check, aggregator, instance, mock_http_get): + instance['resource_filters'] = [ + {'resource': 'sds', 'property': 'name', 'patterns': ['^nonexistent$']}, + ] + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=0, tags=BASE_TAGS + SDS3_TAGS) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', tags=BASE_TAGS + POOL1_TAGS) + + +def test_multiple_filters_same_resource_type(dd_run_check, aggregator, instance, mock_http_get): + instance['resource_filters'] = [ + {'resource': 'sds', 'property': 'name', 'patterns': ['^SDS[12]$']}, + {'resource': 'sds', 'property': 'id', 'type': 'exclude', 'patterns': ['^d1c062b800000001$']}, + ] + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=1, tags=BASE_TAGS + SDS1_TAGS) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=0, tags=BASE_TAGS + SDS2_TAGS) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=0, tags=BASE_TAGS + SDS3_TAGS) + + +@pytest.mark.parametrize( + 'resource_filters, log_message', + [ + pytest.param( + [{'resource': 'invalid_type', 'property': 'name', 'patterns': ['.*']}], + 'Invalid resource type', + id='invalid_resource_type', + ), + pytest.param( + [{'resource': 'sds', 'property': '', 'patterns': ['.*']}], + 'Missing or invalid property', + id='missing_property_in_filter', + ), + pytest.param( + [{'resource': 'sds', 'property': 'name'}], + 'No valid patterns', + id='no_valid_patterns', + ), + pytest.param( + [{'resource': 'sds', 'property': 'name', 'patterns': ['[invalid']}], + 'Invalid regex pattern', + id='invalid_regex', + ), + pytest.param( + [{'resource': 'sds', 'property': 'name', 'type': 'bad', 'patterns': ['.*']}], + 'Invalid filter type', + id='invalid_filter_type', + ), + ], +) +def test_filter_validation_warning( + dd_run_check, aggregator, instance, mock_http_get, caplog, resource_filters, log_message +): + instance['resource_filters'] = resource_filters + caplog.set_level(logging.WARNING) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + assert log_message in caplog.text + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + + +def test_invalid_filter_type_is_skipped(dd_run_check, aggregator, instance, mock_http_get): + # test that a restrictive pattern with an invalid type is skipped + instance['resource_filters'] = [ + {'resource': 'sds', 'property': 'name', 'type': 'exculde', 'patterns': ['^nonexistent$']}, + ] + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + aggregator.assert_metric('dell_powerflex.sds.count', at_least=1) + + +def test_include_filter_missing_property(dd_run_check, aggregator, instance, mock_http_get, caplog): + instance['resource_filters'] = [ + {'resource': 'sds', 'property': 'nonexistent_field', 'patterns': ['.*']}, + ] + caplog.set_level(logging.DEBUG) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + aggregator.assert_metric('dell_powerflex.sds.count', count=0) + assert 'property nonexistent_field not found' in caplog.text + + +@pytest.mark.parametrize( + 'resource_filters', + [ + pytest.param( + [{'resource': 'sds', 'property': 'nonexistent_field', 'type': 'exclude', 'patterns': ['.*']}], + id='exclude_filter_missing_property', + ), + pytest.param( + [{'resource': 'sds', 'property': 'name', 'patterns': [123, '.*']}], + id='non_string_pattern_skipped', + ), + ], +) +def test_invalid_filter_still_collects_metrics(dd_run_check, aggregator, instance, mock_http_get, resource_filters): + instance['resource_filters'] = resource_filters + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + sds3_tags = BASE_TAGS + SDS3_TAGS + aggregator.assert_metric('dell_powerflex.sds.count', value=1, tags=sds3_tags) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', tags=sds3_tags) + + +@pytest.mark.parametrize( + 'resource, property, stats_metric', + [ + ('volume', 'name', 'dell_powerflex.num_of_child_volumes'), + ('storage_pool', 'name', 'dell_powerflex.capacity.in_use_in_kb'), + ('protection_domain', 'name', 'dell_powerflex.capacity.in_use_in_kb'), + ('sds', 'name', 'dell_powerflex.capacity.in_use_in_kb'), + ('sdc', 'sdcType', 'dell_powerflex.num_of_mapped_volumes'), + ('device', 'name', 'dell_powerflex.capacity.in_use_in_kb'), + ], +) +def test_collect_statistics_false_per_resource( + dd_run_check, aggregator, instance, mock_http_get, resource, property, stats_metric +): + instance['resource_filters'] = [ + {'resource': resource, 'property': property, 'patterns': ['.*'], 'collect_statistics': False}, + ] + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + aggregator.assert_metric(f'dell_powerflex.{resource}.count', at_least=1) + aggregator.assert_metric_has_tag(stats_metric, f'dell_type:{resource}', count=0) + + +@pytest.mark.parametrize( + 'resource, property, patterns', + [ + ('protection_domain', 'name', ['^nonexistent$']), + ('sdc', 'sdcIp', ['^192\\.168\\.']), + ('device', 'name', ['^nonexistent$']), + ], +) +def test_filter_excludes_all_resources(dd_run_check, aggregator, instance, mock_http_get, resource, property, patterns): + instance['resource_filters'] = [ + {'resource': resource, 'property': property, 'patterns': patterns}, + ] + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + aggregator.assert_metric(f'dell_powerflex.{resource}.count', count=0) + + +def test_collect_events(dd_run_check, aggregator, instance, mock_http_get): + instance['collect_events'] = True + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + events = aggregator.events + # Fixture has 3 CRITICAL + 1 MAJOR + 1 MINOR; filter keeps CRITICAL and MAJOR only + assert len(events) == 4 + for event in events: + assert event['alert_type'] == 'error' + assert event['event_type'] == 'dell_powerflex.event' + assert event['source_type_name'] == 'dell-powerflex' + assert f'powerflex_gateway_url:{DEFAULT_GATEWAY_URL}' in event['tags'] + + severities = {tag for e in events for tag in e['tags'] if tag.startswith('severity:')} + assert severities == {'severity:CRITICAL', 'severity:MAJOR'} + + titles = [e['msg_title'] for e in events] + assert 'Health Check Failed' in titles + assert 'Unknown Snmp Trap' in titles + assert 'Postgres Instance Different Timeline' in titles + + health_check_event = next(e for e in events if e['msg_title'] == 'Health Check Failed') + assert ( + 'pfm-asmmanager: Health check failed: SDNAS Gateway pod failed to response.' in health_check_event['msg_text'] + ) + assert 'powerflex_event_name:HEALTH_CHECK_FAILED' in health_check_event['tags'] + assert 'category:AUDIT' in health_check_event['tags'] + assert 'domain:MANAGEMENT' in health_check_event['tags'] + assert 'service_name:pfm-asmmanager' in health_check_event['tags'] + + major_event = next(e for e in events if 'severity:MAJOR' in e['tags']) + assert major_event['msg_title'] == 'Postgres Instance Different Timeline' + assert check.read_persistent_cache('last_event_timestamp') is not None + + +def test_collect_events_subsequent_run_uses_cached_time(dd_run_check, aggregator, instance, mock_http_get, mocker): + instance['collect_events'] = True + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + cached_timestamp = check.read_persistent_cache('last_event_timestamp') + spy = mocker.spy(check._api, 'get_events') + dd_run_check(check) + + assert spy.call_args.kwargs['since'] == cached_timestamp + + +def test_collect_alerts_subsequent_run_uses_cached_time(dd_run_check, aggregator, instance, mock_http_get, mocker): + instance['collect_alerts'] = True + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + cached_timestamp = check.read_persistent_cache('last_alert_timestamp') + spy = mocker.spy(check._api, 'get_alerts') + dd_run_check(check) + + assert spy.call_args.kwargs['since'] == cached_timestamp + + +@pytest.mark.parametrize('config_key', ['collect_events', 'collect_alerts']) +def test_collect_disabled(dd_run_check, aggregator, instance, mock_http_get, config_key): + instance[config_key] = False + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + assert len(aggregator.events) == 0 + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + + +@pytest.mark.parametrize( + 'config_key, mock_target, log_message', + [ + ('collect_events', 'datadog_checks.dell_powerflex.api.PowerFlexAPI.get_events', 'Failed to collect events'), + ('collect_alerts', 'datadog_checks.dell_powerflex.api.PowerFlexAPI.get_alerts', 'Failed to collect alerts'), + ], +) +def test_collect_failure( + dd_run_check, aggregator, instance, mock_http_get, mocker, caplog, config_key, mock_target, log_message +): + instance[config_key] = True + mocker.patch(mock_target, side_effect=Exception(f'{config_key} API failed')) + caplog.set_level(logging.WARNING) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + cache_key = 'last_event_timestamp' if config_key == 'collect_events' else 'last_alert_timestamp' + dd_run_check(check) # initialize persistent cache prefix + previous_timestamp = '2020-01-01T00:00:00.000000Z' + check.write_persistent_cache(cache_key, previous_timestamp) + dd_run_check(check) + assert log_message in caplog.text + assert len(aggregator.events) == 0 + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1) + assert check.read_persistent_cache(cache_key) == previous_timestamp + + +@pytest.mark.parametrize( + 'config_key, mock_target, log_message', + [ + ( + 'collect_events', + 'datadog_checks.dell_powerflex.api.PowerFlexAPI.get_events', + 'Skipping malformed event', + ), + ( + 'collect_alerts', + 'datadog_checks.dell_powerflex.api.PowerFlexAPI.get_alerts', + 'Skipping malformed alert', + ), + ], +) +def test_malformed_record_is_skipped( + dd_run_check, aggregator, instance, mock_http_get, mocker, caplog, config_key, mock_target, log_message +): + instance[config_key] = True + valid_record = {'name': 'VALID_EVENT', 'timestamp': '2026-03-18T03:40:16.253Z', 'severity': 'CRITICAL'} + malformed_record = {'name': 'BAD_EVENT', 'timestamp': 'not-a-timestamp', 'severity': 'CRITICAL'} + mocker.patch(mock_target, return_value=[valid_record, malformed_record]) + caplog.set_level(logging.WARNING) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + assert log_message in caplog.text + assert len(aggregator.events) == 1 + + +def test_collect_alerts(dd_run_check, aggregator, instance, mock_http_get): + instance['collect_alerts'] = True + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + alerts = aggregator.events + assert len(alerts) == 2 + for alert in alerts: + assert alert['event_type'] == 'dell_powerflex.alert' + assert alert['source_type_name'] == 'dell-powerflex' + assert f'powerflex_gateway_url:{DEFAULT_GATEWAY_URL}' in alert['tags'] + + mdm_alert = next(a for a in alerts if a['msg_title'] == 'Unable To Receive Mdm Events') + assert mdm_alert['alert_type'] == 'error' + assert 'MDM1: Unable to receive mdm events from [10.0.1.250] .' in mdm_alert['msg_text'] + assert 'powerflex_alert_name:UNABLE_TO_RECEIVE_MDM_EVENTS' in mdm_alert['tags'] + assert 'severity:MAJOR' in mdm_alert['tags'] + assert 'category:MAINTENANCE' in mdm_alert['tags'] + assert 'domain:BLOCK' in mdm_alert['tags'] + assert 'dell_type:mdms' in mdm_alert['tags'] + assert 'service_name:block-events-gw' in mdm_alert['tags'] + + license_alert = next(a for a in alerts if a['msg_title'] == 'Trial License Used') + assert license_alert['alert_type'] == 'warning' + assert 'PowerFlex is using a trial license' in license_alert['msg_text'] + assert 'severity:MINOR' in license_alert['tags'] + assert check.read_persistent_cache('last_alert_timestamp') is not None + + +def test_statistics_failure_does_not_block_other_resources( + dd_run_check, aggregator, instance, mock_http_get, mocker, caplog +): + sds2_stats = {'capacityInUseInKb': 350208, 'unusedCapacityInKb': 103406592, 'numOfDevices': 1} + + def selective_fail(sds_id): + if sds_id == 'd1c062b700000000': + raise Exception('stats API error') + return sds2_stats + + mocker.patch( + 'datadog_checks.dell_powerflex.api.PowerFlexAPI.get_sds_statistics', + side_effect=selective_fail, + ) + caplog.set_level(logging.WARNING) + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + assert 'Failed to collect statistics for d1c062b700000000' in caplog.text + + # SDS3 (d1c062b700000000) inventory metric should exist but stats should be missing + sds3_tags = BASE_TAGS + SDS3_TAGS + aggregator.assert_metric('dell_powerflex.sds.count', value=1, tags=sds3_tags) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', count=0, tags=sds3_tags) + + # SDS2 (d1c062b800000001) stats should still be collected + sds2_tags = BASE_TAGS + SDS2_TAGS + aggregator.assert_metric('dell_powerflex.sds.count', value=1, tags=sds2_tags) + aggregator.assert_metric('dell_powerflex.capacity.in_use_in_kb', value=350208, tags=sds2_tags) + + +def test_user_configured_tags(dd_run_check, aggregator, instance, mock_http_get): + instance['tags'] = ['env:prod', 'cluster:powerflex-01'] + instance['collect_events'] = True + instance['collect_alerts'] = True + check = DellPowerflexCheck('dell_powerflex', {}, [instance]) + dd_run_check(check) + + custom_tags = ['env:prod', 'cluster:powerflex-01'] + base_tags = BASE_TAGS + custom_tags + + # Verify metrics include user-configured tags + aggregator.assert_metric('dell_powerflex.api.can_connect', value=1, tags=base_tags) + aggregator.assert_metric('dell_powerflex.system.count', value=1, tags=base_tags + SYSTEM_TAGS) + aggregator.assert_metric('dell_powerflex.storage_pool.count', value=1, tags=base_tags + POOL1_TAGS) + + # Verify events include user-configured tags + for event in aggregator.events: + for tag in custom_tags: + assert tag in event['tags'], f"Expected tag '{tag}' in event tags: {event['tags']}" diff --git a/kafka_actions/changelog.d/23951.fixed b/kafka_actions/changelog.d/23951.fixed new file mode 100644 index 0000000000000..ce6c07836c46e --- /dev/null +++ b/kafka_actions/changelog.d/23951.fixed @@ -0,0 +1 @@ +Fall back to string deserialization when schema registry magic byte is absent. diff --git a/kafka_actions/datadog_checks/kafka_actions/message_deserializer.py b/kafka_actions/datadog_checks/kafka_actions/message_deserializer.py index fe09e8035745f..aa63994102b56 100644 --- a/kafka_actions/datadog_checks/kafka_actions/message_deserializer.py +++ b/kafka_actions/datadog_checks/kafka_actions/message_deserializer.py @@ -243,20 +243,21 @@ def _deserialize_bytes_maybe_schema_registry( ) -> tuple[str | None, int | None]: """Deserialize message, handling Schema Registry format if present.""" if uses_schema_registry: - if len(message) < 5 or message[0] != SCHEMA_REGISTRY_MAGIC_BYTE: - msg_hex = message[:5].hex() if len(message) >= 5 else message.hex() - raise ValueError( - f"Expected schema registry format (magic byte 0x00 + 4-byte schema ID), " - f"but message is too short or has wrong magic byte: {msg_hex}" - ) - schema_id = int.from_bytes(message[1:5], 'big') - message = message[5:] # Skip the magic byte and schema ID bytes + if len(message) >= 5 and message[0] == SCHEMA_REGISTRY_MAGIC_BYTE: + schema_id = int.from_bytes(message[1:5], 'big') + message = message[5:] # Skip the magic byte and schema ID bytes - actual_format = message_format - if self.schema_registry is not None: - schema, actual_format = self._fetch_and_build_schema(schema_id, message_format) + actual_format = message_format + if self.schema_registry is not None: + schema, actual_format = self._fetch_and_build_schema(schema_id, message_format) - return self._deserialize_bytes(message, actual_format, schema, uses_schema_registry=True), schema_id + return self._deserialize_bytes(message, actual_format, schema, uses_schema_registry=True), schema_id + else: + self.log.debug( + "Expected schema registry format (magic byte 0x00 + 4-byte schema ID), " + "but message is too short or has wrong magic byte, falling back to string", + ) + return self._deserialize_bytes(message, 'string', None, uses_schema_registry=False), None else: # Fallback behavior: try without schema registry format first, then with it try: diff --git a/kafka_actions/tests/test_message_deserializer.py b/kafka_actions/tests/test_message_deserializer.py index af67af7d328ba..954ff300ec283 100644 --- a/kafka_actions/tests/test_message_deserializer.py +++ b/kafka_actions/tests/test_message_deserializer.py @@ -270,9 +270,11 @@ def test_avro_explicit_schema_registry_configuration(self): assert result[1] is None, "Should have no schema ID" assert 'The Go Programming Language' in result[0] - # Test 2: uses_schema_registry=True with plain Avro message - should fail (missing magic byte) + # Test 2: uses_schema_registry=True with plain Avro message - falls back to string (non-UTF-8 bytes → error) result = deserializer.deserialize_message(avro_message_no_sr, 'avro', avro_schema, True) - assert result[0].startswith(" dict[str, Any]: try: if self._cancel_event.is_set(): raise Exception("Job loop cancelled. Aborting query.") - with conn.cursor() as cursor: - cursor.execute(query_spec.query) - # cursor.description is None when the query produced no result set - # (e.g. INSERT, UPDATE, DELETE, or a syntax error that executed without - # raising). RC-delivered queries must be SELECTs; treat this as a - # per-query error so subsequent queries in the list still run. - if cursor.description is None: - raise psycopg.errors.ProgrammingError( - "Query returned no result set — only SELECT statements are supported" + # query_timeout is in milliseconds, matching the instance-level query_timeout unit. + timeout_ms = query_spec.query_timeout + # Pool connections run with autocommit=True, so the timeout must be + # applied inside an explicit transaction and reverts on commit, + # avoiding timeout leakage onto the shared connection. + # set_config() is used instead of "SET LOCAL statement_timeout = %s" + # because psycopg3 uses server-side binding (extended query protocol) + # for parameterized execute() calls, and PostgreSQL rejects bound + # parameters in SET statements under the extended protocol. set_config() + # is a regular function that accepts parameters normally; is_local=true + # gives the same scope as SET LOCAL (current transaction only). + with conn.transaction(): + with conn.cursor() as cursor: + cursor.execute( + "SELECT set_config('statement_timeout', %s, true)", + (str(int(timeout_ms)),), ) - columns = [desc[0] for desc in cursor.description] - rows = [list(row) for row in cursor.fetchmany(MAX_RESULT_ROWS)] + cursor.execute(query_spec.query) + # cursor.description is None when the query produced no result set + # (e.g. INSERT, UPDATE, DELETE, or a syntax error that executed without + # raising). RC-delivered queries must be SELECTs; treat this as a + # per-query error so subsequent queries in the list still run. + if cursor.description is None: + raise psycopg.errors.ProgrammingError( + "Query returned no result set — only SELECT statements are supported" + ) + columns = [desc[0] for desc in cursor.description] + rows = [list(row) for row in cursor.fetchmany(MAX_RESULT_ROWS)] duration = time.time() - start return { 'status': 'success', diff --git a/postgres/datadog_checks/postgres/schemas.py b/postgres/datadog_checks/postgres/schemas.py index fb4dca189b19e..22e11347af663 100644 --- a/postgres/datadog_checks/postgres/schemas.py +++ b/postgres/datadog_checks/postgres/schemas.py @@ -230,10 +230,12 @@ def _get_databases(self): def _get_cursor(self, database_name): with self._check.db_pool.get_connection(database_name) as conn: with conn.cursor(row_factory=dict_row) as cursor: - query, params = self.get_rows_query() - cursor.execute(f"SET statement_timeout = '{self._config.max_query_duration}s';") - cursor.execute(query, params) - yield cursor + # Explicitly wrap these queries in a transaction so we can set the statement timeout locally + with conn.transaction(): + query, params = self.get_rows_query() + cursor.execute(f"SET LOCAL statement_timeout = '{self._config.max_query_duration}s';") + cursor.execute(query, params) + yield cursor def _get_schemas_query(self): query = SCHEMA_QUERY diff --git a/postgres/tests/test_config.py b/postgres/tests/test_config.py index 026ec07eeeaba..90eb5dd94d1c1 100644 --- a/postgres/tests/test_config.py +++ b/postgres/tests/test_config.py @@ -628,6 +628,7 @@ def test_do_query_schedule_field_defaults_to_none(mock_check, minimal_instance): 'dbname': 'mydb', 'query': 'SELECT 1', 'interval_seconds': 60, + 'query_timeout': 30_000, 'entity': { 'platform': 'aws', 'account': '123', @@ -658,6 +659,7 @@ def test_do_query_schedule_field_parsed(mock_check, minimal_instance): 'dbname': 'mydb', 'query': 'SELECT 1', 'schedule': '20 * * * *', + 'query_timeout': 30_000, 'entity': { 'platform': 'aws', 'account': '123', @@ -689,6 +691,7 @@ def test_do_query_both_schedule_and_interval_parsed(mock_check, minimal_instance 'query': 'SELECT 1', 'schedule': '0 * * * *', 'interval_seconds': 3600, + 'query_timeout': 30_000, 'entity': { 'platform': 'aws', 'account': '123', diff --git a/postgres/tests/test_data_observability.py b/postgres/tests/test_data_observability.py index ab13e139ed24c..0523f9318c002 100644 --- a/postgres/tests/test_data_observability.py +++ b/postgres/tests/test_data_observability.py @@ -12,6 +12,7 @@ import psycopg import pytest +import yaml from datadog_checks.postgres import PostgreSql from datadog_checks.postgres.data_observability import EVENT_TRACK_TYPE @@ -26,6 +27,7 @@ 'dbname': 'test_db', 'query': 'SELECT count(*) FROM orders', 'interval_seconds': 60, + 'query_timeout': 30_000, 'type': 'freshness', 'entity': { 'platform': 'aws', @@ -43,6 +45,7 @@ 'dbname': 'test_db', 'query': 'SELECT count(*) FROM users', 'interval_seconds': 120, + 'query_timeout': 30_000, 'type': 'freshness', 'entity': { 'platform': 'aws', @@ -279,15 +282,47 @@ def execute_side_effect(sql, *args, **kwargs): assert len(status_metrics) == 2 +def _get_local_timeout_ms(mock_cursor): + """Return the statement_timeout (ms) passed to the set_config execute, or None.""" + for call in mock_cursor.execute.call_args_list: + sql = call.args[0] + if "set_config('statement_timeout'" in sql.lower(): + return int(call.args[1][0]) + return None + + +def test_query_timeout_applied_in_transaction(pg_instance): + """The query's query_timeout is applied via set_config inside a transaction.""" + mock_conn, mock_cursor = _make_mock_conn() + + _setup_and_run(pg_instance, mock_conn=mock_conn, mock_cursor=mock_cursor) + + mock_conn.transaction.assert_called_once() + assert _get_local_timeout_ms(mock_cursor) == 30_000 + + +def test_query_timeout_passed_directly_to_set_config(pg_instance): + """query_timeout (milliseconds) is forwarded as-is to SET LOCAL statement_timeout.""" + query = deepcopy(BASE_QUERY) + query['query_timeout'] = 180_000 + mock_conn, mock_cursor = _make_mock_conn() + + _setup_and_run(pg_instance, queries=[query], mock_conn=mock_conn, mock_cursor=mock_cursor) + + assert _get_local_timeout_ms(mock_cursor) == 180_000 + + def test_no_description_does_not_block_subsequent(aggregator, pg_instance): """First query returns None description (non-SELECT), second query still runs.""" mock_conn, mock_cursor = _make_mock_conn() - call_count = 0 + query_count = 0 def execute_side_effect(sql, *args, **kwargs): - nonlocal call_count - call_count += 1 - mock_cursor.description = None if call_count == 1 else [('count',)] + nonlocal query_count + if "set_config('statement_timeout'" in sql.lower(): + return + query_count += 1 + mock_cursor.description = None if query_count == 1 else [('count',)] mock_cursor.execute = MagicMock(side_effect=execute_side_effect) @@ -513,6 +548,8 @@ def test_multi_query_different_dbnames(aggregator, pg_instance): 'dbname': 'test_db', 'query': 'SELECT 1', 'schedule': '50 * * * *', # every hour at :50 + 'interval_seconds': 3600, + 'query_timeout': 30_000, 'type': 'freshness', 'entity': { 'platform': 'aws', @@ -670,6 +707,7 @@ def test_query_without_schedule_or_positive_interval_filtered_at_init(pg_instanc 'monitor_id': 30, 'dbname': 'test_db', 'query': 'SELECT 1', + 'query_timeout': 30_000, 'type': 'freshness', 'entity': { 'platform': 'aws', @@ -986,3 +1024,122 @@ def boom(*args, **kwargs): assert len(failures) == 1 assert any(t.startswith('exc_class:JSONDecodeError') for t in failures[0].tags) assert 'monitor_id:1' in failures[0].tags + + +# --- Agent YAML-delivery round-trip tests --- +# +# The DO queries originate from a Remote Configuration payload handled by the Datadog Agent's +# Go RC handler (comp/dataobs/queryactions/impl/handler.go). The agent injects them into the +# postgres instance config, serializes the instance to YAML, and hands that YAML *string* to +# this check; datadog_checks.base parses it with yaml.safe_load before the dict ever reaches +# PostgreSql.__init__. +# +# DO query strings are multi-line SQL that routinely mixes indented lines with a trailing +# column-0 "-- Datadog {...}" annotation. yaml.v3 used to serialize such a string as a literal +# block scalar ("|") whose later, less-indented line escaped the block and produced YAML that +# neither go-yaml nor PyYAML can parse (a "did not find expected key" / ParserError). The agent +# now forces a double-quoted scalar for the query so the round-trip is exact. The tests above +# all inject a Python dict directly and therefore never exercise this serialize→safe_load +# boundary; these tests close that gap from the consumer side. + +# Indented SELECT lines followed by a column-0 "-- Datadog" comment — the canonical failing shape. +MULTILINE_QUERY = ( + " SELECT count(*) AS dd_value\n" + " FROM events.clicks c\n" + " LEFT JOIN events.page_views pv\n" + " ON c.user_id = pv.user_id AND c.page_url = pv.url\n" + " WHERE pv.id IS NULL\n" + '-- Datadog {"monitor_ids":[26724188]}\n' +) + + +def _instance_yaml_as_agent_delivers(pg_instance, query): + """Render the DO instance to a YAML string the way the agent delivers it to the check. + + The query is emitted as a double-quoted scalar (matching the agent's yaml.Node fix); the + rest of the instance is dumped normally. Returns the YAML text, which mirrors exactly what + datadog_checks.base feeds to yaml.safe_load. + """ + instance = _make_do_instance(pg_instance, queries=[{**deepcopy(BASE_QUERY), 'query': query}]) + do = instance.pop('data_observability') + queries_block = do.pop('queries') + q = queries_block[0] + # Build the query entry by hand so the SQL is a double-quoted scalar, as the agent emits it. + # json.dumps produces a valid YAML double-quoted flow scalar for these characters. + query_entry_lines = [f" - query: {json.dumps(query)}"] + for key, value in q.items(): + if key == 'query': + continue + query_entry_lines.append(f" {key}: {json.dumps(value)}") + do_yaml = ["data_observability:"] + for key, value in do.items(): + do_yaml.append(f" {key}: {json.dumps(value)}") + do_yaml.append(" queries:") + do_yaml.extend(query_entry_lines) + return yaml.safe_dump(instance, default_flow_style=False) + "\n".join(do_yaml) + "\n" + + +@pytest.mark.parametrize( + 'query', + [ + pytest.param(MULTILINE_QUERY, id='indented_then_col0_comment'), + pytest.param('SELECT 23 as dd_value;\n-- Datadog {"monitor_ids":[26386160]}\n', id='trailing_comment'), + pytest.param( + 'SELECT COUNT(1) AS dd_a_1, COUNT(DISTINCT "customer_id") AS dd_b_2 FROM "testdb"."shop"."orders"\n' + '-- Datadog {"monitor_ids":[26358412,26386112]}\n', + id='embedded_quotes', + ), + pytest.param('SELECT 1', id='simple_single_line'), + ], +) +def test_agent_yaml_delivery_round_trips_query(pg_instance, query): + """The query survives the agent's YAML serialization + safe_load round-trip byte-for-byte + and reaches cursor.execute unchanged. This is the consumer-side guard for the yaml.v3 + block-scalar bug fixed in the agent (handler.go).""" + instance_yaml = _instance_yaml_as_agent_delivers(pg_instance, query) + + # The agent → Python boundary: base.load_config feeds this YAML string to yaml.safe_load. + parsed = yaml.safe_load(instance_yaml) + assert parsed['data_observability']['queries'][0]['query'] == query, "query must survive safe_load intact" + + mock_conn, mock_cursor = _make_mock_conn() + check = PostgreSql('postgres', {}, [parsed]) + check.db_pool = _mock_db_pool(mock_conn) + check.data_observability.run_job() + + # The exact SQL string (not the set_config timeout call) must reach the driver. + executed = [call.args[0] for call in mock_cursor.execute.call_args_list] + assert query in executed, f"original query string must be executed verbatim; got {executed!r}" + + +# The YAML the agent emitted for MULTILINE_QUERY *before* the handler.go fix: a literal block +# scalar ("|4") whose trailing "-- Datadog" line is indented 12 spaces while the block content +# sits at 14. Being less-indented than the block, that line terminates the scalar and is then +# read as a sibling node at indent 12 — deeper than the mapping keys at 10 — and the ":" inside +# it makes the parser expect a key. yaml.v3 emitted this without error; both go-yaml and PyYAML +# fail to parse it. In production this never reached the check: the agent's autodiscovery digest +# re-parsed the YAML first, failed, and dropped the config before scheduling. +BROKEN_AGENT_YAML = ( + "data_observability:\n" + " enabled: true\n" + " queries:\n" + " - dbname: analyticsdb\n" + " interval_seconds: 3600\n" + " query: |4\n" + " SELECT count(*) AS dd_value\n" + " FROM events.clicks c\n" + " LEFT JOIN events.page_views pv\n" + " ON c.user_id = pv.user_id AND c.page_url = pv.url\n" + " WHERE pv.id IS NULL\n" + ' -- Datadog {"monitor_ids":[26724188]}\n' + " query_timeout: 300\n" + " type: run_query\n" +) + + +def test_pre_fix_agent_yaml_was_unparseable(): + """Documents the cross-language bug. The agent's old literal-block output fails yaml.safe_load + with the same parse error go-yaml hit. The agent now emits a double-quoted scalar (handler.go), + so this shape is no longer produced; test_agent_yaml_delivery_round_trips_query covers the fix.""" + with pytest.raises(yaml.YAMLError): + yaml.safe_load(BROKEN_AGENT_YAML) diff --git a/postgres/tests/test_statements.py b/postgres/tests/test_statements.py index 6583c6e58bb06..69f4167703922 100644 --- a/postgres/tests/test_statements.py +++ b/postgres/tests/test_statements.py @@ -1960,6 +1960,7 @@ def test_pg_stat_statements_max_warning( def test_pg_stat_statements_dealloc(aggregator, integration_check, dbm_instance_replica2): dbm_instance_replica2['query_samples'] = {'enabled': False} dbm_instance_replica2['query_activity'] = {'enabled': False} + dbm_instance_replica2['collect_schemas'] = {'enabled': False} with _get_superconn(dbm_instance_replica2) as superconn: with superconn.cursor() as cur: cur.execute("select pg_stat_statements_reset();") diff --git a/prometheus/manifest.json b/prometheus/manifest.json index c187f91555490..3face1a59be6f 100644 --- a/prometheus/manifest.json +++ b/prometheus/manifest.json @@ -48,7 +48,7 @@ "spec": "assets/configuration/spec.yaml" }, "events": { - "creates_events": false + "creates_events": true }, "service_checks": { "metadata_path": "assets/service_checks.json" @@ -57,4 +57,4 @@ "auto_install": true } } -} \ No newline at end of file +}