Move provision to orchestrator domain and rename provision references to orchestration

- Move src/playbooks/provision/ to src/orchestrator/ (independent domain)
- Move src/playbooks/discovery/ to src/discovery/ (independent domain)
- Rename provision_validations role to orchestrator_validations
- Rename provision.yml to orchestrator.yml
- Rename provision_* variables to orchestration_* across orchestrator roles
- Rename provision_completion.yml to orchestration_completion.yml
- Rename provision_mapping_nodes.yml to orchestration_mapping_nodes.yml
- Update user-facing messages to reference orchestrator.yml
- Add orchestrator_config.json schema and validation
- Fix admin_nic_ip variable scoping in deploy_openchami role
- Add retry logic for SMD group data query (3 retries, 15s delay)
- Add deploy_openchami role (moved from prepare_oim)
- Add ARCHITECTURE_REFACTOR.md documenting the refactoring

Author: SAYUK09

ARCHITECTURE_REFACTOR.md

@@ -52,6 +52,7 @@
     "network_spec": "network_spec.yml",
     "omnia_config": "omnia_config.yml",
     "provision_config": "provision_config.yml",
+    "orchestrator_config": "orchestrator_config.yml",
     "security_config": "security_config.yml",
     "software_config": "software_config.json",
     "storage_config": "storage_config.yml",
@@ -81,6 +82,10 @@
         files["software_config"],
         # files["high_availability_config"]
     ],
+    "orchestrator": [
+        files["orchestrator_config"],
+        files["network_spec"],
+    ],
     "security": [
         files["security_config"]
     ],

src/common/library/module_utils/input_validation/common_utils/config.py

@@ -58,6 +58,7 @@ def validate_input_logic(
     # Based on the file_name, run validation function
     validation_functions = {
         "provision_config.yml": provision_validation.validate_provision_config,
+        "orchestrator_config.yml": provision_validation.validate_orchestrator_config,
         "software_config.json": common_validation.validate_software_config,
         "network_spec.yml": provision_validation.validate_network_spec,
         "omnia_config.yml": common_validation.validate_omnia_config,

src/common/library/module_utils/input_validation/common_utils/logical_validation.py

@@ -0,0 +1,72 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "pxe_mapping_file_path": {
+      "type": "string",
+      "description": "Path to the PXE mapping file."
+    },
+    "language": {
+      "type": "string",
+      "description": "Language setting.",
+      "default": "en_US.UTF-8"
+    },
+    "default_lease_time": {
+      "type": ["string", "integer"],
+      "description": "Default lease time for DHCP.",
+      "default": 86400
+    },
+    "dns_enabled": {
+      "type": "boolean",
+      "description": "Enable DNS-based hostname resolution via coresmd.",
+      "default": false
+    },
+    "kernel_version_override": {
+      "type": "string",
+      "description": "Optional kernel version to pin for boot image selection. Leave empty to auto-select latest.",
+      "pattern": "^(|[0-9]+\\.[0-9]+\\.[0-9]+-.+)$",
+      "default": ""
+    },
+    "additional_cloud_init_config_file": {
+      "type": "string",
+      "description": "Path to additional cloud-init configuration file for stateless node provisioning. Leave empty to disable.",
+      "default": ""
+    },
+    "s3_storage_provider": {
+      "type": "string",
+      "description": "S3 storage provider: minio, powerscale, or external.",
+      "enum": ["minio", "powerscale", "external"],
+      "default": "minio"
+    },
+    "s3_endpoint": {
+      "type": "string",
+      "description": "S3-compatible endpoint URL. Required for powerscale/external providers.",
+      "default": ""
+    },
+    "s3_access_key": {
+      "type": "string",
+      "description": "S3 access key.",
+      "default": ""
+    },
+    "s3_secret_key": {
+      "type": "string",
+      "description": "S3 secret key.",
+      "default": ""
+    },
+    "s3_boot_images_bucket": {
+      "type": "string",
+      "description": "S3 bucket name for boot images.",
+      "default": "boot-images"
+    },
+    "s3_verify_ssl": {
+      "type": "boolean",
+      "description": "Whether to verify SSL certificates for S3 connections.",
+      "default": false
+    }
+  },
+  "required": [
+    "pxe_mapping_file_path",
+    "language",
+    "default_lease_time"
+  ]
+}

src/common/library/module_utils/input_validation/schema/orchestrator_config.json

@@ -1451,6 +1451,128 @@ def validate_provision_config(
 
     return errors
 
+
+def validate_orchestrator_config(
+    input_file_path, data, logger, module, omnia_base_dir, module_utils_base, project_name
+):
+    """
+    Validates the orchestrator configuration.
+    Subset of validate_provision_config — skips software_config and build_stream
+    validation since the orchestrator does not own those files.
+    """
+    errors = []
+
+    # Validate language setting
+    language = data.get("language", "")
+    if not language:
+        errors.append(
+            create_error_msg("language", input_file_path, en_us_validation_msg.LANGUAGE_EMPTY_MSG)
+        )
+    elif "en_US.UTF-8" not in language:
+        errors.append(
+            create_error_msg("language", input_file_path, en_us_validation_msg.LANGUAGE_FAIL_MSG)
+        )
+
+    pxe_mapping_file_path = data.get("pxe_mapping_file_path", "")
+    if pxe_mapping_file_path and validation_utils.verify_path(pxe_mapping_file_path):
+        try:
+            validate_mapping_file_entries(pxe_mapping_file_path)
+            validate_functional_groups_in_mapping_file(pxe_mapping_file_path)
+            validate_duplicate_service_tags_in_mapping_file(pxe_mapping_file_path)
+            validate_duplicate_hostnames_in_mapping_file(pxe_mapping_file_path)
+            validate_duplicate_admin_ips_in_mapping_file(pxe_mapping_file_path)
+            validate_duplicate_ib_ips_in_mapping_file(pxe_mapping_file_path)
+            validate_ib_nic_name_format_in_mapping_file(pxe_mapping_file_path)
+            validate_group_parent_service_tag_consistency_in_mapping_file(pxe_mapping_file_path)
+            validate_functional_groups_separation(pxe_mapping_file_path)
+            validate_parent_service_tag_hierarchy(pxe_mapping_file_path)
+            validate_slurm_login_compiler_prefix(pxe_mapping_file_path)
+            validate_aarch64_local_path_compatibility(pxe_mapping_file_path)
+            dns_enabled = data.get("dns_enabled", False)
+            if isinstance(dns_enabled, str):
+                dns_enabled = dns_enabled.lower() in ("true", "yes", "1")
+            validate_hostname_nid_format_when_dns_enabled(pxe_mapping_file_path, bool(dns_enabled))
+
+            # Validate ADMIN_IPs against network_spec.yml subnets
+            network_spec_path = create_file_path(input_file_path, file_names["network_spec"])
+            if os.path.isfile(network_spec_path):
+                try:
+                    with open(network_spec_path, "r", encoding="utf-8") as f:
+                        network_spec_json = yaml.safe_load(f)
+
+                    admin_netmaskbits = None
+                    oim_admin_ip = None
+                    additional_subnets = []
+
+                    for network in (network_spec_json or {}).get("Networks", []):
+                        if "admin_network" in network and isinstance(network["admin_network"], dict):
+                            admin_net = network["admin_network"]
+                            admin_netmaskbits = admin_net.get("netmask_bits")
+                            oim_admin_ip = admin_net.get("primary_oim_admin_ip")
+                            additional_subnets = admin_net.get("additional_subnets") or []
+                            break
+
+                    if admin_netmaskbits and oim_admin_ip:
+                        validate_pxe_admin_ips_subnet_consistency(
+                            errors, pxe_mapping_file_path,
+                            oim_admin_ip, admin_netmaskbits,
+                            additional_subnets
+                        )
+                except (yaml.YAMLError, IOError) as e:
+                    errors.append(
+                        create_error_msg(
+                            "network_spec.yml",
+                            network_spec_path,
+                            f"Failed to load or parse network_spec.yml: {str(e)}"
+                        )
+                    )
+        except ValueError as e:
+            errors.append(
+                create_error_msg(
+                    "pxe_mapping_file_path",
+                    pxe_mapping_file_path,
+                    str(e),
+                )
+            )
+    else:
+        errors.append(
+            create_error_msg(
+                "pxe_mapping_file_path",
+                pxe_mapping_file_path,
+                en_us_validation_msg.PXE_MAPPING_FILE_PATH_FAIL_MSG,
+            )
+        )
+
+    default_lease_time = data["default_lease_time"]
+    if not validation_utils.validate_default_lease_time(default_lease_time):
+        errors.append(
+            create_error_msg(
+                "default_lease_time",
+                default_lease_time,
+                en_us_validation_msg.DEFAULT_LEASE_TIME_FAIL_MSG,
+            )
+        )
+
+    kernel_version_override = data.get("kernel_version_override", "")
+    if kernel_version_override:
+        if not re.match(r"^[0-9]+\.[0-9]+\.[0-9]+-.+$", kernel_version_override):
+            errors.append(
+                create_error_msg(
+                    "kernel_version_override",
+                    kernel_version_override,
+                    en_us_validation_msg.KERNEL_VERSION_OVERRIDE_FAIL_MSG,
+                )
+            )
+
+    # Validate additional cloud-init config file
+    aci_path = data.get("additional_cloud_init_config_file", "")
+    if aci_path:
+        aci_errors = validate_additional_cloud_init_config(aci_path, pxe_mapping_file_path)
+        errors.extend(aci_errors)
+
+    return errors
+
+
 def validate_network_spec(
     input_file_path, data, logger, module, omnia_base_dir, module_utils_base, project_name
 ):

src/common/library/module_utils/input_validation/validation_flows/provision_validation.py

@@ -115,6 +115,7 @@ def createlogger(project_name, tag_name=None):
         log_filename = f"validation_omnia_{project_name}.log"
 
     log_file_path = os.path.join(config.INPUT_VALIDATOR_LOG_PATH, log_filename)
+    os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
     logging.basicConfig(
         filename=log_file_path,
         format="%(asctime)s %(message)s",

src/common/library/modules/validate_input.py

@@ -0,0 +1,80 @@
+# Discovery Domain – Input/Output Contracts
+
+## Overview
+
+The Discovery domain is responsible for discovering hardware (servers) via
+management platforms (e.g., Dell OME) and producing a PXE mapping file that
+serves as the primary data contract between Discovery and the Orchestrator.
+
+## Inputs
+
+All inputs are read from:
+
+```
+/opt/omnia/input/<project_name>/discovery/
+```
+
+### discovery_config.yml
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `enable_bmc_discovery` | bool | Yes | Enable BMC discovery via OME |
+| `ome_ip` | string | Yes (when OME) | IP address of Dell OME instance |
+
+### network_spec.yml
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `Networks[].admin_network.subnet` | string | Yes | Admin network subnet for IP derivation |
+| `Networks[].ib_network.subnet` | string | No | InfiniBand subnet for IB IP derivation |
+
+### Credentials (from shared credential utility)
+
+| Parameter | Source | Description |
+|-----------|--------|-------------|
+| `ome_username` | omnia_config_credentials.yml | OME login username |
+| `ome_password` | omnia_config_credentials.yml | OME login password (vault-encrypted) |
+
+## Outputs
+
+All outputs are written to:
+
+```
+/opt/omnia/output/<project_name>/discovery/
+```
+
+### bmc_pxe_mapping_file.csv (Primary Contract)
+
+This is the **primary output** consumed by the Orchestrator domain.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| `FUNCTIONAL_GROUP_NAME` | string | Node functional group (e.g., `slurm_node_aarch64`) |
+| `GROUP_NAME` | string | Scalable Unit / group identifier |
+| `SERVICE_TAG` | string | Dell server service tag |
+| `PARENT_SERVICE_TAG` | string | Parent node service tag (for slurm child nodes) |
+| `HOSTNAME` | string | Generated hostname (e.g., `nid00001`) |
+| `ADMIN_MAC` | string | Admin NIC MAC address |
+| `ADMIN_IP` | string | Admin IP (derived from admin_subnet + BMC IP) |
+| `BMC_MAC` | string | BMC/iDRAC MAC address |
+| `BMC_IP` | string | BMC/iDRAC IP address |
+| `IB_NIC_NAME` | string | InfiniBand NIC FQDD (if present) |
+| `IB_IP` | string | InfiniBand IP (derived from ib_subnet + BMC IP) |
+
+### bmc_discovery_report.csv (Informational)
+
+NIC link status report for operator review. Not consumed programmatically.
+
+## Data Flow
+
+```
+Discovery Input                    Discovery Output                    Orchestrator Input
+─────────────────                  ─────────────────                   ──────────────────
+discovery_config.yml ─┐
+                      ├─► OME ─► bmc_pxe_mapping_file.csv ──copy──► pxe_mapping_file.csv
+network_spec.yml ─────┘            bmc_discovery_report.csv
+```
+
+The mapping file must be **manually reviewed and copied** to the orchestrator
+input directory before running orchestrator.yml. This deliberate handoff point
+ensures operators can review/edit node assignments before provisioning.

src/discovery/CONTRACTS.md

@@ -1,5 +1,5 @@
 [defaults]
-log_path = /opt/omnia/log/core/playbooks/provision.log
+log_path = /opt/omnia/log/core/discovery/discovery.log
 remote_tmp = /opt/omnia/tmp/.ansible/tmp/
 host_key_checking = false
 forks = 5
@@ -9,9 +9,9 @@ interpreter_python = /usr/bin/python3
 deprecation_warnings = false
 show_task_path_on_failure = false
 stdout_callback = omnia_default
-callback_plugins = ../../common/callback_plugins
-library = library:../../common/library/modules
-module_utils = ../../common/library/module_utils
+callback_plugins = ../common/callback_plugins
+library = roles/ome_discovery/library:../common/library/modules
+module_utils = ../common/library/module_utils
 
 [persistent_connection]
 command_timeout = 180