Skip to content

Commit 9011d13

Browse files
author
Nate Thornton
authored
Continued changes for NNF fencing (#22)
* Changes for NNF Fencing Signed-off-by: Nate Thornton <nate.thornton@hpe.com>
1 parent 788e87d commit 9011d13

3 files changed

Lines changed: 249 additions & 18 deletions

File tree

‎docs/guides/ha-cluster/fence_nnf.py‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
#!@PYTHON@
2+
3+
# Fence agent for Near Node Flash devices
4+
5+
import sys
6+
import logging
7+
import atexit
8+
import http
9+
10+
sys.path.append("@FENCEAGENTSLIBDIR@")
11+
12+
from fencing import *
13+
from fencing import fail, run_delay
14+
15+
try:
16+
from kubernetes import client
17+
except ImportError:
18+
logging.error("Couldn't import kubernetes.client - not found or not accessible")
19+
20+
def get_power_status(api_client, options):
21+
# Power status is the status of the Kubernetes Node resource if it is down, or
22+
# the Fencing status of the NNF Node resource if the Kubernetes Node is up.
23+
24+
name = options.get("--nnf-node-name")
25+
logging.debug("Reading node resource %s", name)
26+
27+
try:
28+
node = client.CoreV1Api(api_client).read_node(name)
29+
30+
except client.exceptions.ApiException as e:
31+
logging.debug("Exception when reading node: %s %s", e, type(e))
32+
if e.status == http.HTTPStatus.UNAUTHORIZED:
33+
fail(EC_LOGIN_DENIED)
34+
elif e.status == http.HTTPStatus.NOT_FOUND:
35+
fail(EC_STATUS)
36+
37+
logging.error("Exception when reading node: %s %s", e, type(e))
38+
39+
for condition in node.status.conditions:
40+
if condition.type == "Ready" and condition.status == "True":
41+
return _get_nnf_node_power_status(api_client, options)
42+
43+
logging.info("Node %s is not ready", name)
44+
return "off"
45+
46+
def _get_nnf_node_power_status(api_client, options):
47+
48+
version = options.get("--api-version")
49+
namespace = options.get("--nnf-node-name")
50+
logging.debug("Reading NNF node resource %s/%s", version, namespace)
51+
52+
try:
53+
node = client.CustomObjectsApi(api_client).get_namespaced_custom_object(
54+
"nnf.cray.hpe.com",
55+
version,
56+
namespace,
57+
"nnfnodes",
58+
"nnf-nlc"
59+
)
60+
61+
except client.exceptions.ApiException as e:
62+
logging.debug("Exception when reading NNF node: %s %s", e, type(e))
63+
if e.status == http.HTTPStatus.NOT_FOUND:
64+
fail(EC_STATUS)
65+
66+
logging.error("Exception when reading NNF node: %s %s", e, type(e))
67+
68+
fenced = node["status"].get("fenced", False)
69+
logging.info("NNF Node fenced status: %s", fenced)
70+
71+
return "off" if fenced else "on"
72+
73+
def set_power_status(api_client, options):
74+
75+
if options.get("--action") == "on":
76+
# NNF Fencing Agent is not permitted to enable the power status; that is done via
77+
# adminsitrator actions.
78+
return
79+
80+
api_object = client.CustomObjectsApi(api_client)
81+
82+
version = options.get("--api-version")
83+
namespace = options.get("--nnf-node-name")
84+
85+
node = api_object.get_namespaced_custom_object(
86+
"nnf.cray.hpe.com",
87+
version,
88+
namespace,
89+
"nnfnodes",
90+
"nnf-nlc"
91+
)
92+
93+
if not node["status"].get("fenced", False):
94+
node["status"]["fenced"] = True
95+
api_object.patch_namespaced_custom_object_status(
96+
"nnf.cray.hpe.com",
97+
version,
98+
namespace,
99+
"nnfnodes",
100+
"nnf-nlc",
101+
node
102+
)
103+
104+
105+
def define_new_opts():
106+
all_opt["kubernetes-service-host"] = {
107+
"getopt" : ":",
108+
"longopt" : "kubernetes-service-host",
109+
"help" : "--kubernetes-service-host=[ADDRESS] The IP Address of the kubeapi server",
110+
"shortdesc" : "Kubernetes API service IP address or hostname",
111+
"required" : "0",
112+
"order" : 1
113+
}
114+
115+
all_opt["kubernetes-service-port"] = {
116+
"getopt" : ":",
117+
"longopt" : "kubernetes-service-port",
118+
"help" : "--kubernetes-service-port=[PORT] The listen port of the kubeapi server",
119+
"shortdesc" : "Kubernetes API service port",
120+
"required" : "0",
121+
"order" : 1
122+
}
123+
124+
all_opt["service-token-file"] = {
125+
"getopt" : ":",
126+
"longopt" : "service-token-file",
127+
"help" : "--service-token-file=[PATH] The path to the service token file",
128+
"shortdesc" : "Path to the service token file",
129+
"required" : "1",
130+
"order" : 2
131+
}
132+
133+
all_opt["service-cert-file"] = {
134+
"getopt" : ":",
135+
"longopt" : "service-cert-file",
136+
"help" : "--service-cert-file=[PATH] The path to the service certificate file",
137+
"shortdesc" : "Path to the service certificate file",
138+
"required" : "1",
139+
"order" : 2
140+
}
141+
142+
all_opt["nnf-node-name"] = {
143+
"getopt" : ":",
144+
"longopt" : "nnf-node-name",
145+
"help" : "--nnf-node-name=[PATH] The name of the NNF node",
146+
"shortdesc" : "The name of the NNF node as listed in the system configuration",
147+
"required" : "1",
148+
"order" : 3
149+
}
150+
151+
all_opt["api-version"] = {
152+
"getopt" : ":",
153+
"longopt" : "api-version",
154+
"help" : "--api-version=[VERSION] Version of the NNF Node API",
155+
"shortdesc" : "The API Version of the NNF node resource",
156+
"required" : "0",
157+
"default" : "v1alpha1",
158+
"order" : 4
159+
}
160+
161+
all_opt["localconfig"] = {
162+
"getopt" : "",
163+
"longopt" : "localconfig",
164+
"help" : "--localconfig Use the local kubernetes config",
165+
"required" : "0",
166+
"order" : 4
167+
}
168+
169+
def main():
170+
atexit.register(atexit_handler)
171+
172+
device_opt = [
173+
"kubernetes-service-host", "kubernetes-service-port",
174+
"service-token-file", "service-cert-file",
175+
"nnf-node-name",
176+
"api-version",
177+
"localconfig",
178+
"no_password" # signals to fencing.py to disable password requirement
179+
]
180+
181+
define_new_opts()
182+
183+
opt = process_input(device_opt)
184+
options = check_input(device_opt, opt)
185+
186+
docs = {}
187+
docs["shortdesc"] = "Fencing agent for Near Node Flash"
188+
docs["longdesc"] = "fence_nnf is a fencing agent for Near Node Flash storage nodes."
189+
docs["vendorurl"] = "https://nearnodeflash.github.io/"
190+
show_docs(options, docs)
191+
192+
run_delay(options)
193+
194+
if "--localconfig" in options:
195+
from kubernetes import config
196+
configuration = config.load_kube_config()
197+
else:
198+
configuration = client.Configuration(
199+
host="https://%s:%s" % (
200+
options.get("--kubernetes-service-host"),
201+
options.get("--kubernetes-service-port")
202+
)
203+
)
204+
205+
with open(options.get("--service-token-file"), "r") as f:
206+
token = f.read()
207+
208+
configuration.api_key = { "authorization" : token }
209+
configuration.api_key_prefix = { "authorization" : "Bearer" }
210+
configuration.ssl_ca_cert = options.get("--service-cert-file")
211+
212+
with client.ApiClient(configuration) as api_client:
213+
result = fence_action(api_client, options, set_power_status, get_power_status)
214+
215+
sys.exit(result)
216+
217+
if __name__ == "__main__":
218+
main()

‎docs/guides/ha-cluster/notes.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pcs stonith create stonith-rabbit-node-1 fence_nnf pcmk_host_list=rabbit-node-1 kubernetes-service-host=10.30.107.247 kubernetes-service-port=6443 service-token-file=/etc/nnf/service.token service-cert-file=/etc/nnf/service.cert nnf-node-name=rabbit-node-1 verbose=1
2+
3+
pcs stonith create stonith-rabbit-compute-2 fence_redfish pcmk_host_list="rabbit-compute-2" ip=10.30.105.237 port=80 systems-uri=/redfish/v1/Systems/1 username=root password=REDACTED ssl_insecure=true verbose=1
4+
5+
pcs stonith create stonith-rabbit-compute-3 fence_redfish pcmk_host_list="rabbit-compute-3" ip=10.30.105.253 port=80 systems-uri=/redfish/v1/Systems/1 username=root password=REDACTED ssl_insecure=true verbose=1

‎docs/guides/ha-cluster/readme.md‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ categories: setup
55

66
# High Availability Cluster
77

8-
Rabbit software supports provisioning of Red Hat GFS2 (Global File System 2) storage. Per Red Hat:
8+
NNF software supports provisioning of Red Hat GFS2 (Global File System 2) storage. Per RedHat:
99
> GFS2 allows multiple nodes to share storage at a block level as if the storage were connected locally to each cluster node. GFS2 cluster file system requires a cluster infrastructure.
1010
11-
Therefore, in order to use GFS2, the Rabbit and its associated compute nodes must form a high availability cluster.
11+
Therefore, in order to use GFS2, the NNF node and its associated compute nodes must form a high availability cluster.
1212

1313
## Cluster Setup
1414

@@ -24,34 +24,42 @@ HPE hardware implements software known as the Hardware System Supervisor (HSS),
2424

2525
### Compute Fencing
2626

27-
!!! warning
28-
Usage of the Redfish fencing agent is not yet verified
29-
3027
The [Redfish fencing agent](https://github.com/ClusterLabs/fence-agents/tree/main/agents/redfish) from [ClusterLabs](https://github.com/ClusterLabs/fence-agents) should be used for Compute nodes in the cluster. Configure the agent with the following parameters:
3128

3229
| Argument | Definition |
3330
| -------- | ---------- |
34-
| `--ip=[ADDRESS]` | The IP address or hostname of the compute node's HSS node controller |
35-
|`--systems-uri=[URI]` | The URI of the Systems object. Must be `/redfish/v1/Systems/Node0` |
36-
|`--ssl-insecure` | Instructs the use of an insecure SSL exchange |
31+
| `ip=[ADDRESS]` | The IP address or hostname of the HSS controller |
32+
| `port=80` | The Port of the HSS controller. Must be `80` |
33+
| `systems-uri=/redfish/v1/Systems/Node0` | The URI of the Systems object. Must be `/redfish/v1/Systems/Node0` |
34+
| `ssl-insecure=true` | Instructs the use of an insecure SSL exchange. Must be `true` |
35+
| `username=[USER]` | The user name for connecting to the HSS controller |
36+
| `password=[PASSWORD]` | the password for connecting to the HSS controller |
3737

3838

39-
### Rabbit Fencing
39+
### NNF Fencing
4040

4141
!!! info
42-
Rabbit fencing agent is in active development; the description below is subject to change.
43-
44-
Since the Rabbit node is connected to 16 compute blades, careful coordination around fencing of a Rabbit node is required to minimize the impact of the outage. When a Rabbit node is fenced, the corresponding Kubernetes Storage resource (`storages.dws.cray.hpe.com`) is updated with a status of 'Fenced'. The workload manager must observe this change and handle the movement of resources off the Rabbit node and clear the 'Fenced' status before forcibly rebooting the node.
42+
NNF fencing agent is in active development; the description below is subject to change.
4543

46-
Configure the Rabbit agent with the following parameters:
44+
Configure the NNF agent with the following parameters:
4745

4846
| Argument | Definition |
4947
| -------- | ---------- |
50-
| `--kubernetes-service-host=[ADDRESS]` | The IP address of the kubeapi server |
51-
| `--kubernetes-service-port=[PORT]` | The listening port of the kubeapi server |
52-
| `--service-token-file=[PATH]` | The location of the service token file. The file must be present on all nodes within the cluster |
53-
| `--service-cert-file=[PATH]` | The location of the service certificate file. The file must be present on all nodes within the cluster |
54-
| `--nnf-node-name=[RABBIT-NODE-NAME]` | Name of the rabbit node |
48+
| `kubernetes-service-host=[ADDRESS]` | The IP address of the kubeapi server |
49+
| `kubernetes-service-port=[PORT]` | The listening port of the kubeapi server |
50+
| `service-token-file=[PATH]` | The location of the service token file. The file must be present on all nodes within the cluster |
51+
| `service-cert-file=[PATH]` | The location of the service certificate file. The file must be present on all nodes within the cluster |
52+
| `nnf-node-name=[NNF-NODE-NAME]` | Name of the NNF node as it is appears in the System Configuration |
53+
| `api-version=[VERSION]` | The API Version of the NNF Node resource. Defaults to "v1alpha1" |
54+
55+
Since the NNF node is connected to 16 compute blades, careful coordination around fencing of a NNF node is required to minimize the impact of the outage. When a Rabbit node is fenced, the corresponding DWS Storage resource (`storages.dws.cray.hpe.com`) status changes. The workload manager must observe this change and follow the procedure below to recover from the fencing status.
56+
57+
1. Observed the `storage.Status` changed and that `storage.Status.RequiresReboot == True`
58+
2. Set the `storage.Spec.State := Disabled`
59+
4. Wait for a change to the Storage status `storage.Status.State == Disabled`
60+
5. Reboot the NNF node
61+
6. Set the `storage.Spec.State := Enabled`
62+
7. Wait for `storage.Status.State == Enabled`
5563

5664
### Dummy Fencing
5765

0 commit comments

Comments
 (0)