Skip to content

Commit 6817253

Browse files
sbernauersbernauer
committed
Get rid of Zookeeper, simply url gathering, better waiting
1 parent 9147fad commit 6817253

4 files changed

Lines changed: 39 additions & 215 deletions

File tree

‎docs/modules/nifi/examples/getting_started/getting_started.sh‎

Lines changed: 19 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ echo "Installing Operators with Helm"
2929
helm install --wait commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator --version 0.0.0-dev
3030
helm install --wait secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator --version 0.0.0-dev
3131
helm install --wait listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator --version 0.0.0-dev
32-
helm install --wait zookeeper-operator oci://oci.stackable.tech/sdp-charts/zookeeper-operator --version 0.0.0-dev
3332
helm install --wait nifi-operator oci://oci.stackable.tech/sdp-charts/nifi-operator --version 0.0.0-dev
3433
# end::helm-install-operators[]
3534
;;
@@ -40,7 +39,6 @@ stackablectl operator install \
4039
commons=0.0.0-dev \
4140
secret=0.0.0-dev \
4241
listener=0.0.0-dev \
43-
zookeeper=0.0.0-dev \
4442
nifi=0.0.0-dev
4543
# end::stackablectl-install-operators[]
4644
;;
@@ -50,45 +48,6 @@ exit 1
5048
;;
5149
esac
5250

53-
echo "Installing ZooKeeper"
54-
# tag::install-zookeeper[]
55-
kubectl apply -f - <<EOF
56-
---
57-
apiVersion: zookeeper.stackable.tech/v1alpha1
58-
kind: ZookeeperCluster
59-
metadata:
60-
name: simple-zk
61-
spec:
62-
image:
63-
productVersion: 3.9.3
64-
servers:
65-
roleGroups:
66-
default:
67-
replicas: 1
68-
EOF
69-
# end::install-zookeeper[]
70-
71-
echo "Create a ZNode"
72-
# tag::install-znode[]
73-
kubectl apply -f - <<EOF
74-
---
75-
apiVersion: zookeeper.stackable.tech/v1alpha1
76-
kind: ZookeeperZnode
77-
metadata:
78-
name: simple-nifi-znode
79-
spec:
80-
clusterRef:
81-
name: simple-zk
82-
EOF
83-
# end::install-znode[]
84-
85-
sleep 15
86-
87-
echo "Awaiting ZooKeeper rollout finish"
88-
# tag::watch-zookeeper-rollout[]
89-
kubectl rollout status --watch --timeout=5m statefulset/simple-zk-server-default
90-
# end::watch-zookeeper-rollout[]
91-
9251
echo "Create NiFi admin credentials"
9352
# tag::install-nifi-credentials[]
9453
kubectl apply -f - <<EOF
@@ -129,7 +88,6 @@ spec:
12988
sensitiveProperties:
13089
keySecret: nifi-sensitive-property-key
13190
autoGenerate: true
132-
zookeeperConfigMapName: simple-nifi-znode
13391
nodes:
13492
roleConfig:
13593
listenerClass: external-unstable
@@ -139,50 +97,32 @@ spec:
13997
EOF
14098
# end::install-nifi[]
14199

142-
sleep 5
143-
144100
echo "Awaiting NiFi rollout finish"
145101
# tag::wait-nifi-rollout[]
146-
kubectl wait -l statefulset.kubernetes.io/pod-name=simple-nifi-node-default-0 \
147-
--for=condition=ready pod --timeout=1200s
102+
kubectl wait --for=condition=available --timeout=20m nificluster/simple-nifi
148103
# end::wait-nifi-rollout[]
149104

150-
sleep 5
151-
152105
case "$1" in
153-
"helm")
154-
echo "Getting the NiFi endpoint with kubectl"
155-
156-
echo "Get first node address from Listener"
157-
# tag::get-nifi-node-address[]
158-
nifi_node_address=$(kubectl get listener simple-nifi-node -o 'jsonpath={.status.ingressAddresses[0].address}') && \
159-
echo "NodeAddress: $nifi_node_address"
160-
# end::get-nifi-node-address[]
161-
162-
echo "Get HTTPS node port from Listener"
163-
# tag::get-nifi-node-port[]
164-
nifi_node_port=$(kubectl get listener simple-nifi-node -o 'jsonpath={.status.nodePorts.https}') && \
165-
echo "NodePort: $nifi_node_port"
166-
# end::get-nifi-node-port[]
106+
"helm")
107+
echo "Getting the NiFi URL with kubectl"
167108

168-
echo "Create NiFi url"
169-
# tag::create-nifi-url[]
170-
nifi_url="https://$nifi_node_address:$nifi_node_port" && \
171-
echo "NiFi web interface: $nifi_url"
172-
# end::create-nifi-url[]
109+
# tag::get-nifi-url[]
110+
nifi_url=$(kubectl get listener simple-nifi-node -o 'jsonpath=https://{.status.ingressAddresses[0].address}:{.status.ingressAddresses[0].ports.https}') && \
111+
echo "NiFi URL: $nifi_url"
112+
# end::get-nifi-url[]
173113

174-
;;
175-
"stackablectl")
176-
echo "Getting NiFi endpoint with stackablectl ..."
177-
# tag::stackablectl-nifi-url[]
178-
nifi_url=$(stackablectl stacklet ls -o json | jq --raw-output '.[] | select(.name == "simple-nifi") | .endpoints["node-https"]')
179-
# end::stackablectl-nifi-url[]
180-
echo "Endpoint: $nifi_url"
181-
;;
182-
*)
183-
echo "Need to provide 'helm' or 'stackablectl' as an argument for which installation method to use!"
184-
exit 1
185-
;;
114+
;;
115+
"stackablectl")
116+
echo "Getting NiFi URL with stackablectl ..."
117+
# tag::stackablectl-nifi-url[]
118+
nifi_url=$(stackablectl stacklet ls -o json | jq --raw-output '.[] | select(.name == "simple-nifi") | .endpoints["node-https"]')
119+
# end::stackablectl-nifi-url[]
120+
echo "NiFi URL: $nifi_url"
121+
;;
122+
*)
123+
echo "Need to provide 'helm' or 'stackablectl' as an argument for which installation method to use!"
124+
exit 1
125+
;;
186126
esac
187127

188128
echo "Starting nifi tests"

‎docs/modules/nifi/examples/getting_started/getting_started.sh.j2‎

Lines changed: 8 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ echo "Installing Operators with Helm"
2929
helm install --wait commons-operator oci://{{ helm.repo_url }}/{{ helm.repo_name }}/commons-operator --version {{ versions.commons }}
3030
helm install --wait secret-operator oci://{{ helm.repo_url }}/{{ helm.repo_name }}/secret-operator --version {{ versions.secret }}
3131
helm install --wait listener-operator oci://{{ helm.repo_url }}/{{ helm.repo_name }}/listener-operator --version {{ versions.listener }}
32-
helm install --wait zookeeper-operator oci://{{ helm.repo_url }}/{{ helm.repo_name }}/zookeeper-operator --version {{ versions.zookeeper }}
3332
helm install --wait nifi-operator oci://{{ helm.repo_url }}/{{ helm.repo_name }}/nifi-operator --version {{ versions.nifi }}
3433
# end::helm-install-operators[]
3534
;;
@@ -40,7 +39,6 @@ stackablectl operator install \
4039
commons={{ versions.commons }} \
4140
secret={{ versions.secret }} \
4241
listener={{ versions.listener }} \
43-
zookeeper={{ versions.zookeeper }} \
4442
nifi={{ versions.nifi }}
4543
# end::stackablectl-install-operators[]
4644
;;
@@ -50,45 +48,6 @@ exit 1
5048
;;
5149
esac
5250

53-
echo "Installing ZooKeeper"
54-
# tag::install-zookeeper[]
55-
kubectl apply -f - <<EOF
56-
---
57-
apiVersion: zookeeper.stackable.tech/v1alpha1
58-
kind: ZookeeperCluster
59-
metadata:
60-
name: simple-zk
61-
spec:
62-
image:
63-
productVersion: 3.9.3
64-
servers:
65-
roleGroups:
66-
default:
67-
replicas: 1
68-
EOF
69-
# end::install-zookeeper[]
70-
71-
echo "Create a ZNode"
72-
# tag::install-znode[]
73-
kubectl apply -f - <<EOF
74-
---
75-
apiVersion: zookeeper.stackable.tech/v1alpha1
76-
kind: ZookeeperZnode
77-
metadata:
78-
name: simple-nifi-znode
79-
spec:
80-
clusterRef:
81-
name: simple-zk
82-
EOF
83-
# end::install-znode[]
84-
85-
sleep 15
86-
87-
echo "Awaiting ZooKeeper rollout finish"
88-
# tag::watch-zookeeper-rollout[]
89-
kubectl rollout status --watch --timeout=5m statefulset/simple-zk-server-default
90-
# end::watch-zookeeper-rollout[]
91-
9251
echo "Create NiFi admin credentials"
9352
# tag::install-nifi-credentials[]
9453
kubectl apply -f - <<EOF
@@ -129,7 +88,6 @@ spec:
12988
sensitiveProperties:
13089
keySecret: nifi-sensitive-property-key
13190
autoGenerate: true
132-
zookeeperConfigMapName: simple-nifi-znode
13391
nodes:
13492
roleConfig:
13593
listenerClass: external-unstable
@@ -139,45 +97,27 @@ spec:
13997
EOF
14098
# end::install-nifi[]
14199

142-
sleep 5
143-
144100
echo "Awaiting NiFi rollout finish"
145101
# tag::wait-nifi-rollout[]
146-
kubectl wait -l statefulset.kubernetes.io/pod-name=simple-nifi-node-default-0 \
147-
--for=condition=ready pod --timeout=1200s
102+
kubectl wait --for=condition=available --timeout=20m nificluster/simple-nifi
148103
# end::wait-nifi-rollout[]
149104

150-
sleep 5
151-
152105
case "$1" in
153106
"helm")
154-
echo "Getting the NiFi endpoint with kubectl"
155-
156-
echo "Get first node address from Listener"
157-
# tag::get-nifi-node-address[]
158-
nifi_node_address=$(kubectl get listener simple-nifi-node -o 'jsonpath={.status.ingressAddresses[0].address}') && \
159-
echo "NodeAddress: $nifi_node_address"
160-
# end::get-nifi-node-address[]
161-
162-
echo "Get HTTPS node port from Listener"
163-
# tag::get-nifi-node-port[]
164-
nifi_node_port=$(kubectl get listener simple-nifi-node -o 'jsonpath={.status.nodePorts.https}') && \
165-
echo "NodePort: $nifi_node_port"
166-
# end::get-nifi-node-port[]
107+
echo "Getting the NiFi URL with kubectl"
167108

168-
echo "Create NiFi url"
169-
# tag::create-nifi-url[]
170-
nifi_url="https://$nifi_node_address:$nifi_node_port" && \
171-
echo "NiFi web interface: $nifi_url"
172-
# end::create-nifi-url[]
109+
# tag::get-nifi-url[]
110+
nifi_url=$(kubectl get listener simple-nifi-node -o 'jsonpath=https://{.status.ingressAddresses[0].address}:{.status.ingressAddresses[0].ports.https}') && \
111+
echo "NiFi URL: $nifi_url"
112+
# end::get-nifi-url[]
173113

174114
;;
175115
"stackablectl")
176-
echo "Getting NiFi endpoint with stackablectl ..."
116+
echo "Getting NiFi URL with stackablectl ..."
177117
# tag::stackablectl-nifi-url[]
178118
nifi_url=$(stackablectl stacklet ls -o json | jq --raw-output '.[] | select(.name == "simple-nifi") | .endpoints["node-https"]')
179119
# end::stackablectl-nifi-url[]
180-
echo "Endpoint: $nifi_url"
120+
echo "NiFi URL: $nifi_url"
181121
;;
182122
*)
183123
echo "Need to provide 'helm' or 'stackablectl' as an argument for which installation method to use!"

‎docs/modules/nifi/pages/getting_started/first_steps.adoc‎

Lines changed: 10 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,13 @@
11
= First steps
2-
:description: Deploy Apache NiFi and ZooKeeper on Kubernetes using manifest files. Verify setup and access NiFi's web interface with stackablectl or kubectl.
2+
:description: Deploy Apache NiFi on Kubernetes using manifest files. Verify setup and access NiFi's web interface with stackablectl or kubectl.
33

44
After going through the xref:getting_started/installation.adoc[] section and having installed all the operators, you deploy a NiFi cluster and the required dependencies.
55
Afterward you can <<_verify_that_it_works, verify that it works>> by querying the REST API.
66

77
== Setup
88

9-
Two things need to be installed to create a NiFi cluster:
10-
11-
* A ZooKeeper cluster for internal use by NiFi
12-
* The NiFi cluster itself
13-
14-
Create them in this order, each one is created by applying a manifest file.
15-
The operators create the resources according to the manifest.
16-
17-
=== Apache ZooKeeper
18-
19-
To create a ZooKeeper instance run the following command:
20-
21-
[source,bash]
22-
----
23-
include::example$getting_started/getting_started.sh[tag=install-zookeeper]
24-
----
25-
26-
Create a Znode object:
27-
28-
[source,bash]
29-
----
30-
include::example$getting_started/getting_started.sh[tag=install-znode]
31-
----
32-
33-
The xref:zookeeper:znodes.adoc[ZNode] makes sure that the NiFi cluster operates in its own separated directory in ZooKeeper.
34-
35-
=== Apache NiFi
9+
We are going to install a NiFi cluster by applying a manifest file.
10+
The operators create the resources according to the manifests.
3611

3712
The NiFi cluster requires authentication.
3813
Create a set of credentials for this purpose:
@@ -42,7 +17,7 @@ Create a set of credentials for this purpose:
4217
include::example$getting_started/getting_started.sh[tag=install-nifi-credentials]
4318
----
4419

45-
Finally create a NiFi instance:
20+
Afterwards create a NiFi instance:
4621

4722
[source,bash]
4823
----
@@ -92,9 +67,8 @@ stackablectl stacklet list
9267
which should return something like this:
9368

9469
----
95-
PRODUCT NAME NAMESPACE ENDPOINTS CONDITIONS
96-
nifi simple-nifi default node-https https://172.18.0.2:30528 Available, Reconciling, Running
97-
zookeeper simple-zk default server-zk simple-zk-server.default.svc.cluster.local:2282 Available, Reconciling, Running
70+
PRODUCT NAME NAMESPACE ENDPOINTS CONDITIONS
71+
nifi simple-nifi default node-https https://172.18.0.2:30528 Available, Reconciling, Running
9872
----
9973

10074
You can also use the `json` output and parse the endpoint:
@@ -110,48 +84,18 @@ image::nifi-web-ui.png[]
11084

11185
==== Via kubectl
11286

113-
You can also extract the IP and port from the xref:listener-operator:listener.adoc[Listener] status via `kubectl`.
114-
115-
Retrieve the Kubernetes node address where the NiFi pod is running:
116-
117-
[source,bash]
118-
----
119-
include::example$getting_started/getting_started.sh[tag=get-nifi-node-address]
120-
----
121-
122-
which should output the Kubernetes node address:
123-
124-
[source]
125-
----
126-
NodeAddress: 172.18.0.2
127-
----
128-
129-
Finally, retrieve the NodePort of the `simple-nifi` listener:
130-
131-
[source,bash]
132-
----
133-
include::example$getting_started/getting_started.sh[tag=get-nifi-node-port]
134-
----
135-
136-
which should output the NodePort:
137-
138-
[source]
139-
----
140-
NodePort: 30528
141-
----
142-
143-
Now build the full URL:
87+
You can also extract the endpoint from the xref:listener-operator:listener.adoc[Listener] status via `kubectl`:
14488

14589
[source,bash]
14690
----
147-
include::example$getting_started/getting_started.sh[tag=create-nifi-url]
91+
include::example$getting_started/getting_started.sh[tag=get-nifi-url]
14892
----
14993

150-
which should output a URL to connect to the NiFi web interface:
94+
should output a URL to connect to the NiFi web interface:
15195

15296
[source]
15397
----
154-
NiFi web interface: https://172.18.0.2:30528
98+
NiFi URL: https://172.18.0.2:30528
15599
----
156100

157101
Then connect to `https://172.18.0.2:30528/nifi` and you should see the NiFi web login. After providing the username `admin` and password `admin` you are redirected to the NiFi web interface.

‎docs/modules/nifi/pages/getting_started/installation.adoc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
= Installation
22
:description: Install the Stackable operator for Apache NiFi and its dependencies using stackablectl or Helm. Follow steps for a complete setup on Kubernetes.
33

4-
On this page you install the Stackable operator for Apache NiFi and operators for its dependencies -- ZooKeeper --
5-
as well as the commons, secret and listener operator which are required by all Stackable operators.
4+
On this page you install the Stackable operator for Apache NiFi as well as the commons, secret and listener operator,
5+
which are required by all Stackable operators.
66

77
There are multiple ways to install the Stackable operators.
88
xref:management:stackablectl:index.adoc[] is the preferred way, but Helm is also supported.

0 commit comments

Comments
 (0)