cleaned up docs, lowered logging level for informer

Author: adwk67

src/main/java/tech/stackable/hadoop/StackableTopologyProvider.java

@@ -582,7 +582,7 @@ public void onAdd(Pod pod) {
                 for (PodIP ip : pod.getStatus().getPodIPs()) {
                   cache.putPod(ip.getIp(), pod);
                 }
-                LOG.info("Pod {} added", pod.getMetadata().getName());
+                LOG.debug("Pod {} added", pod.getMetadata().getName());
               }
 
               @Override
@@ -591,7 +591,7 @@ public void onUpdate(Pod oldPod, Pod newPod) {
                 for (PodIP ip : oldPod.getStatus().getPodIPs()) {
                   cache.putPod(ip.getIp(), newPod);
                 }
-                LOG.info("Pod {} updated", oldPod.getMetadata().getName());
+                LOG.trace("Pod {} updated", oldPod.getMetadata().getName());
               }
 
               @Override
@@ -600,7 +600,7 @@ public void onDelete(Pod pod, boolean deletedFinalStateUnknown) {
                 for (PodIP ip : pod.getStatus().getPodIPs()) {
                   cache.deletePod(ip.getIp());
                 }
-                LOG.info("Pod {} deleted", pod.getMetadata().getName());
+                LOG.debug("Pod {} deleted", pod.getMetadata().getName());
               }
             },
             INFORMER_POLL_SECONDS * 1000L);

test/topology-provider/stack/03-hdfs.yaml

@@ -19,7 +19,7 @@ metadata:
 spec:
   image:
     productVersion: 3.4.2
-    custom: oci.stackable.tech/sandbox/andrew/hadoop:3.4.2-stackable0.0.0-topprov
+    custom: oci.stackable.tech/sandbox/hadoop:3.4.2-stackable0.0.0-dev
     pullPolicy: IfNotPresent
   clusterConfig:
     dfsReplication: 1
@@ -48,6 +48,8 @@ spec:
     configOverrides: &configOverrides
       core-site.xml:
         hadoop.user.group.static.mapping.overrides: "dr.who=;nn=;nm=;jn=;testuser=supergroup;"
+    envOverrides:
+      HADOOP_NAMENODE_OPTS: -agentlib:jdwp=transport=dt_socket,address=*:5005,server=y,suspend=n
     roleGroups:
       default:
         replicas: 2

test/topology-provider/stack/04-spark.yaml

@@ -12,7 +12,7 @@ spec:
   mainApplicationFile: local:///tmp/spark-terasort-1.2-SNAPSHOT.jar
   mainClass: com.github.ehiggs.spark.terasort.TeraGen
   args:
-    - "100M"
+    - "10M"
     - "hdfs://simple-hdfs/user/stackable/teragen_output"
   sparkConf:
     "spark.driver.extraClassPath": "/etc/hadoop/conf/:/stackable/spark/extra-jars/*"

test/topology-provider/stack/05-access-hdfs.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: access-hdfs
-          image: oci.stackable.tech/sandbox/andrew/hadoop:3.4.2-stackable0.0.0-topprov
+          image: oci.stackable.tech/sandbox/hadoop:3.4.2-stackable0.0.0-dev
           imagePullPolicy: IfNotPresent
           env:
             - name: HADOOP_CONF_DIR
@@ -26,14 +26,13 @@ spec:
               kinit -kt /stackable/kerberos/keytab testuser/access-hdfs.default.svc.cluster.local
               klist
 
-              #bin/hdfs dfs -mkdir /stackable
-              #bin/hdfs dfs -chown -R access-hive /stackable
-              #bin/hdfs dfs -mkdir /access-hive
-              #bin/hdfs dfs -chown -R access-hive /access-hive
+
+              bin/hdfs dfs -mkdir /access-hdfs
+              bin/hdfs dfs -chown -R access-hdfs /access-hdfs
               bin/hdfs dfs -ls /
-              #bin/hdfs dfs -ls -d /
+              bin/hdfs dfsadmin -printTopology
+              echo "Hello HDFS" | bin/hdfs dfs -put - /access-hdfs/file.txt
 
-              sleep infinity
           volumeMounts:
             - name: hdfs-config
               mountPath: /stackable/conf/hdfs

test/topology-provider/stack/README.md

@@ -0,0 +1,17 @@
+# Testing
+
+## Install operators
+
+```bash
+stackablectl op in secret commons listener secret zookeeper hdfs spark-k8s
+```
+
+## Run scripts in the `default` namespace
+
+```bash
+kubectl apply -f ./hdfs-utils/test/topology-provider/stack/01-install-krb5-kdc.yaml
+kubectl apply -f ./hdfs-utils/test/topology-provider/stack/02-create-kerberos-secretclass.yaml
+kubectl apply -f ./hdfs-utils/test/topology-provider/stack/03-hdfs.yaml
+kubectl apply -f ./hdfs-utils/test/topology-provider/stack/04-spark.yaml
+kubectl apply -f ./hdfs-utils/test/topology-provider/stack/05-access-hdfs.yaml
+```