wip

Author: maltesander

docs/modules/hdfs/pages/reference/troubleshooting.adoc

@@ -0,0 +1,27 @@
+= Troubleshooting
+
+[#init-container-format-namenode-fails]
+== Init container format-namenodes fails
+
+When creating fresh HDFS clusters, unexpected pod restarts might corrupt the initial namenode formatting.
+This leaves the namenode data PVC in a dangling state, where e.g. the `../current/VERSION` file is created, but `fsimage_xxx` files are missing.
+
+After a restart corrupted the namenode formatting, reformatting again fails due to directories and files existing.
+We do not want to force (override) the formatting process to avoid data loss and other implications.
+
+[source]
+----
+Running in non-interactive mode, and data appears to exist in Storage Directory root= /stackable/data/namenode; location= null. Not formatting.
+----
+
+Another error message indicating a corrupt formatting state appears in the namenode main container during startup.
+
+[source]
+----
+java.io.FileNotFoundException: No valid image files found
+----
+
+WARNING: The following fix should only be applied to fresh clusters. For existing clusters please consider support.
+
+1. Remove the PVC called `data-<cluster-name>-namenode-<rolegroup>-0` for a failed namenode 0. 
+2. Restart the namenode afterwards.

docs/modules/hdfs/partials/nav.adoc

@@ -23,3 +23,4 @@
 ** xref:hdfs:reference/discovery.adoc[]
 ** xref:hdfs:reference/commandline-parameters.adoc[]
 ** xref:hdfs:reference/environment-variables.adoc[]
+* xref:hdfs:reference/troubleshooting.adoc[]

rust/operator-binary/src/container.rs

@@ -709,16 +709,6 @@ impl ContainerConfig {
 
                     if [ ! -f "{NAMENODE_ROOT_DATA_DIR}/current/VERSION" ]
                     then
-                      # Sanity check for data corruption: VERSION file exists but no fsimage_xxx files were created.
-                      FSIMAGE_COUNT=$(find "{NAMENODE_ROOT_DATA_DIR}/current" -maxdepth 1 -regextype posix-egrep -regex ".*/fsimage_[0-9]+" | wc -l)
-
-                      if [ "${{FSIMAGE_COUNT}}" -eq 0 ] 
-                      then
-                        echo "WARNING: {NAMENODE_ROOT_DATA_DIR}/current/VERSION file exists but no fsimage file(s) found."
-                        echo "This indicates an incomplete and corrupted namenode formatting. Please check the troubleshooting guide.
-                        exit 1
-                      fi
-
                       if [ -z ${{ACTIVE_NAMENODE+x}} ]
                       then
                         echo "No active namenode found. Formatting $POD_NAME as active."
@@ -728,6 +718,16 @@ impl ContainerConfig {
                         exclude_from_capture {hadoop_home}/bin/hdfs namenode -bootstrapStandby -nonInteractive
                       fi
                     else
+                      # Sanity check for initial format data corruption: VERSION file exists but no fsimage files were created.
+                      FSIMAGE_COUNT=$(find "{NAMENODE_ROOT_DATA_DIR}/current" -maxdepth 1 -regextype posix-egrep -regex ".*/fsimage_[0-9]+" | wc -l)
+
+                      if [ "${{FSIMAGE_COUNT}}" -eq 0 ] 
+                      then
+                        echo "WARNING: {NAMENODE_ROOT_DATA_DIR}/current/VERSION file exists but no fsimage files were found."
+                        echo "This indicates an incomplete and corrupted namenode formatting. Please check the troubleshooting guide."
+                        exit 1
+                      fi
+
                       cat "{NAMENODE_ROOT_DATA_DIR}/current/VERSION"
                       echo "Pod $POD_NAME already formatted. Skipping..."
                     fi

rust/operator-binary/src/hdfs_controller.rs

@@ -22,7 +22,6 @@ use stackable_operator::{
         product_image_selection::{self, ResolvedProductImage},
         rbac::build_rbac_resources,
     },
-    constants::RESTART_CONTROLLER_ENABLED_LABEL,
     iter::reverse_if,
     k8s_openapi::{
         DeepMerge,
@@ -901,13 +900,8 @@ fn rolegroup_statefulset(
         ..StatefulSetSpec::default()
     };
 
-    let sts_metadata = metadata
-        .clone()
-        .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned())
-        .build();
-
     Ok(StatefulSet {
-        metadata: sts_metadata,
+        metadata: metadata.build(),
         spec: Some(statefulset_spec),
         status: None,
     })

tests/templates/kuttl/smoke/30-assert.yaml.j2

@@ -7,9 +7,6 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: hdfs-namenode-default
-  generation: 1 # There should be no unneeded Pod restarts
-  labels:
-    restarter.stackable.tech/enabled: "true"
 spec:
   template:
     spec:
@@ -35,9 +32,6 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: hdfs-journalnode-default
-  generation: 1 # There should be no unneeded Pod restarts
-  labels:
-    restarter.stackable.tech/enabled: "true"
 spec:
   template:
     spec:
@@ -62,9 +56,6 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: hdfs-datanode-default
-  generation: 1 # There should be no unneeded Pod restarts
-  labels:
-    restarter.stackable.tech/enabled: "true"
 spec:
   template:
     spec: