Skip to content

Commit 8a6ab18

Browse files
maltesandermaltesander
authored
Improve shell capturing in init containers (#746)
* wip - test format-namenodes container logging improvements * fix bash helpers * extend shell capture for wait-for-namenodes and format-zooekeeper * cleanup * adapted changelog * precommit * fix pre-commit * fix typo * remove set +x and add format-zookeeper test * fix capture_shell script * cleanup
1 parent 807779e commit 8a6ab18

3 files changed

Lines changed: 175 additions & 57 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,13 @@ All notable changes to this project will be documented in this file.
1010
See [objectOverrides concepts page](https://docs.stackable.tech/home/nightly/concepts/overrides/#object-overrides) for details ([#741]).
1111
- Enable the [restart-controller](https://docs.stackable.tech/home/nightly/commons-operator/restarter/), so that the Pods are automatically restarted on config changes ([#743]).
1212

13+
### Fixed
14+
15+
- Previously, some shell output of init-containers was not logged properly and therefore not aggregated, which is fixed now ([#746]).
16+
1317
[#741]: https://github.com/stackabletech/hdfs-operator/pull/741
1418
[#743]: https://github.com/stackabletech/hdfs-operator/pull/743
19+
[#746]: https://github.com/stackabletech/hdfs-operator/pull/746
1520

1621
## [25.11.0] - 2025-11-07
1722

‎rust/operator-binary/src/container.rs‎

Lines changed: 88 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ use stackable_operator::{
4444
LoggingError, create_vector_shutdown_file_command, remove_vector_shutdown_file_command,
4545
},
4646
spec::{
47-
ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice,
48-
CustomContainerLogConfig,
47+
AutomaticContainerLogConfig, ConfigMapLogConfig, ContainerLogConfig,
48+
ContainerLogConfigChoice, CustomContainerLogConfig,
4949
},
5050
},
5151
role_utils::RoleGroupRef,
@@ -627,22 +627,22 @@ impl ContainerConfig {
627627
&merged_config.hdfs_logging(),
628628
));
629629

630-
args.push_str(&format!(
630+
args.push_str(&formatdoc!(
631631
r#"\
632-
{COMMON_BASH_TRAP_FUNCTIONS}
633-
{remove_vector_shutdown_file_command}
634-
prepare_signal_handlers
635-
containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &
636-
if [[ -d {LISTENER_VOLUME_DIR} ]]; then
637-
export POD_ADDRESS=$(cat {LISTENER_VOLUME_DIR}/default-address/address)
638-
for i in {LISTENER_VOLUME_DIR}/default-address/ports/*; do
639-
export $(basename $i | tr a-z- A-Z_)_PORT="$(cat $i)"
640-
done
641-
fi
642-
{hadoop_home}/bin/hdfs {role} {upgrade_args} &
643-
wait_for_termination $!
644-
{create_vector_shutdown_file_command}
645-
"#,
632+
{COMMON_BASH_TRAP_FUNCTIONS}
633+
{remove_vector_shutdown_file_command}
634+
prepare_signal_handlers
635+
containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &
636+
if [[ -d {LISTENER_VOLUME_DIR} ]]; then
637+
export POD_ADDRESS=$(cat {LISTENER_VOLUME_DIR}/default-address/address)
638+
for i in {LISTENER_VOLUME_DIR}/default-address/ports/*; do
639+
export $(basename $i | tr a-z- A-Z_)_PORT="$(cat $i)"
640+
done
641+
fi
642+
{hadoop_home}/bin/hdfs {role} {upgrade_args} &
643+
wait_for_termination $!
644+
{create_vector_shutdown_file_command}
645+
"#,
646646
hadoop_home = Self::HADOOP_HOME,
647647
remove_vector_shutdown_file_command =
648648
remove_vector_shutdown_file_command(STACKABLE_LOG_DIR),
@@ -664,7 +664,9 @@ wait_for_termination $!
664664
hadoop_home = Self::HADOOP_HOME
665665
));
666666
}
667-
ContainerConfig::FormatNameNodes { .. } => {
667+
ContainerConfig::FormatNameNodes { container_name, .. } => {
668+
args.push_str(&bash_capture_shell_helper(container_name));
669+
668670
if let Some(container_config) = merged_config.as_namenode().map(|node| {
669671
node.logging
670672
.for_container(&NameNodeContainer::FormatNameNodes)
@@ -690,32 +692,36 @@ wait_for_termination $!
690692
for namenode_id in {pod_names}
691693
do
692694
echo -n "Checking pod $namenode_id... "
693-
{get_service_state_command}
695+
696+
# We only redirect 2 (stderr) to 4 (console).
697+
# We leave 1 (stdout) alone so the $(...) can catch it.
698+
SERVICE_STATE=$({hadoop_home}/bin/hdfs haadmin -getServiceState "$namenode_id" 2>&4 | tail -n1 || true)
699+
694700
if [ "$SERVICE_STATE" == "active" ]
695701
then
696-
ACTIVE_NAMENODE=$namenode_id
702+
ACTIVE_NAMENODE="$namenode_id"
697703
echo "active"
698704
break
705+
else
706+
echo "unknown / unreachable"
699707
fi
700-
echo ""
701708
done
702709
703710
if [ ! -f "{NAMENODE_ROOT_DATA_DIR}/current/VERSION" ]
704711
then
705712
if [ -z ${{ACTIVE_NAMENODE+x}} ]
706713
then
707-
echo "Create pod $POD_NAME as active namenode."
708-
{hadoop_home}/bin/hdfs namenode -format -noninteractive
714+
echo "No active namenode found. Formatting $POD_NAME as active."
715+
exclude_from_capture {hadoop_home}/bin/hdfs namenode -format -noninteractive
709716
else
710-
echo "Create pod $POD_NAME as standby namenode."
711-
{hadoop_home}/bin/hdfs namenode -bootstrapStandby -nonInteractive
717+
echo "Active namenode is $ACTIVE_NAMENODE. Bootstrapping standby."
718+
exclude_from_capture {hadoop_home}/bin/hdfs namenode -bootstrapStandby -nonInteractive
712719
fi
713720
else
714721
cat "{NAMENODE_ROOT_DATA_DIR}/current/VERSION"
715722
echo "Pod $POD_NAME already formatted. Skipping..."
716723
fi
717724
"###,
718-
get_service_state_command = Self::get_namenode_service_state_command(),
719725
hadoop_home = Self::HADOOP_HOME,
720726
pod_names = namenode_podrefs
721727
.iter()
@@ -724,7 +730,9 @@ wait_for_termination $!
724730
.join(" "),
725731
));
726732
}
727-
ContainerConfig::FormatZooKeeper { .. } => {
733+
ContainerConfig::FormatZooKeeper { container_name, .. } => {
734+
args.push_str(&bash_capture_shell_helper(container_name));
735+
728736
if let Some(container_config) = merged_config.as_namenode().map(|node| {
729737
node.logging
730738
.for_container(&NameNodeContainer::FormatZooKeeper)
@@ -736,29 +744,27 @@ wait_for_termination $!
736744
}
737745
args.push_str(&formatdoc!(
738746
r###"
739-
echo "Attempt to format ZooKeeper..."
747+
echo "Attempt to format ZooKeeper ZNode for $POD_NAME ..."
740748
if [[ "0" -eq "$(echo $POD_NAME | sed -e 's/.*-//')" ]] ; then
741-
set +e
742-
{hadoop_home}/bin/hdfs zkfc -formatZK -nonInteractive
743-
EXITCODE=$?
744-
set -e
749+
EXITCODE=$(exclude_from_capture {hadoop_home}/bin/hdfs zkfc -formatZK -nonInteractive)
745750
if [[ $EXITCODE -eq 0 ]]; then
746-
echo "Successfully formatted"
751+
echo "Successfully formatted ZooKeeper ZNode."
747752
elif [[ $EXITCODE -eq 2 ]]; then
748-
echo "ZNode already existed, did nothing"
753+
echo "ZNode already exists, nothing to do."
749754
else
750-
echo "Zookeeper format failed with exit code $EXITCODE"
755+
echo "ZooKeeper format ZNode failed with exit code $EXITCODE".
751756
exit $EXITCODE
752757
fi
753-
754758
else
755-
echo "ZooKeeper already formatted!"
759+
echo "ZooKeeper ZNode already formatted!"
756760
fi
757761
"###,
758-
hadoop_home = Self::HADOOP_HOME
762+
hadoop_home = Self::HADOOP_HOME,
759763
));
760764
}
761-
ContainerConfig::WaitForNameNodes { .. } => {
765+
ContainerConfig::WaitForNameNodes { container_name, .. } => {
766+
args.push_str(&bash_capture_shell_helper(container_name));
767+
762768
if let Some(container_config) = merged_config.as_datanode().map(|node| {
763769
node.logging
764770
.for_container(&DataNodeContainer::WaitForNameNodes)
@@ -781,7 +787,11 @@ wait_for_termination $!
781787
for namenode_id in {pod_names}
782788
do
783789
echo -n "Checking pod $namenode_id... "
784-
{get_service_state_command}
790+
791+
# We only redirect 2 (stderr) to 4 (console).
792+
# We leave 1 (stdout) alone so the $(...) can catch it.
793+
SERVICE_STATE=$({hadoop_home}/bin/hdfs haadmin -getServiceState "$namenode_id" 2>&4 | tail -n1 || true)
794+
785795
if [ "$SERVICE_STATE" = "active" ] || [ "$SERVICE_STATE" = "standby" ]
786796
then
787797
echo "$SERVICE_STATE"
@@ -800,7 +810,7 @@ wait_for_termination $!
800810
sleep 5
801811
done
802812
"###,
803-
get_service_state_command = Self::get_namenode_service_state_command(),
813+
hadoop_home = Self::HADOOP_HOME,
804814
pod_names = namenode_podrefs
805815
.iter()
806816
.map(|pod_ref| pod_ref.pod_name.as_ref())
@@ -842,14 +852,6 @@ wait_for_termination $!
842852
))
843853
}
844854

845-
fn get_namenode_service_state_command() -> String {
846-
formatdoc!(
847-
r###"
848-
SERVICE_STATE=$({hadoop_home}/bin/hdfs haadmin -getServiceState $namenode_id | tail -n1 || true)"###,
849-
hadoop_home = Self::HADOOP_HOME,
850-
)
851-
}
852-
853855
/// Returns the container env variables.
854856
fn env(
855857
&self,
@@ -1565,3 +1567,40 @@ impl TryFrom<&str> for ContainerVolumeDirs {
15651567
})
15661568
}
15671569
}
1570+
1571+
fn bash_capture_shell_helper(container_name: &str) -> String {
1572+
let capture_shell_output = product_logging::framework::capture_shell_output(
1573+
STACKABLE_LOG_DIR,
1574+
container_name,
1575+
// we do not access any of the crd config options for this and just log it to file
1576+
&AutomaticContainerLogConfig::default(),
1577+
);
1578+
1579+
formatdoc! {
1580+
r###"
1581+
# Store the original stdout/stderr globally so we can always find our way back
1582+
# 3 and 4 are usually safe, but we'll be explicit.
1583+
exec 3>&1
1584+
exec 4>&2
1585+
1586+
start_capture() {{
1587+
# We redirect 1 and 2 to the background tee processes
1588+
{capture_shell_output}
1589+
}}
1590+
1591+
stop_capture() {{
1592+
# Restore stdout and stderr from our saved descriptors
1593+
exec 1>&3 2>&4
1594+
}}
1595+
1596+
exclude_from_capture() {{
1597+
# Temporarily restore original FDs just for the duration of this command
1598+
# We use 'local' for the exit code to keep things clean
1599+
"$@" 1>&3 2>&4
1600+
echo $?
1601+
}}
1602+
1603+
start_capture
1604+
"###
1605+
}
1606+
}

‎tests/templates/kuttl/logging/hdfs-vector-aggregator-values.yaml.j2‎

Lines changed: 82 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -60,24 +60,83 @@ customConfig:
6060
condition: >-
6161
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
6262
.container == "vector"
63-
filteredAutomaticLogConfigNameNode0FormatNameNode:
63+
filteredAutomaticLogConfigNameNode0FormatNameNodeLog4j:
6464
type: filter
6565
inputs: [validEvents]
6666
condition: >-
6767
.pod == "test-hdfs-automatic-log-namenode-default-0" &&
68-
.container == "format-namenodes"
69-
filteredAutomaticLogConfigNameNode1FormatNameNode:
68+
.container == "format-namenodes" &&
69+
.file == "format-namenodes.log4j.xml"
70+
filteredAutomaticLogConfigNameNode0FormatNameNodeStdout:
71+
type: filter
72+
inputs: [validEvents]
73+
condition: >-
74+
.pod == "test-hdfs-automatic-log-namenode-default-0" &&
75+
.container == "format-namenodes" &&
76+
.file == "container.stdout.log"
77+
filteredAutomaticLogConfigNameNode0FormatNameNodeStderr:
78+
type: filter
79+
inputs: [validEvents]
80+
condition: >-
81+
.pod == "test-hdfs-automatic-log-namenode-default-0" &&
82+
.container == "format-namenodes" &&
83+
.file == "container.stderr.log"
84+
filteredAutomaticLogConfigNameNode1FormatNameNodeLog4j:
7085
type: filter
7186
inputs: [validEvents]
7287
condition: >-
7388
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
74-
.container == "format-namenodes"
75-
filteredAutomaticLogConfigNameNode0FormatZookeeper:
89+
.container == "format-namenodes" &&
90+
.file == "format-namenodes.log4j.xml"
91+
filteredAutomaticLogConfigNameNode1FormatNameNodeStdout:
92+
type: filter
93+
inputs: [validEvents]
94+
condition: >-
95+
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
96+
.container == "format-namenodes" &&
97+
.file == "container.stdout.log"
98+
filteredAutomaticLogConfigNameNode1FormatNameNodeStderr:
99+
type: filter
100+
inputs: [validEvents]
101+
condition: >-
102+
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
103+
.container == "format-namenodes" &&
104+
.file == "container.stderr.log"
105+
filteredAutomaticLogConfigNameNode0FormatZookeeperLog4j:
76106
type: filter
77107
inputs: [validEvents]
78108
condition: >-
79109
.pod == "test-hdfs-automatic-log-namenode-default-0" &&
80-
.container == "format-zookeeper"
110+
.container == "format-zookeeper" &&
111+
.file == "format-zookeeper.log4j.xml"
112+
filteredAutomaticLogConfigNameNode0FormatZookeeperStdout:
113+
type: filter
114+
inputs: [validEvents]
115+
condition: >-
116+
.pod == "test-hdfs-automatic-log-namenode-default-0" &&
117+
.container == "format-zookeeper" &&
118+
.file == "container.stdout.log"
119+
filteredAutomaticLogConfigNameNode0FormatZookeeperStderr:
120+
type: filter
121+
inputs: [validEvents]
122+
condition: >-
123+
.pod == "test-hdfs-automatic-log-namenode-default-0" &&
124+
.container == "format-zookeeper" &&
125+
.file == "container.stderr.log"
126+
filteredAutomaticLogConfigNameNode1FormatZookeeperStdout:
127+
type: filter
128+
inputs: [validEvents]
129+
condition: >-
130+
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
131+
.container == "format-zookeeper" &&
132+
.file == "container.stdout.log"
133+
filteredAutomaticLogConfigNameNode1FormatZookeeperStderr:
134+
type: filter
135+
inputs: [validEvents]
136+
condition: >-
137+
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
138+
.container == "format-zookeeper" &&
139+
.file == "container.stderr.log"
81140
filteredAutomaticLogConfigDataNode0:
82141
type: filter
83142
inputs: [validEvents]
@@ -90,12 +149,27 @@ customConfig:
90149
condition: >-
91150
.pod == "test-hdfs-automatic-log-datanode-default-0" &&
92151
.container == "vector"
93-
filteredAutomaticLogConfigDataNode0WaitForNameNodes:
152+
filteredAutomaticLogConfigDataNode0WaitForNameNodesLog4j:
94153
type: filter
95154
inputs: [validEvents]
96155
condition: >-
97156
.pod == "test-hdfs-automatic-log-datanode-default-0" &&
98-
.container == "wait-for-namenodes"
157+
.container == "wait-for-namenodes" &&
158+
.file == "wait-for-namenodes.log4j.xml"
159+
filteredAutomaticLogConfigDataNode0WaitForNameNodesStdout:
160+
type: filter
161+
inputs: [validEvents]
162+
condition: >-
163+
.pod == "test-hdfs-automatic-log-datanode-default-0" &&
164+
.container == "wait-for-namenodes" &&
165+
.file == "container.stdout.log"
166+
filteredAutomaticLogConfigDataNode0WaitForNameNodesStderr:
167+
type: filter
168+
inputs: [validEvents]
169+
condition: >-
170+
.pod == "test-hdfs-automatic-log-datanode-default-0" &&
171+
.container == "wait-for-namenodes" &&
172+
.file == "container.stderr.log"
99173
filteredAutomaticLogConfigJournalNode0:
100174
type: filter
101175
inputs: [validEvents]

0 commit comments

Comments
 (0)