Skip to content

Commit 46600a6

Browse files
maltesandermaltesander
committed
wip - test format-namenodes container logging improvements
1 parent 807779e commit 46600a6

2 files changed

Lines changed: 91 additions & 27 deletions

File tree

‎rust/operator-binary/src/container.rs‎

Lines changed: 57 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
//!
1212
use std::{collections::BTreeMap, str::FromStr};
1313

14-
use indoc::formatdoc;
14+
use indoc::{formatdoc, indoc};
1515
use snafu::{OptionExt, ResultExt, Snafu};
1616
use stackable_operator::{
1717
builder::{
@@ -41,11 +41,12 @@ use stackable_operator::{
4141
product_logging::{
4242
self,
4343
framework::{
44-
LoggingError, create_vector_shutdown_file_command, remove_vector_shutdown_file_command,
44+
LoggingError, capture_shell_output, create_vector_shutdown_file_command,
45+
remove_vector_shutdown_file_command,
4546
},
4647
spec::{
47-
ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice,
48-
CustomContainerLogConfig,
48+
AutomaticContainerLogConfig, ConfigMapLogConfig, ContainerLogConfig,
49+
ContainerLogConfigChoice, CustomContainerLogConfig,
4950
},
5051
},
5152
role_utils::RoleGroupRef,
@@ -627,22 +628,22 @@ impl ContainerConfig {
627628
&merged_config.hdfs_logging(),
628629
));
629630

630-
args.push_str(&format!(
631+
args.push_str(&formatdoc!(
631632
r#"\
632-
{COMMON_BASH_TRAP_FUNCTIONS}
633-
{remove_vector_shutdown_file_command}
634-
prepare_signal_handlers
635-
containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &
636-
if [[ -d {LISTENER_VOLUME_DIR} ]]; then
637-
export POD_ADDRESS=$(cat {LISTENER_VOLUME_DIR}/default-address/address)
638-
for i in {LISTENER_VOLUME_DIR}/default-address/ports/*; do
639-
export $(basename $i | tr a-z- A-Z_)_PORT="$(cat $i)"
640-
done
641-
fi
642-
{hadoop_home}/bin/hdfs {role} {upgrade_args} &
643-
wait_for_termination $!
644-
{create_vector_shutdown_file_command}
645-
"#,
633+
{COMMON_BASH_TRAP_FUNCTIONS}
634+
{remove_vector_shutdown_file_command}
635+
prepare_signal_handlers
636+
containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &
637+
if [[ -d {LISTENER_VOLUME_DIR} ]]; then
638+
export POD_ADDRESS=$(cat {LISTENER_VOLUME_DIR}/default-address/address)
639+
for i in {LISTENER_VOLUME_DIR}/default-address/ports/*; do
640+
export $(basename $i | tr a-z- A-Z_)_PORT="$(cat $i)"
641+
done
642+
fi
643+
{hadoop_home}/bin/hdfs {role} {upgrade_args} &
644+
wait_for_termination $!
645+
{create_vector_shutdown_file_command}
646+
"#,
646647
hadoop_home = Self::HADOOP_HOME,
647648
remove_vector_shutdown_file_command =
648649
remove_vector_shutdown_file_command(STACKABLE_LOG_DIR),
@@ -664,7 +665,9 @@ wait_for_termination $!
664665
hadoop_home = Self::HADOOP_HOME
665666
));
666667
}
667-
ContainerConfig::FormatNameNodes { .. } => {
668+
ContainerConfig::FormatNameNodes { container_name, .. } => {
669+
args.push_str(&add_capture_shell_output(container_name));
670+
668671
if let Some(container_config) = merged_config.as_namenode().map(|node| {
669672
node.logging
670673
.for_container(&NameNodeContainer::FormatNameNodes)
@@ -705,9 +708,17 @@ wait_for_termination $!
705708
if [ -z ${{ACTIVE_NAMENODE+x}} ]
706709
then
707710
echo "Create pod $POD_NAME as active namenode."
711+
# Restore original stdout/stderr from FD 3 and 4
712+
exec 1>&3 2>&4
713+
# Clean up (close FD 3 and 4)
714+
exec 3>&- 4>&-
708715
{hadoop_home}/bin/hdfs namenode -format -noninteractive
709716
else
710717
echo "Create pod $POD_NAME as standby namenode."
718+
# Restore original stdout/stderr from FD 3 and 4
719+
exec 1>&3 2>&4
720+
# Clean up (close FD 3 and 4)
721+
exec 3>&- 4>&-
711722
{hadoop_home}/bin/hdfs namenode -bootstrapStandby -nonInteractive
712723
fi
713724
else
@@ -724,7 +735,9 @@ wait_for_termination $!
724735
.join(" "),
725736
));
726737
}
727-
ContainerConfig::FormatZooKeeper { .. } => {
738+
ContainerConfig::FormatZooKeeper { container_name, .. } => {
739+
args.push_str(&add_capture_shell_output(container_name));
740+
728741
if let Some(container_config) = merged_config.as_namenode().map(|node| {
729742
node.logging
730743
.for_container(&NameNodeContainer::FormatZooKeeper)
@@ -739,6 +752,10 @@ wait_for_termination $!
739752
echo "Attempt to format ZooKeeper..."
740753
if [[ "0" -eq "$(echo $POD_NAME | sed -e 's/.*-//')" ]] ; then
741754
set +e
755+
# Restore original stdout/stderr from FD 3 and 4
756+
exec 1>&3 2>&4
757+
# Clean up (close FD 3 and 4)
758+
exec 3>&- 4>&-
742759
{hadoop_home}/bin/hdfs zkfc -formatZK -nonInteractive
743760
EXITCODE=$?
744761
set -e
@@ -755,10 +772,12 @@ wait_for_termination $!
755772
echo "ZooKeeper already formatted!"
756773
fi
757774
"###,
758-
hadoop_home = Self::HADOOP_HOME
775+
hadoop_home = Self::HADOOP_HOME,
759776
));
760777
}
761-
ContainerConfig::WaitForNameNodes { .. } => {
778+
ContainerConfig::WaitForNameNodes { container_name, .. } => {
779+
args.push_str(&add_capture_shell_output(container_name));
780+
762781
if let Some(container_config) = merged_config.as_datanode().map(|node| {
763782
node.logging
764783
.for_container(&DataNodeContainer::WaitForNameNodes)
@@ -1565,3 +1584,18 @@ impl TryFrom<&str> for ContainerVolumeDirs {
15651584
})
15661585
}
15671586
}
1587+
1588+
fn add_capture_shell_output(container_name: &str) -> String {
1589+
let capture_shell_output = product_logging::framework::capture_shell_output(
1590+
STACKABLE_LOG_DIR,
1591+
container_name,
1592+
// we do not access any of the crd config options for this and just log it to file
1593+
&AutomaticContainerLogConfig::default(),
1594+
);
1595+
formatdoc! {r###"
1596+
# Save original stdout/stderr to FD 3 and 4
1597+
exec 3>&1 4>&2
1598+
{capture_shell_output}
1599+
"###
1600+
}
1601+
}

‎tests/templates/kuttl/logging/hdfs-vector-aggregator-values.yaml.j2‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,18 +60,48 @@ customConfig:
6060
condition: >-
6161
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
6262
.container == "vector"
63-
filteredAutomaticLogConfigNameNode0FormatNameNode:
63+
filteredAutomaticLogConfigNameNode0FormatNameNodeLog4j:
6464
type: filter
6565
inputs: [validEvents]
6666
condition: >-
6767
.pod == "test-hdfs-automatic-log-namenode-default-0" &&
68-
.container == "format-namenodes"
69-
filteredAutomaticLogConfigNameNode1FormatNameNode:
68+
.container == "format-namenodes" &&
69+
.file == "format-namenodes.log4j.xml"
70+
filteredAutomaticLogConfigNameNode0FormatNameNodeStdout:
71+
type: filter
72+
inputs: [validEvents]
73+
condition: >-
74+
.pod == "test-hdfs-automatic-log-namenode-default-0" &&
75+
.container == "format-namenodes" &&
76+
.file == "container.stdout.log"
77+
filteredAutomaticLogConfigNameNode0FormatNameNodeStderr:
78+
type: filter
79+
inputs: [validEvents]
80+
condition: >-
81+
.pod == "test-hdfs-automatic-log-namenode-default-0" &&
82+
.container == "format-namenodes" &&
83+
.file == "container.stderr.log"
84+
filteredAutomaticLogConfigNameNode1FormatNameNodeLog4j:
7085
type: filter
7186
inputs: [validEvents]
7287
condition: >-
7388
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
74-
.container == "format-namenodes"
89+
.container == "format-namenodes" &&
90+
.file == "format-namenodes.log4j.xml"
91+
filteredAutomaticLogConfigNameNode1FormatNameNodeStdout:
92+
type: filter
93+
inputs: [validEvents]
94+
condition: >-
95+
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
96+
.container == "format-namenodes" &&
97+
.file == "container.stdout.log"
98+
filteredAutomaticLogConfigNameNode1FormatNameNodeStderr:
99+
type: filter
100+
inputs: [validEvents]
101+
condition: >-
102+
.pod == "test-hdfs-automatic-log-namenode-default-1" &&
103+
.container == "format-namenodes" &&
104+
.file == "container.stderr.log"
75105
filteredAutomaticLogConfigNameNode0FormatZookeeper:
76106
type: filter
77107
inputs: [validEvents]

0 commit comments

Comments
 (0)