Skip to content

Commit 8425d01

Browse files
siegfriedwebersiegfriedweber
authored
fix: Log file rollover (#107)
* fix: Log file rollover * Fix the scope of the constants in node_config; Fix a failing unit test * test(smoke): Fix assertion * test(metrics): Fix the test on OpenShift * test(metrics): Fix the test on OpenShift for long namespace names * test(metrics): Make the script compatible to Dash
1 parent da0084b commit 8425d01

8 files changed

Lines changed: 198 additions & 69 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ All notable changes to this project will be documented in this file.
3030

3131
- Bump testing-tools to `0.3.0-stackable0.0.0-dev` ([#91]).
3232

33+
### Fixed
34+
35+
- Log file rollover fixed ([#107]).
36+
3337
[#55]: https://github.com/stackabletech/opensearch-operator/pull/55
3438
[#63]: https://github.com/stackabletech/opensearch-operator/pull/63
3539
[#76]: https://github.com/stackabletech/opensearch-operator/pull/76
@@ -38,6 +42,7 @@ All notable changes to this project will be documented in this file.
3842
[#94]: https://github.com/stackabletech/opensearch-operator/pull/94
3943
[#97]: https://github.com/stackabletech/opensearch-operator/pull/97
4044
[#100]: https://github.com/stackabletech/opensearch-operator/pull/100
45+
[#107]: https://github.com/stackabletech/opensearch-operator/pull/107
4146

4247
## [25.11.0] - 2025-11-07
4348

‎rust/operator-binary/src/controller/build/node_config.rs‎

Lines changed: 36 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use crate::{
1313
crd::v1alpha1,
1414
framework::{
1515
builder::pod::container::{EnvVarName, EnvVarSet},
16+
product_logging::framework::STACKABLE_LOG_DIR,
1617
role_group_utils,
1718
types::{kubernetes::ServiceName, operator::RoleGroupName},
1819
},
@@ -23,96 +24,101 @@ pub const CONFIGURATION_FILE_OPENSEARCH_YML: &str = "opensearch.yml";
2324

2425
/// The cluster name.
2526
/// Type: string
26-
pub const CONFIG_OPTION_CLUSTER_NAME: &str = "cluster.name";
27+
const CONFIG_OPTION_CLUSTER_NAME: &str = "cluster.name";
2728

2829
/// The list of hosts that perform discovery when a node is started.
2930
/// Type: (comma-separated) list of strings
30-
pub const CONFIG_OPTION_DISCOVERY_SEED_HOSTS: &str = "discovery.seed_hosts";
31+
const CONFIG_OPTION_DISCOVERY_SEED_HOSTS: &str = "discovery.seed_hosts";
3132

3233
/// By default, OpenSearch forms a multi-node cluster. Set `discovery.type` to `single-node` to
3334
/// form a single-node cluster.
3435
/// Type: string
35-
pub const CONFIG_OPTION_DISCOVERY_TYPE: &str = "discovery.type";
36+
const CONFIG_OPTION_DISCOVERY_TYPE: &str = "discovery.type";
3637

3738
/// Specifies an address or addresses that an OpenSearch node publishes to other nodes for HTTP
3839
/// communication.
3940
/// Type: (comma-separated) list of strings
40-
pub const CONFIG_OPTION_HTTP_PUBLISH_HOST: &str = "http.publish_host";
41+
const CONFIG_OPTION_HTTP_PUBLISH_HOST: &str = "http.publish_host";
4142

4243
/// A list of cluster-manager-eligible nodes used to bootstrap the cluster.
4344
/// Type: (comma-separated) list of strings
44-
pub const CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES: &str =
45-
"cluster.initial_cluster_manager_nodes";
45+
const CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES: &str = "cluster.initial_cluster_manager_nodes";
4646

4747
/// Binds an OpenSearch node to an address.
4848
/// Type: string
49-
pub const CONFIG_OPTION_NETWORK_HOST: &str = "network.host";
49+
const CONFIG_OPTION_NETWORK_HOST: &str = "network.host";
5050

5151
/// Specifies an address or addresses that an OpenSearch node publishes to other nodes in the
5252
/// cluster so that they can connect to it.
5353
/// Type: (comma-separated) list of strings
54-
pub const CONFIG_OPTION_NETWORK_PUBLISH_HOST: &str = "network.publish_host";
54+
const CONFIG_OPTION_NETWORK_PUBLISH_HOST: &str = "network.publish_host";
5555

5656
/// The custom node attribute "role-group"
5757
/// Type: string
58-
pub const CONFIG_OPTION_NODE_ATTR_ROLE_GROUP: &str = "node.attr.role-group";
58+
const CONFIG_OPTION_NODE_ATTR_ROLE_GROUP: &str = "node.attr.role-group";
5959

6060
/// A descriptive name for the node.
6161
/// Type: string
62-
pub const CONFIG_OPTION_NODE_NAME: &str = "node.name";
62+
const CONFIG_OPTION_NODE_NAME: &str = "node.name";
6363

6464
/// Defines one or more roles for an OpenSearch node.
6565
/// Type: (comma-separated) list of strings
66-
pub const CONFIG_OPTION_NODE_ROLES: &str = "node.roles";
66+
const CONFIG_OPTION_NODE_ROLES: &str = "node.roles";
67+
68+
/// Defines the path for the logs
69+
/// OpenSearch grants the required access rights, see
70+
/// https://github.com/opensearch-project/OpenSearch/blob/3.4.0/server/src/main/java/org/opensearch/bootstrap/Security.java#L369
71+
/// The permissions "write" and "delete" are required for the log file rollover.
72+
/// Type: string
73+
const CONFIG_OPTION_PATH_LOGS: &str = "path.logs";
6774

6875
/// Specifies a list of distinguished names (DNs) that denote the other nodes in the cluster.
6976
/// Type: (comma-separated) list of strings
70-
pub const CONFIG_OPTION_PLUGINS_SECURITY_NODES_DN: &str = "plugins.security.nodes_dn";
77+
const CONFIG_OPTION_PLUGINS_SECURITY_NODES_DN: &str = "plugins.security.nodes_dn";
7178

7279
/// Whether to enable TLS on the REST layer. If enabled, only HTTPS is allowed.
7380
/// Type: boolean
74-
pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_ENABLED: &str =
75-
"plugins.security.ssl.http.enabled";
81+
const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_ENABLED: &str = "plugins.security.ssl.http.enabled";
7682

7783
/// Path to the cert PEM file used for TLS on the HTTP PORT.
7884
/// type: string
79-
pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMCERT_FILEPATH: &str =
85+
const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMCERT_FILEPATH: &str =
8086
"plugins.security.ssl.http.pemcert_filepath";
8187

8288
/// Path to the key PEM file used for TLS on the HTTP PORT.
8389
/// type: string
84-
pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMKEY_FILEPATH: &str =
90+
const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMKEY_FILEPATH: &str =
8591
"plugins.security.ssl.http.pemkey_filepath";
8692

8793
/// Path to the trusted CAs PEM file used for TLS on the HTTP PORT.
8894
/// type: string
89-
pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMTRUSTEDCAS_FILEPATH: &str =
95+
const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMTRUSTEDCAS_FILEPATH: &str =
9096
"plugins.security.ssl.http.pemtrustedcas_filepath";
9197

9298
/// Whether to enable TLS on internal node-to-node communication using the transport port.
9399
/// type: boolean
94-
pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_ENABLED: &str =
100+
const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_ENABLED: &str =
95101
"plugins.security.ssl.transport.enabled";
96102

97103
/// Path to the cert PEM file used for TLS on the transport PORT.
98104
/// type: string
99-
pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMCERT_FILEPATH: &str =
105+
const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMCERT_FILEPATH: &str =
100106
"plugins.security.ssl.transport.pemcert_filepath";
101107

102108
/// Path to the key PEM file used for TLS on the transport PORT.
103109
/// type: string
104-
pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMKEY_FILEPATH: &str =
110+
const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMKEY_FILEPATH: &str =
105111
"plugins.security.ssl.transport.pemkey_filepath";
106112

107113
/// Path to the trusted CAs PEM file used for TLS on the transport PORT.
108114
/// type: string
109-
pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMTRUSTEDCAS_FILEPATH: &str =
115+
const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMTRUSTEDCAS_FILEPATH: &str =
110116
"plugins.security.ssl.transport.pemtrustedcas_filepath";
111117

112118
/// Specifies an address or addresses that an OpenSearch node publishes to other nodes for
113119
/// transport communication.
114120
/// Type: (comma-separated) list of strings
115-
pub const CONFIG_OPTION_TRANSPORT_PUBLISH_HOST: &str = "transport.publish_host";
121+
const CONFIG_OPTION_TRANSPORT_PUBLISH_HOST: &str = "transport.publish_host";
116122

117123
const DEFAULT_OPENSEARCH_HOME: &str = "/stackable/opensearch";
118124

@@ -203,6 +209,13 @@ impl NodeConfig {
203209
CONFIG_OPTION_NODE_ATTR_ROLE_GROUP.to_owned(),
204210
json!(self.role_group_name),
205211
);
212+
config.insert(
213+
CONFIG_OPTION_PATH_LOGS.to_owned(),
214+
json!(format!(
215+
"{STACKABLE_LOG_DIR}/{container}",
216+
container = v1alpha1::Container::OpenSearch.to_container_name()
217+
)),
218+
);
206219

207220
config
208221
}
@@ -616,6 +629,7 @@ mod tests {
616629
"discovery.type: \"zen\"\n",
617630
"network.host: \"0.0.0.0\"\n",
618631
"node.attr.role-group: \"data\"\n",
632+
"path.logs: \"/stackable/log/opensearch\"\n",
619633
"plugins.security.nodes_dn: [\"CN=generated certificate for pod\"]\n",
620634
"plugins.security.ssl.http.enabled: true\n",
621635
"plugins.security.ssl.http.pemcert_filepath: \"/stackable/opensearch/config/tls/server/tls.crt\"\n",

‎tests/templates/kuttl/logging/20-install-opensearch.yaml.j2‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,34 @@ data:
1313
appender.FILE.layout.type = OpenSearchJsonLayout
1414
appender.FILE.layout.type_name = server
1515
---
16+
# Expose the API of the Vector agent
17+
apiVersion: v1
18+
kind: ConfigMap
19+
metadata:
20+
name: vector-api-config
21+
data:
22+
vector-api-config.yaml: |
23+
api:
24+
address: 0.0.0.0:8686
25+
enabled: true
26+
---
27+
# Expose the API of the Vector agent of the pod opensearch-nodes-automatic-0
28+
apiVersion: v1
29+
kind: Service
30+
metadata:
31+
name: opensearch-nodes-automatic-vector
32+
spec:
33+
ports:
34+
- name: vector
35+
port: 8686
36+
protocol: TCP
37+
selector:
38+
app.kubernetes.io/component: nodes
39+
app.kubernetes.io/instance: opensearch
40+
app.kubernetes.io/name: opensearch
41+
app.kubernetes.io/role-group: automatic
42+
type: ClusterIP
43+
---
1644
apiVersion: opensearch.stackable.tech/v1alpha1
1745
kind: OpenSearchCluster
1846
metadata:
@@ -37,10 +65,10 @@ spec:
3765
console:
3866
level: INFO
3967
file:
40-
level: INFO
68+
level: TRACE
4169
loggers:
4270
ROOT:
43-
level: INFO
71+
level: TRACE
4472
vector:
4573
console:
4674
level: INFO
@@ -78,10 +106,26 @@ spec:
78106
- name: security-config
79107
mountPath: /stackable/opensearch/config/opensearch-security
80108
readOnly: true
109+
- name: vector
110+
env:
111+
- name: VECTOR_CONFIG_YAML
112+
value: /stackable/config/vector.yaml,/stackable/config/vector-api-config.yaml
113+
ports:
114+
- name: vector
115+
containerPort: 8686
116+
protocol: TCP
117+
volumeMounts:
118+
- name: vector-api-config
119+
mountPath: /stackable/config/vector-api-config.yaml
120+
readOnly: true
121+
subPath: vector-api-config.yaml
81122
volumes:
82123
- name: security-config
83124
secret:
84125
secretName: opensearch-security-config
126+
- name: vector-api-config
127+
configMap:
128+
name: vector-api-config
85129
---
86130
apiVersion: v1
87131
kind: Secret

‎tests/templates/kuttl/logging/30-test-opensearch.yaml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ spec:
3737
securityContext:
3838
fsGroup: 1000
3939
restartPolicy: OnFailure
40+
backoffLimit: 10
4041
---
4142
apiVersion: v1
4243
kind: ConfigMap
@@ -88,6 +89,50 @@ data:
8889
f'No events were sent in "{componentId}".'
8990
9091
92+
def check_log_file_rollover():
93+
response = requests.post(
94+
'http://opensearch-nodes-automatic-vector:8686/graphql',
95+
json={
96+
'query': """
97+
{
98+
sources {
99+
nodes {
100+
componentId
101+
metrics {
102+
receivedBytesTotal {
103+
receivedBytesTotal
104+
}
105+
}
106+
}
107+
}
108+
}
109+
"""
110+
}
111+
)
112+
113+
assert response.status_code == 200, \
114+
'Cannot access the API of the vector agent.'
115+
116+
result = response.json()
117+
118+
sources = result['data']['sources']['nodes']
119+
for source in sources:
120+
receivedBytes = source['metrics']['receivedBytesTotal']
121+
componentId = source['componentId']
122+
123+
if componentId == 'files_opensearch_server':
124+
assert receivedBytes is not None
125+
receivedBytes = receivedBytes['receivedBytesTotal']
126+
MAX_LOG_FILE_SIZE = 5_500_000
127+
expectedBytes = 2 * MAX_LOG_FILE_SIZE
128+
assert receivedBytes >= expectedBytes, \
129+
'Log file rollover did not yet happen twice ' \
130+
f'({receivedBytes:,.0f} Bytes of {expectedBytes:,d} Bytes received). ' \
131+
'The first rollover requires write permission to rename the log file, ' \
132+
'the second rollover additionally requires delete permission to remove the old log file.'
133+
134+
91135
if __name__ == '__main__':
92136
check_sent_events()
137+
check_log_file_rollover()
93138
print('Test successful!')

‎tests/templates/kuttl/metrics/01-rbac.yaml‎

Lines changed: 17 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,18 @@
11
---
2-
apiVersion: v1
3-
kind: ServiceAccount
4-
metadata:
5-
name: test-service-account
6-
---
7-
kind: Role
8-
apiVersion: rbac.authorization.k8s.io/v1
9-
metadata:
10-
name: test-role
11-
rules:
12-
- apiGroups:
13-
- security.openshift.io
14-
resources:
15-
- securitycontextconstraints
16-
resourceNames:
17-
- privileged
18-
verbs:
19-
- use
20-
---
21-
kind: RoleBinding
22-
apiVersion: rbac.authorization.k8s.io/v1
23-
metadata:
24-
name: test-role-binding
25-
subjects:
26-
- kind: ServiceAccount
27-
name: test-service-account
28-
- kind: ServiceAccount
29-
name: prometheus-stack-kube-prom-admission
30-
- kind: ServiceAccount
31-
name: prometheus-stack-kube-prom-operator
32-
- kind: ServiceAccount
33-
name: prometheus-stack-kube-prom-prometheus
34-
roleRef:
35-
apiGroup: rbac.authorization.k8s.io
36-
kind: Role
37-
name: test-role
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestStep
4+
commands:
5+
- script: |
6+
PROMETHEUS_HELM_RELEASE_NAME=prometheus-$NAMESPACE
7+
PROMETHEUS_HELM_CHART_NAME=kube-prometheus-stack
8+
9+
# `kube-prometheus-stack.fullname` in the Prometheus Helm Chart is used to create the
10+
# ServiceAccount names. It is set to `<.Release.Name>-<.Chart.Name>` and truncated to
11+
# 26 characters without dashes at the end, see
12+
# https://github.com/prometheus-community/helm-charts/blob/kube-prometheus-stack-81.2.2/charts/kube-prometheus-stack/templates/_helpers.tpl#L22
13+
KUBE_PROMETHEUS_STACK_FULLNAME=$PROMETHEUS_HELM_RELEASE_NAME-$PROMETHEUS_HELM_CHART_NAME
14+
KUBE_PROMETHEUS_STACK_FULLNAME=$(echo -n $KUBE_PROMETHEUS_STACK_FULLNAME | head --bytes=26)
15+
export KUBE_PROMETHEUS_STACK_FULLNAME=$(echo -n $KUBE_PROMETHEUS_STACK_FULLNAME | sed 's/-*$//')
16+
17+
envsubst '$KUBE_PROMETHEUS_STACK_FULLNAME' < 01_rbac.yaml | \
18+
kubectl apply --namespace=$NAMESPACE --filename=-

‎tests/templates/kuttl/metrics/01_rbac.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
---
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
name: test-service-account
6+
---
7+
kind: Role
8+
apiVersion: rbac.authorization.k8s.io/v1
9+
metadata:
10+
name: test-role
11+
rules:
12+
- apiGroups:
13+
- security.openshift.io
14+
resources:
15+
- securitycontextconstraints
16+
resourceNames:
17+
- privileged
18+
verbs:
19+
- use
20+
---
21+
kind: RoleBinding
22+
apiVersion: rbac.authorization.k8s.io/v1
23+
metadata:
24+
name: test-role-binding
25+
subjects:
26+
- kind: ServiceAccount
27+
name: test-service-account
28+
- kind: ServiceAccount
29+
name: $KUBE_PROMETHEUS_STACK_FULLNAME-admission
30+
- kind: ServiceAccount
31+
name: $KUBE_PROMETHEUS_STACK_FULLNAME-operator
32+
- kind: ServiceAccount
33+
name: $KUBE_PROMETHEUS_STACK_FULLNAME-prometheus
34+
roleRef:
35+
apiGroup: rbac.authorization.k8s.io
36+
kind: Role
37+
name: test-role

0 commit comments

Comments
 (0)