Skip to content

Commit d0e3c51

Browse files
pmatsinopoulospmatsinopoulos
committed
Cloudwatch logs configuration update for base and celo
1 parent 848e32f commit d0e3c51

11 files changed

Lines changed: 201 additions & 35 deletions

‎docs/TODO.md‎

Lines changed: 4 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
- [ ] Mainnets
1010
- [ ] `mainnet`
1111
- [ ] `base`
12-
- [ ] `celo-mainnet`
12+
- [ ] `celo`
1313
- [ ] Latest `reth` version. Upgrade `reth` to latest version.
1414
- [ ] Testnets
1515
- [ ] `sepolia`
@@ -21,31 +21,14 @@
2121
- [ ] Testnets
2222
- [ ] `celo-sepolia`
2323
- [ ] `op-geth`
24-
- [ ] `nginx`
25-
- [ ] `access`
26-
- [ ] `error`
27-
- [ ] `ssm-agent`
28-
- [ ] success
29-
- [ ] `error`
3024
- [ ] Mainnets
31-
- [ ] `celo-mainnet`
25+
- [ ] `celo`
3226
- [ ] `op-geth`
33-
- [ ] `nginx`
34-
- [ ] `access`
35-
- [ ] `error`
36-
- [ ] `ssm-agent`
37-
- [ ] success
38-
- [ ] `error`
39-
- [ ] CloudWatch Agent
40-
- [ ] Testnets
41-
- [ ] `celo-sepolia`
42-
- [ ] Mainnets
43-
- [ ] `celo-mainnet`
4427
- [ ] Metrics Exporter
4528
- [ ] Testnets
4629
- [ ] `celo-sepolia`
4730
- [ ] Mainnets
48-
- [ ] `celo-mainnet`
31+
- [ ] `celo`
4932
- [ ] Alarms
5033
- [ ] Testnets
5134
- [ ] `celo-sepolia`
@@ -57,7 +40,7 @@
5740
- [ ] `sepolia`
5841
- [ ] `staging_ethereum` alarms to be renamed to `staging_sepolia`
5942
- [ ] Mainnets
60-
- [ ] `celo-mainnet`
43+
- [ ] `celo`
6144
- [ ] CPU
6245
- [ ] Memory
6346
- [ ] Disk Storage Low

‎terraform/ethereum_reth_cloudwatch_logs.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
resource "aws_cloudwatch_log_group" "ethereum_reth" {
2-
name = "${var.project}-${var.environment}-ethereum-reth"
2+
name = "${var.project}-${var.environment}-${var.ethereum_node.chain}-reth"
33
retention_in_days = var.ethereum_node.logs.retention_days
44

55
tags = {
6-
"Name" = "${var.project}-${var.environment}-ethereum-reth"
6+
"Name" = "${var.project}-${var.environment}-${var.ethereum_node.chain}-reth"
77
}
88
}

‎terraform/locals.tf‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -84,21 +84,22 @@ locals {
8484
celo_common_environment_variables_file = "common-environment-variables.env"
8585
celo_common_environment_variables = templatefile("${local.templates_folder}/celo.${local.celo_common_environment_variables_file}.tftpl", {
8686
chain = var.celo_node.chain
87-
op_node_rpc_endpoint = "http://${aws_route53_record.ethereum_node.name}:8545"
88-
op_node_l1_beacon = "http://${aws_route53_record.ethereum_node.name}:5052"
89-
health_check_reference_rpc_provider = var.celo_node.health_check_reference_rpc_provider
90-
ipc_path = ""
91-
node_public_ip = aws_eip.celo.public_ip
92-
eigenda_proxy_endpoint = ""
93-
eigenda_local_svc_manager_addr = var.celo_node.eigenda_local_svc_manager_addr
9487
eigenda_local_disperser_rpc = var.celo_node.eigenda_local_disperser_rpc
88+
eigenda_local_s3_bucket = var.celo_node.eigenda_local_s3_bucket
9589
eigenda_local_signer_private_key_hex = var.celo_node.eigenda_local_signer_private_key_hex
96-
eigenda_v2_local_svc_manager_addr = var.celo_node.eigenda_v2_local_svc_manager_addr
90+
eigenda_local_svc_manager_addr = var.celo_node.eigenda_local_svc_manager_addr
91+
eigenda_proxy_endpoint = ""
92+
eigenda_v2_local_bls_operator_state_retriever_addr = var.celo_node.eigenda_v2_local_bls_operator_state_retriever_addr
93+
eigenda_v2_local_cert_verifier_addr = var.celo_node.eigenda_v2_local_cert_verifier_addr
9794
eigenda_v2_local_disperser_rpc = var.celo_node.eigenda_v2_local_disperser_rpc
9895
eigenda_v2_local_signer_payment_key_hex = var.celo_node.eigenda_v2_local_signer_payment_key_hex
99-
eigenda_v2_local_cert_verifier_addr = var.celo_node.eigenda_v2_local_cert_verifier_addr
100-
eigenda_v2_local_bls_operator_state_retriever_addr = var.celo_node.eigenda_v2_local_bls_operator_state_retriever_addr
101-
eigenda_local_s3_bucket = var.celo_node.eigenda_local_s3_bucket
96+
eigenda_v2_local_svc_manager_addr = var.celo_node.eigenda_v2_local_svc_manager_addr
97+
health_check_reference_rpc_provider = var.celo_node.health_check_reference_rpc_provider
98+
ipc_path = ""
99+
node_public_ip = aws_eip.celo.public_ip
100+
op_geth_historical_rpc = var.celo_node.op_geth_historical_rpc
101+
op_node_l1_beacon = "http://${aws_route53_record.ethereum_node.name}:5052"
102+
op_node_rpc_endpoint = "http://${aws_route53_record.ethereum_node.name}:8545"
102103
})
103104

104105
celo_cloudwatch_agent_configuration_file = "celo_ec2.amazon-cloudwatch-agent.json"

‎terraform/modules/l2_op_stack_chains/ec2_cloudwatch_agent_configuration_update.tf‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
resource "local_file" "ec2_install_cloudwatch_agent" {
2+
content = local.cloudwatch_agent_configuration_content
3+
filename = "${path.module}/generated-${var.node_name}-${local.cloudwatch_agent_configuration_file}"
4+
}
5+
6+
resource "terraform_data" "ec2_cloudwatch_agent_configuration_update" {
7+
depends_on = [
8+
aws_instance.ec2_instance,
9+
aws_eip.ec2_instance,
10+
module.ec2_install_cloudwatch_agent,
11+
local_file.ec2_install_cloudwatch_agent
12+
]
13+
14+
triggers_replace = {
15+
ec2_cloudwatch_agent_configuration = sha1(local_file.ec2_install_cloudwatch_agent.content)
16+
}
17+
18+
connection {
19+
type = "ssh"
20+
user = var.node.user
21+
port = var.node.ssh_port
22+
host = aws_eip.ec2_instance.public_ip
23+
private_key = file(var.node.key_pair_pem_file_path)
24+
}
25+
26+
provisioner "file" {
27+
source = local_file.ec2_install_cloudwatch_agent.filename
28+
destination = "/home/${var.node.user}/${local.cloudwatch_agent_configuration_file}"
29+
}
30+
31+
provisioner "remote-exec" {
32+
inline = [
33+
"sudo mv /home/${var.node.user}/${local.cloudwatch_agent_configuration_file} /opt/aws/amazon-cloudwatch-agent/etc/${local.cloudwatch_agent_configuration_file}",
34+
"sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/${local.cloudwatch_agent_configuration_file} -s",
35+
"sudo systemctl restart amazon-cloudwatch-agent"
36+
]
37+
}
38+
}

‎terraform/modules/l2_op_stack_chains/locals.tf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,15 @@ locals {
3535

3636
# Ref: https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html#attach-bucket-policy
3737
elb_account_id_for_europe_london = "652711504416"
38+
39+
cloudwatch_agent_configuration_file = "ec2.amazon-cloudwatch-agent.json"
40+
cloudwatch_agent_configuration_content = templatefile("${local.templates_folder}/${local.cloudwatch_agent_configuration_file}.tftpl", {
41+
chain = var.node.chain
42+
namespace = var.cloudwatch_namespace
43+
nginx_log_group_name_access = module.nginx_configuration.cloudwatch_log_group_nginx_access_name
44+
nginx_log_group_name_error = module.nginx_configuration.cloudwatch_log_group_nginx_error_name
45+
reth_log_group_name = aws_cloudwatch_log_group.reth.name
46+
ssm_agent_log_group_name = module.ssm_agent_cloudwatch.cloudwatch_log_group_ssm_agent_name
47+
ssm_agent_log_group_name_error = module.ssm_agent_cloudwatch.cloudwatch_log_group_ssm_agent_error_name
48+
})
3849
}

‎terraform/modules/l2_op_stack_chains/reth_cloudwatch_logs.tf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
resource "aws_cloudwatch_log_group" "reth" {
2+
name = "${var.project}-${var.environment}-${var.node_name}-reth"
3+
retention_in_days = var.node.logs.retention_days
4+
5+
tags = {
6+
"Name" = "${var.project}-${var.environment}-${var.node_name}-reth"
7+
}
8+
}

‎terraform/modules/l2_op_stack_chains/templates/ec2.amazon-cloudwatch-agent.json.tftpl‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
{
2+
"agent": {
3+
"metrics_collection_interval": 60,
4+
"run_as_user": "cwagent",
5+
"debug": true
6+
},
7+
"metrics": {
8+
"namespace": "${namespace}",
9+
"append_dimensions": {
10+
"InstanceId": "$${aws:InstanceId}",
11+
"InstanceType": "$${aws:InstanceType}"
12+
},
13+
"metrics_collected": {
14+
"cpu": {
15+
"measurement": [
16+
"cpu_usage_idle",
17+
"cpu_usage_iowait",
18+
"cpu_usage_user",
19+
"cpu_usage_system",
20+
"cpu_usage_steal",
21+
"cpu_usage_nice",
22+
"cpu_usage_softirq",
23+
"cpu_usage_irq"
24+
],
25+
"totalcpu": true
26+
},
27+
"disk": {
28+
"measurement": [
29+
"used_percent"
30+
],
31+
"resources": [
32+
"/",
33+
"/data"
34+
],
35+
"dimensions": [
36+
[
37+
"path"
38+
]
39+
]
40+
},
41+
"mem": {
42+
"measurement": [
43+
"mem_used_percent"
44+
]
45+
},
46+
"procstat": [
47+
{
48+
"pattern": "docker",
49+
"measurement": [
50+
"cpu_usage",
51+
"memory_rss"
52+
]
53+
},
54+
{
55+
"pattern": "op-reth",
56+
"measurement": [
57+
"cpu_usage",
58+
"memory_rss"
59+
]
60+
},
61+
{
62+
"pattern": "op-node",
63+
"measurement": [
64+
"cpu_usage",
65+
"memory_rss"
66+
]
67+
},
68+
{
69+
"pattern": "ethereum-metrics-exporter",
70+
"measurement": [
71+
"cpu_usage",
72+
"memory_rss"
73+
]
74+
},
75+
{
76+
"pattern": "prometheus",
77+
"measurement": [
78+
"cpu_usage",
79+
"memory_rss"
80+
]
81+
},
82+
{
83+
"pattern": "grafana",
84+
"measurement": [
85+
"cpu_usage",
86+
"memory_rss"
87+
]
88+
}
89+
]
90+
}
91+
},
92+
"logs": {
93+
"logs_collected": {
94+
"files": {
95+
"collect_list": [
96+
{
97+
"file_path": "/var/log/nginx/access.log",
98+
"log_group_name": "${nginx_log_group_name_access}",
99+
"timestamp_format": "[%d/%b/%Y:%H:%M:%S %z]"
100+
},
101+
{
102+
"file_path": "/var/log/nginx/error.log",
103+
"log_group_name": "${nginx_log_group_name_error}",
104+
"timestamp_format": "%Y/%m/%d %H:%M:%S"
105+
},
106+
{
107+
"file_path": "/data/reth/logs/${chain}/reth.log",
108+
"log_group_name": "${reth_log_group_name}"
109+
},
110+
{
111+
"file_path": "/var/log/amazon/ssm/amazon-ssm-agent.log",
112+
"log_group_name": "${ssm_agent_log_group_name}"
113+
},
114+
{
115+
"file_path": "/var/log/amazon/ssm/errors.log",
116+
"log_group_name": "${ssm_agent_log_group_name_error}"
117+
}
118+
]
119+
}
120+
}
121+
}
122+
}

‎terraform/production/terraform.tfvars‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ celo_node = {
108108
eigenda_local_s3_bucket = "eigenda-proxy-cache-celo-mainnet"
109109
geth_bootnodes = "enode://28f4fcb7f38c1b012087f7aef25dcb0a1257ccf1cdc4caa88584dc25416129069b514908c8cead5d0105cb0041dd65cd4ee185ae0d379a586fb07b1447e9de38@34.169.39.223:30303,enode://a9077c3e030206954c5c7f22cc16a32cb5013112aa8985e3575fadda7884a508384e1e63c077b7d9fcb4a15c716465d8585567f047c564ada2e823145591e444@34.169.212.31:30303,enode://029b007a7a56acbaa8ea50ec62cda279484bf3843fae1646f690566f784aca50e7d732a9a0530f0541e5ed82ba9bf2a4e21b9021559c5b8b527b91c9c7a38579@34.82.139.199:30303,enode://f3c96b73a5772c5efb48d5a33bf193e58080d826ba7f03e9d5bdef20c0634a4f83475add92ab6313b7a24aa4f729689efb36f5093e5d527bb25e823f8a377224@34.82.84.247:30303,enode://daa5ad65d16bcb0967cf478d9f20544bf1b6de617634e452dff7b947279f41f408b548261d62483f2034d237f61cbcf92a83fc992dbae884156f28ce68533205@34.168.45.168:30303,enode://c79d596d77268387e599695d23e941c14c220745052ea6642a71ef7df31a13874cb7f2ce2ecf5a8a458cfc9b5d9219ce3e8bc6e5c279656177579605a5533c4f@35.247.32.229:30303,enode://4151336075dd08eb6c75bfd63855e8a4bd6fd0f91ae4a81b14930f2671e16aee55495c139380c16e1094a49691875e69e40a3a5e2b4960c7859e7eb5745f9387@35.205.149.224:30303,enode://ab999db751265c714b171344de1972ed74348162de465a0444f56e50b8cfd048725b213ba1fe48c15e3dfb0638e685ea9a21b8447a54eb2962c6768f43018e5c@34.79.3.199:30303,enode://9d86d92fb38a429330546fe1aefce264e1f55c5d40249b63153e7df744005fa3c1e2da295e307041fd30ab1c618715f362c932c28715bc20bed7ae4fc76dea81@34.77.144.164:30303,enode://c82c31f21dd5bbb8dc35686ff67a4353382b4017c9ec7660a383ccb5b8e3b04c6d7aefe71203e550382f6f892795728570f8190afd885efcb7b78fa398608699@34.76.202.74:30303,enode://3bad5f57ad8de6541f02e36d806b87e7e9ca6d533c956e89a56b3054ae85d608784f2cd948dc685f7d6bbd5a2f6dd1a23cc03e529ea370dd72d880864a2af6a3@104.199.93.87:30303,enode://1decf3b8b9a0d0b8332d15218f3bf0ceb9606b0efe18f352c51effc14bbf1f4f3f46711e1d460230cb361302ceaad2be48b5b187ad946e50d729b34e463268d2@35.240.26.148:30303"
110110
health_check_reference_rpc_provider = "https://forno.celo.org"
111+
op_geth_historical_rpc = "https://forno.celo.org"
111112
instance_type = "c7i.4xlarge"
112113
logs = {
113114
retention_days = 30

‎terraform/staging/terraform.tfvars‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ celo_node = {
107107
eigenda_local_s3_bucket = "eigenda-proxy-cache-celo-sepolia"
108108
geth_bootnodes = "enode://7fd35dfea27042fe008c74ea97c7a41254b293152730419a6e9bcd84bb03c7ced418c1043e2ef6ad63d2facca6fbdacfbf7c4bfcf33ee7e9a0e6b7eb0617595d@34.169.104.197:30303,enode://151bcf170585971fc78129d9c16af355a1a53e1c825ce1ac20700ea754aa33eda60ca83de6f954bfed8d36c53f33295d93dbc3da9d549d6547d09467806b4b3d@104.199.124.11:30303,enode://aa5fb766438ac5a0354eb2eec1c0c002b56bb2ce7ed44f0e76e019cbb931222faa9ecfb0fa0055c0c62a2fcf04492d4129349a1045dfef140585250281885e4b@34.83.115.97:30303,enode://27c81ca466c99016d1595429afc68d66afb3ed9d5a2dd7f6a7797db23a4c826546a177b69b4932f3a75ce374b09d8ccc5b52dad615b3c47dbb8f6217d79ded22@35.247.1.226:30303"
109109
health_check_reference_rpc_provider = "https://forno.celo-sepolia.celo-testnet.org"
110+
op_geth_historical_rpc = "https://forno.celo-sepolia.celo-testnet.org"
110111
instance_type = "c7i.4xlarge"
111112
logs = {
112113
retention_days = 30

‎terraform/templates/celo.common-environment-variables.env.tftpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ HISTORICAL_RPC_DATADIR_PATH=
2020
# Optional provider to serve RPC requests requiring historical state, if set op-geth will proxy
2121
# requests requiring state prior to the L2 start to here. If set this overrides the use of a local Celo L1
2222
# node via HISTORICAL_RPC_DATADIR_PATH.
23-
OP_GETH__HISTORICAL_RPC=
23+
OP_GETH__HISTORICAL_RPC=${op_geth_historical_rpc}
2424

2525
# +snap+ or +full+.
2626
# By default, +snap+ sync.

0 commit comments

Comments
 (0)