diff --git a/.github/tests/mix-public.yaml b/.github/tests/mix-public.yaml index aae8edfa1..e6bea0a09 100644 --- a/.github/tests/mix-public.yaml +++ b/.github/tests/mix-public.yaml @@ -50,6 +50,7 @@ additional_services: - apache - tracoor - spamoor + - txpool_viz ethereum_metrics_exporter_enabled: true snooper_enabled: true keymanager_enabled: true diff --git a/.github/tests/mix-with-tools-mev.yaml b/.github/tests/mix-with-tools-mev.yaml index 65bfc0774..102c0b858 100644 --- a/.github/tests/mix-with-tools-mev.yaml +++ b/.github/tests/mix-with-tools-mev.yaml @@ -25,6 +25,7 @@ additional_services: - apache - tracoor - spamoor + - txpool_viz ethereum_metrics_exporter_enabled: true snooper_enabled: true mev_type: flashbots diff --git a/.github/tests/mix-with-tools-minimal.yaml b/.github/tests/mix-with-tools-minimal.yaml index b5a7811bd..8d5322e51 100644 --- a/.github/tests/mix-with-tools-minimal.yaml +++ b/.github/tests/mix-with-tools-minimal.yaml @@ -27,6 +27,7 @@ additional_services: - apache - tracoor - spamoor + - txpool_viz ethereum_metrics_exporter_enabled: true snooper_enabled: true keymanager_enabled: true diff --git a/.github/tests/mix-with-tools.yaml b/.github/tests/mix-with-tools.yaml index bdc1a8514..17f329a7a 100644 --- a/.github/tests/mix-with-tools.yaml +++ b/.github/tests/mix-with-tools.yaml @@ -26,6 +26,7 @@ additional_services: - apache - tracoor - spamoor + - txpool_viz ethereum_metrics_exporter_enabled: true snooper_enabled: true keymanager_enabled: true diff --git a/README.md b/README.md index 9a27a8798..22b302c2f 100644 --- a/README.md +++ b/README.md @@ -758,6 +758,7 @@ additional_services: - forky - apache - tracoor + - txpool_viz # Configuration place for blockscout explorer - https://github.com/blockscout/blockscout blockscout_params: @@ -1058,6 +1059,20 @@ spamoor_params: # A list of optional params that will be passed to the spamoor command for modifying its behaviour extra_args: [] +# Configuration for txpool_viz. A mempool visualizer. +txpool_viz_params: + # The image to use for txpool_viz + image: punkhazardlabs/txpool-viz:latest + # Polling configuration for txpool_viz + polling: + interval: 0.1s # How often to poll for new mempool data (in seconds) + timeout: 5s # Timeout for polling requests (in seconds) + # Filters to apply to the mempool data + filters: + min_gas_price: 1gwei # Minimum gas price filter for transactions + focil_enabled: "true" # Enable or disable FOCIL (set to "true" to enable) + log_level: "info" # Logging level for txpool_viz (e.g., "info", "debug", "warn", "error") + # Ethereum genesis generator params ethereum_genesis_generator_params: # The image to use for ethereum genesis generator diff --git a/main.star b/main.star index b6bfdad9c..3d94b8cf0 100644 --- a/main.star +++ b/main.star @@ -56,6 +56,7 @@ get_prefunded_accounts = import_module( "./src/prefunded_accounts/get_prefunded_accounts.star" ) spamoor = import_module("./src/spamoor/spamoor.star") +txpool_viz = import_module("./src/txpool_viz/txpool_viz.star") GRAFANA_USER = "admin" GRAFANA_PASSWORD = "admin" @@ -470,6 +471,18 @@ def run(plan, args={}): global_node_selectors, ) plan.print("Successfully launched tx-fuzz") + elif additional_service == "txpool_viz": + plan.print("Launching txpool-viz") + txpool_viz_config_template = read_file( + static_files.TXPOOL_VIZ_CONFIG_TEMPLATE_FILEPATH + ) + txpool_viz.launch_txpool_viz( + plan, + txpool_viz_config_template, + all_participants, + args_with_right_defaults.txpool_viz_params, + global_node_selectors, + ) elif additional_service == "forkmon": plan.print("Launching el forkmon") forkmon_config_template = read_file( diff --git a/src/package_io/constants.star b/src/package_io/constants.star index d415a4dde..111d277cf 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -93,6 +93,7 @@ MEV_RS_MEV_TYPE = "mev-rs" COMMIT_BOOST_MEV_TYPE = "commit-boost" DEFAULT_DORA_IMAGE = "ethpandaops/dora:latest" DEFAULT_SPAMOOR_IMAGE = "ethpandaops/spamoor:latest" +DEFAULT_TXPOOL_VIZ_IMAGE = "punkhazardlabs/txpool-viz:latest" DEFAULT_ASSERTOOR_IMAGE = "ethpandaops/assertoor:latest" DEFAULT_SNOOPER_IMAGE = "ethpandaops/rpc-snooper:latest" DEFAULT_ETHEREUM_GENESIS_GENERATOR_IMAGE = ( diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 73bf18a30..f705c8364 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -74,6 +74,7 @@ ATTR_TO_BE_SKIPPED_AT_ROOT = ( "dora_params", "docker_cache_params", "assertoor_params", + "txpool_viz_params", "prometheus_params", "grafana_params", "tx_fuzz_params", @@ -179,6 +180,8 @@ def input_parser(plan, input_args): for sub_attr in input_args["spamoor_params"]: sub_value = input_args["spamoor_params"][sub_attr] result["spamoor_params"][sub_attr] = sub_value + elif attr == "txpool_viz_params": + result["txpool_viz_params"] = get_txpool_viz_params(input_args) elif attr == "ethereum_genesis_generator_params": for sub_attr in input_args["ethereum_genesis_generator_params"]: sub_value = input_args["ethereum_genesis_generator_params"][sub_attr] @@ -572,6 +575,19 @@ def input_parser(plan, input_args): spammers=result["spamoor_params"]["spammers"], extra_args=result["spamoor_params"]["extra_args"], ), + txpool_viz_params=struct( + image=result["txpool_viz_params"]["image"], + min_cpu=result["txpool_viz_params"]["min_cpu"], + max_cpu=result["txpool_viz_params"]["max_cpu"], + min_mem=result["txpool_viz_params"]["min_mem"], + max_mem=result["txpool_viz_params"]["max_mem"], + extra_args=result["txpool_viz_params"]["extra_args"], + polling=result["txpool_viz_params"]["polling"], + filters=result["txpool_viz_params"]["filters"], + focil_enabled=result["txpool_viz_params"]["focil_enabled"], + log_level=result["txpool_viz_params"]["log_level"], + env=result["txpool_viz_params"]["env"], + ), additional_services=result["additional_services"], wait_for_finalization=result["wait_for_finalization"], global_log_level=result["global_log_level"], @@ -1048,6 +1064,7 @@ def default_input_args(input_args): "public_port_start": None, }, "spamoor_params": get_default_spamoor_params(), + "txpool_viz_params": get_default_txpool_viz_params(), } @@ -1698,6 +1715,7 @@ def docker_cache_image_override(plan, result): "prometheus_params.image", "grafana_params.image", "spamoor_params.image", + "txpool_viz_params.image", "ethereum_genesis_generator_params.image", ] @@ -1835,3 +1853,65 @@ def get_devnet_modified_images(network_name, default_images): modified_images[client_type] = get_devnet_image_tag(network_name, image) return modified_images + + +def get_txpool_viz_params(input_args): + image = input_args.get("txpool_viz_params", {}).get( + "image", constants.DEFAULT_TXPOOL_VIZ_IMAGE + ) + min_cpu = input_args.get("txpool_viz_params", {}).get("min_cpu", False) + max_cpu = input_args.get("txpool_viz_params", {}).get("max_cpu", False) + min_mem = input_args.get("txpool_viz_params", {}).get("min_mem", False) + max_mem = input_args.get("txpool_viz_params", {}).get("max_mem", False) + extra_args = input_args.get("txpool_viz_params", {}).get("extra_args", []) + polling_args = input_args.get("txpool_viz_params", {}).get("polling", {}) + filters_args = input_args.get("txpool_viz_params", {}).get("filters", {}) + focil_enabled = input_args.get("txpool_viz_params", {}).get( + "focil_enabled", "false" + ) + log_level = input_args.get("txpool_viz_params", {}).get("log_level", "info") + env = input_args.get("txpool_viz_params", {}).get("env", {}) + + polling_config = { + "interval": polling_args.get("interval", "0.5s"), + "timeout": polling_args.get("timeout", "3s"), + } + + filters_config = { + "min_gas_price": filters_args.get("min_gas_price", "1gwei"), + } + + return { + "image": image, + "min_cpu": min_cpu, + "max_cpu": max_cpu, + "min_mem": min_mem, + "max_mem": max_mem, + "extra_args": extra_args, + "polling": polling_config, + "filters": filters_config, + "focil_enabled": focil_enabled, + "log_level": log_level, + "env": env, + } + + +def get_default_txpool_viz_params(): + return { + "image": constants.DEFAULT_TXPOOL_VIZ_IMAGE, + "min_cpu": False, + "max_cpu": False, + "min_mem": False, + "max_mem": False, + "extra_args": [], + "polling": { + "interval": "0.5s", + "timeout": "3s", + }, + "filters": { + "min_gas_price": "1gwei", + }, + "focil_enabled": "false", + "log_level": "info", + "env": {}, + } diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index 058b216c6..6a4f33fee 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -299,6 +299,19 @@ SUBCATEGORY_PARAMS = { "mev", "other", ], + "txpool_viz_params": [ + "image", + "min_cpu", + "max_cpu", + "min_mem", + "max_mem", + "extra_args", + "polling", + "filters", + "focil_enabled", + "log_level", + "env", + ], } ADDITIONAL_SERVICES_PARAMS = [ @@ -321,6 +334,7 @@ ADDITIONAL_SERVICES_PARAMS = [ "nginx", "tracoor", "spamoor", + "txpool_viz", ] ADDITIONAL_CATEGORY_PARAMS = { diff --git a/src/static_files/static_files.star b/src/static_files/static_files.star index 049a1fe3b..903778e17 100644 --- a/src/static_files/static_files.star +++ b/src/static_files/static_files.star @@ -123,3 +123,9 @@ FLASHBOTS_RBUILDER_CONFIG_FILEPATH = ( COMMIT_BOOST_CONFIG_FILEPATH = ( STATIC_FILES_DIRPATH + "/mev/commit-boost/cb-config.toml.tmpl" ) + +# txpool-viz config +TXPOOL_VIZ_CONFIG_DIRPATH = "/txpool-viz-config" +TXPOOL_VIZ_CONFIG_TEMPLATE_FILEPATH = ( + STATIC_FILES_DIRPATH + TXPOOL_VIZ_CONFIG_DIRPATH + "/config.yaml.tmpl" +) diff --git a/src/txpool_viz/txpool_viz.star b/src/txpool_viz/txpool_viz.star new file mode 100644 index 000000000..2ca8dbba5 --- /dev/null +++ b/src/txpool_viz/txpool_viz.star @@ -0,0 +1,136 @@ +redis_module = import_module("github.com/kurtosis-tech/redis-package/main.star") +postgres_module = import_module("github.com/kurtosis-tech/postgres-package/main.star") +shared_utils = import_module("../shared_utils/shared_utils.star") + +HTTP_PORT_NUMBER = 8080 +TXPOOL_VIZ_SERVICE_NAME = "txpool-viz" + +TXPOOL_VIZ_CONFIG_FILENAME = "config.yaml" +TXPOOL_VIZ_CONFIG_PATH = "/cfg/" + +# The min/max CPU/memory that txpool-viz can use. +MIN_CPU = 100 +MAX_CPU = 1000 +MIN_MEM = 128 +MAX_MEM = 1024 + + +def launch_txpool_viz( + plan, + config_template, + network_participants, + txpool_viz_params, + global_node_selectors, +): + endpoints_list = [] + for participant in network_participants: + el_metrics_info = participant.el_context.el_metrics_info + if el_metrics_info and len(el_metrics_info) > 0: + name = el_metrics_info[0]["name"] + else: + name = "unknown" + endpoints_list.append( + { + "Name": name, + "RPCUrl": participant.el_context.rpc_http_url, + "Socket": participant.el_context.ws_url, + } + ) + + beacon_endpoints = [] + for participant in network_participants: + beacon_endpoints.append( + { + "Name": participant.cl_context.beacon_service_name, + "BeaconUrl": participant.cl_context.beacon_http_url, + } + ) + + # config data & template + template_data = txpool_viz_config_template_data( + txpool_viz_params, endpoints_list, beacon_endpoints + ) + + template_and_data = shared_utils.new_template_and_data( + config_template, template_data + ) + + file_config = {} + file_config[TXPOOL_VIZ_CONFIG_FILENAME] = template_and_data + + config_files_artifact_name = plan.render_templates( + config=file_config, + name="txpool-viz-config", + ) + + postgres = postgres_module.run( + plan, + service_name="txpool-viz-postgres", + ) + + # attaching redis server + redis = redis_module.run( + plan, + service_name="txpool-viz-redis", + ) + + redis_url = "redis://" + redis.hostname + ":" + str(redis.port_number) + "/0" + + environment_variables = dict(txpool_viz_params.env) + environment_variables.update( + { + "POSTGRES_URL": postgres.url, + "REDIS_URL": redis_url, + "PORT": str(HTTP_PORT_NUMBER), + "ENV": "prod", + } + ) + + service_config = get_service_config( + environment_variables, + txpool_viz_params, + global_node_selectors, + config_files_artifact_name, + ) + + txpoolviz = plan.add_service(TXPOOL_VIZ_SERVICE_NAME, config=service_config) + + +def get_service_config( + environment_variables, txpool_viz_params, node_selectors, files_artifact +): + return ServiceConfig( + image=txpool_viz_params.image, + ports={ + shared_utils.HTTP_APPLICATION_PROTOCOL: PortSpec( + number=HTTP_PORT_NUMBER, + application_protocol=shared_utils.HTTP_APPLICATION_PROTOCOL, + transport_protocol=shared_utils.TCP_PROTOCOL, + ), + }, + env_vars=environment_variables, + min_cpu=txpool_viz_params.min_cpu or MIN_CPU, + max_cpu=txpool_viz_params.max_cpu or MAX_CPU, + min_memory=txpool_viz_params.min_mem or MIN_MEM, + max_memory=txpool_viz_params.max_mem or MAX_MEM, + files={TXPOOL_VIZ_CONFIG_PATH: files_artifact}, + node_selectors=node_selectors, + ) + + +def txpool_viz_config_template_data(config, endpoints_list, beacon_endpoints): + cfg = dict(config.extra_args) + cfg.update( + { + "Endpoints": endpoints_list, + "Polling": config.polling, + "Filters": config.filters, + "LogLevel": config.log_level, + } + ) + + if config.focil_enabled == "true": + cfg["FocilEnabled"] = config.focil_enabled + cfg["BeaconEndpoints"] = beacon_endpoints + + return cfg diff --git a/static_files/txpool-viz-config/config.yaml.tmpl b/static_files/txpool-viz-config/config.yaml.tmpl new file mode 100644 index 000000000..33eab76e6 --- /dev/null +++ b/static_files/txpool-viz-config/config.yaml.tmpl @@ -0,0 +1,26 @@ +endpoints: +{{- range .Endpoints }} + - name: {{ .Name }} + rpc_url: "{{ .RPCUrl }}" + socket: "{{ .Socket }}" +{{- end }} + +{{- if eq .FocilEnabled "true" }} +focil_enabled: true +beacon_urls: +{{- range .BeaconEndpoints }} + - name: {{ .Name }} + beacon_url: "{{ .BeaconUrl }}" +{{- end }} +{{- end }} + +polling: + interval: {{ .Polling.interval }} + timeout: {{ .Polling.timeout }} + +filters: + min_gas_price: {{ .Filters.MinGasPrice }} + +log_level: "{{ .LogLevel }}" + +extra_args: {{ .ExtraArgs }}