feat: add txpool-viz

.github/tests/mix-public.yaml

@@ -50,6 +50,7 @@ additional_services:
   - apache
   - tracoor
   - spamoor
+  - txpool_viz
 ethereum_metrics_exporter_enabled: true
 snooper_enabled: true
 keymanager_enabled: true

.github/tests/mix-with-tools-mev.yaml

@@ -25,6 +25,7 @@ additional_services:
   - apache
   - tracoor
   - spamoor
+  - txpool_viz
 ethereum_metrics_exporter_enabled: true
 snooper_enabled: true
 mev_type: flashbots

.github/tests/mix-with-tools-minimal.yaml

@@ -27,6 +27,7 @@ additional_services:
   - apache
   - tracoor
   - spamoor
+  - txpool_viz
 ethereum_metrics_exporter_enabled: true
 snooper_enabled: true
 keymanager_enabled: true

.github/tests/mix-with-tools.yaml

@@ -26,6 +26,7 @@ additional_services:
   - apache
   - tracoor
   - spamoor
+  - txpool_viz
 ethereum_metrics_exporter_enabled: true
 snooper_enabled: true
 keymanager_enabled: true

README.md

@@ -758,6 +758,7 @@ additional_services:
   - forky
   - apache
   - tracoor
+  - txpool_viz
 
 # Configuration place for blockscout explorer - https://github.com/blockscout/blockscout
 blockscout_params:
@@ -1058,6 +1059,20 @@ spamoor_params:
   # A list of optional params that will be passed to the spamoor command for modifying its behaviour
   extra_args: []
 
+# Configuration for txpool_viz. A mempool visualizer.
+txpool_viz_params:
+  # The image to use for txpool_viz
+  image: punkhazardlabs/txpool-viz:latest
+  # Polling configuration for txpool_viz
+  polling:
+    interval: 0.1s # How often to poll for new mempool data (in seconds)
+    timeout: 5s # Timeout for polling requests (in seconds)
+  # Filters to apply to the mempool data
+  filters:
+    min_gas_price: 1gwei # Minimum gas price filter for transactions
+  focil_enabled: "true" # Enable or disable FOCIL (set to "true" to enable)
+  log_level: "info" # Logging level for txpool_viz (e.g., "info", "debug", "warn", "error")
+
 # Ethereum genesis generator params
 ethereum_genesis_generator_params:
   # The image to use for ethereum genesis generator

main.star

@@ -56,6 +56,7 @@ get_prefunded_accounts = import_module(
     "./src/prefunded_accounts/get_prefunded_accounts.star"
 )
 spamoor = import_module("./src/spamoor/spamoor.star")
+txpool_viz = import_module("./src/txpool_viz/txpool_viz.star")
 
 GRAFANA_USER = "admin"
 GRAFANA_PASSWORD = "admin"
@@ -470,6 +471,18 @@ def run(plan, args={}):
                 global_node_selectors,
             )
             plan.print("Successfully launched tx-fuzz")
+        elif additional_service == "txpool_viz":
+            plan.print("Launching txpool-viz")
+            txpool_viz_config_template = read_file(
+                static_files.TXPOOL_VIZ_CONFIG_TEMPLATE_FILEPATH
+            )
+            txpool_viz.launch_txpool_viz(
+                plan,
+                txpool_viz_config_template,
+                all_participants,
+                args_with_right_defaults.txpool_viz_params,
+                global_node_selectors,
+            )
         elif additional_service == "forkmon":
             plan.print("Launching el forkmon")
             forkmon_config_template = read_file(

src/package_io/constants.star

@@ -93,6 +93,7 @@ MEV_RS_MEV_TYPE = "mev-rs"
 COMMIT_BOOST_MEV_TYPE = "commit-boost"
 DEFAULT_DORA_IMAGE = "ethpandaops/dora:latest"
 DEFAULT_SPAMOOR_IMAGE = "ethpandaops/spamoor:latest"
+DEFAULT_TXPOOL_VIZ_IMAGE = "punkhazardlabs/txpool-viz:latest"
 DEFAULT_ASSERTOOR_IMAGE = "ethpandaops/assertoor:latest"
 DEFAULT_SNOOPER_IMAGE = "ethpandaops/rpc-snooper:latest"
 DEFAULT_ETHEREUM_GENESIS_GENERATOR_IMAGE = (

src/package_io/input_parser.star

@@ -74,6 +74,7 @@ ATTR_TO_BE_SKIPPED_AT_ROOT = (
     "dora_params",
     "docker_cache_params",
     "assertoor_params",
+    "txpool_viz_params",
     "prometheus_params",
     "grafana_params",
     "tx_fuzz_params",
@@ -179,6 +180,8 @@ def input_parser(plan, input_args):
             for sub_attr in input_args["spamoor_params"]:
                 sub_value = input_args["spamoor_params"][sub_attr]
                 result["spamoor_params"][sub_attr] = sub_value
+        elif attr == "txpool_viz_params":
+            result["txpool_viz_params"] = get_txpool_viz_params(input_args)
         elif attr == "ethereum_genesis_generator_params":
             for sub_attr in input_args["ethereum_genesis_generator_params"]:
                 sub_value = input_args["ethereum_genesis_generator_params"][sub_attr]
@@ -572,6 +575,19 @@ def input_parser(plan, input_args):
             spammers=result["spamoor_params"]["spammers"],
             extra_args=result["spamoor_params"]["extra_args"],
         ),
+        txpool_viz_params=struct(
+            image=result["txpool_viz_params"]["image"],
+            min_cpu=result["txpool_viz_params"]["min_cpu"],
+            max_cpu=result["txpool_viz_params"]["max_cpu"],
+            min_mem=result["txpool_viz_params"]["min_mem"],
+            max_mem=result["txpool_viz_params"]["max_mem"],
+            extra_args=result["txpool_viz_params"]["extra_args"],
+            polling=result["txpool_viz_params"]["polling"],
+            filters=result["txpool_viz_params"]["filters"],
+            focil_enabled=result["txpool_viz_params"]["focil_enabled"],
+            log_level=result["txpool_viz_params"]["log_level"],
+            env=result["txpool_viz_params"]["env"],
+        ),
         additional_services=result["additional_services"],
         wait_for_finalization=result["wait_for_finalization"],
         global_log_level=result["global_log_level"],
@@ -1048,6 +1064,7 @@ def default_input_args(input_args):
             "public_port_start": None,
         },
         "spamoor_params": get_default_spamoor_params(),
+        "txpool_viz_params": get_default_txpool_viz_params(),
     }
 
 
@@ -1698,6 +1715,7 @@ def docker_cache_image_override(plan, result):
         "prometheus_params.image",
         "grafana_params.image",
         "spamoor_params.image",
+        "txpool_viz_params.image",
         "ethereum_genesis_generator_params.image",
     ]
 
@@ -1835,3 +1853,65 @@ def get_devnet_modified_images(network_name, default_images):
         modified_images[client_type] = get_devnet_image_tag(network_name, image)
 
     return modified_images
+
+
+def get_txpool_viz_params(input_args):
+    image = input_args.get("txpool_viz_params", {}).get(
+        "image", constants.DEFAULT_TXPOOL_VIZ_IMAGE
+    )
+    min_cpu = input_args.get("txpool_viz_params", {}).get("min_cpu", False)
+    max_cpu = input_args.get("txpool_viz_params", {}).get("max_cpu", False)
+    min_mem = input_args.get("txpool_viz_params", {}).get("min_mem", False)
+    max_mem = input_args.get("txpool_viz_params", {}).get("max_mem", False)
+    extra_args = input_args.get("txpool_viz_params", {}).get("extra_args", [])
+    polling_args = input_args.get("txpool_viz_params", {}).get("polling", {})
+    filters_args = input_args.get("txpool_viz_params", {}).get("filters", {})
+    focil_enabled = input_args.get("txpool_viz_params", {}).get(
+        "focil_enabled", "false"
+    )
+    log_level = input_args.get("txpool_viz_params", {}).get("log_level", "info")
+    env = input_args.get("txpool_viz_params", {}).get("env", {})
+
+    polling_config = {
+        "interval": polling_args.get("interval", "0.5s"),
+        "timeout": polling_args.get("timeout", "3s"),
+    }
+
+    filters_config = {
+        "min_gas_price": filters_args.get("min_gas_price", "1gwei"),
+    }
+
+    return {
+        "image": image,
+        "min_cpu": min_cpu,
+        "max_cpu": max_cpu,
+        "min_mem": min_mem,
+        "max_mem": max_mem,
+        "extra_args": extra_args,
+        "polling": polling_config,
+        "filters": filters_config,
+        "focil_enabled": focil_enabled,
+        "log_level": log_level,
+        "env": env,
+    }
+
+
+def get_default_txpool_viz_params():
+    return {
+        "image": constants.DEFAULT_TXPOOL_VIZ_IMAGE,
+        "min_cpu": False,
+        "max_cpu": False,
+        "min_mem": False,
+        "max_mem": False,
+        "extra_args": [],
+        "polling": {
+            "interval": "0.5s",
+            "timeout": "3s",
+        },
+        "filters": {
+            "min_gas_price": "1gwei",
+        },
+        "focil_enabled": "false",
+        "log_level": "info",
+        "env": {},
+    }

src/package_io/sanity_check.star

@@ -299,6 +299,19 @@ SUBCATEGORY_PARAMS = {
         "mev",
         "other",
     ],
+    "txpool_viz_params": [
+        "image",
+        "min_cpu",
+        "max_cpu",
+        "min_mem",
+        "max_mem",
+        "extra_args",
+        "polling",
+        "filters",
+        "focil_enabled",
+        "log_level",
+        "env",
+    ],
 }
 
 ADDITIONAL_SERVICES_PARAMS = [
@@ -321,6 +334,7 @@ ADDITIONAL_SERVICES_PARAMS = [
     "nginx",
     "tracoor",
     "spamoor",
+    "txpool_viz",
 ]
 
 ADDITIONAL_CATEGORY_PARAMS = {

src/static_files/static_files.star

@@ -123,3 +123,9 @@ FLASHBOTS_RBUILDER_CONFIG_FILEPATH = (
 COMMIT_BOOST_CONFIG_FILEPATH = (
     STATIC_FILES_DIRPATH + "/mev/commit-boost/cb-config.toml.tmpl"
 )
+
+# txpool-viz config
+TXPOOL_VIZ_CONFIG_DIRPATH = "/txpool-viz-config"
+TXPOOL_VIZ_CONFIG_TEMPLATE_FILEPATH = (
+    STATIC_FILES_DIRPATH + TXPOOL_VIZ_CONFIG_DIRPATH + "/config.yaml.tmpl"
+)

src/txpool_viz/txpool_viz.star

@@ -0,0 +1,136 @@
+redis_module = import_module("github.com/kurtosis-tech/redis-package/main.star")
+postgres_module = import_module("github.com/kurtosis-tech/postgres-package/main.star")
+shared_utils = import_module("../shared_utils/shared_utils.star")
+
+HTTP_PORT_NUMBER = 8080
+TXPOOL_VIZ_SERVICE_NAME = "txpool-viz"
+
+TXPOOL_VIZ_CONFIG_FILENAME = "config.yaml"
+TXPOOL_VIZ_CONFIG_PATH = "/cfg/"
+
+# The min/max CPU/memory that txpool-viz can use.
+MIN_CPU = 100
+MAX_CPU = 1000
+MIN_MEM = 128
+MAX_MEM = 1024
+
+
+def launch_txpool_viz(
+    plan,
+    config_template,
+    network_participants,
+    txpool_viz_params,
+    global_node_selectors,
+):
+    endpoints_list = []
+    for participant in network_participants:
+        el_metrics_info = participant.el_context.el_metrics_info
+        if el_metrics_info and len(el_metrics_info) > 0:
+            name = el_metrics_info[0]["name"]
+        else:
+            name = "unknown"
+        endpoints_list.append(
+            {
+                "Name": name,
+                "RPCUrl": participant.el_context.rpc_http_url,
+                "Socket": participant.el_context.ws_url,
+            }
+        )
+
+    beacon_endpoints = []
+    for participant in network_participants:
+        beacon_endpoints.append(
+            {
+                "Name": participant.cl_context.beacon_service_name,
+                "BeaconUrl": participant.cl_context.beacon_http_url,
+            }
+        )
+
+    # config data & template
+    template_data = txpool_viz_config_template_data(
+        txpool_viz_params, endpoints_list, beacon_endpoints
+    )
+
+    template_and_data = shared_utils.new_template_and_data(
+        config_template, template_data
+    )
+
+    file_config = {}
+    file_config[TXPOOL_VIZ_CONFIG_FILENAME] = template_and_data
+
+    config_files_artifact_name = plan.render_templates(
+        config=file_config,
+        name="txpool-viz-config",
+    )
+
+    postgres = postgres_module.run(
+        plan,
+        service_name="txpool-viz-postgres",
+    )
+
+    # attaching redis server
+    redis = redis_module.run(
+        plan,
+        service_name="txpool-viz-redis",
+    )
+
+    redis_url = "redis://" + redis.hostname + ":" + str(redis.port_number) + "/0"
+
+    environment_variables = dict(txpool_viz_params.env)
+    environment_variables.update(
+        {
+            "POSTGRES_URL": postgres.url,
+            "REDIS_URL": redis_url,
+            "PORT": str(HTTP_PORT_NUMBER),
+            "ENV": "prod",
+        }
+    )
+
+    service_config = get_service_config(
+        environment_variables,
+        txpool_viz_params,
+        global_node_selectors,
+        config_files_artifact_name,
+    )
+
+    txpoolviz = plan.add_service(TXPOOL_VIZ_SERVICE_NAME, config=service_config)
+
+
+def get_service_config(
+    environment_variables, txpool_viz_params, node_selectors, files_artifact
+):
+    return ServiceConfig(
+        image=txpool_viz_params.image,
+        ports={
+            shared_utils.HTTP_APPLICATION_PROTOCOL: PortSpec(
+                number=HTTP_PORT_NUMBER,
+                application_protocol=shared_utils.HTTP_APPLICATION_PROTOCOL,
+                transport_protocol=shared_utils.TCP_PROTOCOL,
+            ),
+        },
+        env_vars=environment_variables,
+        min_cpu=txpool_viz_params.min_cpu or MIN_CPU,
+        max_cpu=txpool_viz_params.max_cpu or MAX_CPU,
+        min_memory=txpool_viz_params.min_mem or MIN_MEM,
+        max_memory=txpool_viz_params.max_mem or MAX_MEM,
+        files={TXPOOL_VIZ_CONFIG_PATH: files_artifact},
+        node_selectors=node_selectors,
+    )
+
+
+def txpool_viz_config_template_data(config, endpoints_list, beacon_endpoints):
+    cfg = dict(config.extra_args)
+    cfg.update(
+        {
+            "Endpoints": endpoints_list,
+            "Polling": config.polling,
+            "Filters": config.filters,
+            "LogLevel": config.log_level,
+        }
+    )
+
+    if config.focil_enabled == "true":
+        cfg["FocilEnabled"] = config.focil_enabled
+        cfg["BeaconEndpoints"] = beacon_endpoints
+
+    return cfg