From b0418f8e733b8d421448bbebd30c21b1ebf13401 Mon Sep 17 00:00:00 2001 From: skudasov Date: Thu, 19 Mar 2026 11:23:41 +0100 Subject: [PATCH 1/2] remove stale automation/vrf tests --- .github/e2e-tests.yml | 95 --- .../on-demand-vrfv2-performance-test.yml | 104 --- .../on-demand-vrfv2plus-performance-test.yml | 106 --- .../automationv2_1/automationv2_1_test.go | 759 ------------------ integration-tests/load/automationv2_1/gun.go | 123 --- .../load/automationv2_1/helpers.go | 207 ----- integration-tests/load/vrfv2/README.md | 22 - integration-tests/load/vrfv2/gun.go | 140 ---- .../load/vrfv2/onchain_monitoring.go | 56 -- integration-tests/load/vrfv2/vrfv2_test.go | 383 --------- .../load/vrfv2/vrfv2cmd/dashboard.go | 102 --- integration-tests/load/vrfv2plus/README.md | 22 - integration-tests/load/vrfv2plus/gun.go | 144 ---- .../load/vrfv2plus/onchain_monitoring.go | 56 -- .../load/vrfv2plus/vrfv2plus_test.go | 397 --------- .../load/vrfv2plus/vrfv2pluscmd/dashboard.go | 102 --- 16 files changed, 2818 deletions(-) delete mode 100644 .github/workflows/on-demand-vrfv2-performance-test.yml delete mode 100644 .github/workflows/on-demand-vrfv2plus-performance-test.yml delete mode 100644 integration-tests/load/automationv2_1/automationv2_1_test.go delete mode 100644 integration-tests/load/automationv2_1/gun.go delete mode 100644 integration-tests/load/automationv2_1/helpers.go delete mode 100644 integration-tests/load/vrfv2/README.md delete mode 100644 integration-tests/load/vrfv2/gun.go delete mode 100644 integration-tests/load/vrfv2/onchain_monitoring.go delete mode 100644 integration-tests/load/vrfv2/vrfv2_test.go delete mode 100644 integration-tests/load/vrfv2/vrfv2cmd/dashboard.go delete mode 100644 integration-tests/load/vrfv2plus/README.md delete mode 100644 integration-tests/load/vrfv2plus/gun.go delete mode 100644 integration-tests/load/vrfv2plus/onchain_monitoring.go delete mode 100644 integration-tests/load/vrfv2plus/vrfv2plus_test.go delete mode 100644 integration-tests/load/vrfv2plus/vrfv2pluscmd/dashboard.go diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 01a6ae84769..28adf15845c 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -13,25 +13,6 @@ runner-test-matrix: # START: Automation tests - - id: load/automationv2_1/automationv2_1_test.go:TestLogTrigger - path: integration-tests/load/automationv2_1/automationv2_1_test.go - runs_on: ubuntu-latest - test_env_type: k8s-remote-runner - test_cmd: | - cd load/automationv2_1 && \ - gotestsum \ - --junitfile=/tmp/junit.xml \ - --jsonfile=/tmp/gotest.log \ - --format=github-actions \ - -- -v -run "TestLogTrigger" -parallel=1 -timeout 60m -count=1 - remote_runner_memory: 4Gi - test_secrets_required: true - test_env_vars: - TEST_LOG_LEVEL: info - TEST_SUITE: automationv2_1 - pyroscope_env: automation-load-test - test_go_project_path: integration-tests - - id: reorg/automation_reorg_test.go^TestAutomationReorg/registry_2_0 path: integration-tests/reorg/automation_reorg_test.go runs_on: ubuntu-latest @@ -198,82 +179,6 @@ runner-test-matrix: - On Demand VRFV2 Smoke Test (Ethereum clients) test_go_project_path: integration-tests - - id: load/vrfv2plus/vrfv2plus_test.go:^TestVRFV2PlusPerformance$Smoke - path: integration-tests/load/vrfv2plus/vrfv2plus_test.go - runs_on: ubuntu22.04-8cores-32GB - test_env_type: docker - test_cmd: | - cd load/vrfv2plus && \ - gotestsum \ - --junitfile=/tmp/junit.xml \ - --jsonfile=/tmp/gotest.log \ - --format=github-actions \ - -- -v -run "^TestVRFV2PlusPerformance$" -parallel=1 -timeout 24h -count=1 - test_config_override_required: true - test_secrets_required: true - test_env_vars: - TEST_TYPE: Smoke - triggers: - - On Demand VRFV2 Plus Performance Test - test_go_project_path: integration-tests - - - id: load/vrfv2plus/vrfv2plus_test.go:^TestVRFV2PlusBHSPerformance$Smoke - path: integration-tests/load/vrfv2plus/vrfv2plus_test.go - runs_on: ubuntu22.04-8cores-32GB - test_env_type: docker - test_cmd: | - cd load/vrfv2plus && \ - gotestsum \ - --junitfile=/tmp/junit.xml \ - --jsonfile=/tmp/gotest.log \ - --format=github-actions \ - -- -v -run "^TestVRFV2PlusBHSPerformance$" -parallel=1 -timeout 24h -count=1 - test_config_override_required: true - test_secrets_required: true - test_env_vars: - TEST_TYPE: Smoke - triggers: - - On Demand VRFV2 Plus Performance Test - test_go_project_path: integration-tests - - - id: load/vrfv2/vrfv2_test.go:^TestVRFV2Performance$Smoke - path: integration-tests/load/vrfv2/vrfv2_test.go - runs_on: ubuntu22.04-8cores-32GB - test_env_type: docker - test_cmd: | - cd load/vrfv2 && \ - gotestsum \ - --junitfile=/tmp/junit.xml \ - --jsonfile=/tmp/gotest.log \ - --format=github-actions \ - -- -v -run "^TestVRFV2Performance$" -parallel=1 -timeout 24h -count=1 - test_config_override_required: true - test_secrets_required: true - test_env_vars: - TEST_TYPE: Smoke - triggers: - - On Demand VRFV2 Performance Test - test_go_project_path: integration-tests - - - id: load/vrfv2/vrfv2_test.go:^TestVRFV2PlusBHSPerformance$Smoke - path: integration-tests/load/vrfv2/vrfv2_test.go - runs_on: ubuntu22.04-8cores-32GB - test_env_type: docker - test_cmd: | - cd load/vrfv2 && \ - gotestsum \ - --junitfile=/tmp/junit.xml \ - --jsonfile=/tmp/gotest.log \ - --format=github-actions \ - -- -v -run "^TestVRFV2PlusBHSPerformance$" -parallel=1 -timeout 24h -count=1 - test_config_override_required: true - test_secrets_required: true - test_env_vars: - TEST_TYPE: Smoke - triggers: - - On Demand VRFV2 Performance Test - test_go_project_path: integration-tests - - id: smoke/vrf_test.go:* path: integration-tests/smoke/vrf_test.go test_env_type: docker diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml deleted file mode 100644 index c12580de006..00000000000 --- a/.github/workflows/on-demand-vrfv2-performance-test.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: On Demand VRFV2 Performance Test -on: - workflow_dispatch: - inputs: - performanceTestType: - description: Performance Test Type of test to run - type: choice - options: - - "Smoke" - - "Soak" - - "Load" - - "Stress" - - "Spike" - test_list_regex: - description: "Regex for tests to run" - required: false - default: "(TestVRFV2Performance)" - test_config_override_path: - description: Path to a test config file used to override the default test config - required: false - type: string - test_secrets_override_key: - description: 'Key to run tests with custom test secrets' - required: false - type: string - notify_user_id_on_failure: - description: 'Enter Slack user ID to notify on test failure' - required: false - type: string - -jobs: - set-tests-to-run: - name: Set tests to run - runs-on: ubuntu-latest - outputs: - test_list: ${{ steps.set-tests.outputs.test_list }} - env: - GH_INPUTS_TEST_LIST_REGEX: ${{ inputs.test_list_regex }} - GH_INPUTS_TEST_CONFIG_OVERRIDE_PATH: ${{ inputs.test_config_override_path }} - GH_INPUTS_PERFORMANCE_TEST_TYPE: ${{ inputs.performanceTestType }} - steps: - - name: Generate Test List JSON - id: set-tests - run: | - TEST_CMD='cd integration-tests/load && go test -v -count=1 -timeout 24h -run "$GH_INPUTS_TEST_LIST_REGEX" ./vrfv2' - TEST_CONFIG_OVERRIDE_PATH=$GH_INPUTS_TEST_CONFIG_OVERRIDE_PATH - TEST_TYPE=$GH_INPUTS_PERFORMANCE_TEST_TYPE - - TEST_LIST=$(jq -n -c \ - --arg test_cmd "$TEST_CMD" \ - --arg test_config_override_path "$TEST_CONFIG_OVERRIDE_PATH" \ - --arg TEST_TYPE "$TEST_TYPE" \ - '{ - "tests": [ - { - "id": "TestVRFv2Plus_Performance", - "path": "integration-tests/load/vrfv2plus/vrfv2plus_test.go", - "runs_on": "ubuntu22.04-8cores-32GB", - "test_env_type": "docker", - "test_cmd": $test_cmd, - "test_config_override_path": $test_config_override_path, - "test_env_vars": { - "TEST_TYPE": $TEST_TYPE - } - } - ] - }') - - echo "test_list=$TEST_LIST" >> $GITHUB_OUTPUT - - run-e2e-tests-workflow: - name: Run E2E Tests - needs: set-tests-to-run - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@2e0f146b5352cbf5b00525687ae65ccc9828abb6 - with: - custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }} - chainlink_version: ${{ inputs.chainlink_version }} - slack_notification_after_tests: always - slack_notification_after_tests_name: "VRF V2 Performance Tests with test config: ${{ inputs.test_config_override_path || 'default' }}" - slack_notification_after_tests_notify_user_id_on_failure: ${{ inputs.notify_user_id_on_failure }} - secrets: - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - PROD_AWS_ACCOUNT_NUMBER: ${{ secrets.AWS_ACCOUNT_ID_PROD }} - QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} - QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} - GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }} - LOKI_URL: ${{ secrets.LOKI_URL }} - LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }} - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} - SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} - SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }} - MAIN_DNS_ZONE_PUBLIC_SDLC: ${{ secrets.MAIN_DNS_ZONE_PUBLIC_SDLC }} - AWS_K8S_CLUSTER_NAME_SDLC: ${{ secrets.AWS_K8S_CLUSTER_NAME_SDLC }} diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml deleted file mode 100644 index 7045e3ef37d..00000000000 --- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: On Demand VRFV2 Plus Performance Test -on: - workflow_dispatch: - inputs: - performanceTestType: - description: Performance Test Type of test to run - type: string - required: true - test_list_regex: - description: "Regex for tests to run" - required: false - default: "(TestVRFV2PlusPerformance)" - test_config_override_path: - description: Path to a test config file used to override the default test config - required: false - type: string - test_secrets_override_key: - description: 'Key to run tests with custom test secrets' - required: false - type: string - chainlink_version: - description: Chainlink image version to use - default: develop - required: false - type: string - notify_user_id_on_failure: - description: 'Enter Slack user ID to notify on test failure' - required: false - type: string - -jobs: - set-tests-to-run: - name: Set tests to run - runs-on: ubuntu-latest - outputs: - test_list: ${{ steps.set-tests.outputs.test_list }} - env: - GH_INPUTS_TEST_LIST_REGEX: ${{ inputs.test_list_regex }} - GH_INPUTS_TEST_CONFIG_OVERRIDE_PATH: ${{ inputs.test_config_override_path }} - GH_INPUTS_PERFORMANCE_TEST_TYPE: ${{ inputs.performanceTestType }} - steps: - - name: Generate Test List JSON - id: set-tests - run: | - TEST_CMD='cd integration-tests/load && go test -v -count=1 -timeout 24h -run "$GH_INPUTS_TEST_LIST_REGEX" ./vrfv2plus' - TEST_CONFIG_OVERRIDE_PATH=$GH_INPUTS_TEST_CONFIG_OVERRIDE_PATH - TEST_TYPE=$GH_INPUTS_PERFORMANCE_TEST_TYPE - - TEST_LIST=$(jq -n -c \ - --arg test_cmd "$TEST_CMD" \ - --arg test_config_override_path "$TEST_CONFIG_OVERRIDE_PATH" \ - --arg TEST_TYPE "$TEST_TYPE" \ - '{ - "tests": [ - { - "id": "TestVRFv2Plus_Performance", - "path": "integration-tests/load/vrfv2plus/vrfv2plus_test.go", - "runs_on": "ubuntu-latest", - "test_env_type": "docker", - "test_cmd": $test_cmd, - "test_config_override_path": $test_config_override_path, - "test_env_vars": { - "TEST_TYPE": $TEST_TYPE - } - } - ] - }') - - echo "test_list=$TEST_LIST" >> $GITHUB_OUTPUT - - run-e2e-tests-workflow: - name: Run E2E Tests - needs: set-tests-to-run - uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@3f881c41965743bc3af7f2223242f66f21931322 - with: - custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }} - chainlink_version: ${{ inputs.chainlink_version }} - slack_notification_after_tests: always - slack_notification_after_tests_name: "VRF V2 Plus Performance Tests with test config: ${{ inputs.test_config_override_path || 'default' }}" - slack_notification_after_tests_notify_user_id_on_failure: ${{ inputs.notify_user_id_on_failure }} - test_secrets_override_key: ${{ inputs.test_secrets_override_key }} - secrets: - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - PROD_AWS_ACCOUNT_NUMBER: ${{ secrets.AWS_ACCOUNT_ID_PROD }} - QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} - QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} - GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }} - LOKI_URL: ${{ secrets.LOKI_URL }} - LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }} - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} - SLACK_NOTIFICATION_AFTER_TESTS_CHANNEL_ID: ${{ secrets.QA_VRF_SLACK_CHANNEL }} - SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} - SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }} - MAIN_DNS_ZONE_PUBLIC_SDLC: ${{ secrets.MAIN_DNS_ZONE_PUBLIC_SDLC }} - AWS_K8S_CLUSTER_NAME_SDLC: ${{ secrets.AWS_K8S_CLUSTER_NAME_SDLC }} diff --git a/integration-tests/load/automationv2_1/automationv2_1_test.go b/integration-tests/load/automationv2_1/automationv2_1_test.go deleted file mode 100644 index e40171ec390..00000000000 --- a/integration-tests/load/automationv2_1/automationv2_1_test.go +++ /dev/null @@ -1,759 +0,0 @@ -package automation - -import ( - "context" - "encoding/hex" - "fmt" - "io" - "math" - "math/big" - "net/http" - "strconv" - "strings" - "testing" - "time" - - "github.com/pkg/errors" - - geth "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/pelletier/go-toml/v2" - "github.com/slack-go/slack" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - - "github.com/smartcontractkit/chainlink-testing-framework/lib/k8s/environment" - "github.com/smartcontractkit/chainlink-testing-framework/lib/k8s/pkg/helm/chainlink" - "github.com/smartcontractkit/chainlink-testing-framework/lib/k8s/pkg/helm/ethereum" - "github.com/smartcontractkit/chainlink-testing-framework/lib/k8s/pkg/helm/wiremock" - "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" - "github.com/smartcontractkit/chainlink-testing-framework/lib/networks" - seth_utils "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/seth" - - ctfconfig "github.com/smartcontractkit/chainlink-testing-framework/lib/config" - - gowiremock "github.com/wiremock/go-wiremock" - - "github.com/smartcontractkit/chainlink/deployment/environment/nodeclient" - - ac "github.com/smartcontractkit/chainlink-evm/gethwrappers/generated/automation_compatible_utils" - "github.com/smartcontractkit/chainlink-evm/gethwrappers/generated/simple_log_upkeep_counter_wrapper" - "github.com/smartcontractkit/chainlink-evm/gethwrappers/shared/generated/initial/log_emitter" - "github.com/smartcontractkit/chainlink/integration-tests/actions" - "github.com/smartcontractkit/chainlink/integration-tests/actions/automationv2" - "github.com/smartcontractkit/chainlink/integration-tests/contracts" - contractseth "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum" - tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" - aconfig "github.com/smartcontractkit/chainlink/integration-tests/testconfig/automation" - "github.com/smartcontractkit/chainlink/integration-tests/testreporters" -) - -const ( - StartupWaitTime = 30 * time.Second - StopWaitTime = 60 * time.Second -) - -var ( - secretsTOML = `[Mercury.Credentials.%s] -LegacyURL = '%s' -URL = '%s' -Username = '%s' -Password = '%s'` - - minimumNodeSpec = map[string]any{ - "resources": map[string]any{ - "requests": map[string]any{ - "cpu": "2000m", - "memory": "4Gi", - }, - "limits": map[string]any{ - "cpu": "2000m", - "memory": "4Gi", - }, - }, - } - - minimumDbSpec = map[string]any{ - "resources": map[string]any{ - "requests": map[string]any{ - "cpu": "4000m", - "memory": "4Gi", - }, - "limits": map[string]any{ - "cpu": "4000m", - "memory": "4Gi", - }, - }, - "stateful": true, - "capacity": "20Gi", - "enablePrometheusPostgresExporter": true, - } - - recNodeSpec = map[string]any{ - "resources": map[string]any{ - "requests": map[string]any{ - "cpu": "4000m", - "memory": "8Gi", - }, - "limits": map[string]any{ - "cpu": "4000m", - "memory": "8Gi", - }, - }, - } - - recDbSpec = minimumDbSpec - - gethNodeSpec = map[string]any{ - "requests": map[string]any{ - "cpu": "8000m", - "memory": "8Gi", - }, - "limits": map[string]any{ - "cpu": "16000m", - "memory": "16Gi", - }, - } -) - -func setUpDataStreamsWireMock(ctx context.Context, url string) error { - wm := gowiremock.NewClient(url) - rule200 := gowiremock.Get(gowiremock.URLPathEqualTo("/api/v1/reports/bulk")). - WithQueryParam("feedIDs", gowiremock.EqualTo("0x000200")). - WillReturnResponse(gowiremock.NewResponse(). - WithBody(`{"reports":[{"feedID":"0x000200","validFromTimestamp":0,"observationsTimestamp":0,"fullReport":"0x000abc"}]}`). - WithStatus(200)) - err := wm.StubFor(rule200) - if err != nil { - return err - } - req, err := http.NewRequestWithContext(ctx, "POST", url+"/__admin/mappings/save", nil) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return errors.New("error saving wiremock mappings") - } - defer func(Body io.ReadCloser) { - err := Body.Close() - if err != nil { - panic(err) - } - }(resp.Body) - - if resp.StatusCode != 200 { - return errors.New("error saving wiremock mappings") - } - - return nil -} - -func TestLogTrigger(t *testing.T) { - ctx := t.Context() - l := logging.GetTestLogger(t) - registryVersion := contractseth.RegistryVersion_2_1 - - loadedTestConfig, err := tc.GetConfig([]string{"Load"}, tc.Automation) - if err != nil { - t.Fatal(err) - } - l.Info().Interface("loadedTestConfig", loadedTestConfig).Msg("Loaded Test Config") - - version := *loadedTestConfig.ChainlinkImage.Version - image := *loadedTestConfig.ChainlinkImage.Image - - l.Info().Msg("Starting automation v2.1 log trigger load test") - l.Info(). - Int("Number of Nodes", *loadedTestConfig.Automation.General.NumberOfNodes). - Int("Duration", *loadedTestConfig.Automation.General.Duration). - Int("Block Time", *loadedTestConfig.Automation.General.BlockTime). - Str("Spec Type", *loadedTestConfig.Automation.General.SpecType). - Str("Log Level", *loadedTestConfig.Automation.General.ChainlinkNodeLogLevel). - Str("Image", image). - Str("Tag", version). - Msg("Test Config") - - testConfigFormat := `Number of Nodes: %d -Duration: %d -Block Time: %d -Spec Type: %s -Log Level: %s -Image: %s -Tag: %s - -Load Config: -%s` - - prettyLoadConfig, err := toml.Marshal(loadedTestConfig.Automation.Load) - require.NoError(t, err, "Error marshalling load config") - - testConfig := fmt.Sprintf(testConfigFormat, *loadedTestConfig.Automation.General.NumberOfNodes, *loadedTestConfig.Automation.General.Duration, - *loadedTestConfig.Automation.General.BlockTime, *loadedTestConfig.Automation.General.SpecType, *loadedTestConfig.Automation.General.ChainlinkNodeLogLevel, image, version, string(prettyLoadConfig)) - l.Info().Str("testConfig", testConfig).Msg("Test Config") - - testNetwork := networks.MustGetSelectedNetworkConfig(loadedTestConfig.Network)[0] - testType := "load" - loadDuration := time.Duration(*loadedTestConfig.Automation.General.Duration) * time.Second - automationDefaultLinkFunds := big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(10000))) // 10000 LINK - - nsLabels, err := environment.GetRequiredChainLinkNamespaceLabels(string(tc.Automation), testType) - require.NoError(t, err, "Error creating required chain.link labels for namespace") - - workloadPodLabels, err := environment.GetRequiredChainLinkWorkloadAndPodLabels(string(tc.Automation), testType) - require.NoError(t, err, "Error creating required chain.link labels for workloads and pods") - - testEnvironment := environment.New(&environment.Config{ - TTL: loadDuration.Round(time.Hour) + time.Hour, - NamespacePrefix: fmt.Sprintf( - "automation-%s-%s", - testType, - strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-"), - ), - Labels: nsLabels, - WorkloadLabels: workloadPodLabels, - PodLabels: workloadPodLabels, - Test: t, - PreventPodEviction: true, - }) - - testEnvironment. - AddHelm(ethereum.New(ðereum.Props{ - NetworkName: testNetwork.Name, - Simulated: testNetwork.Simulated, - WsURLs: testNetwork.URLs, - Values: map[string]any{ - "resources": gethNodeSpec, - "geth": map[string]any{ - "blocktime": *loadedTestConfig.Automation.General.BlockTime, - "capacity": "20Gi", - "startGaslimit": "20000000", - "targetGasLimit": "30000000", - }, - }, - })) - - err = testEnvironment.Run() - require.NoError(t, err, "Error launching test environment") - - if testEnvironment.WillUseRemoteRunner() { - return - } - - var ( - nodeSpec = minimumNodeSpec - dbSpec = minimumDbSpec - ) - - switch *loadedTestConfig.Automation.General.SpecType { - case "recommended": - nodeSpec = recNodeSpec - dbSpec = recDbSpec - case "local": - nodeSpec = map[string]any{} - dbSpec = map[string]any{"stateful": true} - default: - // minimum: - } - - if *loadedTestConfig.Pyroscope.Enabled { - loadedTestConfig.Pyroscope.Environment = &testEnvironment.Cfg.Namespace - } - - if *loadedTestConfig.Automation.DataStreams.Enabled { - if loadedTestConfig.Automation.DataStreams.URL == nil || *loadedTestConfig.Automation.DataStreams.URL == "" { - testEnvironment.AddHelm(wiremock.New(nil)) - err := testEnvironment.Run() - require.NoError(t, err, "Error running wiremock server") - wiremockURL := testEnvironment.URLs[wiremock.InternalURLsKey][0] - secretsTOML = fmt.Sprintf( - secretsTOML, "cred1", - wiremockURL, wiremockURL, - "username", "password", - ) - if !testEnvironment.Cfg.InsideK8s { - wiremockURL = testEnvironment.URLs[wiremock.LocalURLsKey][0] - } - err = setUpDataStreamsWireMock(ctx, wiremockURL) - require.NoError(t, err, "Error setting up wiremock server") - } else { - secretsTOML = fmt.Sprintf( - secretsTOML, "cred1", - *loadedTestConfig.Automation.DataStreams.URL, *loadedTestConfig.Automation.DataStreams.URL, - *loadedTestConfig.Automation.DataStreams.Username, *loadedTestConfig.Automation.DataStreams.Password, - ) - } - } else { - secretsTOML = "" - } - - numberOfUpkeeps := *loadedTestConfig.Automation.General.NumberOfNodes - - for i := 0; i < numberOfUpkeeps+1; i++ { // +1 for the OCR boot node - var overrideFn = func(_ any, target any) { - ctfconfig.MustConfigOverrideChainlinkVersion(loadedTestConfig.GetChainlinkImageConfig(), target) - ctfconfig.MightConfigOverridePyroscopeKey(loadedTestConfig.GetPyroscopeConfig(), target) - } - - tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(&loadedTestConfig, testNetwork) - require.NoError(t, err, "Error building TOML config") - - cd := chainlink.NewWithOverride(i, map[string]any{ - "toml": tomlConfig, - "chainlink": nodeSpec, - "db": dbSpec, - "prometheus": *loadedTestConfig.Automation.General.UsePrometheus, - "secretsToml": secretsTOML, - }, loadedTestConfig.ChainlinkImage, overrideFn) - testEnvironment.AddHelm(cd) - } - - err = testEnvironment.Run() - require.NoError(t, err, "Error running chainlink DON") - - testNetwork = seth_utils.MustReplaceSimulatedNetworkUrlWithK8(l, testNetwork, *testEnvironment) - chainClient, err := seth_utils.GetChainClientWithConfigFunction(loadedTestConfig, testNetwork, seth_utils.OneEphemeralKeysLiveTestnetCheckFn) - require.NoError(t, err, "Error creating seth client") - - chainlinkNodes, err := nodeclient.ConnectChainlinkNodes(testEnvironment) - require.NoError(t, err, "Error connecting to chainlink nodes") - - multicallAddress, err := contracts.DeployMultiCallContract(chainClient) - require.NoError(t, err, "Error deploying multicall contract") - - a := automationv2.NewAutomationTestK8s(l, chainClient, chainlinkNodes, &loadedTestConfig) - a.RegistrySettings = actions.ReadRegistryConfig(loadedTestConfig) - a.RegistrySettings.RegistryVersion = registryVersion - a.PluginConfig = actions.ReadPluginConfig(loadedTestConfig) - a.PublicConfig = actions.ReadPublicConfig(loadedTestConfig) - a.RegistrarSettings = contracts.KeeperRegistrarSettings{ - AutoApproveConfigType: uint8(2), - AutoApproveMaxAllowed: 1000, - MinLinkJuels: big.NewInt(0), - } - - if *loadedTestConfig.Automation.DataStreams.Enabled { - a.SetMercuryCredentialName("cred1") - } - - startTimeTestSetup := time.Now() - l.Info().Str("START_TIME", startTimeTestSetup.String()).Msg("Test setup started") - - a.SetupAutomationDeployment(t) - - err = actions.FundChainlinkNodesFromRootAddress(l, a.ChainClient, contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(chainlinkNodes[1:]), big.NewFloat(*loadedTestConfig.Common.ChainlinkNodeFunding)) - require.NoError(t, err, "Error funding chainlink nodes") - - consumerContracts := make([]contracts.KeeperConsumer, 0) - triggerContracts := make([]contracts.LogEmitter, 0) - triggerAddresses := make([]common.Address, 0) - - convenienceABI, err := ac.AutomationCompatibleUtilsMetaData.GetAbi() - require.NoError(t, err, "Error getting automation utils abi") - emitterABI, err := log_emitter.LogEmitterMetaData.GetAbi() - require.NoError(t, err, "Error getting log emitter abi") - consumerABI, err := simple_log_upkeep_counter_wrapper.SimpleLogUpkeepCounterMetaData.GetAbi() - require.NoError(t, err, "Error getting consumer abi") - - var bytes0 = [32]byte{ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - } - - var bytes1 = [32]byte{ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - } - - upkeepConfigs := make([]automationv2.UpkeepConfig, 0) - loadConfigs := make([]aconfig.Load, 0) - - expectedTotalUpkeepCount := 0 - for _, u := range loadedTestConfig.Automation.Load { - expectedTotalUpkeepCount += *u.NumberOfUpkeeps - } - - maxDeploymentConcurrency := 100 - - for _, u := range loadedTestConfig.Automation.Load { - deploymentData, err := deployConsumerAndTriggerContracts(l, u, a.ChainClient, multicallAddress, maxDeploymentConcurrency, automationDefaultLinkFunds, a.LinkToken) - require.NoError(t, err, "Error deploying consumer and trigger contracts") - - consumerContracts = append(consumerContracts, deploymentData.ConsumerContracts...) - triggerContracts = append(triggerContracts, deploymentData.TriggerContracts...) - triggerAddresses = append(triggerAddresses, deploymentData.TriggerAddresses...) - loadConfigs = append(loadConfigs, deploymentData.LoadConfigs...) - } - - require.Len(t, consumerContracts, expectedTotalUpkeepCount, "Incorrect number of consumer/trigger contracts deployed") - - for i, consumerContract := range consumerContracts { - logTriggerConfigStruct := ac.IAutomationV21PlusCommonLogTriggerConfig{ - ContractAddress: triggerAddresses[i], - FilterSelector: 1, - Topic0: emitterABI.Events["Log4"].ID, - Topic1: bytes1, - Topic2: bytes0, - Topic3: bytes0, - } - encodedLogTriggerConfig, err := convenienceABI.Methods["_logTriggerConfig"].Inputs.Pack(&logTriggerConfigStruct) - require.NoError(t, err, "Error encoding log trigger config") - l.Debug(). - Interface("logTriggerConfigStruct", logTriggerConfigStruct). - Str("Encoded Log Trigger Config", hex.EncodeToString(encodedLogTriggerConfig)).Msg("Encoded Log Trigger Config") - - checkDataStruct := simple_log_upkeep_counter_wrapper.CheckData{ - CheckBurnAmount: loadConfigs[i].CheckBurnAmount, - PerformBurnAmount: loadConfigs[i].PerformBurnAmount, - EventSig: bytes1, - Feeds: loadConfigs[i].Feeds, - } - - encodedCheckDataStruct, err := consumerABI.Methods["_checkDataConfig"].Inputs.Pack(&checkDataStruct) - require.NoError(t, err, "Error encoding check data struct") - l.Debug(). - Interface("checkDataStruct", checkDataStruct). - Str("Encoded Check Data Struct", hex.EncodeToString(encodedCheckDataStruct)).Msg("Encoded Check Data Struct") - - upkeepConfig := automationv2.UpkeepConfig{ - UpkeepName: fmt.Sprintf("LogTriggerUpkeep-%d", i), - EncryptedEmail: []byte("test@mail.com"), - UpkeepContract: common.HexToAddress(consumerContract.Address()), - GasLimit: *loadConfigs[i].UpkeepGasLimit, - AdminAddress: chainClient.MustGetRootKeyAddress(), - TriggerType: uint8(1), - CheckData: encodedCheckDataStruct, - TriggerConfig: encodedLogTriggerConfig, - OffchainConfig: []byte(""), - FundingAmount: automationDefaultLinkFunds, - } - l.Debug().Interface("Upkeep Config", upkeepConfig).Msg("Upkeep Config") - upkeepConfigs = append(upkeepConfigs, upkeepConfig) - } - - require.Len(t, upkeepConfigs, expectedTotalUpkeepCount, "Incorrect number of upkeep configs created") - registrationTxHashes, err := a.RegisterUpkeeps(upkeepConfigs, maxDeploymentConcurrency) - require.NoError(t, err, "Error registering upkeeps") - - upkeepIDs, err := a.ConfirmUpkeepsRegistered(registrationTxHashes, maxDeploymentConcurrency) - require.NoError(t, err, "Error confirming upkeeps registered") - require.Len(t, upkeepIDs, expectedTotalUpkeepCount, "Incorrect number of upkeeps registered") - - l.Info().Msg("Successfully registered all Automation Upkeeps") - l.Info().Interface("Upkeep IDs", upkeepIDs).Msg("Upkeeps Registered") - l.Info().Str("STARTUP_WAIT_TIME", StartupWaitTime.String()).Msg("Waiting for plugin to start") - time.Sleep(StartupWaitTime) - - startBlock, err := a.ChainClient.Client.BlockNumber(ctx) - require.NoError(t, err, "Error getting latest block number") - - p := wasp.NewProfile() - - configs := make([]LogTriggerConfig, 0) - var numberOfEventsEmitted int64 - var numberOfEventsEmittedPerSec int64 - - for i, triggerContract := range triggerContracts { - c := LogTriggerConfig{ - Address: triggerContract.Address().String(), - NumberOfEvents: int64(*loadConfigs[i].NumberOfEvents), - NumberOfSpamMatchingEvents: int64(*loadConfigs[i].NumberOfSpamMatchingEvents), - NumberOfSpamNonMatchingEvents: int64(*loadConfigs[i].NumberOfSpamNonMatchingEvents), - } - numberOfEventsEmittedPerSec += int64(*loadConfigs[i].NumberOfEvents) - configs = append(configs, c) - } - - endTimeTestSetup := time.Now() - testSetupDuration := endTimeTestSetup.Sub(startTimeTestSetup) - l.Info(). - Str("END_TIME", endTimeTestSetup.String()). - Str("Duration", testSetupDuration.String()). - Msg("Test setup ended") - - ts, err := sendSlackNotification("Started :white_check_mark:", l, &loadedTestConfig, testEnvironment.Cfg.Namespace, strconv.Itoa(*loadedTestConfig.Automation.General.NumberOfNodes), - strconv.FormatInt(startTimeTestSetup.UnixMilli(), 10), "now", - []slack.Block{extraBlockWithText("\bTest Config\b\n```" + testConfig + "```")}, slack.MsgOptionBlocks()) - if err != nil { - l.Error().Err(err).Msg("Error sending slack notification") - } - - g, err := wasp.NewGenerator(&wasp.Config{ - T: t, - LoadType: wasp.RPS, - GenName: "log_trigger_gen", - CallTimeout: time.Minute * 3, - Schedule: wasp.Plain( - 1, - loadDuration, - ), - Gun: NewLogTriggerUser( - l, - configs, - a.ChainClient, - multicallAddress.Hex(), - ), - CallResultBufLen: 1000, - }) - p.Add(g, err) - - startTimeTestEx := time.Now() - l.Info().Str("START_TIME", startTimeTestEx.String()).Msg("Test execution started") - - l.Info().Msg("Starting load generators") - _, err = p.Run(true) - require.NoError(t, err, "Error running load generators") - - l.Info().Msg("Finished load generators") - l.Info().Str("STOP_WAIT_TIME", StopWaitTime.String()).Msg("Waiting for upkeeps to be performed") - time.Sleep(StopWaitTime) - l.Info().Msg("Finished waiting 60s for upkeeps to be performed") - endTimeTestEx := time.Now() - testExDuration := endTimeTestEx.Sub(startTimeTestEx) - l.Info(). - Str("END_TIME", endTimeTestEx.String()). - Str("Duration", testExDuration.String()). - Msg("Test execution ended") - - l.Info().Str("Duration", testExDuration.String()).Msg("Test Execution Duration") - endBlock, err := chainClient.Client.BlockNumber(ctx) - require.NoError(t, err, "Error getting latest block number") - l.Info().Uint64("Starting Block", startBlock).Uint64("Ending Block", endBlock).Msg("Test Block Range") - - startTimeTestReport := time.Now() - l.Info().Str("START_TIME", startTimeTestReport.String()).Msg("Test reporting started") - - for _, gen := range p.Generators { - if len(gen.Errors()) != 0 { - l.Error().Strs("Errors", gen.Errors()).Msg("Error in load gen") - t.Fail() - } - } - - upkeepDelaysFast := make([][]int64, 0) - upkeepDelaysRecovery := make([][]int64, 0) - - var batchSize uint64 = 500 - - if endBlock-startBlock < batchSize { - batchSize = endBlock - startBlock - } - - for _, consumerContract := range consumerContracts { - var ( - logs []types.Log - address = common.HexToAddress(consumerContract.Address()) - timeout = 5 * time.Second - ) - for fromBlock := startBlock; fromBlock < endBlock; fromBlock += batchSize + 1 { - filterQuery := geth.FilterQuery{ - Addresses: []common.Address{address}, - FromBlock: big.NewInt(0).SetUint64(fromBlock), - ToBlock: big.NewInt(0).SetUint64(fromBlock + batchSize), - Topics: [][]common.Hash{{consumerABI.Events["PerformingUpkeep"].ID}}, - } - err = errors.New("initial error") // to ensure our for loop runs at least once - for err != nil { - var ( - logsInBatch []types.Log - ) - ctx2, cancel := context.WithTimeout(ctx, timeout) - logsInBatch, err = a.ChainClient.Client.FilterLogs(ctx2, filterQuery) - cancel() - if err != nil { - l.Error().Err(err). - Interface("FilterQuery", filterQuery). - Str("Contract Address", consumerContract.Address()). - Str("Timeout", timeout.String()). - Msg("Error getting consumer contract logs") - timeout = time.Duration(math.Min(float64(timeout)*2, float64(2*time.Minute))) - continue - } - l.Debug(). - Interface("FilterQuery", filterQuery). - Str("Contract Address", consumerContract.Address()). - Str("Timeout", timeout.String()). - Int("Number of Logs", len(logsInBatch)). - Msg("Collected consumer contract logs") - logs = append(logs, logsInBatch...) - } - } - - if len(logs) > 0 { - delayFast := make([]int64, 0) - delayRecovery := make([]int64, 0) - for _, log := range logs { - eventDetails, err := consumerABI.EventByID(log.Topics[0]) - require.NoError(t, err, "Error getting event details") - consumer, err := simple_log_upkeep_counter_wrapper.NewSimpleLogUpkeepCounter( - address, a.ChainClient.Client, - ) - require.NoError(t, err, "Error getting consumer contract") - if eventDetails.Name == "PerformingUpkeep" { - parsedLog, err := consumer.ParsePerformingUpkeep(log) - require.NoError(t, err, "Error parsing log") - if parsedLog.IsRecovered { - delayRecovery = append(delayRecovery, parsedLog.TimeToPerform.Int64()) - } else { - delayFast = append(delayFast, parsedLog.TimeToPerform.Int64()) - } - } - } - upkeepDelaysFast = append(upkeepDelaysFast, delayFast) - upkeepDelaysRecovery = append(upkeepDelaysRecovery, delayRecovery) - } - } - - for _, triggerContract := range triggerContracts { - var ( - logs []types.Log - address = triggerContract.Address() - timeout = 5 * time.Second - ) - for fromBlock := startBlock; fromBlock < endBlock; fromBlock += batchSize + 1 { - filterQuery := geth.FilterQuery{ - Addresses: []common.Address{address}, - FromBlock: big.NewInt(0).SetUint64(fromBlock), - ToBlock: big.NewInt(0).SetUint64(fromBlock + batchSize), - Topics: [][]common.Hash{{emitterABI.Events["Log4"].ID}, {bytes1}, {bytes1}}, - } - err = errors.New("initial error") // to ensure our for loop runs at least once - for err != nil { - var ( - logsInBatch []types.Log - ) - ctx2, cancel := context.WithTimeout(ctx, timeout) - logsInBatch, err = chainClient.Client.FilterLogs(ctx2, filterQuery) - cancel() - if err != nil { - l.Error().Err(err). - Interface("FilterQuery", filterQuery). - Str("Contract Address", address.Hex()). - Str("Timeout", timeout.String()). - Msg("Error getting trigger contract logs") - timeout = time.Duration(math.Min(float64(timeout)*2, float64(2*time.Minute))) - continue - } - l.Debug(). - Interface("FilterQuery", filterQuery). - Str("Contract Address", address.Hex()). - Str("Timeout", timeout.String()). - Int("Number of Logs", len(logsInBatch)). - Msg("Collected trigger contract logs") - logs = append(logs, logsInBatch...) - } - } - numberOfEventsEmitted += int64(len(logs)) - } - - l.Info().Int64("Number of Events Emitted", numberOfEventsEmitted).Msg("Number of Events Emitted") - - l.Info(). - Interface("Upkeep Delays Fast", upkeepDelaysFast). - Interface("Upkeep Delays Recovered", upkeepDelaysRecovery). - Msg("Upkeep Delays") - - var allUpkeepDelays []int64 - var allUpkeepDelaysFast []int64 - var allUpkeepDelaysRecovery []int64 - - for _, upkeepDelay := range upkeepDelaysFast { - allUpkeepDelays = append(allUpkeepDelays, upkeepDelay...) - allUpkeepDelaysFast = append(allUpkeepDelaysFast, upkeepDelay...) - } - - for _, upkeepDelay := range upkeepDelaysRecovery { - allUpkeepDelays = append(allUpkeepDelays, upkeepDelay...) - allUpkeepDelaysRecovery = append(allUpkeepDelaysRecovery, upkeepDelay...) - } - - avgF, medianF, ninetyPctF, ninetyNinePctF, maximumF := testreporters.IntListStats(allUpkeepDelaysFast) - avgR, medianR, ninetyPctR, ninetyNinePctR, maximumR := testreporters.IntListStats(allUpkeepDelaysRecovery) - eventsMissed := (numberOfEventsEmitted) - int64(len(allUpkeepDelays)) - percentMissed := float64(eventsMissed) / float64(numberOfEventsEmitted) * 100 - l.Info(). - Float64("Average", avgF).Int64("Median", medianF). - Int64("90th Percentile", ninetyPctF).Int64("99th Percentile", ninetyNinePctF). - Int64("Max", maximumF).Msg("Upkeep Delays Fast Execution in seconds") - l.Info(). - Float64("Average", avgR).Int64("Median", medianR). - Int64("90th Percentile", ninetyPctR).Int64("99th Percentile", ninetyNinePctR). - Int64("Max", maximumR).Msg("Upkeep Delays Recovery Execution in seconds") - l.Info(). - Int("Total Perform Count", len(allUpkeepDelays)). - Int("Perform Count Fast Execution", len(allUpkeepDelaysFast)). - Int("Perform Count Recovery Execution", len(allUpkeepDelaysRecovery)). - Int64("Total Events Emitted", numberOfEventsEmitted). - Int64("Total Events Missed", eventsMissed). - Float64("Percent Missed", percentMissed). - Msg("Test completed") - - testReportFormat := `Upkeep Delays in seconds - Fast Execution -Average: %f -Median: %d -90th Percentile: %d -99th Percentile: %d -Max: %d - -Upkeep Delays in seconds - Recovery Execution -Average: %f -Median: %d -90th Percentile: %d -99th Percentile: %d -Max: %d - -Total Perform Count: %d -Perform Count Fast Execution: %d -Perform Count Recovery Execution: %d -Total Expected Log Triggering Events: %d -Total Log Triggering Events Emitted: %d -Total Events Missed: %d -Percent Missed: %f -Test Duration: %s` - - endTimeTestReport := time.Now() - testReDuration := endTimeTestReport.Sub(startTimeTestReport) - l.Info(). - Str("END_TIME", endTimeTestReport.String()). - Str("Duration", testReDuration.String()). - Msg("Test reporting ended") - - numberOfExpectedEvents := numberOfEventsEmittedPerSec * int64(loadDuration.Seconds()) - if numberOfEventsEmitted < numberOfExpectedEvents { - l.Error().Msg("Number of events emitted is less than expected") - t.Fail() - } - testReport := fmt.Sprintf(testReportFormat, avgF, medianF, ninetyPctF, ninetyNinePctF, maximumF, - avgR, medianR, ninetyPctR, ninetyNinePctR, maximumR, len(allUpkeepDelays), len(allUpkeepDelaysFast), - len(allUpkeepDelaysRecovery), numberOfExpectedEvents, numberOfEventsEmitted, eventsMissed, percentMissed, testExDuration.String()) - l.Info().Str("Test Report", testReport).Msg("Test Report prepared") - - testStatus := "Failed :x:" - if !t.Failed() { - testStatus = "Finished :white_check_mark:" - } - - _, err = sendSlackNotification(testStatus, l, &loadedTestConfig, testEnvironment.Cfg.Namespace, strconv.Itoa(*loadedTestConfig.Automation.General.NumberOfNodes), - strconv.FormatInt(startTimeTestSetup.UnixMilli(), 10), strconv.FormatInt(time.Now().UnixMilli(), 10), - []slack.Block{extraBlockWithText("\bTest Report\b\n```" + testReport + "```")}, slack.MsgOptionTS(ts)) - if err != nil { - l.Error().Err(err).Msg("Error sending slack notification") - } - - t.Cleanup(func() { - if err = actions.TeardownRemoteSuite(t, chainClient, testEnvironment.Cfg.Namespace, chainlinkNodes, nil, &loadedTestConfig); err != nil { - l.Error().Err(err).Msg("Error when tearing down remote suite") - testEnvironment.Cfg.TTL += time.Hour * 48 - err := testEnvironment.Run() - if err != nil { - l.Error().Err(err).Msg("Error increasing TTL of namespace") - } - } else if chainClient.Cfg.IsSimulatedNetwork() && *loadedTestConfig.Automation.General.RemoveNamespace { - err := testEnvironment.Client.RemoveNamespace(testEnvironment.Cfg.Namespace) - if err != nil { - l.Error().Err(err).Msg("Error removing namespace") - } - } - }) -} diff --git a/integration-tests/load/automationv2_1/gun.go b/integration-tests/load/automationv2_1/gun.go deleted file mode 100644 index 330b7cc7f73..00000000000 --- a/integration-tests/load/automationv2_1/gun.go +++ /dev/null @@ -1,123 +0,0 @@ -package automation - -import ( - "math/big" - "sync" - - "github.com/rs/zerolog" - - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - - "github.com/smartcontractkit/chainlink-testing-framework/seth" - - "github.com/smartcontractkit/chainlink-evm/gethwrappers/shared/generated/initial/log_emitter" - - "github.com/smartcontractkit/chainlink/integration-tests/contracts" -) - -type LogTriggerConfig struct { - Address string - NumberOfEvents int64 - NumberOfSpamMatchingEvents int64 - NumberOfSpamNonMatchingEvents int64 -} - -type LogTriggerGun struct { - data [][]byte - addresses []string - multiCallAddress string - client *seth.Client - logger zerolog.Logger -} - -func generateCallData(int1 int64, int2 int64, count int64) []byte { - abi, err := log_emitter.LogEmitterMetaData.GetAbi() - if err != nil { - panic(err) - } - data, err := abi.Pack("EmitLog4", big.NewInt(int1), big.NewInt(int2), big.NewInt(count)) - if err != nil { - panic(err) - } - return data -} - -func NewLogTriggerUser( - logger zerolog.Logger, - triggerConfigs []LogTriggerConfig, - client *seth.Client, - multicallAddress string, -) *LogTriggerGun { - var data [][]byte - var addresses []string - - for _, c := range triggerConfigs { - if c.NumberOfEvents > 0 { - d := generateCallData(1, 1, c.NumberOfEvents) - data = append(data, d) - addresses = append(addresses, c.Address) - } - if c.NumberOfSpamMatchingEvents > 0 { - d := generateCallData(1, 2, c.NumberOfSpamMatchingEvents) - data = append(data, d) - addresses = append(addresses, c.Address) - } - if c.NumberOfSpamNonMatchingEvents > 0 { - d := generateCallData(2, 2, c.NumberOfSpamNonMatchingEvents) - data = append(data, d) - addresses = append(addresses, c.Address) - } - } - - return &LogTriggerGun{ - addresses: addresses, - data: data, - logger: logger, - multiCallAddress: multicallAddress, - client: client, - } -} - -func (m *LogTriggerGun) Call(_ *wasp.Generator) *wasp.Response { - var wg sync.WaitGroup - var dividedData [][][]byte - d := m.data - chunkSize := 100 - for i := 0; i < len(d); i += chunkSize { - end := min(i+chunkSize, len(d)) - dividedData = append(dividedData, d[i:end]) - } - - resultCh := make(chan *wasp.Response, len(dividedData)) - - for _, a := range dividedData { - wg.Add(1) - go func(a [][]byte, m *LogTriggerGun) { - defer wg.Done() - - _, err := contracts.MultiCallLogTriggerLoadGen(m.client, m.multiCallAddress, m.addresses, a) - if err != nil { - m.logger.Error().Err(err).Msg("Error calling MultiCallLogTriggerLoadGen") - resultCh <- &wasp.Response{Error: err.Error(), Failed: true} - return - } - resultCh <- &wasp.Response{} - }(a, m) - } - wg.Wait() - close(resultCh) - - r := &wasp.Response{} - for result := range resultCh { - if result.Failed { - r.Failed = true - if r.Error != "" { - r.Error += "; " + result.Error - } else { - r.Error = result.Error - } - } - } - - return r -} diff --git a/integration-tests/load/automationv2_1/helpers.go b/integration-tests/load/automationv2_1/helpers.go deleted file mode 100644 index b8b164871b3..00000000000 --- a/integration-tests/load/automationv2_1/helpers.go +++ /dev/null @@ -1,207 +0,0 @@ -package automation - -import ( - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/pkg/errors" - "github.com/rs/zerolog" - "github.com/slack-go/slack" - - "github.com/smartcontractkit/chainlink-testing-framework/seth" - - ctf_concurrency "github.com/smartcontractkit/chainlink-testing-framework/lib/concurrency" - reportModel "github.com/smartcontractkit/chainlink-testing-framework/lib/testreporters" - "github.com/smartcontractkit/chainlink/integration-tests/actions" - "github.com/smartcontractkit/chainlink/integration-tests/contracts" - tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" - aconfig "github.com/smartcontractkit/chainlink/integration-tests/testconfig/automation" -) - -func extraBlockWithText(text string) slack.Block { - return slack.NewSectionBlock(slack.NewTextBlockObject( - "mrkdwn", text, false, false), nil, nil) -} - -func sendSlackNotification(header string, l zerolog.Logger, config *tc.TestConfig, namespace string, numberOfNodes, - startingTime string, endingTime string, extraBlocks []slack.Block, msgOption slack.MsgOption) (string, error) { - slackClient := slack.New(reportModel.SlackAPIKey) - - headerText := ":chainlink-keepers: Automation Load Test " + header - - grafanaURL, err := config.GetGrafanaBaseURL() - if err != nil { - return "", err - } - - dashboardURL, err := config.GetGrafanaDashboardURL() - if err != nil { - return "", err - } - - formattedDashboardURL := fmt.Sprintf("%s%s?orgId=1&from=%s&to=%s&var-namespace=%s&var-number_of_nodes=%s", grafanaURL, dashboardURL, startingTime, endingTime, namespace, numberOfNodes) - l.Info().Str("Dashboard", formattedDashboardURL).Msg("Dashboard URL") - - var notificationBlocks []slack.Block - - notificationBlocks = append(notificationBlocks, - slack.NewHeaderBlock(slack.NewTextBlockObject("plain_text", headerText, true, false))) - notificationBlocks = append(notificationBlocks, - slack.NewContextBlock("context_block", slack.NewTextBlockObject("plain_text", namespace, false, false))) - notificationBlocks = append(notificationBlocks, slack.NewDividerBlock()) - if *config.Pyroscope.Enabled { - pyroscopeServer := *config.Pyroscope.ServerUrl - pyroscopeEnvironment := *config.Pyroscope.Environment - - formattedPyroscopeURL := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%s&to=%s", pyroscopeServer, pyroscopeEnvironment, startingTime, endingTime) - l.Info().Str("Pyroscope", formattedPyroscopeURL).Msg("Dashboard URL") - notificationBlocks = append(notificationBlocks, slack.NewSectionBlock(slack.NewTextBlockObject("mrkdwn", - fmt.Sprintf("<%s|Pyroscope>", - formattedPyroscopeURL), false, true), nil, nil)) - } - notificationBlocks = append(notificationBlocks, slack.NewSectionBlock(slack.NewTextBlockObject("mrkdwn", - fmt.Sprintf("<%s|Test Dashboard> \nNotifying <@%s>", - formattedDashboardURL, reportModel.SlackUserID), false, true), nil, nil)) - - if len(extraBlocks) > 0 { - notificationBlocks = append(notificationBlocks, extraBlocks...) - } - - ts, err := reportModel.SendSlackMessage(slackClient, slack.MsgOptionBlocks(notificationBlocks...), msgOption) - l.Info().Str("ts", ts).Msg("Sent Slack Message") - return ts, err -} - -type DeploymentData struct { - ConsumerContracts []contracts.KeeperConsumer - TriggerContracts []contracts.LogEmitter - TriggerAddresses []common.Address - LoadConfigs []aconfig.Load -} - -type deployedContractData struct { - consumerContract contracts.KeeperConsumer - triggerContract contracts.LogEmitter - triggerAddress common.Address - loadConfig aconfig.Load -} - -func (d deployedContractData) GetResult() deployedContractData { - return d -} - -type task struct { - deployTrigger bool -} - -func deployConsumerAndTriggerContracts(l zerolog.Logger, loadConfig aconfig.Load, chainClient *seth.Client, multicallAddress common.Address, maxConcurrency int, automationDefaultLinkFunds *big.Int, linkToken contracts.LinkToken) (DeploymentData, error) { - data := DeploymentData{} - - concurrency, err := actions.GetAndAssertCorrectConcurrency(chainClient, 1) - if err != nil { - return DeploymentData{}, err - } - - if concurrency > maxConcurrency { - concurrency = maxConcurrency - l.Debug(). - Msgf("Concurrency is higher than max concurrency, setting concurrency to %d", concurrency) - } - - l.Debug(). - Int("Number of Upkeeps", *loadConfig.NumberOfUpkeeps). - Int("Concurrency", concurrency). - Msg("Deployment parallelisation info") - - tasks := []task{} - for i := 0; i < *loadConfig.NumberOfUpkeeps; i++ { - if *loadConfig.SharedTrigger { - if i == 0 { - tasks = append(tasks, task{deployTrigger: true}) - } else { - tasks = append(tasks, task{deployTrigger: false}) - } - continue - } - tasks = append(tasks, task{deployTrigger: true}) - } - - var deployContractFn = func(deployedCh chan deployedContractData, errorCh chan error, keyNum int, task task) { - data := deployedContractData{} - consumerContract, err := contracts.DeployAutomationSimpleLogTriggerConsumerFromKey(chainClient, *loadConfig.IsStreamsLookup, keyNum) - if err != nil { - errorCh <- errors.Wrapf(err, "Error deploying simple log trigger contract") - return - } - - data.consumerContract = consumerContract - - loadCfg := aconfig.Load{ - NumberOfEvents: loadConfig.NumberOfEvents, - NumberOfSpamMatchingEvents: loadConfig.NumberOfSpamMatchingEvents, - NumberOfSpamNonMatchingEvents: loadConfig.NumberOfSpamNonMatchingEvents, - CheckBurnAmount: loadConfig.CheckBurnAmount, - PerformBurnAmount: loadConfig.PerformBurnAmount, - UpkeepGasLimit: loadConfig.UpkeepGasLimit, - SharedTrigger: loadConfig.SharedTrigger, - Feeds: []string{}, - } - - if *loadConfig.IsStreamsLookup { - loadCfg.Feeds = loadConfig.Feeds - } - - data.loadConfig = loadCfg - - if !task.deployTrigger { - deployedCh <- data - return - } - - triggerContract, err := contracts.DeployLogEmitterContractFromKey(l, chainClient, keyNum) - if err != nil { - errorCh <- errors.Wrapf(err, "Error deploying log emitter contract") - return - } - - data.triggerContract = triggerContract - data.triggerAddress = triggerContract.Address() - deployedCh <- data - } - - executor := ctf_concurrency.NewConcurrentExecutor[deployedContractData, deployedContractData, task](l) - results, err := executor.Execute(concurrency, tasks, deployContractFn) - if err != nil { - return DeploymentData{}, err - } - - for _, result := range results { - if result.GetResult().triggerContract != nil { - data.TriggerContracts = append(data.TriggerContracts, result.GetResult().triggerContract) - data.TriggerAddresses = append(data.TriggerAddresses, result.GetResult().triggerAddress) - } - data.ConsumerContracts = append(data.ConsumerContracts, result.GetResult().consumerContract) - data.LoadConfigs = append(data.LoadConfigs, result.GetResult().loadConfig) - } - - // if there's more than 1 upkeep and it's a shared trigger, then we should use only the first address in triggerAddresses - // as triggerAddresses array - if *loadConfig.SharedTrigger { - if len(data.TriggerAddresses) == 0 { - return DeploymentData{}, errors.New("No trigger addresses found") - } - triggerAddress := data.TriggerAddresses[0] - data.TriggerAddresses = make([]common.Address, 0) - for i := 0; i < *loadConfig.NumberOfUpkeeps; i++ { - data.TriggerAddresses = append(data.TriggerAddresses, triggerAddress) - } - } - - sendErr := actions.SendLinkFundsToDeploymentAddresses(chainClient, concurrency, *loadConfig.NumberOfUpkeeps, *loadConfig.NumberOfUpkeeps/concurrency, multicallAddress, automationDefaultLinkFunds, linkToken) - if sendErr != nil { - return DeploymentData{}, sendErr - } - - return data, nil -} diff --git a/integration-tests/load/vrfv2/README.md b/integration-tests/load/vrfv2/README.md deleted file mode 100644 index 4b00d4ad0d2..00000000000 --- a/integration-tests/load/vrfv2/README.md +++ /dev/null @@ -1,22 +0,0 @@ -### VRFv2 Load tests - -## Usage -``` -export LOKI_TOKEN=... -export LOKI_URL=... - -go test -v -run TestVRFV2Load/vrfv2_soak_test -``` - -### Dashboards - -Deploying dashboard: -``` -export GRAFANA_URL=... -export GRAFANA_TOKEN=... -export DATA_SOURCE_NAME=grafanacloud-logs -export DASHBOARD_FOLDER=LoadTests -export DASHBOARD_NAME=${JobTypeName, for example WaspVRFv2} - -go run dashboard.go -``` \ No newline at end of file diff --git a/integration-tests/load/vrfv2/gun.go b/integration-tests/load/vrfv2/gun.go deleted file mode 100644 index 691926ff6a8..00000000000 --- a/integration-tests/load/vrfv2/gun.go +++ /dev/null @@ -1,140 +0,0 @@ -package loadvrfv2 - -import ( - "math/rand" - - "github.com/rs/zerolog" - "golang.org/x/exp/constraints" - - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - - "github.com/smartcontractkit/chainlink-testing-framework/seth" - - seth_utils "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/seth" - - vrfcommon "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/common" - "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/vrfv2" - vrfv2_config "github.com/smartcontractkit/chainlink/integration-tests/testconfig/vrfv2" -) - -type BHSTestGun struct { - contracts *vrfcommon.VRFContracts - subIDs []uint64 - keyHash [32]byte - testConfig *vrfv2_config.Config - logger zerolog.Logger - sethClient *seth.Client -} - -func NewBHSTestGun( - contracts *vrfcommon.VRFContracts, - keyHash [32]byte, - subIDs []uint64, - testConfig *vrfv2_config.Config, - logger zerolog.Logger, - sethClient *seth.Client, -) *BHSTestGun { - return &BHSTestGun{ - contracts: contracts, - subIDs: subIDs, - keyHash: keyHash, - testConfig: testConfig, - logger: logger, - sethClient: sethClient, - } -} - -// Call implements example gun call, assertions on response bodies should be done here -func (m *BHSTestGun) Call(_ *wasp.Generator) *wasp.Response { - _, err := vrfv2.RequestRandomness( - m.logger, - m.contracts.VRFV2Consumers[0], - m.contracts.CoordinatorV2, - m.subIDs[0], - &vrfcommon.VRFKeyData{KeyHash: m.keyHash}, - *m.testConfig.General.MinimumConfirmations, - *m.testConfig.General.CallbackGasLimit, - *m.testConfig.General.NumberOfWords, - *m.testConfig.General.RandomnessRequestCountPerRequest, - *m.testConfig.General.RandomnessRequestCountPerRequestDeviation, - seth_utils.AvailableSethKeyNum(m.sethClient), - ) - // todo - might need to store randRequestBlockNumber and blockhash to verify that it was stored in BHS contract at the end of the test - if err != nil { - return &wasp.Response{Error: err.Error(), Failed: true} - } - return &wasp.Response{} -} - -type SingleHashGun struct { - contracts *vrfcommon.VRFContracts - keyHash [32]byte - subIDs []uint64 - testConfig *vrfv2_config.Config - logger zerolog.Logger - sethClient *seth.Client -} - -func NewSingleHashGun( - contracts *vrfcommon.VRFContracts, - keyHash [32]byte, - subIDs []uint64, - testConfig *vrfv2_config.Config, - logger zerolog.Logger, - sethClient *seth.Client, -) *SingleHashGun { - return &SingleHashGun{ - contracts: contracts, - keyHash: keyHash, - subIDs: subIDs, - testConfig: testConfig, - logger: logger, - sethClient: sethClient, - } -} - -// Call implements example gun call, assertions on response bodies should be done here -func (m *SingleHashGun) Call(_ *wasp.Generator) *wasp.Response { - // todo - should work with multiple consumers and consumers having different keyhashes and wallets - - vrfv2Config := m.testConfig.General - // randomly increase/decrease randomness request count per TX - randomnessRequestCountPerRequest := deviateValue(*vrfv2Config.RandomnessRequestCountPerRequest, *vrfv2Config.RandomnessRequestCountPerRequestDeviation) - _, _, err := vrfv2.RequestRandomnessAndWaitForFulfillment( - m.logger, - // the same consumer is used for all requests and in all subs - m.contracts.VRFV2Consumers[0], - m.contracts.CoordinatorV2, - // randomly pick a subID from pool of subIDs - m.subIDs[randInRange(0, len(m.subIDs)-1)], - &vrfcommon.VRFKeyData{KeyHash: m.keyHash}, - *vrfv2Config.MinimumConfirmations, - *vrfv2Config.CallbackGasLimit, - *vrfv2Config.NumberOfWords, - randomnessRequestCountPerRequest, - *vrfv2Config.RandomnessRequestCountPerRequestDeviation, - vrfv2Config.RandomWordsFulfilledEventTimeout.Duration, - seth_utils.AvailableSethKeyNum(m.sethClient), - ) - if err != nil { - return &wasp.Response{Error: err.Error(), Failed: true} - } - return &wasp.Response{} -} - -func deviateValue(requestCountPerTX uint16, deviation uint16) uint16 { - if randBool() && requestCountPerTX > deviation { - requestCountPerTX -= randInRange(0, deviation) - } else { - requestCountPerTX += randInRange(0, deviation) - } - return requestCountPerTX -} - -func randBool() bool { - return rand.Intn(2) == 1 -} - -func randInRange[I constraints.Integer](lower, upper I) I { - return I(rand.Intn(int(upper-lower)+1)) + lower -} diff --git a/integration-tests/load/vrfv2/onchain_monitoring.go b/integration-tests/load/vrfv2/onchain_monitoring.go deleted file mode 100644 index ed8f7c74852..00000000000 --- a/integration-tests/load/vrfv2/onchain_monitoring.go +++ /dev/null @@ -1,56 +0,0 @@ -package loadvrfv2 - -import ( - "context" - "maps" - "testing" - "time" - - "github.com/rs/zerolog/log" - - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - - "github.com/smartcontractkit/chainlink/integration-tests/contracts" -) - -/* Monitors on-chain stats of LoadConsumer and pushes them to Loki every second */ - -const ( - LokiTypeLabel = "vrfv2_contracts_load_summary" - ErrMetrics = "failed to get VRFv2 load test metrics" - ErrLokiClient = "failed to create Loki client for monitoring" - ErrLokiPush = "failed to push monitoring metrics to Loki" -) - -func MonitorLoadStats(ctx context.Context, lc *wasp.LokiClient, consumer contracts.VRFv2LoadTestConsumer, labels map[string]string) { - go func() { - for { - time.Sleep(1 * time.Second) - metrics := GetLoadTestMetrics(ctx, consumer) - SendMetricsToLoki(metrics, lc, labels) - } - }() -} - -func UpdateLabels(labels map[string]string, t *testing.T) map[string]string { - updatedLabels := make(map[string]string) - maps.Copy(updatedLabels, labels) - updatedLabels["type"] = LokiTypeLabel - updatedLabels["go_test_name"] = t.Name() - updatedLabels["gen_name"] = "performance" - return updatedLabels -} - -func SendMetricsToLoki(metrics *contracts.VRFLoadTestMetrics, lc *wasp.LokiClient, updatedLabels map[string]string) { - if err := lc.HandleStruct(wasp.LabelsMapToModel(updatedLabels), time.Now(), metrics); err != nil { - log.Error().Err(err).Msg(ErrLokiPush) - } -} - -func GetLoadTestMetrics(ctx context.Context, consumer contracts.VRFv2LoadTestConsumer) *contracts.VRFLoadTestMetrics { - metrics, err := consumer.GetLoadTestMetrics(ctx) - if err != nil { - log.Error().Err(err).Msg(ErrMetrics) - } - return metrics -} diff --git a/integration-tests/load/vrfv2/vrfv2_test.go b/integration-tests/load/vrfv2/vrfv2_test.go deleted file mode 100644 index 265829c80ff..00000000000 --- a/integration-tests/load/vrfv2/vrfv2_test.go +++ /dev/null @@ -1,383 +0,0 @@ -package loadvrfv2 - -import ( - "math/big" - "strconv" - "sync" - "testing" - "time" - - "github.com/smartcontractkit/chainlink-testing-framework/seth" - - "github.com/rs/zerolog/log" - - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - - "github.com/smartcontractkit/chainlink/integration-tests/actions" - vrfcommon "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/common" - "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/vrfv2" - "github.com/smartcontractkit/chainlink/integration-tests/contracts" - "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" - "github.com/smartcontractkit/chainlink/integration-tests/testreporters" - - "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" - "github.com/smartcontractkit/chainlink-testing-framework/lib/networks" - "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/ptr" - "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" - - "github.com/stretchr/testify/require" - - tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" -) - -var ( - labels = map[string]string{ - "branch": "vrfv2_healthcheck", - "commit": "vrfv2_healthcheck", - } -) - -func TestVRFV2Performance(t *testing.T) { - var ( - testEnv *test_env.CLClusterTestEnv - vrfContracts *vrfcommon.VRFContracts - subIDsForCancellingAfterTest []uint64 - vrfKey *vrfcommon.VRFKeyData - sethClient *seth.Client - ) - l := logging.GetTestLogger(t) - testType, err := tc.GetConfigurationNameFromEnv() - require.NoError(t, err) - testConfig, err := tc.GetChainAndTestTypeSpecificConfig(testType, tc.VRFv2) - require.NoError(t, err) - cfgl := testConfig.Logging.Loki - - vrfv2Config := testConfig.VRFv2 - testReporter := &testreporters.VRFV2TestReporter{} - - lokiConfig := wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken) - lc, err := wasp.NewLokiClient(lokiConfig) - if err != nil { - l.Error().Err(err).Msg(ErrLokiClient) - return - } - network := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0] - chainID := network.ChainID - updatedLabels := UpdateLabels(labels, t) - - l.Info(). - Str("Test Type", testType). - Str("Test Duration", vrfv2Config.Performance.TestDuration.Duration.Truncate(time.Second).String()). - Int64("RPS", *vrfv2Config.Performance.RPS). - Str("RateLimitUnitDuration", vrfv2Config.Performance.RateLimitUnitDuration.String()). - Uint16("RandomnessRequestCountPerRequest", *vrfv2Config.General.RandomnessRequestCountPerRequest). - Uint16("RandomnessRequestCountPerRequestDeviation", *vrfv2Config.General.RandomnessRequestCountPerRequestDeviation). - Bool("UseExistingEnv", *vrfv2Config.General.UseExistingEnv). - Msg("Performance Test Configuration") - - cleanupFn := func() { - teardown(t, vrfContracts.VRFV2Consumers[0], lc, updatedLabels, testReporter, testType, &testConfig) - - if sethClient.Cfg.IsSimulatedNetwork() { - l.Info(). - Str("Network Name", sethClient.Cfg.Network.Name). - Msg("Network is a simulated network. Skipping fund return for Coordinator Subscriptions.") - } else if *vrfv2Config.General.CancelSubsAfterTestRun { - // wait for all txs to be mined in order to avoid nonce issues - time.Sleep(10 * time.Second) - // cancel subs and return funds to sub owner - vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, sethClient.MustGetRootKeyAddress().Hex(), subIDsForCancellingAfterTest, l) - } - if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { - l.Error().Err(err).Msg("Error cleaning up test environment") - } - } - } - vrfEnvConfig := vrfcommon.VRFEnvConfig{ - TestConfig: testConfig, - ChainID: chainID, - CleanupFn: cleanupFn, - } - newEnvConfig := vrfcommon.NewEnvConfig{ - NodesToCreate: []vrfcommon.VRFNodeType{vrfcommon.VRF}, - NumberOfTxKeysToCreate: *vrfv2Config.General.NumberOfSendingKeysToCreate, - UseVRFOwner: true, - UseTestCoordinator: true, - ChainlinkNodeLogScannerSettings: test_env.DefaultChainlinkNodeLogScannerSettings, - } - testEnv, vrfContracts, vrfKey, _, sethClient, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, vrfEnvConfig, newEnvConfig, l) - require.NoError(t, err, "error setting up VRFV2 universe") - - var consumers []contracts.VRFv2LoadTestConsumer - subIDs, consumers, err := vrfv2.SetupSubsAndConsumersForExistingEnv( - sethClient, - vrfContracts.CoordinatorV2, - vrfContracts.LinkToken, - 1, - *vrfv2Config.General.NumberOfSubToCreate, - testConfig, - l, - ) - require.NoError(t, err, "error setting up new consumers and subs") - for _, subID := range subIDs { - subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID) - require.NoError(t, err, "error getting subscription information for subscription %d", subID) - vrfcommon.LogSubDetails(l, subscription, strconv.FormatUint(subID, 10), vrfContracts.CoordinatorV2) - } - subIDsForCancellingAfterTest = subIDs - l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test") - - vrfContracts.VRFV2Consumers = consumers - - // is our "job" stable at all, no memory leaks, no flaking performance under some RPS? - t.Run("vrfv2 performance test", func(t *testing.T) { - require.Len(t, vrfContracts.VRFV2Consumers, 1, "only one consumer should be created for Load Test") - err = vrfContracts.VRFV2Consumers[0].ResetMetrics() - require.NoError(t, err) - MonitorLoadStats(testcontext.Get(t), lc, vrfContracts.VRFV2Consumers[0], updatedLabels) - - singleFeedConfig := &wasp.Config{ - T: t, - LoadType: wasp.RPS, - GenName: "gun", - RateLimitUnitDuration: vrfv2Config.Performance.RateLimitUnitDuration.Duration, - Gun: NewSingleHashGun( - vrfContracts, - vrfKey.KeyHash, - subIDs, - vrfv2Config, - l, - sethClient, - ), - Labels: labels, - LokiConfig: lokiConfig, - CallTimeout: 2 * time.Minute, - } - - singleFeedConfig.Schedule = wasp.Plain( - *vrfv2Config.Performance.RPS, - vrfv2Config.Performance.TestDuration.Duration, - ) - _, err = wasp.NewProfile(). - Add(wasp.NewGenerator(singleFeedConfig)). - Run(true) - require.NoError(t, err) - - var wg sync.WaitGroup - wg.Add(1) - // todo - timeout should be configurable depending on the perf test type - requestCount, fulfilmentCount, err := vrfcommon.WaitForRequestCountEqualToFulfilmentCount(testcontext.Get(t), vrfContracts.VRFV2Consumers[0], 2*time.Minute, &wg) - require.NoError(t, err) - wg.Wait() - - l.Info(). - Interface("Request Count", requestCount). - Interface("Fulfilment Count", fulfilmentCount). - Msg("Final Request/Fulfilment Stats") - }) -} - -func TestVRFV2BHSPerformance(t *testing.T) { - var ( - testEnv *test_env.CLClusterTestEnv - vrfContracts *vrfcommon.VRFContracts - subIDsForCancellingAfterTest []uint64 - vrfKey *vrfcommon.VRFKeyData - sethClient *seth.Client - ) - l := logging.GetTestLogger(t) - - testType, err := tc.GetConfigurationNameFromEnv() - require.NoError(t, err) - testConfig, err := tc.GetChainAndTestTypeSpecificConfig(testType, tc.VRFv2) - require.NoError(t, err) - vrfv2Config := testConfig.VRFv2 - testReporter := &testreporters.VRFV2TestReporter{} - cfgl := testConfig.Logging.Loki - lokiConfig := wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken) - lc, err := wasp.NewLokiClient(lokiConfig) - if err != nil { - l.Error().Err(err).Msg(ErrLokiClient) - return - } - - updatedLabels := UpdateLabels(labels, t) - - l.Info(). - Str("Test Type", testType). - Str("Test Duration", vrfv2Config.Performance.TestDuration.Duration.Truncate(time.Second).String()). - Int64("RPS", *vrfv2Config.Performance.RPS). - Str("RateLimitUnitDuration", vrfv2Config.Performance.RateLimitUnitDuration.String()). - Uint16("RandomnessRequestCountPerRequest", *vrfv2Config.General.RandomnessRequestCountPerRequest). - Uint16("RandomnessRequestCountPerRequestDeviation", *vrfv2Config.General.RandomnessRequestCountPerRequestDeviation). - Bool("UseExistingEnv", *vrfv2Config.General.UseExistingEnv). - Msg("Performance Test Configuration") - - network := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0] - chainID := network.ChainID - - cleanupFn := func() { - teardown(t, vrfContracts.VRFV2Consumers[0], lc, updatedLabels, testReporter, testType, &testConfig) - if sethClient.Cfg.IsSimulatedNetwork() { - l.Info(). - Str("Network Name", sethClient.Cfg.Network.Name). - Msg("Network is a simulated network. Skipping fund return for Coordinator Subscriptions.") - } else if *vrfv2Config.General.CancelSubsAfterTestRun { - // cancel subs and return funds to sub owner - vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, sethClient.MustGetRootKeyAddress().Hex(), subIDsForCancellingAfterTest, l) - } - if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { - l.Error().Err(err).Msg("Error cleaning up test environment") - } - } - } - vrfEnvConfig := vrfcommon.VRFEnvConfig{ - TestConfig: testConfig, - ChainID: chainID, - CleanupFn: cleanupFn, - } - newEnvConfig := vrfcommon.NewEnvConfig{ - NodesToCreate: []vrfcommon.VRFNodeType{vrfcommon.VRF}, - NumberOfTxKeysToCreate: *vrfv2Config.General.NumberOfSendingKeysToCreate, - UseVRFOwner: true, - UseTestCoordinator: true, - ChainlinkNodeLogScannerSettings: test_env.DefaultChainlinkNodeLogScannerSettings, - } - testEnv, vrfContracts, vrfKey, _, sethClient, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, vrfEnvConfig, newEnvConfig, l) - require.NoError(t, err, "error setting up VRFV2 universe") - - t.Run("vrfv2 and bhs performance test", func(t *testing.T) { - configCopy := testConfig.MustCopy().(tc.TestConfig) - // Underfund Subscription - configCopy.VRFv2.General.SubscriptionFundingAmountLink = ptr.Ptr(float64(0)) - - underfundedSubIDs, consumers, err := vrfv2.SetupSubsAndConsumersForExistingEnv( - sethClient, - vrfContracts.CoordinatorV2, - vrfContracts.LinkToken, - 1, - *vrfv2Config.General.NumberOfSubToCreate, - configCopy, - l, - ) - require.NoError(t, err, "error setting up new consumers and subs") - for _, subID := range underfundedSubIDs { - subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID) - require.NoError(t, err, "error getting subscription information for subscription %d", subID) - vrfcommon.LogSubDetails(l, subscription, strconv.FormatUint(subID, 10), vrfContracts.CoordinatorV2) - } - subIDsForCancellingAfterTest = underfundedSubIDs - l.Debug().Int("Number of Subs", len(underfundedSubIDs)).Msg("Subs involved in the test") - vrfContracts.VRFV2Consumers = consumers - require.Len(t, vrfContracts.VRFV2Consumers, 1, "only one consumer should be created for Load Test") - err = vrfContracts.VRFV2Consumers[0].ResetMetrics() - require.NoError(t, err, "error resetting consumer metrics") - MonitorLoadStats(testcontext.Get(t), lc, vrfContracts.VRFV2Consumers[0], updatedLabels) - - singleFeedConfig := &wasp.Config{ - T: t, - LoadType: wasp.RPS, - GenName: "gun", - RateLimitUnitDuration: configCopy.VRFv2.Performance.BHSTestRateLimitUnitDuration.Duration, - Gun: NewBHSTestGun( - vrfContracts, - vrfKey.KeyHash, - underfundedSubIDs, - configCopy.VRFv2, - l, - sethClient, - ), - Labels: labels, - LokiConfig: lokiConfig, - CallTimeout: 2 * time.Minute, - } - - singleFeedConfig.Schedule = wasp.Plain( - *configCopy.VRFv2.Performance.BHSTestRPS, - configCopy.VRFv2.Performance.BHSTestDuration.Duration, - ) - _, err = wasp.NewProfile(). - Add(wasp.NewGenerator(singleFeedConfig)). - Run(true) - require.NoError(t, err) - - var wgBlockNumberTobe sync.WaitGroup - wgBlockNumberTobe.Add(1) - // Wait at least 256 blocks - latestBlockNumber, err := sethClient.Client.BlockNumber(testcontext.Get(t)) - require.NoError(t, err, "error getting latest block number") - _, err = actions.WaitForBlockNumberToBe( - testcontext.Get(t), - latestBlockNumber+uint64(257), - sethClient, - &wgBlockNumberTobe, - nil, - configCopy.VRFv2.General.WaitFor256BlocksTimeout.Duration, - l, - ) - wgBlockNumberTobe.Wait() - require.NoError(t, err, "error waiting for block number to be") - - metrics, err := consumers[0].GetLoadTestMetrics(testcontext.Get(t)) - require.NoError(t, err) - require.Equal(t, 0, metrics.FulfilmentCount.Cmp(big.NewInt(0)), "Fulfilment count should be 0 since sub is underfunded. Check if the sub is actually funded") - - var subIDsString []uint64 - subIDsString = append(subIDsString, underfundedSubIDs...) - l.Info(). - Float64("SubscriptionRefundingAmountLink", *configCopy.VRFv2.General.SubscriptionRefundingAmountLink). - Uints64("SubIDs", subIDsString). - Msg("Funding Subscriptions with Link and Native Tokens") - err = vrfv2.FundSubscriptions( - big.NewFloat(*configCopy.VRFv2.General.SubscriptionRefundingAmountLink), - vrfContracts.LinkToken, - vrfContracts.CoordinatorV2, - underfundedSubIDs, - ) - require.NoError(t, err, "error funding subscriptions") - - var wgAllRequestsFulfilled sync.WaitGroup - wgAllRequestsFulfilled.Add(1) - requestCount, fulfilmentCount, err := vrfcommon.WaitForRequestCountEqualToFulfilmentCount(testcontext.Get(t), vrfContracts.VRFV2Consumers[0], 2*time.Minute, &wgAllRequestsFulfilled) - require.NoError(t, err) - wgAllRequestsFulfilled.Wait() - - l.Info(). - Interface("Request Count", requestCount). - Interface("Fulfilment Count", fulfilmentCount). - Msg("Final Request/Fulfilment Stats") - }) -} -func teardown( - t *testing.T, - consumer contracts.VRFv2LoadTestConsumer, - lc *wasp.LokiClient, - updatedLabels map[string]string, - testReporter *testreporters.VRFV2TestReporter, - testType string, - testConfig *tc.TestConfig, -) { - // send final results to Loki - metrics := GetLoadTestMetrics(testcontext.Get(t), consumer) - SendMetricsToLoki(metrics, lc, updatedLabels) - // set report data for Slack notification - testReporter.SetReportData( - testType, - testreporters.VRFLoadTestMetrics{ - RequestCount: metrics.RequestCount, - FulfilmentCount: metrics.FulfilmentCount, - AverageFulfillmentInMillions: metrics.AverageFulfillmentInMillions, - SlowestFulfillment: metrics.SlowestFulfillment, - FastestFulfillment: metrics.FastestFulfillment, - }, - testConfig, - ) - - // send Slack notification - err := testReporter.SendSlackNotification(t, nil) - if err != nil { - log.Warn().Err(err).Msg("Error sending Slack notification") - } -} diff --git a/integration-tests/load/vrfv2/vrfv2cmd/dashboard.go b/integration-tests/load/vrfv2/vrfv2cmd/dashboard.go deleted file mode 100644 index c048666d886..00000000000 --- a/integration-tests/load/vrfv2/vrfv2cmd/dashboard.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "os" - - db "github.com/smartcontractkit/chainlink-testing-framework/wasp/dashboard" - - "github.com/K-Phoen/grabana/dashboard" - "github.com/K-Phoen/grabana/logs" - "github.com/K-Phoen/grabana/row" - "github.com/K-Phoen/grabana/target/prometheus" - "github.com/K-Phoen/grabana/timeseries" - "github.com/K-Phoen/grabana/timeseries/axis" -) - -func main() { - // TODO switch to TOML too? - lokiDS := os.Getenv("DATA_SOURCE_NAME") - d, err := db.NewDashboard(nil, - []dashboard.Option{ - dashboard.Row("LoadContractMetrics", - row.WithTimeSeries( - "RequestCount + FulfilmentCount", - timeseries.Span(12), - timeseries.Height("300px"), - timeseries.DataSource(lokiDS), - timeseries.Axis( - axis.Unit("Requests"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap RequestCount [$__interval]) by (node_id, go_test_name, gen_name) - `, prometheus.Legend("{{go_test_name}} requests"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap FulfilmentCount [$__interval]) by (node_id, go_test_name, gen_name) - `, prometheus.Legend("{{go_test_name}} fulfillments"), - ), - ), - row.WithTimeSeries( - "Fulfillment time (blocks)", - timeseries.Span(12), - timeseries.Height("300px"), - timeseries.DataSource(lokiDS), - timeseries.Axis( - axis.Unit("Blocks"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap AverageFulfillmentInMillions [$__interval]) by (node_id, go_test_name, gen_name) / 1e6 - `, prometheus.Legend("{{go_test_name}} avg"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap SlowestFulfillment [$__interval]) by (node_id, go_test_name, gen_name) - `, prometheus.Legend("{{go_test_name}} slowest"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap FastestFulfillment [$__interval]) by (node_id, go_test_name, gen_name) - `, prometheus.Legend("{{go_test_name}} fastest"), - ), - ), - ), - dashboard.Row("CL nodes logs", - row.Collapse(), - row.WithLogs( - "CL nodes logs", - logs.DataSource(lokiDS), - logs.Span(12), - logs.Height("300px"), - logs.Transparent(), - logs.WithLokiTarget(` - {type="log_watch"} - `), - )), - }, - ) - if err != nil { - panic(err) - } - // set env vars - //export GRAFANA_URL=... - //export GRAFANA_TOKEN=... - //export DATA_SOURCE_NAME=Loki - //export DASHBOARD_FOLDER=LoadTests - //export DASHBOARD_NAME=WaspVRFv2 - if _, err := d.Deploy(); err != nil { - panic(err) - } -} diff --git a/integration-tests/load/vrfv2plus/README.md b/integration-tests/load/vrfv2plus/README.md deleted file mode 100644 index 1013a3f4c16..00000000000 --- a/integration-tests/load/vrfv2plus/README.md +++ /dev/null @@ -1,22 +0,0 @@ -### VRFv2Plus Load tests - -## Usage -``` -export LOKI_TOKEN=... -export LOKI_URL=... - -go test -v -run TestVRFV2PlusLoad/vrfv2plus_soak_test -``` - -### Dashboards - -Deploying dashboard: -``` -export GRAFANA_URL=... -export GRAFANA_TOKEN=... -export DATA_SOURCE_NAME=grafanacloud-logs -export DASHBOARD_FOLDER=LoadTests -export DASHBOARD_NAME=${JobTypeName, for example WaspVRFv2Plus} - -go run dashboard.go -``` \ No newline at end of file diff --git a/integration-tests/load/vrfv2plus/gun.go b/integration-tests/load/vrfv2plus/gun.go deleted file mode 100644 index 0229cdf371f..00000000000 --- a/integration-tests/load/vrfv2plus/gun.go +++ /dev/null @@ -1,144 +0,0 @@ -package loadvrfv2plus - -import ( - "math/big" - "math/rand" - - "github.com/rs/zerolog" - "golang.org/x/exp/constraints" - - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - - "github.com/smartcontractkit/chainlink-testing-framework/seth" - - seth_utils "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/seth" - "github.com/smartcontractkit/chainlink/integration-tests/actions" - - vrfcommon "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/common" - "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/vrfv2plus" - vrfv2plus_config "github.com/smartcontractkit/chainlink/integration-tests/testconfig/vrfv2plus" -) - -type BHSTestGun struct { - contracts *vrfcommon.VRFContracts - keyHash [32]byte - subIDs []*big.Int - testConfig *vrfv2plus_config.Config - logger zerolog.Logger - sethClient *seth.Client -} - -func NewBHSTestGun( - contracts *vrfcommon.VRFContracts, - keyHash [32]byte, - subIDs []*big.Int, - testConfig *vrfv2plus_config.Config, - logger zerolog.Logger, - sethClient *seth.Client, -) *BHSTestGun { - return &BHSTestGun{ - contracts: contracts, - subIDs: subIDs, - keyHash: keyHash, - testConfig: testConfig, - logger: logger, - sethClient: sethClient, - } -} - -// Call implements example gun call, assertions on response bodies should be done here -func (m *BHSTestGun) Call(_ *wasp.Generator) *wasp.Response { - vrfv2PlusConfig := m.testConfig.General - billingType, err := vrfv2plus.SelectBillingTypeWithDistribution(*vrfv2PlusConfig.SubscriptionBillingType, actions.RandBool) - if err != nil { - return &wasp.Response{Error: err.Error(), Failed: true} - } - _, err = vrfv2plus.RequestRandomness( - // the same consumer is used for all requests and in all subs - m.contracts.VRFV2PlusConsumer[0], - m.contracts.CoordinatorV2Plus, - &vrfcommon.VRFKeyData{KeyHash: m.keyHash}, - // randomly pick a subID from pool of subIDs - m.subIDs[randInRange(0, len(m.subIDs)-1)], - billingType, - vrfv2PlusConfig, - m.logger, - seth_utils.AvailableSethKeyNum(m.sethClient), - ) - // todo - might need to store randRequestBlockNumber and blockhash to verify that it was stored in BHS contract at the end of the test - if err != nil { - return &wasp.Response{Error: err.Error(), Failed: true} - } - return &wasp.Response{} -} - -/* SingleHashGun is a gun that constantly requests randomness for one feed */ -type SingleHashGun struct { - contracts *vrfcommon.VRFContracts - keyHash [32]byte - subIDs []*big.Int - testConfig *vrfv2plus_config.Config - logger zerolog.Logger - sethClient *seth.Client -} - -func NewSingleHashGun( - contracts *vrfcommon.VRFContracts, - keyHash [32]byte, - subIDs []*big.Int, - testConfig *vrfv2plus_config.Config, - logger zerolog.Logger, - sethClient *seth.Client, -) *SingleHashGun { - return &SingleHashGun{ - contracts: contracts, - keyHash: keyHash, - subIDs: subIDs, - testConfig: testConfig, - logger: logger, - sethClient: sethClient, - } -} - -// Call implements example gun call, assertions on response bodies should be done here -func (m *SingleHashGun) Call(_ *wasp.Generator) *wasp.Response { - // todo - should work with multiple consumers and consumers having different keyhashes and wallets - vrfv2PlusConfig := m.testConfig.General - billingType, err := vrfv2plus.SelectBillingTypeWithDistribution(*vrfv2PlusConfig.SubscriptionBillingType, actions.RandBool) - if err != nil { - return &wasp.Response{Error: err.Error(), Failed: true} - } - - // randomly increase/decrease randomness request count per TX - reqCount := deviateValue(*m.testConfig.General.RandomnessRequestCountPerRequest, *m.testConfig.General.RandomnessRequestCountPerRequestDeviation) - m.testConfig.General.RandomnessRequestCountPerRequest = &reqCount - _, _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( - // the same consumer is used for all requests and in all subs - m.contracts.VRFV2PlusConsumer[0], - m.contracts.CoordinatorV2Plus, - &vrfcommon.VRFKeyData{KeyHash: m.keyHash}, - // randomly pick a subID from pool of subIDs - m.subIDs[randInRange(0, len(m.subIDs)-1)], - billingType, - vrfv2PlusConfig, - m.logger, - seth_utils.AvailableSethKeyNum(m.sethClient), - ) - if err != nil { - return &wasp.Response{Error: err.Error(), Failed: true} - } - return &wasp.Response{} -} - -func deviateValue(requestCountPerTX uint16, deviation uint16) uint16 { - if actions.RandBool() && requestCountPerTX > deviation { - requestCountPerTX -= randInRange(0, deviation) - } else { - requestCountPerTX += randInRange(0, deviation) - } - return requestCountPerTX -} - -func randInRange[I constraints.Integer](lower, upper I) I { - return I(rand.Intn(int(upper-lower)+1)) + lower -} diff --git a/integration-tests/load/vrfv2plus/onchain_monitoring.go b/integration-tests/load/vrfv2plus/onchain_monitoring.go deleted file mode 100644 index c7e3b771481..00000000000 --- a/integration-tests/load/vrfv2plus/onchain_monitoring.go +++ /dev/null @@ -1,56 +0,0 @@ -package loadvrfv2plus - -import ( - "context" - "maps" - "testing" - "time" - - "github.com/rs/zerolog/log" - - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - - "github.com/smartcontractkit/chainlink/integration-tests/contracts" -) - -/* Monitors on-chain stats of LoadConsumer and pushes them to Loki every second */ - -const ( - LokiTypeLabel = "vrfv2plus_contracts_load_summary" - ErrMetrics = "failed to get VRFv2Plus load test metrics" - ErrLokiClient = "failed to create Loki client for monitoring" - ErrLokiPush = "failed to push monitoring metrics to Loki" -) - -func MonitorLoadStats(ctx context.Context, lc *wasp.LokiClient, consumer contracts.VRFv2PlusLoadTestConsumer, labels map[string]string) { - go func() { - for { - time.Sleep(1 * time.Second) - metrics := GetLoadTestMetrics(ctx, consumer) - SendMetricsToLoki(metrics, lc, labels) - } - }() -} - -func UpdateLabels(labels map[string]string, t *testing.T) map[string]string { - updatedLabels := make(map[string]string) - maps.Copy(updatedLabels, labels) - updatedLabels["type"] = LokiTypeLabel - updatedLabels["go_test_name"] = t.Name() - updatedLabels["gen_name"] = "performance" - return updatedLabels -} - -func SendMetricsToLoki(metrics *contracts.VRFLoadTestMetrics, lc *wasp.LokiClient, updatedLabels map[string]string) { - if err := lc.HandleStruct(wasp.LabelsMapToModel(updatedLabels), time.Now(), metrics); err != nil { - log.Error().Err(err).Msg(ErrLokiPush) - } -} - -func GetLoadTestMetrics(ctx context.Context, consumer contracts.VRFv2PlusLoadTestConsumer) *contracts.VRFLoadTestMetrics { - metrics, err := consumer.GetLoadTestMetrics(ctx) - if err != nil { - log.Error().Err(err).Msg(ErrMetrics) - } - return metrics -} diff --git a/integration-tests/load/vrfv2plus/vrfv2plus_test.go b/integration-tests/load/vrfv2plus/vrfv2plus_test.go deleted file mode 100644 index 64f77da02bb..00000000000 --- a/integration-tests/load/vrfv2plus/vrfv2plus_test.go +++ /dev/null @@ -1,397 +0,0 @@ -package loadvrfv2plus - -import ( - "math/big" - "sync" - "testing" - "time" - - "github.com/smartcontractkit/chainlink-testing-framework/seth" - - "github.com/rs/zerolog/log" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - - "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" - "github.com/smartcontractkit/chainlink-testing-framework/lib/networks" - "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/ptr" - "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" - "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/vrfv2plus" - "github.com/smartcontractkit/chainlink/integration-tests/contracts" - "github.com/smartcontractkit/chainlink/integration-tests/testreporters" - - "github.com/smartcontractkit/chainlink/integration-tests/actions" - vrfcommon "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/common" - "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" - - tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" -) - -var ( - labels = map[string]string{ - "branch": "vrfv2Plus_healthcheck", - "commit": "vrfv2Plus_healthcheck", - } -) - -func TestVRFV2PlusPerformance(t *testing.T) { - var ( - testEnv *test_env.CLClusterTestEnv - vrfContracts *vrfcommon.VRFContracts - subIDsForCancellingAfterTest []*big.Int - vrfKey *vrfcommon.VRFKeyData - sethClient *seth.Client - ) - l := logging.GetTestLogger(t) - testType, err := tc.GetConfigurationNameFromEnv() - require.NoError(t, err) - testConfig, err := tc.GetChainAndTestTypeSpecificConfig(testType, tc.VRFv2Plus) - require.NoError(t, err) - cfgl := testConfig.Logging.Loki - - vrfv2PlusConfig := testConfig.VRFv2Plus - testReporter := &testreporters.VRFV2PlusTestReporter{} - - lokiConfig := wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken) - lc, err := wasp.NewLokiClient(lokiConfig) - if err != nil { - l.Error().Err(err).Msg(ErrLokiClient) - return - } - network := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0] - chainID := network.ChainID - updatedLabels := UpdateLabels(labels, t) - - l.Info(). - Str("Test Type", testType). - Str("Test Duration", vrfv2PlusConfig.Performance.TestDuration.Duration.Truncate(time.Second).String()). - Int64("RPS", *vrfv2PlusConfig.Performance.RPS). - Str("RateLimitUnitDuration", vrfv2PlusConfig.Performance.RateLimitUnitDuration.String()). - Uint16("RandomnessRequestCountPerRequest", *vrfv2PlusConfig.General.RandomnessRequestCountPerRequest). - Uint16("RandomnessRequestCountPerRequestDeviation", *vrfv2PlusConfig.General.RandomnessRequestCountPerRequestDeviation). - Bool("UseExistingEnv", *vrfv2PlusConfig.General.UseExistingEnv). - Msg("Performance Test Configuration") - - cleanupFn := func() { - teardown(t, vrfContracts.VRFV2PlusConsumer[0], lc, updatedLabels, testReporter, testType, &testConfig) - - if sethClient.Cfg.IsSimulatedNetwork() { - l.Info(). - Str("Network Name", sethClient.Cfg.Network.Name). - Msg("Network is a simulated network. Skipping fund return for Coordinator Subscriptions.") - } else if *testConfig.VRFv2Plus.General.CancelSubsAfterTestRun { - // wait for all txs to be mined in order to avoid nonce issues - time.Sleep(10 * time.Second) - // cancel subs and return funds to sub owner - vrfv2plus.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, sethClient.MustGetRootKeyAddress().Hex(), subIDsForCancellingAfterTest, l) - } - if !*testConfig.VRFv2Plus.General.UseExistingEnv { - if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { - l.Error().Err(err).Msg("Error cleaning up test environment") - } - } - } - vrfEnvConfig := vrfcommon.VRFEnvConfig{ - TestConfig: testConfig, - ChainID: chainID, - CleanupFn: cleanupFn, - } - newEnvConfig := vrfcommon.NewEnvConfig{ - NodesToCreate: []vrfcommon.VRFNodeType{vrfcommon.VRF}, - NumberOfTxKeysToCreate: *vrfv2PlusConfig.General.NumberOfSendingKeysToCreate, - ChainlinkNodeLogScannerSettings: test_env.DefaultChainlinkNodeLogScannerSettings, - } - testEnv, vrfContracts, vrfKey, _, sethClient, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, vrfEnvConfig, newEnvConfig, l) - require.NoError(t, err, "error setting up VRFV2Plus universe") - - var consumers []contracts.VRFv2PlusLoadTestConsumer - - subIDs, consumers, err := vrfv2plus.SetupSubsAndConsumersForExistingEnv( - testcontext.Get(t), - sethClient, - vrfContracts.CoordinatorV2Plus, - vrfContracts.LinkToken, - 1, - *vrfv2PlusConfig.General.NumberOfSubToCreate, - testConfig, - l, - ) - require.NoError(t, err, "error setting up new consumers and subs") - for _, subID := range subIDs { - subscription, err := vrfContracts.CoordinatorV2Plus.GetSubscription(testcontext.Get(t), subID) - require.NoError(t, err, "error getting subscription information for subscription %s", subID.String()) - vrfcommon.LogSubDetails(l, subscription, subID.String(), vrfContracts.CoordinatorV2Plus) - } - subIDsForCancellingAfterTest = subIDs - l.Info().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test") - - vrfContracts.VRFV2PlusConsumer = consumers - - // is our "job" stable at all, no memory leaks, no flaking performance under some RPS? - t.Run("vrfv2plus performance test", func(t *testing.T) { - require.Len(t, vrfContracts.VRFV2PlusConsumer, 1, "only one consumer should be created for Load Test") - consumer := vrfContracts.VRFV2PlusConsumer[0] - err = consumer.ResetMetrics() - require.NoError(t, err) - MonitorLoadStats(testcontext.Get(t), lc, consumer, updatedLabels) - - singleFeedConfig := &wasp.Config{ - T: t, - LoadType: wasp.RPS, - GenName: "gun", - RateLimitUnitDuration: vrfv2PlusConfig.Performance.RateLimitUnitDuration.Duration, - Gun: NewSingleHashGun( - vrfContracts, - vrfKey.KeyHash, - subIDs, - vrfv2PlusConfig, - l, - sethClient, - ), - Labels: labels, - LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), - CallTimeout: 2 * time.Minute, - } - - singleFeedConfig.Schedule = wasp.Plain( - *vrfv2PlusConfig.Performance.RPS, - vrfv2PlusConfig.Performance.TestDuration.Duration, - ) - _, err = wasp.NewProfile(). - Add(wasp.NewGenerator(singleFeedConfig)). - Run(true) - require.NoError(t, err) - - var wg sync.WaitGroup - wg.Add(1) - // todo - timeout should be configurable depending on the perf test type - requestCount, fulfilmentCount, err := vrfcommon.WaitForRequestCountEqualToFulfilmentCount(testcontext.Get(t), consumer, 2*time.Minute, &wg) - require.NoError(t, err) - wg.Wait() - - l.Info(). - Interface("Request Count", requestCount). - Interface("Fulfilment Count", fulfilmentCount). - Msg("Final Request/Fulfilment Stats") - }) -} - -func TestVRFV2PlusBHSPerformance(t *testing.T) { - var ( - testEnv *test_env.CLClusterTestEnv - vrfContracts *vrfcommon.VRFContracts - subIDsForCancellingAfterTest []*big.Int - vrfKey *vrfcommon.VRFKeyData - sethClient *seth.Client - ) - l := logging.GetTestLogger(t) - - testType, err := tc.GetConfigurationNameFromEnv() - require.NoError(t, err) - testConfig, err := tc.GetChainAndTestTypeSpecificConfig(testType, tc.VRFv2Plus) - require.NoError(t, err) - vrfv2PlusConfig := testConfig.VRFv2Plus - testReporter := &testreporters.VRFV2PlusTestReporter{} - cfgl := testConfig.Logging.Loki - lokiConfig := wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken) - lc, err := wasp.NewLokiClient(lokiConfig) - if err != nil { - l.Error().Err(err).Msg(ErrLokiClient) - return - } - - updatedLabels := UpdateLabels(labels, t) - - l.Info(). - Str("Test Type", testType). - Str("Test Duration", vrfv2PlusConfig.Performance.TestDuration.Duration.Truncate(time.Second).String()). - Int64("RPS", *vrfv2PlusConfig.Performance.RPS). - Str("RateLimitUnitDuration", vrfv2PlusConfig.Performance.RateLimitUnitDuration.String()). - Uint16("RandomnessRequestCountPerRequest", *vrfv2PlusConfig.General.RandomnessRequestCountPerRequest). - Uint16("RandomnessRequestCountPerRequestDeviation", *vrfv2PlusConfig.General.RandomnessRequestCountPerRequestDeviation). - Bool("UseExistingEnv", *vrfv2PlusConfig.General.UseExistingEnv). - Msg("Performance Test Configuration") - - network := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0] - chainID := network.ChainID - - cleanupFn := func() { - teardown(t, vrfContracts.VRFV2PlusConsumer[0], lc, updatedLabels, testReporter, testType, &testConfig) - if sethClient.Cfg.IsSimulatedNetwork() { - l.Info(). - Str("Network Name", sethClient.Cfg.Network.Name). - Msg("Network is a simulated network. Skipping fund return for Coordinator Subscriptions.") - } else if *testConfig.VRFv2Plus.General.CancelSubsAfterTestRun { - // cancel subs and return funds to sub owner - vrfv2plus.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, sethClient.MustGetRootKeyAddress().Hex(), subIDsForCancellingAfterTest, l) - } - if !*testConfig.VRFv2Plus.General.UseExistingEnv { - if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { - l.Error().Err(err).Msg("Error cleaning up test environment") - } - } - } - - vrfEnvConfig := vrfcommon.VRFEnvConfig{ - TestConfig: testConfig, - ChainID: chainID, - CleanupFn: cleanupFn, - } - newEnvConfig := vrfcommon.NewEnvConfig{ - NodesToCreate: []vrfcommon.VRFNodeType{vrfcommon.VRF, vrfcommon.BHS}, - NumberOfTxKeysToCreate: *vrfv2PlusConfig.General.NumberOfSendingKeysToCreate, - ChainlinkNodeLogScannerSettings: test_env.DefaultChainlinkNodeLogScannerSettings, - } - testEnv, vrfContracts, vrfKey, _, sethClient, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, vrfEnvConfig, newEnvConfig, l) - require.NoError(t, err, "error setting up VRFV2Plus universe") - - t.Run("vrfv2plus and bhs performance test", func(t *testing.T) { - configCopy := testConfig.MustCopy().(tc.TestConfig) - // Underfund Subscription - configCopy.VRFv2Plus.General.SubscriptionFundingAmountLink = ptr.Ptr(float64(0)) - configCopy.VRFv2Plus.General.SubscriptionFundingAmountNative = ptr.Ptr(float64(0)) - - underfundedSubIDs, consumers, err := vrfv2plus.SetupSubsAndConsumersForExistingEnv( - testcontext.Get(t), - sethClient, - vrfContracts.CoordinatorV2Plus, - vrfContracts.LinkToken, - 1, - *vrfv2PlusConfig.General.NumberOfSubToCreate, - configCopy, - l, - ) - require.NoError(t, err, "error setting up new consumers and subs for Load Test") - for _, subID := range underfundedSubIDs { - subscription, err := vrfContracts.CoordinatorV2Plus.GetSubscription(testcontext.Get(t), subID) - require.NoError(t, err, "error getting subscription information for subscription %s", subID.String()) - vrfcommon.LogSubDetails(l, subscription, subID.String(), vrfContracts.CoordinatorV2Plus) - } - subIDsForCancellingAfterTest = underfundedSubIDs - l.Debug().Int("Number of Subs", len(underfundedSubIDs)).Msg("Subs involved in the test") - vrfContracts.VRFV2PlusConsumer = consumers - require.Len(t, vrfContracts.VRFV2PlusConsumer, 1, "only one consumer should be created for Load Test") - consumer := vrfContracts.VRFV2PlusConsumer[0] - err = consumer.ResetMetrics() - require.NoError(t, err) - MonitorLoadStats(testcontext.Get(t), lc, consumer, updatedLabels) - - singleFeedConfig := &wasp.Config{ - T: t, - LoadType: wasp.RPS, - GenName: "gun", - RateLimitUnitDuration: configCopy.VRFv2Plus.Performance.BHSTestRateLimitUnitDuration.Duration, - Gun: NewBHSTestGun( - vrfContracts, - vrfKey.KeyHash, - underfundedSubIDs, - configCopy.VRFv2Plus, - l, - sethClient, - ), - Labels: labels, - LokiConfig: lokiConfig, - CallTimeout: 2 * time.Minute, - } - - singleFeedConfig.Schedule = wasp.Plain( - *configCopy.VRFv2Plus.Performance.BHSTestRPS, - configCopy.VRFv2Plus.Performance.BHSTestDuration.Duration, - ) - _, err = wasp.NewProfile(). - Add(wasp.NewGenerator(singleFeedConfig)). - Run(true) - require.NoError(t, err) - - var wgBlockNumberTobe sync.WaitGroup - wgBlockNumberTobe.Add(1) - // Wait at least 256 blocks - latestBlockNumber, err := sethClient.Client.BlockNumber(testcontext.Get(t)) - require.NoError(t, err, "error getting latest block number") - _, err = actions.WaitForBlockNumberToBe( - testcontext.Get(t), - latestBlockNumber+uint64(257), - sethClient, - &wgBlockNumberTobe, - nil, - configCopy.VRFv2Plus.General.WaitFor256BlocksTimeout.Duration, - l, - ) - wgBlockNumberTobe.Wait() - require.NoError(t, err, "error waiting for block number to be") - - metrics, err := consumers[0].GetLoadTestMetrics(testcontext.Get(t)) - require.NoError(t, err) - require.Equal(t, 0, metrics.FulfilmentCount.Cmp(big.NewInt(0)), "Fulfilment count should be 0 since sub is underfunded. Check if the sub is actually funded") - - var subIDsString []string - for _, subID := range underfundedSubIDs { - subIDsString = append(subIDsString, subID.String()) - } - - l.Info(). - Float64("SubscriptionRefundingAmountNative", *configCopy.VRFv2Plus.General.SubscriptionRefundingAmountNative). - Float64("SubscriptionRefundingAmountLink", *configCopy.VRFv2Plus.General.SubscriptionRefundingAmountLink). - Strs("SubIDs", subIDsString). - Str("Funding type", *configCopy.VRFv2Plus.General.SubscriptionBillingType). - Msg("Funding Subscriptions with Link and/or Native Tokens") - err = vrfv2plus.FundSubscriptions( - big.NewFloat(*configCopy.VRFv2Plus.General.SubscriptionRefundingAmountNative), - big.NewFloat(*configCopy.VRFv2Plus.General.SubscriptionRefundingAmountLink), - vrfContracts.LinkToken, - vrfContracts.CoordinatorV2Plus, - underfundedSubIDs, - *configCopy.VRFv2Plus.General.SubscriptionBillingType, - ) - require.NoError(t, err, "error funding subscriptions") - - var wgAllRequestsFulfilled sync.WaitGroup - wgAllRequestsFulfilled.Add(1) - requestCount, fulfilmentCount, err := vrfcommon.WaitForRequestCountEqualToFulfilmentCount(testcontext.Get(t), consumer, 2*time.Minute, &wgAllRequestsFulfilled) - require.NoError(t, err) - wgAllRequestsFulfilled.Wait() - - l.Info(). - Interface("Request Count", requestCount). - Interface("Fulfilment Count", fulfilmentCount). - Msg("Final Request/Fulfilment Stats") - }) -} - -func teardown( - t *testing.T, - consumer contracts.VRFv2PlusLoadTestConsumer, - lc *wasp.LokiClient, - updatedLabels map[string]string, - testReporter *testreporters.VRFV2PlusTestReporter, - testType string, - testConfig *tc.TestConfig, -) { - // send final results to Loki - metrics := GetLoadTestMetrics(testcontext.Get(t), consumer) - SendMetricsToLoki(metrics, lc, updatedLabels) - // set report data for Slack notification - testReporter.SetReportData( - testType, - testreporters.VRFLoadTestMetrics{ - RequestCount: metrics.RequestCount, - FulfilmentCount: metrics.FulfilmentCount, - AverageFulfillmentInMillions: metrics.AverageFulfillmentInMillions, - SlowestFulfillment: metrics.SlowestFulfillment, - FastestFulfillment: metrics.FastestFulfillment, - P90FulfillmentBlockTime: metrics.P90FulfillmentBlockTime, - P95FulfillmentBlockTime: metrics.P95FulfillmentBlockTime, - AverageResponseTimeInSecondsMillions: metrics.AverageResponseTimeInSecondsMillions, - SlowestResponseTimeInSeconds: metrics.SlowestResponseTimeInSeconds, - FastestResponseTimeInSeconds: metrics.FastestResponseTimeInSeconds, - }, - testConfig, - ) - - // send Slack notification - err := testReporter.SendSlackNotification(t, nil, testConfig) - if err != nil { - log.Warn().Err(err).Msg("Error sending Slack notification") - } -} diff --git a/integration-tests/load/vrfv2plus/vrfv2pluscmd/dashboard.go b/integration-tests/load/vrfv2plus/vrfv2pluscmd/dashboard.go deleted file mode 100644 index 51574683559..00000000000 --- a/integration-tests/load/vrfv2plus/vrfv2pluscmd/dashboard.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "os" - - db "github.com/smartcontractkit/chainlink-testing-framework/wasp/dashboard" - - "github.com/K-Phoen/grabana/dashboard" - "github.com/K-Phoen/grabana/logs" - "github.com/K-Phoen/grabana/row" - "github.com/K-Phoen/grabana/target/prometheus" - "github.com/K-Phoen/grabana/timeseries" - "github.com/K-Phoen/grabana/timeseries/axis" -) - -func main() { - // TODO switch to TOML too? - lokiDS := os.Getenv("DATA_SOURCE_NAME") - d, err := db.NewDashboard(nil, - []dashboard.Option{ - dashboard.Row("LoadContractMetrics", - row.WithTimeSeries( - "RequestCount + FulfilmentCount", - timeseries.Span(12), - timeseries.Height("300px"), - timeseries.DataSource(lokiDS), - timeseries.Axis( - axis.Unit("Requests"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2plus_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap RequestCount [$__interval]) by (node_id, go_test_name, gen_name) - `, prometheus.Legend("{{go_test_name}} requests"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2plus_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap FulfilmentCount [$__interval]) by (node_id, go_test_name, gen_name) - `, prometheus.Legend("{{go_test_name}} fulfillments"), - ), - ), - row.WithTimeSeries( - "Fulfillment time (blocks)", - timeseries.Span(12), - timeseries.Height("300px"), - timeseries.DataSource(lokiDS), - timeseries.Axis( - axis.Unit("Blocks"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2plus_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap AverageFulfillmentInMillions [$__interval]) by (node_id, go_test_name, gen_name) / 1e6 - `, prometheus.Legend("{{go_test_name}} avg"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2plus_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap SlowestFulfillment [$__interval]) by (node_id, go_test_name, gen_name) - `, prometheus.Legend("{{go_test_name}} slowest"), - ), - timeseries.WithPrometheusTarget( - ` - last_over_time({type="vrfv2plus_contracts_load_summary", go_test_name=~"${go_test_name:pipe}", branch=~"${branch:pipe}", commit=~"${commit:pipe}", gen_name=~"${gen_name:pipe}"} - | json - | unwrap FastestFulfillment [$__interval]) by (node_id, go_test_name, gen_name) - `, prometheus.Legend("{{go_test_name}} fastest"), - ), - ), - ), - dashboard.Row("CL nodes logs", - row.Collapse(), - row.WithLogs( - "CL nodes logs", - logs.DataSource(lokiDS), - logs.Span(12), - logs.Height("300px"), - logs.Transparent(), - logs.WithLokiTarget(` - {type="log_watch"} - `), - )), - }, - ) - if err != nil { - panic(err) - } - // set env vars - //export GRAFANA_URL=... - //export GRAFANA_TOKEN=... - //export DATA_SOURCE_NAME=Loki - //export DASHBOARD_FOLDER=LoadTests - //export DASHBOARD_NAME=Waspvrfv2plus - if _, err := d.Deploy(); err != nil { - panic(err) - } -} From 084321111e643c3373fe31fb5cb21c01fc66596b Mon Sep 17 00:00:00 2001 From: skudasov Date: Thu, 19 Mar 2026 11:58:27 +0100 Subject: [PATCH 2/2] tidy --- integration-tests/load/README.md | 90 -------------------------------- integration-tests/load/go.mod | 17 +++--- integration-tests/load/go.sum | 10 ---- 3 files changed, 6 insertions(+), 111 deletions(-) delete mode 100644 integration-tests/load/README.md diff --git a/integration-tests/load/README.md b/integration-tests/load/README.md deleted file mode 100644 index ce9d8b9ae6d..00000000000 --- a/integration-tests/load/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Performance tests for CL jobs - -This folder container performance e2e tests for different job types, currently implemented: - -- VRFv2 -- CCIP (wip) - -All the tests have 4 groups: - -- one product soak -- one product load -- multiple product instances soak -- multiple product instances load - -When you develop an e2e performance suite for a new product you can implement the tests one by one to answer the questions: - -What are performance characteristics of a one instance of a product (jobs + contracts): - -- is our product stable at all, no memory leaks, no flaking performance under some RPS? (test #1) -- what are the limits for one product instance, figuring out the max/optimal performance params by increasing RPS and varying configuration (test #2) -- update test #1 with optimal params and workload to constantly run in CI - -What are performance and capacity characteristics of Chainlink node(s) that run multiple products of the same type simultaneously: - -- how many products of the same type we can run at once at a stable load with optimal configuration? (test #3) -- what are the limits if we add more and more products of the same type, each product have a stable RPS, we vary only amount of products -- update test #3 with optimal params and workload to constantly run in CI - -Implementing test #1 is **mandatory** for each product. -Tests #2,#3,#4 are optional if you need to figure out your product scaling or node/cluster capacity. - -## Usage - -```sh -export LOKI_TOKEN=... -export LOKI_URL=... - -go test -v -run TestVRFV2Load/vrfv2_soak_test -``` - -### Dashboards - -Each product has its own generated dashboard in `cmd/dashboard.go` - -Deploying dashboard: - -```sh -export GRAFANA_URL=... -export GRAFANA_TOKEN=... -export DATA_SOURCE_NAME=grafanacloud-logs -export DASHBOARD_FOLDER=LoadTests -export DASHBOARD_NAME=${JobTypeName, for example WaspVRFv2} - -go run dashboard.go -``` - -### Assertions - -You can implement your product requirements assertions in `onchain_monitoring.go`. For Chainlink products (VRF/OCR/Keepers/Automation) it's easier to assert on-chain part - -If you need to assert some metrics in `Prometheus/Loki`, here is an [example](https://github.com/smartcontractkit/wasp/blob/master/examples/alerts/main_test.go#L88) - -Do not mix workload logic with assertions, separate them. - -### Implementation - -To implement a standard e2e performance suite for a new product please look at `gun.go` and `vu.go`. - -Gun should be working with one instance of your product. - -VU(Virtual user) creates a new instance of your product and works with it in `Call()` - -### Cluster mode (k8s) - -Add configuration to `overrides.toml` - -```toml -[WaspAutoBuild] -namespace = "wasp" -update_image = true -repo_image_version_uri = "${staging_ecr_registry}/wasp-tests:wb-core" -test_binary_name = "ocr.test" -test_name = "TestOCRLoad" -test_timeout = "24h" -wasp_log_level = "debug" -wasp_jobs = "1" -keep_jobs = true -``` - -And run your tests using `go test -v -run TestClusterEntrypoint` diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index b7c50bcbf76..d35a3cb2d01 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -18,14 +18,10 @@ require ( ) require ( - github.com/K-Phoen/grabana v0.22.2 github.com/aptos-labs/aptos-go-sdk v1.12.1-0.20260318141106-21b6ef4ed363 github.com/ethereum/go-ethereum v1.17.1 github.com/gagliardetto/solana-go v1.13.0 - github.com/pelletier/go-toml/v2 v2.2.4 - github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.34.0 - github.com/slack-go/slack v0.15.0 github.com/smartcontractkit/chain-selectors v1.0.97 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260318173523-755cafb24200 github.com/smartcontractkit/chainlink-ccip v0.1.1-solana.0.20260317185256-d5f7db87ae70 @@ -34,16 +30,12 @@ require ( github.com/smartcontractkit/chainlink-common v0.10.1-0.20260317233127-178dd2eeaa87 github.com/smartcontractkit/chainlink-deployments-framework v0.86.3 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260318010722-59d4165024f1 - github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260119171452-39c98c3b33cd github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.3 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.5 - github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.7 github.com/smartcontractkit/chainlink-testing-framework/seth v1.51.5 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.51.2 github.com/stretchr/testify v1.11.1 - github.com/wiremock/go-wiremock v1.9.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa golang.org/x/sync v0.20.0 ) @@ -75,7 +67,6 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect github.com/BurntSushi/toml v1.5.0 // indirect github.com/DataDog/zstd v1.5.6 // indirect - github.com/K-Phoen/sdk v0.12.4 // indirect github.com/Khan/genqlient v0.8.1 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -288,8 +279,6 @@ require ( github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/sessions v1.2.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/gosimple/slug v1.13.1 // indirect - github.com/gosimple/unidecode v1.0.1 // indirect github.com/grafana/dskit v0.0.0-20250617101305-c93a1bb09ecb // indirect github.com/grafana/gomemcache v0.0.0-20250318131618-74242eea118d // indirect github.com/grafana/grafana-foundation-sdk/go v0.0.0-20240326122733-6f96a993222b // indirect @@ -431,6 +420,7 @@ require ( github.com/otiai10/mint v1.6.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect @@ -441,6 +431,7 @@ require ( github.com/pion/transport/v3 v3.0.7 // indirect github.com/pires/go-proxyproto v0.7.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pressly/goose/v3 v3.26.0 // indirect @@ -476,6 +467,7 @@ require ( github.com/shopspring/decimal v1.4.0 // indirect github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3 // indirect github.com/sirupsen/logrus v1.9.4 // indirect + github.com/slack-go/slack v0.15.0 // indirect github.com/smartcontractkit/ccip-contract-examples/chains/evm v0.0.0-20260129135848-c86808ba5cb9 // indirect github.com/smartcontractkit/ccip-owner-contracts v0.1.0 // indirect github.com/smartcontractkit/chainlink-automation v0.8.1 // indirect @@ -487,6 +479,7 @@ require ( github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 // indirect github.com/smartcontractkit/chainlink-data-streams v0.1.12 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260107191744-4b93f62cffe3 // indirect + github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260119171452-39c98c3b33cd // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20250818175541-3389ac08a563 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260317132927-e8bc2c7b01f1 // indirect @@ -510,6 +503,7 @@ require ( github.com/smartcontractkit/chainlink-solana v1.1.2-0.20260318003508-442c25062e9a // indirect github.com/smartcontractkit/chainlink-sui v0.0.0-20260304150206-c64e48eb0cb0 // indirect github.com/smartcontractkit/chainlink-sui/deployment v0.0.0-20260304150206-c64e48eb0cb0 // indirect + github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.7 // indirect github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.51.0 // indirect github.com/smartcontractkit/chainlink-testing-framework/parrot v0.6.2 // indirect github.com/smartcontractkit/chainlink-testing-framework/sentinel v0.1.2 // indirect @@ -622,6 +616,7 @@ require ( go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect golang.org/x/arch v0.11.0 // indirect golang.org/x/crypto v0.48.0 // indirect + golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.50.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 81fa9f6eff0..e77093c0a69 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -118,10 +118,6 @@ github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/K-Phoen/grabana v0.22.2 h1:tMiSvcKHnDbXi3IgBCax2+sg5qL6x0G6wMURHgjGDag= -github.com/K-Phoen/grabana v0.22.2/go.mod h1:TbgU7jM55UlExzQyzu6AMiSUXr9jiaXmCu9AN28WXHk= -github.com/K-Phoen/sdk v0.12.4 h1:j2EYuBJm3zDTD0fGKACVFWxAXtkR0q5QzfVqxmHSeGQ= -github.com/K-Phoen/sdk v0.12.4/go.mod h1:qmM0wO23CtoDux528MXPpYvS4XkRWkWX6rvX9Za8EVU= github.com/Khan/genqlient v0.8.1 h1:wtOCc8N9rNynRLXN3k3CnfzheCUNKBcvXmVv5zt6WCs= github.com/Khan/genqlient v0.8.1/go.mod h1:R2G6DzjBvCbhjsEajfRjbWdVglSH/73kSivC9TLWVjU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -869,10 +865,6 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gosimple/slug v1.13.1 h1:bQ+kpX9Qa6tHRaK+fZR0A0M2Kd7Pa5eHPPsb1JpHD+Q= -github.com/gosimple/slug v1.13.1/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= -github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= -github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/grafana/dskit v0.0.0-20250617101305-c93a1bb09ecb h1:jQ5UjRFq1xdRC+Mr60mix44jBJGVQ/jUOEDkXuYR5kc= github.com/grafana/dskit v0.0.0-20250617101305-c93a1bb09ecb/go.mod h1:JO5e0gs13dzqKYIF5cSa5YCNU5ydrziLP6dDzw0mPqc= github.com/grafana/gomemcache v0.0.0-20250318131618-74242eea118d h1:oXRJlb9UjVsl6LhqBdbyAQ9YFhExwsj4bjh5vwMNRZY= @@ -1856,8 +1848,6 @@ github.com/vektah/gqlparser/v2 v2.5.19 h1:bhCPCX1D4WWzCDvkPl4+TP1N8/kLrWnp43egpl github.com/vektah/gqlparser/v2 v2.5.19/go.mod h1:y7kvl5bBlDeuWIvLtA9849ncyvx6/lj06RsMrEjVy3U= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= -github.com/wiremock/go-wiremock v1.9.0 h1:9xcU4/IoEfgCaH4TGhQTtiQyBh2eMtu9JB6ppWduK+E= -github.com/wiremock/go-wiremock v1.9.0/go.mod h1:/uvO0XFheyy8XetvQqm4TbNQRsGPlByeNegzLzvXs0c= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=