optimism/op-acceptance-tests/acceptance-tests.yaml at develop · ChainSafe/optimism · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
# Configuration file for acceptance tests (op-acceptor)
#
# All acceptance tests need to be registered here for op-acceptor to run them.
# As a rule of thumb, we recommend that each fork gate inherits from the
# base gate as well as any earlier fork gates.

gates:
  # ============================================================================
  # FLAKE-SHAKE GATE
  # ============================================================================
  # The flake-shake gate is a quarantine area for new or potentially flaky tests.
  # Tests here are run repeatedly (100+ iterations daily) to detect flakiness
  # before they can be promoted to production gates.
  #
  # HOW TO ADD A NEW TEST:
  # ----------------------
  # 1. Add your test entry to the flake-shake gate (see example below)
  # 2. Include the `target_gate` metadata to indicate where the test should be
  #    promoted once it passes stability validation
  # 3. Include the `owner` metadata for notifications and accountability
  #
  # Example entry format:
  #   - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/myfeature
  #     name: TestMyFeature        # Optional: specific test name (omit to run all tests in package)
  #     timeout: 10m               # Optional: test timeout (default varies by gate)
  #     metadata:
  #       owner: "my-team"         # Required: team/person responsible for this test
  #       target_gate: "base"      # Required: destination gate after promotion
  #
  # PROMOTION CRITERIA:
  # -------------------
  # Tests are automatically promoted when they meet ALL of these criteria:
  #   - 100% pass rate (zero failures allowed)
  #   - Minimum 300 total runs accumulated
  #   - Been in flake-shake for at least 2-3 days
  #   - No failures in the last 24 hours
  #
  # WHAT HAPPENS AT PROMOTION:
  # --------------------------
  # When a test meets the promotion criteria, an automated PR is created that:
  #   1. Removes the test entry from the flake-shake gate
  #   2. The PR reviewer must then manually add the test to the target gate
  #      (as specified in the `target_gate` metadata)
  #
  # This two-step process ensures human review of all gate promotions.
  #
  # For more details, see the README.md section on "Flake-Shake: Test Stability Validation"
  # ============================================================================
  - id: flake-shake
    description: "Quarantine gate for new and potentially flaky tests requiring stability validation."
    tests:
     - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/reqressyncdisabled
       name: TestUnsafeChainNotStalling_DisabledReqRespSync
       timeout: 10m
       metadata:
         owner: "anton evangelatov"
         target_gate: "depreqres"
     - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/supernode/interop/activation
       name: TestSupernodeInteropActivationAfterGenesis
       timeout: 10m
       metadata:
         owner: "adrian sutton"
         target_gate: "supernode-interop"
     - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/supernode/interop
       name: TestSupernodeInteropChainLag
       timeout: 15m
       metadata:
         owner: "axel kingsley"
         target_gate: "supernode-interop"
     - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/syncmodereqressync/elsync
       name: TestUnsafeChainNotStalling_ELSync_Short
       timeout: 10m
       metadata:
         owner: "anton evangelatov"
         target_gate: "depreqres"
     - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/syncmodereqressync/elsync
       name: TestUnsafeChainNotStalling_ELSync_Long
       timeout: 10m
       metadata:
         owner: "anton evangelatov"
         target_gate: "depreqres"
     - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/syncmodereqressync/elsync
       name: TestUnsafeChainNotStalling_ELSync_RestartOpNode_Long
       timeout: 10m
       metadata:
         owner: "anton evangelatov"
         target_gate: "depreqres"

  - id: jovian
    description: "Jovian network tests."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/jovian/bpo2
        timeout: 10m
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/jovian/pectra
        timeout: 10m

  - id: isthmus
    description: "Isthmus network tests."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus
        timeout: 6h
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/operator_fee
        timeout: 6h
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/withdrawal_root
        timeout: 20m
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/erc20_bridge
        timeout: 10m
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/pectra
        timeout: 10m

  - id: base
    description: "Sanity/smoke acceptance tests for all networks."
    inherits:
      - isthmus
      - jovian
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/deposit
        timeout: 10m
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/chain
        timeout: 10m
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/ecotone
        timeout: 10m
      # TODO(infra#401): Re-enable the test once the remaining infra gap is resolved.
      #- package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/withdrawal
      #  timeout: 10m
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/fjord

  - id: conductor
    description: "Sanity/smoke acceptance tests for networks with conductors."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base/conductor
        timeout: 10m

  - id: pre-interop
    inherits:
      - base
    description: "Pre-interop network tests."
    tests:
      - name: TestInteropReadiness
        package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/isthmus/preinterop
        timeout: 20m

  - id: interop
    inherits:
      - base
    description: "Interop network tests."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop
        timeout: 10m
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/message
        timeout: 30m
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/sync
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/smoke
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/contract

  - id: interop-loadtest
    description: "Interop network loadtests."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/interop/loadtest
        timeout: 10m

  - id: depreqres
    description: "Deprecate Req/Res CL sync protocol"
    tests:

  - id: flashblocks
    inherits:
      - base
    description: "Flashblocks network tests."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/flashblocks
        timeout: 5m

  - id: flashblocks-with-isthmus
    inherits:
      - base
    description: "Flashblocks network tests with Isthmus."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/flashblocks
        timeout: 5m

  - id: cgt
    description: "Custom Gas Token (CGT) network tests."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/custom_gas_token
        timeout: 10m

  - id: supernode
    description: "Supernode tests - tests for the op-supernode multi-chain consensus layer."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/supernode
        timeout: 10m

  - id: supernode-interop
    inherits:
      - supernode
    description: "Supernode interop tests - tests for supernode's cross-chain message verification."
    tests:
      - package: github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/supernode/interop/...
        timeout: 15m