Skip to content

Commit 4d9a688

Browse files
thevaizmanclaude
thevaizman
and
claude
committed
feat(dex): deterministic 0x Settler aggregator volume — supersede PR #9795 (CUR2-2844)
Replaces the heuristic maker/taker leg-matching (and the PR #9795 5x-divergence band-aid) for 0x Settler trades in dex_aggregator.trades with a deterministic decode. Per settler call, emits one 0x-API aggregator row = the user's net swap: - token_bought = AllowedSlippage.buyToken (from calldata, deterministic). - amount = the unique Transfer(buyToken, to=receiver) when resolvable, else the minAmountOut floor (verified tight, ~1% under actual). receiver = tx-level sender (execute) / msgSender (executeMetaTxn) — verified to be the true buyToken recipient, NOT the calldata recipient (an intermediate routing hop). Anchoring on a single calldata-named token to the verified user means it cannot mis-bind to internal routing hops the way the heuristic did. - token_sold = best-effort (unique Transfer out of the user, token != buyToken). - volume_usd priced via either leg (add_amount_usd), so an unpriced buyToken still values off the sell leg (cuts the unpriced-NULL bucket ~20% -> ~5%). New macro zeroex_settler_agg; settler-txs staging extended with the AllowedSlippage fields (buy_token, min_amount_out, settler_msgsender); all 17 settler chains' zeroex_v2_<chain>_trades rewired to it. Verified on-chain (example 0xf61374… : current $0.75 -> deterministic ~$490). NOTE: requires a coordinated full-refresh of zeroex_v2_<chain>_trades + dex_aggregator_trades (merge cannot delete the old mis-bound rows). Old-vs-new regression to be validated on CI-built tables before merge. Stacked on CUR2-2843 (Phase 1). CUR2-2844 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent ed06f94 commit 4d9a688

19 files changed

Lines changed: 273 additions & 783 deletions

‎dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_settler_agg.sql‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
{% macro zeroex_settler_agg(blockchain, start_date='2024-07-15') %}
2+
{%- if target.name == 'ci' -%}
3+
{%- set start_date = (modules.datetime.date.today() - modules.datetime.timedelta(days=14)).strftime('%Y-%m-%d') -%}
4+
{%- endif -%}
5+
{%- set weth = '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2' -%}
6+
{%- set native_tokens = '(0x0000000000000000000000000000000000000000, 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee)' -%}
7+
{%- set erc20_transfer_topic = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' -%}
8+
9+
-- Deterministic 0x Settler aggregator decode (replaces the heuristic leg-matching + the PR #9795 band-aid).
10+
-- Per settler call, emits ONE 0x-API aggregator row = the user's net swap:
11+
-- token_bought = AllowedSlippage.buyToken (from calldata, deterministic).
12+
-- token_bought amount = the unique Transfer(buyToken, to=receiver) when resolvable, else the minAmountOut
13+
-- floor (verified tight, ~1% under actual). receiver = the tx-level sender (execute) / msgSender
14+
-- (executeMetaTxn) — verified to be the true buyToken recipient, NOT the calldata `recipient` (which is
15+
-- often an intermediate routing hop). Anchoring on a single calldata-named token to the verified user
16+
-- means it cannot mis-bind to internal routing hops the way the heuristic did.
17+
-- token_sold = best-effort: the unique Transfer(from=receiver, token != buyToken).
18+
-- volume_usd = priced via either leg (add_amount_usd), so an unpriced buyToken still values off the sell leg.
19+
-- NOTE: native-ETH buyToken is mapped to mainnet WETH for pricing, so it only resolves a price on ethereum;
20+
-- per-chain wrapped-native pricing for native buys is a tracked follow-up (off-mainnet native buys -> NULL volume).
21+
22+
WITH settler AS (
23+
SELECT
24+
tx_hash, block_time, block_number, method_id, settler_address, zid, tag, rn,
25+
buy_token, min_amount_out, settler_msgsender
26+
FROM {{ ref('zeroex_v2_' ~ blockchain ~ '_settler_txs') }}
27+
{% if is_incremental() %}
28+
WHERE {{ incremental_predicate('block_time') }}
29+
{% else %}
30+
WHERE block_time >= DATE '{{ start_date }}'
31+
{% endif %}
32+
),
33+
34+
-- Distinct partition key of the settler txs, so the transactions/logs scans prune by partition
35+
-- (block_time + block_number + tx_hash) rather than a tx_hash-only post-scan filter, per zeroex_v2.sql's norm.
36+
settler_tx_keys AS (
37+
SELECT DISTINCT block_time, block_number, tx_hash FROM settler
38+
),
39+
40+
-- Tx-level sender/recipient (the settler trace's own `from` is an intermediary for nested/Relay-routed calls).
41+
txs AS (
42+
SELECT t.hash AS tx_hash, t."from" AS tx_from, t."to" AS tx_to
43+
FROM {{ source(blockchain, 'transactions') }} t
44+
JOIN settler_tx_keys k
45+
ON k.block_time = t.block_time
46+
AND k.block_number = t.block_number
47+
AND k.tx_hash = t.hash
48+
{% if is_incremental() %}
49+
WHERE {{ incremental_predicate('t.block_time') }}
50+
{% else %}
51+
WHERE t.block_time >= DATE '{{ start_date }}'
52+
{% endif %}
53+
),
54+
55+
calls AS (
56+
SELECT
57+
s.tx_hash, s.block_time, s.block_number, s.settler_address, s.zid, s.tag, s.rn, s.min_amount_out,
58+
t.tx_from, t.tx_to,
59+
-- receiver (the user): execute -> tx-level sender; executeMetaTxn -> msgSender (relayer pays gas).
60+
CASE WHEN s.method_id = 0xfd3ad6d4 THEN s.settler_msgsender ELSE t.tx_from END AS receiver,
61+
-- native sentinels represented/priced as WETH (see header note)
62+
CASE WHEN s.buy_token IN {{ native_tokens }} THEN {{ weth }} ELSE s.buy_token END AS buy_token
63+
FROM settler s
64+
LEFT JOIN txs t ON t.tx_hash = s.tx_hash
65+
),
66+
67+
transfers AS (
68+
SELECT
69+
logs.block_number,
70+
logs.tx_hash,
71+
logs.contract_address AS token,
72+
varbinary_substring(logs.topic1, 13, 20) AS transfer_from,
73+
varbinary_substring(logs.topic2, 13, 20) AS transfer_to,
74+
bytearray_to_uint256(logs.data) AS amount
75+
FROM {{ source(blockchain, 'logs') }} AS logs
76+
JOIN settler_tx_keys k
77+
ON k.block_time = logs.block_time
78+
AND k.block_number = logs.block_number
79+
AND k.tx_hash = logs.tx_hash
80+
WHERE logs.topic0 = {{ erc20_transfer_topic }}
81+
{% if is_incremental() %}
82+
AND {{ incremental_predicate('logs.block_time') }}
83+
{% else %}
84+
AND logs.block_time >= DATE '{{ start_date }}'
85+
{% endif %}
86+
),
87+
88+
-- Single pass over the transfers: buy leg = unique Transfer(buyToken, to=receiver); sell leg (best-effort) =
89+
-- unique Transfer out of the user of a token other than buyToken.
90+
legs AS (
91+
SELECT c.tx_hash, c.rn,
92+
count(*) FILTER (WHERE t.token = c.buy_token AND t.transfer_to = c.receiver) AS buy_n,
93+
arbitrary(t.amount) FILTER (WHERE t.token = c.buy_token AND t.transfer_to = c.receiver) AS buy_amount,
94+
count(*) FILTER (WHERE t.transfer_from = c.receiver AND t.token <> c.buy_token) AS sell_n,
95+
arbitrary(t.token) FILTER (WHERE t.transfer_from = c.receiver AND t.token <> c.buy_token) AS sell_token,
96+
arbitrary(t.amount) FILTER (WHERE t.transfer_from = c.receiver AND t.token <> c.buy_token) AS sell_amount
97+
FROM calls c
98+
JOIN transfers t
99+
ON t.tx_hash = c.tx_hash
100+
AND t.block_number = c.block_number
101+
AND t.amount > UINT256 '0'
102+
AND (
103+
(t.token = c.buy_token AND t.transfer_to = c.receiver)
104+
OR (t.transfer_from = c.receiver AND t.token <> c.buy_token)
105+
)
106+
GROUP BY c.tx_hash, c.rn
107+
),
108+
109+
token_metadata AS (
110+
SELECT contract_address, symbol, decimals
111+
FROM {{ source('tokens', 'erc20') }}
112+
WHERE blockchain = '{{ blockchain }}'
113+
),
114+
115+
trades AS (
116+
SELECT
117+
c.block_time, c.block_number, c.tx_hash, c.tx_from, c.tx_to, c.zid, c.tag,
118+
c.rn AS evt_index, c.settler_address AS contract_address, c.receiver AS taker,
119+
c.buy_token AS token_bought_address,
120+
CASE WHEN l.buy_n = 1 THEN l.buy_amount ELSE c.min_amount_out END AS token_bought_amount_raw,
121+
CASE WHEN l.sell_n = 1 THEN l.sell_token END AS token_sold_address,
122+
CASE WHEN l.sell_n = 1 THEN l.sell_amount END AS token_sold_amount_raw
123+
FROM calls c
124+
LEFT JOIN legs l ON l.tx_hash = c.tx_hash AND l.rn = c.rn
125+
),
126+
127+
results AS (
128+
SELECT
129+
'{{ blockchain }}' AS blockchain,
130+
trades.block_time, trades.block_number, trades.tx_hash, trades.tx_from, trades.tx_to,
131+
trades.zid, trades.tag, trades.evt_index, trades.contract_address, trades.taker,
132+
CAST(NULL AS varbinary) AS maker,
133+
trades.token_bought_address,
134+
bt.symbol AS bought_symbol,
135+
trades.token_bought_amount_raw,
136+
trades.token_bought_amount_raw / POW(10, bt.decimals) AS token_bought_amount,
137+
trades.token_sold_address,
138+
st.symbol AS sold_symbol,
139+
trades.token_sold_amount_raw,
140+
trades.token_sold_amount_raw / POW(10, st.decimals) AS token_sold_amount
141+
FROM trades
142+
LEFT JOIN token_metadata bt ON bt.contract_address = trades.token_bought_address
143+
LEFT JOIN token_metadata st ON st.contract_address = trades.token_sold_address
144+
),
145+
146+
results_usd AS (
147+
{{ add_amount_usd(trades_cte = 'results') }}
148+
)
149+
150+
SELECT
151+
'{{ blockchain }}' AS blockchain,
152+
'0x-API' AS project,
153+
'settler' AS version,
154+
cast(DATE_TRUNC('day', block_time) as date) AS block_date,
155+
cast(DATE_TRUNC('month', block_time) as date) AS block_month,
156+
block_time,
157+
block_number,
158+
sold_symbol AS taker_symbol,
159+
bought_symbol AS maker_symbol,
160+
CASE WHEN LOWER(sold_symbol) > LOWER(bought_symbol) THEN CONCAT(bought_symbol, '-', sold_symbol) ELSE CONCAT(sold_symbol, '-', bought_symbol) END AS token_pair,
161+
token_sold_amount AS taker_token_amount,
162+
token_bought_amount AS maker_token_amount,
163+
token_sold_amount_raw AS taker_token_amount_raw,
164+
token_bought_amount_raw AS maker_token_amount_raw,
165+
amount_usd AS volume_usd,
166+
token_sold_address AS taker_token,
167+
token_bought_address AS maker_token,
168+
taker,
169+
maker,
170+
tag,
171+
zid,
172+
tx_hash,
173+
tx_from,
174+
tx_to,
175+
evt_index,
176+
(ARRAY[-1]) AS trace_address,
177+
'settler' AS type,
178+
TRUE AS swap_flag,
179+
contract_address
180+
FROM results_usd
181+
{% endmacro %}

‎dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_settler_txs_cte.sql‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,13 @@ settler_txs AS (
112112
tx_from,
113113
-- Keep raw calldata only for RFQ-bearing settler calls (plain RFQ action 0xd92aadfb);
114114
-- consumed by the zeroex_settler_rfq macro. NULL otherwise to avoid bloating the staging table.
115-
CASE WHEN varbinary_position(input, 0xd92aadfb) <> 0 THEN input END AS rfq_input
115+
CASE WHEN varbinary_position(input, 0xd92aadfb) <> 0 THEN input END AS rfq_input,
116+
-- AllowedSlippage fields (present in every execute/executeMetaTxn call, fixed offsets) — used by
117+
-- the deterministic aggregator decode (zeroex_settler_agg): buyToken, minAmountOut, and the
118+
-- executeMetaTxn msgSender (the order signer / fund owner; tx-level from is the relayer there).
119+
varbinary_substring(input, 49, 20) AS buy_token,
120+
bytearray_to_uint256(varbinary_substring(input, 69, 32)) AS min_amount_out,
121+
CASE WHEN method_id = 0xfd3ad6d4 THEN varbinary_substring(input, 177, 20) END AS settler_msgsender
116122
FROM
117123
settler_trace_data
118124
WHERE

‎dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_v2_arbitrum_trades.sql‎

Lines changed: 5 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -10,49 +10,8 @@
1010
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
1111
)}}
1212

13-
{% set zeroex_settler_start_date = '2024-07-15' %}
14-
{% set blockchain = 'arbitrum' %}
15-
16-
WITH zeroex_tx AS (
17-
-- Read the pre-materialized settler transactions instead of inlining the
18-
-- zeroex_settler_txs_cte macro, which Trino re-expanded into ~14 arbitrum.traces scans.
19-
select
20-
tx_hash,
21-
block_time,
22-
block_number,
23-
method_id,
24-
contract_address,
25-
settler_address,
26-
zid,
27-
tag,
28-
rn,
29-
cow_rn,
30-
taker
31-
from {{ ref('zeroex_v2_arbitrum_settler_txs') }}
32-
{% if is_incremental() %}
33-
where {{ incremental_predicate('block_time') }}
34-
{% endif %}
35-
),
36-
zeroex_v2_trades AS (
37-
{{
38-
zeroex_v2_trades(
39-
blockchain = blockchain,
40-
start_date = zeroex_settler_start_date
41-
42-
)
43-
}}
44-
),
45-
46-
trade_details as (
47-
{{
48-
zeroex_v2_trades_detail(
49-
blockchain = blockchain,
50-
start_date = zeroex_settler_start_date
51-
52-
)
53-
}}
54-
55-
)
56-
select
57-
*
58-
from trade_details
13+
{{
14+
zeroex_settler_agg(
15+
blockchain = 'arbitrum'
16+
)
17+
}}

‎dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_v2_avalanche_c_trades.sql‎

Lines changed: 5 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -10,49 +10,8 @@
1010
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
1111
)}}
1212

13-
{% set zeroex_settler_start_date = '2024-07-15' %}
14-
{% set blockchain = 'avalanche_c' %}
15-
16-
WITH zeroex_tx AS (
17-
-- Read the pre-materialized settler transactions instead of inlining the
18-
-- zeroex_settler_txs_cte macro, which Trino re-expanded into ~14 avalanche_c.traces scans.
19-
select
20-
tx_hash,
21-
block_time,
22-
block_number,
23-
method_id,
24-
contract_address,
25-
settler_address,
26-
zid,
27-
tag,
28-
rn,
29-
cow_rn,
30-
taker
31-
from {{ ref('zeroex_v2_avalanche_c_settler_txs') }}
32-
{% if is_incremental() %}
33-
where {{ incremental_predicate('block_time') }}
34-
{% endif %}
35-
),
36-
zeroex_v2_trades AS (
37-
{{
38-
zeroex_v2_trades(
39-
blockchain = blockchain,
40-
start_date = zeroex_settler_start_date
41-
42-
)
43-
}}
44-
),
45-
46-
trade_details as (
47-
{{
48-
zeroex_v2_trades_detail(
49-
blockchain = blockchain,
50-
start_date = zeroex_settler_start_date
51-
52-
)
53-
}}
54-
55-
)
56-
select
57-
*
58-
from trade_details
13+
{{
14+
zeroex_settler_agg(
15+
blockchain = 'avalanche_c'
16+
)
17+
}}

‎dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v2_base_trades.sql‎

Lines changed: 5 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -10,49 +10,8 @@
1010
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
1111
)}}
1212

13-
{% set zeroex_settler_start_date = '2024-07-15' %}
14-
{% set blockchain = 'base' %}
15-
16-
WITH zeroex_tx AS (
17-
-- Read the pre-materialized settler transactions instead of inlining the
18-
-- zeroex_settler_txs_cte macro, which Trino re-expanded into ~14 base.traces scans.
19-
select
20-
tx_hash,
21-
block_time,
22-
block_number,
23-
method_id,
24-
contract_address,
25-
settler_address,
26-
zid,
27-
tag,
28-
rn,
29-
cow_rn,
30-
taker
31-
from {{ ref('zeroex_v2_base_settler_txs') }}
32-
{% if is_incremental() %}
33-
where {{ incremental_predicate('block_time') }}
34-
{% endif %}
35-
),
36-
zeroex_v2_trades AS (
37-
{{
38-
zeroex_v2_trades(
39-
blockchain = blockchain,
40-
start_date = zeroex_settler_start_date
41-
42-
)
43-
}}
44-
),
45-
46-
trade_details as (
47-
{{
48-
zeroex_v2_trades_detail(
49-
blockchain = blockchain,
50-
start_date = zeroex_settler_start_date
51-
52-
)
53-
}}
54-
55-
)
56-
select
57-
*
58-
from trade_details
13+
{{
14+
zeroex_settler_agg(
15+
blockchain = 'base'
16+
)
17+
}}

0 commit comments

Comments
 (0)