Skip to content

Commit 47d2196

Browse files
thevaizmanclaude
thevaizman
and
claude
committed
feat(dex): deterministic 0x Settler aggregator volume — supersede PR #9795 (CUR2-2844)
Replaces the heuristic maker/taker leg-matching (and the PR #9795 5x-divergence band-aid) for 0x Settler trades in dex_aggregator.trades with a deterministic decode. Per settler call, emits one 0x-API aggregator row = the user's net swap: - token_bought = AllowedSlippage.buyToken (from calldata, deterministic). - amount = the unique Transfer(buyToken, to=receiver) when resolvable, else the minAmountOut floor (verified tight, ~1% under actual). receiver = tx-level sender (execute) / msgSender (executeMetaTxn) — verified to be the true buyToken recipient, NOT the calldata recipient (an intermediate routing hop). Anchoring on a single calldata-named token to the verified user means it cannot mis-bind to internal routing hops the way the heuristic did. - token_sold = best-effort (unique Transfer out of the user, token != buyToken). - volume_usd priced via either leg (add_amount_usd), so an unpriced buyToken still values off the sell leg (cuts the unpriced-NULL bucket ~20% -> ~5%). New macro zeroex_settler_agg; settler-txs staging extended with the AllowedSlippage fields (buy_token, min_amount_out, settler_msgsender); all 17 settler chains' zeroex_v2_<chain>_trades rewired to it. Verified on-chain (example 0xf61374… : current $0.75 -> deterministic ~$490). NOTE: requires a coordinated full-refresh of zeroex_v2_<chain>_trades + dex_aggregator_trades (merge cannot delete the old mis-bound rows). Old-vs-new regression to be validated on CI-built tables before merge. Stacked on CUR2-2843 (Phase 1). CUR2-2844 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent a00011e commit 47d2196

19 files changed

Lines changed: 279 additions & 783 deletions

‎dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_settler_agg.sql‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
{% macro zeroex_settler_agg(blockchain, start_date='2024-07-15') %}
2+
{%- if target.name == 'ci' -%}
3+
{%- set start_date = (modules.datetime.date.today() - modules.datetime.timedelta(days=14)).strftime('%Y-%m-%d') -%}
4+
{%- endif -%}
5+
{%- set weth = '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2' -%}
6+
{%- set native_tokens = '(0x0000000000000000000000000000000000000000, 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee)' -%}
7+
{%- set erc20_transfer_topic = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' -%}
8+
9+
-- Deterministic 0x Settler aggregator decode (replaces the heuristic leg-matching + the PR #9795 band-aid).
10+
-- Per settler call, emits ONE 0x-API aggregator row = the user's net swap:
11+
-- token_bought = AllowedSlippage.buyToken (from calldata, deterministic).
12+
-- token_bought amount = the unique Transfer(buyToken, to=receiver) when resolvable, else the minAmountOut
13+
-- floor (verified tight, ~1% under actual). receiver = the tx-level sender (execute) / msgSender
14+
-- (executeMetaTxn) — verified to be the true buyToken recipient, NOT the calldata `recipient` (which is
15+
-- often an intermediate routing hop). Anchoring on a single calldata-named token to the verified user
16+
-- means it cannot mis-bind to internal routing hops the way the heuristic did.
17+
-- token_sold = best-effort: the unique Transfer(from=receiver, token != buyToken).
18+
-- volume_usd = priced via either leg (add_amount_usd), so an unpriced buyToken still values off the sell leg.
19+
-- NOTE: native-ETH buyToken is mapped to mainnet WETH for pricing, so it only resolves a price on ethereum;
20+
-- per-chain wrapped-native pricing for native buys is a tracked follow-up (off-mainnet native buys -> NULL volume).
21+
22+
WITH settler AS (
23+
SELECT
24+
tx_hash, block_time, block_number, method_id, settler_address, zid, tag, rn,
25+
buy_token, min_amount_out, settler_msgsender
26+
FROM {{ ref('zeroex_v2_' ~ blockchain ~ '_settler_txs') }}
27+
{% if is_incremental() %}
28+
WHERE {{ incremental_predicate('block_time') }}
29+
{% else %}
30+
WHERE block_time >= DATE '{{ start_date }}'
31+
{% endif %}
32+
),
33+
34+
-- Distinct partition key of the settler txs, so the transactions/logs scans prune by partition
35+
-- (block_time + block_number + tx_hash) rather than a tx_hash-only post-scan filter, per zeroex_v2.sql's norm.
36+
settler_tx_keys AS (
37+
SELECT DISTINCT block_time, block_number, tx_hash FROM settler
38+
),
39+
40+
-- Tx-level sender/recipient (the settler trace's own `from` is an intermediary for nested/Relay-routed calls).
41+
txs AS (
42+
SELECT t.hash AS tx_hash, t."from" AS tx_from, t."to" AS tx_to
43+
FROM {{ source(blockchain, 'transactions') }} t
44+
JOIN settler_tx_keys k
45+
ON k.block_time = t.block_time
46+
AND k.block_number = t.block_number
47+
AND k.tx_hash = t.hash
48+
{% if is_incremental() %}
49+
WHERE {{ incremental_predicate('t.block_time') }}
50+
{% else %}
51+
WHERE t.block_time >= DATE '{{ start_date }}'
52+
{% endif %}
53+
),
54+
55+
calls AS (
56+
SELECT
57+
s.tx_hash, s.block_time, s.block_number, s.settler_address, s.zid, s.tag, s.rn, s.min_amount_out,
58+
t.tx_from, t.tx_to,
59+
-- receiver (the user): execute -> tx-level sender; executeMetaTxn -> msgSender (relayer pays gas).
60+
CASE WHEN s.method_id = 0xfd3ad6d4 THEN s.settler_msgsender ELSE t.tx_from END AS receiver,
61+
-- native sentinels represented/priced as WETH (see header note)
62+
CASE WHEN s.buy_token IN {{ native_tokens }} THEN {{ weth }} ELSE s.buy_token END AS buy_token
63+
FROM settler s
64+
LEFT JOIN txs t ON t.tx_hash = s.tx_hash
65+
),
66+
67+
transfers AS (
68+
SELECT
69+
logs.block_number,
70+
logs.tx_hash,
71+
logs.contract_address AS token,
72+
varbinary_substring(logs.topic1, 13, 20) AS transfer_from,
73+
varbinary_substring(logs.topic2, 13, 20) AS transfer_to,
74+
-- CASE-guard the conversion (Trino only evaluates the THEN branch when the WHEN holds): a non-standard
75+
-- Transfer-topic log with >32-byte data would otherwise overflow bytearray_to_uint256. The WHERE filter
76+
-- below is not sufficient on its own — Trino may evaluate this projection before applying it.
77+
CASE WHEN varbinary_length(logs.data) = 32 THEN bytearray_to_uint256(logs.data) END AS amount
78+
FROM {{ source(blockchain, 'logs') }} AS logs
79+
JOIN settler_tx_keys k
80+
ON k.block_time = logs.block_time
81+
AND k.block_number = logs.block_number
82+
AND k.tx_hash = logs.tx_hash
83+
WHERE logs.topic0 = {{ erc20_transfer_topic }}
84+
-- standard ERC20 value transfers only (uint256 amount is exactly 32 bytes): drops NFT/ERC721 (0-byte
85+
-- data) and non-standard >32-byte Transfer-topic logs. Row-reducer; the overflow guard is the CASE above.
86+
AND varbinary_length(logs.data) = 32
87+
{% if is_incremental() %}
88+
AND {{ incremental_predicate('logs.block_time') }}
89+
{% else %}
90+
AND logs.block_time >= DATE '{{ start_date }}'
91+
{% endif %}
92+
),
93+
94+
-- Single pass over the transfers: buy leg = unique Transfer(buyToken, to=receiver); sell leg (best-effort) =
95+
-- unique Transfer out of the user of a token other than buyToken.
96+
legs AS (
97+
SELECT c.tx_hash, c.rn,
98+
count(*) FILTER (WHERE t.token = c.buy_token AND t.transfer_to = c.receiver) AS buy_n,
99+
arbitrary(t.amount) FILTER (WHERE t.token = c.buy_token AND t.transfer_to = c.receiver) AS buy_amount,
100+
count(*) FILTER (WHERE t.transfer_from = c.receiver AND t.token <> c.buy_token) AS sell_n,
101+
arbitrary(t.token) FILTER (WHERE t.transfer_from = c.receiver AND t.token <> c.buy_token) AS sell_token,
102+
arbitrary(t.amount) FILTER (WHERE t.transfer_from = c.receiver AND t.token <> c.buy_token) AS sell_amount
103+
FROM calls c
104+
JOIN transfers t
105+
ON t.tx_hash = c.tx_hash
106+
AND t.block_number = c.block_number
107+
AND t.amount > UINT256 '0'
108+
AND (
109+
(t.token = c.buy_token AND t.transfer_to = c.receiver)
110+
OR (t.transfer_from = c.receiver AND t.token <> c.buy_token)
111+
)
112+
GROUP BY c.tx_hash, c.rn
113+
),
114+
115+
token_metadata AS (
116+
SELECT contract_address, symbol, decimals
117+
FROM {{ source('tokens', 'erc20') }}
118+
WHERE blockchain = '{{ blockchain }}'
119+
),
120+
121+
trades AS (
122+
SELECT
123+
c.block_time, c.block_number, c.tx_hash, c.tx_from, c.tx_to, c.zid, c.tag,
124+
c.rn AS evt_index, c.settler_address AS contract_address, c.receiver AS taker,
125+
c.buy_token AS token_bought_address,
126+
CASE WHEN l.buy_n = 1 THEN l.buy_amount ELSE c.min_amount_out END AS token_bought_amount_raw,
127+
CASE WHEN l.sell_n = 1 THEN l.sell_token END AS token_sold_address,
128+
CASE WHEN l.sell_n = 1 THEN l.sell_amount END AS token_sold_amount_raw
129+
FROM calls c
130+
LEFT JOIN legs l ON l.tx_hash = c.tx_hash AND l.rn = c.rn
131+
),
132+
133+
results AS (
134+
SELECT
135+
'{{ blockchain }}' AS blockchain,
136+
trades.block_time, trades.block_number, trades.tx_hash, trades.tx_from, trades.tx_to,
137+
trades.zid, trades.tag, trades.evt_index, trades.contract_address, trades.taker,
138+
CAST(NULL AS varbinary) AS maker,
139+
trades.token_bought_address,
140+
bt.symbol AS bought_symbol,
141+
trades.token_bought_amount_raw,
142+
trades.token_bought_amount_raw / POW(10, bt.decimals) AS token_bought_amount,
143+
trades.token_sold_address,
144+
st.symbol AS sold_symbol,
145+
trades.token_sold_amount_raw,
146+
trades.token_sold_amount_raw / POW(10, st.decimals) AS token_sold_amount
147+
FROM trades
148+
LEFT JOIN token_metadata bt ON bt.contract_address = trades.token_bought_address
149+
LEFT JOIN token_metadata st ON st.contract_address = trades.token_sold_address
150+
),
151+
152+
results_usd AS (
153+
{{ add_amount_usd(trades_cte = 'results') }}
154+
)
155+
156+
SELECT
157+
'{{ blockchain }}' AS blockchain,
158+
'0x-API' AS project,
159+
'settler' AS version,
160+
cast(DATE_TRUNC('day', block_time) as date) AS block_date,
161+
cast(DATE_TRUNC('month', block_time) as date) AS block_month,
162+
block_time,
163+
block_number,
164+
sold_symbol AS taker_symbol,
165+
bought_symbol AS maker_symbol,
166+
CASE WHEN LOWER(sold_symbol) > LOWER(bought_symbol) THEN CONCAT(bought_symbol, '-', sold_symbol) ELSE CONCAT(sold_symbol, '-', bought_symbol) END AS token_pair,
167+
token_sold_amount AS taker_token_amount,
168+
token_bought_amount AS maker_token_amount,
169+
token_sold_amount_raw AS taker_token_amount_raw,
170+
token_bought_amount_raw AS maker_token_amount_raw,
171+
amount_usd AS volume_usd,
172+
token_sold_address AS taker_token,
173+
token_bought_address AS maker_token,
174+
taker,
175+
maker,
176+
tag,
177+
zid,
178+
tx_hash,
179+
tx_from,
180+
tx_to,
181+
evt_index,
182+
(ARRAY[-1]) AS trace_address,
183+
'settler' AS type,
184+
TRUE AS swap_flag,
185+
contract_address
186+
FROM results_usd
187+
{% endmacro %}

‎dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_settler_txs_cte.sql‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,13 @@ settler_txs AS (
110110
END AS taker,
111111
-- Keep raw calldata only for RFQ-bearing settler calls (plain RFQ action 0xd92aadfb);
112112
-- consumed by the zeroex_settler_rfq macro. NULL otherwise to avoid bloating the staging table.
113-
CASE WHEN varbinary_position(input, 0xd92aadfb) <> 0 THEN input END AS rfq_input
113+
CASE WHEN varbinary_position(input, 0xd92aadfb) <> 0 THEN input END AS rfq_input,
114+
-- AllowedSlippage fields (present in every execute/executeMetaTxn call, fixed offsets) — used by
115+
-- the deterministic aggregator decode (zeroex_settler_agg): buyToken, minAmountOut, and the
116+
-- executeMetaTxn msgSender (the order signer / fund owner; tx-level from is the relayer there).
117+
varbinary_substring(input, 49, 20) AS buy_token,
118+
bytearray_to_uint256(varbinary_substring(input, 69, 32)) AS min_amount_out,
119+
CASE WHEN method_id = 0xfd3ad6d4 THEN varbinary_substring(input, 177, 20) END AS settler_msgsender
114120
FROM
115121
settler_trace_data
116122
WHERE

‎dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_v2_arbitrum_trades.sql‎

Lines changed: 5 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -10,49 +10,8 @@
1010
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
1111
)}}
1212

13-
{% set zeroex_settler_start_date = '2024-07-15' %}
14-
{% set blockchain = 'arbitrum' %}
15-
16-
WITH zeroex_tx AS (
17-
-- Read the pre-materialized settler transactions instead of inlining the
18-
-- zeroex_settler_txs_cte macro, which Trino re-expanded into ~14 arbitrum.traces scans.
19-
select
20-
tx_hash,
21-
block_time,
22-
block_number,
23-
method_id,
24-
contract_address,
25-
settler_address,
26-
zid,
27-
tag,
28-
rn,
29-
cow_rn,
30-
taker
31-
from {{ ref('zeroex_v2_arbitrum_settler_txs') }}
32-
{% if is_incremental() %}
33-
where {{ incremental_predicate('block_time') }}
34-
{% endif %}
35-
),
36-
zeroex_v2_trades AS (
37-
{{
38-
zeroex_v2_trades(
39-
blockchain = blockchain,
40-
start_date = zeroex_settler_start_date
41-
42-
)
43-
}}
44-
),
45-
46-
trade_details as (
47-
{{
48-
zeroex_v2_trades_detail(
49-
blockchain = blockchain,
50-
start_date = zeroex_settler_start_date
51-
52-
)
53-
}}
54-
55-
)
56-
select
57-
*
58-
from trade_details
13+
{{
14+
zeroex_settler_agg(
15+
blockchain = 'arbitrum'
16+
)
17+
}}

‎dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_v2_avalanche_c_trades.sql‎

Lines changed: 5 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -10,49 +10,8 @@
1010
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
1111
)}}
1212

13-
{% set zeroex_settler_start_date = '2024-07-15' %}
14-
{% set blockchain = 'avalanche_c' %}
15-
16-
WITH zeroex_tx AS (
17-
-- Read the pre-materialized settler transactions instead of inlining the
18-
-- zeroex_settler_txs_cte macro, which Trino re-expanded into ~14 avalanche_c.traces scans.
19-
select
20-
tx_hash,
21-
block_time,
22-
block_number,
23-
method_id,
24-
contract_address,
25-
settler_address,
26-
zid,
27-
tag,
28-
rn,
29-
cow_rn,
30-
taker
31-
from {{ ref('zeroex_v2_avalanche_c_settler_txs') }}
32-
{% if is_incremental() %}
33-
where {{ incremental_predicate('block_time') }}
34-
{% endif %}
35-
),
36-
zeroex_v2_trades AS (
37-
{{
38-
zeroex_v2_trades(
39-
blockchain = blockchain,
40-
start_date = zeroex_settler_start_date
41-
42-
)
43-
}}
44-
),
45-
46-
trade_details as (
47-
{{
48-
zeroex_v2_trades_detail(
49-
blockchain = blockchain,
50-
start_date = zeroex_settler_start_date
51-
52-
)
53-
}}
54-
55-
)
56-
select
57-
*
58-
from trade_details
13+
{{
14+
zeroex_settler_agg(
15+
blockchain = 'avalanche_c'
16+
)
17+
}}

‎dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v2_base_trades.sql‎

Lines changed: 5 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -10,49 +10,8 @@
1010
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
1111
)}}
1212

13-
{% set zeroex_settler_start_date = '2024-07-15' %}
14-
{% set blockchain = 'base' %}
15-
16-
WITH zeroex_tx AS (
17-
-- Read the pre-materialized settler transactions instead of inlining the
18-
-- zeroex_settler_txs_cte macro, which Trino re-expanded into ~14 base.traces scans.
19-
select
20-
tx_hash,
21-
block_time,
22-
block_number,
23-
method_id,
24-
contract_address,
25-
settler_address,
26-
zid,
27-
tag,
28-
rn,
29-
cow_rn,
30-
taker
31-
from {{ ref('zeroex_v2_base_settler_txs') }}
32-
{% if is_incremental() %}
33-
where {{ incremental_predicate('block_time') }}
34-
{% endif %}
35-
),
36-
zeroex_v2_trades AS (
37-
{{
38-
zeroex_v2_trades(
39-
blockchain = blockchain,
40-
start_date = zeroex_settler_start_date
41-
42-
)
43-
}}
44-
),
45-
46-
trade_details as (
47-
{{
48-
zeroex_v2_trades_detail(
49-
blockchain = blockchain,
50-
start_date = zeroex_settler_start_date
51-
52-
)
53-
}}
54-
55-
)
56-
select
57-
*
58-
from trade_details
13+
{{
14+
zeroex_settler_agg(
15+
blockchain = 'base'
16+
)
17+
}}

0 commit comments

Comments
 (0)