Skip to content

Commit ea4cb16

Browse files
nyx-discochucknyx-discochuck
authored
Merge branch 'main' into add-ramsesxyz-hyperevm-dex-trades
2 parents 0420082 + e3a7dc9 commit ea4cb16

11 files changed

Lines changed: 202 additions & 68 deletions

‎dbt_subprojects/hourly_spellbook/macros/project/safe_native_transfers.sql‎

Lines changed: 4 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -12,42 +12,16 @@ from
1212
CAST(date_trunc('day', et.block_time) as date) as block_date,
1313
CAST(date_trunc('month', et.block_time) as DATE) as block_month,
1414
et.block_time,
15-
-CAST(et.value AS INT256) as amount_raw,
15+
case when side.is_from then -CAST(et.value AS INT256) else CAST(et.value AS INT256) end as amount_raw,
1616
et.tx_hash,
1717
array_join(et.trace_address, ',') as trace_address
1818
from
1919
{{ source(blockchain, 'traces') }} et
20+
-- fan each trace out to its (from, to) sides so traces is scanned once instead of once per join side
21+
cross join unnest(array[et."from", et.to], array[true, false]) as side(side_address, is_from)
2022
join
2123
{{ ref('safe_' ~ blockchain ~ '_safes') }} s
22-
on et."from" = s.address
23-
where
24-
et."from" != et.to -- exclude calls to self to guarantee unique key property
25-
and et.success = true
26-
and (lower(et.call_type) not in ('delegatecall', 'callcode', 'staticcall') or et.call_type is null)
27-
and et.value > UINT256 '0' -- et.value is uint256 type
28-
{% if not is_incremental() %}
29-
and et.block_time >= TIMESTAMP '{{ project_start_date }}'
30-
{% else %}
31-
and et.block_time > date_trunc('day', now() - interval '10' day) -- to prevent potential counterfactual safe deployment issues we take a bigger interval
32-
{% endif %}
33-
34-
union all
35-
36-
select
37-
'{{ blockchain }}' as blockchain,
38-
'{{ native_token_symbol }}' as symbol,
39-
s.address,
40-
CAST(date_trunc('day', et.block_time) as date) as block_date,
41-
CAST(date_trunc('month', et.block_time) as DATE) as block_month,
42-
et.block_time,
43-
CAST(et.value AS INT256) as amount_raw,
44-
et.tx_hash,
45-
array_join(et.trace_address, ',') as trace_address
46-
from
47-
{{ source(blockchain, 'traces') }} et
48-
join
49-
{{ ref('safe_' ~ blockchain ~ '_safes') }} s
50-
on et.to = s.address
24+
on side.side_address = s.address
5125
where
5226
et."from" != et.to -- exclude calls to self to guarantee unique key property
5327
and et.success = true

‎dbt_subprojects/hourly_spellbook/macros/sector/bridges/layerzero_v1_deposits.sql‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,17 @@ WITH send_calls AS (
1717
)
1818

1919
, distinct_calls AS (
20-
SELECT block_number
20+
SELECT block_time
21+
, block_number
2122
, tx_hash
2223
, MAX(call_send_index) AS max_call_send_index
2324
FROM send_calls
24-
GROUP BY 1, 2
25+
GROUP BY 1, 2, 3
2526
)
2627

2728
, transfers AS (
28-
SELECT block_number
29+
SELECT block_time
30+
, block_number
2931
, tx_hash
3032
, sender
3133
, recipient
@@ -36,7 +38,8 @@ WITH send_calls AS (
3638
, unique_key
3739
, rn
3840
FROM (
39-
SELECT t.block_number
41+
SELECT t.block_time
42+
, t.block_number
4043
, t.tx_hash
4144
, t."from" AS sender
4245
, t.to AS recipient
@@ -45,11 +48,15 @@ WITH send_calls AS (
4548
, t.contract_address AS deposit_token_address
4649
, t.evt_index
4750
, t.unique_key
48-
, ROW_NUMBER() OVER (PARTITION BY t.tx_hash ORDER BY COALESCE(t.trace_address, ARRAY[t.evt_index])) AS rn
51+
-- block_time in the window partition keys is redundant for correctness (tx_hash
52+
-- determines the block) but lets the planner propagate the incremental
53+
-- block_time predicate below the window instead of scanning all history
54+
, ROW_NUMBER() OVER (PARTITION BY t.block_time, t.block_number, t.tx_hash ORDER BY COALESCE(t.trace_address, ARRAY[t.evt_index])) AS rn
4955
, sc.max_call_send_index
5056
FROM {{ source('tokens_' + blockchain, 'transfers') }} t
5157
INNER JOIN {{ ref('bridges_layerzero_chain_indexes') }} i ON i.blockchain='{{blockchain}}'
52-
INNER JOIN distinct_calls sc ON t.block_number=sc.block_number
58+
INNER JOIN distinct_calls sc ON t.block_time=sc.block_time
59+
AND t.block_number=sc.block_number
5360
AND t.tx_hash=sc.tx_hash
5461
AND t.to=i.endpoint_address
5562
)
@@ -75,7 +82,8 @@ SELECT distinct '{{blockchain}}' AS deposit_chain
7582
, sc.contract_address
7683
, {{ dbt_utils.generate_surrogate_key(['sc.tx_hash', 't.evt_index', 'sc.call_send_index']) }} as bridge_transfer_id
7784
FROM send_calls sc
78-
LEFT JOIN transfers t ON t.block_number=sc.block_number
85+
LEFT JOIN transfers t ON t.block_time=sc.block_time
86+
AND t.block_number=sc.block_number
7987
AND t.tx_hash=sc.tx_hash
8088
AND t.rn=sc.call_send_index
8189
LEFT JOIN {{ ref('bridges_layerzero_chain_indexes') }} ci ON sc.withdrawal_chain_id=ci.id

‎dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_buffer_inflow.sql‎

Lines changed: 39 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,50 @@
22
schema='lido_accounting_ethereum',
33
alias = 'buffer_inflow',
44

5-
materialized = 'table',
6-
file_format = 'delta'
5+
materialized = 'incremental',
6+
file_format = 'delta',
7+
incremental_strategy = 'append'
78
, post_hook='{{ hide_spells() }}'
89
)
910
}}
1011

11-
SELECT evt_block_time as period, amount as amount,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 as token, evt_tx_hash, date_trunc('day', evt_block_time) as day
12-
FROM {{source('lido_ethereum','steth_evt_Submitted')}}
12+
with buffer_inflow as (
13+
SELECT evt_block_time as period, amount as amount,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 as token, evt_tx_hash, date_trunc('day', evt_block_time) as day
14+
FROM {{source('lido_ethereum','steth_evt_Submitted')}}
15+
{% if is_incremental() %}
16+
WHERE {{ incremental_predicate('evt_block_time') }}
17+
{% endif %}
1318

14-
union all
19+
union all
1520

16-
SELECT evt_block_time, amount, 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2, evt_tx_hash, date_trunc('day', evt_block_time) as day
17-
FROM {{source('lido_ethereum','steth_evt_ELRewardsReceived')}}
21+
SELECT evt_block_time, amount, 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2, evt_tx_hash, date_trunc('day', evt_block_time) as day
22+
FROM {{source('lido_ethereum','steth_evt_ELRewardsReceived')}}
23+
{% if is_incremental() %}
24+
WHERE {{ incremental_predicate('evt_block_time') }}
25+
{% endif %}
1826

19-
union all
27+
union all
2028

21-
SELECT evt_block_time, amount , 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2, evt_tx_hash, date_trunc('day', evt_block_time) as day
22-
FROM {{source('lido_ethereum','steth_evt_WithdrawalsReceived')}}
29+
SELECT evt_block_time, amount , 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2, evt_tx_hash, date_trunc('day', evt_block_time) as day
30+
FROM {{source('lido_ethereum','steth_evt_WithdrawalsReceived')}}
31+
{% if is_incremental() %}
32+
WHERE {{ incremental_predicate('evt_block_time') }}
33+
{% endif %}
34+
)
35+
36+
SELECT period, amount, token, evt_tx_hash, day
37+
FROM buffer_inflow b
38+
{% if is_incremental() %}
39+
-- append-only dedup: drop rows already inserted by a previous run inside the
40+
-- incremental window (no event index in the output, so a merge unique_key would
41+
-- collapse legitimately duplicated events within one tx)
42+
WHERE not exists (
43+
select 1
44+
from {{ this }} t
45+
where t.period = b.period
46+
and t.amount = b.amount
47+
and t.token = b.token
48+
and t.evt_tx_hash = b.evt_tx_hash
49+
and t.day = b.day
50+
)
51+
{% endif %}

‎dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_buffer_outflow.sql‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,27 @@
22
schema='lido_accounting_ethereum',
33
alias = 'buffer_outflow',
44

5-
materialized = 'table',
6-
file_format = 'delta'
5+
materialized = 'incremental',
6+
file_format = 'delta',
7+
incremental_strategy = 'append'
78
, post_hook='{{ hide_spells() }}'
89
)
910
}}
1011

1112
SELECT evt_block_time as period, amountOfETHLocked as amount, 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 AS token, evt_tx_hash, date_trunc('day', evt_block_time) as day
12-
FROM {{source('lido_ethereum','WithdrawalQueueERC721_evt_WithdrawalsFinalized')}}
13+
FROM {{source('lido_ethereum','WithdrawalQueueERC721_evt_WithdrawalsFinalized')}} w
14+
{% if is_incremental() %}
15+
WHERE {{ incremental_predicate('evt_block_time') }}
16+
-- append-only dedup: drop rows already inserted by a previous run inside the
17+
-- incremental window (no event index in the output, so a merge unique_key would
18+
-- collapse legitimately duplicated events within one tx)
19+
AND not exists (
20+
select 1
21+
from {{ this }} t
22+
where t.period = w.evt_block_time
23+
and t.amount = w.amountOfETHLocked
24+
and t.token = 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
25+
and t.evt_tx_hash = w.evt_tx_hash
26+
and t.day = date_trunc('day', w.evt_block_time)
27+
)
28+
{% endif %}

‎dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_dai_referral_payment.sql‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
schema='lido_accounting_ethereum',
33
alias = 'dai_referral_payment',
44

5-
materialized = 'table',
6-
file_format = 'delta'
5+
materialized = 'incremental',
6+
file_format = 'delta',
7+
incremental_strategy = 'append'
78
, post_hook='{{ hide_spells() }}'
89
)
910
}}
@@ -28,6 +29,9 @@ dai_referral_payment_txns AS (
2829
)
2930
AND evt_block_time >= CAST('2023-01-01 00:00' AS TIMESTAMP)
3031
AND contract_address = 0x6B175474E89094C44Da98b954EedeAC495271d0F
32+
{% if is_incremental() %}
33+
AND {{ incremental_predicate('evt_block_time') }}
34+
{% endif %}
3135
ORDER BY evt_block_time
3236
)
3337

@@ -36,4 +40,17 @@ dai_referral_payment_txns AS (
3640
evt_tx_hash,
3741
contract_address AS token,
3842
value AS amount_token
39-
FROM dai_referral_payment_txns
43+
FROM dai_referral_payment_txns d
44+
{% if is_incremental() %}
45+
-- append-only dedup: drop rows already inserted by a previous run inside the
46+
-- incremental window (no event index in the output, so a merge unique_key would
47+
-- collapse legitimately duplicated transfers within one tx)
48+
WHERE not exists (
49+
select 1
50+
from {{ this }} t
51+
where t.period = d.evt_block_time
52+
and t.evt_tx_hash = d.evt_tx_hash
53+
and t.token = d.contract_address
54+
and t.amount_token = d.value
55+
)
56+
{% endif %}

‎dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_ldo_referral_payment.sql‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
schema='lido_accounting_ethereum',
33
alias = 'ldo_referral_payment',
44

5-
materialized = 'table',
6-
file_format = 'delta'
5+
materialized = 'incremental',
6+
file_format = 'delta',
7+
incremental_strategy = 'append'
78
, post_hook='{{ hide_spells() }}'
89
)
910
}}
@@ -62,6 +63,9 @@ ldo_referral_payment_txns AS ( --only LDO referral program, need to add DAI refe
6263
AND _to IN (
6364
SELECT address FROM ldo_referral_payments_addr
6465
)
66+
{% if is_incremental() %}
67+
AND {{ incremental_predicate('evt_block_time') }}
68+
{% endif %}
6569
ORDER BY evt_block_time
6670

6771
)
@@ -70,6 +74,19 @@ ldo_referral_payment_txns AS ( --only LDO referral program, need to add DAI refe
7074
evt_tx_hash,
7175
contract_address AS token,
7276
amnt AS amount_token
73-
FROM ldo_referral_payment_txns
77+
FROM ldo_referral_payment_txns l
7478
WHERE amnt != 0
79+
{% if is_incremental() %}
80+
-- append-only dedup: drop rows already inserted by a previous run inside the
81+
-- incremental window (no event index in the output, so a merge unique_key would
82+
-- collapse legitimately duplicated transfers within one tx)
83+
AND not exists (
84+
select 1
85+
from {{ this }} t
86+
where t.period = l.evt_block_time
87+
and t.evt_tx_hash = l.evt_tx_hash
88+
and t.token = l.contract_address
89+
and t.amount_token = l.amnt
90+
)
91+
{% endif %}
7592

‎dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_lego_expenses.sql‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
schema='lido_accounting_ethereum',
33
alias = 'lego_expenses',
44

5-
materialized = 'table',
5+
materialized = 'incremental',
6+
incremental_strategy = 'append',
67
file_format = 'delta'
78
, post_hook='{{ hide_spells() }}'
89
)
@@ -90,6 +91,9 @@ lego_expenses_txns AS (
9091
contract_address
9192
FROM {{source('erc20_ethereum','evt_Transfer')}}
9293
WHERE contract_address IN (SELECT address FROM tokens)
94+
{% if is_incremental() %}
95+
AND {{ incremental_predicate('evt_block_time') }}
96+
{% endif %}
9397
AND "from" IN (
9498
SELECT
9599
address
@@ -112,6 +116,9 @@ lego_expenses_txns AS (
112116
contract_address
113117
FROM {{source('erc20_ethereum','evt_Transfer')}}
114118
WHERE contract_address IN (SELECT address FROM tokens)
119+
{% if is_incremental() %}
120+
AND {{ incremental_predicate('evt_block_time') }}
121+
{% endif %}
115122
AND to IN (
116123
SELECT
117124
address
@@ -131,6 +138,19 @@ lego_expenses_txns AS (
131138
contract_address AS token,
132139
value AS amount_token,
133140
evt_tx_hash
134-
FROM lego_expenses_txns
141+
FROM lego_expenses_txns l
135142
WHERE contract_address IN (SELECT address FROM tokens)
136143
and value != 0
144+
{% if is_incremental() %}
145+
-- append-only dedup: drop rows already inserted by a previous run inside the
146+
-- incremental window (the model has no event index, so a merge unique_key would
147+
-- collapse legitimately duplicated transfers within one tx)
148+
and not exists (
149+
select 1
150+
from {{ this }} t
151+
where t.period = l.evt_block_time
152+
and t.token = l.contract_address
153+
and t.amount_token = l.value
154+
and t.evt_tx_hash = l.evt_tx_hash
155+
)
156+
{% endif %}

‎dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_steth_referral_payment.sql‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
schema='lido_accounting_ethereum',
33
alias = 'steth_referral_payment',
44

5-
materialized = 'table',
6-
file_format = 'delta'
5+
materialized = 'incremental',
6+
file_format = 'delta',
7+
incremental_strategy = 'append'
78
, post_hook='{{ hide_spells() }}'
89
)
910
}}
@@ -58,6 +59,9 @@ referral_payment_txns AS (
5859
)
5960
AND evt_block_time >= CAST('2023-08-01 00:00' AS TIMESTAMP)
6061
AND contract_address in (select address from tokens)
62+
{% if is_incremental() %}
63+
AND {{ incremental_predicate('evt_block_time') }}
64+
{% endif %}
6165

6266
UNION ALL
6367

@@ -72,6 +76,9 @@ referral_payment_txns AS (
7276
)
7377
AND evt_block_time >= CAST('2023-08-01 00:00' AS TIMESTAMP)
7478
AND contract_address in (select address from tokens)
79+
{% if is_incremental() %}
80+
AND {{ incremental_predicate('evt_block_time') }}
81+
{% endif %}
7582
ORDER BY evt_block_time
7683
)
7784

@@ -80,4 +87,17 @@ referral_payment_txns AS (
8087
evt_tx_hash,
8188
contract_address AS token,
8289
value AS amount_token
83-
FROM referral_payment_txns
90+
FROM referral_payment_txns r
91+
{% if is_incremental() %}
92+
-- append-only dedup: drop rows already inserted by a previous run inside the
93+
-- incremental window (no event index in the output, so a merge unique_key would
94+
-- collapse legitimately duplicated transfers within one tx)
95+
WHERE not exists (
96+
select 1
97+
from {{ this }} t
98+
where t.period = r.evt_block_time
99+
and t.evt_tx_hash = r.evt_tx_hash
100+
and t.token = r.contract_address
101+
and t.amount_token = r.value
102+
)
103+
{% endif %}

0 commit comments

Comments
 (0)