Skip to content

Commit 55a2cf4

Browse files
rysudsrysuds
committed
Minor refactors & cleanup
1 parent eed71a6 commit 55a2cf4

3 files changed

Lines changed: 70 additions & 166 deletions

File tree

‎demo.ipynb‎

Lines changed: 54 additions & 154 deletions
Original file line numberDiff line numberDiff line change
@@ -29,42 +29,57 @@
2929
},
3030
{
3131
"cell_type": "code",
32-
"execution_count": 5,
32+
"execution_count": 4,
3333
"metadata": {},
3434
"outputs": [
3535
{
3636
"data": {
3737
"text/plain": [
38-
"{'map_pair_created_events': {'output_type': 'proto:messari.uniswap.v2.PairCreatedEvents',\n",
38+
"{'map_pair_created_events': {'is_map': True,\n",
39+
" 'output_type': 'proto:messari.uniswap.v2.PairCreatedEvents',\n",
40+
" 'initial_block': 10000835},\n",
41+
" 'map_mint_events': {'is_map': True,\n",
42+
" 'output_type': 'proto:messari.uniswap.v2.MintEvents',\n",
3943
" 'initial_block': 10000835},\n",
40-
" 'map_mint_events': {'output_type': 'proto:messari.uniswap.v2.MintEvents',\n",
44+
" 'map_burn_events': {'is_map': True,\n",
45+
" 'output_type': 'proto:messari.uniswap.v2.BurnEvents',\n",
4146
" 'initial_block': 10000835},\n",
42-
" 'map_burn_events': {'output_type': 'proto:messari.uniswap.v2.BurnEvents',\n",
47+
" 'map_swap_events': {'is_map': True,\n",
48+
" 'output_type': 'proto:messari.uniswap.v2.SwapEvents',\n",
4349
" 'initial_block': 10000835},\n",
44-
" 'map_swap_events': {'output_type': 'proto:messari.uniswap.v2.SwapEvents',\n",
50+
" 'store_pair_created_events': {'is_map': False,\n",
51+
" 'output_type': 'proto:messari.uniswap.v2.PairCreatedEvent',\n",
4552
" 'initial_block': 10000835},\n",
46-
" 'store_pair_created_events': {'output_type': 'proto:messari.uniswap.v2.PairCreatedEvent',\n",
53+
" 'store_mint_events': {'is_map': False,\n",
54+
" 'output_type': 'proto:messari.uniswap.v2.MintEvent',\n",
4755
" 'initial_block': 10000835},\n",
48-
" 'store_mint_events': {'output_type': 'proto:messari.uniswap.v2.MintEvent',\n",
56+
" 'store_burn_events': {'is_map': False,\n",
57+
" 'output_type': 'proto:messari.uniswap.v2.BurnEvent',\n",
4958
" 'initial_block': 10000835},\n",
50-
" 'store_burn_events': {'output_type': 'proto:messari.uniswap.v2.BurnEvent',\n",
59+
" 'store_swap_events': {'is_map': False,\n",
60+
" 'output_type': 'proto:messari.uniswap.v2.SwapEvent',\n",
5161
" 'initial_block': 10000835},\n",
52-
" 'store_swap_events': {'output_type': 'proto:messari.uniswap.v2.SwapEvent',\n",
62+
" 'map_usage_events': {'is_map': True,\n",
63+
" 'output_type': 'proto:messari.dex_amm.v1.UsageEvents',\n",
5364
" 'initial_block': 10000835},\n",
54-
" 'map_usage_events': {'output_type': 'proto:messari.dex_amm.v1.UsageEvents',\n",
65+
" 'store_usage_events': {'is_map': False,\n",
66+
" 'output_type': 'proto:messari.dex_amm.v1.UsageEvents',\n",
5567
" 'initial_block': 10000835},\n",
56-
" 'store_usage_events': {'output_type': 'proto:messari.dex_amm.v1.UsageEvents',\n",
68+
" 'store_usage_counts': {'is_map': False,\n",
69+
" 'output_type': 'int64',\n",
5770
" 'initial_block': 10000835},\n",
58-
" 'store_usage_counts': {'output_type': 'int64', 'initial_block': 10000835},\n",
59-
" 'store_volumes_from_swaps': {'output_type': 'bigint',\n",
71+
" 'store_volumes_from_swaps': {'is_map': False,\n",
72+
" 'output_type': 'bigint',\n",
6073
" 'initial_block': 10000835},\n",
61-
" 'map_block_to_transfers': {'output_type': 'proto:messari.erc20.v1.TransferEvents',\n",
74+
" 'map_block_to_transfers': {'is_map': True,\n",
75+
" 'output_type': 'proto:messari.erc20.v1.TransferEvents',\n",
6276
" 'initial_block': 0},\n",
63-
" 'store_transfers': {'output_type': 'proto:messari.erc20.v1.TransferEvents',\n",
77+
" 'store_transfers': {'is_map': False,\n",
78+
" 'output_type': 'proto:messari.erc20.v1.TransferEvents',\n",
6479
" 'initial_block': 0}}"
6580
]
6681
},
67-
"execution_count": 5,
82+
"execution_count": 4,
6883
"metadata": {},
6984
"output_type": "execute_result"
7085
}
@@ -75,16 +90,24 @@
7590
},
7691
{
7792
"cell_type": "code",
78-
"execution_count": 16,
93+
"execution_count": 5,
7994
"metadata": {},
80-
"outputs": [],
95+
"outputs": [
96+
{
97+
"name": "stdout",
98+
"output_type": "stream",
99+
"text": [
100+
"data block # 10010835\n"
101+
]
102+
}
103+
],
81104
"source": [
82-
"results = sb.poll([\"store_swap_events\"], start_block=10000835+40_000, end_block=10000835+40_001)"
105+
"results = sb.poll([\"map_swap_events\"], start_block=10000835+10_000, end_block=10000835+10_001)"
83106
]
84107
},
85108
{
86109
"cell_type": "code",
87-
"execution_count": 18,
110+
"execution_count": 6,
88111
"metadata": {},
89112
"outputs": [
90113
{
@@ -108,158 +131,35 @@
108131
" <thead>\n",
109132
" <tr style=\"text-align: right;\">\n",
110133
" <th></th>\n",
111-
" <th>txHash</th>\n",
112-
" <th>logIndex</th>\n",
113-
" <th>logOrdinal</th>\n",
114-
" <th>poolAddress</th>\n",
115-
" <th>sender</th>\n",
116-
" <th>amount0In</th>\n",
117-
" <th>amount1In</th>\n",
118-
" <th>amount0Out</th>\n",
119-
" <th>amount1Out</th>\n",
120-
" <th>to</th>\n",
121134
" <th>output_module</th>\n",
122135
" </tr>\n",
123136
" </thead>\n",
124137
" <tbody>\n",
125-
" <tr>\n",
126-
" <th>0</th>\n",
127-
" <td>697b7aaca56a80a8d3a2f560ed7f1ecb97c22b2edd6edd...</td>\n",
128-
" <td>4</td>\n",
129-
" <td>107</td>\n",
130-
" <td>b4e16d0168e52d35cacd2c6185b44281ec28c9dc</td>\n",
131-
" <td>f164fc0ec4e93095b804a4795bbe1e041497b92a</td>\n",
132-
" <td>0</td>\n",
133-
" <td>49280731798118</td>\n",
134-
" <td>10000</td>\n",
135-
" <td>0</td>\n",
136-
" <td>11e4857bb9993a50c685a79afad4e6f65d518dda</td>\n",
137-
" <td>store_swap_events</td>\n",
138-
" </tr>\n",
139-
" <tr>\n",
140-
" <th>1</th>\n",
141-
" <td>932cb88306450d481a0e43365a3ed832625b68f036e988...</td>\n",
142-
" <td>4</td>\n",
143-
" <td>97</td>\n",
144-
" <td>b4e16d0168e52d35cacd2c6185b44281ec28c9dc</td>\n",
145-
" <td>f164fc0ec4e93095b804a4795bbe1e041497b92a</td>\n",
146-
" <td>0</td>\n",
147-
" <td>4873849709289</td>\n",
148-
" <td>1000</td>\n",
149-
" <td>0</td>\n",
150-
" <td>8688a84fcfd84d8f78020d0fc0b35987cc58911f</td>\n",
151-
" <td>store_swap_events</td>\n",
152-
" </tr>\n",
153-
" <tr>\n",
154-
" <th>2</th>\n",
155-
" <td>f7e7f3b82d8944e0cc89ab16a015cbdcfe7d0806d48946...</td>\n",
156-
" <td>3</td>\n",
157-
" <td>502</td>\n",
158-
" <td>12ede161c702d1494612d19f05992f43aa6a26fb</td>\n",
159-
" <td>f164fc0ec4e93095b804a4795bbe1e041497b92a</td>\n",
160-
" <td>700000000000000</td>\n",
161-
" <td>0</td>\n",
162-
" <td>0</td>\n",
163-
" <td>3391967105717</td>\n",
164-
" <td>f164fc0ec4e93095b804a4795bbe1e041497b92a</td>\n",
165-
" <td>store_swap_events</td>\n",
166-
" </tr>\n",
167-
" <tr>\n",
168-
" <th>3</th>\n",
169-
" <td>735cf98e86a5df67b6a837ae50de1d7a589d9f6baaf0e1...</td>\n",
170-
" <td>6</td>\n",
171-
" <td>1369</td>\n",
172-
" <td>12ede161c702d1494612d19f05992f43aa6a26fb</td>\n",
173-
" <td>f164fc0ec4e93095b804a4795bbe1e041497b92a</td>\n",
174-
" <td>0</td>\n",
175-
" <td>3412398195916</td>\n",
176-
" <td>700000000000000</td>\n",
177-
" <td>0</td>\n",
178-
" <td>8688a84fcfd84d8f78020d0fc0b35987cc58911f</td>\n",
179-
" <td>store_swap_events</td>\n",
180-
" </tr>\n",
181-
" <tr>\n",
182-
" <th>4</th>\n",
183-
" <td>735cf98e86a5df67b6a837ae50de1d7a589d9f6baaf0e1...</td>\n",
184-
" <td>3</td>\n",
185-
" <td>1331</td>\n",
186-
" <td>b4e16d0168e52d35cacd2c6185b44281ec28c9dc</td>\n",
187-
" <td>f164fc0ec4e93095b804a4795bbe1e041497b92a</td>\n",
188-
" <td>689</td>\n",
189-
" <td>0</td>\n",
190-
" <td>0</td>\n",
191-
" <td>3412398195916</td>\n",
192-
" <td>12ede161c702d1494612d19f05992f43aa6a26fb</td>\n",
193-
" <td>store_swap_events</td>\n",
194-
" </tr>\n",
195-
" <tr>\n",
196-
" <th>5</th>\n",
197-
" <td>43b6bfd06dde0814fe9c1b63ce98ec4c67c72d96169d73...</td>\n",
198-
" <td>4</td>\n",
199-
" <td>2165</td>\n",
200-
" <td>b4e16d0168e52d35cacd2c6185b44281ec28c9dc</td>\n",
201-
" <td>f164fc0ec4e93095b804a4795bbe1e041497b92a</td>\n",
202-
" <td>0</td>\n",
203-
" <td>4983090479442</td>\n",
204-
" <td>1000</td>\n",
205-
" <td>0</td>\n",
206-
" <td>8688a84fcfd84d8f78020d0fc0b35987cc58911f</td>\n",
207-
" <td>store_swap_events</td>\n",
208-
" </tr>\n",
209138
" </tbody>\n",
210139
"</table>\n",
211140
"</div>"
212141
],
213142
"text/plain": [
214-
" txHash logIndex logOrdinal \\\n",
215-
"0 697b7aaca56a80a8d3a2f560ed7f1ecb97c22b2edd6edd... 4 107 \n",
216-
"1 932cb88306450d481a0e43365a3ed832625b68f036e988... 4 97 \n",
217-
"2 f7e7f3b82d8944e0cc89ab16a015cbdcfe7d0806d48946... 3 502 \n",
218-
"3 735cf98e86a5df67b6a837ae50de1d7a589d9f6baaf0e1... 6 1369 \n",
219-
"4 735cf98e86a5df67b6a837ae50de1d7a589d9f6baaf0e1... 3 1331 \n",
220-
"5 43b6bfd06dde0814fe9c1b63ce98ec4c67c72d96169d73... 4 2165 \n",
221-
"\n",
222-
" poolAddress \\\n",
223-
"0 b4e16d0168e52d35cacd2c6185b44281ec28c9dc \n",
224-
"1 b4e16d0168e52d35cacd2c6185b44281ec28c9dc \n",
225-
"2 12ede161c702d1494612d19f05992f43aa6a26fb \n",
226-
"3 12ede161c702d1494612d19f05992f43aa6a26fb \n",
227-
"4 b4e16d0168e52d35cacd2c6185b44281ec28c9dc \n",
228-
"5 b4e16d0168e52d35cacd2c6185b44281ec28c9dc \n",
229-
"\n",
230-
" sender amount0In amount1In \\\n",
231-
"0 f164fc0ec4e93095b804a4795bbe1e041497b92a 0 49280731798118 \n",
232-
"1 f164fc0ec4e93095b804a4795bbe1e041497b92a 0 4873849709289 \n",
233-
"2 f164fc0ec4e93095b804a4795bbe1e041497b92a 700000000000000 0 \n",
234-
"3 f164fc0ec4e93095b804a4795bbe1e041497b92a 0 3412398195916 \n",
235-
"4 f164fc0ec4e93095b804a4795bbe1e041497b92a 689 0 \n",
236-
"5 f164fc0ec4e93095b804a4795bbe1e041497b92a 0 4983090479442 \n",
237-
"\n",
238-
" amount0Out amount1Out to \\\n",
239-
"0 10000 0 11e4857bb9993a50c685a79afad4e6f65d518dda \n",
240-
"1 1000 0 8688a84fcfd84d8f78020d0fc0b35987cc58911f \n",
241-
"2 0 3391967105717 f164fc0ec4e93095b804a4795bbe1e041497b92a \n",
242-
"3 700000000000000 0 8688a84fcfd84d8f78020d0fc0b35987cc58911f \n",
243-
"4 0 3412398195916 12ede161c702d1494612d19f05992f43aa6a26fb \n",
244-
"5 1000 0 8688a84fcfd84d8f78020d0fc0b35987cc58911f \n",
245-
"\n",
246-
" output_module \n",
247-
"0 store_swap_events \n",
248-
"1 store_swap_events \n",
249-
"2 store_swap_events \n",
250-
"3 store_swap_events \n",
251-
"4 store_swap_events \n",
252-
"5 store_swap_events "
143+
"Empty DataFrame\n",
144+
"Columns: [output_module]\n",
145+
"Index: []"
253146
]
254147
},
255-
"execution_count": 18,
148+
"execution_count": 6,
256149
"metadata": {},
257150
"output_type": "execute_result"
258151
}
259152
],
260153
"source": [
261154
"results[0].snapshots"
262155
]
156+
},
157+
{
158+
"cell_type": "code",
159+
"execution_count": null,
160+
"metadata": {},
161+
"outputs": [],
162+
"source": []
263163
}
264164
],
265165
"metadata": {

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ name = "substreams"
77
version = "0.0.3"
88
authors = [
99
{ name="Ryan Sudhakaran", email="ryan.sudhakaran@messari.io" },
10-
{ name="Michael Carroll", email="michaelcarroll1999@gmail.com}
10+
{ name="Michael Carroll", email="michaelcarroll1999@gmail.com" }
1111
]
1212
description = "WIP Substreams Python Adapter"
1313
readme = "README.md"

‎substreams/substream.py‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -139,8 +139,6 @@ def _parse_data_deltas(self, data: dict) -> list[dict]:
139139
return deltas
140140

141141
def _parse_data_outputs(self, data: dict) -> list[dict]:
142-
module_name: str = data["outputs"][0]["name"]
143-
obj_class = self._class_from_module(module_name)
144142
outputs = list()
145143
for output in data["outputs"]:
146144
map_output = output["mapOutput"]
@@ -177,7 +175,13 @@ def proto_file_map(self) -> dict[str, DescriptorProto]:
177175
return name_map
178176

179177
# TODO how do I type annotate this stuff?
180-
def poll(self, output_modules: list[str], start_block: int, end_block: int, initial_snapshot = False):
178+
def poll(
179+
self,
180+
output_modules: list[str],
181+
start_block: int,
182+
end_block: int,
183+
initial_snapshot=False,
184+
):
181185
# TODO make this general
182186
from sf.substreams.v1.substreams_pb2 import STEP_IRREVERSIBLE, Request
183187

@@ -193,7 +197,9 @@ def poll(self, output_modules: list[str], start_block: int, end_block: int, init
193197
fork_steps=[STEP_IRREVERSIBLE],
194198
modules=self.pkg.modules,
195199
output_modules=output_modules,
196-
initial_store_snapshot_for_modules=output_modules if initial_snapshot else None,
200+
initial_store_snapshot_for_modules=output_modules
201+
if initial_snapshot
202+
else None,
197203
)
198204
)
199205
raw_results = defaultdict(lambda: {"data": list(), "snapshots": list()})
@@ -205,15 +211,13 @@ def poll(self, output_modules: list[str], start_block: int, end_block: int, init
205211
snapshot_deltas = self._parse_snapshot_deltas(snapshot)
206212
raw_results[module_name]["snapshots"].extend(snapshot_deltas)
207213
if data:
208-
print('data block #', data["clock"]["number"])
214+
print("data block #", data["clock"]["number"])
209215
if self.output_modules[module]["is_map"]:
210-
module_name: str = data["outputs"][0]["name"]
211-
data_outputs = self._parse_data_outputs(data)
212-
raw_results[module_name]["data"].extend(data_outputs)
216+
parsed = self._parse_data_outputs(data)
213217
else:
214-
module_name: str = data["outputs"][0]["name"]
215-
data_deltas = self._parse_data_deltas(data)
216-
raw_results[module_name]["data"].extend(data_deltas)
218+
parsed = self._parse_data_deltas(data)
219+
module_name: str = data["outputs"][0]["name"]
220+
raw_results[module_name]["data"].extend(parsed)
217221

218222
results = []
219223
for output_module in output_modules:

0 commit comments

Comments
 (0)