Skip to content

Commit 70ad7e4

Browse files
author
Yasir
authored
Merge pull request #113 from bitfinity-network/fix-insertion-of-data
fix: insertion of data and deployment of docker images
2 parents 98e4df8 + f7bf495 commit 70ad7e4

5 files changed

Lines changed: 249 additions & 95 deletions

File tree

‎.github/workflows/block-extractor-scheduler.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,10 @@ jobs:
4848
run: |
4949
./evm-block-extractor \
5050
--rpc-url ${{ env.RPC_URL }} \
51+
${{ matrix.network == 'testnet' && '--reset-db-on-state-change' || '' }} \
5152
--bigquery \
5253
--project-id ${{ secrets.BIGQUERY_PROJECT_ID }} \
53-
--dataset-id ${{ matrix.network }} \
54-
${{ matrix.network == 'testnet' && '--reset_db_on_state_change' || '' }}
55-
54+
--dataset-id ${{ matrix.network }}
5655
# Only run the following steps if the network is mainnet and the database is postgres
5756
- name: Run evm-block-extractor for Postgres
5857
if: matrix.database == 'postgres' && matrix.network == 'mainnet'

‎src/evm-block-extractor/Dockerfile‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM lukemathwalker/cargo-chef:latest-rust-1.74.1 as chef
1+
FROM lukemathwalker/cargo-chef:latest-rust-1.75 as chef
22
WORKDIR /app
33

44
FROM chef as planner
@@ -16,9 +16,14 @@ RUN cargo build --release --bin evm-block-extractor-server
1616
FROM debian:bookworm-slim AS runtime
1717

1818
# GCP BigQuery Environment Variables
19-
ENV DATASET=
20-
ENV PROJECT_ID=
21-
ENV GCP_BLOCK_EXTRACTOR_SA_KEY=
19+
ARG DATASET
20+
ARG PROJECT_ID
21+
ARG GCP_BLOCK_EXTRACTOR_SA_KEY
22+
23+
# Set environment variables
24+
ENV DATASET=$DATASET \
25+
PROJECT_ID=$PROJECT_ID \
26+
GCP_BLOCK_EXTRACTOR_SA_KEY=$GCP_BLOCK_EXTRACTOR_SA_KEY
2227

2328
WORKDIR /app
2429
RUN apt-get update -y \
@@ -31,8 +36,10 @@ RUN apt-get update -y \
3136

3237
COPY --from=builder /app/target/release/evm-block-extractor-server /app/evm-block-extractor-server
3338

39+
40+
3441
EXPOSE 8080
3542

36-
ENTRYPOINT ["./evm-block-extractor-server"]
3743

38-
CMD ["-s", "0.0.0.0:8080", "--", "bigquery", "-d", ${DATASET}, "-p", ${PROJECT_ID}, "-k", ${GCP_BLOCK_EXTRACTOR_SA_KEY}]
44+
CMD ./evm-block-extractor-server -s 0.0.0.0:8080 --bigquery -p $PROJECT_ID -d $DATASET
45+

‎src/evm-block-extractor/src/database/big_query_db_client.rs‎

Lines changed: 74 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use serde_json::Value;
2020

2121
use super::DatabaseClient;
2222

23-
const BQ_RECEIPTS_TABLE_ID: &str = "receipts";
23+
const BQ_EXE_RESULTS_TABLE_ID: &str = "exe_results";
2424
const BQ_BLOCKS_TABLE_ID: &str = "blocks";
2525
const BQ_TRANSACTIONS_TABLE_ID: &str = "transactions";
2626

@@ -131,10 +131,10 @@ impl BigQueryDbClient {
131131
],
132132
),
133133
(
134-
BQ_RECEIPTS_TABLE_ID,
134+
BQ_EXE_RESULTS_TABLE_ID,
135135
vec![
136136
TableFieldSchema::string("tx_hash"),
137-
TableFieldSchema::json("receipt"),
137+
TableFieldSchema::json("exe_result"),
138138
],
139139
),
140140
(
@@ -208,7 +208,7 @@ impl DatabaseClient for BigQueryDbClient {
208208
};
209209

210210
delete_table(BQ_BLOCKS_TABLE_ID.to_owned()).await?;
211-
delete_table(BQ_RECEIPTS_TABLE_ID.to_owned()).await?;
211+
delete_table(BQ_EXE_RESULTS_TABLE_ID.to_owned()).await?;
212212
delete_table(BQ_TRANSACTIONS_TABLE_ID.to_owned()).await?;
213213

214214
Ok(())
@@ -303,66 +303,77 @@ impl DatabaseClient for BigQueryDbClient {
303303
);
304304
};
305305

306-
let receipts = receipts
307-
.iter()
308-
.map(|r| {
309-
let tx_hash = &r.transaction_hash;
310-
let receipt = ReceiptRow {
311-
tx_hash: format!("0x{:x}", tx_hash),
312-
receipt: serde_json::to_value(r).expect("Failed to serialize receipt"),
313-
};
314-
315-
TableDataInsertAllRequestRows {
316-
insert_id: Some(format!("0x{:x}", tx_hash)),
317-
json: serde_json::to_value(receipt).expect("Failed to serialize receipt"),
318-
}
319-
})
320-
.collect::<Vec<_>>();
306+
if !receipts.is_empty() {
307+
let receipts = receipts
308+
.iter()
309+
.map(|r| {
310+
let tx_hash = &r.transaction_hash;
311+
let receipt = ExeResultRow {
312+
tx_hash: format!("0x{:x}", tx_hash),
313+
exe_result: serde_json::to_value(r).expect("Failed to serialize receipt"),
314+
};
315+
316+
TableDataInsertAllRequestRows {
317+
insert_id: Some(format!("0x{:x}", tx_hash)),
318+
json: serde_json::to_value(receipt).expect("Failed to serialize receipt"),
319+
}
320+
})
321+
.collect::<Vec<_>>();
321322

322-
self.insert_batch_data(BQ_RECEIPTS_TABLE_ID, receipts)
323-
.await?;
323+
log::debug!("Inserting {} receipts", receipts.len());
324324

325-
let blocks = blocks
326-
.iter()
327-
.map(|b| {
328-
let block_id = b.number.0.as_u64();
325+
self.insert_batch_data(BQ_EXE_RESULTS_TABLE_ID, receipts)
326+
.await?;
327+
}
329328

330-
let block_row = BlockRow {
331-
id: block_id,
332-
body: serde_json::to_value(b).expect("Failed to serialize block"),
333-
};
329+
if !blocks.is_empty() {
330+
let blocks = blocks
331+
.iter()
332+
.map(|b| {
333+
let block_id = b.number.0.as_u64();
334+
335+
let block_row = BlockRow {
336+
id: block_id,
337+
body: serde_json::to_value(b).expect("Failed to serialize block"),
338+
};
339+
340+
TableDataInsertAllRequestRows {
341+
insert_id: Some(b.hash.to_hex_str()),
342+
json: serde_json::to_value(block_row).expect("Failed to serialize block"),
343+
}
344+
})
345+
.collect::<Vec<_>>();
334346

335-
TableDataInsertAllRequestRows {
336-
insert_id: Some(b.hash.to_hex_str()),
337-
json: serde_json::to_value(block_row).expect("Failed to serialize block"),
338-
}
339-
})
340-
.collect::<Vec<_>>();
341-
342-
self.insert_batch_data(BQ_BLOCKS_TABLE_ID, blocks).await?;
343-
344-
// Insert transactions
345-
let transactions = transactions
346-
.iter()
347-
.map(|txn| {
348-
let tx_hash = &txn.hash;
349-
350-
let txn = TransactionRow {
351-
tx_hash: format!("0x{:x}", tx_hash),
352-
transaction: serde_json::to_value(txn)
353-
.expect("Failed to serialize transaction"),
354-
block_number: txn.block_number.expect("Block number not found").0.as_u64(),
355-
};
356-
357-
TableDataInsertAllRequestRows {
358-
insert_id: Some(format!("0x{:x}", tx_hash)),
359-
json: serde_json::to_value(txn).expect("Failed to serialize transaction"),
360-
}
361-
})
362-
.collect::<Vec<_>>();
347+
log::debug!("Inserting {} blocks", blocks.len());
363348

364-
self.insert_batch_data(BQ_TRANSACTIONS_TABLE_ID, transactions)
365-
.await?;
349+
self.insert_batch_data(BQ_BLOCKS_TABLE_ID, blocks).await?;
350+
}
351+
352+
if !transactions.is_empty() {
353+
let transactions = transactions
354+
.iter()
355+
.map(|txn| {
356+
let tx_hash = &txn.hash;
357+
358+
let txn = TransactionRow {
359+
tx_hash: format!("0x{:x}", tx_hash),
360+
transaction: serde_json::to_value(txn)
361+
.expect("Failed to serialize transaction"),
362+
block_number: txn.block_number.expect("Block number not found").0.as_u64(),
363+
};
364+
365+
TableDataInsertAllRequestRows {
366+
insert_id: Some(format!("0x{:x}", tx_hash)),
367+
json: serde_json::to_value(txn).expect("Failed to serialize transaction"),
368+
}
369+
})
370+
.collect::<Vec<_>>();
371+
372+
log::debug!("Inserting {} transactions", transactions.len());
373+
374+
self.insert_batch_data(BQ_TRANSACTIONS_TABLE_ID, transactions)
375+
.await?;
376+
}
366377

367378
Ok(())
368379
}
@@ -383,10 +394,10 @@ impl DatabaseClient for BigQueryDbClient {
383394
},
384395
]),
385396
query:format!(
386-
"SELECT receipt FROM `{project_id}.{dataset_id}.{table_id}` WHERE tx_hash = @tx_hash",
397+
"SELECT exe_result FROM `{project_id}.{dataset_id}.{table_id}` WHERE tx_hash = @tx_hash",
387398
project_id = self.project_id,
388399
dataset_id = self.dataset_id,
389-
table_id = BQ_RECEIPTS_TABLE_ID,
400+
table_id = BQ_EXE_RESULTS_TABLE_ID,
390401
),
391402
..Default::default()
392403
};
@@ -458,10 +469,10 @@ pub struct BlockRow {
458469
}
459470

460471
#[derive(Debug, Serialize, Clone)]
461-
pub struct ReceiptRow {
472+
pub struct ExeResultRow {
462473
tx_hash: String,
463474
#[serde(serialize_with = "serialize_json_as_string")]
464-
receipt: Value,
475+
exe_result: Value,
465476
}
466477

467478
#[derive(Debug, Serialize, Clone)]

‎src/evm-block-extractor/src/database/postgres_db_client.rs‎

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -97,35 +97,43 @@ impl DatabaseClient for PostgresDbClient {
9797

9898
let mut tx = self.pool.begin().await?;
9999

100-
for block in blocks {
101-
let block_id = block.number.0.as_u64();
102-
103-
sqlx::query("INSERT INTO EVM_BLOCK (id, data) VALUES ($1, $2)")
104-
.bind(block_id as i64)
105-
.bind(serde_json::to_value(block)?)
106-
.execute(&mut *tx)
107-
.await
108-
.map_err(|e| anyhow::anyhow!("Error inserting block {}: {:?}", block_id, e))
109-
.map(|_| ())?;
100+
if !blocks.is_empty() {
101+
for block in blocks {
102+
let block_id = block.number.0.as_u64();
103+
104+
sqlx::query("INSERT INTO EVM_BLOCK (id, data) VALUES ($1, $2)")
105+
.bind(block_id as i64)
106+
.bind(serde_json::to_value(block)?)
107+
.execute(&mut *tx)
108+
.await
109+
.map_err(|e| anyhow::anyhow!("Error inserting block {}: {:?}", block_id, e))
110+
.map(|_| ())?;
111+
}
110112
}
111113

112-
for receipt in receipts {
113-
let hex_tx_hash = did::H256::from(receipt.transaction_hash.clone()).to_hex_str();
114-
sqlx::query("INSERT INTO EVM_TRANSACTION_EXE_RESULT (id, data) VALUES ($1, $2)")
115-
.bind(&hex_tx_hash)
116-
.bind(serde_json::to_value(receipt)?)
117-
.execute(&mut *tx)
118-
.await?;
114+
if !receipts.is_empty() {
115+
for receipt in receipts {
116+
let hex_tx_hash = did::H256::from(receipt.transaction_hash.clone()).to_hex_str();
117+
sqlx::query("INSERT INTO EVM_TRANSACTION_EXE_RESULT (id, data) VALUES ($1, $2)")
118+
.bind(&hex_tx_hash)
119+
.bind(serde_json::to_value(receipt)?)
120+
.execute(&mut *tx)
121+
.await?;
122+
}
119123
}
120124

121-
for txn in transactions {
122-
let hex_tx_hash = txn.hash.to_hex_str();
123-
sqlx::query("INSERT INTO EVM_TRANSACTION (id, data,block_number) VALUES ($1, $2,$3)")
125+
if !transactions.is_empty() {
126+
for txn in transactions {
127+
let hex_tx_hash = txn.hash.to_hex_str();
128+
sqlx::query(
129+
"INSERT INTO EVM_TRANSACTION (id, data,block_number) VALUES ($1, $2,$3)",
130+
)
124131
.bind(&hex_tx_hash)
125132
.bind(serde_json::to_value(txn)?)
126133
.bind(txn.block_number.expect("Block number not found").0.as_u64() as i64)
127134
.execute(&mut *tx)
128135
.await?;
136+
}
129137
}
130138

131139
tx.commit().await?;

0 commit comments

Comments
 (0)