Merge pull request #116 from bitfinity-network/evm_block_extractor_daemon

[EPROD-707] Evm block extractor daemon

Author: ufoscout

.github/workflows/block-extractor-scheduler.yml

@@ -1,49 +1,53 @@
-name: 'Deploy EVM Block Extractor to GCP Registry'
+name: 'Deploy EVM Block Extractor docker image'
 
 on:
   workflow_dispatch: {}
 
   push:
-    branches:
-      - 'main'
+    branches: [main]
+    tags:
+      - 'v*'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{github.repository_owner}}/evm-block-extractor
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
 jobs:
-  publishing-image:
+  build-and-push-image:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        network: [mainnet, testnet]
-
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      # 
     steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - name: Login to GCP Registry
+      - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:
-          registry: us-east4-docker.pkg.dev
-          username: _json_key
-          password: ${{ secrets.GCP_DOCKER_KEY }}
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Checkout
-        uses: actions/checkout@v3
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      - name: Building CloudRun image for BigQuery
+      - name: Build and push Docker image
         uses: docker/build-push-action@v5
         with:
-          push: true
           context: .
           file: ./src/evm-block-extractor/Dockerfile
-          tags: |
-            us-east4-docker.pkg.dev/extractor-410310/block-extractor-repo/evm-block-extractor-server:${{ matrix.network }}-latest
-          build-args: |
-            PROJECT_ID=${{ secrets.BIGQUERY_PROJECT_ID }}
-            DATASET=${{ matrix.network }}
-            GCP_BLOCK_EXTRACTOR_SA_KEY=${{ secrets.GCP_BLOCK_EXTRACTOR_SA_KEY }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          # labels: ${{ steps.meta.outputs.labels }}

.github/workflows/publish-evm-block-extractor.yml

@@ -50,6 +50,7 @@ ic-storage = { git = "https://github.com/bitfinity-network/canister-sdk", packag
 itertools = "0.12"
 jsonrpc-core = "18.0"
 jsonrpsee = { version = "0.21", features = ["server", "macros"] }
+lightspeed_scheduler = "0.57"
 log = "0.4"
 murmur3 = "0.5"
 num = "0.4"

Cargo.toml

@@ -0,0 +1,26 @@
+version: "3.3"
+
+#
+# This docker-compose file is used to start the services for local testing.
+# It starts a evm-blockchain-extractor connected to a local postgres database.
+#
+
+services:
+
+  db:
+    image: "postgres:11-alpine"
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_USER: postgres
+
+
+  extractor:
+    image: ghcr.io/bitfinity-network/evm-block-extractor:main
+    # image: "evm-block-extractor:latest"
+    # build: 
+    #   dockerfile: ./src/evm-block-extractor/Dockerfile
+    ports:
+      - "8080:8080"
+    command: --rpc-url https://testnet.bitfinity.network --postgres --username postgres --password postgres --database-name postgres --database-url db

docker-compose.yml

@@ -3,15 +3,6 @@ name = "evm-block-extractor"
 version = "0.1.0"
 edition = "2021"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[[bin]]
-name = "evm-block-extractor"
-path = "src/bin/extractor.rs"
-
-[[bin]]
-name = "evm-block-extractor-server"
-path = "src/bin/server.rs"
 
 [dependencies]
 anyhow = { workspace = true }
@@ -24,8 +15,8 @@ ethers-core = { workspace = true }
 futures = { workspace = true }
 gcp-bigquery-client = { workspace = true }
 hex = { workspace = true }
-itertools = { workspace = true }
 jsonrpsee = { workspace = true }
+lightspeed_scheduler = { workspace = true }
 log = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -34,6 +25,7 @@ tokio = { workspace = true }
 yup-oauth2 = { workspace = true }
 zip = { workspace = true }
 
+
 [dev-dependencies]
 jsonrpc-core = { workspace = true }
 port_check = { workspace = true }

src/evm-block-extractor/Cargo.toml

@@ -1,31 +1,18 @@
-FROM lukemathwalker/cargo-chef:latest-rust-1.75 as chef
+#
+# This dockerfile has to be called from the root folder of the project
+# > docker build -f src/evm-block-extractor/Dockerfile -t evm-block-extractor .
+#
+FROM rust:slim-bookworm as builder
 WORKDIR /app
 
-FROM chef as planner
-COPY ../../ .
+ADD . .
 
-RUN cargo chef prepare  --recipe-path recipe.json
-
-FROM chef as builder
-COPY --from=planner /app/recipe.json recipe.json
-RUN cargo chef cook --release --recipe-path recipe.json
-
-COPY . .
-RUN cargo build --release --bin evm-block-extractor-server
+RUN cargo build --release --bin evm-block-extractor
 
 FROM debian:bookworm-slim AS runtime
 
-# GCP BigQuery Environment Variables
-ARG DATASET
-ARG PROJECT_ID
-ARG GCP_BLOCK_EXTRACTOR_SA_KEY
-
-# Set environment variables
-ENV DATASET=$DATASET \
-    PROJECT_ID=$PROJECT_ID \
-    GCP_BLOCK_EXTRACTOR_SA_KEY=$GCP_BLOCK_EXTRACTOR_SA_KEY
-
 WORKDIR /app
+
 RUN apt-get update -y \
     && apt-get install -y --no-install-recommends ca-certificates \
     && update-ca-certificates \
@@ -34,12 +21,9 @@ RUN apt-get update -y \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/*
 
-COPY --from=builder /app/target/release/evm-block-extractor-server /app/evm-block-extractor-server
-
-
+COPY --from=builder /app/target/release/evm-block-extractor /app/evm-block-extractor
 
 EXPOSE 8080
 
-
-CMD ./evm-block-extractor-server -s 0.0.0.0:8080 --bigquery -p $PROJECT_ID -d $DATASET
+ENTRYPOINT ["./evm-block-extractor"]
 

src/evm-block-extractor/Dockerfile

@@ -1,58 +1,66 @@
-# EVM Block Extractor And Server
+# EVM Block Extractor
 
-## EVM Block Extractor
+## Introduction
 
-### Introduction
+The EVM block extractor is an advanced tool used to collect EVM blocks and transactions, and send them to a specified data storage. 
+This version is enhanced to handle parallel requests efficiently and integrates with Google Cloud Platform's BigQuery service or Postgres.
 
-The EVM block extractor is an advanced tool used to collect EVM blocks and transactions, and send them to a specified BigQuery dataset endpoint. This version is enhanced to handle parallel requests efficiently and integrates with Google Cloud Platform's BigQuery service.
+## Configuration
 
-### Usage
+### Usage with BigQuery
 
 ```sh
 evm-block-extractor
+  --server-address <server-address>
   --rpc-url <evmc-rpc-url>
-  --dataset-id <bigquery-dataset-id>
   --max-number-of-requests <max-parallel-requests>
   --rpc-batch-size <rpc-batch-size>
+  --bigquery
+  --project-id <bigquery-project-id>
+  --dataset-id <bigquery-dataset-id>
   --sa-key <service-account-key>
 ```
 
 Where:
 
+- **server-address:** The address where the server will be hosted (default: 127.0.0.1:8080).
 - **rpc-url**: is the endpoint of the EVMC json-rpc url
-- **dataset-id**: is the BigQuery dataset id where the data will be sent
 - **max-number-of-requests**: is the maximum number of parallel requests to be sent to the EVMC json-rpc endpoint
 - **rpc-batch-size**: is the number of blocks to be requested in a single batch
+- **dataset-id**: is the BigQuery dataset id where the data will be sent
 - **sa-key**: the service account key in JSON format for GCP authentication.
 
-### Output
 
-The data is sent and stored in the specified BigQuery dataset. This allows for enhanced querying and analysis capabilities using BigQuery's features.
-
-## EVM Block Extractor Server
-
-### Introduction
-
-The EVM block extractor server is a JSON-RPC server for the EVM block extractor. It is integrated with BigQuery and allows for querying the data stored in the BigQuery dataset.
-
-### Usage
+### Usage with Postgres
 
 ```sh
-evm-block-extractor-server
-  --dataset-id <bigquery-dataset-id>
+evm-block-extractor
   --server-address <server-address>
-  --sa-key <service-account-key>
+  --rpc-url <evmc-rpc-url>
+  --max-number-of-requests <max-parallel-requests>
+  --rpc-batch-size <rpc-batch-size>
+  --postgres
+  --username <postgres-db-username>
+  --password <postgres-db-password>
+  --database_name <postgres-db-name>
+  --database_url <postgres-db-url>
+  --database_port <postgres-db-port>
+  --require_ssl <postgres-db-require-ssl>
 ```
 
 Where:
 
-- **dataset-id**: The dataset ID of the BigQuery table.
-- **server-address:** The address where the server will be hosted (default: 127.0.0.1:8080).
-- **sa-key**: The service account key in JSON format for GCP authentication.
+- **username**: Username for the database connection
+- **password**: Password for the database connection
+- **database_name**: database name
+- **database_url**: database IP or URL
+- **database_port**: database port
+- **require_ssl**: whether to use ssl (true/false)
+
 
-### Endpoints
+## Endpoints
 
-This is minimal version of the Ethereum JSON-RPC server. It supports the following endpoints:
+Th evm-block-extracor is also a minimal version of the Ethereum JSON-RPC server which supports the following endpoints:
 
 - **eth_blockNumber**: Returns the number of most recent block.
 - **eth_getBlockByNumber**: Returns information about a block by block number.
@@ -62,5 +70,15 @@ This is minimal version of the Ethereum JSON-RPC server. It supports the followi
 ### Example
 
 ```sh
-curl -X POST --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' http://127.0.0.1:8080
+curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' http://127.0.0.1:8080
 ```
+
+## Docker image
+
+The evm-block-extractor docker image is a debian slim based image that allows for simple installation of the service.
+The docker image accepts the same configuration arguments of the plain executor. 
+E.g.:
+```sh
+docker run ghcr.io/bitfinity-network/evm-block-extractor:main --rpc-url https://testnet.bitfinity.network --postgres --username postgres --password postgres --database-name postgres --database-url 127.0.0.1:5432
+```
+