Merge pull request #87 from multiversx/feat/spica

Merge feat/spica into main

Author: andreibancioiu

.env

@@ -0,0 +1,14 @@
+# Default (placeholder): mnemonic of "testwallets" (friends: Alice, Bob, Carol etc.):
+# https://github.com/multiversx/mx-sdk-testwallets/blob/main/users/mnemonic.txt.
+# This works fine for localnet.
+# For the Github workflows, a repository secret is being used.
+export USERS_MNEMONIC="moral volcano peasant pass circle pen over picture flat shop clap goat never lyrics gather prepare woman film husband gravity behind test tiger improve"
+
+# Default (placeholder): Alice's secret key:
+# https://github.com/multiversx/mx-sdk-testwallets/blob/main/users/alice.pem.
+# This works fine for localnet.
+# For the Github workflows, a repository secret is being used.
+export SPONSOR_SECRET_KEY=413f42575f7f26fad3317a778771212fdb80245850981e48b58a4f25e344e8f9
+
+# For the Github workflows, a repository secret (private deep history instance) is being used.
+export MAINNET_PROXY_URL="https://gateway.multiversx.com"

.github/workflows/check_with_mesh_cli.yml

@@ -17,7 +17,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip3 install requests bottle
+          pip install -r ./requirements-dev.txt
           curl -sSfL https://raw.githubusercontent.com/coinbase/mesh-cli/master/scripts/install.sh | sh -s -- -b "$HOME/.local/bin"
           echo "$HOME/.local/bin" >> $GITHUB_PATH
 
@@ -26,12 +26,26 @@ jobs:
           cd $GITHUB_WORKSPACE/cmd/rosetta && go build .
           cd $GITHUB_WORKSPACE/systemtests && go build ./proxyToObserverAdapter.go
 
-      - name: check:data
+      - name: check:construction (native)
         run: |
-          PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=data --network=testnet
+          PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=construction-native --network=testnet
+          sleep 30
+
+      - name: check:construction (custom)
+        run: |
+          PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=construction-custom --network=testnet
+          sleep 30
 
-      - name: check:construction
+      - name: Generate sample transactions for check:data
         run: |
-          PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=construction --network=testnet
+          PYTHONPATH=. python3 ./systemtests/generate_testdata_on_network.py setup --network=testnet
+          PYTHONPATH=. python3 ./systemtests/generate_testdata_on_network.py run --network=testnet
+        env:
+          USERS_MNEMONIC: ${{ secrets.USERS_MNEMONIC }}
+          SPONSOR_SECRET_KEY: ${{ secrets.SPONSOR_SECRET_KEY }}
+
+      - name: check:data
+        run: |
+          PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=data --network=testnet
 
 

.github/workflows/check_with_mesh_cli_on_mainnet.yml

@@ -0,0 +1,34 @@
+name: Check with Mesh CLI (mainnet)
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          pip install -r ./requirements-dev.txt
+          curl -sSfL https://raw.githubusercontent.com/coinbase/mesh-cli/master/scripts/install.sh | sh -s -- -b "$HOME/.local/bin"
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Build
+        run: |
+          cd $GITHUB_WORKSPACE/cmd/rosetta && go build .
+          cd $GITHUB_WORKSPACE/systemtests && go build ./proxyToObserverAdapter.go
+
+      - name: check:data
+        run: |
+          PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=data --network=mainnet
+        env:
+          MAINNET_PROXY_URL: ${{ secrets.MAINNET_PROXY_URL }}
+

.gitignore

@@ -1,7 +1,9 @@
 cmd/rosetta/rosetta
 cmd/rosetta/logs/**
 systemtests/logs/**
+systemtests/memento/**
 systemtests/**/check-data/**
 logs/**
 
 **/__pycache__/**
+venv/**

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "[json]": {
+        "editor.tabSize": 4
+    },
+    "prettier"
+}

README.md

@@ -222,8 +222,43 @@ oldest_block_identifier = first block of oldest_epoch
  - We do not support the `related_transactions` property, since it's not feasible to properly filter the related transactions of a given transaction by source / destination shard (with respect to the observed shard).
  - The endpoint `/block/transaction` is not implemented, since all transactions are returned by the endpoint `/block`.
  - We chose not to support the optional property `Operation.related_operations`. Although the smart contract results (also known as _unsigned transactions_) form a DAG (directed acyclic graph) at the protocol level, operations within a transaction are in a simple sequence.
- - Balance-changing operations that affect Smart Contract accounts are not emitted by our Rosetta implementation (thus are not available on the Rosetta API).
+ - Balance-changing operations that affect Smart Contract accounts are only emitted if Rosetta is started with the flag `--handle-contracts`.
 
 ## Implementation validation
 
 In order to validate the Rosetta implementation using `rosetta-cli`, please follow [MultiversX/rosetta-checks](https://github.com/multiversx/mx-chain-rosetta-checks).
+
+## System tests
+
+### Virtual environment
+
+Create a virtual environment and install the dependencies:
+
+```
+python3 -m venv ./venv
+source ./venv/bin/activate
+pip install -r ./requirements-dev.txt --upgrade
+```
+
+### Check:data
+
+Optional: generate sample data (transactions):
+
+```
+source .env
+
+PYTHONPATH=. python3 ./systemtests/generate_testdata_on_network.py setup --network testnet
+PYTHONPATH=. python3 ./systemtests/generate_testdata_on_network.py run --network testnet
+```
+
+Run the checks:
+
+```
+PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=data --network=testnet
+```
+
+### Check:construction
+
+```
+PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=construction-native --network=testnet
+```

cmd/rosetta/cli.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"math"
+
 	logger "github.com/multiversx/mx-chain-logger-go"
 	"github.com/urfave/cli"
 )
@@ -104,13 +106,13 @@ VERSION:
 
 	cliFlagMinGasPrice = cli.Uint64Flag{
 		Name:  "min-gas-price",
-		Usage: "Specifies the minimum gas price (required in offline mode).",
+		Usage: "Specifies the minimum gas price (for transaction construction).",
 		Value: 1000000000,
 	}
 
 	cliFlagMinGasLimit = cli.UintFlag{
 		Name:  "min-gas-limit",
-		Usage: "Specifies the minimum gas limit (required in offline mode).",
+		Usage: "Specifies the minimum gas limit (for transaction construction).",
 		Value: 50000,
 	}
 
@@ -122,10 +124,22 @@ VERSION:
 
 	cliFlagGasPerDataByte = cli.UintFlag{
 		Name:  "gas-per-data-byte",
-		Usage: "Specifies the gas required per data byte (required in offline mode).",
+		Usage: "Specifies the gas required per data byte (for transaction construction).",
 		Value: 1500,
 	}
 
+	cliFlagGasPriceModifier = cli.Float64Flag{
+		Name:  "gas-price-modifier",
+		Usage: "Specifies the gas price modifier (for transaction construction).",
+		Value: 0.01,
+	}
+
+	cliFlagGasLimitCustomTransfer = cli.Float64Flag{
+		Name:  "gas-limit-custom-transfer",
+		Usage: "Specifies the necessary gas limit for a custom transfer (for transaction construction).",
+		Value: 200000,
+	}
+
 	cliFlagNativeCurrencySymbol = cli.StringFlag{
 		Name:  "native-currency",
 		Usage: "Specifies the symbol of the native currency (must be EGLD for mainnet, XeGLD for testnet and devnet).",
@@ -143,6 +157,36 @@ VERSION:
 		Usage:    "Provides a hint for the number of historical epochs to be kept.",
 		Required: true,
 	}
+
+	cliFlagShouldHandleContracts = cli.BoolFlag{
+		Name:  "handle-contracts",
+		Usage: "Whether to handle balance changes of smart contracts or not.",
+	}
+
+	cliFlagConfigFileCustomCurrencies = cli.StringFlag{
+		Name:     "config-custom-currencies",
+		Usage:    "Specifies the configuration file for custom currencies.",
+		Required: false,
+	}
+
+	cliFlagActivationEpochSirius = cli.UintFlag{
+		Name:     "activation-epoch-sirius",
+		Usage:    "Specifies the activation epoch for Sirius release.",
+		Required: false,
+		Value:    1265,
+	}
+
+	cliFlagActivationEpochSpica = cli.UintFlag{
+		Name:     "activation-epoch-spica",
+		Usage:    "Specifies the activation epoch for Spica release.",
+		Required: false,
+		Value:    math.MaxUint32,
+	}
+
+	cliFlagShouldEnablePprofEndpoints = cli.BoolFlag{
+		Name:  "pprof",
+		Usage: "Whether to enable pprof HTTP endpoints.",
+	}
 )
 
 func getAllCliFlags() []cli.Flag {
@@ -164,9 +208,16 @@ func getAllCliFlags() []cli.Flag {
 		cliFlagMinGasLimit,
 		cliFlagExtraGasLimitGuardedTx,
 		cliFlagGasPerDataByte,
+		cliFlagGasPriceModifier,
+		cliFlagGasLimitCustomTransfer,
 		cliFlagNativeCurrencySymbol,
 		cliFlagFirstHistoricalEpoch,
 		cliFlagNumHistoricalEpochs,
+		cliFlagShouldHandleContracts,
+		cliFlagConfigFileCustomCurrencies,
+		cliFlagActivationEpochSirius,
+		cliFlagActivationEpochSpica,
+		cliFlagShouldEnablePprofEndpoints,
 	}
 }
 
@@ -189,9 +240,16 @@ type parsedCliFlags struct {
 	minGasLimit                 uint64
 	extraGasLimitGuardedTx      uint64
 	gasPerDataByte              uint64
+	gasPriceModifier            float64
+	gasLimitCustomTransfer      uint64
 	nativeCurrencySymbol        string
 	firstHistoricalEpoch        uint32
 	numHistoricalEpochs         uint32
+	shouldHandleContracts       bool
+	configFileCustomCurrencies  string
+	activationEpochSirius       uint32
+	activationEpochSpica        uint32
+	shouldEnablePprofEndpoints  bool
 }
 
 func getParsedCliFlags(ctx *cli.Context) parsedCliFlags {
@@ -214,8 +272,15 @@ func getParsedCliFlags(ctx *cli.Context) parsedCliFlags {
 		minGasLimit:                 ctx.GlobalUint64(cliFlagMinGasLimit.Name),
 		extraGasLimitGuardedTx:      ctx.GlobalUint64(cliFlagExtraGasLimitGuardedTx.Name),
 		gasPerDataByte:              ctx.GlobalUint64(cliFlagGasPerDataByte.Name),
+		gasPriceModifier:            ctx.GlobalFloat64(cliFlagGasPriceModifier.Name),
+		gasLimitCustomTransfer:      ctx.GlobalUint64(cliFlagGasLimitCustomTransfer.Name),
 		nativeCurrencySymbol:        ctx.GlobalString(cliFlagNativeCurrencySymbol.Name),
 		firstHistoricalEpoch:        uint32(ctx.GlobalUint(cliFlagFirstHistoricalEpoch.Name)),
 		numHistoricalEpochs:         uint32(ctx.GlobalUint(cliFlagNumHistoricalEpochs.Name)),
+		shouldHandleContracts:       ctx.GlobalBool(cliFlagShouldHandleContracts.Name),
+		configFileCustomCurrencies:  ctx.GlobalString(cliFlagConfigFileCustomCurrencies.Name),
+		activationEpochSirius:       uint32(ctx.GlobalUint(cliFlagActivationEpochSirius.Name)),
+		activationEpochSpica:        uint32(ctx.GlobalUint(cliFlagActivationEpochSpica.Name)),
+		shouldEnablePprofEndpoints:  ctx.GlobalBool(cliFlagShouldEnablePprofEndpoints.Name),
 	}
 }

cmd/rosetta/config.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/multiversx/mx-chain-rosetta/server/resources"
+)
+
+func decideCustomCurrencies(configFileCustomCurrencies string) ([]resources.Currency, error) {
+	if len(configFileCustomCurrencies) == 0 {
+		return make([]resources.Currency, 0), nil
+	}
+
+	return loadConfigOfCustomCurrencies(configFileCustomCurrencies)
+}
+
+func loadConfigOfCustomCurrencies(configFile string) ([]resources.Currency, error) {
+	fileContent, err := os.ReadFile(configFile)
+	if err != nil {
+		return nil, fmt.Errorf("error when reading custom currencies config file: %w", err)
+	}
+
+	var customCurrencies []resources.Currency
+
+	err = json.Unmarshal(fileContent, &customCurrencies)
+	if err != nil {
+		return nil, fmt.Errorf("error when loading custom currencies from file: %w", err)
+	}
+
+	return customCurrencies, nil
+}

cmd/rosetta/config_test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/multiversx/mx-chain-rosetta/server/resources"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecideCustomCurrencies(t *testing.T) {
+	t.Run("with success (file provided)", func(t *testing.T) {
+		customCurrencies, err := decideCustomCurrencies("testdata/custom-currencies.json")
+		require.NoError(t, err)
+		require.Len(t, customCurrencies, 2)
+	})
+
+	t.Run("with success (file not provided)", func(t *testing.T) {
+		customCurrencies, err := decideCustomCurrencies("")
+		require.NoError(t, err)
+		require.Empty(t, customCurrencies)
+	})
+}
+
+func TestLoadConfigOfCustomCurrencies(t *testing.T) {
+	t.Run("with success", func(t *testing.T) {
+		customCurrencies, err := loadConfigOfCustomCurrencies("testdata/custom-currencies.json")
+		require.NoError(t, err)
+		require.NoError(t, err)
+		require.Equal(t, []resources.Currency{
+			{
+				Symbol:   "WEGLD-bd4d79",
+				Decimals: 18,
+			},
+			{
+				Symbol:   "USDC-c76f1f",
+				Decimals: 6,
+			},
+		}, customCurrencies)
+	})
+
+	t.Run("with error (missing file)", func(t *testing.T) {
+		_, err := loadConfigOfCustomCurrencies("testdata/missing-file.json")
+		require.ErrorContains(t, err, "error when reading custom currencies config file")
+	})
+
+	t.Run("with error (invalid file)", func(t *testing.T) {
+		_, err := loadConfigOfCustomCurrencies("testdata/custom-currencies-bad.json")
+		require.ErrorContains(t, err, "error when loading custom currencies from file")
+	})
+}