Merge pull request #4960 from harmony-one/dev

Author: GheisMohammadi

.github/workflows/build-docker-proto-image.yaml

@@ -27,8 +27,8 @@ jobs:
       - name: Build and push
         uses: docker/build-push-action@v6
         with:
-          context: ./harmony/api/service/legacysync/downloader
-          file: ./harmony/api/service/legacysync/downloader/Proto.Dockerfile
+          context: ./harmony/api/service/synchronize/legacysync/downloader
+          file: ./harmony/api/service/synchronize/legacysync/downloader/Proto.Dockerfile
           push: true
           platforms: linux/amd64,linux/arm64
           tags: |

.gitignore

@@ -76,9 +76,11 @@ blspass.txt
 # local test db
 db-127.0.0.1-*
 
-# dht storage
+# dht and peer storage
 .dht
 .dht-*
+.ps
+.ps-*
 .tern-port
 
 # testdata

.hmy/wallet.ini

@@ -4,7 +4,7 @@ bootnode = /ip4/100.26.90.187/tcp/9874/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvv
 bootnode = /ip4/54.213.43.194/tcp/9874/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
 bootnode = /ip4/13.113.101.219/tcp/12019/p2p/QmQayinFSgMMw5cSpDUiD9pQ2WeP6WNmGxpZ6ou3mdVFJX
 bootnode = /ip4/99.81.170.167/tcp/12019/p2p/QmRVbTpEYup8dSaURZfF6ByrMTSKa4UyUzJhSjahFzRqNj
-shards = 4
+shards = 2
 network = mainnet
 
 [main.shard0.rpc]
@@ -15,14 +15,6 @@ rpc = s0.t.hmny.io:14555
 rpc = l1.t.hmny.io:14555
 rpc = s1.t.hmny.io:14555
 
-[main.shard2.rpc]
-rpc = l2.t.hmny.io:14555
-rpc = s2.t.hmny.io:14555
-
-[main.shard3.rpc]
-rpc = l3.t.hmny.io:14555
-rpc = s3.t.hmny.io:14555
-
 [local]
 chain_id = 2
 bootnode = /ip4/127.0.0.1/tcp/19876/p2p/Qmc1V6W7BwX8Ugb42Ti8RnXF1rY5PF7nnZ6bKBryCgi6cv
@@ -59,7 +51,7 @@ rpc = s1.b.hmny.io:14555
 chain_id = 3
 bootnode = /ip4/54.86.126.90/tcp/9889/p2p/Qmdfjtk6hPoyrH1zVD9PEH4zfWLo38dP2mDvvKXfh3tnEv
 bootnode = /ip4/52.40.84.2/tcp/9889/p2p/QmZJJx6AdaoEkGLrYG4JeLCKeCKDjnFz2wfHNHxAqFSGA9
-shards = 4
+shards = 2
 network = pangaea
 
 [pangaea.shard0.rpc]
@@ -69,11 +61,3 @@ rpc = s0.p.hmny.io:14555
 [pangaea.shard1.rpc]
 rpc = l1.p.hmny.io:14555
 rpc = s1.p.hmny.io:14555
-
-[pangaea.shard2.rpc]
-rpc = l2.p.hmny.io:14555
-rpc = s2.p.hmny.io:14555
-
-[pangaea.shard3.rpc]
-rpc = l3.p.hmny.io:14555
-rpc = s3.p.hmny.io:14555

.travis.yml

@@ -13,13 +13,12 @@ env:
   - TEST="make"
   - TEST="bash ./scripts/travis_go_checker.sh"
   - TEST="bash ./scripts/travis_rpc_checker.sh"
-  - TEST="bash ./scripts/travis_rosetta_checker.sh"
   - TEST="bash ./scripts/travis_pyhmy_checker.sh"
 
-# We enable Travis on the main/dev branches or forked repositories here.
-# PRs workflow isn't affected by this condition
+# We enable Travis on the main/dev branches.
+# pull_request type includes both main repo/fork PR workflow
 if: >
-  repo != harmony-one/harmony OR
+  type = pull_request OR
   (repo = harmony-one/harmony AND (branch = main OR branch = dev))
 
 # upgrade docker to latest stable version
@@ -40,7 +39,7 @@ install:
 #  - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.41.1
   - make go-get
   - go install golang.org/x/tools/cmd/goimports@v0.30.0
-  - go install github.com/fjl/gencodec@v0.1.0
+  - go install github.com/fjl/gencodec@v0.1.1
   - echo "[WARN] - workaround for the GOPATH:"
   # sometimes Travis decides to respect GOPATH and creates a symlink, thus we have || true for such cases
   - rm $GOPATH/src/github.com/harmony-one/harmony || true;

Makefile

@@ -11,6 +11,8 @@ RELEASE?=$(shell git describe --long | cut -f2 -d-)
 RPMBUILD=$(HOME)/rpmbuild
 DEBBUILD=$(HOME)/debbuild
 SHELL := bash
+EPOCH_TO_WAIT ?=5
+EXTRA_NODES_FILE ?="./test/configs/local-extra-nodes.txt"
 
 .PHONY: all help libs exe race trace-pointer debug debug-ext debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker go-vet go-test docker build_localnet_validator protofiles travis_go_checker travis_rpc_checker travis_rosetta_checker debug-start-log debug-stop-log debug-restart-log debug-delete-log
 
@@ -38,6 +40,7 @@ help:
 	@echo "test-rosetta-attach - attach onto the rosetta testing docker container for inspection"
 	@echo "linux_static - static build the harmony binary & bootnode along with the MCL & BLS libs (for linux)"
 	@echo "linux_static_quick - static build the harmony binary & bootnode more quickly without recompiling dependencies (for linux)"
+	@echo "linux_static_cross_build - cross-compile static Linux binaries for Harmony and Bootnode from macOS"
 	@echo "rpm_init - prepare the RPM build environment by creating directories, copying files, and generating the spec file and source tarball"
 	@echo "rpm_build - build an RPM package for x86_64 architecture using the spec file (<RPMBUILD>/SPECS/harmony.spec)"
 	@echo "rpm - build a harmony RPM pacakge"
@@ -56,6 +59,7 @@ help:
 	@echo "travis_rosetta_checker - run the Travis Rosetta checker script, defaulting the test branch to 'master' unless overridden by TEST_REPO_BRANCH"
 	@echo "debug_external - cleans up environment, rebuilds the binary, and deploys with external nodes"
 	@echo "debug-multi-bls - cleans up environment, rebuilds the binary, and deploys with external nodes in configuration 1 harmony process -> 2 validators"
+	@echo "debug-add-extra-nodes-to-the-running-network wait for EPOCH_TO_WAIT and add nodes from EXTRA_NODES_FILE file"
 	@echo "build_localnet_validator - imports validator keys, funds validator accounts, waits for the epoch, and creates external validators on a local network"
 	@echo "debug-start-log - start a docker compose Promtail->Loki->Grafana stack against localnet logs, creates"\
 		"persistent volume to store parsed logs between localnet runs, needs docker compose and started localnet"
@@ -79,10 +83,9 @@ trace-pointer:
 	bash ./scripts/go_executable_build.sh -t
 
 debug:
-	rm -rf .dht-127.0.0.1*
 	# uncomment the following lines to enable debug logging for libp2p, it produces a lot of logs, so disabled by default
-	#export GOLOG_LOG_LEVEL=debug
-	#export GOLOG_OUTPUT=stdout
+	# export GOLOG_LOG_LEVEL=debug
+	# export GOLOG_OUTPUT=stdout
 	# add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging
 	# add LEGACY_SYNC=true before bash  or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync
 	bash ./test/debug.sh ./test/configs/local-resharding.txt
@@ -117,6 +120,11 @@ debug-multi-bls-with-terminal:
 	bash ./test/build-localnet-validator.sh
 	screen -r localnet
 
+debug-add-extra-nodes-to-the-running-network:
+	echo waiting for the $(EPOCH_TO_WAIT) epoch on the localnet
+	bash ./test/wait_till_n_epoch.sh $(EPOCH_TO_WAIT)
+	./test/debug.sh $(EXTRA_NODES_FILE) 64 64 false &
+
 debug-multi-bls-multi-ext-node:
 	# add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging
 	# add LEGACY_SYNC=true before bash  or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync
@@ -127,6 +135,7 @@ debug-multi-bls-multi-ext-node:
 
 clean:
 	rm -rf ./tmp_log/*
+	rm -rf ./.ps*
 	rm -rf ./.dht*
 	rm -rf ./db-*
 	rm -rf ./latest
@@ -168,6 +177,9 @@ linux_static:
 linux_static_quick:
 	bash ./scripts/go_executable_build.sh -s
 
+linux_static_cross_build:
+	bash ./scripts/linux_executable_from_macos.sh -s
+
 deb_init:
 	rm -rf $(DEBBUILD)
 	mkdir -p $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/{etc/systemd/system,usr/sbin,etc/sysctl.d,etc/harmony}
@@ -257,3 +269,6 @@ debug-delete-log:
 	@echo "[WARN] - it needs sudo to remove folder created with loki docker image user"
 	sudo rm -rf test/logs_aggregator/loki
 
+
+docker-go-test:
+	docker run --rm -it -v "$PWD":/go/src/github.com/harmony-one/harmony frozen621/harmony-test bash -c 'make go-test'

Test.Dockerfile

@@ -0,0 +1,22 @@
+FROM golang:1.24
+
+ARG ENV=dev
+
+RUN apt-get update && \
+    apt-get install -y build-essential make git curl libgmp-dev libssl-dev gcc g++ && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /go/src/github.com/harmony-one
+
+RUN git clone https://github.com/harmony-one/mcl.git && \
+    git clone https://github.com/harmony-one/bls.git
+
+RUN echo "Cloning branch: ${ENV}" && \
+    git clone -b ${ENV} https://github.com/harmony-one/harmony.git && \
+    cd harmony && \
+    go mod tidy && \
+    make deps
+
+WORKDIR /go/src/github.com/harmony-one/harmony
+
+CMD ["make", "go-test"]

api/service/manager.go

@@ -23,7 +23,6 @@ const (
 	Prometheus
 	Synchronize
 	CrosslinkSending
-	StagedStreamSync
 )
 
 func (t Type) String() string {
@@ -46,8 +45,6 @@ func (t Type) String() string {
 		return "Synchronize"
 	case CrosslinkSending:
 		return "CrosslinkSending"
-	case StagedStreamSync:
-		return "StagedStreamSync"
 	default:
 		return "Unknown"
 	}