#724 Enable URL liveness check for release builds

.github/workflows/release.yml

@@ -24,6 +24,7 @@ on:
 env:
   BIN_NAME: dcld
   COSMOVISOR_VERSION: 1.5.0
+  URL_LIVENESS_CHECK_ENABLED: true
 
 jobs:
 

.github/workflows/verify.yml

@@ -112,7 +112,8 @@ jobs:
         run: |
           set -euo pipefail
           # shellcheck disable=SC2046
-          go test -json -v $(go list ./... | grep -v '/integration_tests') -coverprofile=./cover.out -covermode=set -coverpkg=./... 2>&1 | tee /tmp/gotest.log | gotestfmt
+          go test -tags=dev -json -v $(go list ./... | grep -v '/integration_tests') -coverprofile=./cover.out -covermode=set -coverpkg=./... 2>&1 | tee /tmp/gotest.log | gotestfmt
+          go test -v utils/cli/url_liveness.go  utils/cli/url_liveness_test.go
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
           GOCOVER_MODE: set
@@ -147,6 +148,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
           UBUNTU_VERSION: ${{ 24.04 }}
           GOCOVER_MODE: set  # Ensure integration tests use the same mode as unit tests
+          URL_LIVENESS_CHECK_ENABLED: false # Disable network calls to check URLs for integration tests
       - name: Generate test coverage
         shell: bash
         run: go tool covdata textfmt -i=integration_tests/gocover -o cover.out

Dockerfile-build

@@ -40,6 +40,7 @@ COPY app ./app/
 COPY cmd ./cmd/
 COPY testutil ./testutil/
 COPY utils ./utils/
+COPY internal ./internal/
 COPY x ./x/
 COPY types ./types/
 COPY go.mod go.sum Makefile ./

Makefile

@@ -1,4 +1,5 @@
 PACKAGES = $(shell go list ./... | grep -v '/integration_tests')
+TESTS_WITH_DEV_TAG_DISABLED = $(shell grep -rl '//go:build !dev' --include='*_test.go' .)
 
 ifndef DCL_VERSION
 DCL_VERSION := $(shell echo $(shell git describe --tags --always) | sed 's/^v//')
@@ -11,6 +12,7 @@ endif
 NAME ?= dcl
 APPNAME ?= $(NAME)d
 LEDGER_ENABLED ?= true
+URL_LIVENESS_CHECK_ENABLED ?= true
 CGO_ENABLED ?= 0
 
 OUTPUT_DIR ?= build
@@ -58,6 +60,10 @@ ifeq (cleveldb,$(findstring cleveldb,$(COSMOS_BUILD_OPTIONS)))
   build_tags += gcc
 endif
 
+ifeq ($(URL_LIVENESS_CHECK_ENABLED),false)
+  build_tags += dev
+endif
+
 whitespace :=
 whitespace += $(whitespace)
 comma := ,
@@ -112,7 +118,11 @@ go.sum: go.mod
 	GO111MODULE=on go mod verify
 
 test:
-	go test -v $(PACKAGES)
+	go test -tags=dev -v $(PACKAGES)
+	@for f in $(TESTS_WITH_DEV_TAG_DISABLED); do \
+		d=$$(dirname $$f); \
+		go test -v $$(ls $$d/*.go | grep -v _test.go) $$f || exit 1; \
+	done
 
 lint:
 	golangci-lint run ./... --timeout 5m0s

integration_tests/deploy/Dockerfile-build

@@ -25,6 +25,7 @@ COPY app ./app/
 COPY cmd ./cmd/
 COPY testutil ./testutil/
 COPY utils ./utils/
+COPY internal ./internal/
 COPY x ./x/
 COPY types ./types/
 COPY go.mod go.sum Makefile ./

integration_tests/run-all.sh

@@ -155,8 +155,8 @@ set -euo pipefail
 log "Verifying the environment"
 check_env
 
-log "Compiling local binaries"
-make install &>${DETAILED_OUTPUT_TARGET}
+log "Compiling local binaries. URL liveness check is DISABLED"
+URL_LIVENESS_CHECK_ENABLED=false make install &>${DETAILED_OUTPUT_TARGET}
 
 log "Building docker image"
 make image &>${DETAILED_OUTPUT_TARGET}

internal/config/config.go

@@ -0,0 +1,5 @@
+//go:build !dev
+
+package config
+
+const URLLivenessCheckEnabled = true

internal/config/config_dev.go

@@ -0,0 +1,5 @@
+//go:build dev
+
+package config
+
+const URLLivenessCheckEnabled = false

utils/cli/url_liveness.go

@@ -0,0 +1,88 @@
+package cli
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/zigbee-alliance/distributed-compliance-ledger/internal/config"
+)
+
+const (
+	livenessCheckTimeout = 10 * time.Second
+)
+
+var allowed4XXStatusCodes = []int{
+	http.StatusUnauthorized,
+	http.StatusForbidden,
+	http.StatusUnavailableForLegalReasons,
+	http.StatusMethodNotAllowed,
+}
+var httpClient = &http.Client{Timeout: livenessCheckTimeout}
+
+func IsLiveURL(u string) bool {
+	if !config.URLLivenessCheckEnabled {
+		return true
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), livenessCheckTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil)
+	if err != nil {
+		return false
+	}
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusBadRequest {
+		return true
+	}
+
+	for _, code := range allowed4XXStatusCodes {
+		if code == resp.StatusCode {
+			return true
+		}
+	}
+
+	return false
+}
+
+// CheckURLsForLiveness checks the liveness of the given URLs concurrently and
+// returns unreachable URLs as a list.
+// Empty strings are skipped.
+//
+// Returns an empty list if all non-empty URLs are reachable.
+func CheckURLsForLiveness(urls ...string) []string {
+	results := make([]string, len(urls))
+
+	var wg sync.WaitGroup
+	for i, u := range urls {
+		if u == "" {
+			continue
+		}
+		// Call each URL concurrently
+		wg.Add(1)
+		go func(i int, u string) {
+			defer wg.Done()
+			if !IsLiveURL(u) {
+				results[i] = u
+			}
+		}(i, u)
+	}
+	wg.Wait()
+
+	var unreachable []string
+	for _, u := range results {
+		if u != "" {
+			unreachable = append(unreachable, u)
+		}
+	}
+
+	return unreachable
+}

utils/cli/url_liveness_test.go

@@ -0,0 +1,134 @@
+// Copyright 2020 DSR Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !dev
+
+package cli
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+const unreachableURL = "http://192.0.2.1:1"
+
+func TestIsLiveURL(t *testing.T) {
+	tests := []struct {
+		name       string
+		statusCode int
+		want       bool
+	}{
+		{"200 OK", http.StatusOK, true},
+		{"301 redirect", http.StatusMovedPermanently, true},
+		{"401 unauthorized", http.StatusUnauthorized, true},
+		{"403 forbidden", http.StatusForbidden, true},
+		{"405 method not allowed", http.StatusMethodNotAllowed, true},
+		{"451 unavailable for legal reasons", http.StatusUnavailableForLegalReasons, true},
+		{"404 not found", http.StatusNotFound, false},
+		{"500 internal server error", http.StatusInternalServerError, false},
+		{"502 bad gateway", http.StatusBadGateway, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, http.MethodHead, r.Method)
+				w.WriteHeader(tt.statusCode)
+			}))
+			defer srv.Close()
+
+			u, err := url.ParseRequestURI(srv.URL)
+			require.NoError(t, err)
+
+			require.Equal(t, tt.want, IsLiveURL(u.String()))
+		})
+	}
+}
+
+func TestIsLiveURLUnreachable(t *testing.T) {
+	u, err := url.ParseRequestURI(unreachableURL)
+	require.NoError(t, err)
+
+	require.False(t, IsLiveURL(u.String()))
+}
+
+func TestCheckURLsForLiveness(t *testing.T) {
+	okSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer okSrv.Close()
+
+	notFoundSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer notFoundSrv.Close()
+
+	tests := []struct {
+		name string
+		urls []string
+		want []string
+	}{
+		{"no URLs", nil, nil},
+		{"all empty strings", []string{"", "", ""}, nil},
+		{"all reachable", []string{okSrv.URL, okSrv.URL}, nil},
+		{"single unreachable", []string{okSrv.URL, notFoundSrv.URL}, []string{notFoundSrv.URL}},
+		{"empties skipped", []string{"", okSrv.URL, ""}, nil},
+		{
+			"multiple unreachable preserve input order",
+			[]string{okSrv.URL, notFoundSrv.URL, unreachableURL},
+			[]string{notFoundSrv.URL, unreachableURL},
+		},
+		{
+			"unreachable later in list",
+			[]string{okSrv.URL, "", notFoundSrv.URL},
+			[]string{notFoundSrv.URL},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.want, CheckURLsForLiveness(tt.urls...))
+		})
+	}
+}
+
+func TestCheckURLsForLivenessRunsConcurrently(t *testing.T) {
+	const handlerDelay = 200 * time.Millisecond
+	const concurrentURLs = 5
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(handlerDelay)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	urls := make([]string, concurrentURLs)
+	for i := range urls {
+		urls[i] = srv.URL
+	}
+
+	start := time.Now()
+	require.Empty(t, CheckURLsForLiveness(urls...))
+	elapsed := time.Since(start)
+
+	// Sequential calls would take approximately concurrentURLs*handlerDelay time
+	// Concurrent execution should finish in roughly handlerDelay.
+	require.Less(t, elapsed, time.Duration(concurrentURLs)*handlerDelay/2,
+		"URL checks did not run concurrently (elapsed %s)", elapsed)
+}

utils/validator/validations.go

@@ -49,27 +49,30 @@ func requiredIfBit0Set(fl validator.FieldLevel) bool {
 }
 
 func isValidHttpOrHttpsUrl(fl validator.FieldLevel) bool { //nolint:stylecheck
-	return _validURL(fl, "http", "https")
+	return validURL(fl, "http", "https")
 }
 
 func isValidHttpsUrl(fl validator.FieldLevel) bool { //nolint:stylecheck
-	return _validURL(fl, "https")
+	return validURL(fl, "https")
 }
 
-func _validURL(fl validator.FieldLevel, allowedSchemas ...string) bool {
+func validURL(fl validator.FieldLevel, allowedSchemes ...string) bool {
 	raw := fl.Field().String()
-	// Field is empty or omitempty is set, skip checks
 	if raw == "" {
 		return true
 	}
 
-	u, _ := url.Parse(raw)
-	if u.Host == "" {
+	u, err := url.ParseRequestURI(raw)
+	if err != nil || u.Host == "" {
 		return false
 	}
 
-	for _, schema := range allowedSchemas {
-		if u.Scheme == schema {
+	return isSchemeAllowed(u.Scheme, allowedSchemes)
+}
+
+func isSchemeAllowed(scheme string, allowed []string) bool {
+	for _, s := range allowed {
+		if scheme == s {
 			return true
 		}
 	}