Enable URL liveness check for release builds

Signed-off-by: Abdulbois <abdulbois.tursunov@dsr-corporation.com>

Author: Abdulbois

.github/workflows/release.yml

@@ -24,6 +24,7 @@ on:
 env:
   BIN_NAME: dcld
   COSMOVISOR_VERSION: 1.5.0
+  URL_LIVENESS_CHECK_ENABLED: true
 
 jobs:
 

Makefile

@@ -11,7 +11,7 @@ endif
 NAME ?= dcl
 APPNAME ?= $(NAME)d
 LEDGER_ENABLED ?= true
-
+URL_LIVENESS_CHECK_ENABLED ?= true
 OUTPUT_DIR ?= build
 
 ### Process ld flags
@@ -56,6 +56,10 @@ ifeq (cleveldb,$(findstring cleveldb,$(COSMOS_BUILD_OPTIONS)))
   build_tags += gcc
 endif
 
+ifeq ($(URL_LIVENESS_CHECK_ENABLED),false)
+  build_tags += dev
+endif
+
 whitespace :=
 whitespace += $(whitespace)
 comma := ,
@@ -110,7 +114,7 @@ go.sum: go.mod
 	GO111MODULE=on go mod verify
 
 test:
-	go test -v $(PACKAGES)
+	URL_LIVENESS_CHECK_ENABLED=false go test -tags=dev -v $(PACKAGES)
 
 lint:
 	golangci-lint run ./... --timeout 5m0s

internal/config/config.go

@@ -0,0 +1,5 @@
+//go:build !dev
+
+package config
+
+const DisableURLLivenessCheck = false

internal/config/config_dev.go

@@ -0,0 +1,5 @@
+//go:build dev
+
+package config
+
+const DisableURLLivenessCheck = true

utils/validator/validations.go

@@ -15,10 +15,13 @@
 package validator
 
 import (
+	"net/http"
 	"net/url"
 	"reflect"
+	"time"
 
 	"github.com/go-playground/validator/v10"
+	"github.com/zigbee-alliance/distributed-compliance-ledger/internal/config"
 )
 
 func requiredIfBit0Set(fl validator.FieldLevel) bool {
@@ -56,20 +59,58 @@ func isValidHttpsUrl(fl validator.FieldLevel) bool { //nolint:stylecheck
 	return _validURL(fl, "https")
 }
 
+var allowed4XXStatusCodes = []int{
+	http.StatusUnauthorized,
+	http.StatusForbidden,
+	http.StatusUnavailableForLegalReasons,
+}
+var httpClient = &http.Client{Timeout: 10 * time.Second}
+
 func _validURL(fl validator.FieldLevel, allowedSchemas ...string) bool {
 	raw := fl.Field().String()
-	// Field is empty or omitempty is set, skip checks
+	// Field is empty, or omitempty is set, skip checks
 	if raw == "" {
 		return true
 	}
 
-	u, _ := url.Parse(raw)
-	if u.Host == "" {
+	u, err := url.ParseRequestURI(raw)
+	if err != nil || u.Host == "" {
 		return false
 	}
 
+	isSchemaAllowed := false
 	for _, schema := range allowedSchemas {
 		if u.Scheme == schema {
+			isSchemaAllowed = true
+			break
+		}
+	}
+
+	if _isLiveURL(u) || !isSchemaAllowed {
+		return isSchemaAllowed
+	}
+
+	return false
+}
+
+func _isLiveURL(u *url.URL) bool {
+	if config.DisableURLLivenessCheck {
+		return true
+	}
+
+	// HEAD request only retrieves headers, not the body
+	resp, err := httpClient.Head(u.String())
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 200 && resp.StatusCode < 400 {
+		return true
+	}
+
+	for _, code := range allowed4XXStatusCodes {
+		if code == resp.StatusCode {
 			return true
 		}
 	}

utils/validator/validations_test.go

@@ -0,0 +1,52 @@
+// Copyright 2020 DSR Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !dev
+
+package validator
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestURLLivenessCheck(t *testing.T) {
+	negativeTests := []string{
+		"https://dcl-test.org",
+		"https://httpbin.org/status/404",
+		"https://httpbin.org/status/500",
+	}
+	positiveTests := []string{
+		"http://github.com/",             // Redirects to https://github.com/
+		"https://httpbin.org/status/401", // Private repo
+		"https://httpbin.org/status/403", // Unavailable for some reason
+	}
+
+	for _, testUrl := range negativeTests {
+		u, err := url.ParseRequestURI(testUrl)
+		require.NoError(t, err)
+
+		require.False(t, _isLiveURL(u))
+	}
+
+	for _, testUrl := range positiveTests {
+		u, err := url.ParseRequestURI(testUrl)
+		require.NoError(t, err)
+
+		require.True(t, _isLiveURL(u))
+	}
+
+}