diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cb1e2c60d..ca15a1ee6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,7 @@ on: env: BIN_NAME: dcld COSMOVISOR_VERSION: 1.5.0 + URL_LIVENESS_CHECK_ENABLED: true jobs: diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 10f6aece0..343da0a75 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -107,7 +107,8 @@ jobs: run: | set -euo pipefail # shellcheck disable=SC2046 - go test -json -v $(go list ./... | grep -v '/integration_tests') -coverprofile=./cover.out -covermode=set -coverpkg=./... 2>&1 | tee /tmp/gotest.log | gotestfmt + go test -tags=dev -json -v $(go list ./... | grep -v '/integration_tests') -coverprofile=./cover.out -covermode=set -coverpkg=./... 2>&1 | tee /tmp/gotest.log | gotestfmt + go test -v utils/cli/url_liveness.go utils/cli/url_liveness_test.go env: GH_TOKEN: ${{ secrets.GH_TOKEN }} GOCOVER_MODE: set @@ -141,6 +142,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GH_TOKEN }} GOCOVER_MODE: set # Ensure integration tests use the same mode as unit tests + URL_LIVENESS_CHECK_ENABLED: false # Disable network calls to check URLs for integration tests - name: Generate test coverage shell: bash run: go tool covdata textfmt -i=integration_tests/gocover -o cover.out diff --git a/Dockerfile-build b/Dockerfile-build index 8f2eaafdb..68d845c8e 100644 --- a/Dockerfile-build +++ b/Dockerfile-build @@ -39,6 +39,7 @@ COPY app ./app/ COPY cmd ./cmd/ COPY testutil ./testutil/ COPY utils ./utils/ +COPY internal ./internal/ COPY x ./x/ COPY types ./types/ COPY go.mod go.sum Makefile ./ diff --git a/Makefile b/Makefile index bf6a61daa..1a00fff62 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ endif NAME ?= dcl APPNAME ?= $(NAME)d LEDGER_ENABLED ?= true - +URL_LIVENESS_CHECK_ENABLED ?= true OUTPUT_DIR ?= build ### Process ld flags @@ -56,6 +56,10 @@ ifeq (cleveldb,$(findstring cleveldb,$(COSMOS_BUILD_OPTIONS))) build_tags += gcc endif +ifeq ($(URL_LIVENESS_CHECK_ENABLED),false) + build_tags += dev +endif + whitespace := whitespace += $(whitespace) comma := , @@ -110,7 +114,8 @@ go.sum: go.mod GO111MODULE=on go mod verify test: - go test -v $(PACKAGES) + go test -tags=dev -v $(PACKAGES) + go test -v utils/cli/url_liveness.go utils/cli/url_liveness_test.go lint: golangci-lint run ./... --timeout 5m0s diff --git a/integration_tests/deploy/Dockerfile-build b/integration_tests/deploy/Dockerfile-build index d5b1b5a91..ea0f94d07 100644 --- a/integration_tests/deploy/Dockerfile-build +++ b/integration_tests/deploy/Dockerfile-build @@ -25,6 +25,7 @@ COPY app ./app/ COPY cmd ./cmd/ COPY testutil ./testutil/ COPY utils ./utils/ +COPY internal ./internal/ COPY x ./x/ COPY types ./types/ COPY go.mod go.sum Makefile ./ diff --git a/integration_tests/run-all.sh b/integration_tests/run-all.sh index 8e1e6e98c..84ff4aa6b 100755 --- a/integration_tests/run-all.sh +++ b/integration_tests/run-all.sh @@ -155,8 +155,8 @@ set -euo pipefail log "Verifying the environment" check_env -log "Compiling local binaries" -make install &>${DETAILED_OUTPUT_TARGET} +log "Compiling local binaries. URL liveness check is DISABLED" +URL_LIVENESS_CHECK_ENABLED=false make install &>${DETAILED_OUTPUT_TARGET} log "Building docker image" make image &>${DETAILED_OUTPUT_TARGET} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 000000000..d1aefea1f --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,5 @@ +//go:build !dev + +package config + +const URLLivenessCheckEnabled = true diff --git a/internal/config/config_dev.go b/internal/config/config_dev.go new file mode 100644 index 000000000..9e82b500d --- /dev/null +++ b/internal/config/config_dev.go @@ -0,0 +1,5 @@ +//go:build dev + +package config + +const URLLivenessCheckEnabled = false diff --git a/utils/cli/url_liveness.go b/utils/cli/url_liveness.go new file mode 100644 index 000000000..f980126d0 --- /dev/null +++ b/utils/cli/url_liveness.go @@ -0,0 +1,88 @@ +package cli + +import ( + "context" + "net/http" + "sync" + "time" + + "github.com/zigbee-alliance/distributed-compliance-ledger/internal/config" +) + +const ( + livenessCheckTimeout = 10 * time.Second +) + +var allowed4XXStatusCodes = []int{ + http.StatusUnauthorized, + http.StatusForbidden, + http.StatusUnavailableForLegalReasons, + http.StatusMethodNotAllowed, +} +var httpClient = &http.Client{Timeout: livenessCheckTimeout} + +func IsLiveURL(u string) bool { + if !config.URLLivenessCheckEnabled { + return true + } + + ctx, cancel := context.WithTimeout(context.Background(), livenessCheckTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil) + if err != nil { + return false + } + + resp, err := httpClient.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + + if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusBadRequest { + return true + } + + for _, code := range allowed4XXStatusCodes { + if code == resp.StatusCode { + return true + } + } + + return false +} + +// CheckURLsForLiveness checks the liveness of the given URLs concurrently and +// returns unreachable URLs as a list. +// Empty strings are skipped. +// +// Returns an empty list if all non-empty URLs are reachable. +func CheckURLsForLiveness(urls ...string) []string { + results := make([]string, len(urls)) + + var wg sync.WaitGroup + for i, u := range urls { + if u == "" { + continue + } + // Call each URL concurrently + wg.Add(1) + go func(i int, u string) { + defer wg.Done() + if !IsLiveURL(u) { + results[i] = u + } + }(i, u) + } + wg.Wait() + + var unreachable []string + for _, u := range results { + if u != "" { + unreachable = append(unreachable, u) + } + } + + return unreachable +} diff --git a/utils/cli/url_liveness_test.go b/utils/cli/url_liveness_test.go new file mode 100644 index 000000000..ddf353a6b --- /dev/null +++ b/utils/cli/url_liveness_test.go @@ -0,0 +1,134 @@ +// Copyright 2020 DSR Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !dev + +package cli + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const unreachableURL = "http://192.0.2.1:1" + +func TestIsLiveURL(t *testing.T) { + tests := []struct { + name string + statusCode int + want bool + }{ + {"200 OK", http.StatusOK, true}, + {"301 redirect", http.StatusMovedPermanently, true}, + {"401 unauthorized", http.StatusUnauthorized, true}, + {"403 forbidden", http.StatusForbidden, true}, + {"405 method not allowed", http.StatusMethodNotAllowed, true}, + {"451 unavailable for legal reasons", http.StatusUnavailableForLegalReasons, true}, + {"404 not found", http.StatusNotFound, false}, + {"500 internal server error", http.StatusInternalServerError, false}, + {"502 bad gateway", http.StatusBadGateway, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, http.MethodHead, r.Method) + w.WriteHeader(tt.statusCode) + })) + defer srv.Close() + + u, err := url.ParseRequestURI(srv.URL) + require.NoError(t, err) + + require.Equal(t, tt.want, IsLiveURL(u.String())) + }) + } +} + +func TestIsLiveURLUnreachable(t *testing.T) { + u, err := url.ParseRequestURI(unreachableURL) + require.NoError(t, err) + + require.False(t, IsLiveURL(u.String())) +} + +func TestCheckURLsForLiveness(t *testing.T) { + okSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer okSrv.Close() + + notFoundSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer notFoundSrv.Close() + + tests := []struct { + name string + urls []string + want []string + }{ + {"no URLs", nil, nil}, + {"all empty strings", []string{"", "", ""}, nil}, + {"all reachable", []string{okSrv.URL, okSrv.URL}, nil}, + {"single unreachable", []string{okSrv.URL, notFoundSrv.URL}, []string{notFoundSrv.URL}}, + {"empties skipped", []string{"", okSrv.URL, ""}, nil}, + { + "multiple unreachable preserve input order", + []string{okSrv.URL, notFoundSrv.URL, unreachableURL}, + []string{notFoundSrv.URL, unreachableURL}, + }, + { + "unreachable later in list", + []string{okSrv.URL, "", notFoundSrv.URL}, + []string{notFoundSrv.URL}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, CheckURLsForLiveness(tt.urls...)) + }) + } +} + +func TestCheckURLsForLivenessRunsConcurrently(t *testing.T) { + const handlerDelay = 200 * time.Millisecond + const concurrentURLs = 5 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(handlerDelay) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + urls := make([]string, concurrentURLs) + for i := range urls { + urls[i] = srv.URL + } + + start := time.Now() + require.Empty(t, CheckURLsForLiveness(urls...)) + elapsed := time.Since(start) + + // Sequential calls would take approximately concurrentURLs*handlerDelay time + // Concurrent execution should finish in roughly handlerDelay. + require.Less(t, elapsed, time.Duration(concurrentURLs)*handlerDelay/2, + "URL checks did not run concurrently (elapsed %s)", elapsed) +} diff --git a/utils/validator/validations.go b/utils/validator/validations.go index 83a10789c..5eccf435e 100644 --- a/utils/validator/validations.go +++ b/utils/validator/validations.go @@ -49,27 +49,30 @@ func requiredIfBit0Set(fl validator.FieldLevel) bool { } func isValidHttpOrHttpsUrl(fl validator.FieldLevel) bool { //nolint:stylecheck - return _validURL(fl, "http", "https") + return validURL(fl, "http", "https") } func isValidHttpsUrl(fl validator.FieldLevel) bool { //nolint:stylecheck - return _validURL(fl, "https") + return validURL(fl, "https") } -func _validURL(fl validator.FieldLevel, allowedSchemas ...string) bool { +func validURL(fl validator.FieldLevel, allowedSchemes ...string) bool { raw := fl.Field().String() - // Field is empty or omitempty is set, skip checks if raw == "" { return true } - u, _ := url.Parse(raw) - if u.Host == "" { + u, err := url.ParseRequestURI(raw) + if err != nil || u.Host == "" { return false } - for _, schema := range allowedSchemas { - if u.Scheme == schema { + return isSchemeAllowed(u.Scheme, allowedSchemes) +} + +func isSchemeAllowed(scheme string, allowed []string) bool { + for _, s := range allowed { + if scheme == s { return true } } diff --git a/x/model/client/cli/tx_model.go b/x/model/client/cli/tx_model.go index efa6ef82a..9f15bcba2 100644 --- a/x/model/client/cli/tx_model.go +++ b/x/model/client/cli/tx_model.go @@ -1,6 +1,9 @@ package cli import ( + "fmt" + "strings" + "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/client/tx" @@ -99,6 +102,12 @@ func CmdCreateModel() *cobra.Command { commissioningFallbackURL, ) + unreachable := cli.CheckURLsForLiveness(commissioningCustomFlowURL, userManualURL, supportURL, productURL, lsfURL, + enhancedSetupFlowTCURL, enhancedSetupFlowMaintenanceURL, commissioningFallbackURL) + if len(unreachable) > 0 { + return fmt.Errorf("URLs not reachable: %s", strings.Join(unreachable, ", ")) + } + // validate basic will be called in GenerateOrBroadcastTxCLI err = tx.GenerateOrBroadcastTxCLI(clientCtx, cmd.Flags(), msg) if cli.IsWriteInsteadReadRPCError(err) { @@ -284,6 +293,12 @@ func CmdUpdateModel() *cobra.Command { factoryResetStepsInstruction, ) + unreachable := cli.CheckURLsForLiveness(commissioningCustomFlowURL, userManualURL, supportURL, productURL, lsfURL, + enhancedSetupFlowTCURL, enhancedSetupFlowMaintenanceURL, commissioningFallbackURL) + if len(unreachable) > 0 { + return fmt.Errorf("URLs not reachable: %s", strings.Join(unreachable, ", ")) + } + // validate basic will be called in GenerateOrBroadcastTxCLI err = tx.GenerateOrBroadcastTxCLI(clientCtx, cmd.Flags(), msg) if cli.IsWriteInsteadReadRPCError(err) { diff --git a/x/model/client/cli/tx_model_version.go b/x/model/client/cli/tx_model_version.go index 4ec678abe..6a66987dd 100644 --- a/x/model/client/cli/tx_model_version.go +++ b/x/model/client/cli/tx_model_version.go @@ -1,6 +1,9 @@ package cli import ( + "fmt" + "strings" + "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/client/tx" @@ -59,6 +62,10 @@ func CmdCreateModelVersion() *cobra.Command { specificationVersion, ) + unreachable := cli.CheckURLsForLiveness(otaURL, releaseNotesURL) + if len(unreachable) > 0 { + return fmt.Errorf("URLs not reachable: %s", strings.Join(unreachable, ", ")) + } // validate basic will be called in GenerateOrBroadcastTxCLI err = tx.GenerateOrBroadcastTxCLI(clientCtx, cmd.Flags(), msg) if cli.IsWriteInsteadReadRPCError(err) { @@ -166,6 +173,11 @@ func CmdUpdateModelVersion() *cobra.Command { schemaVersion, ) + unreachable := cli.CheckURLsForLiveness(otaURL, releaseNotesURL) + if len(unreachable) > 0 { + return fmt.Errorf("URLs not reachable: %s", strings.Join(unreachable, ", ")) + } + // validate basic will be called in GenerateOrBroadcastTxCLI err = tx.GenerateOrBroadcastTxCLI(clientCtx, cmd.Flags(), msg) if cli.IsWriteInsteadReadRPCError(err) { diff --git a/x/pki/client/cli/tx_add_pki_revocation_distribution_point.go b/x/pki/client/cli/tx_add_pki_revocation_distribution_point.go index 0fa50d384..efff903c1 100644 --- a/x/pki/client/cli/tx_add_pki_revocation_distribution_point.go +++ b/x/pki/client/cli/tx_add_pki_revocation_distribution_point.go @@ -1,6 +1,7 @@ package cli import ( + "fmt" "strconv" "github.com/cosmos/cosmos-sdk/client" @@ -69,6 +70,10 @@ func CmdAddPkiRevocationDistributionPoint() *cobra.Command { schemaVersion, ) + if dataURL != "" && !cli.IsLiveURL(dataURL) { + return fmt.Errorf("%s is not reachable", dataURL) + } + // validate basic will be called in GenerateOrBroadcastTxCLI err = tx.GenerateOrBroadcastTxCLI(clientCtx, cmd.Flags(), msg) if cli.IsWriteInsteadReadRPCError(err) { diff --git a/x/pki/client/cli/tx_update_pki_revocation_distribution_point.go b/x/pki/client/cli/tx_update_pki_revocation_distribution_point.go index deca28601..5f1c14ea8 100644 --- a/x/pki/client/cli/tx_update_pki_revocation_distribution_point.go +++ b/x/pki/client/cli/tx_update_pki_revocation_distribution_point.go @@ -1,6 +1,7 @@ package cli import ( + "fmt" "strconv" "github.com/cosmos/cosmos-sdk/client" @@ -63,6 +64,10 @@ func CmdUpdatePkiRevocationDistributionPoint() *cobra.Command { schemaVersion, ) + if dataURL != "" && !cli.IsLiveURL(dataURL) { + return fmt.Errorf("%s is not reachable", dataURL) + } + // validate basic will be called in GenerateOrBroadcastTxCLI err = tx.GenerateOrBroadcastTxCLI(clientCtx, cmd.Flags(), msg) if cli.IsWriteInsteadReadRPCError(err) { diff --git a/x/vendorinfo/client/cli/tx_vendor_info.go b/x/vendorinfo/client/cli/tx_vendor_info.go index edf498f25..526a2754a 100644 --- a/x/vendorinfo/client/cli/tx_vendor_info.go +++ b/x/vendorinfo/client/cli/tx_vendor_info.go @@ -1,6 +1,8 @@ package cli import ( + "fmt" + "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/client/tx" @@ -40,6 +42,10 @@ func CmdCreateVendorInfo() *cobra.Command { schemaVersion, ) + if vendorLandingPageURL != "" && !cli.IsLiveURL(vendorLandingPageURL) { + return fmt.Errorf("%s is not reachable", vendorLandingPageURL) + } + // validate basic will be called in GenerateOrBroadcastTxCLI err = tx.GenerateOrBroadcastTxCLI(clientCtx, cmd.Flags(), msg) if cli.IsWriteInsteadReadRPCError(err) { @@ -101,6 +107,10 @@ func CmdUpdateVendorInfo() *cobra.Command { schemaVersion, ) + if vendorLandingPageURL != "" && !cli.IsLiveURL(vendorLandingPageURL) { + return fmt.Errorf("%s is not reachable", vendorLandingPageURL) + } + // validate basic will be called in GenerateOrBroadcastTxCLI err = tx.GenerateOrBroadcastTxCLI(clientCtx, cmd.Flags(), msg) if cli.IsWriteInsteadReadRPCError(err) {