From a7a505709365c76d0b5fd72ef577aad4387241fb Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 30 May 2025 00:05:59 +0100 Subject: [PATCH 1/3] :sparkles: Generic map and slice serialisers --- .secrets.baseline | 2 +- changes/20250530113842.bugfix | 1 + changes/20250530114030.bugfix | 1 + changes/20250530114133.feature | 1 + changes/20250530114405.feature | 1 + utils/charset/iconv/interfaces.go | 2 +- utils/collection/pagination/interfaces.go | 2 +- utils/collection/parseLists.go | 7 +- utils/config/interfaces.go | 2 +- utils/config/service_configuration.go | 2 +- utils/encryption/interface.go | 2 +- utils/environment/interfaces.go | 2 +- utils/filesystem/interfaces.go | 2 +- utils/go.mod | 14 +- utils/go.sum | 14 +- utils/hashing/interfaces.go | 2 +- utils/http/interfaces.go | 2 +- utils/logs/interfaces.go | 2 +- utils/maps/ReadME.md | 4 + utils/maps/expand.go | 143 ++++++++++++++ utils/maps/expand_test.go | 210 +++++++++++++++++++++ utils/maps/flatten.go | 147 +++++++++++++++ utils/maps/flatten_test.go | 220 ++++++++++++++++++++++ utils/maps/map.go | 88 +++++++++ utils/maps/map_test.go | 114 +++++++++++ utils/maps/overall_test.go | 50 +++++ utils/proc/interfaces.go | 2 +- utils/resource/interfaces.go | 2 +- utils/serialization/maps/map.go | 141 ++++++++++++++ utils/serialization/maps/map_test.go | 138 ++++++++++++++ utils/serialization/slice/slice.go | 33 ++++ utils/serialization/slice/slice_test.go | 157 +++++++++++++++ utils/sharedcache/interface.go | 2 +- utils/subprocess/supervisor/interface.go | 2 +- 34 files changed, 1481 insertions(+), 33 deletions(-) create mode 100644 changes/20250530113842.bugfix create mode 100644 changes/20250530114030.bugfix create mode 100644 changes/20250530114133.feature create mode 100644 changes/20250530114405.feature create mode 100644 utils/maps/ReadME.md create mode 100644 utils/maps/expand.go create mode 100644 utils/maps/expand_test.go create mode 100644 utils/maps/flatten.go create mode 100644 utils/maps/flatten_test.go create mode 100644 utils/maps/map.go create mode 100644 utils/maps/map_test.go create mode 100644 utils/maps/overall_test.go create mode 100644 utils/serialization/maps/map.go create mode 100644 utils/serialization/maps/map_test.go create mode 100644 utils/serialization/slice/slice.go create mode 100644 utils/serialization/slice/slice_test.go diff --git a/.secrets.baseline b/.secrets.baseline index 931a4b6e81..466543e52b 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -265,5 +265,5 @@ } ] }, - "generated_at": "2025-05-12T17:01:47Z" + "generated_at": "2025-05-30T10:38:21Z" } diff --git a/changes/20250530113842.bugfix b/changes/20250530113842.bugfix new file mode 100644 index 0000000000..39355773f9 --- /dev/null +++ b/changes/20250530113842.bugfix @@ -0,0 +1 @@ +:camel: Upgrade dependencies diff --git a/changes/20250530114030.bugfix b/changes/20250530114030.bugfix new file mode 100644 index 0000000000..33ac61015b --- /dev/null +++ b/changes/20250530114030.bugfix @@ -0,0 +1 @@ +:gear: Move dependency on mapstructure to use https://github.com/go-viper/mapstructure as https://github.com/mitchellh/mapstructure is [no longer maintained](https://github.com/go-viper/mapstructure?tab=readme-ov-file#migrating-from-githubcommitchellhmapstructure) diff --git a/changes/20250530114133.feature b/changes/20250530114133.feature new file mode 100644 index 0000000000..8fd49976c8 --- /dev/null +++ b/changes/20250530114133.feature @@ -0,0 +1 @@ +:sparkles: [serialization] Added ways to serialise structs into maps or slice diff --git a/changes/20250530114405.feature b/changes/20250530114405.feature new file mode 100644 index 0000000000..3c5e678618 --- /dev/null +++ b/changes/20250530114405.feature @@ -0,0 +1 @@ +:sparkles: [maps] Added ways to expand and flatten maps in a similar fashion to https://github.com/krakend/flatmap and no longer available https://pkg.go.dev/github.com/hashicorp/terraform/flatmap diff --git a/utils/charset/iconv/interfaces.go b/utils/charset/iconv/interfaces.go index 1586c1ed34..3e464c4458 100644 --- a/utils/charset/iconv/interfaces.go +++ b/utils/charset/iconv/interfaces.go @@ -11,7 +11,7 @@ import ( "io" ) -//go:generate mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/charset/$GOPACKAGE ICharsetConverter +//go:generate go tool mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/charset/$GOPACKAGE ICharsetConverter type ICharsetConverter interface { // ConvertString converts the charset of an input string diff --git a/utils/collection/pagination/interfaces.go b/utils/collection/pagination/interfaces.go index 6d72a8fed4..d8fb9ff2ac 100644 --- a/utils/collection/pagination/interfaces.go +++ b/utils/collection/pagination/interfaces.go @@ -12,7 +12,7 @@ import ( "io" ) -//go:generate mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/collection/$GOPACKAGE IStaticPage,IPage,IStaticPageStream,IStream,IIterator,IPaginator,IPaginatorAndPageFetcher,IStreamPaginator,IStreamPaginatorAndPageFetcher +//go:generate go tool mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/collection/$GOPACKAGE IStaticPage,IPage,IStaticPageStream,IStream,IIterator,IPaginator,IPaginatorAndPageFetcher,IStreamPaginator,IStreamPaginatorAndPageFetcher // IIterator defines an iterator over a collection of items. type IIterator interface { diff --git a/utils/collection/parseLists.go b/utils/collection/parseLists.go index 39e434f7f6..eccdfb1bef 100644 --- a/utils/collection/parseLists.go +++ b/utils/collection/parseLists.go @@ -6,6 +6,7 @@ package collection import ( "fmt" + "slices" "strings" "unicode" @@ -75,11 +76,9 @@ func ConvertSliceToMap[T comparable](input []T) (pairs map[T]T, err error) { } pairs = make(map[T]T, numElements/2) - // TODO use slices.Chunk introduced in go 23 when library is upgraded - for i := 0; i < numElements; i += 2 { - pairs[input[i]] = input[i+1] + for pair := range slices.Chunk(input, 2) { + pairs[pair[0]] = pair[1] } - return } diff --git a/utils/config/interfaces.go b/utils/config/interfaces.go index 247621d7f7..bad6443cf8 100644 --- a/utils/config/interfaces.go +++ b/utils/config/interfaces.go @@ -6,7 +6,7 @@ // Package config provides utilities to load configuration from an environment and perform validation at load time. package config -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IServiceConfiguration,Validator +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IServiceConfiguration,Validator // IServiceConfiguration defines a typical service configuration. type IServiceConfiguration interface { diff --git a/utils/config/service_configuration.go b/utils/config/service_configuration.go index e5b0fca46a..bccc849742 100644 --- a/utils/config/service_configuration.go +++ b/utils/config/service_configuration.go @@ -9,8 +9,8 @@ import ( "fmt" "strings" + "github.com/go-viper/mapstructure/v2" "github.com/joho/godotenv" - "github.com/mitchellh/mapstructure" "github.com/spf13/pflag" "github.com/spf13/viper" diff --git a/utils/encryption/interface.go b/utils/encryption/interface.go index 01be2d69a6..feb294af54 100644 --- a/utils/encryption/interface.go +++ b/utils/encryption/interface.go @@ -6,7 +6,7 @@ import ( "fmt" ) -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IKeyPair +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IKeyPair // IKeyPair defines an asymmetric key pair for cryptography. type IKeyPair interface { diff --git a/utils/environment/interfaces.go b/utils/environment/interfaces.go index b4d73fa68d..592b45512a 100644 --- a/utils/environment/interfaces.go +++ b/utils/environment/interfaces.go @@ -9,7 +9,7 @@ import ( "github.com/ARM-software/golang-utils/utils/filesystem" ) -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IEnvironmentVariable,IEnvironment +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IEnvironmentVariable,IEnvironment // IEnvironmentVariable defines an environment variable to be set for the commands to run. type IEnvironmentVariable interface { diff --git a/utils/filesystem/interfaces.go b/utils/filesystem/interfaces.go index 9804c11747..3ec95bc66c 100644 --- a/utils/filesystem/interfaces.go +++ b/utils/filesystem/interfaces.go @@ -15,7 +15,7 @@ import ( "github.com/ARM-software/golang-utils/utils/config" ) -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IFileHash,IChowner,ILinker,File,DiskUsage,FileTimeInfo,ILock,ILimits,FS,ICloseableFS,IForceRemover,IStater,ILinkReader,ISymLinker +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IFileHash,IChowner,ILinker,File,DiskUsage,FileTimeInfo,ILock,ILimits,FS,ICloseableFS,IForceRemover,IStater,ILinkReader,ISymLinker // IFileHash defines a file hash. // For reference. diff --git a/utils/go.mod b/utils/go.mod index 80b7131bf2..13d658cb79 100644 --- a/utils/go.mod +++ b/utils/go.mod @@ -16,10 +16,11 @@ require ( github.com/go-faker/faker/v4 v4.6.1 github.com/go-git/go-git/v5 v5.16.0 github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a - github.com/go-logr/logr v1.4.2 + github.com/go-logr/logr v1.4.3 github.com/go-logr/stdr v1.2.2 github.com/go-logr/zapr v1.3.0 github.com/go-ozzo/ozzo-validation/v4 v4.3.0 + github.com/go-viper/mapstructure/v2 v2.2.1 github.com/gofrs/uuid/v5 v5.3.2 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f github.com/hashicorp/go-cleanhttp v0.5.2 @@ -28,7 +29,6 @@ require ( github.com/iamacarpet/go-win64api v0.0.0-20230324134531-ef6dbdd6db97 github.com/joho/godotenv v1.5.1 github.com/mitchellh/go-homedir v1.1.0 - github.com/mitchellh/mapstructure v1.5.0 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 github.com/rs/zerolog v1.34.0 github.com/sasha-s/go-deadlock v0.3.5 @@ -45,7 +45,7 @@ require ( go.uber.org/mock v0.5.2 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.38.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 golang.org/x/mod v0.24.0 golang.org/x/net v0.40.0 golang.org/x/oauth2 v0.30.0 @@ -70,7 +70,6 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/cabbie v1.0.2 // indirect github.com/google/glazier v0.0.0-20211029225403-9f766cca891d // indirect @@ -97,9 +96,12 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/tools v0.30.0 // indirect + golang.org/x/tools v0.33.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) -tool github.com/dmarkham/enumer +tool ( + github.com/dmarkham/enumer + go.uber.org/mock/mockgen +) diff --git a/utils/go.sum b/utils/go.sum index e7f515c58d..351feb7e9a 100644 --- a/utils/go.sum +++ b/utils/go.sum @@ -73,8 +73,8 @@ github.com/go-git/go-git/v5 v5.16.0/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lo github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -159,8 +159,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -254,8 +252,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -313,8 +311,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= -golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/utils/hashing/interfaces.go b/utils/hashing/interfaces.go index 5549b88367..3f70ff3b02 100644 --- a/utils/hashing/interfaces.go +++ b/utils/hashing/interfaces.go @@ -11,7 +11,7 @@ import ( "io" ) -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IHash +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IHash // IHash defines a hashing algorithm. type IHash interface { diff --git a/utils/http/interfaces.go b/utils/http/interfaces.go index a1c485d190..bb3699649f 100644 --- a/utils/http/interfaces.go +++ b/utils/http/interfaces.go @@ -26,7 +26,7 @@ import ( "github.com/hashicorp/go-retryablehttp" ) -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IClient,IRetryWaitPolicy +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IClient,IRetryWaitPolicy // IClient defines an HTTP client similar to http.Client but without shared state with other clients used in the same program. // See https://github.com/hashicorp/go-cleanhttp for more details. diff --git a/utils/logs/interfaces.go b/utils/logs/interfaces.go index 6fc391ccce..9ad56e5c70 100644 --- a/utils/logs/interfaces.go +++ b/utils/logs/interfaces.go @@ -12,7 +12,7 @@ import ( "github.com/go-logr/logr" ) -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE Loggers,IMultipleLoggers,WriterWithSource,StdLogger +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE Loggers,IMultipleLoggers,WriterWithSource,StdLogger // Loggers defines generic loggers which separate common logging messages from errors. // This is to use in cases where it is necessary to separate the two streams e.g. remote procedure call (RPC) diff --git a/utils/maps/ReadME.md b/utils/maps/ReadME.md new file mode 100644 index 0000000000..2fd911e5b5 --- /dev/null +++ b/utils/maps/ReadME.md @@ -0,0 +1,4 @@ +This module was initially created to vendor https://pkg.go.dev/github.com/hashicorp/terraform/flatmap which has been +removed from the terraform project. + +code has been updated and is also inspired from https://github.com/astaxie/flatmap \ No newline at end of file diff --git a/utils/maps/expand.go b/utils/maps/expand.go new file mode 100644 index 0000000000..998e1966f7 --- /dev/null +++ b/utils/maps/expand.go @@ -0,0 +1,143 @@ +package maps + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +func Expand(value map[string]string) (expandedMap any, err error) { + if len(value) == 0 { + return + } + m := make(map[string]any, len(value)) + for k := range value { + key, _, found := strings.Cut(k, separator) + if found { + subMap, subErr := ExpandPrefixed(value, key) + if subErr != nil { + err = subErr + return + } + m[key] = subMap + } else { + m[k] = value[k] + } + } + expandedMap = m + return +} + +// ExpandPrefixed takes a maps and a prefix and expands that value into +// a more complex structure. This is the reverse of the Flatten operation. +func ExpandPrefixed(m map[string]string, key string) (result any, err error) { + // If the key is exactly a key in the maps, just return it + if v, ok := m[key]; ok { + if v == "true" { + result = true + return + } else if v == "false" { + result = false + return + } + + result = v + return + } + + // Check if the key is an array, and if so, expand the array + arrayKey := fmt.Sprintf("%v%v%d", key, separator, 0) + if _, ok := m[arrayKey]; ok { + result, err = expandArray(m, key) + return + } + arrayKey = fmt.Sprintf("%v%v", arrayKey, separator) + for k := range m { + if strings.HasPrefix(k, arrayKey) { + result, err = expandArray(m, key) + return + } + } + + // Check if this is a prefix in the maps + prefix := key + separator + for k := range m { + if strings.HasPrefix(k, prefix) { + result, err = expandMap(m, prefix) + return + } + } + + result = nil + return +} + +func expandArray(m map[string]string, prefix string) (result []any, err error) { + keySet := map[int]bool{} + for k := range m { + if !strings.HasPrefix(k, prefix+separator) { + continue + } + + key := k[len(prefix)+1:] + idx := strings.Index(key, separator) + if idx != -1 { + key = key[:idx] + } + + k, subErr := strconv.Atoi(key) + if subErr != nil { + err = subErr + return + } + keySet[k] = true + } + + var keysList []int + for key := range keySet { + keysList = append(keysList, key) + } + sort.Ints(keysList) + + r := make([]any, len(keysList)) + for i, key := range keysList { + keyString := strconv.Itoa(key) + item, subErr := ExpandPrefixed(m, fmt.Sprintf("%s.%s", prefix, keyString)) + r[i] = item + if subErr != nil { + err = subErr + return + } + } + result = r + return +} + +func expandMap(m map[string]string, prefix string) (r map[string]any, err error) { + result := make(map[string]any) + for k := range m { + if !strings.HasPrefix(k, prefix) { + continue + } + + key := k[len(prefix):] + idx := strings.Index(key, separator) + if idx != -1 { + key = key[:idx] + } + if _, ok := result[key]; ok { + continue + } + + item, subErr := ExpandPrefixed(m, k[:len(prefix)+len(key)]) + if subErr != nil { + err = subErr + return + } + result[key] = item + } + + r = result + return +} diff --git a/utils/maps/expand_test.go b/utils/maps/expand_test.go new file mode 100644 index 0000000000..d651412387 --- /dev/null +++ b/utils/maps/expand_test.go @@ -0,0 +1,210 @@ +package maps + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExpand(t *testing.T) { + cases := []struct { + Map map[string]string + Key string + Output any + NoKeyOutput any + }{ + { + Map: map[string]string{ + "foo": "bar", + "bar": "baz", + }, + Key: "foo", + Output: "bar", + NoKeyOutput: map[string]any{ + "foo": "bar", + "bar": "baz", + }, + }, + + { + Map: map[string]string{ + "foo.0": "one", + "foo.1": "two", + }, + Key: "foo", + Output: []any{ + "one", + "two", + }, + }, + + { + Map: map[string]string{ + "foo.0": "one", + "foo.1": "two", + "foo.2": "three", + }, + Key: "foo", + Output: []any{ + "one", + "two", + "three", + }, + }, + + { + Map: map[string]string{ + "foo.0": "one", + "foo.1": "two", + "foo.2": "three", + }, + Key: "foo", + Output: []any{ + "one", + "two", + "three", + }, + }, + + { + Map: map[string]string{ + "foo.0.name": "bar", + "foo.0.port": "3000", + "foo.0.enabled": "true", + }, + Key: "foo", + Output: []any{ + map[string]any{ + "name": "bar", + "port": "3000", + "enabled": true, + }, + }, + }, + + { + Map: map[string]string{ + "foo.0.name": "bar", + "foo.0.ports.0": "1", + "foo.0.ports.1": "2", + }, + Key: "foo", + Output: []any{ + map[string]any{ + "name": "bar", + "ports": []any{ + "1", + "2", + }, + }, + }, + }, + { + Map: map[string]string{ + "list_of_map.0.a": "1", + "list_of_map.1.b": "2", + "list_of_map.1.c": "3", + }, + Key: "list_of_map", + Output: []any{ + map[string]any{ + "a": "1", + }, + map[string]any{ + "b": "2", + "c": "3", + }, + }, + NoKeyOutput: map[string]any{ + "list_of_map": []any{ + map[string]any{"a": "1"}, + map[string]any{ + "b": "2", + "c": "3", + }, + }, + }, + }, + + { + Map: map[string]string{ + "map_of_list.list2.0": "c", + "map_of_list.list1.0": "a", + "map_of_list.list1.1": "b", + }, + Key: "map_of_list", + Output: map[string]any{ + "list1": []any{"a", "b"}, + "list2": []any{"c"}, + }, + }, + { + Map: map[string]string{ + "struct.0.name": "hello", + }, + Key: "struct", + Output: []any{ + map[string]any{ + "name": "hello", + }, + }, + }, + { + Map: map[string]string{ + "struct.0.name": "hello", + "struct.0.set.0.key": "value", + }, + Key: "struct", + Output: []any{ + map[string]any{ + "name": "hello", + "set": []any{ + map[string]any{ + "key": "value", + }, + }, + }, + }, + }, + { + Map: map[string]string{ + "struct.0b.name": "hello", + "struct.0b.key": "value", + }, + Key: "struct", + Output: map[string]any{ + "0b": map[string]any{ + "name": "hello", + "key": "value", + }, + }, + }, + } + + t.Run("ExpandPrefixed", func(t *testing.T) { + for i := range cases { + tc := cases[i] + t.Run(tc.Key, func(t *testing.T) { + actual, err := ExpandPrefixed(tc.Map, tc.Key) + require.NoError(t, err) + assert.Equal(t, tc.Output, actual) + }) + } + }) + t.Run("Expand", func(t *testing.T) { + for i := range cases { + tc := cases[i] + t.Run(tc.Key, func(t *testing.T) { + actual, err := Expand(tc.Map) + require.NoError(t, err) + if tc.NoKeyOutput == nil { + assert.Equal(t, map[string]any{tc.Key: tc.Output}, actual) + } else { + assert.Equal(t, tc.NoKeyOutput, actual) + } + + }) + } + }) +} diff --git a/utils/maps/flatten.go b/utils/maps/flatten.go new file mode 100644 index 0000000000..9e6cbeb9b7 --- /dev/null +++ b/utils/maps/flatten.go @@ -0,0 +1,147 @@ +package maps + +import ( + "fmt" + "reflect" + "strconv" + "time" + + "github.com/ARM-software/golang-utils/utils/commonerrors" + "github.com/ARM-software/golang-utils/utils/reflection" + "github.com/ARM-software/golang-utils/utils/safecast" +) + +// Flatten takes a structure and turns into a flat maps[string]string. +// +// Within the "thing" parameter, only primitive values are allowed. Structs are +// not supported. Therefore, it can only be slices, maps, primitives, and +// any combination of those together. +// +// See the tests for examples of what inputs are turned into. +func Flatten(thing map[string]any) (result Map, err error) { + result = make(map[string]string) + + for k, raw := range thing { + subErr := flatten(result, k, reflect.ValueOf(raw)) + if subErr != nil { + err = subErr + return + } + } + + return +} + +func flatten(result map[string]string, prefix string, v reflect.Value) (err error) { + if v.Kind() == reflect.Interface { + v = v.Elem() + } + switch v.Kind() { + case reflect.Bool: + if v.Bool() { + result[prefix] = "true" + } else { + result[prefix] = "false" + } + case reflect.Int64: + switch v.Type() { + case reflect.TypeOf(time.Duration(5)): + result[prefix] = v.Interface().(time.Duration).String() + default: + result[prefix] = strconv.FormatInt(safecast.ToInt64(v.Int()), 10) + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32: + result[prefix] = strconv.FormatInt(safecast.ToInt64(v.Int()), 10) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + result[prefix] = strconv.FormatUint(safecast.ToUint64(v.Uint()), 10) + case reflect.Float64, reflect.Float32: + result[prefix] = strconv.FormatFloat(v.Float(), 'g', -1, 64) + case reflect.Map: + err = flattenMap(result, prefix, v) + if err != nil { + return err + } + case reflect.Slice, reflect.Array: + err = flattenSlice(result, prefix, v) + if err != nil { + return err + } + case reflect.Interface: + case reflect.Struct: + switch v.Type().String() { + case "time.Time": + result[prefix] = v.Interface().(time.Time).Format(time.RFC3339Nano) + return + default: + err = flattenStruct(result, prefix, v) + if err != nil { + return + } + } + case reflect.String: + result[prefix] = v.String() + case reflect.Invalid: + result[prefix] = "" + default: + if v.IsZero() { + result[prefix] = "" + } else { + err = commonerrors.Newf(commonerrors.ErrUnknown, "unknown %v", v) + } + } + return +} + +func flattenMap(result Map, prefix string, v reflect.Value) (err error) { + for _, k := range v.MapKeys() { + if k.Kind() == reflect.Interface { + k = k.Elem() + } + + if k.Kind() != reflect.String { + err = commonerrors.Newf(commonerrors.ErrInvalid, "%s: maps key is not string: %s", prefix, k) + return + + } + + keyString := k.String() + subPrefix := "" + if reflection.IsEmpty(keyString) { + subPrefix = prefix + } else { + subPrefix = fmt.Sprintf("%s%s%s", prefix, separator, k.String()) + } + subErr := flatten(result, subPrefix, v.MapIndex(k)) + if subErr != nil { + err = subErr + return + } + } + return +} + +func flattenSlice(result Map, prefix string, v reflect.Value) (err error) { + prefix = prefix + separator + + for i := 0; i < v.Len(); i++ { + subErr := flatten(result, fmt.Sprintf("%s%d", prefix, i), v.Index(i)) + if subErr != nil { + err = subErr + return + } + } + return +} + +func flattenStruct(result Map, prefix string, v reflect.Value) (err error) { + prefix = prefix + separator + ty := v.Type() + for i := 0; i < ty.NumField(); i++ { + subErr := flatten(result, fmt.Sprintf("%s%s", prefix, ty.Field(i).Name), v.Field(i)) + if subErr != nil { + err = subErr + return + } + } + return +} diff --git a/utils/maps/flatten_test.go b/utils/maps/flatten_test.go new file mode 100644 index 0000000000..64007bd071 --- /dev/null +++ b/utils/maps/flatten_test.go @@ -0,0 +1,220 @@ +package maps + +import ( + "fmt" + "testing" + "time" + + "github.com/go-faker/faker/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var randomNumber = faker.RandomUnixTime() + +func TestFlatten(t *testing.T) { + cases := []struct { + Input map[string]any + Output map[string]string + }{ + { + Input: map[string]any{ + "foo": "bar", + "bar": "baz", + }, + Output: map[string]string{ + "foo": "bar", + "bar": "baz", + }, + }, + + { + Input: map[string]any{ + "foo": []string{ + "one", + "two", + }, + }, + Output: map[string]string{ + "foo.0": "one", + "foo.1": "two", + }, + }, + + { + Input: map[string]any{ + "foo": []map[any]any{ + map[any]any{ + "name": "bar", + "port": 3000, + "enabled": true, + }, + }, + }, + Output: map[string]string{ + "foo.0.name": "bar", + "foo.0.port": "3000", + "foo.0.enabled": "true", + }, + }, + + { + Input: map[string]any{ + "foo": []map[any]any{ + map[any]any{ + "name": "bar", + "ports": []string{ + "1", + "2", + }, + }, + }, + }, + Output: map[string]string{ + "foo.0.name": "bar", + "foo.0.ports.0": "1", + "foo.0.ports.1": "2", + }, + }, + } + + for i := range cases { + test := cases[i] + t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { + result, err := Flatten(test.Input) + require.NoError(t, err) + assert.Equal(t, test.Output, result.AsMap()) + }) + } +} + +func TestFlatten2(t *testing.T) { + now := time.Now().UTC() + cases := []struct { + Input map[string]any + Output map[string]string + }{ + { + Input: map[string]any{ + "foo": "bar", + "bar": "baz", + }, + Output: map[string]string{ + "foo": "bar", + "bar": "baz", + }, + }, + { + Input: map[string]any{ + "foo": []string{ + "one", + "two", + }, + }, + Output: map[string]string{ + "foo.0": "one", + "foo.1": "two", + }, + }, + { + Input: map[string]any{ + "foo": []map[any]any{ + { + "name": "bar", + "port": 3000, + "enabled": true, + }, + }, + }, + Output: map[string]string{ + "foo.0.name": "bar", + "foo.0.port": "3000", + "foo.0.enabled": "true", + }, + }, + { + Input: map[string]any{ + "foo": []map[any]any{ + { + "name": "bar", + "ports": []string{ + "1", + "2", + }, + }, + }, + }, + Output: map[string]string{ + "foo.0.name": "bar", + "foo.0.ports.0": "1", + "foo.0.ports.1": "2", + }, + }, + { + Input: map[string]any{ + "foo": struct { + Name string + Age int + }{ + "astaxie", + 30, + }, + }, + Output: map[string]string{ + "foo.Name": "astaxie", + "foo.Age": "30", + }, + }, + { + Input: map[string]any{ + "foo": struct { + Name string + Age int + Test int64 + }{ + "astaxie", + 30, + randomNumber, + }, + }, + Output: map[string]string{ + "foo.Name": "astaxie", + "foo.Age": "30", + "foo.Test": fmt.Sprintf("%d", randomNumber), + }, + }, + { + Input: map[string]any{ + "foo": struct { + SomeTime time.Time + }{ + now, + }, + }, + Output: map[string]string{ + "foo.SomeTime": now.UTC().Format(time.RFC3339Nano), + }, + }, + { + Input: map[string]any{ + "foo": struct { + SomeDuration time.Duration + }{ + 56 * time.Minute, + }, + }, + Output: map[string]string{ + "foo.SomeDuration": time.Duration(56 * time.Minute).String(), + }, + }, + } + + for i := range cases { + test := cases[i] + t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { + result, err := Flatten(test.Input) + require.NoError(t, err) + assert.Equal(t, test.Output, result.AsMap()) + }) + } +} diff --git a/utils/maps/map.go b/utils/maps/map.go new file mode 100644 index 0000000000..1c023a58d9 --- /dev/null +++ b/utils/maps/map.go @@ -0,0 +1,88 @@ +package maps + +import ( + "strings" +) + +const separator = "." + +// Map is a wrapper around maps[string]string that provides some helpers +// above it that assume the maps is in the format that flatmap expects +// (the result of Flatten). +// +// All modifying functions such as Delete are done in-place unless +// otherwise noted. +type Map map[string]string + +// Contains returns true if the maps contains the given key. +func (m Map) Contains(key string) bool { + for _, k := range m.Keys() { + if k == key { + return true + } + } + + return false +} + +func (m Map) AsMap() map[string]string { + return m +} + +// Delete deletes a key out of the maps with the given prefix. +func (m Map) Delete(prefix string) { + for k, _ := range m { + match := k == prefix + if !match { + if !strings.HasPrefix(k, prefix) { + continue + } + + if k[len(prefix):len(prefix)+1] != separator { + continue + } + } + + delete(m, k) + } +} + +// Keys returns all the top-level keys in this maps +func (m Map) Keys() []string { + ks := make(map[string]struct{}) + for k, _ := range m { + idx := strings.Index(k, separator) + if idx == -1 { + idx = len(k) + } + + ks[k[:idx]] = struct{}{} + } + + result := make([]string, 0, len(ks)) + for k, _ := range ks { + result = append(result, k) + } + + return result +} + +// Merge merges the contents of the other Map into this one. +// +// This merge is smarter than a simple maps iteration because it +// will fully replace arrays and other complex structures that +// are present in this maps with the other maps's. For example, if +// this maps has a 3 element "foo" list, and m2 has a 2 element "foo" +// list, then the result will be that m has a 2 element "foo" +// list. +func (m Map) Merge(m2 Map) { + for _, prefix := range m2.Keys() { + m.Delete(prefix) + + for k, v := range m2 { + if strings.HasPrefix(k, prefix) { + m[k] = v + } + } + } +} diff --git a/utils/maps/map_test.go b/utils/maps/map_test.go new file mode 100644 index 0000000000..788b775422 --- /dev/null +++ b/utils/maps/map_test.go @@ -0,0 +1,114 @@ +package maps + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMapContains(t *testing.T) { + cases := []struct { + Input map[string]string + Key string + Result bool + }{ + { + Input: map[string]string{ + "foo": "bar", + "bar": "nope", + }, + Key: "foo", + Result: true, + }, + + { + Input: map[string]string{ + "foo": "bar", + "bar": "nope", + }, + Key: "baz", + Result: false, + }, + } + + for _, tc := range cases { + actual := Map(tc.Input).Contains(tc.Key) + assert.Equal(t, tc.Result, actual) + } +} + +func TestMapDelete(t *testing.T) { + m, err := Flatten(map[string]any{ + "foo": "bar", + "routes": []map[string]string{ + { + "foo": "bar", + }, + }, + }) + require.NoError(t, err) + + m.Delete("routes") + + expected := Map(map[string]string{"foo": "bar"}) + assert.Equal(t, expected, m) +} + +func TestMapKeys(t *testing.T) { + cases := []struct { + Input map[string]string + Output []string + }{ + { + Input: map[string]string{ + "foo": "bar", + "bar.#": "bar", + "bar.0.foo": "bar", + "bar.0.baz": "bar", + }, + Output: []string{ + "bar", + "foo", + }, + }, + } + + for _, tc := range cases { + actual := Map(tc.Input).Keys() + + // Sort so we have a consistent view of the output + sort.Strings(actual) + assert.Equal(t, tc.Output, actual) + } +} + +func TestMapMerge(t *testing.T) { + cases := []struct { + One map[string]string + Two map[string]string + Result map[string]string + }{ + { + One: map[string]string{ + "foo": "bar", + "bar": "nope", + }, + Two: map[string]string{ + "bar": "baz", + "baz": "buz", + }, + Result: map[string]string{ + "foo": "bar", + "bar": "baz", + "baz": "buz", + }, + }, + } + + for _, tc := range cases { + Map(tc.One).Merge(tc.Two) + assert.Equal(t, tc.One, tc.Result) + } +} diff --git a/utils/maps/overall_test.go b/utils/maps/overall_test.go new file mode 100644 index 0000000000..4c0592d6b0 --- /dev/null +++ b/utils/maps/overall_test.go @@ -0,0 +1,50 @@ +package maps + +import ( + "strconv" + "testing" + "time" + + "github.com/go-faker/faker/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMapFlattenExpand(t *testing.T) { + for i := 0; i < 10; i++ { + t.Run(strconv.Itoa(i), func(t *testing.T) { + testCase := map[string]any{ + "test1": map[string]any{ + "test1.1": faker.DomainName(), + "test2": map[string]any{ + faker.Name(): faker.Paragraph(), + "test3": map[string]any{ + faker.Word(): faker.Phonenumber(), + "test3.1": 5.54, + "some time": time.Now().UTC(), + "some float": 45454.454545812, + faker.UUIDDigit(): faker.RandomUnixTime(), + faker.Password(): time.Now().UTC(), + "test4": map[string]any{ + "test5": map[string]time.Duration{ + faker.DomainName(): 5 * time.Hour, + }, + }, + }, + }, + }, + } + + flattened, err := Flatten(testCase) + require.NoError(t, err) + expanded, err := Expand(flattened) + require.NoError(t, err) + flattened, err = Flatten(testCase) + require.NoError(t, err) + expanded2, err := Expand(flattened) + require.NoError(t, err) + assert.NotEmpty(t, expanded) + assert.Equal(t, expanded, expanded2) + }) + } +} diff --git a/utils/proc/interfaces.go b/utils/proc/interfaces.go index 9306c2a5f6..b8eb14dd4e 100644 --- a/utils/proc/interfaces.go +++ b/utils/proc/interfaces.go @@ -6,7 +6,7 @@ package proc import "context" -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IProcess +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IProcess // IProcess is the generic interface that is implemented on every platform // and provides common operations for processes. diff --git a/utils/resource/interfaces.go b/utils/resource/interfaces.go index 64462400a5..dae00a8249 100644 --- a/utils/resource/interfaces.go +++ b/utils/resource/interfaces.go @@ -11,7 +11,7 @@ import ( "io" ) -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE ICloseableResource +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE ICloseableResource // ICloseableResource defines a resource which must be closed after use e.g. an open file. type ICloseableResource interface { diff --git a/utils/serialization/maps/map.go b/utils/serialization/maps/map.go new file mode 100644 index 0000000000..c506a61f1b --- /dev/null +++ b/utils/serialization/maps/map.go @@ -0,0 +1,141 @@ +package maps + +import ( + "reflect" + "time" + + "github.com/go-viper/mapstructure/v2" + + "github.com/ARM-software/golang-utils/utils/commonerrors" + "github.com/ARM-software/golang-utils/utils/maps" +) + +// ToMap converts a struct to a flat map using (mapstructure)[https://github.com/go-viper/mapstructure] +func ToMap[T any](o *T) (m map[string]string, err error) { + if o == nil { + err = commonerrors.UndefinedVariable("object") + return + } + mapAny := map[string]any{} + err = mapstructureDecoder(o, &mapAny) + if err != nil { + err = commonerrors.WrapError(commonerrors.ErrMarshalling, err, "failed to serialise object") + return + } + m, err = maps.Flatten(mapAny) + if err != nil { + err = commonerrors.WrapError(commonerrors.ErrMarshalling, err, "failed to flatten map") + } + return +} + +// FromMap deserialises a flatten map into a struct using (mapstructure)[https://github.com/go-viper/mapstructure] +func FromMap[T any](m map[string]string, o *T) (err error) { + if o == nil { + err = commonerrors.UndefinedVariable("object") + return + } + if len(m) == 0 { + return + } + expandedMap, err := maps.Expand(m) + if err != nil { + err = commonerrors.WrapError(commonerrors.ErrMarshalling, err, "failed to expand the map") + return + } + + err = mapstructureDecoder(expandedMap, o) + if err != nil { + err = commonerrors.WrapError(commonerrors.ErrMarshalling, err, "failed to deserialise upload request") + } + return +} + +func timeHookFunc() mapstructure.DecodeHookFunc { + return func(f reflect.Type, t reflect.Type, data any) (any, error) { + switch { + case t == reflect.TypeOf(time.Time{}): + return toTime(f, t, data) + case f == reflect.TypeOf(time.Time{}) || f == reflect.TypeOf(&time.Time{}): + return fromTime(f, t, data) + default: + return data, nil + } + } +} + +func fromTime(f, t reflect.Type, data any) (any, error) { + switch f { + case reflect.TypeOf(time.Time{}): + subtime := data.(time.Time) + value := subtime.Format(time.RFC3339Nano) + return convertTo(value, data, t) + case reflect.TypeOf(&time.Time{}): + subtime := data.(*time.Time) + if subtime == nil { + return nil, nil + } + value := subtime.Format(time.RFC3339Nano) + return convertTo(value, data, t) + default: + return data, nil + } +} + +func convertTo(value string, rawValue any, t reflect.Type) (any, error) { + switch t.Kind() { + case reflect.String: + return value, nil + case reflect.Map: + return map[string]any{"": value}, nil + case reflect.Slice: + return []string{value}, nil + default: + return rawValue, nil + } +} + +func toTime(f reflect.Type, t reflect.Type, data any) (any, error) { + if t != reflect.TypeOf(time.Time{}) { + return data, nil + } + + switch f.Kind() { + case reflect.String: + return time.Parse(time.RFC3339Nano, data.(string)) + case reflect.Float64: + return time.Unix(0, int64(data.(float64))*int64(time.Millisecond)), nil + case reflect.Int64: + return time.Unix(0, data.(int64)*int64(time.Millisecond)), nil + default: + return data, nil + } +} + +func fromTimeDurationHookFunc() mapstructure.DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data any, + ) (any, error) { + switch f { + case reflect.TypeOf(time.Duration(5)): + return convertTo(f.String(), data, t) + default: + return data, nil + } + } +} + +func mapstructureDecoder(input, result any) error { + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + WeaklyTypedInput: true, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + timeHookFunc(), mapstructure.StringToTimeDurationHookFunc(), mapstructure.StringToURLHookFunc(), mapstructure.StringToIPHookFunc()), + Result: result, + }) + if err != nil { + return err + } + return decoder.Decode(input) +} diff --git a/utils/serialization/maps/map_test.go b/utils/serialization/maps/map_test.go new file mode 100644 index 0000000000..bc49cb98ec --- /dev/null +++ b/utils/serialization/maps/map_test.go @@ -0,0 +1,138 @@ +package maps + +import ( + "testing" + "time" + + "github.com/go-faker/faker/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type TestStruct0 struct { + Number int + BigNumber int64 + Float64 float64 + Uint uint + LongString string + OtherString string + Domain string `faker:"domain_name"` + Array []string + Bool bool + Latitude float32 `faker:"lat"` + Longitude float32 `faker:"long"` + RealAddress faker.RealAddress `faker:"real_address"` + CreditCardNumber string `faker:"cc_number"` + CreditCardType string `faker:"cc_type"` + Email string `faker:"email"` + DomainName string `faker:"domain_name"` + IPV4 string `faker:"ipv4"` + IPV6 string `faker:"ipv6"` + Password string `faker:"password"` + Jwt string `faker:"jwt"` + PhoneNumber string `faker:"phone_number"` + MacAddress string `faker:"mac_address"` + URL string `faker:"url"` + UserName string `faker:"username"` + TollFreeNumber string `faker:"toll_free_number"` + E164PhoneNumber string `faker:"e_164_phone_number"` + TitleMale string `faker:"title_male"` + TitleFemale string `faker:"title_female"` + FirstName string `faker:"first_name"` + FirstNameMale string `faker:"first_name_male"` + FirstNameFemale string `faker:"first_name_female"` + LastName string `faker:"last_name"` + Name string `faker:"name"` + UnixTime int64 `faker:"unix_time"` + Date string `faker:"date"` + Time string `faker:"time"` + MonthName string `faker:"month_name"` + Year string `faker:"year"` + DayOfWeek string `faker:"day_of_week"` + DayOfMonth string `faker:"day_of_month"` + Timestamp string `faker:"timestamp"` + Century string `faker:"century"` + TimeZone string `faker:"timezone"` + TimePeriod string `faker:"time_period"` + Word string `faker:"word"` + Sentence string `faker:"sentence"` + Paragraph string `faker:"paragraph"` + Currency string `faker:"currency"` + Amount float64 `faker:"amount"` + AmountWithCurrency string `faker:"amount_with_currency"` + UUIDHypenated string `faker:"uuid_hyphenated"` + UUID string `faker:"uuid_digit"` + PaymentMethod string `faker:"oneof: cc, paypal, check, money order"` // oneof will randomly pick one of the comma-separated values supplied in the tag + AccountID int `faker:"oneof: 15, 27, 61"` // use commas to separate the values for now. Future support for other separator characters may be added + Price32 float32 `faker:"oneof: 4.95, 9.99, 31997.97"` + Price64 float64 `faker:"oneof: 47463.9463525, 993747.95662529, 11131997.978767990"` + NumS64 int64 `faker:"oneof: 1, 2"` + NumS32 int32 `faker:"oneof: -3, 4"` + NumS16 int16 `faker:"oneof: -5, 6"` + NumS8 int8 `faker:"oneof: 7, -8"` + NumU64 uint64 `faker:"oneof: 9, 10"` + NumU32 uint32 `faker:"oneof: 11, 12"` + NumU16 uint16 `faker:"oneof: 13, 14"` + NumU8 uint8 `faker:"oneof: 15, 16"` + NumU uint `faker:"oneof: 17, 18"` +} + +type TestStruct1 struct { + Name string + Number int + BigNumber int64 + Float64 float64 + Duration int `mapstructure:"time_duration"` + Uint uint + LongString string + OtherString string + Domain string `faker:"domain_name"` + UUID string + Array []int + Bool bool + Struct TestStruct0 +} + +type TestStruct2WithTime struct { + Time time.Time `mapstructure:"some_time"` + Duration time.Duration `mapstructure:"some_duration"` +} +type TestStruct3WithTime struct { + Time time.Time `mapstructure:"some_time"` + Duration time.Duration `mapstructure:"some_duration"` + Struct TestStruct2WithTime +} + +func TestToMap(t *testing.T) { + t.Run("generic", func(t *testing.T) { + testStruct := TestStruct1{} + require.NoError(t, faker.FakeData(&testStruct)) + + structMap, err := ToMap[TestStruct1](&testStruct) + require.NoError(t, err) + newStruct := TestStruct1{} + require.NoError(t, FromMap[TestStruct1](structMap, &newStruct)) + assert.Equal(t, testStruct, newStruct) + }) + t.Run("with time", func(t *testing.T) { + random, err := faker.RandomInt(0, 1000, 2) + require.NoError(t, err) + testStruct := TestStruct3WithTime{ + Time: time.Now().UTC(), + Duration: time.Duration(random[0]) * time.Minute, + Struct: TestStruct2WithTime{ + Time: time.Unix(faker.RandomUnixTime(), 0), + Duration: time.Duration(random[1]) * time.Second, + }, + } + structMap, err := ToMap[TestStruct3WithTime](&testStruct) + require.NoError(t, err) + newStruct := TestStruct3WithTime{} + require.NoError(t, FromMap[TestStruct3WithTime](structMap, &newStruct)) + assert.WithinDuration(t, testStruct.Time, newStruct.Time, 0) + assert.Equal(t, testStruct.Duration, newStruct.Duration) + assert.WithinDuration(t, testStruct.Struct.Time, newStruct.Struct.Time, 0) + assert.Equal(t, testStruct.Struct.Duration, newStruct.Struct.Duration) + }) + +} diff --git a/utils/serialization/slice/slice.go b/utils/serialization/slice/slice.go new file mode 100644 index 0000000000..d7c519ca94 --- /dev/null +++ b/utils/serialization/slice/slice.go @@ -0,0 +1,33 @@ +package slice + +import ( + "github.com/ARM-software/golang-utils/utils/collection" + "github.com/ARM-software/golang-utils/utils/commonerrors" + "github.com/ARM-software/golang-utils/utils/serialization/maps" +) + +// ToSlice converts a struct to a list of key values. +func ToSlice[T any](s *T) (list []string, err error) { + r, err := maps.ToMap(s) + if err != nil { + return + } + list = collection.ConvertMapToSlice(r) + return +} + +// FromSlice converts a slice of key,values into a struct i +func FromSlice[T any](s []string, o *T) (err error) { + m, err := collection.ConvertSliceToMap(s) + if err != nil { + err = commonerrors.WrapErrorf(commonerrors.ErrMarshalling, err, "could not convert slice to map so it can be converted into `%T`", o) + return + } + err = maps.FromMap[T](m, o) + return +} + +// FromArgs converts a list of args into a struct o +func FromArgs[T any](o *T, args ...string) error { + return FromSlice(args, o) +} diff --git a/utils/serialization/slice/slice_test.go b/utils/serialization/slice/slice_test.go new file mode 100644 index 0000000000..8611ec9e93 --- /dev/null +++ b/utils/serialization/slice/slice_test.go @@ -0,0 +1,157 @@ +package slice + +import ( + "testing" + "time" + + "github.com/go-faker/faker/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type TestStruct0 struct { + Number int + BigNumber int64 + Float64 float64 + Uint uint + LongString string + OtherString string + Domain string `faker:"domain_name"` + Array []string + Bool bool + Latitude float32 `faker:"lat"` + Longitude float32 `faker:"long"` + RealAddress faker.RealAddress `faker:"real_address"` + CreditCardNumber string `faker:"cc_number"` + CreditCardType string `faker:"cc_type"` + Email string `faker:"email"` + DomainName string `faker:"domain_name"` + IPV4 string `faker:"ipv4"` + IPV6 string `faker:"ipv6"` + Password string `faker:"password"` + Jwt string `faker:"jwt"` + PhoneNumber string `faker:"phone_number"` + MacAddress string `faker:"mac_address"` + URL string `faker:"url"` + UserName string `faker:"username"` + TollFreeNumber string `faker:"toll_free_number"` + E164PhoneNumber string `faker:"e_164_phone_number"` + TitleMale string `faker:"title_male"` + TitleFemale string `faker:"title_female"` + FirstName string `faker:"first_name"` + FirstNameMale string `faker:"first_name_male"` + FirstNameFemale string `faker:"first_name_female"` + LastName string `faker:"last_name"` + Name string `faker:"name"` + UnixTime int64 `faker:"unix_time"` + Date string `faker:"date"` + Time string `faker:"time"` + MonthName string `faker:"month_name"` + Year string `faker:"year"` + DayOfWeek string `faker:"day_of_week"` + DayOfMonth string `faker:"day_of_month"` + Timestamp string `faker:"timestamp"` + Century string `faker:"century"` + TimeZone string `faker:"timezone"` + TimePeriod string `faker:"time_period"` + Word string `faker:"word"` + Sentence string `faker:"sentence"` + Paragraph string `faker:"paragraph"` + Currency string `faker:"currency"` + Amount float64 `faker:"amount"` + AmountWithCurrency string `faker:"amount_with_currency"` + UUIDHypenated string `faker:"uuid_hyphenated"` + UUID string `faker:"uuid_digit"` + PaymentMethod string `faker:"oneof: cc, paypal, check, money order"` // oneof will randomly pick one of the comma-separated values supplied in the tag + AccountID int `faker:"oneof: 15, 27, 61"` // use commas to separate the values for now. Future support for other separator characters may be added + Price32 float32 `faker:"oneof: 4.95, 9.99, 31997.97"` + Price64 float64 `faker:"oneof: 47463.9463525, 993747.95662529, 11131997.978767990"` + NumS64 int64 `faker:"oneof: 1, 2"` + NumS32 int32 `faker:"oneof: -3, 4"` + NumS16 int16 `faker:"oneof: -5, 6"` + NumS8 int8 `faker:"oneof: 7, -8"` + NumU64 uint64 `faker:"oneof: 9, 10"` + NumU32 uint32 `faker:"oneof: 11, 12"` + NumU16 uint16 `faker:"oneof: 13, 14"` + NumU8 uint8 `faker:"oneof: 15, 16"` + NumU uint `faker:"oneof: 17, 18"` +} + +type TestStruct1 struct { + Name string + Number int + BigNumber int64 + Float64 float64 + Duration int `mapstructure:"time_duration"` + Uint uint + LongString string + OtherString string + Domain string `faker:"domain_name"` + UUID string + Array []int + Bool bool + Struct TestStruct0 +} + +type TestStruct2WithTime struct { + Time time.Time `mapstructure:"some_time"` + Duration time.Duration `mapstructure:"some_duration"` +} +type TestStruct3WithTime struct { + Time time.Time `mapstructure:"some_time"` + Duration time.Duration `mapstructure:"some_duration"` + Struct TestStruct2WithTime +} + +type TestStruct4 struct { + Duration time.Duration `mapstructure:"some_duration"` + Test string + Struct TestStruct2WithTime +} + +func TestToSlice(t *testing.T) { + t.Run("generic", func(t *testing.T) { + testStruct := TestStruct1{} + require.NoError(t, faker.FakeData(&testStruct)) + + structSlice, err := ToSlice[TestStruct1](&testStruct) + require.NoError(t, err) + newStruct := TestStruct1{} + require.NoError(t, FromSlice[TestStruct1](structSlice, &newStruct)) + assert.Equal(t, structSlice, newStruct) + }) + t.Run("with time", func(t *testing.T) { + random, err := faker.RandomInt(0, 1000, 2) + require.NoError(t, err) + testStruct := TestStruct3WithTime{ + Time: time.Now().UTC(), + Duration: time.Duration(random[0]) * time.Minute, + Struct: TestStruct2WithTime{ + Time: time.Unix(faker.RandomUnixTime(), 0), + Duration: time.Duration(random[1]) * time.Second, + }, + } + structSlice, err := ToSlice[TestStruct3WithTime](&testStruct) + require.NoError(t, err) + newStruct := TestStruct3WithTime{} + require.NoError(t, FromSlice[TestStruct3WithTime](structSlice, &newStruct)) + assert.WithinDuration(t, testStruct.Time, newStruct.Time, 0) + assert.Equal(t, testStruct.Duration, newStruct.Duration) + assert.WithinDuration(t, testStruct.Struct.Time, newStruct.Struct.Time, 0) + assert.Equal(t, testStruct.Struct.Duration, newStruct.Struct.Duration) + }) + + t.Run("args", func(t *testing.T) { + testStruct := TestStruct4{} + someText := faker.Paragraph() + now := time.Now().UTC() + require.NoError(t, FromArgs[TestStruct4](&testStruct, "some_duration", "15m", "test", someText, "struct.some_duration", "5h10m50s", "struct.some_time", now.Format(time.RFC3339Nano))) + duration, err := time.ParseDuration("5h10m50s") + require.NoError(t, err) + assert.Equal(t, duration, testStruct.Struct.Duration) + assert.Equal(t, 15*time.Minute, testStruct.Duration) + assert.Equal(t, someText, testStruct.Test) + assert.WithinDuration(t, now, testStruct.Struct.Time, 0) + }) + +} diff --git a/utils/sharedcache/interface.go b/utils/sharedcache/interface.go index 9d18b0b27b..d82b2834e2 100644 --- a/utils/sharedcache/interface.go +++ b/utils/sharedcache/interface.go @@ -7,7 +7,7 @@ import ( // Mocks are generated using `go generate ./...` // Add interfaces to the following command for a mock to be generated -//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE ISharedCacheRepository +//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE ISharedCacheRepository // ISharedCacheRepository defines a cache stored on a remote location and shared by separate processes. type ISharedCacheRepository interface { diff --git a/utils/subprocess/supervisor/interface.go b/utils/subprocess/supervisor/interface.go index 48a4d24fb6..f648cc4a63 100644 --- a/utils/subprocess/supervisor/interface.go +++ b/utils/subprocess/supervisor/interface.go @@ -2,7 +2,7 @@ package supervisor import "context" -//go:generate mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/subprocess/$GOPACKAGE ISupervisor +//go:generate go tool mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/subprocess/$GOPACKAGE ISupervisor // ISupervisor will run a command and automatically restart it if it exits. Hooks can be used to execute code at // different points in the execution lifecyle. Restarts can be delayed From 1a3df47f8a5d5a8f8629907195d82c308c670fa7 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 30 May 2025 12:48:19 +0100 Subject: [PATCH 2/3] :green_heart: linting --- utils/maps/flatten.go | 4 ++-- utils/maps/flatten_test.go | 2 +- utils/maps/map.go | 6 +++--- utils/serialization/maps/map.go | 15 --------------- utils/serialization/slice/slice.go | 2 +- 5 files changed, 7 insertions(+), 22 deletions(-) diff --git a/utils/maps/flatten.go b/utils/maps/flatten.go index 9e6cbeb9b7..5802f733ae 100644 --- a/utils/maps/flatten.go +++ b/utils/maps/flatten.go @@ -121,7 +121,7 @@ func flattenMap(result Map, prefix string, v reflect.Value) (err error) { } func flattenSlice(result Map, prefix string, v reflect.Value) (err error) { - prefix = prefix + separator + prefix += separator for i := 0; i < v.Len(); i++ { subErr := flatten(result, fmt.Sprintf("%s%d", prefix, i), v.Index(i)) @@ -134,7 +134,7 @@ func flattenSlice(result Map, prefix string, v reflect.Value) (err error) { } func flattenStruct(result Map, prefix string, v reflect.Value) (err error) { - prefix = prefix + separator + prefix += separator ty := v.Type() for i := 0; i < ty.NumField(); i++ { subErr := flatten(result, fmt.Sprintf("%s%s", prefix, ty.Field(i).Name), v.Field(i)) diff --git a/utils/maps/flatten_test.go b/utils/maps/flatten_test.go index 64007bd071..ba886ae8f9 100644 --- a/utils/maps/flatten_test.go +++ b/utils/maps/flatten_test.go @@ -204,7 +204,7 @@ func TestFlatten2(t *testing.T) { }, }, Output: map[string]string{ - "foo.SomeDuration": time.Duration(56 * time.Minute).String(), + "foo.SomeDuration": (56 * time.Minute).String(), }, }, } diff --git a/utils/maps/map.go b/utils/maps/map.go index 1c023a58d9..973001dc6f 100644 --- a/utils/maps/map.go +++ b/utils/maps/map.go @@ -31,7 +31,7 @@ func (m Map) AsMap() map[string]string { // Delete deletes a key out of the maps with the given prefix. func (m Map) Delete(prefix string) { - for k, _ := range m { + for k := range m { match := k == prefix if !match { if !strings.HasPrefix(k, prefix) { @@ -50,7 +50,7 @@ func (m Map) Delete(prefix string) { // Keys returns all the top-level keys in this maps func (m Map) Keys() []string { ks := make(map[string]struct{}) - for k, _ := range m { + for k := range m { idx := strings.Index(k, separator) if idx == -1 { idx = len(k) @@ -60,7 +60,7 @@ func (m Map) Keys() []string { } result := make([]string, 0, len(ks)) - for k, _ := range ks { + for k := range ks { result = append(result, k) } diff --git a/utils/serialization/maps/map.go b/utils/serialization/maps/map.go index c506a61f1b..ab0e38ebc2 100644 --- a/utils/serialization/maps/map.go +++ b/utils/serialization/maps/map.go @@ -112,21 +112,6 @@ func toTime(f reflect.Type, t reflect.Type, data any) (any, error) { } } -func fromTimeDurationHookFunc() mapstructure.DecodeHookFunc { - return func( - f reflect.Type, - t reflect.Type, - data any, - ) (any, error) { - switch f { - case reflect.TypeOf(time.Duration(5)): - return convertTo(f.String(), data, t) - default: - return data, nil - } - } -} - func mapstructureDecoder(input, result any) error { decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ WeaklyTypedInput: true, diff --git a/utils/serialization/slice/slice.go b/utils/serialization/slice/slice.go index d7c519ca94..8a9c1365d1 100644 --- a/utils/serialization/slice/slice.go +++ b/utils/serialization/slice/slice.go @@ -3,7 +3,7 @@ package slice import ( "github.com/ARM-software/golang-utils/utils/collection" "github.com/ARM-software/golang-utils/utils/commonerrors" - "github.com/ARM-software/golang-utils/utils/serialization/maps" + "github.com/ARM-software/golang-utils/utils/serialization/maps" //nolint:misspell ) // ToSlice converts a struct to a list of key values. From 83d340bb95aba46227cca131565e1f8fbead6dde Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Fri, 30 May 2025 15:03:30 +0100 Subject: [PATCH 3/3] :green_heart: Fix test --- utils/serialization/slice/slice_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/serialization/slice/slice_test.go b/utils/serialization/slice/slice_test.go index 8611ec9e93..616a3e468f 100644 --- a/utils/serialization/slice/slice_test.go +++ b/utils/serialization/slice/slice_test.go @@ -118,7 +118,7 @@ func TestToSlice(t *testing.T) { require.NoError(t, err) newStruct := TestStruct1{} require.NoError(t, FromSlice[TestStruct1](structSlice, &newStruct)) - assert.Equal(t, structSlice, newStruct) + assert.Equal(t, testStruct, newStruct) }) t.Run("with time", func(t *testing.T) { random, err := faker.RandomInt(0, 1000, 2)