feature: implement onboard device to hub (#101)

* implement onboard and change pb.RemoteConfiguration to similar as in hub

---------

Co-authored-by: Patrik Matiaško <patrik.matiasko@gmail.com>

Author: jkralik

.vscode/settings.json

@@ -17,6 +17,8 @@
         "TEST_ROOT_CA_KEY": "${workspaceFolder}/.tmp/certs/rootcakey.pem",
         "TEST_ROOT_CA_CERT": "${workspaceFolder}/.tmp/certs/rootcacrt.pem",
         "TEST_CLOUD_SID": "adebc667-1f2b-41e3-bf5c-6d6eabc68cc6",
+        "TEST_COAP_GW_CERT_FILE": "${workspaceFolder}/.tmp/certs/coap.crt",
+        "TEST_COAP_GW_KEY_FILE": "${workspaceFolder}/.tmp/certs/coap.key",
         // "GOMAXPROCS": 1,
         // "GOFLAGS":"-mod=vendor",
         // "GRPC_VERBOSITY":"DEBUG",

Makefile

@@ -31,19 +31,45 @@ certificates:
 	docker pull $(CERT_TOOL_IMAGE)
 	docker run --rm --user $(USER_ID):$(GROUP_ID) -v $(CERT_PATH):/out $(CERT_TOOL_IMAGE) --outCert=/out/rootcacrt.pem --outKey=/out/rootcakey.pem --cert.subject.cn="ca" --cmd.generateRootCA
 	docker run --rm --user $(USER_ID):$(GROUP_ID) -v $(CERT_PATH):/out $(CERT_TOOL_IMAGE) --signerCert=/out/rootcacrt.pem --signerKey=/out/rootcakey.pem --outCert=/out/httpcrt.pem --outKey=/out/httpkey.pem --cert.san.domain=localhost --cert.san.ip=127.0.0.1 --cert.subject.cn="http-server" --cmd.generateCertificate
+	docker run --rm --user $(USER_ID):$(GROUP_ID) -v $(CERT_PATH):/out $(CERT_TOOL_IMAGE) --signerCert=/out/rootcacrt.pem --signerKey=/out/rootcakey.pem --outCert=/out/coap.crt --outKey=/out/coap.key --cmd.generateIdentityCertificate=$(CLOUD_SID)  --cert.san.domain=localhost
+	cat $(WORKING_DIRECTORY)/.tmp/certs/httpcrt.pem > $(WORKING_DIRECTORY)/.tmp/certs/mongo.key
+	cat $(WORKING_DIRECTORY)/.tmp/certs/httpkey.pem >> $(WORKING_DIRECTORY)/.tmp/certs/mongo.key
 	mkdir -p $(MFG_CERT_PATH)
 	docker run --rm --user $(USER_ID):$(GROUP_ID) -v $(MFG_CERT_PATH):/out $(CERT_TOOL_IMAGE) --outCert=/out/cloudca.pem --outKey=/out/cloudcakey.pem --cert.subject.cn=MfgRootCA --cmd.generateRootCA
 	docker run --rm --user $(USER_ID):$(GROUP_ID) -v $(MFG_CERT_PATH):/out $(CERT_TOOL_IMAGE) --signerCert=/out/cloudca.pem --signerKey=/out/cloudcakey.pem --outCert=/out/mfgcrt.pem --outKey=/out/mfgkey.pem --cert.subject.cn="mfg-device-cert" --cmd.generateCertificate
 	docker run --rm --user $(USER_ID):$(GROUP_ID) -v $(MFG_CERT_PATH):/out $(CERT_TOOL_IMAGE) --signerCert=/out/cloudca.pem --signerKey=/out/cloudcakey.pem --outCert=/out/mfgclientapplicationcrt.pem --outKey=/out/mfgclientapplicationkey.pem --cert.subject.cn="mfg-client-application-cert" --cmd.generateCertificate
 .PHONY: certificates
 
+nats:
+	mkdir -p $(WORKING_DIRECTORY)/.tmp/jetstream/cloud
+	docker run \
+	    -d \
+		--network=host \
+		--name=nats \
+		-v $(WORKING_DIRECTORY)/.tmp/certs:/certs \
+		-v $(WORKING_DIRECTORY)/.tmp/jetstream/cloud:/data \
+		--user $(USER_ID):$(GROUP_ID) \
+		nats --jetstream --store_dir /data --tls --tlsverify --tlscert=/certs/httpcrt.pem --tlskey=/certs/httpkey.pem --tlscacert=/certs/rootcacrt.pem
+.PHONY: nats
+
+mongo: certificates
+	mkdir -p $(WORKING_DIRECTORY)/.tmp/mongo
+	docker run \
+		-d \
+		--network=host \
+		--name=mongo \
+		-v $(WORKING_DIRECTORY)/.tmp/mongo:/data/db \
+		-v $(WORKING_DIRECTORY)/.tmp/certs:/certs --user $(USER_ID):$(GROUP_ID) \
+		mongo --tlsMode requireTLS --tlsCAFile /certs/rootcacrt.pem --tlsCertificateKeyFile /certs/mongo.key
+.PHONY: mongo
+
 privateKeys:
 	mkdir -p $(OAUTH_SERVER_PATH)
 	openssl genrsa -out $(OAUTH_SERVER_ID_TOKEN_PRIVATE_KEY) 4096
 	openssl ecparam -name prime256v1 -genkey -noout -out $(OAUTH_SERVER_ACCESS_TOKEN_PRIVATE_KEY)
 .PHONY: privateKeys
 
-env: clean certificates privateKeys
+env: clean certificates privateKeys nats mongo
 	mkdir -p $(DEVSIM_PATH)
 	docker pull $(DEVSIM_IMAGE)
 	docker run -d \
@@ -56,8 +82,10 @@ env: clean certificates privateKeys
 .PHONY: env
 
 clean:
+	docker rm -f mongo || true
+	docker rm -f nats || true
 	docker rm -f devsim || true
-	rm -rf $(TMP_PATH) || true
+	sudo rm -rf $(TMP_PATH) || true
 .PHONY: clean
 
 
@@ -92,6 +120,8 @@ test: env
 	export TEST_OAUTH_SERVER_ACCESS_TOKEN_PRIVATE_KEY=$(OAUTH_SERVER_ACCESS_TOKEN_PRIVATE_KEY); \
 	export TEST_ROOT_CA_KEY=$(WORKING_DIRECTORY)/.tmp/certs/rootcakey.pem; \
 	export TEST_ROOT_CA_CERT=$(WORKING_DIRECTORY)/.tmp/certs/rootcacrt.pem; \
+	export TEST_COAP_GW_CERT_FILE=$(WORKING_DIRECTORY)/.tmp/certs/coap.crt; \
+	export TEST_COAP_GW_KEY_FILE=$(WORKING_DIRECTORY)/.tmp/certs/coap.key; \
 	export TEST_CLOUD_SID=$(CLOUD_SID); \
 	if [ -n "$${JSON_REPORT}" ]; then \
 		go test -v --race -p 1 -covermode=atomic -coverpkg=./... -coverprofile=$${COVERAGE_FILE} -json ./... > "$${JSON_REPORT_FILE}" ; \
@@ -121,10 +151,13 @@ proto/generate:
 	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/disown_device.proto
 	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/clear_cache.proto
 	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/get_configuration.proto
+	protoc-go-inject-tag -input=$(WORKING_DIRECTORY)/pb/get_configuration.pb.go
 	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/get_identity_certificate.proto
 	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/get_json_web_keys.proto
 	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/initialize.proto
 	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/reset.proto
+	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/onboard_device.proto
+	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/offboard_device.proto
 
 	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) -I=$(GOOGLEAPIS_PATH) -I=$(GRPCGATEWAY_MODULE_PATH) --go-grpc_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/service.proto
 	protoc -I=. -I=$(GOPATH)/src -I=$(PLGDHUB_MODULE_PATH) -I=$(GOOGLEAPIS_PATH) -I=$(GRPCGATEWAY_MODULE_PATH) --openapiv2_out=$(GOPATH)/src \

README.md

@@ -66,7 +66,7 @@ HTTP API of the client application service as defined [swagger](./pb/service.swa
 | `apis.http.ui.enabled` | bool | `Set to true if you would like to run the web UI.` | `false` |
 | `apis.http.ui.directory` | string | `A path to the directory with web UI files. When it is not present, it creates <client_application_binary>/www with default ui.` | `""` |
 | `apis.http.tls.enabled` | bool | `Enable HTTPS.` | `false` |
-| `apis.http.tls.caPool` | string | `File path to the root certificate in PEM format which might contain multiple certificates in a single file.` |  `""` |
+| `apis.http.tls.caPool` | []string | `File path to the root certificate in PEM format which might contain multiple certificates in a single file.` |  `""` |
 | `apis.http.tls.keyFile` | string | `File path to private key in PEM format.` | `""` |
 | `apis.http.tls.certFile` | string | `File path to certificate in PEM format.` | `""` |
 | `apis.http.tls.clientCertificateRequired` | bool | `If true, require client certificate.` | `true` |
@@ -86,7 +86,7 @@ gRPC API of the client application service as defined [service](./pb/service.pro
 | `apis.grpc.keepAlive.maxConnectionAgeGrace` | string | `An additive period after MaxConnectionAge after which the connection will be forcibly closed. 0s means infinity.` | `0s` |
 | `apis.grpc.keepAlive.time` | string | `After a duration of this time if the server doesn't see any activity it pings the client to see if the transport is still alive.` | `2h` |
 | `apis.grpc.keepAlive.timeout` | string | `After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed.` | `20s` |
-| `apis.grpc.tls.caPool` | string | `File path to the root certificate in PEM format which might contain multiple certificates in a single file.` |  `""` |
+| `apis.grpc.tls.caPool` | []string | `File path to the root certificate in PEM format which might contain multiple certificates in a single file.` |  `""` |
 | `apis.grpc.tls.enabled` | bool | `Enable TLS for grpc.` | `false` |
 | `apis.grpc.tls.keyFile` | string | `File path to private key in PEM format.` | `""` |
 | `apis.grpc.tls.certFile` | string | `File path to certificate in PEM format.` | `""` |
@@ -121,12 +121,19 @@ Supported modes:
 | Property | Type | Description | Default |
 | ---------- | -------- | -------------- | ------- |
 | `remoteProvisioning.mode` | string | `Provides remote provisioning mode. In "userAgent" mode all signing certificates goes through the user agent (browser,cli). "" means none. The supported values are: "", "userAgent"` | `""` |
-| `remoteProvisioning.userAgent.certificateAuthorityAddress` | string | `Certificate authority server address in format {DNS}:{PORT}` | `""` |
+| `remoteProvisioning.certificateAuthority` | string | `Certificate authority server address in format {SCHEME}://{DNS}:{PORT}` | `""` |
 | `remoteProvisioning.userAgent.csrChallengeStateExpiration` | string | `Defines how long is valid csr challenge.` | `"1m"` |
-| `remoteProvisioning.authorization.authority` | string | `Authority is the address of the token-issuing authentication server.` | `""` |
-| `remoteProvisioning.authorization.clientId` | string | `Client ID to exchange an authorization code for an access token.` | `""` |
-| `remoteProvisioning.authorization.audience` | string | `Identifier of the API configured in your OAuth provider.` | `""` |
-| `remoteProvisioning.authorization.scopes` | []string | `List of required scopes.` | `[]` |
-| `remoteProvisioning.authorization.ownerClaim` | string | `Claim used to identify owner of the device.` | `"sub"` |
+| `remoteProvisioning.authority` | string | `Authority is the address of the token-issuing authentication server.` | `""` |
+| `remoteProvisioning.webOAuthClient.clientID` | string | `Client ID to exchange an authorization code for an access token.` | `""` |
+| `remoteProvisioning.webOAuthClient.audience` | string | `Identifier of the API configured in your OAuth provider.` | `""` |
+| `remoteProvisioning.webOAuthClient.scopes` | []string | `List of required scopes.` | `[]` |
+| `remoteProvisioning.ownerClaim` | string | `Claim used to identify owner of the device.` | `"sub"` |
+| `remoteProvisioning.hubID` | string | `Plgd hub id.` | `""` |
+| `remoteProvisioning.coapGateway` | string | `CoAP gateway for onboard device.` | `""` |
+| `remoteProvisioning.caPool` | []string | `File paths to root CAs which was used to sign coap-gw certificate.` | `""` |
+| `remoteProvisioning.deviceOAuthClient.clientID` | string | `Client ID to exchange an authorization code for an access token.` | `""` |
+| `remoteProvisioning.deviceOAuthClient.audience` | string | `Identifier of the API configured in your OAuth provider.` | `""` |
+| `remoteProvisioning.deviceOAuthClient.scopes` | []string | `List of required scopes.` | `[]` |
+| `remoteProvisioning.deviceOAuthClient.providerName` | string | `Name of provider, which needs to be set to cloud resource during cloud provisioning.` | `""` |
 
 > Note that the string type related to time (i.e. timeout, idleConnTimeout, expirationTime) is decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "1.5h" or "2h45m". Valid time units are "ns", "us", "ms", "s", "m", "h".

cmd/defaultConfig.go

@@ -24,6 +24,7 @@ import (
 	"github.com/plgd-dev/client-application/service/config"
 )
 
+// resolveDefaultConfig creates default config file if it doesn't exist and sets --config argument.
 func resolveDefaultConfig(configPath string) error {
 	configPathWasSet := true
 	if configPath == "" {

cmd/main.go

@@ -51,6 +51,8 @@ func main() {
 		log.Errorf("cannot create default config: %v", err)
 		return
 	}
+	// parse line arguments again because resolveDefaultConfig can set config path
+	_, _ = flags.NewParser(&opts, flags.Default|flags.IgnoreUnknown).Parse()
 	cfg, err := config.New(opts.ConfigPath)
 	if err != nil {
 		log.Errorf("cannot load config: %v", err)

go.mod

@@ -4,35 +4,36 @@ go 1.18
 
 require (
 	cloud.google.com/go v0.105.0
+	github.com/favadi/protoc-go-inject-tag v1.4.0
 	github.com/fullstorydev/grpchan v1.1.1
-	github.com/golang-jwt/jwt/v4 v4.4.2
+	github.com/golang-jwt/jwt/v4 v4.4.3
 	github.com/google/uuid v1.3.0
 	github.com/goreleaser/goreleaser v1.9.2
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.13.0
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.14.0
 	github.com/jellydator/ttlcache/v3 v3.0.0
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/lestrrat-go/jwx v1.2.25
 	github.com/pion/dtls/v2 v2.1.6-0.20221001075407-984d41b9981a
-	github.com/plgd-dev/device/v2 v2.0.1
-	github.com/plgd-dev/go-coap/v3 v3.0.1-0.20221107080825-5cfe76a3720f
-	github.com/plgd-dev/hub/v2 v2.6.2-0.20221114160043-0fae0566c217
+	github.com/plgd-dev/device/v2 v2.0.2-0.20221202214050-f9f57f7c9a61
+	github.com/plgd-dev/go-coap/v3 v3.0.2-0.20221201101543-2e2c858a13f2
+	github.com/plgd-dev/hub/v2 v2.7.4-0.20221213114839-98df2c84d8eb
 	github.com/plgd-dev/kit/v2 v2.0.0-20211006190727-057b33161b90
 	github.com/stretchr/testify v1.8.1
 	go.opentelemetry.io/otel/trace v1.11.1
 	go.uber.org/atomic v1.10.0
-	go.uber.org/zap v1.23.0
-	google.golang.org/api v0.102.0
-	google.golang.org/grpc v1.50.1
+	go.uber.org/zap v1.24.0
+	google.golang.org/api v0.103.0
+	google.golang.org/grpc v1.51.0
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	cloud.google.com/go/compute v1.12.1 // indirect
-	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/compute v1.13.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.2 // indirect
 	cloud.google.com/go/iam v0.7.0 // indirect
 	cloud.google.com/go/kms v1.6.0 // indirect
 	cloud.google.com/go/storage v1.27.0 // indirect
@@ -81,7 +82,7 @@ require (
 	github.com/caarlos0/go-reddit/v3 v3.0.1 // indirect
 	github.com/caarlos0/go-shellwords v1.0.12 // indirect
 	github.com/cavaliergopher/cpio v1.0.1 // indirect
-	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
@@ -102,7 +103,7 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/goccy/go-json v0.9.11 // indirect
+	github.com/goccy/go-json v0.10.0 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -120,7 +121,7 @@ require (
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2 // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
@@ -159,12 +160,12 @@ require (
 	github.com/muesli/mango-pflag v0.1.0 // indirect
 	github.com/muesli/roff v0.1.0 // indirect
 	github.com/nats-io/jwt/v2 v2.2.0 // indirect
-	github.com/nats-io/nats.go v1.19.1 // indirect
+	github.com/nats-io/nats.go v1.20.0 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/panjf2000/ants/v2 v2.6.0 // indirect
 	github.com/pion/logging v0.2.2 // indirect
-	github.com/pion/transport v0.13.1 // indirect
+	github.com/pion/transport v0.14.1 // indirect
 	github.com/pion/udp v0.1.2-0.20221011090648-2589407f52c9 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -198,17 +199,17 @@ require (
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	gocloud.dev v0.24.0 // indirect
-	golang.org/x/crypto v0.1.0 // indirect
-	golang.org/x/exp v0.0.0-20221106115401-f9659909a136 // indirect
-	golang.org/x/net v0.1.0 // indirect
-	golang.org/x/oauth2 v0.1.0 // indirect
+	golang.org/x/crypto v0.3.0 // indirect
+	golang.org/x/exp v0.0.0-20221204150635-6dcec336b2bb // indirect
+	golang.org/x/net v0.2.0 // indirect
+	golang.org/x/oauth2 v0.2.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.2.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/text v0.5.0 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20221107162902-2d387536bcdd // indirect
+	google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/mail.v2 v2.3.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect