chore: Double the limit of grpc messages (#2131)

#### Summary

<!-- 🎉 Thank you for making CloudQuery awesome by submitting a PR 🎉 -->
Following up on the transformer hanging issue reported by a customer and
addressed in PR
[#2130](#2130):
Currently, the transformer plugin can increase the message size beyond
the limit enforced in the sync method of plugin-sdk
[1](https://github.com/cloudquery/plugin-sdk/blob/bc50fd3d2be414edba8f8ad5bb7739a012840bf1/internal/servers/plugin/v3/plugin.go#L267).
To accommodate scenarios where transformers or destination plugins need
to handle larger messages than the source plugin, we should consider
increasing this limit to provide more flexibility.

<!--
Explain what problem this PR addresses
-->

---

Use the following steps to ensure your PR is ready to be reviewed

- [ ] Read the [contribution guidelines](../blob/main/CONTRIBUTING.md)
🧑‍🎓
- [ ] Run `go fmt` to format your code 🖊
- [ ] Lint your changes via `golangci-lint run` 🚨 (install golangci-lint
[here](https://golangci-lint.run/usage/install/#local-installation))
- [ ] Update or add tests 🧪
- [ ] Ensure the status checks below are successful ✅

---------

Co-authored-by: Bartosz Lesniewski <bartosz@cloudquery.io>

Author: Przemysław Stępień

examples/simple_plugin/go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cloudquery/cloudquery-api-go v1.13.8 // indirect
 	github.com/cloudquery/codegen v0.3.26 // indirect
-	github.com/cloudquery/plugin-pb-go v1.26.9 // indirect
+	github.com/cloudquery/plugin-pb-go v1.26.10 // indirect
 	github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect

examples/simple_plugin/go.sum

@@ -54,8 +54,8 @@ github.com/cloudquery/cloudquery-api-go v1.13.8 h1:8n5D0G2wynbUdexr1GS8ND8i0uOwm
 github.com/cloudquery/cloudquery-api-go v1.13.8/go.mod h1:ZhEjPkDGDL2KZKlQLUnsgQ0mPz3qC7qftr37q3q+IcA=
 github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMpb5ZY=
 github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs=
-github.com/cloudquery/plugin-pb-go v1.26.9 h1:lkgxqIzabD6yvDm7D7oJvgO/T/bYIh7SSOojEgbMpnA=
-github.com/cloudquery/plugin-pb-go v1.26.9/go.mod h1:euhtVJKRtmWzukBxOjJyCKHPU9O9Gs5vasiBCaZVFRA=
+github.com/cloudquery/plugin-pb-go v1.26.10 h1:VNRk3JMLR7+pCXGCk4729I8r3vTrn64qonCs+4KY7+M=
+github.com/cloudquery/plugin-pb-go v1.26.10/go.mod h1:OVk5GXAlz8u6+sNlBgYxzpP+pYU+TqyyQV6FqMBBcIM=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=

go.mod

@@ -12,7 +12,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.29.0
 	github.com/bradleyjkemp/cupaloy/v2 v2.8.0
 	github.com/cloudquery/cloudquery-api-go v1.13.8
-	github.com/cloudquery/plugin-pb-go v1.26.9
+	github.com/cloudquery/codegen v0.3.26
+	github.com/cloudquery/plugin-pb-go v1.26.10
 	github.com/cloudquery/plugin-sdk/v2 v2.7.0
 	github.com/goccy/go-json v0.10.5
 	github.com/golang/mock v1.6.0
@@ -65,7 +66,6 @@ require (
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
-	github.com/cloudquery/codegen v0.3.26 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect

go.sum

@@ -56,8 +56,8 @@ github.com/cloudquery/codegen v0.3.26 h1:cWORVpObYW5/0LnjC0KO/Ocg1+vbZivJfFd+sMp
 github.com/cloudquery/codegen v0.3.26/go.mod h1:bg/M1JxFvNVABMLMFb/uAQmTGAyI2L/E4zL4kho9RFs=
 github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66 h1:OZLPSIBYEfvkAUeOeM8CwTgVQy5zhayI99ishCrsFV0=
 github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
-github.com/cloudquery/plugin-pb-go v1.26.9 h1:lkgxqIzabD6yvDm7D7oJvgO/T/bYIh7SSOojEgbMpnA=
-github.com/cloudquery/plugin-pb-go v1.26.9/go.mod h1:euhtVJKRtmWzukBxOjJyCKHPU9O9Gs5vasiBCaZVFRA=
+github.com/cloudquery/plugin-pb-go v1.26.10 h1:VNRk3JMLR7+pCXGCk4729I8r3vTrn64qonCs+4KY7+M=
+github.com/cloudquery/plugin-pb-go v1.26.10/go.mod h1:OVk5GXAlz8u6+sNlBgYxzpP+pYU+TqyyQV6FqMBBcIM=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=

serve/constants.go

@@ -6,7 +6,7 @@ import (
 
 const (
 	// bufSize used for unit testing grpc server and client
-	testBufSize  = 1024 * 1024
-	flushTimeout = 5 * time.Second
-	MaxMsgSize   = 100 * 1024 * 1024 // 100 MiB
+	testBufSize    = 1024 * 1024
+	flushTimeout   = 5 * time.Second
+	MaxGrpcMsgSize = 200 * 1024 * 1024 // 200 MiB
 )

serve/plugin.go

@@ -179,8 +179,8 @@ func (s *PluginServe) newCmdPluginServe() *cobra.Command {
 				grpc.ChainStreamInterceptor(
 					logging.StreamServerInterceptor(grpczerolog.InterceptorLogger(logger)),
 				),
-				grpc.MaxRecvMsgSize(MaxMsgSize),
-				grpc.MaxSendMsgSize(MaxMsgSize),
+				grpc.MaxRecvMsgSize(MaxGrpcMsgSize),
+				grpc.MaxSendMsgSize(MaxGrpcMsgSize),
 			)
 			s.plugin.SetLogger(logger)
 			pbv3.RegisterPluginServer(grpcServer, &serversv3.Server{