Lakshman single datasource generation (#14598)

Author: luckyswaminathan

mmv1/api/resource.go

@@ -226,6 +226,9 @@ type Resource struct {
 	// If true, resource is not importable
 	ExcludeImport bool `yaml:"exclude_import,omitempty"`
 
+	// If true, resource should be autogenerated as a data source
+	GenerateDatasource bool `yaml:"generate_datasource,omitempty"`
+
 	// If true, skip sweeper generation for this resource
 	ExcludeSweeper bool `yaml:"exclude_sweeper,omitempty"`
 
@@ -1993,6 +1996,44 @@ func urlContainsOnlyAllowedKeys(templateURL string, allowedKeys []string) bool {
 	return true
 }
 
+func (r Resource) ShouldGenerateSingularDataSource() bool {
+	return r.GenerateDatasource
+}
+
+// DatasourceOptionalFields returns a list of fields from the resource's URI
+// that should be marked as "Required".
+func (r Resource) DatasourceRequiredFields() []string {
+	requiredFields := []string{}
+	uriParts := strings.Split(r.SelfLink, "/")
+
+	for _, part := range uriParts {
+		if strings.HasPrefix(part, "{{") && strings.HasSuffix(part, "}}") {
+			field := strings.TrimSuffix(strings.TrimPrefix(part, "{{"), "}}")
+			if field != "region" && field != "project" && field != "zone" {
+				requiredFields = append(requiredFields, field)
+			}
+		}
+	}
+	return requiredFields
+}
+
+// DatasourceOptionalFields returns a list of fields from the resource's URI
+// that should be marked as "Optional".
+func (r Resource) DatasourceOptionalFields() []string {
+	optionalFields := []string{}
+	uriParts := strings.Split(r.SelfLink, "/")
+
+	for _, part := range uriParts {
+		if strings.HasPrefix(part, "{{") && strings.HasSuffix(part, "}}") {
+			field := strings.TrimSuffix(strings.TrimPrefix(part, "{{"), "}}")
+			if field == "region" || field == "project" || field == "zone" {
+				optionalFields = append(optionalFields, field)
+			}
+		}
+	}
+	return optionalFields
+}
+
 func (r Resource) ShouldGenerateSweepers() bool {
 	if !r.ExcludeSweeper && !utils.IsEmpty(r.Sweeper) {
 		return true

mmv1/products/iap/Client.yaml

@@ -33,6 +33,7 @@ self_link: '{{brand}}/identityAwareProxyClients/{{client_id}}'
 immutable: true
 import_format:
   - '{{brand}}/identityAwareProxyClients/{{client_id}}'
+generate_datasource: true
 timeouts:
   insert_minutes: 20
   update_minutes: 20

mmv1/provider/template_data.go

@@ -93,6 +93,14 @@ func (td *TemplateData) GenerateMetadataFile(filePath string, resource api.Resou
 	td.GenerateFile(filePath, templatePath, resource, false, templates...)
 }
 
+func (td *TemplateData) GenerateDataSourceFile(filePath string, resource api.Resource) {
+	templatePath := "templates/terraform/datasource.go.tmpl"
+	templates := []string{
+		templatePath,
+	}
+	td.GenerateFile(filePath, templatePath, resource, true, templates...)
+}
+
 func (td *TemplateData) GenerateProductFile(filePath string, product api.Product) {
 	templatePath := "templates/terraform/product.go.tmpl"
 	templates := []string{

mmv1/provider/terraform.go

@@ -107,6 +107,7 @@ func (t *Terraform) GenerateObject(object api.Resource, outputFolder, productPat
 			// log.Printf("Generating %s tests", object.Name)
 			t.GenerateResourceTests(object, *templateData, outputFolder)
 			t.GenerateResourceSweeper(object, *templateData, outputFolder)
+			t.GenerateSingularDataSource(object, *templateData, outputFolder)
 			// log.Printf("Generating %s metadata", object.Name)
 			t.GenerateResourceMetadata(object, *templateData, outputFolder)
 		}
@@ -188,6 +189,20 @@ func (t *Terraform) GenerateResourceSweeper(object api.Resource, templateData Te
 	templateData.GenerateSweeperFile(targetFilePath, object)
 }
 
+func (t *Terraform) GenerateSingularDataSource(object api.Resource, templateData TemplateData, outputFolder string) {
+	if !object.ShouldGenerateSingularDataSource() {
+		return
+	}
+
+	productName := t.Product.ApiName
+	targetFolder := path.Join(outputFolder, t.FolderName(), "services", productName)
+	if err := os.MkdirAll(targetFolder, os.ModePerm); err != nil {
+		log.Println(fmt.Errorf("error creating parent directory %v: %v", targetFolder, err))
+	}
+	targetFilePath := path.Join(targetFolder, fmt.Sprintf("data_source_%s.go", t.ResourceGoFilename(object)))
+	templateData.GenerateDataSourceFile(targetFilePath, object)
+}
+
 // GenerateProduct creates the product.go file for a given service directory.
 // This will be used to seed the directory and add a package-level comment
 // specific to the product.

mmv1/templates/terraform/datasource.go.tmpl

@@ -0,0 +1,92 @@
+{{/* The license inside this block applies to this file
+  Copyright 2024 Google LLC. All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. */ -}}
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+{{$.CodeHeader TemplatePath}}
+package {{ lower $.ProductMetadata.Name }}
+
+import (
+
+    "fmt"
+    "log"
+    "net/http"
+    "reflect"
+{{- if $.SupportsIndirectUserProjectOverride }}
+    "regexp"
+{{- end }}
+{{- if or (and (not $.Immutable) ($.UpdateMask)) $.LegacyLongFormProject }}
+    "strings"
+{{- end }}
+    "time"
+
+{{/*     # We list all the v2 imports here, because we run 'goimports' to guess the correct */}}
+{{/*     # set of imports, which will never guess the major version correctly. */}}
+    "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
+    "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff"
+    "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+    "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
+    "github.com/hashicorp/terraform-plugin-sdk/v2/helper/structure"
+    "github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
+    "github.com/hashicorp/terraform-plugin-sdk/v2/diag"
+    "github.com/hashicorp/terraform-plugin-sdk/v2/helper/logging"
+    "github.com/hashicorp/go-cty/cty"
+
+    "{{ $.ImportPath }}/tpgresource"
+    transport_tpg "{{ $.ImportPath }}/transport"
+    "{{ $.ImportPath }}/verify"
+
+{{ if $.FlattenedProperties }}
+    "google.golang.org/api/googleapi"
+{{- end}}
+)
+
+func DataSource{{ .ResourceName -}}() *schema.Resource {
+	rs := Resource{{ .ResourceName -}}().Schema
+
+	dsSchema := tpgresource.DatasourceSchemaFromResourceSchema(rs)
+
+  // Set 'Required' schema elements
+  tpgresource.AddRequiredFieldsToSchema(dsSchema, {{range $index, $field := .DatasourceRequiredFields}}{{if gt $index 0}}, {{end}}{{printf "%q" $field}}{{end}})
+
+  // Set 'Optional' schema elements
+  tpgresource.AddOptionalFieldsToSchema(dsSchema, {{range $index, $field := .DatasourceOptionalFields}}{{if gt $index 0}}, {{end}}{{printf "%q" $field}}{{end}})
+
+  return &schema.Resource{
+		Read:   dataSource{{ $.ResourceName -}}Read,
+		Schema: dsSchema,
+	}
+}
+
+func dataSource{{ $.ResourceName -}}Read(d *schema.ResourceData, meta interface{}) error {
+  config := meta.(*transport_tpg.Config)
+
+  id, err := tpgresource.ReplaceVars{{if $.LegacyLongFormProject -}}ForId{{ end -}}(d, config, "{{$.SelfLinkUri}}{{$.ReadQueryParams}}")
+    if err != nil {
+        return err
+    }
+  d.SetId(id)
+
+  err = resource{{ $.ResourceName -}}Read(d, meta)
+	if err != nil {
+		return err
+	}
+
+  if d.Id() == "" {
+		return fmt.Errorf("%s not found", id)
+	}
+
+  return nil
+}

mmv1/third_party/terraform/provider/provider_mmv1_resources.go.tmpl

@@ -151,7 +151,7 @@ var handwrittenDatasources = map[string]*schema.Resource{
 	"google_iam_testable_permissions":                  resourcemanager.DataSourceGoogleIamTestablePermissions(),
 	"google_iam_workload_identity_pool":                iambeta.DataSourceIAMBetaWorkloadIdentityPool(),
 	"google_iam_workload_identity_pool_provider":       iambeta.DataSourceIAMBetaWorkloadIdentityPoolProvider(),
-	"google_iap_client":                                iap.DataSourceGoogleIapClient(),
+	"google_iap_client":                                iap.DataSourceIapClient(),
 	"google_kms_crypto_key":                            kms.DataSourceGoogleKmsCryptoKey(),
 	"google_kms_crypto_keys":                           kms.DataSourceGoogleKmsCryptoKeys(),
 	"google_kms_crypto_key_version":                    kms.DataSourceGoogleKmsCryptoKeyVersion(),