feat(codegen): Add support for terraform lists/queries

Author: kklimonda-cl

assets/terraform/internal/manager/entry_import.go

@@ -18,6 +18,7 @@ type SDKImportableEntryService[E EntryObject, L EntryLocation, IL ImportLocation
 	Delete(context.Context, L, []IL, ...string) error
 	ImportToLocations(context.Context, L, []IL, string) error
 	UnimportFromLocations(context.Context, L, []IL, []string) error
+	ListWithXpath(context.Context, string, string, string, string) ([]E, error)
 }
 
 type ImportableEntryObjectManager[E EntryObject, L EntryLocation, IL ImportLocation, IS SDKImportableEntryService[E, L, IL]] struct {
@@ -38,8 +39,22 @@ func NewImportableEntryObjectManager[E EntryObject, L EntryLocation, IL ImportLo
 	}
 }
 
-func (o *ImportableEntryObjectManager[E, L, IL, IS]) ReadMany(ctx context.Context, location L, entries []E) ([]E, error) {
-	return nil, &Error{err: ErrInternal, message: "called ReadMany on an importable singular resource"}
+func (o *ImportableEntryObjectManager[E, L, IL, IS]) ReadMany(ctx context.Context, location L, components []string) ([]E, error) {
+	xpath, err := location.XpathWithComponents(o.client.Versioning(), append(components, util.AsEntryXpath(""))...)
+	if err != nil {
+		return nil, err
+	}
+
+	existing, err := o.service.ListWithXpath(ctx, util.AsXpath(xpath), "get", "", "")
+	if err != nil {
+		if sdkerrors.IsObjectNotFound(err) {
+			return nil, ErrObjectNotFound
+		} else {
+			return nil, &Error{err: err, message: "Failed to read entries from the server"}
+		}
+	}
+
+	return existing, nil
 }
 
 func (o *ImportableEntryObjectManager[E, L, IL, IS]) Read(ctx context.Context, location L, components []string, name string) (E, error) {

pkg/commands/codegen/codegen.go

@@ -76,6 +76,7 @@ func (c *Command) Execute() error {
 	var dataSourceList []string
 	var ephemeralResourceList []string
 	var actionsList []string
+	var listResourceList []string
 	specMetadata := make(map[string]properties.TerraformProviderSpecMetadata)
 
 	for _, specPath := range c.specs {
@@ -138,6 +139,7 @@ func (c *Command) Execute() error {
 				dataSourceList = append(dataSourceList, data.DataSources...)
 				ephemeralResourceList = append(ephemeralResourceList, data.EphemeralResources...)
 				actionsList = append(actionsList, data.Actions...)
+				listResourceList = append(listResourceList, data.ListResources...)
 
 				for k, v := range data.SpecMetadata {
 					specMetadata[k] = v
@@ -168,6 +170,7 @@ func (c *Command) Execute() error {
 				dataSourceList = append(dataSourceList, data.DataSources...)
 				ephemeralResourceList = append(ephemeralResourceList, data.EphemeralResources...)
 				actionsList = append(actionsList, data.Actions...)
+				listResourceList = append(listResourceList, data.ListResources...)
 
 				for k, v := range data.SpecMetadata {
 					specMetadata[k] = v
@@ -190,6 +193,7 @@ func (c *Command) Execute() error {
 		newProviderObject.DataSources = append(newProviderObject.DataSources, dataSourceList...)
 		newProviderObject.Resources = append(newProviderObject.Resources, resourceList...)
 		newProviderObject.EphemeralResources = append(newProviderObject.EphemeralResources, ephemeralResourceList...)
+		newProviderObject.ListResources = append(newProviderObject.ListResources, listResourceList...)
 		newProviderObject.Actions = append(newProviderObject.Actions, actionsList...)
 		newProviderObject.SpecMetadata = specMetadata
 

pkg/generate/generator.go

@@ -60,6 +60,7 @@ func (c *Creator) RenderTemplate() error {
 
 type TerraformProviderFileData struct {
 	EphemeralResources []string
+	ListResources      []string
 	Resources          []string
 	DataSources        []string
 	Actions            []string
@@ -90,6 +91,12 @@ func (c *Creator) RenderTerraformProviderFile(spec *properties.Normalization, ty
 		return nil, err
 	}
 
+	if !*spec.TerraformProviderConfig.SkipListResource && (typ == properties.ResourceEntry || typ == properties.ResourceUuid) {
+		if err := tfp.GenerateTerraformListResource(typ, spec, terraformProvider); err != nil {
+			return nil, err
+		}
+	}
+
 	if spec.TerraformProviderConfig.Action {
 		if err := tfp.GenerateTerraformAction(spec, terraformProvider); err != nil {
 			return nil, err
@@ -126,6 +133,7 @@ func (c *Creator) RenderTerraformProviderFile(spec *properties.Normalization, ty
 		EphemeralResources: terraformProvider.EphemeralResources,
 		Actions:            terraformProvider.Actions,
 		SpecMetadata:       terraformProvider.SpecMetadata,
+		ListResources:      terraformProvider.ListResources,
 	}
 	return data, nil
 }

pkg/properties/normalized.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io/fs"
 	"maps"
-	"os"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -85,6 +84,7 @@ type TerraformProviderConfig struct {
 	Action                bool                       `json:"action" yaml:"action"`
 	CustomValidation      bool                       `json:"custom_validation" yaml:"custom_validation"`
 	SkipResource          bool                       `json:"skip_resource" yaml:"skip_resource"`
+	SkipListResource      *bool                      `json:"skip_list_resource", yaml:"skip_list_resource"`
 	SkipDatasource        bool                       `json:"skip_datasource" yaml:"skip_datasource"`
 	SkipDatasourceListing bool                       `json:"skip_datasource_listing" yaml:"skip_datasource_listing"`
 	ResourceType          TerraformResourceType      `json:"resource_type" yaml:"resource_type"`
@@ -722,6 +722,7 @@ func schemaToSpec(object object.Object) (*Normalization, error) {
 			Action:                object.TerraformConfig.Action,
 			CustomValidation:      object.TerraformConfig.CustomValidation,
 			SkipResource:          object.TerraformConfig.SkipResource,
+			SkipListResource:      object.TerraformConfig.SkipListResource,
 			SkipDatasource:        object.TerraformConfig.SkipDatasource,
 			SkipDatasourceListing: object.TerraformConfig.SkipdatasourceListing,
 			ResourceType:          TerraformResourceType(object.TerraformConfig.ResourceType),
@@ -1126,26 +1127,20 @@ func (spec *Normalization) ResourceXpathVariablesCount() int {
 	return len(spec.PanosXpath.Variables)
 }
 
-// loadTemplate loads a template from either sdk or terraform-provider templates directory
-func loadTemplate(templateType string, templatePath string) (string, error) {
-	fullPath := filepath.Join("templates", templateType, templatePath)
-	content, err := os.ReadFile(fullPath)
-	if err != nil {
-		// Try from parent directories (for when running from subdirectories)
-		for i := 1; i <= 3; i++ {
-			prefix := strings.Repeat("../", i)
-			altPath := filepath.Join(prefix, "templates", templateType, templatePath)
-			content, err = os.ReadFile(altPath)
-			if err == nil {
-				break
-			}
-		}
-		if err != nil {
-			return "", fmt.Errorf("failed to read template %s: %w", fullPath, err)
-		}
-	}
-	return string(content), nil
+const attributesFromXpathComponentsTmpl = `
+{{- range .Components }}
+  {{ if eq .Type "value" }}
+	{{ $.Target }}.{{ .Name }} = types.StringValue(components[{{ .Idx }}])
+  {{- else if eq .Type "entry" }}
+{
+	component := components[{{ .Idx }}]
+	component = strings.TrimPrefix(component, "entry[@name='")
+	component = strings.TrimSuffix(component, "']")
+	{{ $.Target }}.{{ .Name.CamelCase }} = types.StringValue(component)
 }
+  {{- end }}
+{{- end }}
+`
 
 func (spec *Normalization) AttributesFromXpathComponents(target string) (string, error) {
 	type componentCtx struct {
@@ -1182,43 +1177,66 @@ func (spec *Normalization) AttributesFromXpathComponents(target string) (string,
 		Components: components,
 	}
 
-	tmplContent, err := loadTemplate("terraform-provider", "partials/attributes_from_xpath_components.tmpl")
-	if err != nil {
-		return "", err
-	}
-
-	tmpl := template.Must(template.New("attributes-from-xpath-components").Parse(tmplContent))
+	tmpl := template.Must(template.New("attributes-from-xpath-components").Parse(attributesFromXpathComponentsTmpl))
 
 	var buf bytes.Buffer
-	err = tmpl.Execute(&buf, data)
+	err := tmpl.Execute(&buf, data)
 	if err != nil {
 		panic(err)
 	}
 
 	return buf.String(), nil
 }
 
+const resourceXpathAssignmentsTmpl = `
+{{- range .Variables }}
+  {{- if or (eq .Type "entry") (eq .Type "value") }}
+	ans = append(ans, components[{{ .Idx }}])
+  {{- else if eq .Type "static" }}
+	ans = append(ans, "{{ .Name.Original }}")
+  {{- end }}
+{{- end }}
+`
+
 func (spec *Normalization) ResourceXpathAssignments() (string, error) {
 	data := xpathVariablesCtx{
 		Variables: createXpathVariablesContext(spec),
 	}
 
-	tmplContent, err := loadTemplate("sdk", "partials/resource_xpath_assignments.tmpl")
-	if err != nil {
-		return "", err
-	}
-
-	tmpl := template.Must(template.New("resource-xpath-assignments").Parse(tmplContent))
+	tmpl := template.Must(template.New("resource-xpath-assignments").Parse(resourceXpathAssignmentsTmpl))
 
 	var buf bytes.Buffer
-	err = tmpl.Execute(&buf, data)
+	err := tmpl.Execute(&buf, data)
 	if err != nil {
 		panic(err)
 	}
 
 	return buf.String(), nil
 }
 
+const xpathVariableChecksTmpl = `
+{{- range .Variables }}
+  {{- if eq .Type "entry" }}
+{
+	component := components[{{ .Idx }}]
+  {{- if .AllowEmpty }}
+	if component != "entry" {
+  {{- end }}
+	if !strings.HasPrefix(component, "entry[@name=\"]") && !strings.HasPrefix(component, "entry[@name='") {
+		return nil, errors.NewInvalidXpathComponentError(fmt.Sprintf("{{ .Name.CamelCase }} must be formatted as entry: %s", component))
+	}
+
+	if !strings.HasSuffix(component, "\"]") && !strings.HasSuffix(component, "']") {
+		return nil, errors.NewInvalidXpathComponentError(fmt.Sprintf("{{ .Name.CamelCase }} must be formatted as entry: %s", component))
+	}
+  {{- if .AllowEmpty }}
+	}
+  {{- end }}
+}
+  {{- end }}
+{{- end }}
+`
+
 type xpathVariableCtx struct {
 	Type       object.PanosXpathVariableType
 	Name       *NameVariant
@@ -1283,15 +1301,10 @@ func (spec *Normalization) ResourceXpathVariableChecks() (string, error) {
 		Variables: createXpathVariablesContext(spec),
 	}
 
-	tmplContent, err := loadTemplate("sdk", "partials/xpath_variable_checks.tmpl")
-	if err != nil {
-		return "", err
-	}
-
 	var buf bytes.Buffer
-	tmpl := template.Must(template.New("xpath-variable-checks").Parse(tmplContent))
+	tmpl := template.Must(template.New("xpath-variable-checks").Parse(xpathVariableChecksTmpl))
 
-	err = tmpl.Execute(&buf, data)
+	err := tmpl.Execute(&buf, data)
 	if err != nil {
 		panic(err)
 	}

pkg/properties/provider_file.go

@@ -68,6 +68,10 @@ func (o TerraformNameProvider) IdentityModelStructName() string {
 	return fmt.Sprintf("%sIdentityModel", o.ResourceStructName)
 }
 
+func (o TerraformNameProvider) ListResourceStructName() string {
+	return fmt.Sprintf("%sListResource", o.StructName)
+}
+
 type TerraformSpecFlags uint
 
 const (
@@ -91,6 +95,7 @@ type TerraformProviderFile struct {
 	DataSources        []string
 	Resources          []string
 	EphemeralResources []string
+	ListResources      []string
 	Actions            []string
 	SpecMetadata       map[string]TerraformProviderSpecMetadata
 	Code               *strings.Builder

pkg/properties/resourcetype.go

@@ -16,8 +16,10 @@ type SchemaType string
 const (
 	SchemaResource          SchemaType = "resource"
 	SchemaEphemeralResource SchemaType = "ephemeral-resource"
+	SchemaListResource      SchemaType = "list-resource"
 	SchemaAction            SchemaType = "action"
 	SchemaDataSource        SchemaType = "datasource"
 	SchemaCommon            SchemaType = "common"
 	SchemaProvider          SchemaType = "provider"
+	SchemaCustom            SchemaType = "custom"
 )

pkg/schema/object/object.go

@@ -43,6 +43,7 @@ type TerraformConfig struct {
 	SkipResource          bool                       `yaml:"skip_resource"`
 	SkipDatasource        bool                       `yaml:"skip_datasource"`
 	SkipdatasourceListing bool                       `yaml:"skip_datasource_listing"`
+	SkipListResource      *bool                      `yaml:"skip_list_resource"`
 	ResourceType          TerraformResourceType      `yaml:"resource_type"`
 	XmlNode               *string                    `yaml:"xml_node"`
 	CustomFunctions       map[string]bool            `yaml:"custom_functions"`
@@ -167,6 +168,11 @@ func (o *Object) UnmarshalYAML(n *yaml.Node) error {
 		return fmt.Errorf("failed to unmarshal yaml spec: plural_type must be list for uuid resource types")
 	}
 
+	if obj.TerraformConfig.SkipListResource == nil {
+		skipListResource := true
+		obj.TerraformConfig.SkipListResource = &skipListResource
+	}
+
 	return nil
 }
 

pkg/translate/terraform_provider/crud_operations.go

@@ -35,7 +35,7 @@ func ResourceCreateFunction(resourceTyp properties.ResourceType, names *NameProv
 			return RendeCreateUpdateMovementRequired(state, entries)
 		},
 		"RenderLocationsStateToPango": func(source string, dest string) (string, error) {
-			return RenderLocationsStateToPango(names, paramSpec, source, dest)
+			return RenderLocationsStateToPango(names, paramSpec, source, dest, "resp.Diagnostics")
 		},
 	}
 
@@ -151,7 +151,7 @@ func DataSourceReadFunction(resourceTyp properties.ResourceType, names *NameProv
 			return RenderLocationsPangoToState(names, paramSpec, source, dest)
 		},
 		"RenderLocationsStateToPango": func(source string, dest string) (string, error) {
-			return RenderLocationsStateToPango(names, paramSpec, source, dest)
+			return RenderLocationsStateToPango(names, paramSpec, source, dest, "resp.Diagnostics")
 		},
 	}
 
@@ -219,7 +219,7 @@ func ResourceReadFunction(resourceTyp properties.ResourceType, names *NameProvid
 			return RenderLocationsPangoToState(names, paramSpec, source, dest)
 		},
 		"RenderLocationsStateToPango": func(source string, dest string) (string, error) {
-			return RenderLocationsStateToPango(names, paramSpec, source, dest)
+			return RenderLocationsStateToPango(names, paramSpec, source, dest, "resp.Diagnostics")
 		},
 	}
 
@@ -281,7 +281,7 @@ func ResourceUpdateFunction(resourceTyp properties.ResourceType, names *NameProv
 			return RendeCreateUpdateMovementRequired(state, entries)
 		},
 		"RenderLocationsStateToPango": func(source string, dest string) (string, error) {
-			return RenderLocationsStateToPango(names, paramSpec, source, dest)
+			return RenderLocationsStateToPango(names, paramSpec, source, dest, "resp.Diagnostics")
 		},
 		"RenderLocationsPangoToState": func(source string, dest string) (string, error) {
 			return RenderLocationsPangoToState(names, paramSpec, source, dest)
@@ -345,7 +345,7 @@ func ResourceDeleteFunction(resourceTyp properties.ResourceType, names *NameProv
 			return RenderImportLocationAssignment(names, paramSpec, source, dest)
 		},
 		"RenderLocationsStateToPango": func(source string, dest string) (string, error) {
-			return RenderLocationsStateToPango(names, paramSpec, source, dest)
+			return RenderLocationsStateToPango(names, paramSpec, source, dest, "resp.Diagnostics")
 		},
 	}