Skip to content

Commit 6172dba

Browse files
authored
Update Test Generation: Launch (GoogleCloudPlatform#16931)
1 parent d489351 commit 6172dba

8 files changed

Lines changed: 582 additions & 10 deletions

File tree

GNUmakefile

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,16 @@ ifneq ($(SKIP_CLEAN),)
3232
endif
3333
endif
3434

35-
terraform build provider: validate_environment clean-provider mmv1
35+
TEMP_DIR := $(shell mktemp -d)
36+
export TEMP_DIR
37+
38+
terraform build provider: validate_environment clean-provider mmv1 clean-temp
3639
@echo "Provider generation process finished for $(VERSION) in $(OUTPUT_PATH)"
3740

3841

39-
mmv1:
42+
mmv1: prepare-temp
4043
@echo "Executing mmv1 build for $(OUTPUT_PATH)";
41-
@cd mmv1;\
44+
@cd $(TEMP_DIR)/mmv1;\
4245
if [ "$(VERSION)" = "ga" ]; then \
4346
go run . --output $(OUTPUT_PATH) --version ga --no-docs $(mmv1_compile) \
4447
&& go run . --output $(OUTPUT_PATH) --version beta --no-code $(mmv1_compile); \
@@ -87,15 +90,17 @@ clean-tgc:
8790
rm -rf ./test/*;\
8891
rm -rf ./cmd/*;\
8992

90-
tgc:
91-
cd mmv1;\
93+
tgc: prepare-temp
94+
cd $(TEMP_DIR)/mmv1;\
9295
go run . --version beta --provider tgc --output $(OUTPUT_PATH)/tfplan2cai $(mmv1_compile)\
9396
&& go run . --version ga --provider tgc_cai2hcl --output $(OUTPUT_PATH)/cai2hcl $(mmv1_compile)\
94-
&& go run . --version ga --provider tgc_next --output $(OUTPUT_PATH) $(mmv1_compile);\
97+
&& go run . --version ga --provider tgc_next --output $(OUTPUT_PATH) $(mmv1_compile);
98+
$(MAKE) clean-temp\
9599

96-
tf-oics:
97-
cd mmv1;\
98-
go run . --version ga --provider oics --output $(OUTPUT_PATH) $(mmv1_compile);\
100+
tf-oics: prepare-temp
101+
cd $(TEMP_DIR)/mmv1;\
102+
go run . --version ga --provider oics --output $(OUTPUT_PATH) $(mmv1_compile);
103+
$(MAKE) clean-temp\
99104

100105
test:
101106
cd mmv1; \
@@ -122,4 +127,34 @@ check_safe_build:
122127
doctor:
123128
./scripts/doctor
124129

125-
.PHONY: mmv1 test clean-provider validate_environment doctor
130+
prepare-temp:
131+
@echo "Setting up temporary workspace for conversion at $(TEMP_DIR)..."
132+
@rm -rf $(TEMP_DIR)
133+
@mkdir -p $(TEMP_DIR)/mmv1 $(TEMP_DIR)/tpgtools $(TEMP_DIR)/tools
134+
@echo "Copying files to $(TEMP_DIR)..."
135+
@cp -R ./mmv1/. $(TEMP_DIR)/mmv1/
136+
@cp -R ./tpgtools/. $(TEMP_DIR)/tpgtools/
137+
@cp -R ./tools/. $(TEMP_DIR)/tools/
138+
@echo "Building resource template converter in temp..."
139+
cd $(TEMP_DIR)/tools/resource-template-converter && go env -w GO111MODULE=on && go mod tidy && go build -v -o $(TEMP_DIR)/tools/convert-resource-template . && echo "BUILD SUCCEEDED" || (echo "BUILD FAILED"; exit 1)
140+
@echo "Build finished, running converter..."
141+
@ls -la $(TEMP_DIR)/tools/convert-resource-template
142+
@$(TEMP_DIR)/tools/convert-resource-template convert-resource-template $(TEMP_DIR)
143+
@echo "Temporary workspace setup complete."
144+
145+
test-convert:
146+
@echo $(TEMP_DIR)
147+
148+
clean-temp:
149+
@echo "Cleaning up temporary workspace $(TEMP_DIR)..."
150+
@rm -rf $(TEMP_DIR)
151+
152+
convert-templates:
153+
@echo "Checking and running resource template converter..."
154+
@if [ ! -f ./convert-resource-template ]; then \
155+
echo "Building convert-resource-template tool..."; \
156+
(cd tools/resource-template-converter && go build -o ../../convert-resource-template); \
157+
fi
158+
@./convert-resource-template convert-resource-template .
159+
160+
.PHONY: mmv1 test clean-provider validate_environment doctor convert-templates prepare-temp clean-temp
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"os"
7+
"path/filepath"
8+
"regexp"
9+
10+
"github.com/GoogleCloudPlatform/magic-modules/tools/resource-template-converter/copy"
11+
"github.com/GoogleCloudPlatform/magic-modules/tools/resource-template-converter/migrate"
12+
"github.com/spf13/cobra"
13+
)
14+
15+
var resourceFileRegex = regexp.MustCompile(`mmv1/products/([^/]+)/([^/]+\.yaml)`)
16+
17+
var convertResourceTemplateCmd = &cobra.Command{
18+
Use: "convert-resource-template",
19+
Short: "convert resource template from using examples to samples",
20+
Long: `This command convert resource yaml template to use new version samples within existing legacy examples.
21+
22+
23+
The command expects the following argument(s):
24+
1. Root directory path
25+
26+
It then performs the following operations:
27+
1. Updates existing example config to new vars and then copy them to the new samples/services/<service> dir
28+
2. Updates resource yaml teplate to use new samples blocks from existing legacy examples block`,
29+
30+
Args: cobra.ExactArgs(1),
31+
RunE: func(cmd *cobra.Command, args []string) error {
32+
root := args[0]
33+
return exeCconvertResourceTemplate(root)
34+
},
35+
}
36+
37+
func exeCconvertResourceTemplate(basePath string) error {
38+
if _, err := os.Stat(filepath.Join(basePath, "mmv1")); os.IsNotExist(err) {
39+
log.Fatalf("magic-modules directory structure not found. Please ensure this tool is run from 'magic-modules/tools/example-split'.")
40+
}
41+
42+
productsPath := filepath.Join(basePath, "mmv1", "products")
43+
templatesPath := filepath.Join(basePath, "mmv1", "templates", "terraform")
44+
examplesSourceDir := filepath.Join(templatesPath, "examples")
45+
samplesDestDir := filepath.Join(templatesPath, "samples", "services")
46+
fmt.Printf("Starting processing of product YAML files in: %s\n", productsPath)
47+
48+
err := filepath.Walk(productsPath, func(path string, info os.FileInfo, err error) error {
49+
if err != nil {
50+
return err
51+
}
52+
if !info.IsDir() && filepath.Ext(path) == ".yaml" {
53+
matches := resourceFileRegex.FindStringSubmatch(path)
54+
if matches == nil {
55+
log.Printf("Skipping non-resource file: %s\n", path)
56+
return nil
57+
}
58+
serviceName := matches[1]
59+
60+
if err := copy.ProcessResourceFile(path, serviceName, examplesSourceDir, samplesDestDir); err != nil {
61+
log.Printf("Error copying templates registered in file %s: %v\n", path, err)
62+
// Continue processing other files even if one fails.
63+
}
64+
if err := migrate.MigrateFile(path, serviceName); err != nil {
65+
log.Printf("Failed to migrate file %s: %v\n", path, err)
66+
// Continue migrating other files even if one fails.
67+
}
68+
}
69+
return nil
70+
})
71+
72+
if err != nil {
73+
log.Fatalf("Error walking the products path %q: %v\n", productsPath, err)
74+
}
75+
76+
fmt.Println("Processing complete.")
77+
return nil
78+
}
79+
80+
func init() {
81+
rootCmd.AddCommand(convertResourceTemplateCmd)
82+
}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"os"
6+
7+
"github.com/spf13/cobra"
8+
)
9+
10+
// rootCmd represents the base command when called without any subcommands
11+
12+
var rootCmd = &cobra.Command{
13+
Use: "resource-template-coverter",
14+
Short: "Tool for migrating resource yaml templates",
15+
Long: `Tool for migrating resource yaml templates`,
16+
}
17+
18+
// Execute adds all child commands to the root command and sets flags appropriately.
19+
// This is called by main.main(). It only needs to happen once to the rootCmd.
20+
func Execute() {
21+
err := rootCmd.Execute()
22+
if err != nil {
23+
fmt.Println(err)
24+
os.Exit(1)
25+
}
26+
}
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package copy
2+
3+
import (
4+
"fmt"
5+
"io/ioutil"
6+
"log"
7+
"os"
8+
"path/filepath"
9+
"strings"
10+
11+
"gopkg.in/yaml.v2"
12+
)
13+
14+
type Resource struct {
15+
Examples []Example `yaml:"examples"`
16+
}
17+
18+
type Example struct {
19+
Name string `yaml:"name"`
20+
ConfigPath string `yaml:"config_path,omitempty"`
21+
}
22+
23+
func ProcessResourceFile(filePath, serviceName, examplesSourceDir, samplesDestDir string) error {
24+
originalBytes, err := ioutil.ReadFile(filePath)
25+
if err != nil {
26+
return fmt.Errorf("failed to read YAML file: %w", err)
27+
}
28+
29+
var resource Resource
30+
err = yaml.Unmarshal(originalBytes, &resource)
31+
if err != nil {
32+
return fmt.Errorf("failed to unmarshal YAML: %w", err)
33+
}
34+
35+
if len(resource.Examples) == 0 {
36+
return nil
37+
}
38+
39+
// fmt.Printf("Found %d examples in %s for service '%s'\n", len(resource.Examples), filepath.Base(filePath), serviceName)
40+
41+
for _, example := range resource.Examples {
42+
if example.Name == "" {
43+
log.Printf("Skipping example with empty name in file: %s\n", filePath)
44+
continue
45+
}
46+
47+
// Determine the source file name. Use config_path if it exists,
48+
// otherwise fall back to the example's name.
49+
var sourceFileName string
50+
if example.ConfigPath != "" {
51+
sourceFileName = filepath.Base(example.ConfigPath)
52+
} else {
53+
sourceFileName = fmt.Sprintf("%s.tf.tmpl", example.Name)
54+
}
55+
56+
sourcePath := filepath.Join(examplesSourceDir, sourceFileName)
57+
58+
// Check if the source template file actually exists.
59+
if _, err := os.Stat(sourcePath); os.IsNotExist(err) {
60+
// Skip if the template doesn't exist.
61+
continue
62+
}
63+
64+
destDir := filepath.Join(samplesDestDir, serviceName)
65+
destPath := filepath.Join(destDir, sourceFileName)
66+
67+
err := copyFile(sourcePath, destPath)
68+
if err != nil {
69+
// Log the error but don't stop processing other examples.
70+
log.Printf("Failed to copy '%s' to '%s': %v\n", sourcePath, destPath, err)
71+
} else {
72+
// fmt.Printf(" - Copied '%s'\n", sourceFileName)
73+
fmt.Print()
74+
}
75+
}
76+
77+
return nil
78+
}
79+
80+
func copyFile(src, dst string) error {
81+
input, err := ioutil.ReadFile(src)
82+
if err != nil {
83+
return fmt.Errorf("could not read source file %s: %w", src, err)
84+
}
85+
86+
// Switch existing Vars to ResourceIdVars
87+
output := strings.ReplaceAll(string(input), "$.Vars", "$.ResourceIdVars")
88+
89+
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
90+
return fmt.Errorf("could not create destination directory for %s: %w", dst, err)
91+
}
92+
93+
// Get the source file's permissions to apply to the destination file.
94+
info, err := os.Stat(src)
95+
if err != nil {
96+
return fmt.Errorf("could not stat source file %s: %w", src, err)
97+
}
98+
99+
// Write the modified content to the destination file.
100+
err = ioutil.WriteFile(dst, []byte(output), info.Mode())
101+
if err != nil {
102+
return fmt.Errorf("could not write to destination file %s: %w", dst, err)
103+
}
104+
105+
return nil
106+
}
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
module github.com/GoogleCloudPlatform/magic-modules/tools/resource-template-converter
2+
3+
go 1.23.2
4+
5+
require (
6+
github.com/spf13/cobra v1.10.1
7+
gopkg.in/yaml.v2 v2.4.0
8+
)
9+
10+
require (
11+
github.com/inconshreveable/mousetrap v1.1.0 // indirect
12+
github.com/spf13/pflag v1.0.9 // indirect
13+
gopkg.in/yaml.v3 v3.0.1 // indirect
14+
)
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
2+
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
3+
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
4+
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
5+
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
6+
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
7+
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
8+
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
9+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
10+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
11+
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
12+
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
13+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
14+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package main
2+
3+
import (
4+
"github.com/GoogleCloudPlatform/magic-modules/tools/resource-template-converter/cmd"
5+
)
6+
7+
func main() {
8+
cmd.Execute()
9+
}

0 commit comments

Comments
 (0)