diff --git a/desktop/src-tauri/src/types/generated/flowfile.rs b/desktop/src-tauri/src/types/generated/flowfile.rs index 5763ca08..f4a221cf 100644 --- a/desktop/src-tauri/src/types/generated/flowfile.rs +++ b/desktop/src-tauri/src/types/generated/flowfile.rs @@ -2203,6 +2203,11 @@ impl ::std::default::Default for ExecutableVerb { #[doc = " }"] #[doc = " },"] #[doc = " \"fromFile\": {"] +#[doc = " \"description\": \"DEPRECATED: Use `imports` instead\","] +#[doc = " \"default\": [],"] +#[doc = " \"$ref\": \"#/definitions/FromFile\""] +#[doc = " },"] +#[doc = " \"imports\": {"] #[doc = " \"default\": [],"] #[doc = " \"$ref\": \"#/definitions/FromFile\""] #[doc = " },"] @@ -2236,8 +2241,11 @@ pub struct FlowFile { pub description_file: ::std::string::String, #[serde(default, skip_serializing_if = "::std::vec::Vec::is_empty")] pub executables: ::std::vec::Vec, + #[doc = "DEPRECATED: Use `imports` instead"] #[serde(rename = "fromFile", default = "defaults::flow_file_from_file")] pub from_file: FromFile, + #[serde(default = "defaults::flow_file_imports")] + pub imports: FromFile, #[doc = "The namespace to be given to all executables in the flow file.\nIf not set, the executables in the file will be grouped into the root (*) namespace. \nNamespaces can be reused across multiple flow files.\n\nNamespaces are used to reference executables in the CLI using the format `workspace:namespace/name`.\n"] #[serde(default)] pub namespace: ::std::string::String, @@ -2259,6 +2267,7 @@ impl ::std::default::Default for FlowFile { description_file: Default::default(), executables: Default::default(), from_file: defaults::flow_file_from_file(), + imports: defaults::flow_file_imports(), namespace: Default::default(), tags: Default::default(), visibility: Default::default(), @@ -3813,6 +3822,7 @@ pub mod builder { executables: ::std::result::Result<::std::vec::Vec, ::std::string::String>, from_file: ::std::result::Result, + imports: ::std::result::Result, namespace: ::std::result::Result<::std::string::String, ::std::string::String>, tags: ::std::result::Result<::std::vec::Vec<::std::string::String>, ::std::string::String>, visibility: ::std::result::Result< @@ -3827,6 +3837,7 @@ pub mod builder { description_file: Ok(Default::default()), executables: Ok(Default::default()), from_file: Ok(super::defaults::flow_file_from_file()), + imports: Ok(super::defaults::flow_file_imports()), namespace: Ok(Default::default()), tags: Ok(Default::default()), visibility: Ok(Default::default()), @@ -3877,6 +3888,16 @@ pub mod builder { .map_err(|e| format!("error converting supplied value for from_file: {}", e)); self } + pub fn imports(mut self, value: T) -> Self + where + T: ::std::convert::TryInto, + T::Error: ::std::fmt::Display, + { + self.imports = value + .try_into() + .map_err(|e| format!("error converting supplied value for imports: {}", e)); + self + } pub fn namespace(mut self, value: T) -> Self where T: ::std::convert::TryInto<::std::string::String>, @@ -3916,6 +3937,7 @@ pub mod builder { description_file: value.description_file?, executables: value.executables?, from_file: value.from_file?, + imports: value.imports?, namespace: value.namespace?, tags: value.tags?, visibility: value.visibility?, @@ -3929,6 +3951,7 @@ pub mod builder { description_file: Ok(value.description_file), executables: Ok(value.executables), from_file: Ok(value.from_file), + imports: Ok(value.imports), namespace: Ok(value.namespace), tags: Ok(value.tags), visibility: Ok(value.visibility), @@ -3992,4 +4015,7 @@ pub mod defaults { pub(super) fn flow_file_from_file() -> super::FromFile { super::FromFile(vec![]) } + pub(super) fn flow_file_imports() -> super::FromFile { + super::FromFile(vec![]) + } } diff --git a/desktop/src/types/generated/flowfile.ts b/desktop/src/types/generated/flowfile.ts index 1d84280f..54ad3fbf 100644 --- a/desktop/src/types/generated/flowfile.ts +++ b/desktop/src/types/generated/flowfile.ts @@ -39,9 +39,13 @@ export type CommonTags = string[]; */ export type CommonVisibility = 'public' | 'private' | 'internal' | 'hidden'; /** - * A list of `.sh` files to convert into generated executables in the file's executable group. + * DEPRECATED: Use `imports` instead */ export type FromFile = string[]; +/** + * A list of `.sh` files to convert into generated executables in the file's executable group. + */ +export type FromFile1 = string[]; /** * Configuration for a group of Flow CLI executables. The file must have the extension `.flow`, `.flow.yaml`, or `.flow.yml` @@ -62,6 +66,7 @@ export interface FlowFile { descriptionFile?: string; executables?: Executable[]; fromFile?: FromFile; + imports?: FromFile1; /** * The namespace to be given to all executables in the flow file. * If not set, the executables in the file will be grouped into the root (*) namespace. diff --git a/docs/guide/executables.md b/docs/guide/executables.md index 78dcb987..4f6ffa81 100644 --- a/docs/guide/executables.md +++ b/docs/guide/executables.md @@ -356,17 +356,31 @@ Current time: {{ .timestamp }} - `templateFile`: Markdown template file (required) - `templateDataFile`: JSON/YAML data file -## Generated Executables +## Importing Executables -Generate executables from shell scripts with special comments: +Generate executables from shell scripts, Makefiles, package.json scripts, or docker-compose services: ```yaml # In flowfile -fromFile: +imports: - "scripts/deploy.sh" - "scripts/backup.sh" + - "Makefile" + - "frontend/package.json" + - "docker-compose.yaml" ``` +All imported executables are automatically tagged with `generated` and their file type (e.g., `docker-compose`, `makefile`, `package.json`). + + + + +#### **Shell Scripts (.sh files)** + +Shell scripts are imported as single executables with the script's filename as the name and `exec` as the default verb. + +You can use special comments to override executable metadata: + ```bash #!/bin/bash # scripts/deploy.sh @@ -394,6 +408,81 @@ kubectl apply -f k8s/ # ``` +#### **Makefiles** + +Makefile targets are imported as executables with a verb and name that best represents the target. + +```makefile +# Makefile + +# f:name=app f:verb=build f:description="Build the application" +build: + go build -o bin/app ./cmd/app + +# Run all tests +test: + go test ./... + +# f:visibility=internal +clean: + rm -rf bin/ +``` + +The same comment parsing syntax works in Makefiles - use `# f:key=value` to override executable configuration. + +#### **Package.json Scripts** + +NPM scripts from package.json are imported as executables with a verb and name that best represents the script name. + +```json +{ + "scripts": { + "build": "webpack --mode production", + "test": "jest", + "dev": "webpack-dev-server --mode development", + "lint": "eslint src/" + } +} +``` + +This creates executables like: +- `build` - Runs the build script +- `test` - Runs the test script +- `start dev` - Runs the development server +- `lint` - Runs the linter + +#### **Docker Compose Services** + +Docker Compose files are imported to create executables for managing services: + +```yaml +# docker-compose.yml +version: '3.8' +services: + app: + build: . + ports: + - "3000:3000" + + db: + image: postgres:13 + environment: + POSTGRES_DB: myapp + + redis: + image: redis:6 +``` + +This creates executables like: +- `start app` - Start the app service +- `start db` - Start the database service +- `start redis` - Start the Redis service +- `start` (alias: all, services) - Start all services +- `stop` (alias: all, services) - Stop all services +- `build app` - Build the app service (if build config exists) + + + ## Executable References Reference other executables to build modular workflows: diff --git a/docs/schemas/flowfile_schema.json b/docs/schemas/flowfile_schema.json index ca33a19d..417d60e4 100644 --- a/docs/schemas/flowfile_schema.json +++ b/docs/schemas/flowfile_schema.json @@ -628,6 +628,11 @@ } }, "fromFile": { + "$ref": "#/definitions/FromFile", + "description": "DEPRECATED: Use `imports` instead", + "default": [] + }, + "imports": { "$ref": "#/definitions/FromFile", "default": [] }, diff --git a/docs/types/flowfile.md b/docs/types/flowfile.md index 4ec9a5fb..fcbe4a4d 100644 --- a/docs/types/flowfile.md +++ b/docs/types/flowfile.md @@ -17,7 +17,8 @@ in order to be discovered by the CLI. It's configuration is used to define a gro | `description` | A description of the executables defined within the flow file. This description will used as a shared description for all executables in the flow file. | `string` | | | | `descriptionFile` | A path to a markdown file that contains the description of the executables defined within the flow file. | `string` | | | | `executables` | | `array` ([Executable](#Executable)) | [] | | -| `fromFile` | | [FromFile](#FromFile) | [] | | +| `fromFile` | DEPRECATED: Use `imports` instead | [FromFile](#FromFile) | [] | | +| `imports` | | [FromFile](#FromFile) | [] | | | `namespace` | The namespace to be given to all executables in the flow file. If not set, the executables in the file will be grouped into the root (*) namespace. Namespaces can be reused across multiple flow files. Namespaces are used to reference executables in the CLI using the format `workspace:namespace/name`. | `string` | | | | `tags` | Tags to be applied to all executables defined within the flow file. | `array` (`string`) | [] | | | `visibility` | | [CommonVisibility](#CommonVisibility) | | | diff --git a/internal/cache/executable_generator.go b/internal/cache/executable_generator.go deleted file mode 100644 index 096a57d4..00000000 --- a/internal/cache/executable_generator.go +++ /dev/null @@ -1,88 +0,0 @@ -package cache - -import ( - "path/filepath" - "strings" - "time" - - "github.com/flowexec/tuikit/io" - "github.com/pkg/errors" - - "github.com/flowexec/flow/internal/fileparser" - "github.com/flowexec/flow/internal/utils" - "github.com/flowexec/flow/types/executable" -) - -const generatedTag = "generated" - -func generatedExecutables( - logger io.Logger, wsName string, flowFile *executable.FlowFile, -) (executable.ExecutableList, error) { - executables := make(executable.ExecutableList, 0) - wsPath := flowFile.WorkspacePath() - flowFilePath := flowFile.ConfigPath() - flowFileNs := flowFile.Namespace - files := flowFile.FromFile - - for _, file := range files { - expandedFile := utils.ExpandPath(logger, file, filepath.Dir(flowFilePath), nil) - exec, err := executablesFromFile(logger, file, expandedFile) - if err != nil { - return nil, err - } - exec.SetContext(wsName, wsPath, flowFileNs, flowFilePath) - exec.SetInheritedFields(flowFile) - executables = append(executables, exec) - } - - return executables, nil -} - -func executablesFromFile(logger io.Logger, fileBase, filePath string) (*executable.Executable, error) { - configMap, err := fileparser.ExecConfigMapFromFile(logger, filePath) - if err != nil { - return nil, err - } - - exec := &executable.Executable{ - Verb: executable.Verb("exec"), - Name: filepath.Base(fileBase), - Exec: &executable.ExecExecutableType{ - File: fileBase, - }, - } - for key, value := range configMap { - switch key { - case fileparser.TimeoutConfigurationKey: - dur, err := time.ParseDuration(value) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse timeout duration %s", value) - } - exec.Timeout = &dur - case fileparser.VerbConfigurationKey: - exec.Verb = executable.Verb(value) - case fileparser.NameConfigurationKey: - exec.Name = value - case fileparser.VisibilityConfigurationKey: - v := executable.ExecutableVisibility(value) - exec.Visibility = &v - case fileparser.DescriptionConfigurationKey: - exec.Description = value - case fileparser.AliasConfigurationKey: - values := make([]string, 0) - for _, v := range strings.Split(value, fileparser.InternalListSeparator) { - values = append(values, strings.TrimSpace(v)) - } - exec.Aliases = values - case fileparser.TagConfigurationKey: - values := make([]string, 0) - for _, v := range strings.Split(value, fileparser.InternalListSeparator) { - values = append(values, strings.TrimSpace(v)) - } - exec.Tags = values - } - } - - exec.Tags = append(exec.Tags, generatedTag) - return exec, nil -} diff --git a/internal/cache/executables_cache.go b/internal/cache/executables_cache.go index 2fb07eac..dd09e807 100644 --- a/internal/cache/executables_cache.go +++ b/internal/cache/executables_cache.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "gopkg.in/yaml.v3" + "github.com/flowexec/flow/internal/fileparser" "github.com/flowexec/flow/internal/filesystem" "github.com/flowexec/flow/types/common" "github.com/flowexec/flow/types/executable" @@ -69,8 +70,8 @@ func (c *ExecutableCacheImpl) Update(logger io.Logger) error { //nolint:gocognit continue } for _, flowFile := range flowFiles { - if len(flowFile.FromFile) > 0 { - generated, err := generatedExecutables(logger, name, flowFile) + if len(flowFile.FromFile) > 0 || len(flowFile.Imports) > 0 { + generated, err := fileparser.ExecutablesFromImports(logger, name, flowFile) if err != nil { logger.Errorx( "failed to generate executables from files", @@ -160,7 +161,7 @@ func (c *ExecutableCacheImpl) GetExecutableByRef(logger io.Logger, ref executabl cfg.SetDefaults() cfg.SetContext(wsInfo.WorkspaceName, wsInfo.WorkspacePath, cfgPath) - generated, err := generatedExecutables(logger, wsInfo.WorkspaceName, cfg) + generated, err := fileparser.ExecutablesFromImports(logger, wsInfo.WorkspaceName, cfg) if err != nil { logger.Warnx( "failed to generate executables from files", @@ -206,7 +207,7 @@ func (c *ExecutableCacheImpl) GetExecutableList(logger io.Logger) (executable.Ex cfg.SetDefaults() cfg.SetContext(wsInfo.WorkspaceName, wsInfo.WorkspacePath, cfgPath) - generated, err := generatedExecutables(logger, wsInfo.WorkspaceName, cfg) + generated, err := fileparser.ExecutablesFromImports(logger, wsInfo.WorkspaceName, cfg) if err != nil { logger.Warnx( "failed to generate executables from files", diff --git a/internal/fileparser/configmap.go b/internal/fileparser/configmap.go new file mode 100644 index 00000000..5970bd48 --- /dev/null +++ b/internal/fileparser/configmap.go @@ -0,0 +1,229 @@ +package fileparser + +import ( + "fmt" + "regexp" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/flowexec/flow/types/executable" +) + +const ( + TimeoutConfigurationKey = "timeout" + VerbConfigurationKey = "verb" + NameConfigurationKey = "name" + AliasConfigurationKey = "alias" + DescriptionConfigurationKey = "description" + VisibilityConfigurationKey = "visibility" + TagConfigurationKey = "tag" + + InternalListSeparator = "," + + commentPrefix = "# " + keyPrefix = "f:" + multiLineKeyPrefix = "f|" + descriptionSeparator = "\n" + descriptionAlias = "desc" +) + +var multiLineDescriptionTag = fmt.Sprintf("<%s%s>", multiLineKeyPrefix, DescriptionConfigurationKey) + +func ExtractExecConfigMap(data, prefix string) (map[string]string, error) { + configMap := make(map[string]string) + processingMultiLineDescription := false + for _, line := range strings.Split(data, "\n") { + isComment := strings.HasPrefix(line, strings.TrimSpace(prefix)) + if trimmedLine := strings.TrimSpace(line); !isComment && trimmedLine != "" { + // If the line is not a comment or empty, break out of the loop. + // All flow executable configuration should be at the top of the file. + break + } + + line = strings.TrimPrefix(line, commentPrefix) + if processingMultiLineDescription = processMultiLineDescription( + line, configMap, processingMultiLineDescription, + ); processingMultiLineDescription { + continue + } + + cfg, err := parseConfigurations(line) + if err != nil { + return nil, fmt.Errorf("unable to extract executable configurations: %w", err) + } + + for key, value := range cfg { + switch key { + case TimeoutConfigurationKey, VerbConfigurationKey, NameConfigurationKey, VisibilityConfigurationKey: + configMap[key] = strings.TrimSpace(value) + case DescriptionConfigurationKey, descriptionAlias: + if existingValue, ok := configMap[DescriptionConfigurationKey]; ok { + configMap[DescriptionConfigurationKey] = + fmt.Sprintf("%s%s%s", existingValue, descriptionSeparator, value) + } else { + configMap[DescriptionConfigurationKey] = value + } + case AliasConfigurationKey, TagConfigurationKey: + value = strings.TrimSpace(value) + if existingValue, ok := configMap[key]; ok { + configMap[key] = fmt.Sprintf("%s%s%s", existingValue, InternalListSeparator, value) + } else { + configMap[key] = value + } + } + } + } + return configMap, nil +} + +func applyConfig(exec *executable.Executable, key, value string) error { + switch key { + case TimeoutConfigurationKey: + dur, err := time.ParseDuration(value) + if err != nil { + return errors.Wrapf(err, "unable to parse timeout duration %s", value) + } + exec.Timeout = &dur + case VerbConfigurationKey: + exec.Verb = executable.Verb(value) + case NameConfigurationKey: + exec.Name = value + case VisibilityConfigurationKey: + v := executable.ExecutableVisibility(value) + exec.Visibility = &v + case DescriptionConfigurationKey: + exec.Description = value + case AliasConfigurationKey: + values := make([]string, 0) + for _, v := range strings.Split(value, InternalListSeparator) { + values = append(values, strings.TrimSpace(v)) + } + exec.Aliases = values + case TagConfigurationKey: + values := make([]string, 0) + for _, v := range strings.Split(value, InternalListSeparator) { + values = append(values, strings.TrimSpace(v)) + } + exec.Tags = values + } + return nil +} + +// This regex is used to extract all flow configurations from a shell script. +// The regex matches the following: +// - f:= +// - f:="" +// - f:='' +// - f:="",='',key= +// - f:exampleKey='Example value\, with comma' +// - f:exampleKey="Example value\, with comma" +// - f:exampleKey="\'Example value with escaped quotes\'" +// - f:exampleKey='\"Example value with escaped quotes\"' +// and so on. +var flowConfigRegex = regexp.MustCompile(`f:\w+=(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|[^, ]+)`) + +func processMultiLineDescription(line string, configMap map[string]string, processing bool) bool { + multiLineDescPrefixed := strings.HasPrefix(line, multiLineDescriptionTag) + multiLineDescSuffixed := strings.HasSuffix(line, multiLineDescriptionTag) + existingValue := configMap[DescriptionConfigurationKey] + switch { + case processing && multiLineDescPrefixed: + processing = false + case processing && multiLineDescSuffixed: + processing = false + line = strings.TrimSuffix(line, multiLineDescriptionTag) + if existingValue != "" { + configMap[DescriptionConfigurationKey] = fmt.Sprintf( + "%s%s%s", + existingValue, + descriptionSeparator, + line, + ) + } else { + configMap[DescriptionConfigurationKey] = line + } + case processing: + if existingValue != "" { + configMap[DescriptionConfigurationKey] = fmt.Sprintf( + "%s%s%s", + existingValue, + descriptionSeparator, + line, + ) + } else { + configMap[DescriptionConfigurationKey] = line + } + case multiLineDescPrefixed: + processing = true + line = strings.TrimPrefix(line, multiLineDescriptionTag) + if trimmedLine := strings.TrimSpace(line); trimmedLine == "" { + return processing + } else if existingValue != "" { + configMap[DescriptionConfigurationKey] = fmt.Sprintf( + "%s%s%s", + existingValue, + descriptionSeparator, + line, + ) + } else { + configMap[DescriptionConfigurationKey] = line + } + } + return processing +} + +func parseConfigurations(line string) (map[string]string, error) { + configMap := make(map[string]string) + matches := flowConfigRegex.FindAllString(line, -1) + for _, match := range matches { + split := strings.SplitN(match, "=", 2) + key := strings.TrimSpace(strings.ToLower(strings.TrimPrefix(split[0], keyPrefix))) + value := split[1] + // Removing quotes if present + if (value[0] == '"' && value[len(value)-1] == '"') || (value[0] == '\'' && value[len(value)-1] == '\'') { + value = value[1 : len(value)-1] + } + // Replace escaped characters + value = strings.ReplaceAll(value, `\"`, `"`) + value = strings.ReplaceAll(value, `\'`, `'`) + value = strings.ReplaceAll(value, `\\`, `\`) + value = strings.ReplaceAll(value, `\,`, `,`) + + if !validateKey(key) { + return nil, fmt.Errorf("invalid key (%s)", key) + } + + switch key { + case AliasConfigurationKey, TagConfigurationKey: + value = strings.TrimSpace(value) + if existingValue, ok := configMap[key]; ok { + configMap[key] = fmt.Sprintf("%s%s%s", existingValue, InternalListSeparator, value) + } else { + configMap[key] = value + } + case DescriptionConfigurationKey, descriptionAlias: + if existingValue, ok := configMap[DescriptionConfigurationKey]; ok { + configMap[DescriptionConfigurationKey] = + fmt.Sprintf("%s%s%s", existingValue, descriptionSeparator, value) + } else { + configMap[DescriptionConfigurationKey] = value + } + default: + configMap[key] = strings.TrimSpace(value) + } + } + return configMap, nil +} + +func validateKey(key string) bool { + switch key { + case VerbConfigurationKey, NameConfigurationKey, DescriptionConfigurationKey, descriptionAlias, + AliasConfigurationKey, VisibilityConfigurationKey, TagConfigurationKey, + TimeoutConfigurationKey: + return true + default: + return false + } +} diff --git a/internal/fileparser/file_parser_test.go b/internal/fileparser/configmap_test.go similarity index 56% rename from internal/fileparser/file_parser_test.go rename to internal/fileparser/configmap_test.go index abb93040..d82f4fbc 100644 --- a/internal/fileparser/file_parser_test.go +++ b/internal/fileparser/configmap_test.go @@ -3,64 +3,21 @@ package fileparser_test import ( "os" "path/filepath" - "testing" - "github.com/flowexec/tuikit/io/mocks" + "github.com/flowexec/flow/internal/fileparser" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "go.uber.org/mock/gomock" - - "github.com/flowexec/flow/internal/fileparser" ) -func TestFileParser(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "FileParser Suite") -} - var _ = Describe("ExecConfigMapFromFile", func() { - var ( - ctrl *gomock.Controller - mockLogger *mocks.MockLogger - ) - BeforeEach(func() { - ctrl = gomock.NewController(GinkgoT()) - mockLogger = mocks.NewMockLogger(ctrl) - }) - - DescribeTable("should error when the file is invalid", func(file string) { - mockLogger.EXPECT().Debugf(gomock.Any(), gomock.Any()).AnyTimes() - wd, err := os.Getwd() - Expect(err).ToNot(HaveOccurred()) - filePath := filepath.Join(wd, "testdata", file) - result, err := fileparser.ExecConfigMapFromFile(mockLogger, filePath) - Expect(err).To(HaveOccurred()) - Expect(result).To(BeNil()) - }, - Entry("non-shell file", "invalidfile"), - Entry("dir instead of file", "invaliddir.sh"), - Entry("file without configs", "empty.sh"), - Entry("non-existent file", "nonexistent.sh"), - ) - - It("should log a warning when configuration key is not recognized", func() { - mockLogger.EXPECT().Debugf(gomock.Any(), gomock.Any()).AnyTimes() - mockLogger.EXPECT().Warnf(gomock.Any(), gomock.Any()).Times(1) - wd, err := os.Getwd() - Expect(err).ToNot(HaveOccurred()) - filePath := filepath.Join(wd, "testdata", "unknownkey.sh") - result, err := fileparser.ExecConfigMapFromFile(mockLogger, filePath) - Expect(err).ToNot(HaveOccurred()) - Expect(result).ToNot(BeNil()) - }) - DescribeTable("should correctly parse configurations", func(file string, expected map[string]string) { - mockLogger.EXPECT().Debugf(gomock.Any(), gomock.Any()).AnyTimes() - wd, err := os.Getwd() + filePath := filepath.Join("testdata", file) + fileBytes, err := os.ReadFile(filepath.Clean(filePath)) Expect(err).ToNot(HaveOccurred()) - filePath := filepath.Join(wd, "testdata", file) - result, err := fileparser.ExecConfigMapFromFile(mockLogger, filePath) + + result, err := fileparser.ExtractExecConfigMap(string(fileBytes), "# ") Expect(err).ToNot(HaveOccurred()) Expect(result).To(Equal(expected)) }, @@ -68,8 +25,8 @@ var _ = Describe("ExecConfigMapFromFile", func() { "simple key-value pairs", "simple.sh", map[string]string{ - fileparser.NameConfigurationKey: "value1", - fileparser.VerbConfigurationKey: "value2", + fileparser.NameConfigurationKey: "hello", + fileparser.VerbConfigurationKey: "show", }), Entry( "values with spaces in quotes", diff --git a/internal/fileparser/docker_compose_parser.go b/internal/fileparser/docker_compose_parser.go new file mode 100644 index 00000000..a8aaa457 --- /dev/null +++ b/internal/fileparser/docker_compose_parser.go @@ -0,0 +1,86 @@ +package fileparser + +import ( + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + + "github.com/flowexec/flow/types/executable" +) + +type composeFile struct { + Services map[string]any `yaml:"services"` +} + +var composeTags = []string{generatedTag, "docker-compose"} + +// ExecutablesFromDockerCompose parses a docker-compose.yml and returns list of Executables for the services +func ExecutablesFromDockerCompose(wsPath, path string) (executable.ExecutableList, error) { + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return nil, fmt.Errorf("failed to open docker compose file: %w", err) + } + defer f.Close() + + var cf composeFile + dec := yaml.NewDecoder(f) + if err := dec.Decode(&cf); err != nil { + return nil, fmt.Errorf("failed to decode docker compose file: %w", err) + } + + execs := make(executable.ExecutableList, 0) + dir := executable.Directory(shortenWsPath(wsPath, filepath.Dir(path))) + // Per-service start/build + for svc, data := range cf.Services { + execs = append(execs, &executable.Executable{ + Name: svc, + Verb: executable.VerbStart, + Tags: composeTags, + Description: fmt.Sprintf("Start service %s via docker-compose", svc), + Exec: &executable.ExecExecutableType{ + Dir: dir, + Cmd: fmt.Sprintf("docker-compose up %s", svc), + }, + }) + + dataMap, ok := data.(map[string]any) + if ok && dataMap["build"] != nil { + execs = append(execs, &executable.Executable{ + Name: svc, + Verb: executable.VerbBuild, + Tags: composeTags, + Description: fmt.Sprintf("Build service %s via docker-compose", svc), + Exec: &executable.ExecExecutableType{ + Dir: dir, + Cmd: fmt.Sprintf("docker-compose build %s", svc), + }, + }) + } + } + + // start and stop all + execs = append(execs, &executable.Executable{ + Verb: executable.VerbStart, + Aliases: []string{"all", "services"}, + Tags: composeTags, + Description: "Start all services via docker-compose", + Exec: &executable.ExecExecutableType{ + Dir: dir, + Cmd: "docker-compose up", + }, + }) + execs = append(execs, &executable.Executable{ + Verb: executable.VerbStop, + Aliases: []string{"all", "services"}, + Tags: composeTags, + Description: "Stop all services via docker-compose", + Exec: &executable.ExecExecutableType{ + Dir: dir, + Cmd: "docker-compose down", + }, + }) + + return execs, nil +} diff --git a/internal/fileparser/docker_compose_parser_test.go b/internal/fileparser/docker_compose_parser_test.go new file mode 100644 index 00000000..8a589027 --- /dev/null +++ b/internal/fileparser/docker_compose_parser_test.go @@ -0,0 +1,43 @@ +package fileparser_test + +import ( + "fmt" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flowexec/flow/internal/fileparser" +) + +var _ = Describe("ExecutablesFromDockerCompose", func() { + const composePath = "testdata/docker-compose.yml" + + It("should parse docker-compose.yml", func() { + execs, err := fileparser.ExecutablesFromDockerCompose("", composePath) + Expect(err).NotTo(HaveOccurred()) + Expect(execs).To(HaveLen(6)) + + found := map[string]bool{ + "start": false, + "stop": false, + "start app": false, + "start db": false, + "start redis": false, + "build app": false, + } + + for _, e := range execs { + Expect(e.Exec).NotTo(BeNil()) + Expect(e.Exec.Cmd).To(ContainSubstring("docker-compose")) + + shortRef := strings.TrimSpace(fmt.Sprintf("%s %s", e.Verb, e.Name)) + if _, ok := found[shortRef]; ok { + found[shortRef] = true + } + } + for ref, found := range found { + Expect(found).To(BeTrue(), "executable %s not found", ref) + } + }) +}) diff --git a/internal/fileparser/fileparser.go b/internal/fileparser/fileparser.go new file mode 100644 index 00000000..4f46599d --- /dev/null +++ b/internal/fileparser/fileparser.go @@ -0,0 +1,95 @@ +package fileparser + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/flowexec/tuikit/io" + + "github.com/flowexec/flow/internal/utils" + "github.com/flowexec/flow/types/executable" +) + +const generatedTag = "generated" + +func ExecutablesFromImports( + logger io.Logger, wsName string, flowFile *executable.FlowFile, +) (executable.ExecutableList, error) { + executables := make(executable.ExecutableList, 0) + wsPath := flowFile.WorkspacePath() + flowFilePath := flowFile.ConfigPath() + flowFileNs := flowFile.Namespace + files := append(flowFile.FromFile, flowFile.Imports...) //nolint:gocritic + + for _, file := range files { + fn := filepath.Base(file) + expandedFile := utils.ExpandPath(logger, file, filepath.Dir(flowFilePath), nil) + + if info, err := os.Stat(expandedFile); err != nil { + logger.Error(err, fmt.Sprintf("unable to import executables from file %s", file)) + continue + } else if info.IsDir() { + logger.Errorx("unable to import executables", "err", fmt.Sprintf("%s is not a file", file)) + continue + } + + switch strings.ToLower(fn) { + case "package.json": + execs, err := ExecutablesFromPackageJSON(wsPath, expandedFile) + if err != nil { + logger.Error(err, fmt.Sprintf("unable to import executables from file (%s)", file)) + } + for _, exec := range execs { + exec.SetContext(wsName, wsPath, flowFileNs, flowFilePath) + exec.SetInheritedFields(flowFile) + executables = append(executables, exec) + } + case "makefile": + execs, err := ExecutablesFromMakefile(wsPath, expandedFile) + if err != nil { + logger.Error(err, fmt.Sprintf("unable to import executables from file (%s)", file)) + } + for _, exec := range execs { + exec.SetContext(wsName, wsPath, flowFileNs, flowFilePath) + exec.SetInheritedFields(flowFile) + executables = append(executables, exec) + } + case "docker-compose.yml", "docker-compose.yaml": + execs, err := ExecutablesFromDockerCompose(wsPath, expandedFile) + if err != nil { + logger.Error(err, fmt.Sprintf("unable to import executables from file (%s)", file)) + } + for _, exec := range execs { + exec.SetContext(wsName, wsPath, flowFileNs, flowFilePath) + exec.SetInheritedFields(flowFile) + executables = append(executables, exec) + } + default: + ext := filepath.Ext(fn) + if ext != ".sh" { + logger.Warnx("unable to import executables - unsupported file type", "file", file) + continue + } + exec, err := ExecutablesFromShFile(wsPath, expandedFile) + if err != nil { + logger.Error(err, fmt.Sprintf("unable to import executables from file (%s)", file)) + continue + } + exec.SetContext(wsName, wsPath, flowFileNs, flowFilePath) + exec.SetInheritedFields(flowFile) + executables = append(executables, exec) + } + } + + return executables, nil +} + +func shortenWsPath(wsPath string, path string) string { + if strings.HasPrefix(path, wsPath) { + return "//" + path[len(wsPath):] + } + + return path +} diff --git a/internal/fileparser/fileparser_test.go b/internal/fileparser/fileparser_test.go new file mode 100644 index 00000000..f98125fd --- /dev/null +++ b/internal/fileparser/fileparser_test.go @@ -0,0 +1,91 @@ +package fileparser_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/flowexec/tuikit/io/mocks" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + + "github.com/flowexec/flow/internal/fileparser" + "github.com/flowexec/flow/types/executable" +) + +func TestFileParser(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "FileParser Suite") +} + +var _ = Describe("ExecutablesFromImports", func() { + var ( + ctrl *gomock.Controller + mockLogger *mocks.MockLogger + flowFile *executable.FlowFile + ) + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + mockLogger = mocks.NewMockLogger(ctrl) + mockLogger.EXPECT().Debugf(gomock.Any(), gomock.Any()).AnyTimes() + + wd, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + ff := filepath.Join(wd, "testdata", "test"+executable.FlowFileExt) + flowFile = &executable.FlowFile{Imports: make(executable.FromFile, 0)} + flowFile.SetContext("ws", filepath.Join(wd, "testdata"), ff) + }) + + It("should return executables from imports", func() { + flowFile.Imports = append( + flowFile.Imports, + "Makefile", + "package.json", + "docker-compose.yml", + "complex.sh", + ) + + result, err := fileparser.ExecutablesFromImports(mockLogger, "ws", flowFile) + Expect(err).NotTo(HaveOccurred()) + Expect(len(result)).To(BeNumerically(">", 10)) + + for _, e := range result { + Expect(e.Exec).ToNot(BeNil()) + Expect(e.Exec.Dir).To(Equal(executable.Directory("//"))) + } + }) + + It("should log a warning for invalid file type", func() { + mockLogger.EXPECT().Warnx(gomock.Any(), "file", "invalidfile").AnyTimes() + flowFile.Imports = append(flowFile.Imports, "invalidfile") + result, err := fileparser.ExecutablesFromImports(mockLogger, "ws", flowFile) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(BeEmpty()) + }) + + It("should log an error for dir instead of file", func() { + mockLogger.EXPECT().Errorx(gomock.Any(), "err", "invaliddir is not a file").AnyTimes() + flowFile.Imports = append(flowFile.Imports, "invaliddir") + result, err := fileparser.ExecutablesFromImports(mockLogger, "ws", flowFile) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(BeEmpty()) + }) + + It("should log an error for non-existent file", func() { + mockLogger.EXPECT().Error(gomock.Any(), gomock.Any()).AnyTimes() + flowFile.Imports = append(flowFile.Imports, "nonexistent.sh") + result, err := fileparser.ExecutablesFromImports(mockLogger, "ws", flowFile) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(BeEmpty()) + }) + + It("should log an error when configuration key is not recognized", func() { + mockLogger.EXPECT().Error(gomock.Any(), gomock.Any()).AnyTimes() + flowFile.Imports = append(flowFile.Imports, "unknownkey.sh") + result, err := fileparser.ExecutablesFromImports(mockLogger, "ws", flowFile) + Expect(err).ToNot(HaveOccurred()) + Expect(result).ToNot(BeNil()) + }) +}) diff --git a/internal/fileparser/makefile_parser.go b/internal/fileparser/makefile_parser.go new file mode 100644 index 00000000..d24a4995 --- /dev/null +++ b/internal/fileparser/makefile_parser.go @@ -0,0 +1,106 @@ +package fileparser + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/flowexec/flow/types/executable" +) + +type makeTarget struct { + name string + description string +} + +// e.g. "target: dep1 dep2" +var ( + targetLine = regexp.MustCompile(`^([a-zA-Z0-9_.-]+)\s*:(.*)$`) + makeTags = []string{generatedTag, "make"} +) + +// ExecutablesFromMakefile parses a Makefile and returns a list of Executables for each makeTarget +func ExecutablesFromMakefile(wsPath, path string) (executable.ExecutableList, error) { + file, err := os.Open(filepath.Clean(path)) + if err != nil { + return nil, fmt.Errorf("failed to open Makefile: %w", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + targets := make(map[string]*makeTarget) + var lastComment string + + for scanner.Scan() { + line := scanner.Text() + trim := strings.TrimSpace(line) + if trim == "" { + lastComment = "" + continue + } + if strings.HasPrefix(trim, "#") { + lastComment = appendComment(lastComment, strings.TrimSpace(strings.TrimPrefix(trim, "#"))) + continue + } + if m := targetLine.FindStringSubmatch(line); m != nil { + name := m[1] + + // Skip special targets and pattern rules + // TODO: add support for these targets + if strings.HasPrefix(name, ".") || strings.Contains(name, "%") { + continue + } + + targets[name] = &makeTarget{name: name, description: lastComment} + lastComment = "" + } + } + + execs := make(executable.ExecutableList, 0, len(targets)) + dir := executable.Directory(shortenWsPath(wsPath, filepath.Dir(path))) + + for _, t := range targets { + verb := InferVerb(t.name) + execName := NormalizeName(t.name, verb.String()) + e := &executable.Executable{ + Name: execName, + Verb: verb, + Description: t.description, + Tags: makeTags, + Exec: &executable.ExecExecutableType{ + Dir: dir, + Cmd: fmt.Sprintf("make %s", t.name), + }, + } + + configMap, err := ExtractExecConfigMap(t.description, "") + if err != nil { + return nil, err + } + + if len(configMap) != 0 { + e.Description = "" + for key, value := range configMap { + if err := applyConfig(e, key, value); err != nil { + return nil, err + } + } + } + + execs = append(execs, e) + } + return execs, nil +} + +func appendComment(s string, comment string) string { + if s == "" { + return comment + } + if comment != "" { + return s + "\n" + comment + } + return s +} diff --git a/internal/fileparser/makefile_parser_test.go b/internal/fileparser/makefile_parser_test.go new file mode 100644 index 00000000..eac8d7dc --- /dev/null +++ b/internal/fileparser/makefile_parser_test.go @@ -0,0 +1,49 @@ +package fileparser_test + +import ( + "fmt" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flowexec/flow/internal/fileparser" +) + +var _ = Describe("ExecutablesFromMakefile", func() { + const makefile = "testdata/Makefile" + + It("should parse Makefile", func() { + execs, err := fileparser.ExecutablesFromMakefile("", makefile) + Expect(err).NotTo(HaveOccurred()) + Expect(execs).To(HaveLen(4)) + + found := map[string]bool{ + "build": false, + "test": false, + "deploy": false, + "run program": false, + } + expectedDesc := map[string]string{ + "build": "Build the application binary", + "test": "Run all tests with coverage", + "deploy": "Deploy to production environment\nDepends on build and test", + "run program": "Run main.go", + } + + for _, e := range execs { + Expect(e.Exec).NotTo(BeNil()) + Expect(e.Exec.Cmd).To(ContainSubstring("make")) + + shortRef := strings.TrimSpace(fmt.Sprintf("%s %s", e.Verb, e.Name)) + if _, ok := found[shortRef]; ok { + found[shortRef] = true + } + Expect(e.Description).To(Equal(expectedDesc[shortRef])) + } + + for ref, found := range found { + Expect(found).To(BeTrue(), "executable %s not found", ref) + } + }) +}) diff --git a/internal/fileparser/package_json_parser.go b/internal/fileparser/package_json_parser.go new file mode 100644 index 00000000..30a6ed26 --- /dev/null +++ b/internal/fileparser/package_json_parser.go @@ -0,0 +1,63 @@ +package fileparser + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/flowexec/flow/types/executable" +) + +type packageJSON struct { + Scripts map[string]string `json:"scripts"` +} + +var packageJSONTags = []string{generatedTag, "npm"} + +// ExecutablesFromPackageJSON parses package.json scripts and returns a list of Executables for them +func ExecutablesFromPackageJSON(wsPath, path string) (executable.ExecutableList, error) { + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return nil, fmt.Errorf("failed to open package.json: %w", err) + } + defer f.Close() + + var pkg packageJSON + dec := json.NewDecoder(f) + if err := dec.Decode(&pkg); err != nil { + return nil, fmt.Errorf("failed to decode package.json: %w", err) + } + + execs := make(executable.ExecutableList, 0) + dir := executable.Directory(shortenWsPath(wsPath, filepath.Dir(path))) + + // default npm install + execs = append(execs, &executable.Executable{ + Verb: executable.VerbInstall, + Aliases: []string{"npm"}, + Description: "Install npm dependencies", + Tags: packageJSONTags, + Exec: &executable.ExecExecutableType{ + Dir: dir, + Cmd: "npm install", + }, + }) + + for name, scriptCmd := range pkg.Scripts { + verb := InferVerb(name) + execName := NormalizeName(name, verb.String()) + e := &executable.Executable{ + Verb: verb, + Name: execName, + Description: fmt.Sprintf("Run npm script %s:\n`%s`", name, scriptCmd), + Tags: packageJSONTags, + Exec: &executable.ExecExecutableType{ + Dir: dir, + Cmd: fmt.Sprintf("npm run %s", name), + }, + } + execs = append(execs, e) + } + return execs, nil +} diff --git a/internal/fileparser/package_json_parser_test.go b/internal/fileparser/package_json_parser_test.go new file mode 100644 index 00000000..506b4c99 --- /dev/null +++ b/internal/fileparser/package_json_parser_test.go @@ -0,0 +1,44 @@ +package fileparser_test + +import ( + "fmt" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flowexec/flow/internal/fileparser" +) + +var _ = Describe("ExecutablesFromPackageJSON", func() { + const pkgPath = "testdata/package.json" + + It("should parse executables from package.json", func() { + execs, err := fileparser.ExecutablesFromPackageJSON("", pkgPath) + Expect(err).NotTo(HaveOccurred()) + Expect(execs).To(HaveLen(7)) + + found := map[string]bool{ + "install": false, + "start dev": false, + "build": false, + "test": false, + "test watch": false, + "lint": false, + "start preview": false, + } + + for _, e := range execs { + Expect(e.Exec).NotTo(BeNil()) + Expect(e.Exec.Cmd).To(ContainSubstring("npm")) + + shortRef := strings.TrimSpace(fmt.Sprintf("%s %s", e.Verb, e.Name)) + if _, ok := found[shortRef]; ok { + found[shortRef] = true + } + } + for ref, found := range found { + Expect(found).To(BeTrue(), "executable %s not found", ref) + } + }) +}) diff --git a/internal/fileparser/ref.go b/internal/fileparser/ref.go new file mode 100644 index 00000000..6b1a1005 --- /dev/null +++ b/internal/fileparser/ref.go @@ -0,0 +1,56 @@ +package fileparser + +import ( + "regexp" + "strings" + + "github.com/flowexec/flow/types/executable" +) + +var verbPatterns = []struct { + verb executable.Verb + regex *regexp.Regexp +}{ + {executable.VerbStart, regexp.MustCompile(`^(start|dev|serve|watch|run|preview|storybook)[\s:_-]?`)}, + {executable.VerbBuild, regexp.MustCompile(`^(build|compile|bundle|transpile)[\s:_-]?`)}, + {executable.VerbTest, regexp.MustCompile(`^(test|coverage|check|ci|e2e|unit)[\s:_-]?`)}, + {executable.VerbLint, regexp.MustCompile(`^(lint|format|fmt|prettier|eslint|stylelint)[\s:_-]?`)}, + {executable.VerbClean, regexp.MustCompile(`^(clean|reset|purge|clear)[\s:_-]?`)}, + {executable.VerbDeploy, regexp.MustCompile(`^(deploy|publish|release|push)[\s:_-]?`)}, + {executable.VerbInstall, regexp.MustCompile(`^(install|bootstrap|setup)[\s:_-]?`)}, + {executable.VerbRemove, regexp.MustCompile(`^(remove|uninstall|delete)[\s:_-]?`)}, + {executable.VerbUpdate, regexp.MustCompile(`^(update|upgrade)[\s:_-]?`)}, + {executable.VerbAnalyze, regexp.MustCompile(`^(analyze|audit|inspect|scan)[\s:_-]?`)}, +} + +// InferVerb infers the most likely Executable verb from a script or makeTarget name. +func InferVerb(name string) executable.Verb { + lower := strings.ToLower(name) + verb := executable.Verb(lower) + if verb.Validate() == nil { + return verb + } + for _, vp := range verbPatterns { + if vp.regex.MatchString(lower) { + return vp.verb + } + } + // Substring match (lower priority) + for _, vp := range verbPatterns { + if vp.regex.FindStringIndex(lower) != nil { + return vp.verb + } + } + return executable.VerbExec +} + +// NormalizeName strips any character that is not a letter, number, dash, or underscore, +// and also removes the verb prefix from the name if present. +func NormalizeName(name, verb string) string { + name = strings.TrimPrefix(name, verb) + name = strings.TrimPrefix(name, ":") + name = strings.TrimPrefix(name, "-") + name = strings.TrimPrefix(name, "_") + + return regexp.MustCompile(`[^a-zA-Z0-9_-]`).ReplaceAllString(name, "-") +} diff --git a/internal/fileparser/shell_file_parser.go b/internal/fileparser/shell_file_parser.go index c34b5f7e..a0ce87d1 100644 --- a/internal/fileparser/shell_file_parser.go +++ b/internal/fileparser/shell_file_parser.go @@ -1,217 +1,41 @@ package fileparser import ( - "fmt" "os" "path/filepath" - "regexp" - "strings" - "github.com/flowexec/tuikit/io" + "github.com/flowexec/flow/types/executable" ) -const ( - TimeoutConfigurationKey = "timeout" - VerbConfigurationKey = "verb" - NameConfigurationKey = "name" - AliasConfigurationKey = "alias" - DescriptionConfigurationKey = "description" - VisibilityConfigurationKey = "visibility" - TagConfigurationKey = "tag" - - InternalListSeparator = "," - - shellCommentPrefix = "# " - keyPrefix = "f:" - multiLineKeyPrefix = "f|" - descriptionSeparator = "\n" - descriptionAlias = "desc" -) - -var multiLineDescriptionTag = fmt.Sprintf("<%s%s>", multiLineKeyPrefix, DescriptionConfigurationKey) - -func ExecConfigMapFromFile(logger io.Logger, file string) (map[string]string, error) { - if err := validateFile(file); err != nil { - return nil, err - } - - fileBytes, err := os.ReadFile(filepath.Clean(file)) +func ExecutablesFromShFile(wsPath, filePath string) (*executable.Executable, error) { + fn := filepath.Base(filepath.Base(filePath)) // remove the ext and the path + verb := InferVerb(fn) + execName := NormalizeName(fn, verb.String()) + dir := executable.Directory(shortenWsPath(wsPath, filepath.Dir(filePath))) + exec := &executable.Executable{ + Verb: verb, + Name: execName, + Exec: &executable.ExecExecutableType{ + Dir: dir, + File: filepath.Base(filePath), + }, + } + + fileBytes, err := os.ReadFile(filepath.Clean(filePath)) if err != nil { return nil, err } - - configMap := make(map[string]string) - processingMultiLineDescription := false - for _, line := range strings.Split(string(fileBytes), "\n") { - isComment := strings.HasPrefix(line, strings.TrimSpace(shellCommentPrefix)) - if trimmedLine := strings.TrimSpace(line); !isComment && trimmedLine != "" { - // If the line is not a comment or empty, break out of the loop. - // All flow executable configuration should be at the top of the file. - break - } - - line = strings.TrimPrefix(line, shellCommentPrefix) - if processingMultiLineDescription = processMultiLineDescription( - line, configMap, processingMultiLineDescription, - ); processingMultiLineDescription { - continue - } - - for key, value := range parseConfigurations(logger, line) { - switch key { - case TimeoutConfigurationKey, VerbConfigurationKey, NameConfigurationKey, VisibilityConfigurationKey: - configMap[key] = strings.TrimSpace(value) - case DescriptionConfigurationKey, descriptionAlias: - if existingValue, ok := configMap[DescriptionConfigurationKey]; ok { - configMap[DescriptionConfigurationKey] = - fmt.Sprintf("%s%s%s", existingValue, descriptionSeparator, value) - } else { - configMap[DescriptionConfigurationKey] = value - } - case AliasConfigurationKey, TagConfigurationKey: - value = strings.TrimSpace(value) - if existingValue, ok := configMap[key]; ok { - configMap[key] = fmt.Sprintf("%s%s%s", existingValue, InternalListSeparator, value) - } else { - configMap[key] = value - } - } - } - } - if len(configMap) == 0 { - return nil, fmt.Errorf("no flow configurations found in file (%s)", file) - } - return configMap, nil -} - -// This regex is used to extract all flow configurations from a shell script. -// The regex matches the following: -// - f:= -// - f:="" -// - f:='' -// - f:="",='',key= -// - f:exampleKey='Example value\, with comma' -// - f:exampleKey="Example value\, with comma" -// - f:exampleKey="\'Example value with escaped quotes\'" -// - f:exampleKey='\"Example value with escaped quotes\"' -// and so on. -var flowConfigRegex = regexp.MustCompile(`f:\w+=(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|[^, ]+)`) - -func validateFile(file string) error { - info, err := os.Stat(file) + configMap, err := ExtractExecConfigMap(string(fileBytes), "# ") if err != nil { - return err - } else if info.IsDir() { - return fmt.Errorf("file (%s) is a directory", file) - } - ext := filepath.Ext(file) - if ext != ".sh" { - return fmt.Errorf("file (%s) is not a shell script", file) - } - return nil -} - -func processMultiLineDescription(line string, configMap map[string]string, processing bool) bool { - multiLineDescPrefixed := strings.HasPrefix(line, multiLineDescriptionTag) - multiLineDescSuffixed := strings.HasSuffix(line, multiLineDescriptionTag) - existingValue := configMap[DescriptionConfigurationKey] - switch { - case processing && multiLineDescPrefixed: - processing = false - case processing && multiLineDescSuffixed: - processing = false - line = strings.TrimSuffix(line, multiLineDescriptionTag) - if existingValue != "" { - configMap[DescriptionConfigurationKey] = fmt.Sprintf( - "%s%s%s", - existingValue, - descriptionSeparator, - line, - ) - } else { - configMap[DescriptionConfigurationKey] = line - } - case processing: - if existingValue != "" { - configMap[DescriptionConfigurationKey] = fmt.Sprintf( - "%s%s%s", - existingValue, - descriptionSeparator, - line, - ) - } else { - configMap[DescriptionConfigurationKey] = line - } - case multiLineDescPrefixed: - processing = true - line = strings.TrimPrefix(line, multiLineDescriptionTag) - if trimmedLine := strings.TrimSpace(line); trimmedLine == "" { - return processing - } else if existingValue != "" { - configMap[DescriptionConfigurationKey] = fmt.Sprintf( - "%s%s%s", - existingValue, - descriptionSeparator, - line, - ) - } else { - configMap[DescriptionConfigurationKey] = line - } + return nil, err } - return processing -} - -func parseConfigurations(logger io.Logger, line string) map[string]string { - configMap := make(map[string]string) - matches := flowConfigRegex.FindAllString(line, -1) - for _, match := range matches { - split := strings.SplitN(match, "=", 2) - key := strings.TrimSpace(strings.ToLower(strings.TrimPrefix(split[0], keyPrefix))) - value := split[1] - // Removing quotes if present - if (value[0] == '"' && value[len(value)-1] == '"') || (value[0] == '\'' && value[len(value)-1] == '\'') { - value = value[1 : len(value)-1] - } - // Replace escaped characters - value = strings.ReplaceAll(value, `\"`, `"`) - value = strings.ReplaceAll(value, `\'`, `'`) - value = strings.ReplaceAll(value, `\\`, `\`) - value = strings.ReplaceAll(value, `\,`, `,`) - if !validateKey(key) { - logger.Warnf("invalid key (%s) in configuration", key) - continue - } - - switch key { - case AliasConfigurationKey, TagConfigurationKey: - value = strings.TrimSpace(value) - if existingValue, ok := configMap[key]; ok { - configMap[key] = fmt.Sprintf("%s%s%s", existingValue, InternalListSeparator, value) - } else { - configMap[key] = value - } - case DescriptionConfigurationKey, descriptionAlias: - if existingValue, ok := configMap[DescriptionConfigurationKey]; ok { - configMap[DescriptionConfigurationKey] = - fmt.Sprintf("%s%s%s", existingValue, descriptionSeparator, value) - } else { - configMap[DescriptionConfigurationKey] = value - } - default: - configMap[key] = strings.TrimSpace(value) + for key, value := range configMap { + if err := applyConfig(exec, key, value); err != nil { + return nil, err } } - return configMap -} -func validateKey(key string) bool { - switch key { - case VerbConfigurationKey, NameConfigurationKey, DescriptionConfigurationKey, descriptionAlias, - AliasConfigurationKey, VisibilityConfigurationKey, TagConfigurationKey, - TimeoutConfigurationKey: - return true - default: - return false - } + exec.Tags = append(exec.Tags, generatedTag) + return exec, nil } diff --git a/internal/fileparser/shell_file_parser_test.go b/internal/fileparser/shell_file_parser_test.go new file mode 100644 index 00000000..77ada0df --- /dev/null +++ b/internal/fileparser/shell_file_parser_test.go @@ -0,0 +1,24 @@ +package fileparser_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flowexec/flow/internal/fileparser" + "github.com/flowexec/flow/types/executable" +) + +var _ = Describe("ExecutablesFromShFile", func() { + const filePath = "testdata/simple.sh" + + It("should parse executables from sh file", func() { + exec, err := fileparser.ExecutablesFromShFile("testdata", filePath) + Expect(err).NotTo(HaveOccurred()) + Expect(exec).NotTo(BeNil()) + Expect(exec.Verb).To(Equal(executable.VerbShow)) + Expect(exec.Name).To(Equal("hello")) + Expect(exec.Exec).NotTo(BeNil()) + Expect(exec.Exec.File).To(Equal("simple.sh")) + Expect(exec.Exec.Dir).To(Equal(executable.Directory("//"))) + }) +}) diff --git a/internal/fileparser/testdata/Makefile b/internal/fileparser/testdata/Makefile new file mode 100644 index 00000000..a9f794b2 --- /dev/null +++ b/internal/fileparser/testdata/Makefile @@ -0,0 +1,24 @@ +# Build the application binary +build: + go build -o bin/app ./cmd/app + +# Run all tests with coverage +test: + go test -v -race -coverprofile=coverage.out ./... + +# Deploy to production environment +# Depends on build and test +deploy: build test + ./scripts/deploy.sh production + +# f:name=program f:verb=run +# f:desc="Run main.go" +target: + go run main.go + +# Should be skipped +.PHONY: build test clean + +# Should be skipped +%.o: %.c + gcc -c $< -o $@ \ No newline at end of file diff --git a/internal/fileparser/testdata/docker-compose.yml b/internal/fileparser/testdata/docker-compose.yml new file mode 100644 index 00000000..64801dcf --- /dev/null +++ b/internal/fileparser/testdata/docker-compose.yml @@ -0,0 +1,10 @@ +services: + app: + build: . + ports: ["3000:3000"] + db: + image: postgres:15 + environment: + POSTGRES_DB: myapp + redis: + image: redis:alpine diff --git a/internal/fileparser/testdata/invaliddir/.keep b/internal/fileparser/testdata/invaliddir/.keep new file mode 100644 index 00000000..e69de29b diff --git a/internal/fileparser/testdata/package.json b/internal/fileparser/testdata/package.json new file mode 100644 index 00000000..78c988e7 --- /dev/null +++ b/internal/fileparser/testdata/package.json @@ -0,0 +1,10 @@ +{ + "scripts": { + "dev": "vite dev --host", + "build": "vite build && tsc", + "test": "vitest run", + "test:watch": "vitest", + "lint": "eslint . --fix", + "preview": "vite preview" + } +} \ No newline at end of file diff --git a/internal/fileparser/testdata/simple.sh b/internal/fileparser/testdata/simple.sh index 980d0016..9922a249 100644 --- a/internal/fileparser/testdata/simple.sh +++ b/internal/fileparser/testdata/simple.sh @@ -1,5 +1,5 @@ #!/bin/sh -# f:name=value1 f:verb=value2 +# f:name=hello f:verb=show echo "Hello, world!" diff --git a/types/executable/flowfile.gen.go b/types/executable/flowfile.gen.go index 4d2bfbc2..01c4a461 100644 --- a/types/executable/flowfile.gen.go +++ b/types/executable/flowfile.gen.go @@ -27,9 +27,12 @@ type FlowFile struct { // Executables corresponds to the JSON schema field "executables". Executables ExecutableList `json:"executables,omitempty" yaml:"executables,omitempty" mapstructure:"executables,omitempty"` - // FromFile corresponds to the JSON schema field "fromFile". + // DEPRECATED: Use `imports` instead FromFile FromFile `json:"fromFile,omitempty" yaml:"fromFile,omitempty" mapstructure:"fromFile,omitempty"` + // Imports corresponds to the JSON schema field "imports". + Imports FromFile `json:"imports,omitempty" yaml:"imports,omitempty" mapstructure:"imports,omitempty"` + // The namespace to be given to all executables in the flow file. // If not set, the executables in the file will be grouped into the root (*) // namespace. diff --git a/types/executable/flowfile_schema.yaml b/types/executable/flowfile_schema.yaml index d4b73580..3915dbbc 100644 --- a/types/executable/flowfile_schema.yaml +++ b/types/executable/flowfile_schema.yaml @@ -25,6 +25,10 @@ properties: fromFile: $ref: '#/definitions/FromFile' default: [] + description: "DEPRECATED: Use `imports` instead" + imports: + $ref: '#/definitions/FromFile' + default: [] namespace: type: string description: |