feat: variable discovery and validation improvements (#104)

Closes #94

- Defaults from environment variables via `env` or `default_from_env` in
variable schema.
- Type coercion: boolean, integer, float/number; string by default.
- Validation: enum, regex/pattern, min/max numeric bounds.
- Required handling: in non-interactive mode, missing required variables
raise a clear error.
- Tests added for env defaults and coercion/validation.

All tests pass (95).

Author: httpdss

docs/cli-reference.md

@@ -59,7 +59,7 @@ Generate the project structure.
 **Usage:**
 
 ```sh
-struct generate [-h] [-l LOG] [-c CONFIG_FILE] [-i LOG_FILE] [-s STRUCTURES_PATH] [-n INPUT_STORE] [-d] [-v VARS] [-b BACKUP] [-f {overwrite,skip,append,rename,backup}] [-p GLOBAL_SYSTEM_PROMPT] [--non-interactive] [--mappings-file MAPPINGS_FILE] [-o {console,file}] structure_definition base_path
+struct generate [-h] [-l LOG] [-c CONFIG_FILE] [-i LOG_FILE] [-s STRUCTURES_PATH] [-n INPUT_STORE] [-d] [--diff] [-v VARS] [-b BACKUP] [-f {overwrite,skip,append,rename,backup}] [-p GLOBAL_SYSTEM_PROMPT] [--non-interactive] [--mappings-file MAPPINGS_FILE] [-o {console,file}] structure_definition base_path
 ```
 
 **Arguments:**
@@ -69,6 +69,7 @@ struct generate [-h] [-l LOG] [-c CONFIG_FILE] [-i LOG_FILE] [-s STRUCTURES_PATH
 - `-s STRUCTURES_PATH, --structures-path STRUCTURES_PATH`: Path to structure definitions.
 - `-n INPUT_STORE, --input-store INPUT_STORE`: Path to the input store.
 - `-d, --dry-run`: Perform a dry run without creating any files or directories.
+- `--diff`: Show unified diffs for files that would be created/modified (works with `--dry-run` and in `-o console` mode).
 - `-v VARS, --vars VARS`: Template variables in the format KEY1=value1,KEY2=value2.
 - `-b BACKUP, --backup BACKUP`: Path to the backup folder.
 - `-f {overwrite,skip,append,rename,backup}, --file-strategy {overwrite,skip,append,rename,backup}`: Strategy for handling existing files.

docs/file-handling.md

@@ -68,7 +68,7 @@ files:
 
 ## Remote File Protocols
 
-STRUCT supports multiple protocols for fetching remote content:
+STRUCT supports multiple protocols for fetching remote content (with caching and robust fallbacks):
 
 ### HTTP/HTTPS
 
@@ -80,6 +80,12 @@ files:
 
 ### GitHub Protocols
 
+STRUCT optimizes single-file fetches from GitHub by preferring `raw.githubusercontent.com` when possible and falling back to `git clone/pull` if necessary. You can control behavior with environment variables:
+
+- `STRUCT_HTTP_TIMEOUT` (seconds, default 10)
+- `STRUCT_HTTP_RETRIES` (default 2)
+- `STRUCT_DENY_NETWORK=1` to skip HTTP attempts and use git fallback directly.
+
 #### Standard GitHub
 
 ```yaml

docs/template-variables.md

@@ -78,8 +78,47 @@ variables:
 
 - `string`: Text values
 - `integer`: Numeric values
+- `number`: Floating-point values
 - `boolean`: True/false values
 
+### Validation and Defaults
+
+Interactive enum selection: when a variable defines `enum` and you are in interactive mode, STRUCT will display numbered choices and accept either the number or the exact value. Press Enter to accept the default (if any).
+
+Example prompt:
+
+```
+❓ Enter value for ENV [dev] (1) dev, (2) prod:
+# Typing `2` selects `prod`, typing `prod` also works.
+```
+
+You can now enforce types and validations in your variables schema:
+
+- `required: true` to require a value (non-interactive runs will error if missing)
+- `enum: [...]` to restrict values to a set
+- `regex`/`pattern` to validate string format
+- `min`/`max` to bound numeric values
+- `env` or `default_from_env` to set defaults from environment variables
+
+Example:
+
+```yaml
+variables:
+  - IS_ENABLED:
+      type: boolean
+      required: true
+  - RETRY:
+      type: integer
+      min: 1
+      max: 5
+  - ENV:
+      type: string
+      enum: [dev, prod]
+  - TOKEN:
+      type: string
+      env: MY_TOKEN
+```
+
 ## Custom Jinja2 Filters
 
 STRUCT includes custom filters for common tasks:

docs/usage.md

@@ -49,6 +49,12 @@ struct generate my-config.yaml ./output
 struct generate file://my-config.yaml ./output
 ```
 
+### Diff Preview Example
+
+```sh
+struct generate --dry-run --diff file://structure.yaml ./output
+```
+
 ### Complete Example
 
 ```sh
@@ -66,6 +72,7 @@ struct generate \
 
 - `--log`: Set logging level (DEBUG, INFO, WARNING, ERROR)
 - `--dry-run`: Preview actions without making changes
+- `--diff`: Show unified diffs for files that would be created/modified (useful with `--dry-run` and console output)
 - `--backup`: Specify backup directory for existing files
 - `--file-strategy`: Choose how to handle existing files (overwrite, skip, append, rename, backup)
 - `--log-file`: Write logs to specified file

struct_module/template_renderer.py

@@ -61,8 +61,13 @@ def get_defaults_from_config(self):
       defaults = {}
       for item in self.config_variables:
         for name, content in item.items():
+          # Explicit default value
           if 'default' in content:
             defaults[name] = content.get('default')
+          # Default from environment variable (env or default_from_env)
+          env_key = content.get('env') or content.get('default_from_env')
+          if env_key and os.environ.get(env_key) is not None:
+            defaults[name] = os.environ.get(env_key)
       return defaults
 
 
@@ -79,19 +84,104 @@ def prompt_for_missing_vars(self, content, vars):
       undeclared_variables = meta.find_undeclared_variables(parsed_content)
       self.logger.debug(f"Undeclared variables: {undeclared_variables}")
 
+      # Build schema lookup
+      schema = {}
+      for item in (self.config_variables or []):
+        for name, conf in item.items():
+          schema[name] = conf or {}
+
       # Prompt the user for any missing variables
       # Suggest a default from the config if available
       default_values = self.get_defaults_from_config()
       self.logger.debug(f"Default values from config: {default_values}")
 
       for var in undeclared_variables:
         if var not in vars:
+          conf = schema.get(var, {})
+          required = conf.get('required', False)
           default = self.input_data.get(var, default_values.get(var, ""))
           if self.non_interactive:
-            user_input = default if default else "NEEDS_TO_BE_SET"
+            if required and (default is None or default == ""):
+              raise ValueError(f"Missing required variable '{var}' in non-interactive mode")
+            user_input = default
           else:
-            user_input = input(f"❓ Enter value for {var} [{default}]: ") or default
-          self.input_store.set_value(var, user_input)
-          vars[var] = user_input
+            # Interactive prompt with enum support (choose by value or index)
+            enum = conf.get('enum')
+            if enum:
+              # Build options list string like "(1) dev, (2) prod)"
+              options = ", ".join([f"({i+1}) {val}" for i, val in enumerate(enum)])
+              while True:
+                raw = input(f"❓ Enter value for {var} [{default}] {options}: ")
+                raw = raw.strip()
+                if raw == "":
+                  user_input = default
+                elif raw.isdigit() and 1 <= int(raw) <= len(enum):
+                  user_input = enum[int(raw) - 1]
+                else:
+                  # accept exact match from enum
+                  if raw in enum:
+                    user_input = raw
+                  else:
+                    print(f"Invalid choice. Please enter one of: {options} or a valid value.")
+                    continue
+                break
+            else:
+              user_input = input(f"❓ Enter value for {var} [{default}]: ") or default
+          # Coerce and validate according to schema
+          coerced = self._coerce_and_validate(var, user_input, conf)
+          self.input_store.set_value(var, coerced)
+          vars[var] = coerced
       self.input_store.save()
       return vars
+
+    def _coerce_and_validate(self, name, value, conf):
+      # Type coercion
+      vtype = (conf.get('type') or 'string').lower()
+      original = value
+      try:
+        if vtype == 'boolean' or vtype == 'bool':
+          if isinstance(value, bool):
+            coerced = value
+          elif isinstance(value, str):
+            coerced = value.strip().lower() in ['1', 'true', 'yes', 'y', 'on']
+          else:
+            coerced = bool(value)
+        elif vtype == 'number' or vtype == 'float':
+          coerced = float(value) if value != '' and value is not None else None
+        elif vtype == 'integer' or vtype == 'int':
+          coerced = int(value) if value not in (None, '') else None
+        else:
+          coerced = '' if value is None else str(value)
+      except Exception:
+        raise ValueError(f"Variable '{name}' could not be coerced to {vtype} (value: {original})")
+
+      # Enum validation
+      enum = conf.get('enum')
+      if enum is not None and coerced not in enum:
+        raise ValueError(f"Variable '{name}' must be one of {enum}, got: {coerced}")
+
+      # Regex validation (only for strings)
+      pattern = conf.get('regex') or conf.get('pattern')
+      if pattern and isinstance(coerced, str):
+        import re as _re
+        if _re.fullmatch(pattern, coerced) is None:
+          raise ValueError(f"Variable '{name}' does not match required pattern: {pattern}")
+
+      # Min/Max validation
+      def _as_num(x):
+        try:
+          return float(x)
+        except Exception:
+          return None
+      minv = conf.get('min')
+      maxv = conf.get('max')
+      if minv is not None:
+        cv = _as_num(coerced)
+        if cv is not None and cv < float(minv):
+          raise ValueError(f"Variable '{name}' must be >= {minv}, got {coerced}")
+      if maxv is not None:
+        cv = _as_num(coerced)
+        if cv is not None and cv > float(maxv):
+          raise ValueError(f"Variable '{name}' must be <= {maxv}, got {coerced}")
+
+      return coerced

tests/test_template_renderer.py

@@ -25,6 +25,52 @@ def test_prompt_for_missing_vars(renderer):
         missing_vars = renderer.prompt_for_missing_vars(content, vars)
         assert missing_vars["var2"] == "Universe"
 
+
+def test_defaults_from_env_renderer(monkeypatch):
+    config_variables = [
+        {"TOKEN": {"type": "string", "env": "MY_TOKEN"}},
+    ]
+    input_store = "/tmp/input.json"
+    non_interactive = True
+    monkeypatch.setenv("MY_TOKEN", "abc123")
+    r = TemplateRenderer(config_variables, input_store, non_interactive)
+    defaults = r.get_defaults_from_config()
+    assert defaults["TOKEN"] == "abc123"
+
+
+def test_type_coercion_and_validation(monkeypatch):
+    config_variables = [
+        {"IS_ENABLED": {"type": "boolean", "required": True}},
+        {"RETRY": {"type": "integer", "min": 1, "max": 5}},
+        {"ENV": {"type": "string", "enum": ["dev", "prod"]}},
+    ]
+    input_store = "/tmp/input.json"
+    non_interactive = False
+    r = TemplateRenderer(config_variables, input_store, non_interactive)
+    content = "{{@ IS_ENABLED @}} {{@ RETRY @}} {{@ ENV @}}"
+
+    # Provide inputs mapped by variable name (order-agnostic)
+    def fake_input(prompt):
+        if 'IS_ENABLED' in prompt:
+            return 'yes'
+        if 'RETRY' in prompt:
+            return '3'
+        if 'ENV' in prompt:
+            return 'prod'
+        return ''
+    with patch('builtins.input', side_effect=fake_input):
+        vars = {}
+        out = r.prompt_for_missing_vars(content, vars)
+        assert out["IS_ENABLED"] is True
+        assert out["RETRY"] == 3
+        assert out["ENV"] == "prod"
+
+    # Enum violation should raise
+    r = TemplateRenderer(config_variables, input_store, non_interactive)
+    with patch('builtins.input', side_effect=["true", "2", "staging"]):
+        with pytest.raises(ValueError):
+            r.prompt_for_missing_vars(content, {})
+
 def test_get_defaults_from_config(renderer):
     defaults = renderer.get_defaults_from_config()
     assert defaults == {"var1": "default1", "var2": "default2"}