Merge pull request #290 from webtech-network/expect-file-artifact

(feat) new I/O test : Expect File Output

Author: ArthurCRodrigues

README.md

@@ -195,6 +195,7 @@ Tests command-line programs by providing inputs and validating outputs.
 | Test Name          | Description                                             | Key Parameters                               |
 |--------------------|---------------------------------------------------------|----------------------------------------------|
 | `expect_output`    | Execute program with inputs and verify output           | `inputs`, `expected_output`, `program_command` |
+| `expect_file_artifact` | Execute program and validate a generated output file | `artifact_path`, `expected_content`, `program_command` |
 | `dont_fail`        | Validates that a program does not crash on a given input | `inputs`, `program_command`                 |
 | `forbidden_import` | Analyzes a file looking for specified libraries imports | `forbidden_imports`        |
 

autograder/template_library/input_output.py

@@ -39,7 +39,7 @@ def run_sandbox_execution(self, sandbox: SandboxContainer, inputs: list = None,
         command = ' '.join(inputs) if isinstance(inputs, list) else str(inputs)
         return sandbox.run_command(command)
 
-    def check_for_base_errors(self, output) -> TestResult:
+    def check_for_base_errors(self, output, **kwargs) -> TestResult:
         """
         Checks for Timeout, Compilation Error, or Runtime Error in the sandbox output.
         Returns a TestResult with score 0.0 if an error is found, or None if successful.
@@ -119,7 +119,7 @@ def execute(self, files, sandbox: SandboxContainer, *args, inputs: list = None,
             )
 
             # Check for generic execution failures
-            error_result = self.check_for_base_errors(output)
+            error_result = self.check_for_base_errors(output, locale=kwargs.get("locale"))
             if error_result:
                 return error_result
 
@@ -191,7 +191,7 @@ def execute(self, files, sandbox: SandboxContainer, *args, user_input: str = "",
             )
 
             # Check for generic execution failures
-            error_result = self.check_for_base_errors(output)
+            error_result = self.check_for_base_errors(output, locale=kwargs.get("locale"))
             if error_result:
                 return error_result
 
@@ -211,6 +211,130 @@ def execute(self, files, sandbox: SandboxContainer, *args, user_input: str = "",
 
 
 
+# ===============================================================
+# TestFunction for File Artifact Validation
+# ===============================================================
+
+class ExpectFileArtifactTest(BaseExecutionTest):
+    """
+    Tests a command-line program by running it, then extracting a generated
+    file from the sandbox and comparing its content against expected values.
+
+    Supports exact, contains, and regex matching modes.
+    """
+
+    @property
+    def name(self):
+        return "expect_file_artifact"
+
+    @property
+    def description(self):
+        return t("io.expect_file_artifact.description")
+
+    @property
+    def required_file(self):
+        return None
+
+    @property
+    def parameter_description(self):
+        return [
+            ParamDescription("program_command", t("io.expect_file_artifact.params.program_command"), "string or dict"),
+            ParamDescription("artifact_path", t("io.expect_file_artifact.params.artifact_path"), "string"),
+            ParamDescription("expected_content", t("io.expect_file_artifact.params.expected_content"), "string"),
+            ParamDescription("match_mode", t("io.expect_file_artifact.params.match_mode"), "string"),
+            ParamDescription("inputs", t("io.expect_file_artifact.params.inputs"), "list of strings"),
+            ParamDescription("normalization", t("io.expect_file_artifact.params.normalization"), "boolean"),
+        ]
+
+    @staticmethod
+    def _validate_artifact_path(artifact_path: str) -> Optional[str]:
+        """Return an error message if the path is unsafe, else None."""
+        if not artifact_path:
+            return "artifact_path is required"
+        if artifact_path.startswith("/") or ".." in artifact_path.split("/"):
+            return f"Invalid artifact_path (absolute or traversal): {artifact_path}"
+        return None
+
+    @staticmethod
+    def _normalize(text: str) -> str:
+        """Normalize line endings and strip trailing whitespace per line."""
+        lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
+        return "\n".join(line.rstrip() for line in lines).strip()
+
+    @staticmethod
+    def _match(actual: str, expected: str, mode: str) -> bool:
+        if mode == "contains":
+            return expected in actual
+        if mode == "regex":
+            return bool(re.search(expected, actual))
+        return actual == expected
+
+    def execute(self, files, sandbox: SandboxContainer, *args,
+                program_command: Optional[str] = None,
+                artifact_path: str = "",
+                expected_content: str = "",
+                match_mode: str = "exact",
+                inputs: list = None,
+                normalization: bool = True,
+                **kwargs) -> TestResult:
+        locale = kwargs.get("locale")
+
+        # Validate artifact_path
+        path_error = self._validate_artifact_path(artifact_path)
+        if path_error:
+            return TestResult(test_name=self.name, score=0.0,
+                              report=t("io.expect_file_artifact.report.invalid_path", locale=locale, error=path_error))
+
+        # Validate match_mode
+        if match_mode not in ("exact", "contains", "regex"):
+            return TestResult(test_name=self.name, score=0.0,
+                              report=t("io.expect_file_artifact.report.invalid_match_mode", locale=locale, mode=match_mode))
+
+        # Pre-compile regex to catch bad patterns early
+        if match_mode == "regex":
+            try:
+                re.compile(expected_content)
+            except re.error as e:
+                return TestResult(test_name=self.name, score=0.0,
+                                  report=t("io.expect_file_artifact.report.invalid_regex", locale=locale, error=str(e)))
+
+        # Execute student program
+        try:
+            output = self.run_sandbox_execution(sandbox=sandbox, inputs=inputs, program_command=program_command)
+            error_result = self.check_for_base_errors(output, locale=locale)
+            if error_result:
+                return error_result
+        except (ValueError, TimeoutError, RuntimeError) as e:
+            return TestResult(test_name=self.name, score=0.0,
+                              report=t("io.expect_output.report.internal_error", locale=locale, error=str(e)))
+
+        # Extract artifact
+        full_path = f"/app/{artifact_path}"
+        try:
+            extracted = sandbox.extract_file(full_path)
+        except FileNotFoundError:
+            return TestResult(test_name=self.name, score=0.0,
+                              report=t("io.expect_file_artifact.report.file_not_found", locale=locale, path=artifact_path))
+        except (ValueError, RuntimeError) as e:
+            return TestResult(test_name=self.name, score=0.0,
+                              report=t("io.expect_file_artifact.report.extraction_error", locale=locale, error=str(e)))
+
+        # Compare content
+        actual = extracted.content_text
+        expected = expected_content
+        if normalization:
+            actual = self._normalize(actual)
+            expected = self._normalize(expected)
+
+        if self._match(actual, expected, match_mode):
+            return TestResult(test_name=self.name, score=100.0,
+                              report=t("io.expect_file_artifact.report.success", locale=locale, path=artifact_path))
+
+        return TestResult(test_name=self.name, score=0.0,
+                          report=t("io.expect_file_artifact.report.mismatch", locale=locale,
+                                   path=artifact_path, expected=expected, actual=actual))
+
+
 # ===============================================================
 # TestFunction for Forbidden Import Detection
 # ===============================================================
@@ -396,6 +520,7 @@ def __init__(self):
         self.tests = {
             "expect_output": ExpectOutputTest(),
             "dont_fail": DontFailTest(),
+            "expect_file_artifact": ExpectFileArtifactTest(),
             "forbidden_import": ForbiddenImportTest(),
         }
 

autograder/translations/en.json

@@ -467,6 +467,26 @@
     "template": {
       "name": "Input/Output",
       "description": "A template for evaluating assignments based on command-line input and output."
+    },
+    "expect_file_artifact": {
+      "description": "Executes the student program, then extracts a generated file from the sandbox and validates its content against expected values.",
+      "params": {
+        "program_command": "(Optional) Command to run the program. Can be a string or dict (multi-lang).",
+        "artifact_path": "Relative path (under /app) of the file the program must generate (e.g. 'output.txt').",
+        "expected_content": "The expected file content or pattern to match against.",
+        "match_mode": "Comparison mode: 'exact' (default), 'contains', or 'regex'.",
+        "inputs": "List of strings to be sent to the program via stdin.",
+        "normalization": "Whether to normalize line endings and trim trailing whitespace (default: true)."
+      },
+      "report": {
+        "success": "Success: Artifact '{path}' matches expected content.",
+        "mismatch": "FAILURE: Content mismatch in artifact '{path}'.\nExpected:\n{expected}\nActual:\n{actual}",
+        "file_not_found": "FAILURE: Expected artifact '{path}' was not found. Make sure your program creates this file.",
+        "extraction_error": "FAILURE: Could not read artifact from sandbox.\nDetails: {error}",
+        "invalid_path": "CONFIGURATION ERROR: Invalid artifact path.\nDetails: {error}",
+        "invalid_match_mode": "CONFIGURATION ERROR: Invalid match_mode '{mode}'. Must be 'exact', 'contains', or 'regex'.",
+        "invalid_regex": "CONFIGURATION ERROR: Invalid regex pattern.\nDetails: {error}"
+      }
     }
   },
   "api_testing": {

autograder/translations/pt_br.json

@@ -467,6 +467,26 @@
     "template": {
       "name": "Entrada/Saída (I/O)",
       "description": "Um modelo para avaliar trabalhos com base na entrada e saída de linha de comando."
+    },
+    "expect_file_artifact": {
+      "description": "Executa o programa do aluno, extrai um arquivo gerado do sandbox e valida seu conteúdo contra os valores esperados.",
+      "params": {
+        "program_command": "(Opcional) Comando para executar o programa. Pode ser uma string ou dict (multi-idioma).",
+        "artifact_path": "Caminho relativo (sob /app) do arquivo que o programa deve gerar (ex: 'output.txt').",
+        "expected_content": "O conteúdo esperado do arquivo ou padrão para comparação.",
+        "match_mode": "Modo de comparação: 'exact' (padrão), 'contains' ou 'regex'.",
+        "inputs": "Lista de strings a serem enviadas para o programa via stdin.",
+        "normalization": "Se deve normalizar quebras de linha e remover espaços em branco no final (padrão: true)."
+      },
+      "report": {
+        "success": "Sucesso: O artefato '{path}' corresponde ao conteúdo esperado.",
+        "mismatch": "FALHA: Conteúdo divergente no artefato '{path}'.\nEsperado:\n{expected}\nObtido:\n{actual}",
+        "file_not_found": "FALHA: O artefato esperado '{path}' não foi encontrado. Verifique se o seu programa cria este arquivo.",
+        "extraction_error": "FALHA: Não foi possível ler o artefato do sandbox.\nDetalhes: {error}",
+        "invalid_path": "ERRO DE CONFIGURAÇÃO: Caminho de artefato inválido.\nDetalhes: {error}",
+        "invalid_match_mode": "ERRO DE CONFIGURAÇÃO: match_mode '{mode}' inválido. Deve ser 'exact', 'contains' ou 'regex'.",
+        "invalid_regex": "ERRO DE CONFIGURAÇÃO: Padrão regex inválido.\nDetalhes: {error}"
+      }
     }
   },
   "api_testing": {

docs/templates/input_output.md

@@ -56,6 +56,84 @@ Executes a program with a series of stdin inputs and compares the program's stdo
 
 ---
 
+### `expect_file_artifact`
+
+Executes a program and then extracts a generated file from the sandbox container, comparing its content against an expected value. Useful for assignments where students must produce output files (e.g., reports, sorted data, metrics).
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `artifact_path` | string | ✓ | Relative path under `/app` of the file the program must generate (e.g., `output.txt`) |
+| `expected_content` | string | ✓ | Expected file content or pattern to match against |
+| `program_command` | string | ✗ | Command to execute (same format as `expect_output`) |
+| `match_mode` | string | ✗ | `exact` (default), `contains`, or `regex` |
+| `inputs` | list[string] | ✗ | List of strings sent to stdin before extraction |
+| `normalization` | boolean | ✗ | Normalize line endings and trim trailing whitespace (default: `true`) |
+
+**Scoring:** 100 if file content matches, 0 otherwise.
+
+**Error handling:** Inherits all execution error detection from `expect_output` (timeouts, compilation errors, runtime errors), plus:
+- Missing artifact file
+- Artifact extraction failure
+- Invalid regex pattern
+- Invalid artifact path (absolute or traversal paths are rejected)
+
+**Artifact path conventions:**
+- Paths are relative to `/app` (the sandbox working directory)
+- Absolute paths (`/etc/passwd`) and traversal paths (`../secret`) are rejected
+- The file must be a regular file (not a directory or symlink)
+- Maximum extracted file size: 1 MB
+
+**Exact match example:**
+```json
+{
+  "name": "expect_file_artifact",
+  "parameters": {
+    "inputs": ["1000"],
+    "program_command": {
+      "python": "python3 main.py",
+      "java": "java Main",
+      "node": "node main.js",
+      "cpp": "./main"
+    },
+    "artifact_path": "matricula_selecao.txt",
+    "match_mode": "exact",
+    "expected_content": "comparacoes: 10\nmovimentacoes: 4\ntempo: 0.001s"
+  },
+  "weight": 100
+}
+```
+
+**Regex match example** (for variable values like execution time):
+```json
+{
+  "name": "expect_file_artifact",
+  "parameters": {
+    "inputs": ["1000"],
+    "program_command": "CMD",
+    "artifact_path": "matricula_insercao.txt",
+    "match_mode": "regex",
+    "expected_content": "comparacoes:\\s*\\d+\\s+movimentacoes:\\s*\\d+\\s+tempo:\\s*[0-9.]+s"
+  },
+  "weight": 100
+}
+```
+
+**Contains match example:**
+```json
+{
+  "name": "expect_file_artifact",
+  "parameters": {
+    "program_command": "python3 main.py",
+    "artifact_path": "report.txt",
+    "match_mode": "contains",
+    "expected_content": "PASSED"
+  },
+  "weight": 50
+}
+```
+
+---
+
 ### `dont_fail`
 
 Executes a program with a specific input and verifies it completes **without crashing**. The program's stdout is ignored — this test only validates that execution succeeds. Useful for testing error handling (e.g., sending invalid input).

sandbox_manager/models/sandbox_models.py

@@ -46,6 +46,16 @@ def __str__(self):
         return self.stdout
 
 
+@dataclass
+class ExtractedFile:
+    """Result of extracting a single file from a sandbox container."""
+    path: str
+    content_bytes: bytes
+    size: int
+    content_text: str = ""
+    encoding: str = "utf-8"
+
+
 class HttpResponse:
     """Wrapper for HTTP responses from containerized applications."""