Add "withMetadata" flag to "listContents" endpoint

- Will no longer include METADATA type files in response
- Made RenderType in charge of the Aerie Metadata type for the backend, as that extension is expected to be fairly constant
- Remove unneeded Optional wrapper from `listFiles` method

Author: Mythicaeda

workspace-server/src/main/java/gov/nasa/jpl/aerie/workspace/server/DirectoryTree.java

@@ -1,13 +1,21 @@
 package gov.nasa.jpl.aerie.workspace.server;
 
 import gov.nasa.jpl.aerie.workspace.server.postgres.RenderType;
+import gov.nasa.jpl.aerie.workspace.server.types.MetadataKeys;
 
 import javax.json.Json;
 import javax.json.JsonArray;
+import javax.json.JsonException;
+import javax.json.JsonObject;
 import javax.json.JsonObjectBuilder;
+import javax.json.JsonValue;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.TreeMap;
 
 /**
@@ -29,7 +37,7 @@ public class DirectoryTree {
    * @param extensionMappings a map of file extensions to RenderTypes.
    *    Used to determine the RenderType of file paths
    */
-  public DirectoryTree(Path root, List<Path> inputList, Map<String, RenderType> extensionMappings) {
+  public DirectoryTree(Path root, List<Path> inputList, Map<String, RenderType> extensionMappings, boolean withMetadata) {
     if(!root.toFile().isDirectory()) {
       throw new IllegalArgumentException("Cannot create a DirectoryTree from a file.");
     }
@@ -39,7 +47,8 @@ public DirectoryTree(Path root, List<Path> inputList, Map<String, RenderType> ex
       if(path.toFile().isDirectory()) {
         this.root.addChild(new DirectoryNode(path));
       } else {
-        this.root.addChild(new FileNode(path, RenderType.getRenderType(path.getFileName().toString(), extensionMappings)));
+        final var rType = RenderType.getRenderType(path.getFileName().toString(), extensionMappings);
+        this.root.addChild(new FileNode(path, rType, withMetadata));
       }
     }
   }
@@ -49,16 +58,125 @@ private static class FileNode {
     final String name;
     final Path path;
 
+    final Optional<JsonObject> metadata;
+    final Optional<MetadataStatus> metadataStatus;
+
+    private enum MetadataStatus {
+      ok, // Metadata file exists and is valid
+      missing, // Metadata file is absent
+      malformed  // Metadata JSON is invalid
+    }
+
     FileNode(Path path, RenderType renderType) {
       this.path = path;
       this.renderType = renderType;
       this.name = path.getFileName().toString();
+      this.metadata = Optional.empty();
+      this.metadataStatus = Optional.empty();
+    }
+
+    FileNode(Path path, RenderType renderType, boolean getMetadata) {
+      this.path = path;
+      this.renderType = renderType;
+      this.name = path.getFileName().toString();
+
+      // Check if we even need to get the file's metadata
+      if (!getMetadata || renderType == RenderType.METADATA) {
+        this.metadata = Optional.empty();
+        this.metadataStatus = Optional.empty();
+        return;
+      }
+
+      // Check if the metadata file exists
+      final var metadataFile = this.path.resolveSibling(RenderType.toMetadataFileName(name)).toFile();
+      if (!metadataFile.exists() || metadataFile.isDirectory()) {
+        this.metadata = Optional.empty();
+        this.metadataStatus = Optional.of(MetadataStatus.missing);
+        return;
+      }
+
+      // Attempt to read the metadata file
+      final JsonObject fileContents;
+      try(final var reader = Json.createReader(new FileReader(metadataFile))){
+        fileContents = reader.readObject();
+      } catch (FileNotFoundException fnf) {
+        this.metadata = Optional.empty();
+        this.metadataStatus = Optional.of(MetadataStatus.missing);
+        return;
+      } catch (JsonException je) {
+        this.metadata = Optional.empty();
+        this.metadataStatus = Optional.of(MetadataStatus.malformed);
+        return;
+      }
+
+      // Validate metadata file
+      if(!validateMetadataFile(fileContents)) {
+        this.metadata = Optional.empty();
+        this.metadataStatus = Optional.of(MetadataStatus.malformed);
+        return;
+      }
+
+      this.metadata = Optional.of(fileContents);
+      this.metadataStatus = Optional.of(MetadataStatus.ok);
+    }
+
+    private static boolean validateMetadataFile(JsonObject metadata) throws JsonException {
+      // Check that there's the right amount of keys
+      if(metadata.size() > MetadataKeys.values().length || metadata.size() < MetadataKeys.mandatoryKeys.size()) {
+        return false;
+      }
+
+      // Check that all the keys are real keys and the mandatory keys are present
+      if(!MetadataKeys.keySet.containsAll(metadata.keySet()) || !metadata.keySet().containsAll(MetadataKeys.mandatoryKeys)) {
+        return false;
+      }
+
+      // Validate that the keys have the correct types
+      try {
+        final var version = metadata.getString("version");
+        // Version should be a recognized version
+        if (!version.equals("1")) {
+          return false;
+        }
+
+        metadata.getString("createdBy");
+
+        final var createdAt = metadata.getString("createdAt");
+        Instant.parse(createdAt);
+
+        metadata.getString("lastEditedBy");
+        final var lastEditedAt = metadata.getString("lastEditedAt");
+        Instant.parse(lastEditedAt);
+
+
+        // Validate that the optional keys have the correct type, if they're present
+        if (metadata.containsKey("readOnly")) {
+          metadata.getBoolean("readOnly");
+        }
+        if(metadata.containsKey("user")) {
+          metadata.getJsonObject("user");
+        }
+      } catch (Exception cce) {
+        return false;
+      }
+      return true;
     }
 
     JsonObjectBuilder toJsonBuilder() {
-      return Json.createObjectBuilder()
-                 .add("name", name)
-                 .add("type", renderType.name());
+      final var builder = Json.createObjectBuilder()
+                              .add("name", name)
+                              .add("type", renderType.name());
+
+      metadataStatus.ifPresent(status -> {
+        builder.add("metadataStatus", status.name());
+        if(status == MetadataStatus.ok && metadata.isPresent()) {
+          builder.add("metadata", metadata.get());
+        } else {
+          builder.add("metadata", JsonValue.NULL);
+        }
+      });
+
+      return builder;
     }
   }
 
@@ -93,7 +211,12 @@ void addChild(FileNode child) {
     @Override
     JsonObjectBuilder toJsonBuilder() {
       final var contentsArray = Json.createArrayBuilder();
-      children.forEach((key, child) -> contentsArray.add(child.toJsonBuilder()));
+      children.forEach((key, child) -> {
+        // Skip Metadata files by default
+        if(child.renderType != RenderType.METADATA) {
+          contentsArray.add(child.toJsonBuilder());
+        }
+      });
       return Json.createObjectBuilder()
                  .add("name", name)
                  .add("type", renderType.name())
@@ -102,11 +225,17 @@ JsonObjectBuilder toJsonBuilder() {
   }
 
   /**
-   * Build a JsonArray representing the contents of this DirectoryTree
+   * Build a JsonArray representing the contents of this DirectoryTree.
+   * By default, skips METADATA type files.
    */
   public JsonArray toJson() {
     final var contentsArray = Json.createArrayBuilder();
-    root.children.forEach((key, child) -> contentsArray.add(child.toJsonBuilder()));
+    root.children.forEach((key, child) -> {
+      // Skip Metadata files
+      if(child.renderType != RenderType.METADATA) {
+        contentsArray.add(child.toJsonBuilder());
+      }
+    });
     return contentsArray.build();
   }
 }

workspace-server/src/main/java/gov/nasa/jpl/aerie/workspace/server/WorkspaceBindings.java

@@ -80,7 +80,7 @@ String fileName() {
     }
 
     String metadataFileName() {
-      return "."+fileName()+".meta.seqdev";
+      return RenderType.toMetadataFileName(fileName());
     }
   }
 
@@ -378,19 +378,17 @@ private void listWorkspaceContents(Context context) {
   private void listContents(Context context) {
     final var workspaceId = Integer.parseInt(context.pathParam("workspaceId"));
 
-    final Optional<Path> directoryPath;
-    if(context.pathParamMap().containsKey("path")) {
-      directoryPath = Optional.of(Path.of(context.pathParam("path")));
-    } else {
-      directoryPath = Optional.empty();
-    }
+    final var directoryPath = context.pathParamMap().containsKey("path")
+        ? Path.of(context.pathParam("path"))
+        : Path.of("");
 
     // Query params
     final var depthString = context.queryParam("depth");
     final int depth = depthString != null ? Integer.parseInt(depthString) : -1;
+    final boolean withMetadata = Boolean.parseBoolean(context.queryParam("withMetadata"));
 
     try {
-      final var fileTree = workspaceService.listFiles(workspaceId, directoryPath, depth);
+      final var fileTree = workspaceService.listFiles(workspaceId, directoryPath, depth, withMetadata);
       if (fileTree == null) {
         context.status(404).json(new FormattedError("No such directory."));
         return;

workspace-server/src/main/java/gov/nasa/jpl/aerie/workspace/server/WorkspaceFileSystemService.java

@@ -127,7 +127,7 @@ private Path resolveMetadataPath(final Path rootPath, final Path filePath) throw
     }
 
     // Convert base file path to metadata file path
-    final var metadataFileName = "."+baseFilePath.getFileName()+".meta.seqdev";
+    final var metadataFileName = RenderType.toMetadataFileName(baseFilePath.getFileName().toString());
     final var metadataFilePath = baseFilePath.resolveSibling(metadataFileName); // Metadata files are hidden sibling files
 
     if(metadataFilePath.toFile().isDirectory()) {
@@ -404,20 +404,20 @@ public boolean deleteFile(final int workspaceId, final Path filePath)
 
   //region Directory Operations
   @Override
-  public DirectoryTree listFiles(final int workspaceId, final Optional<Path> directoryPath, final int depth)
+  public DirectoryTree listFiles(final int workspaceId, final Path directoryPath, final int depth, final boolean withMetadata)
   throws SQLException, NoSuchWorkspaceException, IOException {
-    final var path = resolveReadingPath(workspaceId, directoryPath.orElse(Path.of("")));
+    final var path = resolveReadingPath(workspaceId, directoryPath);
 
     if(!path.toFile().isDirectory()) {
       return null;
     }
 
-    // Converting our API to the Files API
+    // Convert to our API from the Files API
     final var walkDepth = depth == -1 ? Integer.MAX_VALUE : depth + 1;
     try(final Stream<Path> walkOutput = Files.walk(path, walkDepth)) {
       final var walkList = new ArrayList<>(walkOutput.toList());
       walkList.removeFirst(); // remove the initial path
-      return new DirectoryTree(path, walkList, postgresRepository.getExtensionMapping());
+      return new DirectoryTree(path, walkList, postgresRepository.getExtensionMapping(), withMetadata);
     }
   }
 

workspace-server/src/main/java/gov/nasa/jpl/aerie/workspace/server/WorkspaceService.java

@@ -82,7 +82,7 @@ boolean moveFile(final int oldWorkspaceId, final Path oldFilePath, final int new
   boolean deleteFile(final int workspaceId, final Path filePath)
   throws NoSuchWorkspaceException, WorkspaceFileOpException;
 
-  DirectoryTree listFiles(final int workspaceId, final Optional<Path> directoryPath, final int depth)
+  DirectoryTree listFiles(final int workspaceId, final Path directoryPath, final int depth, final boolean withMetadata)
   throws SQLException, NoSuchWorkspaceException, IOException;
 
   boolean createDirectory(final int workspaceId, final Path directoryPath)

workspace-server/src/main/java/gov/nasa/jpl/aerie/workspace/server/postgres/RenderType.java

@@ -57,4 +57,18 @@ public static RenderType getRenderType(String fileName, Map<String, RenderType>
 
     return RenderType.UNKNOWN;
   }
+
+  // Helpers for SeqDev Metadata Files
+  public static final String aerieMetadataExtension = ".meta.seqdev";
+  public static boolean isAerieMetadataFile(String fileName) {
+    return getRenderType(fileName, Map.of(aerieMetadataExtension, METADATA)) == METADATA;
+  }
+
+  /**
+   * Convert a fileName to the naming pattern of a PlanDev metadata file.
+   * @param fileName The base file's filename
+   */
+  public static String toMetadataFileName(String fileName) {
+    return "." + fileName + aerieMetadataExtension;
+  }
 }

workspace-server/src/main/java/gov/nasa/jpl/aerie/workspace/server/types/MetadataKeys.java

@@ -11,5 +11,14 @@ public enum MetadataKeys {
   readOnly,
   user;
 
-  public static final Set<String> whitelist = Set.of("readOnly", "user");
+  public static final Set<String> whitelist = Set.of(readOnly.name(), user.name());
+  public static final Set<String> keySet = Set.of(
+      version.name(),
+      createdBy.name(), createdAt.name(),
+      lastEditedBy.name(), lastEditedAt.name(),
+      readOnly.name(), user.name());
+  public static final Set<String> mandatoryKeys = Set.of(version.name(),
+                                                         createdBy.name(), createdAt.name(),
+                                                         lastEditedBy.name(), lastEditedAt.name());
+
 }