[Feature] Postgres Persistence

dataformat/json/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/dataformat/json/DeserializerWrapper.java

@@ -25,7 +25,7 @@
 public class DeserializerWrapper extends org.eclipse.digitaltwin.aas4j.v3.dataformat.json.JsonDeserializer {
 
     public DeserializerWrapper() {
-
+        useImplementation(Message.class, de.fraunhofer.iosb.ilt.faaast.service.model.api.Message.class);
     }
 
 

docs/source/interfaces/persistence.md

@@ -106,3 +106,39 @@ Each modification of the model results in only writing the specific part to the
 	//...
 }
 ```
+
+## Postgres
+
+The Postgres Persistence stores the AAS model in a Postgres DB according to the AAS specification.
+Therefore, changes are stored permanently even when FA³ST Service is stopped or crashes.
+
+:::{important}
+Each modification of the model results in only writing the specific part to the Postgres table which should improve performance
+:::
+
+### Configuration
+
+:::{table} Configuration properties of MongoDB-based Persistence.
+| Name                     | Allowed Value       | Description                                                                                            | Default Value |
+| -------------------------| ------------------- | ------------------------------------------------------------------------------------------------------ | ------------- |
+| jdbcUrl<br>              | String              | The connection string where the PostgresDB is located.                                                 |               |
+| username<br>             | String              | The username to connect to Postgres.                                                                   |               |
+| password<br>             | String              | The password to connect to Postgres.                                                                   |               |
+| override<br>*(optional)* | Boolean             | If true, FA³ST persistence will always override the previous database, this might result in data loss. | false         |
+
+:::
+
+```{code-block} json
+:caption: Example configuration for Postgres-based Persistence.
+:lineno-start: 1
+{
+	"persistence" : {
+		"@class" : "de.fraunhofer.iosb.ilt.faaast.service.persistence.postgres.PersistencePostgres",
+		"jdbcUrl" : "jdbc:postgresql://localhost:5432/faaast",
+		"username": "faaast",
+		"password": "faaast",
+		"override": true
+	},
+	//...
+}
+```

persistence/file/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/persistence/file/PersistenceFile.java

@@ -61,7 +61,7 @@
 
 
 /**
- * Implementation of {@link de.fraunhofer.iosb.ilt.faaast.service.persistence.Persistence} for a file storage.
+ * Implementation of {@link Persistence} for a file storage.
  *
  * <p>Following types are not supported in the current version:
  * <ul>

persistence/file/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/persistence/file/PersistenceFileConfig.java

@@ -57,9 +57,9 @@ public PersistenceFileConfig() {
     /**
      * Sets the file name according to the configuration parameters.
      *
-     * @throws de.fraunhofer.iosb.ilt.faaast.service.exception.ConfigurationInitializationException if initialModelFile
+     * @throws ConfigurationInitializationException if initialModelFile
      *             is present and cannot be parsed
-     * @throws de.fraunhofer.iosb.ilt.faaast.service.exception.ConfigurationInitializationException if dataDir is not a
+     * @throws ConfigurationInitializationException if dataDir is not a
      *             valid path
      */
     public void init() throws ConfigurationInitializationException {

persistence/memory/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/persistence/memory/PersistenceInMemory.java

@@ -75,7 +75,7 @@
 
 
 /**
- * Implementation of {@link de.fraunhofer.iosb.ilt.faaast.service.persistence.Persistence} for in memory storage.
+ * Implementation of {@link Persistence} for in memory storage.
  *
  * <p>Following types are not supported in the current version:
  * <ul>

persistence/mongo/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/persistence/mongo/PersistenceMongo.java

@@ -211,6 +211,7 @@ public void deleteSubmodelElement(SubmodelElementIdentifier identifier) throws R
 
     @Override
     public void deleteAll() throws PersistenceException {
+        LOGGER.debug("Dropping all AAS collections from MongoDB.");
         aasCollection = resetCollection(AAS_COLLECTION_NAME);
         submodelCollection = resetCollection(SUBMODEL_COLLECTION_NAME);
         cdCollection = resetCollection(CD_COLLECTION_NAME);
@@ -434,9 +435,14 @@ public void start() throws PersistenceException {
 
         if (config.isOverride()) {
             deleteAll();
+            try {
+                saveEnvironment(config.loadInitialModel());
+            }
+            catch (DeserializationException | InvalidConfigurationException | IllegalStateException e) {
+                throw new PersistenceException(e);
+            }
         }
-
-        if (!databaseHasSavedEnvironment(database)) {
+        else if (!databaseHasSavedEnvironment(database)) {
             deleteAll();
             try {
                 saveEnvironment(config.loadInitialModel());
@@ -546,10 +552,14 @@ private <T extends Referable> Stream<T> asPojo(MongoIterable<Document> documents
     private boolean databaseHasSavedEnvironment(MongoDatabase database) {
         List<String> collectionNames = new ArrayList<>();
         database.listCollectionNames().into(collectionNames);
-        return collectionNames.contains(AAS_COLLECTION_NAME)
+        boolean dataPresent = collectionNames.contains(AAS_COLLECTION_NAME)
                 || collectionNames.contains(SUBMODEL_COLLECTION_NAME)
                 || collectionNames.contains(CD_COLLECTION_NAME)
                 || collectionNames.contains(OPERATION_COLLECTION_NAME);
+        if (dataPresent) {
+            LOGGER.error("Database is not empty and model will not be saved.");
+        }
+        return dataPresent;
     }
 
 
@@ -768,6 +778,7 @@ private Bson getSemanticIdFilter(Reference semanticId) throws PersistenceExcepti
 
 
     private void saveEnvironment(Environment environment) throws PersistenceException {
+        LOGGER.debug("Reading environment from file provided in config and storing in mongoDB collections.");
         save(environment.getAssetAdministrationShells(), aasCollection);
         save(environment.getSubmodels(), submodelCollection);
         save(environment.getConceptDescriptions(), cdCollection);

persistence/postgres/pom.xml

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>de.fraunhofer.iosb.ilt.faaast.service</groupId>
+        <artifactId>service</artifactId>
+        <version>1.4.0-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+    <artifactId>persistence-postgres</artifactId>
+    <name>persistence-postgres</name>
+    <properties>
+        <root.basedir>${project.parent.basedir}</root.basedir>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>core</artifactId>
+            <version>${project.version}</version>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>dataformat-json</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>model</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>model</artifactId>
+            <version>${project.version}</version>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>persistence-memory</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+            <version>${hikari.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.zonky.test</groupId>
+            <artifactId>embedded-postgres</artifactId>
+            <version>${embedded-postgres.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>${mockito.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>${postgres.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <version>${slf4j.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-checkstyle-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>

persistence/postgres/src/main/java/de/fraunhofer/iosb/ilt/faaast/service/persistence/postgres/DatabaseSchema.java

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2021 Fraunhofer IOSB, eine rechtlich nicht selbstaendige
+ * Einrichtung der Fraunhofer-Gesellschaft zur Foerderung der angewandten
+ * Forschung e.V.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package de.fraunhofer.iosb.ilt.faaast.service.persistence.postgres;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+
+/**
+ * Database schema definition for PostgreSQL persistence.
+ */
+public final class DatabaseSchema {
+
+    /** Table name for Asset Administration Shells. */
+    public static final String TABLE_AAS = "aas";
+
+    /** Table name for Submodels. */
+    public static final String TABLE_SUBMODEL = "submodels";
+
+    /** Table name for Concept Descriptions. */
+    public static final String TABLE_CONCEPT_DESCRIPTION = "concept_descriptions";
+
+    /** Table name for Operation Results. */
+    public static final String TABLE_OPERATION_RESULT = "operation_results";
+
+    public static final String DROP_TABLE = "DROP TABLE IF EXISTS ";
+    public static final String CASCADE = " CASCADE";
+
+    private DatabaseSchema() {}
+
+
+    /**
+     * Creates all database tables and indexes.
+     *
+     * @param connection the database connection
+     * @throws SQLException if a database error occurs
+     */
+    public static void createTables(final Connection connection) throws SQLException {
+        try (Statement stmt = connection.createStatement()) {
+            stmt.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm");
+            stmt.execute(getAasTableCreate());
+            stmt.execute(getSubmodelsTableCreate());
+            stmt.execute(getConceptDescriptionTableCreate());
+            stmt.execute(getOperationResultTableCreate());
+            stmt.execute(getIndexesCreate());
+        }
+    }
+
+
+    private static String getAasTableCreate() {
+        return """
+                CREATE TABLE IF NOT EXISTS %s (
+                    id TEXT PRIMARY KEY,
+                    id_short TEXT,
+                    content JSONB NOT NULL,
+                    seq BIGSERIAL
+                )
+                """.formatted(TABLE_AAS);
+    }
+
+
+    private static String getSubmodelsTableCreate() {
+        return """
+                CREATE TABLE IF NOT EXISTS %s (
+                    id TEXT PRIMARY KEY,
+                    id_short TEXT,
+                    semantic_id TEXT,
+                    content JSONB NOT NULL,
+                    seq BIGSERIAL
+                )
+                """.formatted(TABLE_SUBMODEL);
+    }
+
+
+    private static String getConceptDescriptionTableCreate() {
+        return """
+                CREATE TABLE IF NOT EXISTS %s (
+                    id TEXT PRIMARY KEY,
+                    id_short TEXT,
+                    content JSONB NOT NULL,
+                    seq BIGSERIAL
+                )
+                """.formatted(TABLE_CONCEPT_DESCRIPTION);
+    }
+
+
+    private static String getOperationResultTableCreate() {
+        return """
+                CREATE TABLE IF NOT EXISTS %s (
+                    id TEXT PRIMARY KEY,
+                    content JSONB NOT NULL,
+                    seq BIGSERIAL
+                )
+                """.formatted(TABLE_OPERATION_RESULT);
+    }
+
+
+    private static String getIndexesCreate() {
+        return """
+                CREATE INDEX IF NOT EXISTS idx_aas_id_short
+                    ON %s(id_short);
+                CREATE INDEX IF NOT EXISTS idx_submodel_id_short
+                    ON %s(id_short);
+                CREATE INDEX IF NOT EXISTS idx_submodel_semantic_id
+                    ON %s USING GIST (semantic_id gist_trgm_ops);
+                CREATE INDEX IF NOT EXISTS idx_concept_description_id_short
+                    ON %s(id_short);
+                """.formatted(
+                TABLE_AAS,
+                TABLE_SUBMODEL,
+                TABLE_SUBMODEL,
+                TABLE_CONCEPT_DESCRIPTION);
+    }
+
+
+    /**
+     * Drops all database tables.
+     *
+     * @param connection the database connection
+     * @throws SQLException if a database error occurs
+     */
+    public static void dropTables(final Connection connection) throws SQLException {
+        try (Statement stmt = connection.createStatement()) {
+            stmt.execute(DROP_TABLE + TABLE_OPERATION_RESULT
+                    + CASCADE);
+            stmt.execute(DROP_TABLE + TABLE_CONCEPT_DESCRIPTION
+                    + CASCADE);
+            stmt.execute(DROP_TABLE + TABLE_SUBMODEL + CASCADE);
+            stmt.execute(DROP_TABLE + TABLE_AAS + CASCADE);
+        }
+    }
+}