281 user friendly decommutation of received reports (#332)

* Grafana updates including: Upgraded to v12, switched internal DB to PostgresSQL, consolidated dashboard+widget definitions and moved to a text provisioning file, and updated anms-ui monitor to use the new dashboard.  Address #282 and #283

* Added to UI a prototype Help page and a Not Found page handler.

* Resolve default database warning in grafana (hopefully).

* Added upgrading file

* Add amp-manager to UI Status page

* added inifinity-datasource plugin and example panels

* updated layout of panels

* Fix core '/services' REST API to return JSON instead of text

* Core REST API bugfixes to detect timeouts and resolve potential error with overloaded status variable.

* initial dev creating a internal transcoder

* working version, added new profile default is internal transcoding

* working transcoder fixed issue with threading and DB

* Automatically create GHCR images when a tag is created. This supplements the formal release process which also triggers publication.

* Fix ghcr name if triggered by tag

* Fixing grafana authnz/demo websocket proxy configuration.

* removed services related to transcoder since they arent up by default

* added --ignore flag for podman volume creaete to avoid name error

* moved grafana db password to enviroment var

* moving adms init

* Updated REST APIs to return correct error codes, and updated dtnma-tools to bring in fixes for SQL concurrency causing reliability issues.

* moved init adms logic into transcoder

* added new folder for adding new ADMS in core

* added gitignore to extra_adms to avoid extra adms be added to main

* Add REFDA startup to allow ipn-scheme managers all access

* Updating all deps references.

* Force CI build to run with no-cache

* moved init logic to avoid loop startup error

* loading default ADMs controlled by a new route

* Revert "Force CI build to run with no-cache"

This reverts commit b19e333.

* Address SonarQube documentation warning.

* Use HTTP status code enums consistently.

* setting cache to store processed reports

* Fixed an additional error return.

* updated to to use newest SQL for reports

* added COPY for startup.uri

* added for querying for report templates

* updated grafana to use new rptsets

* updated to include rpt_item_index

* updated to uses time_offset as a timestamp

* updated test execset

* removed special char

* adding logs

* latest dtnma-tools

* updated to latest dntma-tools

* removed rpt_item_index

* changed time_offset to agent_time

* updated to align with latest dtnma-tools

* combining reports that are the same ARI but stored with nn or no nn

* adding missing _ in transcoder_put_str

* removed extra logs

* addressing sonar issues

---------

Co-authored-by: David Edell <david.edell@jhuapl.edu>
Co-authored-by: David Linko <david.linko@jhuapl.edu>
Co-authored-by: Brian Sipos <brian.sipos@jhuapl.edu>

Author: 4 people

anms-core/anms/components/schemas/ARIs/__init__.py

@@ -53,4 +53,5 @@
 from .registered_agent import RegisteredAgentInDBBase
 from .rpt_entry import RptEntry
 from .rpt_entry import RptEntryName
+from .rpt_entry import RptEntryFull
 from .rpt_entry import RptEntryBaseInDBBase

anms-core/anms/components/schemas/ARIs/rpt_entry.py

@@ -32,12 +32,25 @@ class RptEntryBase(BaseModel):
     class Config:
         arbitrary_types_allowed = True
 
-    ari_rptset_id: Optional[str] = None
-    reference_time: Optional[datetime] = None
-    report_list: Optional[str] = None
-    agent_id: Optional[int] = None
-
+    ari_rptset_id: Optional[int] = None
+    reference_time: Optional[datetime] = None 
+    mgr_time: Optional[datetime] = None 
+    nonce_cbor: Optional[str] = None
+    agent_time: Optional[datetime] = None
+    report_source: Optional[str] = None
+    report_items: Optional[list] = None
+    
+# Shared properties
+class RptEntryFull(RptEntryBase):
+    class Config:
+        arbitrary_types_allowed = True
+        orm_mode = True
 
+    agent_id: Optional[int] = None
+    ari_rptlist_id: Optional[int] = None
+    
+    
+    
 class RptEntryBaseInDBBase(RptEntryBase):
     class Config:
         orm_mode = True

anms-core/anms/models/relational/const.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023 The Johns Hopkins University Applied Physics
+# Laboratory LLC.
+#
+# This file is part of the Asynchronous Network Management System (ANMS).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This work was performed for the Jet Propulsion Laboratory, California
+# Institute of Technology, sponsored by the United States Government under
+# the prime contract 80NM0018D0004 between the Caltech and NASA under
+# subcontract 1658085.
+#
+from typing import Any
+from typing import Dict
+
+from anms.models.relational import Model
+from sqlalchemy import Column
+from sqlalchemy import Integer
+from sqlalchemy import String
+
+# class for vw_ctrl_definition used for build ari
+class Const(Model):
+    __tablename__ = 'vw_const_actual'
+    obj_actual_definition_id        = Column(Integer,  primary_key=True)
+    data_type	 = Column(String)
+    data_value	 = Column(String)
+    use_desc	 = Column(String)
+    obj_metadata_id        = Column(Integer)
+    data_model_name = Column(String)
+    namespace = Column(String)
+    data_type_id        = Column(Integer)
+    name	 = Column(String)
+    data_model_id        = Column(Integer)
+    object_enumeration        = Column(Integer)
+    status	 = Column(String)
+    reference	 = Column(String)
+    description = Column(String)
+
+    def __repr__(self) -> str:
+        return self.as_dict().__repr__()
+
+    def as_dict(self) -> Dict[str, Any]:
+        dict_obj = {
+            c.name: getattr(self, c.name) for c in self.__table__.columns
+        }
+
+        return dict_obj

anms-core/anms/models/relational/execution_set.py

@@ -29,6 +29,7 @@
 from sqlalchemy import Integer
 from sqlalchemy import String
 from sqlalchemy import LargeBinary
+from anms.shared.transmogrifier import TRANSMORGIFIER
 
 
 # class for vw_ctrl_definition used for build ari
@@ -47,7 +48,7 @@ def __repr__(self) -> str:
     def as_dict(self) -> Dict[str, Any]:
         dict_obj = {
                     'execution_set_id': getattr(self, 'execution_set_id'),
-                    'nonce_cbor': getattr(self, 'nonce_cbor'),        
+                    'nonce_cbor': TRANSMORGIFIER._ace_transcode_just_cbor("0x"+getattr(self, 'nonce_cbor').hex()),        
                     'use_desc': getattr(self, 'use_desc'),
                     'agent_id': getattr(self, 'agent_id'),
                     'num_entries': getattr(self, 'num_entries'),

anms-core/anms/models/relational/report.py

@@ -24,33 +24,48 @@
 from typing import Any
 from typing import Dict
 
+from anms.shared.transmogrifier import TRANSMORGIFIER
 from anms.models.relational import Model
 from sqlalchemy import Column
 from sqlalchemy import Integer
-from sqlalchemy import String
+from sqlalchemy import DateTime
+from sqlalchemy import ARRAY
 from sqlalchemy import LargeBinary
-
+from sqlalchemy import orm
 
 # class for vw_ctrl_definition used for build ari
 class Report(Model):
-    __tablename__ = 'ari_rptset'
-    ari_rptset_id = Column(Integer, primary_key=True)
-    nonce_cbor = Column(LargeBinary)
-    reference_time = Column(Integer)
-    report_list	= Column(String)
-    report_list_cbor = Column(LargeBinary)
-    agent_id = Column(Integer)
+    __tablename__ = 'vw_ari_rpt_set'
+    ari_rptset_id	= Column(Integer, primary_key=True)
+    mgr_time	    = Column(DateTime)
+    reference_time	= Column(DateTime)
+    nonce_cbor	    = Column(LargeBinary)
+    agent_id	    = Column(Integer)
+    ari_rptset_cbor	= Column(LargeBinary)
+    ari_rptlist_id	= Column(Integer)
+    agent_time	    = Column( DateTime)
+    report_source	= Column(LargeBinary)
+    report_items	= Column(ARRAY(LargeBinary) )#bytea[] NULL	
+
+    # processing the raw cbor into an ari object
+    @orm.reconstructor
+    def init_on_load(self):
+        self.nonce_cbor =  TRANSMORGIFIER.transcode("0x"+getattr(self, 'nonce_cbor').hex())['uri']        
+        self.report_source =  TRANSMORGIFIER.transcode("0x"+getattr(self, 'report_source').hex())['uri']
+        self.report_items =  [TRANSMORGIFIER.transcode("0x"+x.hex())['uri'] for x in getattr(self, 'report_items')]
+
     def __repr__(self) -> str:
         return self.as_dict().__repr__()
 
     def as_dict(self) -> Dict[str, Any]:
         dict_obj = {
                     'ari_rptset_id': getattr(self, 'ari_rptset_id'),
-                    'nonce_cbor': getattr(self, 'nonce_cbor'),        
                     'reference_time': getattr(self, 'reference_time'),
-                    'report_list': getattr(self, 'report_list'),
-                    'report_list_cbor': getattr(self, 'report_list_cbor'),
-                    'agent_id': getattr(self, 'agent_id')       
+                    'nonce_cbor': getattr(self, 'nonce_cbor'),
+                    'agent_id': getattr(self, 'agent_id'),       
+                    'ari_rptlist_id': getattr(self, 'ari_rptlist_id'),
+                    'agent_time': getattr(self, 'agent_time'),
+                    'report_source': getattr(self, 'report_source'),
+                    'report_items': getattr(self, 'report_items')
                     }
-
         return dict_obj