Merge pull request #27 from sightmachine/ENG-721_show_hidden

[ENG-721] Show_hidden, Bonus Bug Fix

Author: mgoldmansight

docs/entities/machine.md

@@ -4,12 +4,50 @@ Machines are just that, machines in factories.  The machine object is how the Si
 ## Functions
 
 ### get_type_from_machine
-The get_type_from_machine function allows you to get the type of any machine from it's name and is called this way:
+The get_type_from_machine function allows you to get the type of any machine from it's name(or display name) and is called this way:
 ```
 cli.get_type_from_machine(machine_name)
 ```
 
 And will return something like the following:
 ```
 'Lasercut'
+```
+
+### get_machine_schema
+The get_machine_schema functions returns the fields of the machine schema for a given machine source and is called this way:
+```
+cli.get_machine_schema(machine_source, types, show_hidden, return_mtype)
+```
+
+The only required field in this case is the machine_source, we will go over each variable in a second.  The function will return a pandas data frame that looks like the following:
+```
+                            name                       display         type
+0             stats__Alarms__val                        Alarms        float
+1            stats__BLOCKED__val                       BLOCKED        float
+2               stats__DOWN__val                          DOWN        float
+3     stats__DefectCategory__val               Defect Category       string
+```
+
+#### machine_source
+This is the name of the machine that you are trying to grab the schema of.  This will also work with it's display name or source_clean.  This is the only required parameter for this function.
+
+#### types
+This is an optional parameter and is a list of strings.  If this is set the function will only return colomns that match the types given.  For example if we were to pass ['string'] as our types parameter in the previous example we would instead have returned:
+```
+                            name                       display         type
+0     stats__DefectCategory__val               Defect Category       string
+```
+
+#### show_hidden
+This is an optional parameter and is a boolean.  There are a few fields we have set to be hiddden from our ui in our application and by defualt these are also hidden from the return in this function if set to True we will also return these fields.
+
+#### return_mtype
+This is an optional parameter and is a boolean. If set to True this will instead of returning just the pandas dataframe will return a Tupple with a string that is the machine type of the machine_soure for example:
+```
+('Lasercut',                             
+                            name                       display         type
+0             stats__Alarms__val                        Alarms        float
+1            stats__BLOCKED__val                       BLOCKED        float
+...)
 ```

docs/entities/machine_type.md

@@ -0,0 +1,27 @@
+# Machine Types
+Machine Types are the schema for the various machines in a factory. 
+
+## Functions
+
+### get_fields_of_machine_type
+The get_fields_of_machine_type function returns the fields of the machine schema for a given machine type and is called this way:
+```
+cli.get_fields_of_machine_type(machine_type, types, show_hidden)
+```
+
+The only required field in this case is the machine_type, we will go over each variable in a second.  The function will return a pandas list that looks like the following:
+```
+[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}, {'display_name': 'Cycle Start Time', 'unit': '', 'type': 'datetime', 'data_type': 'datetime', 'stream_types': [], 'raw_data_field': '', 'name': 'starttime'},..]
+```
+
+#### machine_type
+This is the name of the machine type that you are trying to grab the fields of.  This is the only required parameter for this function.
+
+#### types
+This is an optional parameter and is a list of strings.  If this is set the function will only return colomns that match the types given.  For example if we were to pass ['string'] as our types parameter in the previous example we would instead have returned:
+```
+[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}]
+```
+
+#### show_hidden
+This is an optional parameter and is a boolean.  There are a few fields we have set to be hiddden from our ui in our application and by defualt these are also hidden from the return in this function if set to True we will also return these fields.

smsdk/client.py

@@ -210,24 +210,6 @@ def get_parts(self, normalize=True, clean_strings_in=True, clean_strings_out=Tru
         df = self.get_data_v1('part_v1', 'get_parts', normalize, *args, **kwargs)
 
         return df
-
-
-    @ClientV0.get_machine_schema_decorator
-    def get_machine_schema(self, machine_source, types=[], return_mtype=False, **kwargs):
-        stats = kwargs.get('stats', [])
-        fields = []
-        for stat in stats:
-            if not stat.get('display', {}).get('ui_hidden', False):
-                if len(types) == 0 or stat['analytics']['columns'][0]['type'] in types:
-                    try:
-                        fields.append({'name': stat['analytics']['columns'][0]['name'],
-                                       'display': stat['display']['title_prefix'],
-                                       'type': stat['analytics']['columns'][0]['type']})
-                    except:
-                        log.warning(
-                            f"Unknow stat schema identified :: machine_type {machine_source} - "
-                            f"title_prefix :: {stat.get('display', {}).get('title_prefix', '')}")
-        return fields
 
     def get_kpis(self, **kwargs):
         kpis = smsdkentities.get('kpi')
@@ -277,4 +259,31 @@ def get_type_from_machine(self, machine_source=None, **kwargs):
             self.config["protocol"], self.tenant, self.config["site.domain"]
         )
         return machine(self.session, base_url).get_type_from_machine_name(machine_source, **kwargs)
+    
+    def get_machine_schema(self, machine_source, types=[], show_hidden=False, return_mtype=False, **kwargs):
+        machineType= smsdkentities.get('machine_type')
+        machine_type = self.get_type_from_machine(machine_source)
+        base_url = get_url(
+            self.config["protocol"], self.tenant, self.config["site.domain"]
+        )
+        fields = machineType(self.session, base_url).get_fields(machine_type, **kwargs)
+        fields = [field for field in fields if not field.get('ui_hidden') or show_hidden]
+        if len(types) > 0:
+            fields = [field for field in fields if field.get('type') in types]
 
+        if return_mtype:
+            return (machine_type, pd.DataFrame(fields))
+
+        return pd.DataFrame(fields)
+    
+    def get_fields_of_machine_type(self, machine_type, types=[], show_hidden=False, **kwargs):
+        machineType= smsdkentities.get('machine_type')
+        base_url = get_url(
+            self.config["protocol"], self.tenant, self.config["site.domain"]
+        )
+        fields = machineType(self.session, base_url).get_fields(machine_type, **kwargs)
+        fields = [field for field in fields if not field.get('ui_hidden') or show_hidden]
+        if len(types) > 0:
+            fields = [field for field in fields if field.get('type') in types]
+
+        return fields

smsdk/client_v0.py

@@ -424,7 +424,7 @@ def inner(self, normalize=True, clean_strings_in=True, clean_strings_out=True, d
     def get_machine_schema_decorator(func):
 
         @functools.wraps(func)
-        def inner(self, machine_source, types=[], return_mtype=False, **kwargs):
+        def inner(self, machine_source, types=[], show_hidden=False, return_mtype=False, **kwargs):
 
             try:
                 machine_type = self.get_machines(source=machine_source)['source_type'][0]
@@ -448,7 +448,7 @@ def inner(self, machine_source, types=[], return_mtype=False, **kwargs):
                 print(f"Exception in getting machine type stats {ex}")
             kwargs['stats'] = stats
 
-            fields = func(self, machine_source, types=[], return_mtype=False, **kwargs)
+            fields = func(self, machine_source, types, show_hidden, return_mtype, **kwargs)
 
             if return_mtype:
                 return machine_type, pd.DataFrame(fields)

smsdk/config/api_endpoints.json

@@ -11,7 +11,8 @@
     "url" : "/api/factory"
   },
   "MachineType": {
-    "url" : "/api/machinetype"
+    "url" : "/api/machinetype",
+    "fields": "/v1/selector/datatab/cycle/{}/field"
   },
   "Machine": {
     "url" : "/api/machine"

smsdk/smsdk_entities/machine/machine.py

@@ -55,6 +55,6 @@ def get_type_from_machine_name(self, machine_source, *args, **kwargs):
         records = self._get_records_v1(url, method="get", **kwargs)[0]["machine"]
         machine_type = ''
         for record in records:
-            if record['name'] == machine_source:
+            if record['name'] == machine_source or record['display_name'] == machine_source:
                 machine_type = record['type']
         return machine_type

smsdk/smsdk_entities/machine_type/machinetype.py

@@ -43,3 +43,16 @@ def get_machine_types(self, *args, **kwargs):
         if not isinstance(records, List):
             raise ValueError("Error - {}".format(records))
         return records
+    
+    @mod_util
+    def get_fields(self, machine_type, *args, **kwargs):
+        """
+        Utility function to get the machine types
+        from the ma machine API
+        Recommend to use 'enable_pagination':True for larger datasets
+        """
+        url = "{}{}".format(self.base_url, ENDPOINTS["MachineType"]["fields"].format(machine_type))
+        records = self._get_records(url, **kwargs)
+        if not isinstance(records, List):
+            raise ValueError("Error - {}".format(records))
+        return records

tests/machine/machine_data.py

@@ -255,3 +255,22 @@
         "source_type": "UpperComponentBacking",
     },
 ]
+
+MACHINE_TYPE =[
+            {
+                'type': 'float',
+                'name': 'stat__test_float',
+                'title_prefix': 'test float'
+            },
+            {
+               'type': 'string', 
+               'name': 'stat__test_string',
+               'title_prefix': 'test string'
+            },
+            {
+               'type': 'string', 
+               'name': 'stat__test_hidden',
+                'title_prefix': 'test hidden',
+                'ui_hidden' : True
+            }
+        ]

tests/machine/test_machine.py

@@ -3,7 +3,7 @@
 import pandas as pd
 from requests.sessions import Session
 from smsdk.client import Client
-from tests.machine.machine_data import JSON_MACHINE
+from tests.machine.machine_data import JSON_MACHINE, MACHINE_TYPE
 from smsdk.smsdk_entities.machine.machine import Machine
 
 
@@ -59,3 +59,62 @@ def json():
 
     # Verify
     assert type == 'test_type'
+
+
+@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields")
+@patch("smsdk.client.Client.get_type_from_machine")
+def test_get_machine_schema(mocked_types, mocked_machines):
+    mocked_machines.return_value = MACHINE_TYPE
+    mocked_types.return_value = 'test'
+    dt = Client("demo")
+
+    # Run
+    fields = dt.get_machine_schema('test')
+    assert fields.shape == (2, 3)
+    # Verify
+    assert fields.name.sort_values().tolist() == ['stat__test_float', 'stat__test_string']
+
+
+@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields")
+@patch("smsdk.client.Client.get_type_from_machine")
+def test_get_machine_schema_hidden(mocked_types, mocked_machines):
+    mocked_machines.return_value = MACHINE_TYPE
+    mocked_types.return_value = 'test'
+    dt = Client("demo")
+
+    # Run
+    fields = dt.get_machine_schema('test', show_hidden=True)
+    assert fields.shape == (3, 4)
+
+    # Verify
+    assert fields.name.sort_values().tolist() == ['stat__test_float', 'stat__test_hidden', 'stat__test_string']
+
+@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields")
+@patch("smsdk.client.Client.get_type_from_machine")
+def test_get_machine_schema_types(mocked_types, mocked_machines):
+    mocked_machines.return_value = MACHINE_TYPE
+    mocked_types.return_value = 'test'
+    dt = Client("demo")
+
+    # Run
+    fields = dt.get_machine_schema('test', types=['float'])
+    assert fields.shape == (1, 3)
+
+    # Verify
+    assert fields.name.sort_values().tolist() == ['stat__test_float']
+
+@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields")
+@patch("smsdk.client.Client.get_type_from_machine")
+def test_get_machine_schema_types_return_mtype(mocked_types, mocked_machines):
+    mocked_machines.return_value = MACHINE_TYPE
+    mocked_types.return_value = 'test'
+    dt = Client("demo")
+
+    # Run
+    fields = dt.get_machine_schema('test', return_mtype=True)
+    assert fields[0] == 'test'
+    assert fields[1].shape == (2, 3)
+    # Verify
+    assert fields[1].name.sort_values().tolist() == ['stat__test_float', 'stat__test_string']
+
+

tests/machine_type/machine_type_data.py

@@ -4411,3 +4411,22 @@
         },
     },
 ]
+
+MACHINE_TYPE_FIELDS =[
+            {
+                'type': 'float',
+                'name': 'stat__test_float',
+                'title_prefix': 'test float'
+            },
+            {
+               'type': 'string', 
+               'name': 'stat__test_string',
+               'title_prefix': 'test string'
+            },
+            {
+               'type': 'string', 
+               'name': 'stat__test_hidden',
+                'title_prefix': 'test hidden',
+                'ui_hidden' : True
+            }
+        ]