#175 and updates for sdk default flag changes

Author: jbutcher21

sz_tools/sz_explorer

@@ -40,12 +40,10 @@ except ImportError as ex:
     sys.exit(1)
 
 
-def print_exception_info(ex):
-    print_message(ex, "error")
-    response = get_char_with_prompt("press T for traceback...")
-    if response.upper() == "T":
+def print_exception(ex):
+    print_message(f"{ex}, press any key to continue", "error")
+    if get_char() in "tTdD":
         print(traceback.format_exc())
-        get_char_with_prompt("press any key any key to continue...")
         print()
 
 
@@ -411,12 +409,13 @@ class EdaReports:
         # (feature_counts)
         for ftype_code in feature_counts:
             ftype_excl = sdk_wrapper.ftype_code_lookup[ftype_code]["FTYPE_EXCL"]
+            ftype_freq = sdk_wrapper.ftype_code_lookup[ftype_code]["FTYPE_FREQ"]
             ftype_count = feature_counts[ftype_code]
             if ftype_count > entity_size:  # a single record has reported multiple
                 continue
             if ftype_excl == "Yes" and ftype_count > 1:
                 review_features.append(ftype_code)
-            elif ftype_count > review_max:
+            elif ftype_freq in ("F1", "FF") and ftype_count > review_max:
                 review_features.append(ftype_code)
         return review_features
 
@@ -561,7 +560,7 @@ class EdaReports:
         tbl = eda_table()
         tbl.title = "Review categories..."
         tbl.columns = [
-            {"name": "row", "width": 5, "align": "center"},
+            {"name": "Row", "width": 5, "align": "center"},
             {"name": "Category", "width": 25, "align": "left"},
             {"name": "Count", "width": 25, "align": "right"},
         ]
@@ -618,7 +617,7 @@ class EdaReports:
         tbl = eda_table()
         tbl.title = f"Entity size breakdown from {self.snapshot_file}"
         tbl.columns = [
-            {"name": "row", "width": 5, "align": "center"},
+            {"name": "Row", "width": 5, "align": "center"},
             {"name": "Size Group", "width": 10, "align": "center"},
             {"name": "Entity Count", "width": 10, "align": "right"},
             {"name": "Review Count", "width": 10, "align": "right"},
@@ -648,7 +647,7 @@ class EdaReports:
         tbl = eda_table()
         tbl.title = f"Data Source Summary from {self.snapshot_file}"
         tbl.columns = [
-            {"name": "\nrow", "width": 5, "align": "center"},
+            {"name": "\nRow", "width": 5, "align": "center"},
             {"name": "\nData Source", "width": 25, "align": "left"},
             {"name": "\nRecords", "width": 15, "align": "right"},
             {"name": "\nEntities", "width": 15, "align": "right"},
@@ -702,7 +701,7 @@ class EdaReports:
         tbl = eda_table()
         tbl.title = f"Cross Source Summary from {self.snapshot_file}"
         tbl.columns = [
-            {"name": "\nrow", "width": 5, "align": "center"},
+            {"name": "\nRow", "width": 5, "align": "center"},
             {"name": "From\nData Source", "width": 25, "align": "center"},
             {"name": "To\nData Source", "width": 25, "align": "center"},
             {"name": "Matched\nRecords", "width": 15, "align": "right"},
@@ -747,25 +746,25 @@ class EdaReports:
         select_levels = ["DATA_SOURCES", "MATCH_LEVEL", "MATCH_KEY"]
         self.drill_into(report_table, report_data, select_levels)
 
-    def multi_source_summary(self, data_source_filter):
+    def entity_source_summary(self, data_source_filter):
         self.check_for_snapshot()
         report_data = {}
         tbl = eda_table()
-        tbl.title = f"Multi-Source Summary from {self.snapshot_file}"
+        tbl.title = f"Entity Source Summary from {self.snapshot_file}"
         tbl.columns = [
-            {"name": "row", "width": 5, "align": "center"},
+            {"name": "Row", "width": 5, "align": "center"},
             {"name": "Data Sources", "width": 100, "align": "left"},
-            {"name": "Records", "width": 15, "align": "right"},
+            {"name": "Entities", "width": 15, "align": "right"},
         ]
         tbl.rows = []
         row_num = 0
-        _data = self.snapshot_data["MULTI_SOURCES"]
+        _data = self.snapshot_data["ENTITY_SOURCES"]
         sorted_data = sorted(_data, key=lambda k: _data[k]["ENTITY_COUNT"], reverse=True)
         for data_sources in sorted_data:
             if data_source_filter and data_source_filter.upper() not in data_sources:
                 continue
             row_num += 1
-            report_segment = self.snapshot_data["MULTI_SOURCES"][data_sources]
+            report_segment = self.snapshot_data["ENTITY_SOURCES"][data_sources]
             row = [
                 colorize(row_num, "bold"),
                 colorize(" | ", "dim").join(colorize_dsrc(x) for x in data_sources.split("||")),
@@ -805,7 +804,7 @@ class EdaReports:
         tbl = eda_table()
         tbl.title = f"Principles Used Report from {self.snapshot_file}"
         tbl.columns = [
-            {"name": "row", "width": 5, "align": "center"},
+            {"name": "Row", "width": 5, "align": "center"},
             {"name": "Match level", "width": 25, "align": "left"},
             {"name": "Count", "width": 15, "align": "right"},
         ]
@@ -840,7 +839,7 @@ class EdaReports:
                 tbl = eda_table()
                 tbl.title = f"Selected {prior_keys}"
                 tbl.columns = [
-                    {"name": "row", "width": 5, "align": "center"},
+                    {"name": "Row", "width": 5, "align": "center"},
                     {"name": select_level.lower(), "width": 100, "align": "left"},
                     {"name": "count", "width": 10, "align": "right"},
                 ]
@@ -2073,7 +2072,7 @@ class EdaSdkWrapper:
                 tree_nodes[related_id].node_desc = rel_desc
             tree_nodes[group_node_id].add_child(tree_nodes[related_id])
 
-            if len(entities[related_id]["RELATIONSHIPS"]) != 0:
+            if len(entities[related_id]["RELATIONSHIPS"]) != 0 and len(parents) < build_out_degree:
                 related_entities = sorted(
                     entities[related_id]["RELATIONSHIPS"],
                     key=lambda k: (k["MATCH_CATEGORY_SORT"], k["ERRULE_ID"], k["ENTITY_ID"]),
@@ -2361,25 +2360,25 @@ class EdaSdkWrapper:
         return tbl.render_table(no_lines=True)
 
     def why_search(self, search_json, **kwargs):
+        search_flag_list = [
+            "SZ_SEARCH_INCLUDE_REQUEST_DETAILS",
+            "SZ_INCLUDE_FEATURE_SCORES",
+            "SZ_INCLUDE_MATCH_KEY_DETAILS",
+            "SZ_ENTITY_DEFAULT_FLAGS",
+            "SZ_ENTITY_INCLUDE_ENTITY_NAME",
+            "SZ_ENTITY_INCLUDE_INTERNAL_FEATURES",
+            "SZ_ENTITY_INCLUDE_FEATURE_STATS",
+            "SZ_ENTITY_INCLUDE_RECORD_FEATURES",
+        ]
+
         if kwargs["search"] == 0:
-            search_flag_list = [
-                "SZ_SEARCH_INCLUDE_REQUEST_DETAILS",
-                "SZ_SEARCH_INCLUDE_ALL_CANDIDATES",
-                "SZ_INCLUDE_MATCH_KEY_DETAILS",
-                "SZ_WHY_ENTITIES_DEFAULT_FLAGS",
-            ]
+            search_flag_list.append("SZ_SEARCH_INCLUDE_ALL_CANDIDATES")
             try:
                 json_data = self.call_sdk("search_by_attributes", search_flag_list, json.dumps(search_json))
             except SzError as err:
                 raise err from err
             matched_list = sdk_wrapper.reorder_search_results(json_data.get("RESOLVED_ENTITIES", []))
         else:
-            search_flag_list = [
-                "SZ_SEARCH_INCLUDE_REQUEST_DETAILS",
-                "SZ_INCLUDE_MATCH_KEY_DETAILS",
-                "SZ_INCLUDE_FEATURE_SCORES",
-                "SZ_WHY_ENTITIES_DEFAULT_FLAGS",
-            ]
             try:
                 json_data = self.call_sdk("why_search", search_flag_list, [json.dumps(search_json), kwargs["search"]])
             except SzError as err:
@@ -2982,18 +2981,22 @@ class EdaCmd(cmd.Cmd):
                 tree {colorize("- see a tree view of an entity's relationships through 1 or 2 degrees.", 'dim')}
                 export {colorize("- export the json records for an entity for debugging or correcting and reloading.", 'dim')}
 
-            {colorize('Snapshot reports:', 'highlight2')} {colorize('(requires a json file created with sz_snapshot)', 'italics')}
+            {colorize('Snapshot reports:', 'highlight2')} {colorize('(requires a json file generated by sz_snapshot)', 'italics')}
                 data_source_summary {colorize('– shows how many duplicates were detected within each data source, as well as ', 'dim')}
                 {colorize('the possible matches and relationships that were derived. For example, how many duplicate customers ', 'dim')}
                 {colorize('there are, and are any of them related to each other.', 'dim')}
                 cross_source_summary {colorize('– shows how many matches were made across data sources.  For example, how many ', 'dim')}
                 {colorize('employees are related to customers.', 'dim')}
+                entity_source_summary {colorize('– shows the number of entities by the set of data sources they can be found in.  For example, ', 'dim')}
+                {colorize('how many entities are only in one data source, how many are only in these two data sources, etc.', 'dim')}
                 entity_size_breakdown {colorize("– shows how many entities of what size were created.  For instance, some entities ", 'dim')}
                 {colorize("are singletons, some might have connected 2 records, some 3, etc.  This report is primarily used to", 'dim')}
                 {colorize("ensure there are no instances of over matching.   For instance, it’s ok for an entity to have hundreds", 'dim')}
                 {colorize("of records as long as there are not too many different names, addresses, identifiers, etc.", 'dim')}
+                principles_used {colorize('– shows what principles and match_keys are firing across all data sources.  For example, ', 'dim')}
+                {colorize('how many name and address matches, how many address only, etc.', 'dim')}
 
-            {colorize('Audit report:', 'highlight2')} {colorize('(requires a json file created with sz_audit)', 'italics')}
+            {colorize('Audit report:', 'highlight2')} {colorize('(requires a json file generated by sz_audit)', 'italics')}
                 audit_summary {colorize("- shows the precision, recall and F1 scores with the ability to browse the entities that", 'dim')}
                 {colorize("were split or merged.", 'dim')}
 
@@ -3194,7 +3197,7 @@ class EdaCmd(cmd.Cmd):
         tbl = eda_table()
         tbl.title = "Data source counts"
         tbl.columns = [
-            {"name": "id", "width": 5, "align": "center"},
+            {"name": "ID", "width": 5, "align": "center"},
             {"name": "DataSource", "width": 30, "align": "left"},
             {"name": "ActualRecordCount", "width": 20, "align": "right"},
             {"name": "DistinctRecordCount", "width": 20, "align": "right"},
@@ -3247,7 +3250,7 @@ class EdaCmd(cmd.Cmd):
         try:
             self.eda_reports.audit_summary()
         except Exception as ex:
-            print_exception_info(ex)
+            print_exception(ex)
 
     def help_entity_size_breakdown(self):
         print(
@@ -3271,7 +3274,7 @@ class EdaCmd(cmd.Cmd):
         try:
             self.eda_reports.entity_size_breakdown()
         except Exception as ex:
-            print_exception_info(ex)
+            print_exception(ex)
 
     def complete_data_source_summary(self, text, line, begidx, endidx):
         possibles = sorted(self.eda_reports.snapshot_data.get("DATA_SOURCES", {}).keys())
@@ -3296,7 +3299,7 @@ class EdaCmd(cmd.Cmd):
         try:
             self.eda_reports.data_source_summary(arg)
         except Exception as ex:
-            print_exception_info(ex)
+            print_exception(ex)
 
     def help_cross_source_summary(self):
         print(
@@ -3321,9 +3324,9 @@ class EdaCmd(cmd.Cmd):
         try:
             self.eda_reports.cross_source_summary(arg)
         except Exception as ex:
-            print_exception_info(ex)
+            print_exception(ex)
 
-    def help_multi_source_summary(self):
+    def help_entity_source_summary(self):
         print(
             textwrap.dedent(
                 f"""\
@@ -3335,22 +3338,22 @@ class EdaCmd(cmd.Cmd):
         can only show the matches, not the possible matches and relationships.
 
         {colorize('Syntax:', 'highlight2')}
-            multi_source_summary [dataSource]
+            entity_source_summary [dataSource]
         """
             )
         )
 
-    def complete_multi_source_summary(self, text, line, begidx, endidx):
+    def complete_entity_source_summary(self, text, line, begidx, endidx):
         possibles = sorted(self.eda_reports.snapshot_data.get("DATA_SOURCES", {}).keys())
         if text:
             return [i for i in possibles if i.startswith(text.upper())]
         return possibles
 
-    def do_multi_source_summary(self, arg):
+    def do_entity_source_summary(self, arg):
         try:
-            self.eda_reports.multi_source_summary(arg)
+            self.eda_reports.entity_source_summary(arg)
         except Exception as ex:
-            print_exception_info(ex)
+            print_exception(ex)
 
     def help_principles_used(self):
         print(
@@ -3370,7 +3373,7 @@ class EdaCmd(cmd.Cmd):
         try:
             self.eda_reports.principles_used_report()
         except Exception as ex:
-            print_exception_info(ex)
+            print_exception(ex)
 
     # adhoc commands
 
@@ -3774,7 +3777,7 @@ class EdaCmd(cmd.Cmd):
             print_message(err, "error")
             return
         except Exception as ex:
-            print_exception_info(ex)
+            print_exception(ex)
             return
         view_report(report)
         if len(arg_tokens) == 1:  # supports previous/next
@@ -3909,7 +3912,8 @@ class EdaCmd(cmd.Cmd):
             why <entity_id1>                                            {colorize('actually runs a columnar how instead!', 'dim')}
             why <entity_id1> <entity_id2>                               {colorize('shows why two or more different entities did not resolve', 'dim')}
             why <data_source1> <record_id1> <data_source2> <record_id2> {colorize('shows if the two data source records could resolve or relate', 'dim')}
-
+            why search [optional entity_id]                             {colorize('shows the features and keys generated by a search against any candidate entities', 'dim')}
+            
         {colorize('Color legend:', 'highlight2')}
             {colorize('green', 'good')} indicates the values matched and contributed to the overall score
             {colorize('red', 'bad')} indicates the values did not match and hurt the overall score
@@ -4225,7 +4229,7 @@ if __name__ == "__main__":
     try:
         sdk_wrapper = EdaSdkWrapper(engine_config, debug_trace=args.debug_trace, webapp_url=args.webapp_url)
     except SzError:
-        print_exception_info()
+        print_exception()
         sys.exit(1)
 
     try: