Dump coclustering JSON report keys in the expected order

Thus, the `write_khiops_json_file` method can be used to dump the JSON
report with the keys exactly in the same order as expected by
MODL_Coclustering.

Thus, coclustering reports are now verbatim-identical to the reference
reports.

Author: popescu-v

khiops/core/coclustering_results.py

@@ -77,6 +77,127 @@ class CoclusteringResults(KhiopsJSONObject):
         Coclustering modeling report.
     """
 
+    # Set coclustering report order key specification
+    # pylint: disable=line-too-long
+    json_key_sort_spec = {
+        "tool": None,
+        "version": None,
+        "shortDescription": None,
+        "coclusteringReport": {
+            "summary": {
+                "instances": None,
+                "cells": None,
+                "nullCost": None,
+                "cost": None,
+                "level": None,
+                "initialDimensions": None,
+                "frequencyVariable": None,
+                "dictionary": None,
+                "database": None,
+                "samplePercentage": None,
+                "samplingMode": None,
+                "selectionVariable": None,
+                "selectionValue": None,
+            },
+            "dimensionSummaries": [
+                {
+                    "name": None,
+                    "isVarPart": None,
+                    "type": None,
+                    "parts": None,
+                    "initialParts": None,
+                    "values": None,
+                    "interest": None,
+                    "description": None,
+                    "min": None,
+                    "max": None,
+                },
+            ],
+            "dimensionPartitions": [
+                {
+                    "name": None,
+                    "type": None,
+                    "innerVariables": {
+                        "dimensionSummaries": [
+                            {
+                                "name": None,
+                                "type": None,
+                                "parts": None,
+                                "initialParts": None,
+                                "values": None,
+                                "interest": None,
+                                "description": None,
+                                "min": None,
+                                "max": None,
+                            }
+                        ],
+                        "dimensionPartitions": [
+                            {
+                                "name": None,
+                                "type": None,
+                                "intervals": [
+                                    {
+                                        "cluster": None,
+                                        "bounds": None,
+                                    }
+                                ],
+                                "valueGroups": [
+                                    {
+                                        "cluster": None,
+                                        "values": None,
+                                        "valueFrequencies": None,
+                                    }
+                                ],
+                                "defaultGroupIndex": None,
+                            }
+                        ],
+                    },
+                    "intervals": [
+                        {
+                            "cluster": None,
+                            "bounds": None,
+                        }
+                    ],
+                    "valueGroups": [
+                        {
+                            "cluster": None,
+                            "values": None,
+                            "valueFrequencies": None,
+                            "valueTypicalities": None,
+                        }
+                    ],
+                    "defaultGroupIndex": None,
+                },
+            ],
+            "dimensionHierarchies": [
+                {
+                    "name": None,
+                    "type": None,
+                    "clusters": [
+                        {
+                            "cluster": None,
+                            "parentCluster": None,
+                            "frequency": None,
+                            "interest": None,
+                            "hierarchicalLevel": None,
+                            "rank": None,
+                            "hierarchicalRank": None,
+                            "isLeaf": None,
+                            "shortDescription": None,
+                            "description": None,
+                        }
+                    ],
+                }
+            ],
+            "cellPartIndexes": None,
+            "cellFrequencies": None,
+        },
+        "khiops_encoding": None,
+        "ansi_chars": None,
+        "colliding_utf8_chars": None,
+    }
+    # pylint: enable=line-too-long
+
     def __init__(self, json_data=None):
         """See class docstring"""
         # Initialize super class

khiops/core/internals/io.py

@@ -16,6 +16,8 @@
 from khiops.core.exceptions import KhiopsJSONError
 from khiops.core.internals.common import (
     deprecation_message,
+    is_dict_like,
+    is_list_like,
     is_string_like,
     type_error_message,
 )
@@ -150,8 +152,20 @@ class KhiopsJSONObject:
         **Deprecated** will be removed in Khiops 12.
     """
 
+    # Set default JSON key sort specification attribute
+    # Can be set in classes that specialize this class
+    json_key_sort_spec = None
+
     def __init__(self, json_data=None):
         """See class docstring"""
+        # Check the type of the json_key_sort_spec class attribute
+        if self.json_key_sort_spec is not None and not isinstance(
+            self.json_key_sort_spec, dict
+        ):
+            raise TypeError(
+                type_error_message("key_sort_spec", self.json_key_sort_spec, dict)
+            )
+
         # Check the type of json_data
         if json_data is not None and not isinstance(json_data, dict):
             raise TypeError(type_error_message("json_data", json_data, dict))
@@ -261,17 +275,83 @@ def to_dict(self):
             report["subTool"] = self.sub_tool
         return report
 
-    def write_khiops_json_file(self, json_file_path):
+    def _json_key_sort_by_spec(self, jdict, key_sort_spec=None):
+        sorted_jdict = {}
+        if key_sort_spec is None:
+            key_sort_spec = self.json_key_sort_spec
+        for spec_key, spec_value in key_sort_spec.items():
+            if not (spec_value is None or isinstance(spec_value, (dict, list))):
+                raise ValueError(
+                    type_error_message(
+                        "specification value",
+                        spec_value,
+                        "'None' or dict or list",
+                    )
+                )
+            if spec_key in jdict:
+                json_value = jdict[spec_key]
+
+                # If value is not a dict, then:
+                # - if not list-like, then add it as such to the output dict
+                # - else, iterate on the list-like value
+                # else, recurse on the dict structure
+                if not is_dict_like(json_value):
+                    if not is_list_like(json_value):
+                        sorted_jdict[spec_key] = json_value
+                    else:
+                        sorted_jdict[spec_key] = []
+                        for json_el in json_value:
+                            if not is_dict_like(json_el):
+                                sorted_jdict[spec_key].append(json_el)
+                            else:
+                                if is_list_like(spec_value):
+                                    sorted_jdict[spec_key].append(
+                                        self._json_key_sort_by_spec(
+                                            json_el, key_sort_spec=spec_value[0]
+                                        )
+                                    )
+                else:
+                    sorted_jdict[spec_key] = self._json_key_sort_by_spec(
+                        json_value, key_sort_spec=spec_value
+                    )
+        return sorted_jdict
+
+    def write_khiops_json_file(
+        self, json_file_path, _sort_keys=False, _ensure_ascii=False
+    ):
         """Write the JSON data of the object to a Khiops JSON file
 
+        The JSON keys are sorted according to ``.KhiopsJSONObject.json_key_sort_spec`
+        class attribute, if set.
+
         Parameters
         ----------
         json_file_path : str
             Path to the Khiops JSON file.
+        _sort_keys : bool, default False
+            Ignored if the ``.KhiopsJSONObject.json_key_sort_spec`` class
+            attribute is set.
+            Otherwise, if True, then sort the keys lexicographically and
+            recursively.
+        _ensure_ascii : bool, default False
+            If True, then non-ASCII characters in the report are escaped. Otherwise,
+            they are dumped as-is.
         """
         # Serialize JSON data to string
         # Do not escape non-ASCII Unicode characters
-        json_string = json.dumps(self.to_dict(), ensure_ascii=False)
+        json_dict = self.to_dict()
+        if self.json_key_sort_spec is not None:
+            json_dict = self._json_key_sort_by_spec(json_dict)
+            json_string = json.dumps(json_dict, indent=4, ensure_ascii=_ensure_ascii)
+        elif _sort_keys:
+            json_string = json.dumps(
+                json_dict,
+                indent=4,
+                ensure_ascii=_ensure_ascii,
+                sort_keys=_sort_keys,
+            )
+        else:
+            json_string = json.dumps(json_dict, indent=4, ensure_ascii=_ensure_ascii)
         with io.BytesIO() as json_stream:
             writer = self.create_output_file_writer(json_stream)
             writer.write(json_string)

tests/test_core.py

@@ -10,6 +10,7 @@
 import json
 import os
 import shutil
+import tempfile
 import textwrap
 import unittest
 import warnings
@@ -33,7 +34,47 @@
 class KhiopsCoreIOTests(unittest.TestCase):
     """Tests the reading/writing of files for the core module classes/functions"""
 
-    def _assert_report_is_dumped_to_correct_json(self, report, ref_json_report):
+    def _assert_coclustering_report_is_written_to_sorted_json_file(
+        self, cc_report, ref_json_report
+    ):
+        # Write the coclustering report to a JSON file, sorted according to
+        # the spec defined in the CoclusteringResults class
+        # Set ensure_ascii, as non-ASCII characters are escaped in the reference
+        # reports
+        tmp_dir = tempfile.mkdtemp()
+        output_report = os.path.join(tmp_dir, "TestCoclustering.khcj")
+        cc_report.write_khiops_json_file(
+            output_report, _sort_keys=True, _ensure_ascii=True
+        )
+
+        # Load JSON Khiops reports into Python dictionaries
+        with open(ref_json_report, encoding="utf-8") as ref_json_file:
+            ref_json = json.load(ref_json_file)
+        with open(output_report, encoding="utf-8") as output_json_file:
+            output_json = json.load(output_json_file)
+        shutil.rmtree(tmp_dir)
+
+        # Dump reports with consistent indentation
+        ref_json_string = json.dumps(ref_json, indent=4)
+        output_json_string = json.dumps(output_json, indent=4)
+
+        # Succeed if the dumped reports are equal
+        if output_json_string == ref_json_string:
+            return
+
+        # On failure print the differences
+        output_json_lines = output_json_string.splitlines(keepends=True)
+        ref_json_lines = ref_json_string.splitlines(keepends=True)
+        out_ref_diff = "".join(unified_diff(ref_json_lines, output_json_lines))
+        if out_ref_diff:
+            self.fail(
+                "CoclusteringResults JSON dump differs from reference "
+                f"'{ref_json_report}':\n{out_ref_diff}"
+            )
+
+    def _assert_analysis_report_is_dumped_to_correct_json(
+        self, report, ref_json_report
+    ):
         # Dump the report as JSON (4-space indented and keys sorted in
         # lexicographic order)
         output_json = report.to_dict()
@@ -54,11 +95,8 @@ def _assert_report_is_dumped_to_correct_json(self, report, ref_json_report):
         ref_json_lines = ref_json_string.splitlines(keepends=True)
         out_ref_diff = "".join(unified_diff(ref_json_lines, output_json_lines))
         if out_ref_diff:
-            report_type = (
-                "Analysis" if ref_json_report.endswith(".khj") else "Coclustering"
-            )
             self.fail(
-                f"{report_type}Results JSON dump differs from reference "
+                f"AnalysisResults JSON dump differs from reference "
                 f"'{ref_json_report}':\n{out_ref_diff}"
             )
 
@@ -111,12 +149,12 @@ def test_analysis_results(self):
                 elif report in reports_warn:
                     with self.assertWarns(UserWarning):
                         results = kh.read_analysis_results_file(ref_json_report)
-                        self._assert_report_is_dumped_to_correct_json(
+                        self._assert_analysis_report_is_dumped_to_correct_json(
                             results, ref_json_report
                         )
                 else:
                     results = kh.read_analysis_results_file(ref_json_report)
-                    self._assert_report_is_dumped_to_correct_json(
+                    self._assert_analysis_report_is_dumped_to_correct_json(
                         results, ref_json_report
                     )
 
@@ -152,7 +190,9 @@ def test_coclustering_results(self):
                         results = kh.read_coclustering_results_file(ref_json_report)
                 else:
                     results = kh.read_coclustering_results_file(ref_json_report)
-                self._assert_report_is_dumped_to_correct_json(results, ref_json_report)
+                self._assert_coclustering_report_is_written_to_sorted_json_file(
+                    results, ref_json_report
+                )
 
     def test_binary_dictionary_domain(self):
         """Test binary dictionary write"""