[json-format] add "any" value kind and fix a display bug

Author: tstack

docs/schemas/format-v1.schema.json

@@ -401,7 +401,8 @@
                                         "struct",
                                         "quoted",
                                         "xml",
-                                        "timestamp"
+                                        "timestamp",
+                                        "any"
                                     ]
                                 },
                                 "collate": {

src/formats/mongodb_json_log.json

@@ -197,16 +197,6 @@
                 "hidden": true,
                 "description": "Additional attributes for the log message"
             },
-            "attr/action/outcome/errorMessage": {
-                "kind": "string",
-                "highlights": {
-                    "error": {
-                        "base-style": {
-                            "color": "Red"
-                        }
-                    }
-                }
-            },
             "attr/authenticationDatabase": {
                 "kind": "string",
                 "description": "The authenticating database for a connection"
@@ -217,7 +207,7 @@
                 "highlights": {
                     "delete": {
                         "base-style": {
-                            "color": "red"
+                            "color": "Red"
                         }
                     }
                 }
@@ -273,6 +263,7 @@
                 }
             },
             "attr/error": {
+                "kind": "any",
                 "highlights": {
                     "error": {
                         "base-style": {

src/log_format.cc

@@ -370,6 +370,7 @@ logline_value_meta::to_chart_type() const
         case value_kind_t::VALUE_FLOAT:
             retval = chart_type_t::spectro;
             break;
+        case value_kind_t::VALUE_ANY:
         case value_kind_t::VALUE_XML:
         case value_kind_t::VALUE_JSON:
         case value_kind_t::VALUE_BOOLEAN:
@@ -435,6 +436,7 @@ logline_value::logline_value(logline_value_meta lvm,
     }
 
     switch (this->lv_meta.lvm_kind) {
+        case value_kind_t::VALUE_ANY:
         case value_kind_t::VALUE_JSON:
         case value_kind_t::VALUE_XML:
         case value_kind_t::VALUE_STRUCT:
@@ -518,6 +520,7 @@ logline_value::to_string() const
         case value_kind_t::VALUE_NULL:
             return "null";
 
+        case value_kind_t::VALUE_ANY:
         case value_kind_t::VALUE_JSON:
         case value_kind_t::VALUE_XML:
         case value_kind_t::VALUE_STRUCT:
@@ -592,6 +595,7 @@ logline_value::to_string_fragment(ArenaAlloc::Alloc<char>& alloc) const
         case value_kind_t::VALUE_NULL:
             return "null"_frag;
 
+        case value_kind_t::VALUE_ANY:
         case value_kind_t::VALUE_JSON:
         case value_kind_t::VALUE_XML:
         case value_kind_t::VALUE_STRUCT:
@@ -1463,6 +1467,10 @@ read_array_end(void* ctx)
         if (field_name == jlu->jlu_format->elf_opid_field) {
             jlu->jlu_opid_desc_frag = json_frag;
         }
+        if (ypc->ypc_path_index_stack.size() > 1 && vd != nullptr) {
+            jlu->add_sub_lines_for(
+                vd, false, std::nullopt, nullptr, -1, nullptr);
+        }
     }
 
     return 1;

src/log_format_fwd.hh

@@ -523,6 +523,7 @@ enum class value_kind_t : int {
     VALUE_W3C_QUOTED,
     VALUE_TIMESTAMP,
     VALUE_XML,
+    VALUE_ANY,
 
     VALUE__MAX
 };

src/log_format_loader.cc

@@ -583,6 +583,7 @@ static constexpr json_path_handler_base::enum_value_t KIND_ENUM[] = {
     {"quoted"_frag, value_kind_t::VALUE_QUOTED},
     {"xml"_frag, value_kind_t::VALUE_XML},
     {"timestamp"_frag, value_kind_t::VALUE_TIMESTAMP},
+    {"any"_frag, value_kind_t::VALUE_ANY},
 
     json_path_handler_base::ENUM_TERMINATOR,
 };

src/log_vtab_impl.cc

@@ -247,6 +247,7 @@ log_vtab_impl::logline_value_to_sqlite_type(value_kind_t kind)
         case value_kind_t::VALUE_W3C_QUOTED:
         case value_kind_t::VALUE_TIMESTAMP:
         case value_kind_t::VALUE_XML:
+        case value_kind_t::VALUE_ANY:
             type = SQLITE3_TEXT;
             break;
         case value_kind_t::VALUE_FLOAT:
@@ -1369,6 +1370,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
                                 sqlite3_result_subtype(ctx, JSON_SUBTYPE);
                                 break;
                             }
+                            case value_kind_t::VALUE_ANY:
                             case value_kind_t::VALUE_STRUCT:
                             case value_kind_t::VALUE_TEXT:
                             case value_kind_t::VALUE_XML: {

test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err

@@ -94,7 +94,7 @@
          Description
            The type of data in the field
          Allowed Values
-           string, integer, float, boolean, json, struct, quoted, xml, timestamp
+           string, integer, float, boolean, json, struct, quoted, xml, timestamp, any
 ✘ error: 'bad' is not a supported log format $schema version
  --> {test_dir}/bad-config/formats/invalid-schema/format.json:2
  |     "$schema": "bad"