Improve documentation regarding json fields and concatenated json fields (#6047)

Author: rdettai-sk

docs/configuration/index-config.md

@@ -358,6 +358,14 @@ fast:
   normalizer: lowercase
 ```
 
+Stored primitive types are inferred from the JSON value types using the following rules:
+- a boolean value `true` or `false` is stored as `bool`
+- numeric values are cast to the first compatible format between `i64`, `u64` or
+  `f64` (in this order)
+- for string values (surrounded with quotes), Tantivy attempts to parse a date
+  in `rfc3339` format. If the parsing fails, the value is stored as `text` using
+  the configured tokenization rules
+
 **Parameters for JSON field**
 
 | Variable      | Description   | Default value |
@@ -435,8 +443,13 @@ record: basic
 Concatenate fields don't support fast fields, and are never stored. They uses their own tokenizer, independently of the
 tokenizer configured on the individual fields.
 At query time, concatenate fields don't support range queries.
-Only the following types are supported inside a concatenate field: text, bool, i64, u64, f64, json. Other types are rejected
-at index creation, or silently discarded during indexation if they are found inside a json field.
+Only the following types are supported inside a concatenate field: text, bool,
+i64, u64, f64, json. Other types are rejected at index creation, or silently
+discarded during indexation if they are found inside a json field. Unlike
+regular JSON fields, JSON fields in a concatenate field don't store RFC3339
+dates as Tantivy dates. This means you can still perform prefix queries,
+e.g `my_default_field:"2025-12-12"*` to work around the lack of support for range
+queries.
 Adding an object field to a concatenate field doesn't automatically add its subfields (yet).
 <!-- typing is made so it wouldn't be too hard to add, as well as things like params_* matching all fields which starts name with params_ , but the feature isn't implemented yet -->
 It isn't possible to add subfields from a json field to a concatenate field. For instance if `attributes` is a json field, it's not possible to add only `attributes.color` to a concatenate field.
@@ -552,6 +565,8 @@ src.port:53
 src.port:53 AND query_params.ctk:e42bb897d
 ```
 
+The stored primitive type inference is the [same as for JSON fields](#json-type).
+
 ### Field name validation rules
 
 Currently Quickwit only accepts field name that matches the following regular expression:

quickwit/quickwit-doc-mapper/src/doc_mapper/doc_mapper_impl.rs

@@ -35,7 +35,7 @@ use super::field_presence::populate_field_presence;
 use super::tantivy_val_to_json::tantivy_value_to_json;
 use crate::doc_mapper::mapping_tree::{
     JsonValueIterator, MappingNode, MappingNodeRoot, build_field_path_from_str, build_mapping_tree,
-    map_primitive_json_to_tantivy,
+    map_primitive_json_to_concatenate_value,
 };
 use crate::doc_mapper::{FieldMappingType, JsonObject, Partition};
 use crate::query_builder::build_query;
@@ -530,7 +530,7 @@ impl DocMapper {
             if !self.concatenate_dynamic_fields.is_empty() {
                 let json_obj_values =
                     JsonValueIterator::new(serde_json::Value::Object(dynamic_json_obj.clone()))
-                        .flat_map(map_primitive_json_to_tantivy);
+                        .flat_map(map_primitive_json_to_concatenate_value);
 
                 for value in json_obj_values {
                     for concatenate_dynamic_field in self.concatenate_dynamic_fields.iter() {

quickwit/quickwit-doc-mapper/src/doc_mapper/mapping_tree.rs

@@ -139,7 +139,11 @@ impl<T, I: Iterator<Item = T>> Iterator for OneOrIter<T, I> {
     }
 }
 
-pub(crate) fn map_primitive_json_to_tantivy(value: JsonValue) -> Option<TantivyValue> {
+/// Similar to the native `From<JsonValue> for TantivyValue` implementation, with a
+/// subtle difference: no automatic parsing to DateTime is performed when the string
+/// is a valid RFC3339 date. This enables some level of range querying through prefix
+/// queries despite concatenate fields not supporting fast fields.
+pub(crate) fn map_primitive_json_to_concatenate_value(value: JsonValue) -> Option<TantivyValue> {
     match value {
         JsonValue::Array(_) | JsonValue::Object(_) | JsonValue::Null => None,
         JsonValue::String(text) => Some(TantivyValue::Str(text)),
@@ -260,7 +264,7 @@ impl LeafType {
         }
     }
 
-    fn tantivy_value_from_json(
+    fn concatenate_values_from_json(
         &self,
         json_val: JsonValue,
     ) -> Result<impl Iterator<Item = TantivyValue>, String> {
@@ -302,7 +306,7 @@ impl LeafType {
                         json_obj
                             .into_iter()
                             .flat_map(|(_key, val)| JsonValueIterator::new(val))
-                            .flat_map(map_primitive_json_to_tantivy),
+                            .flat_map(map_primitive_json_to_concatenate_value),
                     ))
                 } else {
                     Err(format!("expected object, got `{json_val}`"))
@@ -388,7 +392,7 @@ impl MappingLeaf {
                 if !self.concatenate.is_empty() {
                     let concat_values = self
                         .typ
-                        .tantivy_value_from_json(el_json_val.clone())
+                        .concatenate_values_from_json(el_json_val.clone())
                         .map_err(|err_msg| DocParsingError::ValueError(path.join("."), err_msg))?;
                     for concat_value in concat_values {
                         for field in &self.concatenate {
@@ -408,7 +412,7 @@ impl MappingLeaf {
         if !self.concatenate.is_empty() {
             let concat_values = self
                 .typ
-                .tantivy_value_from_json(json_val.clone())
+                .concatenate_values_from_json(json_val.clone())
                 .map_err(|err_msg| DocParsingError::ValueError(path.join("."), err_msg))?;
             for concat_value in concat_values {
                 for field in &self.concatenate {

quickwit/rest-api-tests/scenarii/concat_fields/0001_concat_field.yaml

@@ -163,3 +163,11 @@ params:
 expected:
   num_hits: 1
 ---
+endpoint: concat/search
+params:
+  # concat date values are stored as strings to enable some level of range
+  # querying even though they don't support fast fields
+  query: "concat_raw:\"2024-01-01\"*"
+expected:
+  num_hits: 1
+---

quickwit/rest-api-tests/scenarii/concat_fields/_setup.quickwit.yaml

@@ -67,7 +67,7 @@ ndjson:
   - {"other-field": "otherfieldvalue", "other-field-number": 9, "other-field-bool": false}
   - {"json": {"some_bool": false, "some_int": 10, "nested": {"some_string": "nestedstring"}}}
   - {"float": 1.5}
-  - {"json": {"val:": 2.5}}
+  - {"json": {"val:": 2.5, "date": "2024-01-01T00:13:00Z"}}
   - {"other": 3.5}
   # too big to be a i64, parsed as a u64
   - {"big": 9223372036854775808}