[Backport 2.19-dev] backport markdown doctest support (#4938)

Author: kylehounslow

README.md

@@ -88,7 +88,7 @@ Recently we have been actively improving our query engine primarily for better c
 
 ## Documentation
 
-Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://opensearch.org/docs/latest/search-plugins/sql/index/) for detailed information on installing and configuring plugin.
+Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.md) and [Technical Documentation](https://opensearch.org/docs/latest/search-plugins/sql/index/) for detailed information on installing and configuring plugin.
 
 ## Forum
 

docs/category.json

@@ -4,8 +4,60 @@
     "user/admin/settings.rst"
   ],
   "bash_calcite": [
-    "user/ppl/interfaces/endpoint.rst",
-    "user/ppl/interfaces/protocol.rst"
+    "user/ppl/interfaces/endpoint.md",
+    "user/ppl/interfaces/protocol.md"
+  ],
+  "ppl_cli_calcite": [
+    "user/ppl/cmd/ad.md",
+    "user/ppl/cmd/append.md",
+    "user/ppl/cmd/bin.md",
+    "user/ppl/cmd/dedup.md",
+    "user/ppl/cmd/describe.md",
+    "user/ppl/cmd/eventstats.md",
+    "user/ppl/cmd/eval.md",
+    "user/ppl/cmd/fields.md",
+    "user/ppl/cmd/fillnull.md",
+    "user/ppl/cmd/grok.md",
+    "user/ppl/cmd/head.md",
+    "user/ppl/cmd/join.md",
+    "user/ppl/cmd/lookup.md",
+    "user/ppl/cmd/parse.md",
+    "user/ppl/cmd/patterns.md",
+    "user/ppl/cmd/rare.md",
+    "user/ppl/cmd/regex.md",
+    "user/ppl/cmd/rename.md",
+    "user/ppl/cmd/multisearch.md",
+    "user/ppl/cmd/replace.md",
+    "user/ppl/cmd/rex.md",
+    "user/ppl/cmd/search.md",
+    "user/ppl/cmd/showdatasources.md",
+    "user/ppl/cmd/sort.md",
+    "user/ppl/cmd/spath.md",
+    "user/ppl/cmd/stats.md",
+    "user/ppl/cmd/streamstats.md",
+    "user/ppl/cmd/subquery.md",
+    "user/ppl/cmd/syntax.md",
+    "user/ppl/cmd/chart.md",
+    "user/ppl/cmd/timechart.md",
+    "user/ppl/cmd/top.md",
+    "user/ppl/cmd/trendline.md",
+    "user/ppl/cmd/where.md",
+    "user/ppl/functions/collection.md",
+    "user/ppl/functions/condition.md",
+    "user/ppl/functions/conversion.md",
+    "user/ppl/functions/cryptographic.md",
+    "user/ppl/functions/datetime.md",
+    "user/ppl/functions/expressions.md",
+    "user/ppl/functions/ip.md",
+    "user/ppl/functions/json.md",
+    "user/ppl/functions/math.md",
+    "user/ppl/functions/relevance.md",
+    "user/ppl/functions/statistical.md",
+    "user/ppl/functions/string.md",
+    "user/ppl/functions/system.md",
+    "user/ppl/general/comments.md",
+    "user/ppl/general/datatypes.md",
+    "user/ppl/general/identifiers.md"
   ],
   "sql_cli": [
     "user/dql/expressions.rst",
@@ -21,57 +73,7 @@
     "user/dql/complex.rst",
     "user/dql/metadata.rst"
   ],
-  "ppl_cli_calcite": [
-    "user/ppl/cmd/ad.rst",
-    "user/ppl/cmd/append.rst",
-    "user/ppl/cmd/bin.rst",
-    "user/ppl/cmd/dedup.rst",
-    "user/ppl/cmd/describe.rst",
-    "user/ppl/cmd/eventstats.rst",
-    "user/ppl/cmd/eval.rst",
-    "user/ppl/cmd/fields.rst",
-    "user/ppl/cmd/fillnull.rst",
-    "user/ppl/cmd/grok.rst",
-    "user/ppl/cmd/head.rst",
-    "user/ppl/cmd/join.rst",
-    "user/ppl/cmd/lookup.rst",
-    "user/ppl/cmd/parse.rst",
-    "user/ppl/cmd/patterns.rst",
-    "user/ppl/cmd/rare.rst",
-    "user/ppl/cmd/regex.rst",
-    "user/ppl/cmd/rename.rst",
-    "user/ppl/cmd/multisearch.rst",
-    "user/ppl/cmd/replace.rst",
-    "user/ppl/cmd/rex.rst",
-    "user/ppl/cmd/search.rst",
-    "user/ppl/cmd/showdatasources.rst",
-    "user/ppl/cmd/sort.rst",
-    "user/ppl/cmd/spath.rst",
-    "user/ppl/cmd/stats.rst",
-    "user/ppl/cmd/streamstats.rst",
-    "user/ppl/cmd/subquery.rst",
-    "user/ppl/cmd/syntax.rst",
-    "user/ppl/cmd/chart.rst",
-    "user/ppl/cmd/timechart.rst",
-    "user/ppl/cmd/search.rst",
-    "user/ppl/functions/statistical.rst",
-    "user/ppl/cmd/top.rst",
-    "user/ppl/cmd/trendline.rst",
-    "user/ppl/cmd/where.rst",
-    "user/ppl/functions/collection.rst",
-    "user/ppl/functions/condition.rst",
-    "user/ppl/functions/datetime.rst",
-    "user/ppl/functions/expressions.rst",
-    "user/ppl/functions/ip.rst",
-    "user/ppl/functions/json.rst",
-    "user/ppl/functions/math.rst",
-    "user/ppl/functions/relevance.rst",
-    "user/ppl/functions/string.rst",
-    "user/ppl/functions/conversion.rst",
-    "user/ppl/general/datatypes.rst",
-    "user/ppl/general/identifiers.rst"
-  ],
   "bash_settings": [
-    "user/ppl/admin/settings.rst"
+    "user/ppl/admin/settings.md"
   ]
-}
+}

docs/dev/intro-v3-engine.md

@@ -26,16 +26,18 @@ Find more details in [V3 Architecture](./intro-v3-architecture.md).
 
 In the initial release of the V3 engine (3.0.0), the main new features focus on enhancing the PPL language while maintaining maximum compatibility with V2 behavior.
 
-* **[Join](../user/ppl/cmd/join.rst) Command**
-* **[Lookup](../user/ppl/cmd/lookup.rst) Command**
-* **[Subquery](../user/ppl/cmd/subquery.rst) Command**
+* **[Join](../user/ppl/cmd/join.md) Command**
+* **[Lookup](../user/ppl/cmd/lookup.md) Command**
+* **[Subquery](../user/ppl/cmd/subquery.md) Command**
+
+V3 (Calcite integration) engine is enabled by default in 3.3.0.
 
 ---
 ## 3.What are Changed
 
 ### 3.1 Breaking Changes
 
-Because of implementation changed internally, following behaviors are changed from 3.0.0-beta. (Behaviors in V3 is correct)
+Because of implementation changed internally, following behaviors are changed from 3.0.0. (Behaviors in V3 is correct)
 
 |                       Item                       |    V2     |          V3          |
 |:------------------------------------------------:|:---------:|:--------------------:|
@@ -56,6 +58,13 @@ Because of implementation changed internally, following behaviors are changed fr
 
 ### 3.3 Limitations
 
+For the following commands or functions, we add some defensive restrictions to ensure security.
+
+#### New Restrictions
+- `EVAL` won't allow to use [Metadata Fields of OpenSearch](https://docs.opensearch.org/docs/latest/field-types/metadata-fields/index/) as the fields
+- `RENAME` won't allow renaming to a [Metadata Fields of OpenSearch](https://docs.opensearch.org/docs/latest/field-types/metadata-fields/index/)
+- `as` won't allow to use [Metadata Fields of OpenSearch](https://docs.opensearch.org/docs/latest/field-types/metadata-fields/index/) as the alias name
+
 For the following functionalities in V3 engine, the query will be forwarded to the V2 query engine and thus you cannot use new features in [2. What's New](#2-whats-new).
 
 #### Unsupported functionalities (up to latest)
@@ -80,7 +89,5 @@ If you're interested in the new query engine, please find more details in [V3 Ar
 The following items are on our roadmap with high priority:
 - Resolve the [V3 limitation](#33-limitations).
 - Advancing pushdown optimization and benchmarking
-- Backport to 2.19.x
 - Unified the PPL syntax between [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/opensearch-spark/blob/main/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4)
 - Support more DSL aggregation
-

docs/dev/ppl-commands.md

@@ -54,4 +54,4 @@ If you are working on contributing a new PPL command, please read this guide and
   - Add a test in `CrossClusterSearchIT`
 
 - [ ] **User doc:**
-  - Add a xxx.rst under `docs/user/ppl/cmd` and link the new doc to `docs/user/ppl/index.rst`
+  - Add a xxx.md under `docs/user/ppl/cmd` and link the new doc to `docs/user/ppl/index.md`

docs/dev/testing-doctest.md

@@ -57,11 +57,58 @@ Doctest runs with project build by `./gradlew build`. You can also only run doct
 Make sure you don't have any OpenSearch instance running at `http://localhost:9200`
 
 ### 1.4.2 How to write documentation with doctest?
+
+#### RST Format (SQL docs only. On Deprecation path. Use markdown for PPL)
 1. If you want to add a new doc, you can add it to `docs` folder, under correct sub-folder, in `.rst` format. 
 > **Attention**: For code examples in documentation, a Mixing usage of `cli` and `bash` in one doc is not supported yet.
 2. Add your new doc file path to `docs/category.json` by its category
 3. Run doctest `./gradlew doctest` (optionally with `-DignorePrometheus`) to see if your tests can pass
 
+#### Markdown Format (New - Currently for docs/user/ppl only)
+For PPL documentation, Markdown format is now supported with the following guidelines:
+
+1. **File Format**: Create `.md` file(s) in `docs/user/ppl` folder
+2. **Category Configuration**: Add markdown files to markdown-only categories in `docs/category.json`:
+   - `ppl_cli_calcite`: PPL CLI examples with Calcite engine
+   - `bash_calcite`: Bash/curl examples with Calcite engine  
+   - `bash_settings`: Bash examples for settings/configuration
+
+3. **Code Block Format**: Use **paired** fenced code blocks - each input block must be followed by its expected output block:
+
+```ppl
+search source=accounts | where age > 25 | fields firstname, lastname
+```
+
+Expected output:
+
+```text
++-------------+------------+
+| firstname   | lastname   |
+|-------------+------------|
+| Amber       | Duke       |
+| Hattie      | Bond       |
++-------------+------------+
+```
+
+**Input/Output Pairs**: Each input code fence must be immediately followed by an "Expected output:" section with an output code fence
+- **Supported Input Languages**: `sql`, `ppl`, `bash`, `sh`, `bash ppl`
+- **Supported Output Languages**: `text`, `console`, `output`, `json`, `yaml`
+
+4. **Ignoring Tests**: To skip specific code blocks from testing, add `ignore` attribute:
+
+```ppl ignore
+search source=accounts | head 5
+```
+
+Expected output:
+
+```text
+This output won't be tested
+```
+
+5. **Validation**: Markdown categories only accept `.md` files - mixing with `.rst` files will cause validation errors
+6. **Testing**: Run `./gradlew doctest` to validate your markdown documentation
+
 Currently, there is a `sample` folder under `docs` module to help you get started.
 
 ## 1.5 Future Plan