convert Block Editor HTML to ProseMirror JSON on save (server-side) (dotCMS#36473)

## What

Converts Story Block (Block Editor) field values sent as **HTML** to
Tiptap/ProseMirror JSON server-side, on the shared content save path —
completing the HTML criterion deferred from dotCMS#36002. Non-interactive
clients (AI agents, headless imports) no longer need a human to open and
re-save the contentlet for an HTML value to read back as structured
content.

## How

- New `TiptapHtml` converter (jsoup DOM walk) — same node/mark shapes
and primitive whitelist as `TiptapMarkdown`, so both paths produce
documents the Block Editor loads identically. Sanitizes untrusted HTML:
drops `script`/`style`/`iframe`/… subtrees, allow-lists attributes,
scheme-checks `href`/`src`.
- Wired into `MapToContentletPopulator`: `{` → JSON passthrough, an HTML
tag → `TiptapHtml`, else → `TiptapMarkdown`. Reuses dotCMS#36002's
rich-content overwrite guard and never-block-the-save fallback.
- Adds `org.jsoup:jsoup 1.21.1`; OpenAPI fire note regenerated
("Markdown or HTML").

## Behavior change

An HTML value is now converted and stored as structured JSON instead of
verbatim — only when a client resends the field; stored values are never
rewritten on their own. Additive and rollback-safe.

## Testing

27 unit (`TiptapHtmlTest`) + 9 integration
(`StoryBlockMarkdownPopulatorTest`, including HTML → GraphQL read-back
and an editor-saved primitive doc updated via HTML end to end. Existing
`TiptapMarkdown` tests unchanged.

Closes dotCMS#36470

Author: hassandotcms

bom/application/pom.xml

@@ -31,6 +31,7 @@
         <opensearch.version>3.3.0</opensearch.version>
         <langchain4j.version>1.15.1</langchain4j.version>
         <commonmark.version>0.22.0</commonmark.version>
+        <jsoup.version>1.21.1</jsoup.version>
     </properties>
     <dependencyManagement>
 
@@ -499,6 +500,11 @@
                 <artifactId>commonmark-ext-gfm-strikethrough</artifactId>
                 <version>${commonmark.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.jsoup</groupId>
+                <artifactId>jsoup</artifactId>
+                <version>${jsoup.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.dotcms.lib</groupId>
                 <artifactId>dot.util-taglib</artifactId>

dotCMS/pom.xml

@@ -403,6 +403,10 @@
             <groupId>org.commonmark</groupId>
             <artifactId>commonmark-ext-gfm-strikethrough</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.dotcms.lib</groupId>
             <artifactId>dot.util-taglib</artifactId>

dotCMS/src/main/java/com/dotcms/rest/MapToContentletPopulator.java

@@ -10,6 +10,7 @@
 import com.dotcms.contenttype.model.type.ContentType;
 import com.dotcms.contenttype.transform.field.LegacyFieldTransformer;
 import com.dotcms.rest.api.v1.temp.DotTempFile;
+import com.dotcms.tiptap.TiptapHtml;
 import com.dotcms.tiptap.TiptapMarkdown;
 import com.dotcms.util.CollectionsUtils;
 import com.dotcms.util.DotPreconditions;
@@ -40,6 +41,7 @@
 import com.dotmarketing.util.Logger;
 import com.dotmarketing.util.UtilMethods;
 import com.dotmarketing.util.json.JSONObject;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -56,6 +58,7 @@
 import java.util.*;
 import java.util.Map.Entry;
 import java.util.function.LongSupplier;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import static com.dotmarketing.portlets.contentlet.model.Contentlet.VARIANT_ID;
@@ -78,6 +81,17 @@ public class MapToContentletPopulator  {
     private static final String IDENTIFIER                 = "identifier";
     private static final String INDEX_POLICY = "indexPolicy";
 
+    /**
+     * Matches a value that opens with a real HTML tag or comment, so a Story Block value can be
+     * routed to the HTML converter. A leading {@code '<'} alone is not enough — a CommonMark autolink
+     * ({@code <https://x>}) or text like {@code "<3"} also start with it and must go to Markdown; a
+     * real tag name is ASCII letters/digits terminated by whitespace, {@code '/'} or {@code '>'}.
+     * A leading {@code '<!'} (comment, {@code <!DOCTYPE>}, CDATA) is markup too, so a full HTML
+     * document routes to the HTML converter rather than being mistaken for Markdown.
+     */
+    private static final Pattern HTML_START =
+            Pattern.compile("^<(!|/?[a-zA-Z][a-zA-Z0-9]*[\\s/>])");
+
     @CloseDBIfOpened
     public Contentlet populate(final Contentlet contentlet, final Map<String, Object> stringObjectMap) {
 
@@ -271,11 +285,10 @@ private void fillFields(final Contentlet contentlet,
 
     /**
      * Story Block fields store a Tiptap/ProseMirror JSON document. Non-interactive clients
-     * (AI agents, headless imports) may instead send Markdown. We convert it to ProseMirror
-     * JSON here, on the shared save path, so the field reads back as structured content with
-     * no human editor round-trip. Values that are already JSON (the dominant editor traffic)
-     * or HTML are stored unchanged — Markdown is the only thing converted, and a conversion
-     * failure never blocks the save.
+     * (AI agents, headless imports) may instead send Markdown or HTML. We convert either to
+     * ProseMirror JSON here, on the shared save path, so the field reads back as structured
+     * content with no human editor round-trip. Values that are already JSON (the dominant editor
+     * traffic) are stored unchanged, and a conversion failure never blocks the save.
      */
     private void processStoryBlockField(final Contentlet contentlet, final Field field,
                                         final String value) {
@@ -287,41 +300,64 @@ private void processStoryBlockField(final Contentlet contentlet, final Field fie
     private String toStoryBlockJson(final Contentlet contentlet, final Field field,
                                     final String value) {
 
-        // Editor-authored JSON and (for now) HTML are stored as-is; Markdown is plain text and
-        // begins with neither '{' nor '<'. This mirrors the Block Editor's own client-side
-        // routing and avoids re-parsing an existing document as Markdown.
+        // Editor-authored JSON is stored as-is (it begins with '{'); a value that opens with an HTML
+        // tag is converted from HTML, and anything else is treated as Markdown. This mirrors the Block
+        // Editor's own client-side routing and avoids re-parsing an existing document.
         final String trimmed = value.stripLeading();
-        if (trimmed.isEmpty() || trimmed.charAt(0) == '{' || trimmed.charAt(0) == '<') {
+        if (trimmed.isEmpty() || trimmed.charAt(0) == '{') {
             return value;
         }
+        final boolean html = looksLikeHtml(trimmed);
 
-        // Markdown cannot represent rich blocks (embedded contentlets, video, layout grids). Per the
-        // documented contract (see the fire endpoints' Block Editor note), Markdown is for plain
+        // Markdown/HTML cannot represent rich blocks (embedded contentlets, video, layout grids). Per
+        // the documented contract (see the fire endpoints' Block Editor note), they are for plain
         // content only and must not be used to modify a field that already holds such blocks. If that
-        // is attempted, keep the existing document untouched and log a warning — neither destroying
-        // the rich content nor failing the save. (Markdown -> rich merge is planned as a follow-up.)
+        // is attempted, keep the existing document untouched and log a warning — neither destroying the
+        // rich content nor failing the save. (A rich merge is planned as a follow-up.)
         final String existing = contentlet.getStringProperty(field.getVelocityVarName());
         if (TiptapMarkdown.isTiptapDoc(existing) && !TiptapMarkdown.isMarkdownRepresentable(existing)) {
             Logger.warn(this, String.format(
-                    "Story Block field [%s] holds rich content that Markdown cannot represent; "
-                            + "ignoring the Markdown value and keeping the existing document. Send a "
-                            + "full Tiptap/ProseMirror JSON document to modify this field.",
-                    field.getVelocityVarName()));
+                    "Story Block field [%s] holds rich content that %s cannot represent; ignoring the "
+                            + "value and keeping the existing document. Send a full Tiptap/ProseMirror "
+                            + "JSON document to modify this field.",
+                    field.getVelocityVarName(), html ? "HTML" : "Markdown"));
             return existing;
         }
 
         try {
-            return TiptapMarkdown.toTiptap(value).toString();
+            final ObjectNode converted = html ? TiptapHtml.toTiptap(value) : TiptapMarkdown.toTiptap(value);
+            // Non-blank input whose content is entirely stripped (sanitization / unparseable) converts to
+            // an empty document. Storing it would silently clear a field the client meant to populate and
+            // wipe any prior value, so keep the existing value and warn instead. A genuine field-clear
+            // arrives as a blank value (handled above), so this never blocks an intended clear.
+            if (converted.path("content").size() == 0 && UtilMethods.isSet(existing)) {
+                Logger.warn(this, String.format(
+                        "Story Block field [%s]: %s input converted to an empty document; keeping the "
+                                + "existing value rather than clearing the field.",
+                        field.getVelocityVarName(), html ? "HTML" : "Markdown"));
+                return existing;
+            }
+            return converted.toString();
         } catch (final Exception e) {
-            // Graceful degradation (consistent with the converter's #35728 contract): a parse
-            // failure must never block the save — store the original value and move on.
+            // Graceful degradation (consistent with the converters' contract): a conversion failure
+            // must never block the save — store the original value and move on.
             Logger.warn(this, String.format(
-                    "Story Block field [%s]: Markdown conversion failed, storing value unchanged. %s",
-                    field.getVelocityVarName(), e.getMessage()));
+                    "Story Block field [%s]: %s conversion failed, storing value unchanged. %s",
+                    field.getVelocityVarName(), html ? "HTML" : "Markdown", e.getMessage()));
             return value;
         }
     }
 
+    /**
+     * Distinguishes a genuine HTML fragment (an opening/closing tag or a comment) from a Markdown
+     * value that merely starts with {@code '<'} — e.g. a CommonMark autolink {@code <https://x>} or
+     * text like {@code "<3 things"} — which must be routed to the Markdown converter instead. The
+     * argument is expected to be already left-trimmed.
+     */
+    private static boolean looksLikeHtml(final String trimmed) {
+        return HTML_START.matcher(trimmed).find();
+    }
+
     private static void processPlainValueForBinaryField(final Map<String, Object> map,
                                                         final Field field,
                                                         final Object value,

dotCMS/src/main/java/com/dotcms/rest/api/v1/workflow/WorkflowResource.java

@@ -228,13 +228,13 @@ public class WorkflowResource {
             "leave the default.";
 
     private static final String BLOCK_EDITOR_FIELD_NOTE =
-            "\n\n**Block Editor (Story Block) fields:** send the value as a **Markdown** string — " +
+            "\n\n**Block Editor (Story Block) fields:** send the value as a **Markdown or HTML** string — " +
             "you do not need to hand-author the underlying ProseMirror/JSON document. dotCMS converts it " +
             "to the Block Editor (ProseMirror JSON) structure automatically on save, so the field reads " +
             "back as structured content with no editor round-trip required. A value that is already a " +
-            "valid Tiptap/ProseMirror JSON document is detected and stored unchanged. Markdown is intended " +
-            "for plain content: if the field already holds rich blocks that Markdown cannot represent " +
-            "(embedded contentlets, video or layout blocks), the Markdown value is ignored and the existing " +
+            "valid Tiptap/ProseMirror JSON document is detected and stored unchanged. Markdown and HTML are " +
+            "intended for plain content: if the field already holds rich blocks that they cannot represent " +
+            "(embedded contentlets, video or layout blocks), the value is ignored and the existing " +
             "document is preserved — to modify such a field, send a full Tiptap/ProseMirror JSON document. " +
             "Example: `\"body\": \"## Intro\\n\\nHello **world**.\"`.";