Replace custom markdown parsing with marked, KaTeX, and some custom plugins for matrix things (#727)

<!-- Please read
https://github.com/SableClient/Sable/blob/dev/CONTRIBUTING.md before
submitting your pull request -->

### Description

<!-- Please include a summary of the change. Please also include
relevant motivation and context. List any dependencies that are required
for this change. -->
Replaced Cinny's custom markdown parsing with marked which should
drastically reduce the various random obscure errors and crashes caused
by it.
Added math rendering support with KaTeX, sent using standard formatting
of $ for inline and $$ for block with LaTeX.

Supersedes #726
Fixes #711 
Fixes #587
Fixes #467  
Fixes #684 

Tbh I doubt I've tested this enough so there's probably still a bunch of
bugs, especially in parsing html back to markdown, but hopefully it's
less than our current system?

Might consider using another library like Turndown for converting html
into markdown, currently using a partial custom parser with... well okay
results.

Updated to also remove the WYSIWYG editor because it's largely unused,
can be entirely covered by markdown, and complicates translating html to
markdown. Keyboard shortcuts from the editor have been rewritten to
instead apply markdown syntax.

#### Type of change

- [x] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [x] Breaking change - removes the WYSIWYG editor.
- [ ] This change requires a documentation update

### Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings

### AI disclosure:

- [x] Partially AI assisted (clarify which code was AI assisted and
briefly explain what it does).
- [ ] Fully AI generated (explain what all the generated code does in
moderate detail).
<!-- Write any explanation required here, but do not generate the
explanation using AI!! You must prove you understand what the code in
this PR does. -->

Tests were AI generated and then reviewed by me. I can confirm that the
tests probably test the right stuff.
I had AI do the initial merge and write the marked plugins for the link
preview hiding and spoiler hiding features, which I then tested and
reviewed.

Author: 7w1

.changeset/add_math_and_markdown.md

@@ -0,0 +1,15 @@
+---
+default: minor
+---
+
+# Markdown parser and render updates
+
+Migrated markdown parsing and rendering to use marked, which should fix most (all?) markdown issues involving lists/nested structures, inconsistent/inaccurate code blocks, escape sequences, and all the other bugs with literally everything.
+
+Added math rendering support via marked and KaTeX, uses standard `$$` and `$` delimiters. Only renders a subset of latex tags that will likely need to be expanded so feel free to make issues if needed.
+
+Also adds support for sending markdown tables (although they're rendered rather plainly at the moment), sending valid html directly (such as for colored text), and properly escaping anything with backslashes.
+
+Fixes link previews appearing in code blocks, fixes pmp new line behavior, fixes links not opening in new tabs, and fixes editing arbitrary html messages, probably.
+
+Finally, the old WYSIWYG editor has been completely removed.

package.json

@@ -70,8 +70,10 @@
     "immer": "^9.0.21",
     "is-hotkey": "^0.2.0",
     "jotai": "^2.18.0",
+    "katex": "^0.16.45",
     "linkify-react": "^4.3.2",
     "linkifyjs": "^4.3.2",
+    "marked": "^18.0.2",
     "matrix-js-sdk": "^38.4.0",
     "matrix-widget-api": "^1.16.1",
     "pdfjs-dist": "^5.4.624",

pnpm-lock.yaml

@@ -1,4 +1,4 @@
-import { Scroll, Text } from 'folds';
+import { Text } from 'folds';
 import type { RenderElementProps, RenderLeafProps } from 'slate-react';
 import { useFocused, useSelected, useSlate } from 'slate-react';
 import { useAtomValue } from 'jotai';
@@ -130,72 +130,6 @@ export function RenderElement({ attributes, element, children }: RenderElementPr
           {children}
         </Text>
       );
-    case BlockType.Heading:
-      if (element.level === 1)
-        return (
-          <Text className={css.Heading} as="h2" size="H2" {...attributes}>
-            {children}
-          </Text>
-        );
-      if (element.level === 2)
-        return (
-          <Text className={css.Heading} as="h3" size="H3" {...attributes}>
-            {children}
-          </Text>
-        );
-      if (element.level === 3)
-        return (
-          <Text className={css.Heading} as="h4" size="H4" {...attributes}>
-            {children}
-          </Text>
-        );
-      return (
-        <Text className={css.Heading} as="h3" size="H3" {...attributes}>
-          {children}
-        </Text>
-      );
-    case BlockType.CodeLine:
-      return <div {...attributes}>{children}</div>;
-    case BlockType.CodeBlock:
-      return (
-        <Text as="pre" className={css.CodeBlock} {...attributes}>
-          <Scroll
-            direction="Horizontal"
-            variant="SurfaceVariant"
-            size="300"
-            visibility="Hover"
-            hideTrack
-          >
-            <div className={css.CodeBlockInternal}>{children}</div>
-          </Scroll>
-        </Text>
-      );
-    case BlockType.QuoteLine:
-      return <div {...attributes}>{children}</div>;
-    case BlockType.BlockQuote:
-      return (
-        <Text as="blockquote" className={css.BlockQuote} {...attributes}>
-          {children}
-        </Text>
-      );
-    case BlockType.ListItem:
-      return (
-        <Text as="li" {...attributes}>
-          {children}
-        </Text>
-      );
-    case BlockType.OrderedList:
-      return (
-        <ol className={css.List} {...attributes}>
-          {children}
-        </ol>
-      );
-    case BlockType.UnorderedList:
-      return (
-        <ul className={css.List} {...attributes}>
-          {children}
-        </ul>
-      );
     case BlockType.Mention:
       return (
         <RenderMentionElement attributes={attributes} element={element}>
@@ -220,19 +154,6 @@ export function RenderElement({ attributes, element, children }: RenderElementPr
           {children}
         </RenderCommandElement>
       );
-    case BlockType.Small:
-      return (
-        <Text {...attributes} className={css.Small}>
-          {children}
-        </Text>
-      );
-    case BlockType.HorizontalRule:
-      return (
-        <div {...attributes}>
-          <div contentEditable={false} className={css.HorizontalRule} />
-          {children}
-        </div>
-      );
     default:
       return (
         <Text
@@ -246,52 +167,6 @@ export function RenderElement({ attributes, element, children }: RenderElementPr
   }
 }
 
-export function RenderLeaf({ attributes, leaf, children }: RenderLeafProps) {
-  let child = children;
-  if (leaf.bold)
-    child = (
-      <strong {...attributes}>
-        <InlineChromiumBugfix />
-        {child}
-      </strong>
-    );
-  if (leaf.italic)
-    child = (
-      <i {...attributes}>
-        <InlineChromiumBugfix />
-        {child}
-      </i>
-    );
-  if (leaf.underline)
-    child = (
-      <u {...attributes}>
-        <InlineChromiumBugfix />
-        {child}
-      </u>
-    );
-  if (leaf.strikeThrough)
-    child = (
-      <s {...attributes}>
-        <InlineChromiumBugfix />
-        {child}
-      </s>
-    );
-  if (leaf.code)
-    child = (
-      <code className={css.Code} {...attributes}>
-        <InlineChromiumBugfix />
-        {child}
-      </code>
-    );
-  if (leaf.spoiler)
-    child = (
-      <span className={css.Spoiler()} {...attributes}>
-        <InlineChromiumBugfix />
-        {child}
-      </span>
-    );
-
-  if (child !== children) return child;
-
-  return <span {...attributes}>{child}</span>;
+export function RenderLeaf({ attributes, children }: RenderLeafProps) {
+  return <span {...attributes}>{children}</span>;
 }