feat: add outputFormat: 'tokens' support

hook only and experimental - no renderer yet

Author: AVGVSTVS96

.changeset/token-output-format.md

@@ -0,0 +1,5 @@
+---
+"react-shiki": minor
+---
+
+Add hook-only `outputFormat: "tokens"` support for returning Shiki token output.

package/README.md

@@ -220,7 +220,7 @@ See [Shiki - RegExp Engines](https://shiki.style/guide/regex-engines) for more i
 | `transformers`      | `array`            | `[]`            | Custom Shiki transformers for modifying the highlighting output               |
 | `cssVariablePrefix` | `string`           | `'--shiki'`     | Prefix for CSS variables storing theme colors                                 |
 | `defaultColor`      | `string \| false`  | `'light'`       | Default theme mode when using multiple themes, can also disable default theme |
-| `outputFormat`      | `string`           | `'react'`       | Output format: 'react' for React nodes, 'html' for HTML string                 |
+| `outputFormat`      | `string`           | `'react'`       | Output format: 'react' for React nodes, 'html' for HTML string, or 'tokens' for Shiki tokens in the hook |
 | `tabindex`          | `number`           | `0`             | Tab index for the code block                                                  |
 | `decorations`       | `array`            | `[]`            | Custom decorations to wrap the highlighted tokens with                        |
 | `structure`        | `string`           | `classic`  | The structure of the generated HAST and HTML - `classic` or `inline`               |
@@ -631,7 +631,7 @@ const highlightedCode = useShikiHighlighter(code, "tsx", "github-dark", {
 
 ### Output Format Optimization
 
-`react-shiki` provides two output formats to balance safety and performance:
+`react-shiki` provides output formats to balance safety, performance, and custom rendering:
 
 **React Nodes (Default)** - Safer, no `dangerouslySetInnerHTML` required
 ```tsx
@@ -657,7 +657,24 @@ const highlightedCode = useShikiHighlighter(code, "tsx", "github-dark", {
 </ShikiHighlighter>
 ```
 
-Choose HTML output when performance is critical and you trust the code source. Use the default React output when handling untrusted content or when security is the primary concern.
+**Shiki Tokens** - hook-only output for custom renderers
+```tsx
+const highlighted = useShikiHighlighter(code, "tsx", "github-dark", {
+  outputFormat: "tokens",
+});
+
+const rendered = highlighted?.tokens.map((line, i) => (
+  <div key={i}>
+    {line.map((token, j) => (
+      <span key={j} style={{ color: token.color }}>
+        {token.content}
+      </span>
+    ))}
+  </div>
+));
+```
+
+Choose HTML output when performance is critical and you trust the code source. Use the default React output when handling untrusted content or when security is the primary concern. Use token output when you need to own rendering yourself; it is intentionally available on the hook, not the `ShikiHighlighter` component.
 
 ---
 

package/src/core.ts

@@ -1,6 +1,13 @@
 import { useHighlight } from './lib/hook';
 import { validateCoreHighlighter } from './bundles/core';
-import type { UseShikiHighlighter } from './lib/types';
+import type {
+  BaseHighlighterOptions,
+  Language,
+  OutputFormat,
+  Theme,
+  Themes,
+  UseShikiHighlighter,
+} from './lib/types';
 
 export { isInlineCode, rehypeInlineCodeProperty } from './lib/plugins';
 
@@ -62,16 +69,18 @@ export type { LanguageRegistration } from 'shiki/core';
  *
  * Core bundle (minimal). For plug-and-play: `react-shiki` or `react-shiki/web`
  */
-export const useShikiHighlighter: UseShikiHighlighter = (
-  code,
-  lang,
-  themeInput,
-  options = {}
+export const useShikiHighlighter: UseShikiHighlighter = <
+  F extends OutputFormat = 'react',
+>(
+  code: string,
+  lang: Language,
+  themeInput: Theme | Themes,
+  options: BaseHighlighterOptions & { outputFormat?: F } = {}
 ) => {
   // Validate that highlighter is provided
   const highlighter = validateCoreHighlighter(options.highlighter);
 
-  return useHighlight(
+  return useHighlight<F>(
     code,
     lang,
     themeInput,

package/src/index.ts

@@ -1,4 +1,4 @@
-import { useHighlight } from './lib/hook';
+import { createUseShikiHighlighter } from './lib/hook';
 import { createFullHighlighter } from './bundles/full';
 import type { UseShikiHighlighter } from './lib/types';
 
@@ -50,20 +50,8 @@ export type { LanguageRegistration } from 'shiki';
  *
  * Full bundle (~6.4MB minified, 1.2MB gzipped). For smaller bundles: `react-shiki/web` or `react-shiki/core`
  */
-export const useShikiHighlighter: UseShikiHighlighter = (
-  code,
-  lang,
-  themeInput,
-  options = {}
-) => {
-  return useHighlight(
-    code,
-    lang,
-    themeInput,
-    options,
-    createFullHighlighter
-  );
-};
+export const useShikiHighlighter: UseShikiHighlighter =
+  createUseShikiHighlighter(createFullHighlighter);
 
 /**
  * ShikiHighlighter component using the full bundle.

package/src/lib/component.tsx

@@ -3,7 +3,8 @@ import '../styles/features.css';
 import { clsx } from 'clsx';
 
 import type {
-  HighlighterOptions,
+  ComponentHighlighterOptions,
+  ComponentOutputFormat,
   Language,
   Theme,
   Themes,
@@ -14,7 +15,8 @@ import { forwardRef } from 'react';
 /**
  * Props for the ShikiHighlighter component
  */
-export interface ShikiHighlighterProps extends HighlighterOptions {
+export interface ShikiHighlighterProps
+  extends ComponentHighlighterOptions {
   /**
    * The programming language for syntax highlighting
    * Supports custom textmate grammar objects in addition to Shiki's bundled languages
@@ -126,7 +128,7 @@ export const createShikiHighlighterComponent = (
       ref
     ) => {
       // Destructure some options for use in hook
-      const options: HighlighterOptions = {
+      const options: ComponentHighlighterOptions = {
         delay,
         transformers,
         customLanguages,
@@ -138,12 +140,21 @@ export const createShikiHighlighterComponent = (
         ...shikiOptions,
       };
 
+      if (
+        (options as { outputFormat?: string }).outputFormat === 'tokens'
+      ) {
+        throw new Error(
+          '[react-shiki] outputFormat: "tokens" is hook-only. ' +
+            'Use useShikiHighlighter directly to render tokens.'
+        );
+      }
+
       const displayLanguageId =
         typeof language === 'object'
           ? language.name || null
           : language?.trim() || null;
 
-      const highlightedCode = useShikiHighlighterImpl(
+      const highlightedCode = useShikiHighlighterImpl<ComponentOutputFormat>(
         code,
         language,
         theme,
@@ -167,10 +178,7 @@ export const createShikiHighlighterComponent = (
             <span
               id="language-label"
               data-slot="language-label"
-              className={clsx(
-                'rs-language-label',
-                langClassName
-              )}
+              className={clsx('rs-language-label', langClassName)}
               style={langStyle}
             >
               {displayLanguageId}

package/src/lib/hook.ts

@@ -4,13 +4,16 @@ import { toJsxRuntime } from 'hast-util-to-jsx-runtime';
 import type { CodeToHastOptions } from 'shiki';
 
 import type {
-  HighlightedCode,
+  BaseHighlighterOptions,
   HighlighterFactory,
   HighlighterOptions,
+  HighlightResultMap,
   Language,
+  OutputFormat,
   Theme,
   Themes,
   TimeoutState,
+  UseShikiHighlighter,
 } from './types';
 
 import { throttleHighlighting, useStableValue } from './utils';
@@ -22,22 +25,22 @@ import {
 import { resolveTheme } from './theme';
 import { buildShikiOptions } from './options';
 
-export async function highlight(
+export async function highlight<F extends OutputFormat = 'react'>(
   code: string,
   resolved: {
     languageId: string;
     langsToLoad: Language[];
     themesToLoad: Theme[];
     shikiOptions: CodeToHastOptions;
   },
-  opts: Pick<
-    HighlighterOptions,
-    'highlighter' | 'outputFormat' | 'engine'
-  >,
+  opts: Pick<HighlighterOptions, 'highlighter' | 'engine'> & {
+    outputFormat?: F;
+  },
   factory: HighlighterFactory
-) {
+): Promise<HighlightResultMap[F]> {
   const { languageId, langsToLoad, themesToLoad, shikiOptions } =
     resolved;
+  const format: OutputFormat = opts.outputFormat ?? 'react';
 
   const highlighter =
     opts.highlighter ??
@@ -55,24 +58,30 @@ export async function highlight(
   );
   const options = { ...shikiOptions, lang: language };
 
-  return opts.outputFormat === 'html'
-    ? highlighter.codeToHtml(code, options)
-    : toJsxRuntime(highlighter.codeToHast(code, options), {
+  const outputs: { [K in OutputFormat]: () => HighlightResultMap[K] } = {
+    react: () =>
+      toJsxRuntime(highlighter.codeToHast(code, options), {
         jsx,
         jsxs,
         Fragment,
-      });
+      }),
+    html: () => highlighter.codeToHtml(code, options),
+    tokens: () => highlighter.codeToTokens(code, options),
+  };
+
+  return outputs[format]() as HighlightResultMap[F];
 }
 
-export const useHighlight = (
+export const useHighlight = <F extends OutputFormat = 'react'>(
   code: string,
   lang: Language,
   themeInput: Theme | Themes,
-  options: HighlighterOptions = {},
+  options: BaseHighlighterOptions & { outputFormat?: F } = {},
   highlighterFactory: HighlighterFactory
-) => {
-  const [highlightedCode, setHighlightedCode] =
-    useState<HighlightedCode>(null);
+): HighlightResultMap[F] | null => {
+  const [highlightedCode, setHighlightedCode] = useState<
+    HighlightResultMap[F] | null
+  >(null);
 
   const stableLang = useStableValue(lang);
   const stableTheme = useStableValue(themeInput);
@@ -111,7 +120,7 @@ export const useHighlight = (
 
     const run = async () => {
       try {
-        const result = await highlight(
+        const result = await highlight<F>(
           code,
           resolved,
           stableOpts,
@@ -138,3 +147,18 @@ export const useHighlight = (
 
   return highlightedCode;
 };
+
+/**
+ * Create a `useShikiHighlighter` hook bound to a specific highlighter factory.
+ * Used by the bundled entry points (full / web / core).
+ */
+export const createUseShikiHighlighter = (
+  factory: HighlighterFactory
+): UseShikiHighlighter => {
+  return <F extends OutputFormat = 'react'>(
+    code: string,
+    lang: Language,
+    themeInput: Theme | Themes,
+    options?: BaseHighlighterOptions & { outputFormat?: F }
+  ) => useHighlight<F>(code, lang, themeInput, options, factory);
+};

package/src/lib/theme.ts

@@ -10,7 +10,9 @@ export type ResolvedTheme =
   | { isMulti: true; themes: Themes; themesToLoad: Theme[] }
   | { isMulti: false; theme: Theme; themesToLoad: Theme[] };
 
-const isTextmateTheme = (value: unknown): value is ThemeRegistrationAny => {
+const isTextmateTheme = (
+  value: unknown
+): value is ThemeRegistrationAny => {
   if (typeof value !== 'object' || value === null) return false;
   const v = value as Partial<ThemeRegistrationAny> & {
     settings?: unknown;

package/src/lib/types.ts

@@ -12,6 +12,7 @@ import type {
   SpecialLanguage,
   StringLiteralUnion,
   ThemeRegistrationAny,
+  TokensResult,
 } from 'shiki';
 
 import type { ReactElement } from 'react';
@@ -87,9 +88,10 @@ interface ReactShikiOptions {
    * Output format for the highlighted code.
    * - 'react': Returns React nodes (default, safer)
    * - 'html': Returns HTML string (~15-45% faster, requires dangerouslySetInnerHTML)
+   * - 'tokens': Returns Shiki tokens for custom rendering (hook only)
    * @default 'react'
    */
-  outputFormat?: 'react' | 'html';
+  outputFormat?: OutputFormat;
 
   /**
    * Custom Shiki highlighter instance to use instead of the default one.
@@ -164,22 +166,41 @@ interface TimeoutState {
 }
 
 /**
- * Public API signature for the useShikiHighlighter hook.
+ * Supported output formats and their corresponding result shapes.
  */
-type HighlightedCode = ReactElement | string | null;
+type OutputFormat = 'react' | 'html' | 'tokens';
+type ComponentOutputFormat = Exclude<OutputFormat, 'tokens'>;
+
+interface HighlightResultMap {
+  react: ReactElement;
+  html: string;
+  tokens: TokensResult;
+}
+
+type HighlightResult<F extends OutputFormat = 'react'> =
+  HighlightResultMap[F] | null;
+
+type HighlightedCode = HighlightResult<OutputFormat>;
+type ComponentHighlightedCode = HighlightResult<ComponentOutputFormat>;
+
+type BaseHighlighterOptions = Omit<HighlighterOptions, 'outputFormat'>;
+
+type ComponentHighlighterOptions = BaseHighlighterOptions & {
+  outputFormat?: ComponentOutputFormat;
+};
 
 type HighlighterFactory = (
   langsToLoad: Language[],
   themesToLoad: Theme[],
   engine?: Awaitable<RegexEngine>
 ) => Promise<Highlighter | HighlighterCore>;
 
-export type UseShikiHighlighter = (
+export type UseShikiHighlighter = <F extends OutputFormat = 'react'>(
   code: string,
   lang: Language,
   themeInput: Theme | Themes,
-  options?: HighlighterOptions
-) => HighlightedCode;
+  options?: BaseHighlighterOptions & { outputFormat?: F }
+) => HighlightResult<F>;
 
 export type {
   Language,
@@ -188,6 +209,13 @@ export type {
   Element,
   TimeoutState,
   HighlighterOptions,
+  BaseHighlighterOptions,
+  ComponentHighlighterOptions,
+  ComponentHighlightedCode,
   HighlightedCode,
+  HighlightResult,
+  HighlightResultMap,
+  OutputFormat,
+  ComponentOutputFormat,
   HighlighterFactory,
 };