Implement support for pasting URLs into the prompt field to automatically fetch and convert web page content to Markdown context

Author: robertpiosik

apps/editor/package.json

@@ -1875,12 +1875,15 @@
     "@babel/preset-env": "^7.27.2",
     "@babel/preset-react": "^7.27.1",
     "@babel/preset-typescript": "^7.27.1",
+    "@types/dompurify": "^3.0.5",
     "@types/he": "^1.2.3",
+    "@types/jsdom": "^21.1.7",
     "@types/node": "20.x",
     "@types/react": "^18.2.60",
     "@types/react-dom": "^18.2.19",
     "@types/vscode": "1.92.0",
     "@types/ws": "^8.18.1",
+    "@types/turndown": "^5.0.5",
     "@typescript-eslint/eslint-plugin": "^7.18.0",
     "@typescript-eslint/parser": "^7.18.0",
     "babel-loader": "^9.2.1",
@@ -1900,16 +1903,21 @@
     "webpack-cli": "^6.0.1"
   },
   "dependencies": {
+    "@joplin/turndown": "^4.0.81",
+    "@joplin/turndown-plugin-gfm": "^1.0.63",
+    "@mozilla/readability": "^0.6.0",
     "@svgr/webpack": "^8.1.0",
     "@vscode/codicons": "^0.0.43",
     "axios": "^1.10.0",
     "bufferutil": "^4.0.9",
     "classnames": "^2.5.1",
     "dayjs": "^1.11.10",
     "diff": "^8.0.2",
+    "dompurify": "^3.2.7",
     "glob": "^11.0.3",
     "he": "^1.2.0",
     "ignore": "^6.0.2",
+    "jsdom": "^24.0.0",
     "marked": "^15.0.12",
     "path-browserify": "^1.0.1",
     "react": "^18.2.0",

apps/editor/src/env.d.ts

@@ -0,0 +1,2 @@
+declare module '@joplin/turndown'
+declare module '@joplin/turndown-plugin-gfm'

apps/editor/src/views/panel/backend/message-handlers/handle-code-completion.ts

@@ -25,6 +25,7 @@ import { replace_saved_context_symbol } from '@/views/panel/backend/utils/replac
 import { replace_skill_symbol } from '../utils/replace-skill-symbol'
 import { replace_image_symbol } from '../utils/replace-image-symbol'
 import { replace_document_symbol } from '../utils/replace-document-symbol'
+import { replace_website_symbol } from '../utils/replace-website-symbol'
 
 const get_code_completion_config = async (
   api_providers_manager: ModelProvidersManager,
@@ -394,6 +395,12 @@ export const handle_code_completion = async (
       })
     }
 
+    if (processed_completion_instructions.includes('#Website(')) {
+      processed_completion_instructions = await replace_website_symbol({
+        instruction: processed_completion_instructions
+      })
+    }
+
     const files_collector = new FilesCollector(
       panel_provider.workspace_provider,
       panel_provider.open_editors_provider

apps/editor/src/views/panel/backend/message-handlers/handle-copy-prompt.ts

@@ -11,6 +11,7 @@ import { replace_saved_context_symbol } from '@/views/panel/backend/utils/replac
 import { replace_skill_symbol } from '@/views/panel/backend/utils/replace-skill-symbol'
 import { replace_image_symbol } from '@/views/panel/backend/utils/replace-image-symbol'
 import { replace_document_symbol } from '../utils/replace-document-symbol'
+import { replace_website_symbol } from '../utils/replace-website-symbol'
 import {
   code_at_cursor_instructions_for_panel,
   prune_context_instructions_prefix,
@@ -155,6 +156,12 @@ export const handle_copy_prompt = async (params: {
       })
     }
 
+    if (processed_completion_instructions.includes('#Website(')) {
+      processed_completion_instructions = await replace_website_symbol({
+        instruction: processed_completion_instructions
+      })
+    }
+
     const missing_text_tag = processed_completion_instructions
       ? `<missing_text>${processed_completion_instructions}</missing_text>`
       : '<missing_text>'
@@ -233,6 +240,12 @@ export const handle_copy_prompt = async (params: {
       })
     }
 
+    if (processed_instructions.includes('#Website(')) {
+      processed_instructions = await replace_website_symbol({
+        instruction: processed_instructions
+      })
+    }
+
     let system_instructions_xml = ''
 
     if (params.panel_provider.web_prompt_type == 'edit-context') {

apps/editor/src/views/panel/backend/message-handlers/handle-edit-context.ts

@@ -25,6 +25,7 @@ import { PanelProvider } from '@/views/panel/backend/panel-provider'
 import { replace_skill_symbol } from '@/views/panel/backend/utils/replace-skill-symbol'
 import { replace_image_symbol } from '@/views/panel/backend/utils/replace-image-symbol'
 import { replace_document_symbol } from '../utils/replace-document-symbol'
+import { replace_website_symbol } from '../utils/replace-website-symbol'
 import { apply_reasoning_effort } from '@/utils/apply-reasoning-effort'
 import { EditContextMessage } from '@/views/panel/types/messages'
 import { dictionary } from '@shared/constants/dictionary'
@@ -379,6 +380,12 @@ export const handle_edit_context = async (
     })
   }
 
+  if (processed_instructions.includes('#Website(')) {
+    processed_instructions = await replace_website_symbol({
+      instruction: processed_instructions
+    })
+  }
+
   const is_prune_context = panel_provider.api_prompt_type == 'prune-context'
   const collected_files = await files_collector.collect_files({
     compact: is_prune_context

apps/editor/src/views/panel/backend/message-handlers/handle-paste-url.ts

@@ -0,0 +1,155 @@
+import { PasteUrlMessage } from '../../types/messages'
+import { PanelProvider } from '../panel-provider'
+import axios from 'axios'
+import { JSDOM } from 'jsdom'
+import { Readability, isProbablyReaderable } from '@mozilla/readability'
+import createDOMPurify from 'dompurify'
+import TurndownServiceJoplin from '@joplin/turndown'
+import * as turndownPluginGfm from '@joplin/turndown-plugin-gfm'
+import type TurndownService from 'turndown'
+import * as fs from 'fs'
+import * as os from 'os'
+import * as path from 'path'
+import * as crypto from 'crypto'
+
+const remove_markdown_images = (text: string) => {
+  const without_images = text.replace(/!\[([^\]]*)\]\(([^)]*)\)/g, '')
+  return without_images.replace(/\n{3,}/g, '\n\n')
+}
+
+const create_turndown_service = () => {
+  const turndown_service: TurndownService = new TurndownServiceJoplin({
+    codeBlockStyle: 'fenced'
+  })
+  turndown_service.use(turndownPluginGfm.gfm)
+  turndown_service.addRule('fencedCodeBlock', {
+    filter: (node: any, options: any) => {
+      return (
+        options.codeBlockStyle == 'fenced' &&
+        node.nodeName == 'PRE' &&
+        node.querySelector('code')
+      )
+    },
+    replacement: (_: any, node: any, options: any) => {
+      const element = node as HTMLElement
+      const language = (element
+        .querySelector('code')
+        ?.className.match(/language-(\S+)/) || [null, ''])[1]
+
+      return (
+        '\n\n' +
+        options.fence +
+        language +
+        '\n' +
+        element.textContent +
+        '\n' +
+        options.fence +
+        '\n\n'
+      )
+    }
+  })
+  // Convert math blocks to markdown
+  turndown_service.addRule('multiplemath', {
+    filter(node) {
+      return (
+        node.nodeName == 'SPAN' &&
+        (node as HTMLElement).classList.contains('katex-display')
+      ) // Check if it's a display math block that centers equation
+    },
+    replacement(_, node) {
+      // "<annotation>" element holds expression string, right for markdown
+      const annotation = (node as HTMLElement).querySelector(
+        'annotation'
+      )?.textContent
+      if (!annotation) return ''
+      return `$$\n${annotation}\n$$`
+    }
+  })
+  turndown_service.addRule('multiplemath', {
+    filter(node) {
+      return (
+        node.nodeName == 'SPAN' &&
+        (node as HTMLElement).classList.contains('katex')
+      )
+    },
+    replacement(_, node) {
+      const is_block =
+        node.parentNode?.nodeName == 'P' &&
+        node.parentNode.childNodes.length == 1
+      const annotation = (node as HTMLElement).querySelector(
+        'annotation'
+      )?.textContent
+      if (!annotation) return ''
+      return is_block ? `$$ ${annotation} $$` : `$${annotation}$`
+    }
+  })
+  turndown_service.addRule('stripElements', {
+    filter: ['figure', 'picture', 'sup'],
+    replacement: () => ''
+  })
+  return turndown_service
+}
+
+export const handle_paste_url = async (
+  panel_provider: PanelProvider,
+  message: PasteUrlMessage
+) => {
+  try {
+    const url = message.url
+    const hash = crypto.createHash('md5').update(url).digest('hex')
+    const filename = `cwc-website-${hash}.txt`
+    const file_path = path.join(os.tmpdir(), filename)
+
+    if (fs.existsSync(file_path)) {
+      panel_provider.add_text_at_cursor_position(`#Website(${url})`)
+      return
+    }
+
+    const response = await axios.get(url, {
+      headers: {
+        'User-Agent':
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36'
+      },
+      timeout: 5000
+    })
+
+    if (response.status == 200 && typeof response.data == 'string') {
+      const html = response.data
+      const window = new JSDOM('').window
+      const DOMPurify = createDOMPurify(window as any)
+      const clean_html = DOMPurify.sanitize(html)
+      const dom = new JSDOM(clean_html, { url })
+      const doc = dom.window.document
+
+      if (isProbablyReaderable(doc)) {
+        const reader = new Readability(doc, { keepClasses: true })
+        const article = reader.parse()
+
+        if (article && article.content) {
+          const turndown_service = create_turndown_service()
+          const article_dom = new JSDOM(article.content)
+          let content = turndown_service.turndown(
+            article_dom.window.document.body
+          )
+          content = remove_markdown_images(content)
+
+          if (content && content.trim().length > 0) {
+            if (article.title) {
+              content = `# ${article.title}\n\n${content}`
+            }
+
+            await fs.promises.writeFile(file_path, content, 'utf-8')
+            panel_provider.add_text_at_cursor_position(`#Website(${url})`)
+            return
+          }
+        }
+      }
+
+      panel_provider.add_text_at_cursor_position(url)
+    } else {
+      panel_provider.add_text_at_cursor_position(url)
+    }
+  } catch (error) {
+    panel_provider.add_text_at_cursor_position(message.url)
+  }
+}

apps/editor/src/views/panel/backend/message-handlers/handle-preview-preset.ts

@@ -13,6 +13,7 @@ import { Preset } from '@shared/types/preset'
 import { replace_skill_symbol } from '@/views/panel/backend/utils/replace-skill-symbol'
 import { replace_image_symbol } from '@/views/panel/backend/utils/replace-image-symbol'
 import { replace_document_symbol } from '../utils/replace-document-symbol'
+import { replace_website_symbol } from '../utils/replace-website-symbol'
 import { apply_preset_affixes_to_instruction } from '@/utils/apply-preset-affixes'
 import { dictionary } from '@shared/constants/dictionary'
 import {
@@ -135,6 +136,12 @@ export const handle_preview_preset = async (
       })
     }
 
+    if (processed_completion_instructions.includes('#Website(')) {
+      processed_completion_instructions = await replace_website_symbol({
+        instruction: processed_completion_instructions
+      })
+    }
+
     const missing_text_tag = processed_completion_instructions
       ? `<missing_text>${processed_completion_instructions}</missing_text>`
       : '<missing_text>'
@@ -222,6 +229,12 @@ export const handle_preview_preset = async (
       })
     }
 
+    if (processed_instructions.includes('#Website(')) {
+      processed_instructions = await replace_website_symbol({
+        instruction: processed_instructions
+      })
+    }
+
     let system_instructions_xml = ''
     if (panel_provider.web_prompt_type == 'edit-context') {
       const config = vscode.workspace.getConfiguration('codeWebChat')

apps/editor/src/views/panel/backend/message-handlers/handle-prune-context.ts

@@ -21,6 +21,7 @@ import { PanelProvider } from '@/views/panel/backend/panel-provider'
 import { replace_skill_symbol } from '@/views/panel/backend/utils/replace-skill-symbol'
 import { replace_image_symbol } from '@/views/panel/backend/utils/replace-image-symbol'
 import { replace_document_symbol } from '@/views/panel/backend/utils/replace-document-symbol'
+import { replace_website_symbol } from '@/views/panel/backend/utils/replace-website-symbol'
 import { apply_reasoning_effort } from '@/utils/apply-reasoning-effort'
 import { PruneContextMessage } from '@/views/panel/types/messages'
 import { dictionary } from '@shared/constants/dictionary'
@@ -358,6 +359,12 @@ export const handle_prune_context = async (
     })
   }
 
+  if (processed_instructions.includes('#Website(')) {
+    processed_instructions = await replace_website_symbol({
+      instruction: processed_instructions
+    })
+  }
+
   const collected_files = await files_collector.collect_files({
     compact: true
   })

apps/editor/src/views/panel/backend/message-handlers/handle-send-to-browser.ts

@@ -12,6 +12,7 @@ import {
 import { replace_skill_symbol } from '@/views/panel/backend/utils/replace-skill-symbol'
 import { replace_image_symbol } from '@/views/panel/backend/utils/replace-image-symbol'
 import { replace_document_symbol } from '../utils/replace-document-symbol'
+import { replace_website_symbol } from '../utils/replace-website-symbol'
 import {
   code_at_cursor_instructions_for_panel,
   prune_context_instructions_prefix,
@@ -182,6 +183,12 @@ export const handle_send_to_browser = async (params: {
       })
     }
 
+    if (processed_completion_instructions.includes('#Website(')) {
+      processed_completion_instructions = await replace_website_symbol({
+        instruction: processed_completion_instructions
+      })
+    }
+
     const context_text = await files_collector.collect_files({
       exclude_path: active_path
     })
@@ -297,6 +304,12 @@ export const handle_send_to_browser = async (params: {
           })
         }
 
+        if (processed_instructions.includes('#Website(')) {
+          processed_instructions = await replace_website_symbol({
+            instruction: processed_instructions
+          })
+        }
+
         let system_instructions_xml = ''
         if (params.panel_provider.web_prompt_type == 'edit-context') {
           const config = vscode.workspace.getConfiguration('codeWebChat')

apps/editor/src/views/panel/backend/message-handlers/index.ts

@@ -65,3 +65,4 @@ export * from './handle-save-prompt-image'
 export * from './handle-open-prompt-image'
 export * from './handle-save-prompt-document'
 export * from './handle-open-prompt-document'
+export * from './handle-paste-url'