From 914fe27c86a5d93afac2669aa7689ce5d775d593 Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Mon, 13 May 2024 18:58:48 +0200 Subject: [PATCH 01/12] test --- FeatureFlags.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FeatureFlags.js b/FeatureFlags.js index 5b46c9633..cf50e1886 100644 --- a/FeatureFlags.js +++ b/FeatureFlags.js @@ -45,7 +45,7 @@ module.exports = { // Whether the Chatbot UserInterface and its functionality should be enabled enableChatbot: false, - // ----------------------------------------------------------------------------- + // ------------------------------------------------------------------------------ // Chopping Block // // Planned feature deprecations and breaking changes. Sorted roughly in order of From fed67a34e69e949d0a28d4ec76c3f21474f20ecc Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Mon, 13 May 2024 19:04:10 +0200 Subject: [PATCH 02/12] chatbot UI --- FeatureFlags.js | 5 +- .../processes/[processId]/chatbot-dialog.tsx | 75 +++++++ .../processes/[processId]/modeler-toolbar.tsx | 25 +++ .../processes/[processId]/modeler.tsx | 21 ++ src/management-system-v2/lib/mermaidParser.ts | 212 ++++++++++++++++++ 5 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx create mode 100644 src/management-system-v2/lib/mermaidParser.ts diff --git a/FeatureFlags.js b/FeatureFlags.js index cf50e1886..fbbbb976b 100644 --- a/FeatureFlags.js +++ b/FeatureFlags.js @@ -45,7 +45,10 @@ module.exports = { // Whether the Chatbot UserInterface and its functionality should be enabled enableChatbot: false, - // ------------------------------------------------------------------------------ + // Chatbot for creating and modifying BPMN in modeler + enableBPMNChatbot: false, + + // ----------------------------------------------------------------------------- // Chopping Block // // Planned feature deprecations and breaking changes. Sorted roughly in order of diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx new file mode 100644 index 000000000..2abbf1dbe --- /dev/null +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx @@ -0,0 +1,75 @@ +import { BPMNCanvasRef } from '@/components/bpmn-canvas'; +import { Button, Card, Form, Input, List, Space } from 'antd'; +import React, { useState } from 'react'; + +type PromptAndAnswer = { + userPrompt: string; + chatbotAnswer: string; +}; + +export type ChatbotRequest = { + userPrompt: string; + lastPrompts: PromptAndAnswer[]; +}; + +type ChatbotDialogProps = { + sendPrompt: (request: ChatbotRequest) => Promise; + hidden: boolean; + modeler: BPMNCanvasRef | null; +}; + +type FieldType = { + prompt: string; +}; + +const ChatbotDialog: React.FC = ({ sendPrompt, hidden, modeler }) => { + const [lastPrompts, setLastPrompts] = useState([]); + const [waitForResponse, setWaitForResponse] = useState(false); + + function onPrompt({ prompt }: FieldType) { + setWaitForResponse(true); + sendPrompt({ + lastPrompts: lastPrompts, + userPrompt: prompt, + }) + .then((response) => { + setLastPrompts(lastPrompts.concat([{ userPrompt: prompt, chatbotAnswer: response }])); + }) + .finally(() => setWaitForResponse(false)); + } + + function testModeler() { + const factory = modeler?.getFactory(); + } + + return ( + + ); +}; + +export default ChatbotDialog; diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx index e80917c1f..886ba0e3f 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx @@ -13,6 +13,7 @@ import Icon, { ArrowDownOutlined, FullscreenOutlined, FilePdfOutlined, + RobotOutlined, } from '@ant-design/icons'; import { SvgXML } from '@/components/svg'; import PropertiesPanel from './properties-panel'; @@ -28,6 +29,9 @@ import ModelerShareModalButton from './modeler-share-modal'; import { ProcessExportTypes } from '@/components/process-export'; import { spaceURL } from '@/lib/utils'; import { generateSharedViewerUrl } from '@/lib/sharing/process-sharing'; +import ChatbotDialog, { ChatbotRequest } from './chatbot-dialog'; +import { enableBPMNChatbot } from 'FeatureFlags'; +import { BPMNCanvasRef } from '@/components/bpmn-canvas'; const LATEST_VERSION = { version: -1, name: 'Latest Version', description: '' }; @@ -37,6 +41,8 @@ type ModelerToolbarProps = { canUndo: boolean; canRedo: boolean; versions: { version: number; name: string; description: string }[]; + sendPrompt: (request: ChatbotRequest) => Promise; + modeler: BPMNCanvasRef | null; }; const ModelerToolbar = ({ processId, @@ -44,6 +50,7 @@ const ModelerToolbar = ({ canUndo, canRedo, versions, + sendPrompt, }: ModelerToolbarProps) => { const router = useRouter(); const environment = useEnvironment(); @@ -58,6 +65,7 @@ const ModelerToolbar = ({ const query = useSearchParams(); const subprocessId = query.get('subprocess'); + const [showChatbotDialog, setShowChatbotDialog] = useState(false); const modeler = useModelerStateStore((state) => state.modeler); const selectedElementId = useModelerStateStore((state) => state.selectedElementId); @@ -275,6 +283,15 @@ const ModelerToolbar = ({ )} + + {enableBPMNChatbot && ( + + + + )} {showPropertiesPanel && selectedElement && ( @@ -284,6 +301,14 @@ const ModelerToolbar = ({ selectedElement={selectedElement} /> )} + + {enableBPMNChatbot && ( + + )} diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx index c14d7d0f1..ce7deef37 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx @@ -16,6 +16,8 @@ import { useEnvironment } from '@/components/auth-can'; import styles from './modeler.module.scss'; import ModelerZoombar from './modeler-zoombar'; import { useAddControlCallback } from '@/lib/controls-store'; +import { ChatbotRequest } from './chatbot-dialog'; +import { mermaid2BPMN } from '@/lib/mermaidParser'; type ModelerProps = React.HTMLAttributes & { versionName?: string; @@ -253,6 +255,23 @@ const Modeler = ({ versionName, process, versions, ...divProps }: ModelerProps) setXmlEditorBpmn(undefined); }; + function sendPrompt(request: ChatbotRequest): Promise { + const chatbotResponse = fetch('http://localhost:2000', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(request), + }).then((res) => res.text()); + const xml = modeler.current?.getXML(); + return Promise.all([chatbotResponse, xml]).then(([res, xml]) => { + if (xml) { + handleXmlEditorSave(mermaid2BPMN(res, xml)); + } + return res; + }); + } + const handleXmlEditorSave = async (bpmn: string) => { if (modeler.current) { await modeler.current.loadBPMN(bpmn); @@ -283,6 +302,8 @@ const Modeler = ({ versionName, process, versions, ...divProps }: ModelerProps) versions={versions} canRedo={canRedo} canUndo={canUndo} + sendPrompt={sendPrompt} + modeler={modeler.current} /> )} {selectedVersionId && !showMobileView && } diff --git a/src/management-system-v2/lib/mermaidParser.ts b/src/management-system-v2/lib/mermaidParser.ts new file mode 100644 index 000000000..2557e41f4 --- /dev/null +++ b/src/management-system-v2/lib/mermaidParser.ts @@ -0,0 +1,212 @@ +type shape = { + metaInfo: MetaInfo; + id: string; + label: string; + pos: point; + incoming: Array; + outgoing: Array; +}; + +type point = { + x: number; + y: number; +}; + +type flow = { + id: string; + sourceRef: string; + targetRef: string; + waypoints: Array; + name?: string; + namePoint?: point; + nameWidth?: number; + nameHeight?: number; +}; + +type MetaInfo = { + xmlTag: string; + width: number; + height: number; +}; + +export function mermaid2BPMN(mermaid: string, xml: string): string { + //extract relevant lines describing the graph from mermaid string + //apply regular expression to split lines into groups + const lines = mermaid.split('\n').map((l) => l.trim()); + const regExp = new RegExp( + /(?[A-Za-z0-9äüöÄÖÜ]+)(?[\(\[\{\>\/\\]*)(?[A-Za-z0-9 ?ÄÜÖäüö]*)(?[\)\]\}\/\\]*) -->[\|]?(?[A-Za-z0-9ÄÜÖäüö ]*)[\|]? (?[A-Za-z0-9ÄÖÜäöü]+)(?[\(\[\{\>\/\\]*)(?[A-Za-z0-9 ?ÄÖÜäöü]*)(?[\)\]\}\/\\]*)/, + ); + const matches = lines.map((l) => regExp.exec(l)); + const groups = matches.map((m) => m?.groups).filter((g) => g); + + const shapes: Array = []; + const flows: Array = []; + + for (let group of groups) { + let source: shape, target: shape; + if (group!.sourceLabel) { + source = { + metaInfo: getMetaInfo(group!.sourceBracketsLeft), + id: group!.source, + label: group!.source + ': ' + group!.sourceLabel, + pos: { x: 0, y: 0 }, + incoming: [], + outgoing: [], + }; + shapes.push(source); + } else { + source = shapes.find((e) => e.id == group!.source)!; + } + if (group!.targetLabel) { + const metaInfo = getMetaInfo(group!.targetBracketsLeft); + target = { + metaInfo: metaInfo, + id: group!.target, + label: group!.target + ': ' + group!.targetLabel, + pos: getPos(source, metaInfo), + incoming: [], + outgoing: [], + }; + shapes.push(target); + } else { + target = shapes.find((e) => e.id == group!.target)!; + } + const flow: flow = { + id: source.id + '_' + target.id, + sourceRef: source.id, + targetRef: target.id, + waypoints: [getMid(source), getMid(target)], + }; + if (group!.edgeLabel) { + flow.name = group!.edgeLabel; + flow.namePoint = getMid(source, target); + flow.nameWidth = 20; + flow.nameHeight = 15; + } + flows.push(flow); + source.outgoing.push(flow.id); + target.incoming.push(flow.id); + } + + let newXML = xml.slice(0, xml.indexOf('') + 16); //XML Preamble including ... + //adding element definitions inside ... + for (let shape of shapes) { + newXML += '<' + shape.metaInfo.xmlTag + ' id="' + shape.id + '" name="' + shape.label + '">'; + for (let inFlow of shape.incoming) { + newXML += '' + inFlow + ''; + } + for (let outFlow of shape.outgoing) { + newXML += '' + outFlow + ''; + } + newXML += ''; + } + for (let flow of flows) { + newXML += + ''), + xml.indexOf('>', xml.indexOf('')) { + newXML = newXML.slice(0, -2) + '>'; + } + for (let shape of shapes) { + newXML += ''; + newXML += + ''; + newXML += ''; + } + for (let flow of flows) { + newXML += ''; + for (let waypoint of flow.waypoints) { + newXML += ''; + } + if (flow.name) { + newXML += + ''; + } + newXML += ''; + } + newXML += ''; + console.log(newXML); + + return newXML; +} + +function getMid(shape: shape, shape2?: shape): point { + if (shape2) { + const p1 = getMid(shape); + const p2 = getMid(shape2); + return { + x: (p1.x + p2.x) / 2, + y: (p1.y + p2.y) / 2, + }; + } + return { + x: shape.pos.x + shape.metaInfo.width / 2, + y: shape.pos.y + shape.metaInfo.height / 2, + }; +} + +function getMetaInfo(bracketsLeft: string): MetaInfo { + if (bracketsLeft == '[') { + return { + xmlTag: 'startEvent', + width: 36, + height: 36, + }; + } + if (bracketsLeft == '(') { + return { + xmlTag: 'task', + width: 100, + height: 80, + }; + } + if (bracketsLeft == '{') { + return { + xmlTag: 'exclusiveGateway', + width: 50, + height: 50, + }; + } + return { + xmlTag: 'endEvent', + width: 36, + height: 36, + }; +} + +function getPos(predecessor: shape, current: MetaInfo): point { + return { + x: predecessor.pos.x + predecessor.metaInfo.width + 60, + y: predecessor.pos.y + (predecessor.metaInfo.height - current.height) / 2, + }; +} From 25c6ed7f8f2942fd59c24ae652807a833a89b510 Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Mon, 13 May 2024 19:30:14 +0200 Subject: [PATCH 03/12] moved fetching to chatbot-dialog moved the fetching functionality from the modeler to the chatbot-diaglog file --- .../processes/[processId]/chatbot-dialog.tsx | 23 +++++++++++++++-- .../processes/[processId]/modeler-toolbar.tsx | 6 ++--- .../processes/[processId]/modeler.tsx | 25 +++---------------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx index 2abbf1dbe..3365e6571 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx @@ -1,4 +1,5 @@ import { BPMNCanvasRef } from '@/components/bpmn-canvas'; +import { mermaid2BPMN } from '@/lib/mermaidParser'; import { Button, Card, Form, Input, List, Space } from 'antd'; import React, { useState } from 'react'; @@ -13,7 +14,7 @@ export type ChatbotRequest = { }; type ChatbotDialogProps = { - sendPrompt: (request: ChatbotRequest) => Promise; + handleXmlSave: (bpmn: string) => Promise; hidden: boolean; modeler: BPMNCanvasRef | null; }; @@ -22,10 +23,28 @@ type FieldType = { prompt: string; }; -const ChatbotDialog: React.FC = ({ sendPrompt, hidden, modeler }) => { +const ChatbotDialog: React.FC = ({ handleXmlSave, hidden, modeler }) => { const [lastPrompts, setLastPrompts] = useState([]); + const [waitForResponse, setWaitForResponse] = useState(false); + function sendPrompt(request: ChatbotRequest): Promise { + const chatbotResponse = fetch('http://localhost:2000', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(request), + }).then((res) => res.text()); + const xml = modeler?.getXML(); + return Promise.all([chatbotResponse, xml]).then(([res, xml]) => { + if (xml) { + handleXmlSave(mermaid2BPMN(res, xml)); + } + return res; + }); + } + function onPrompt({ prompt }: FieldType) { setWaitForResponse(true); sendPrompt({ diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx index 886ba0e3f..351e4e122 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx @@ -41,8 +41,8 @@ type ModelerToolbarProps = { canUndo: boolean; canRedo: boolean; versions: { version: number; name: string; description: string }[]; - sendPrompt: (request: ChatbotRequest) => Promise; modeler: BPMNCanvasRef | null; + handleXmlSave: (bpmn: string) => Promise; }; const ModelerToolbar = ({ processId, @@ -50,7 +50,7 @@ const ModelerToolbar = ({ canUndo, canRedo, versions, - sendPrompt, + handleXmlSave, }: ModelerToolbarProps) => { const router = useRouter(); const environment = useEnvironment(); @@ -304,9 +304,9 @@ const ModelerToolbar = ({ {enableBPMNChatbot && ( )} diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx index ce7deef37..1a97d6ee8 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx @@ -16,8 +16,6 @@ import { useEnvironment } from '@/components/auth-can'; import styles from './modeler.module.scss'; import ModelerZoombar from './modeler-zoombar'; import { useAddControlCallback } from '@/lib/controls-store'; -import { ChatbotRequest } from './chatbot-dialog'; -import { mermaid2BPMN } from '@/lib/mermaidParser'; type ModelerProps = React.HTMLAttributes & { versionName?: string; @@ -255,24 +253,7 @@ const Modeler = ({ versionName, process, versions, ...divProps }: ModelerProps) setXmlEditorBpmn(undefined); }; - function sendPrompt(request: ChatbotRequest): Promise { - const chatbotResponse = fetch('http://localhost:2000', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(request), - }).then((res) => res.text()); - const xml = modeler.current?.getXML(); - return Promise.all([chatbotResponse, xml]).then(([res, xml]) => { - if (xml) { - handleXmlEditorSave(mermaid2BPMN(res, xml)); - } - return res; - }); - } - - const handleXmlEditorSave = async (bpmn: string) => { + const handleXmlSave = async (bpmn: string) => { if (modeler.current) { await modeler.current.loadBPMN(bpmn); // If the bpmn contains unexpected content (text content for an element @@ -302,8 +283,8 @@ const Modeler = ({ versionName, process, versions, ...divProps }: ModelerProps) versions={versions} canRedo={canRedo} canUndo={canUndo} - sendPrompt={sendPrompt} modeler={modeler.current} + handleXmlSave={handleXmlSave} /> )} {selectedVersionId && !showMobileView && } @@ -313,7 +294,7 @@ const Modeler = ({ versionName, process, versions, ...divProps }: ModelerProps) bpmn={xmlEditorBpmn} canSave={!selectedVersionId} onClose={handleCloseXmlEditor} - onSaveXml={handleXmlEditorSave} + onSaveXml={handleXmlSave} process={process} versionName={versionName} /> From c08d09bb80b99694368678e90bc1c1e029963905 Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Thu, 16 May 2024 15:54:57 +0200 Subject: [PATCH 04/12] remove mermaid, implement tool-use --- .gitignore | 2 +- FeatureFlags.js | 2 +- .../processes/[processId]/chatbot-dialog.tsx | 94 -------- .../processes/[processId]/modeler-toolbar.tsx | 12 +- .../components/bpmn-canvas.tsx | 5 + .../components/bpmn-chatbot-dialog.tsx | 146 ++++++++++++ src/management-system-v2/lib/mermaidParser.ts | 212 ------------------ 7 files changed, 155 insertions(+), 318 deletions(-) delete mode 100644 src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx create mode 100644 src/management-system-v2/components/bpmn-chatbot-dialog.tsx delete mode 100644 src/management-system-v2/lib/mermaidParser.ts diff --git a/.gitignore b/.gitignore index a7bcceada..e2bffa3af 100644 --- a/.gitignore +++ b/.gitignore @@ -55,4 +55,4 @@ package-lock.json dataEval.json # Ignore generated credentials from google-github-actions/auth -gha-creds-*.json \ No newline at end of file +gha-creds-*.json diff --git a/FeatureFlags.js b/FeatureFlags.js index fbbbb976b..e1746c5f7 100644 --- a/FeatureFlags.js +++ b/FeatureFlags.js @@ -46,7 +46,7 @@ module.exports = { enableChatbot: false, // Chatbot for creating and modifying BPMN in modeler - enableBPMNChatbot: false, + enableBPMNChatbot: true, // ----------------------------------------------------------------------------- // Chopping Block diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx deleted file mode 100644 index 3365e6571..000000000 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/chatbot-dialog.tsx +++ /dev/null @@ -1,94 +0,0 @@ -import { BPMNCanvasRef } from '@/components/bpmn-canvas'; -import { mermaid2BPMN } from '@/lib/mermaidParser'; -import { Button, Card, Form, Input, List, Space } from 'antd'; -import React, { useState } from 'react'; - -type PromptAndAnswer = { - userPrompt: string; - chatbotAnswer: string; -}; - -export type ChatbotRequest = { - userPrompt: string; - lastPrompts: PromptAndAnswer[]; -}; - -type ChatbotDialogProps = { - handleXmlSave: (bpmn: string) => Promise; - hidden: boolean; - modeler: BPMNCanvasRef | null; -}; - -type FieldType = { - prompt: string; -}; - -const ChatbotDialog: React.FC = ({ handleXmlSave, hidden, modeler }) => { - const [lastPrompts, setLastPrompts] = useState([]); - - const [waitForResponse, setWaitForResponse] = useState(false); - - function sendPrompt(request: ChatbotRequest): Promise { - const chatbotResponse = fetch('http://localhost:2000', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(request), - }).then((res) => res.text()); - const xml = modeler?.getXML(); - return Promise.all([chatbotResponse, xml]).then(([res, xml]) => { - if (xml) { - handleXmlSave(mermaid2BPMN(res, xml)); - } - return res; - }); - } - - function onPrompt({ prompt }: FieldType) { - setWaitForResponse(true); - sendPrompt({ - lastPrompts: lastPrompts, - userPrompt: prompt, - }) - .then((response) => { - setLastPrompts(lastPrompts.concat([{ userPrompt: prompt, chatbotAnswer: response }])); - }) - .finally(() => setWaitForResponse(false)); - } - - function testModeler() { - const factory = modeler?.getFactory(); - } - - return ( - - ); -}; - -export default ChatbotDialog; diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx index 351e4e122..a2e7aa1ae 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx @@ -10,8 +10,6 @@ import Icon, { UndoOutlined, RedoOutlined, ArrowUpOutlined, - ArrowDownOutlined, - FullscreenOutlined, FilePdfOutlined, RobotOutlined, } from '@ant-design/icons'; @@ -29,7 +27,7 @@ import ModelerShareModalButton from './modeler-share-modal'; import { ProcessExportTypes } from '@/components/process-export'; import { spaceURL } from '@/lib/utils'; import { generateSharedViewerUrl } from '@/lib/sharing/process-sharing'; -import ChatbotDialog, { ChatbotRequest } from './chatbot-dialog'; +import ChatbotDialog from '@/components/bpmn-chatbot-dialog'; import { enableBPMNChatbot } from 'FeatureFlags'; import { BPMNCanvasRef } from '@/components/bpmn-canvas'; @@ -42,7 +40,6 @@ type ModelerToolbarProps = { canRedo: boolean; versions: { version: number; name: string; description: string }[]; modeler: BPMNCanvasRef | null; - handleXmlSave: (bpmn: string) => Promise; }; const ModelerToolbar = ({ processId, @@ -50,7 +47,6 @@ const ModelerToolbar = ({ canUndo, canRedo, versions, - handleXmlSave, }: ModelerToolbarProps) => { const router = useRouter(); const environment = useEnvironment(); @@ -303,11 +299,7 @@ const ModelerToolbar = ({ )} {enableBPMNChatbot && ( - + )} diff --git a/src/management-system-v2/components/bpmn-canvas.tsx b/src/management-system-v2/components/bpmn-canvas.tsx index fc65db561..9d17cb0d1 100644 --- a/src/management-system-v2/components/bpmn-canvas.tsx +++ b/src/management-system-v2/components/bpmn-canvas.tsx @@ -17,6 +17,7 @@ import schema from '@/lib/schema'; import { copyProcessImage } from '@/lib/process-export/copy-process-image'; import Modeling, { CommandStack } from 'bpmn-js/lib/features/modeling/Modeling'; import { Root, Element } from 'bpmn-js/lib/model/Types'; +import ElementFactory from 'bpmn-js/lib/features/modeling/ElementFactory'; // Conditionally load the BPMN modeler only on the client, because it uses // "window" reference. It won't be included in the initial bundle, but will be @@ -81,6 +82,7 @@ export interface BPMNCanvasRef { getModeling: () => Modeling; getFactory: () => BpmnFactory; loadBPMN: (bpmn: string) => Promise; + getElementFactory: () => ElementFactory; } const BPMNCanvas = forwardRef( @@ -159,6 +161,9 @@ const BPMNCanvas = forwardRef( await modeler.current!.importXML(bpmn); fitViewport(modeler.current!); }, + getElementFactory: () => { + return modeler.current!.get('elementFactory'); + }, })); const [Modeler, Viewer] = use(BPMNJs); diff --git a/src/management-system-v2/components/bpmn-chatbot-dialog.tsx b/src/management-system-v2/components/bpmn-chatbot-dialog.tsx new file mode 100644 index 000000000..9dc9ab5c9 --- /dev/null +++ b/src/management-system-v2/components/bpmn-chatbot-dialog.tsx @@ -0,0 +1,146 @@ +import { BPMNCanvasRef } from '@/components/bpmn-canvas'; +import { Button, Card, Form, Input, List, Space, Tooltip } from 'antd'; +import React, { useState } from 'react'; +import { getNewShapePosition } from 'bpmn-js/lib/features/auto-place/BpmnAutoPlaceUtil'; +import { Shape } from 'bpmn-js/lib/model/Types'; +import { MessageOutlined } from '@ant-design/icons'; + +type ChatbotDialogProps = { + show: boolean; + modeler: BPMNCanvasRef | null; +}; + +type FieldType = { + prompt: string; +}; + +const ChatbotDialog: React.FC = ({ show, modeler }) => { + const [lastPrompts, setLastPrompts] = useState([]); + const [waitForResponse, setWaitForResponse] = useState(false); + const elementFactory = modeler!.getElementFactory(); + const root = modeler!.getCurrentRoot(); + const modeling = modeler!.getModeling(); + + function onPrompt({ prompt }: FieldType) { + setWaitForResponse(true); + getProcessXml() + .then((process) => { + console.log(process); + return fetchClaude(prompt, process); + }) + .then((res) => { + console.log(res); + processResponse(res.content); + setLastPrompts(lastPrompts.concat(prompt)); + }) + .finally(() => setWaitForResponse(false)); + } + + function processResponse(response: any[]) { + const created: Shape[] = []; + response.forEach((res) => { + if (res.type == 'tool_use') { + if (res.name == 'create_and_append_task') { + const task = elementFactory.createShape({ + type: 'bpmn:Task', + name: res.input.new_task_name, + }); + let sourceShape = modeler?.getElement(res.input.source_task_id_or_name) as Shape; + if (!sourceShape) { + sourceShape = created.find((e) => e.name == res.input.source_task_id_or_name)!; + } + const point = getNewShapePosition(sourceShape, task); + modeling.appendShape(sourceShape, task, point, root!); + created.push(task); + } else if (res.name == 'create_end') { + const endEvent = elementFactory.createShape({ + type: 'bpmn:EndEvent', + }); + let sourceShape = modeler?.getElement(res.input.element_id_or_name) as Shape; + if (!sourceShape) { + sourceShape = created.find((e) => e.name == res.input.element_id_or_name)!; + } + const point = getNewShapePosition(sourceShape, endEvent); + modeling.appendShape(sourceShape, endEvent, point, root!); + created.push(endEvent); + } else if (res.name == 'create_start') { + const startEvent = modeling.createShape( + { + type: 'bpmn:StartEvent', + name: res.input.name, + }, + { x: 350, y: 200 }, + root!, + ); + created.push(startEvent); + } + } + }); + } + + function getProcessXml(): Promise { + return modeler!.getXML().then((res) => { + if (res) { + const startIndex = res.indexOf('', startIndex); + if (endIndex == -1) { + return ''; + } + return res.slice(startIndex, endIndex) + ''; + } else { + return ''; + } + }); + } + + function fetchClaude(userPrompt: string, process: string): Promise { + return fetch('http://localhost:2000/', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ userPrompt: userPrompt, process: process }), + }).then((res) => res.json()); + } + + return ( + + ); +}; + +export default ChatbotDialog; diff --git a/src/management-system-v2/lib/mermaidParser.ts b/src/management-system-v2/lib/mermaidParser.ts deleted file mode 100644 index 2557e41f4..000000000 --- a/src/management-system-v2/lib/mermaidParser.ts +++ /dev/null @@ -1,212 +0,0 @@ -type shape = { - metaInfo: MetaInfo; - id: string; - label: string; - pos: point; - incoming: Array; - outgoing: Array; -}; - -type point = { - x: number; - y: number; -}; - -type flow = { - id: string; - sourceRef: string; - targetRef: string; - waypoints: Array; - name?: string; - namePoint?: point; - nameWidth?: number; - nameHeight?: number; -}; - -type MetaInfo = { - xmlTag: string; - width: number; - height: number; -}; - -export function mermaid2BPMN(mermaid: string, xml: string): string { - //extract relevant lines describing the graph from mermaid string - //apply regular expression to split lines into groups - const lines = mermaid.split('\n').map((l) => l.trim()); - const regExp = new RegExp( - /(?[A-Za-z0-9äüöÄÖÜ]+)(?[\(\[\{\>\/\\]*)(?[A-Za-z0-9 ?ÄÜÖäüö]*)(?[\)\]\}\/\\]*) -->[\|]?(?[A-Za-z0-9ÄÜÖäüö ]*)[\|]? (?[A-Za-z0-9ÄÖÜäöü]+)(?[\(\[\{\>\/\\]*)(?[A-Za-z0-9 ?ÄÖÜäöü]*)(?[\)\]\}\/\\]*)/, - ); - const matches = lines.map((l) => regExp.exec(l)); - const groups = matches.map((m) => m?.groups).filter((g) => g); - - const shapes: Array = []; - const flows: Array = []; - - for (let group of groups) { - let source: shape, target: shape; - if (group!.sourceLabel) { - source = { - metaInfo: getMetaInfo(group!.sourceBracketsLeft), - id: group!.source, - label: group!.source + ': ' + group!.sourceLabel, - pos: { x: 0, y: 0 }, - incoming: [], - outgoing: [], - }; - shapes.push(source); - } else { - source = shapes.find((e) => e.id == group!.source)!; - } - if (group!.targetLabel) { - const metaInfo = getMetaInfo(group!.targetBracketsLeft); - target = { - metaInfo: metaInfo, - id: group!.target, - label: group!.target + ': ' + group!.targetLabel, - pos: getPos(source, metaInfo), - incoming: [], - outgoing: [], - }; - shapes.push(target); - } else { - target = shapes.find((e) => e.id == group!.target)!; - } - const flow: flow = { - id: source.id + '_' + target.id, - sourceRef: source.id, - targetRef: target.id, - waypoints: [getMid(source), getMid(target)], - }; - if (group!.edgeLabel) { - flow.name = group!.edgeLabel; - flow.namePoint = getMid(source, target); - flow.nameWidth = 20; - flow.nameHeight = 15; - } - flows.push(flow); - source.outgoing.push(flow.id); - target.incoming.push(flow.id); - } - - let newXML = xml.slice(0, xml.indexOf('') + 16); //XML Preamble including ... - //adding element definitions inside ... - for (let shape of shapes) { - newXML += '<' + shape.metaInfo.xmlTag + ' id="' + shape.id + '" name="' + shape.label + '">'; - for (let inFlow of shape.incoming) { - newXML += '' + inFlow + ''; - } - for (let outFlow of shape.outgoing) { - newXML += '' + outFlow + ''; - } - newXML += ''; - } - for (let flow of flows) { - newXML += - ''), - xml.indexOf('>', xml.indexOf('')) { - newXML = newXML.slice(0, -2) + '>'; - } - for (let shape of shapes) { - newXML += ''; - newXML += - ''; - newXML += ''; - } - for (let flow of flows) { - newXML += ''; - for (let waypoint of flow.waypoints) { - newXML += ''; - } - if (flow.name) { - newXML += - ''; - } - newXML += ''; - } - newXML += ''; - console.log(newXML); - - return newXML; -} - -function getMid(shape: shape, shape2?: shape): point { - if (shape2) { - const p1 = getMid(shape); - const p2 = getMid(shape2); - return { - x: (p1.x + p2.x) / 2, - y: (p1.y + p2.y) / 2, - }; - } - return { - x: shape.pos.x + shape.metaInfo.width / 2, - y: shape.pos.y + shape.metaInfo.height / 2, - }; -} - -function getMetaInfo(bracketsLeft: string): MetaInfo { - if (bracketsLeft == '[') { - return { - xmlTag: 'startEvent', - width: 36, - height: 36, - }; - } - if (bracketsLeft == '(') { - return { - xmlTag: 'task', - width: 100, - height: 80, - }; - } - if (bracketsLeft == '{') { - return { - xmlTag: 'exclusiveGateway', - width: 50, - height: 50, - }; - } - return { - xmlTag: 'endEvent', - width: 36, - height: 36, - }; -} - -function getPos(predecessor: shape, current: MetaInfo): point { - return { - x: predecessor.pos.x + predecessor.metaInfo.width + 60, - y: predecessor.pos.y + (predecessor.metaInfo.height - current.height) / 2, - }; -} From 1c735e1ccb7a323e85b7ef5d27f6504d1ea71f3b Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Fri, 17 May 2024 11:37:50 +0200 Subject: [PATCH 05/12] change file names --- .../[environmentId]/processes/[processId]/modeler-toolbar.tsx | 2 +- .../components/{bpmn-chatbot-dialog.tsx => bpmn-chatbot.tsx} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/management-system-v2/components/{bpmn-chatbot-dialog.tsx => bpmn-chatbot.tsx} (100%) diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx index a2e7aa1ae..a3a36c684 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler-toolbar.tsx @@ -27,7 +27,7 @@ import ModelerShareModalButton from './modeler-share-modal'; import { ProcessExportTypes } from '@/components/process-export'; import { spaceURL } from '@/lib/utils'; import { generateSharedViewerUrl } from '@/lib/sharing/process-sharing'; -import ChatbotDialog from '@/components/bpmn-chatbot-dialog'; +import ChatbotDialog from '@/components/bpmn-chatbot'; import { enableBPMNChatbot } from 'FeatureFlags'; import { BPMNCanvasRef } from '@/components/bpmn-canvas'; diff --git a/src/management-system-v2/components/bpmn-chatbot-dialog.tsx b/src/management-system-v2/components/bpmn-chatbot.tsx similarity index 100% rename from src/management-system-v2/components/bpmn-chatbot-dialog.tsx rename to src/management-system-v2/components/bpmn-chatbot.tsx From df1c6eb936c9aa68a45f6649bb9ddf15dc498787 Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Wed, 22 May 2024 11:10:27 +0200 Subject: [PATCH 06/12] added chatbot response modal --- .../processes/[processId]/modeler.tsx | 1 - .../components/bpmn-chatbot-response.tsx | 36 ++++++ .../components/bpmn-chatbot.tsx | 109 +++++++++++------- 3 files changed, 101 insertions(+), 45 deletions(-) create mode 100644 src/management-system-v2/components/bpmn-chatbot-response.tsx diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx index 1a97d6ee8..8de16fe06 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/processes/[processId]/modeler.tsx @@ -284,7 +284,6 @@ const Modeler = ({ versionName, process, versions, ...divProps }: ModelerProps) canRedo={canRedo} canUndo={canUndo} modeler={modeler.current} - handleXmlSave={handleXmlSave} /> )} {selectedVersionId && !showMobileView && } diff --git a/src/management-system-v2/components/bpmn-chatbot-response.tsx b/src/management-system-v2/components/bpmn-chatbot-response.tsx new file mode 100644 index 000000000..1e441e04c --- /dev/null +++ b/src/management-system-v2/components/bpmn-chatbot-response.tsx @@ -0,0 +1,36 @@ +import { Modal } from 'antd'; +import Title from 'antd/es/typography/Title'; +import React from 'react'; + +type ChatbotResponseModalProps = { + open: boolean; + onClose: () => void; + chatbotInteraction: ChatbotInteraction; +}; + +export type ChatbotInteraction = { + userPrompt: string; + bpmnProcess: string; + chatbotResponse: any[]; +}; + +const ChatbotResponseModal: React.FC = ({ + open, + onClose, + chatbotInteraction, +}) => { + return ( + + User Prompt +

{chatbotInteraction.userPrompt}

+ BPMN Process Element +

{chatbotInteraction.bpmnProcess}

+ Response Content +

+ {JSON.stringify(chatbotInteraction.chatbotResponse)} +

+ + ); +}; + +export default ChatbotResponseModal; diff --git a/src/management-system-v2/components/bpmn-chatbot.tsx b/src/management-system-v2/components/bpmn-chatbot.tsx index 9dc9ab5c9..316c55e33 100644 --- a/src/management-system-v2/components/bpmn-chatbot.tsx +++ b/src/management-system-v2/components/bpmn-chatbot.tsx @@ -4,6 +4,7 @@ import React, { useState } from 'react'; import { getNewShapePosition } from 'bpmn-js/lib/features/auto-place/BpmnAutoPlaceUtil'; import { Shape } from 'bpmn-js/lib/model/Types'; import { MessageOutlined } from '@ant-design/icons'; +import ChatbotResponseModal, { ChatbotInteraction } from './bpmn-chatbot-response'; type ChatbotDialogProps = { show: boolean; @@ -15,23 +16,28 @@ type FieldType = { }; const ChatbotDialog: React.FC = ({ show, modeler }) => { - const [lastPrompts, setLastPrompts] = useState([]); + const [lastPrompts, setLastPrompts] = useState([]); const [waitForResponse, setWaitForResponse] = useState(false); const elementFactory = modeler!.getElementFactory(); const root = modeler!.getCurrentRoot(); const modeling = modeler!.getModeling(); + const [showChatbotResponseModal, setShowChatbotResponseModal] = useState(false); + const [chatbotInteraction, setChatbotInteraction] = useState(); function onPrompt({ prompt }: FieldType) { setWaitForResponse(true); getProcessXml() .then((process) => { - console.log(process); - return fetchClaude(prompt, process); - }) - .then((res) => { - console.log(res); - processResponse(res.content); - setLastPrompts(lastPrompts.concat(prompt)); + fetchClaude(prompt, process).then((res) => { + processResponse(res.content); + setLastPrompts( + lastPrompts.concat({ + userPrompt: prompt, + bpmnProcess: process, + chatbotResponse: res.content, + }), + ); + }); }) .finally(() => setWaitForResponse(false)); } @@ -104,42 +110,57 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { } return ( - + <> + + {chatbotInteraction && ( + setShowChatbotResponseModal(false)} + chatbotInteraction={chatbotInteraction} + > + )} + ); }; From 8b47f4518843c5a3c414a8a56ced277afb3b5757 Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Thu, 23 May 2024 17:10:00 +0200 Subject: [PATCH 07/12] improve tool use tool definitions in extra json file --- .../components/bpmn-chatbot-tools.json | 80 ++++++++++ .../components/bpmn-chatbot.tsx | 140 +++++++++++++----- 2 files changed, 180 insertions(+), 40 deletions(-) create mode 100644 src/management-system-v2/components/bpmn-chatbot-tools.json diff --git a/src/management-system-v2/components/bpmn-chatbot-tools.json b/src/management-system-v2/components/bpmn-chatbot-tools.json new file mode 100644 index 000000000..3ff7d9450 --- /dev/null +++ b/src/management-system-v2/components/bpmn-chatbot-tools.json @@ -0,0 +1,80 @@ +[ + { + "name": "create_element", + "description": "Create a BPMN element. Place it at an arbitrary position or append it to another element.", + "input_schema": { + "type": "object", + "properties": { + "source_element_id_or_name": { + "type": "string", + "description": "The id or name of the source element which to append to. Use the id if the element already exists in the process, otherwise the name. This creates automatically a connection between the new element and the provided source." + }, + "new_element_name": { + "type": "string", + "description": "The name of the new element." + }, + "bpmn_type": { + "type": "string", + "description": "The BPMN type of the element. Possible types are: 'Task','StartEvent','EndEvent','ExclusiveGateway','InclusiveGateway','ParallelGateway'." + }, + "connection_label": { + "type": "string", + "description": "The label of the connection if the new element is appended to another one. This label is used when the source element is a gateway and the connection is taken conditionally. The label tells us what condition has to be met in order to take this connection." + }, + "position": { + "type": "object", + "description": "The position where the new element should be placed. Only needed when the element is not appended to another element. Do not provide this parameter if source_element_id_or_name is provided. Use for the first start event the position x=350 and y=200.", + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": ["x", "y"] + } + }, + "required": ["bpmn_type"] + } + }, + { + "name": "create_connection", + "description": "Create a connection between two elements. Only needed if the connection has not been created yet.", + "input_schema": { + "type": "object", + "properties": { + "label": { + "type": "string", + "description": "The label of the connection. Only needed if the connection comes from a gateway and is only taken, when a certain condition is met." + }, + "source_element_id_or_name": { + "type": "string", + "description": "The id or name of the source element of the connection. User the id if the element already exists in the process, otherwise the name." + }, + "target_element_id_or_name": { + "type": "string", + "description": "The id or name of the target element of the connection. User the id if the element already exists in the process, otherwise the name." + } + }, + "required": ["source_element_id_or_name", "target_element_id_or_name"] + } + }, + { + "name": "remove_elements", + "description": "Remove BPMN elements. Removing an element also removes all associated connections. Connections are also considered elements and can be seperately removed too by including their id here.", + "input_schema": { + "type": "object", + "properties": { + "element_ids": { + "type": "array", + "description": "The ids of the elements to be removed.", + "items": { + "type": "string" + } + } + }, + "required": ["element_ids"] + } + } +] diff --git a/src/management-system-v2/components/bpmn-chatbot.tsx b/src/management-system-v2/components/bpmn-chatbot.tsx index 316c55e33..884425dc7 100644 --- a/src/management-system-v2/components/bpmn-chatbot.tsx +++ b/src/management-system-v2/components/bpmn-chatbot.tsx @@ -2,10 +2,16 @@ import { BPMNCanvasRef } from '@/components/bpmn-canvas'; import { Button, Card, Form, Input, List, Space, Tooltip } from 'antd'; import React, { useState } from 'react'; import { getNewShapePosition } from 'bpmn-js/lib/features/auto-place/BpmnAutoPlaceUtil'; -import { Shape } from 'bpmn-js/lib/model/Types'; +import { Element, Shape } from 'bpmn-js/lib/model/Types'; import { MessageOutlined } from '@ant-design/icons'; import ChatbotResponseModal, { ChatbotInteraction } from './bpmn-chatbot-response'; +/*For chatbots that have features like tool use or function calling. +Defines the modeler functionality through JSON Scheme (see: https://json-schema.org/).*/ +import tools from './bpmn-chatbot-tools.json'; + +import { Point } from 'bpmn-js/lib/features/modeling/BpmnFactory'; + type ChatbotDialogProps = { show: boolean; modeler: BPMNCanvasRef | null; @@ -26,9 +32,9 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { function onPrompt({ prompt }: FieldType) { setWaitForResponse(true); - getProcessXml() - .then((process) => { - fetchClaude(prompt, process).then((res) => { + getProcessXml().then((process) => { + fetchClaude(prompt, process) + .then((res) => { processResponse(res.content); setLastPrompts( lastPrompts.concat({ @@ -37,53 +43,106 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { chatbotResponse: res.content, }), ); - }); - }) - .finally(() => setWaitForResponse(false)); + }) + .finally(() => setWaitForResponse(false)); + }); + } + + //see tools definitions + function create_element( + bpmn_type: string, + created: Shape[], + source_element_id_or_name?: string, + new_element_name?: string, + connection_label?: string, + position?: Point, + ): void { + const element = elementFactory.createShape({ + type: 'bpmn:' + bpmn_type, + }); + let shape: Shape; + if (source_element_id_or_name) { + //append element to existing one + let sourceShape = modeler?.getElement(source_element_id_or_name); + if (!sourceShape) { + sourceShape = created.find((e) => e.name == source_element_id_or_name)!; + } + const point = getNewShapePosition(sourceShape as Shape, element); + shape = modeling.appendShape(sourceShape, element, point, root!, { + connection: { name: connection_label, type: 'bpmn:SequenceFlow' }, + }); + } else { + shape = modeling.createShape(element, position!, root!); + } + if (new_element_name) { + modeling.updateLabel(shape, new_element_name); + } + created.push(shape); + } + function create_connection( + source_element_id_or_name: string, + target_element_id_or_name: string, + created: Shape[], + label?: string, + ): void { + let sourceShape = modeler?.getElement(source_element_id_or_name); + if (!sourceShape) { + sourceShape = created.find((e) => e.name == source_element_id_or_name)!; + } + let targetShape = modeler?.getElement(target_element_id_or_name); + if (!targetShape) { + targetShape = created.find((e) => e.name == target_element_id_or_name)!; + } + const connection = modeling.createConnection( + sourceShape, + targetShape, + { type: 'bpmn:SequenceFlow' }, + root!, + ); + if (label) { + modeling.updateLabel(connection, label); + } + } + function remove_elements(element_ids: string[]): void { + const elements: Element[] = []; + element_ids.forEach((id) => { + let element = modeler?.getElement(id); + if (element) { + elements.push(element); + } + }); + modeling.removeElements(elements); } + //parsing tool uses listed in response function processResponse(response: any[]) { const created: Shape[] = []; response.forEach((res) => { if (res.type == 'tool_use') { - if (res.name == 'create_and_append_task') { - const task = elementFactory.createShape({ - type: 'bpmn:Task', - name: res.input.new_task_name, - }); - let sourceShape = modeler?.getElement(res.input.source_task_id_or_name) as Shape; - if (!sourceShape) { - sourceShape = created.find((e) => e.name == res.input.source_task_id_or_name)!; - } - const point = getNewShapePosition(sourceShape, task); - modeling.appendShape(sourceShape, task, point, root!); - created.push(task); - } else if (res.name == 'create_end') { - const endEvent = elementFactory.createShape({ - type: 'bpmn:EndEvent', - }); - let sourceShape = modeler?.getElement(res.input.element_id_or_name) as Shape; - if (!sourceShape) { - sourceShape = created.find((e) => e.name == res.input.element_id_or_name)!; - } - const point = getNewShapePosition(sourceShape, endEvent); - modeling.appendShape(sourceShape, endEvent, point, root!); - created.push(endEvent); - } else if (res.name == 'create_start') { - const startEvent = modeling.createShape( - { - type: 'bpmn:StartEvent', - name: res.input.name, - }, - { x: 350, y: 200 }, - root!, + if (res.name == 'create_element') { + create_element( + res.input.bpmn_type, + created, + res.input.source_element_id_or_name, + res.input.new_element_name, + res.input.connection_label, + res.input.position, + ); + } else if (res.name == 'create_connection') { + create_connection( + res.input.source_element_id_or_name, + res.input.target_element_id_or_name, + created, + res.input.label, ); - created.push(startEvent); + } else if (res.name == 'remove_elements') { + remove_elements(res.input.element_ids); } } }); } + //get current xml of the ... part only function getProcessXml(): Promise { return modeler!.getXML().then((res) => { if (res) { @@ -99,13 +158,14 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { }); } + //prompting an external service for the chatbot function fetchClaude(userPrompt: string, process: string): Promise { return fetch('http://localhost:2000/', { method: 'POST', headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ userPrompt: userPrompt, process: process }), + body: JSON.stringify({ userPrompt: userPrompt, process: process, tools: tools }), }).then((res) => res.json()); } From dece92eb3fbca1b888a2611df1b158a1e82d2bea Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Fri, 7 Jun 2024 16:24:47 +0200 Subject: [PATCH 08/12] added label functionality --- .../components/bpmn-chatbot-tools.json | 43 +++----- .../components/bpmn-chatbot.tsx | 98 +++++++++---------- 2 files changed, 61 insertions(+), 80 deletions(-) diff --git a/src/management-system-v2/components/bpmn-chatbot-tools.json b/src/management-system-v2/components/bpmn-chatbot-tools.json index 3ff7d9450..4a7cfde61 100644 --- a/src/management-system-v2/components/bpmn-chatbot-tools.json +++ b/src/management-system-v2/components/bpmn-chatbot-tools.json @@ -1,41 +1,28 @@ [ { - "name": "create_element", - "description": "Create a BPMN element. Place it at an arbitrary position or append it to another element.", + "name": "append_element", + "description": "Create a BPMN element and connect it to an already existing element. The element can either be a task, event or gateway. This automatically places the element and creates a connection from the given source element.", "input_schema": { "type": "object", "properties": { - "source_element_id_or_name": { + "bpmn_type": { "type": "string", - "description": "The id or name of the source element which to append to. Use the id if the element already exists in the process, otherwise the name. This creates automatically a connection between the new element and the provided source." + "description": "The BPMN type of the element. Possible types are: 'Task','StartEvent','EndEvent','ExclusiveGateway','InclusiveGateway','ParallelGateway'." }, - "new_element_name": { + "name": { "type": "string", "description": "The name of the new element." }, - "bpmn_type": { + "source_element_id_or_name": { "type": "string", - "description": "The BPMN type of the element. Possible types are: 'Task','StartEvent','EndEvent','ExclusiveGateway','InclusiveGateway','ParallelGateway'." + "description": "The id or name of the source element which to append to. Use the id if the element already exists in the process, otherwise the name." }, - "connection_label": { + "label": { "type": "string", - "description": "The label of the connection if the new element is appended to another one. This label is used when the source element is a gateway and the connection is taken conditionally. The label tells us what condition has to be met in order to take this connection." - }, - "position": { - "type": "object", - "description": "The position where the new element should be placed. Only needed when the element is not appended to another element. Do not provide this parameter if source_element_id_or_name is provided. Use for the first start event the position x=350 and y=200.", - "properties": { - "x": { - "type": "number" - }, - "y": { - "type": "number" - } - }, - "required": ["x", "y"] + "description": "The label of the connection which is automatically created. This is optional." } }, - "required": ["bpmn_type"] + "required": ["bpmn_type", "name", "source_element_id_or_name"] } }, { @@ -44,10 +31,6 @@ "input_schema": { "type": "object", "properties": { - "label": { - "type": "string", - "description": "The label of the connection. Only needed if the connection comes from a gateway and is only taken, when a certain condition is met." - }, "source_element_id_or_name": { "type": "string", "description": "The id or name of the source element of the connection. User the id if the element already exists in the process, otherwise the name." @@ -55,6 +38,10 @@ "target_element_id_or_name": { "type": "string", "description": "The id or name of the target element of the connection. User the id if the element already exists in the process, otherwise the name." + }, + "label": { + "type": "string", + "description": "The optional label of the connection." } }, "required": ["source_element_id_or_name", "target_element_id_or_name"] @@ -62,7 +49,7 @@ }, { "name": "remove_elements", - "description": "Remove BPMN elements. Removing an element also removes all associated connections. Connections are also considered elements and can be seperately removed too by including their id here.", + "description": "Remove BPMN elements. Removing an element with exactly one incomming and one outgoing connection merges the two connections into one. If the deleted element has more than one incomming or outgoing connection all associated connections will be removed. Connections are also considered elements and can be seperately removed too by including their id here.", "input_schema": { "type": "object", "properties": { diff --git a/src/management-system-v2/components/bpmn-chatbot.tsx b/src/management-system-v2/components/bpmn-chatbot.tsx index 884425dc7..5b68c47c1 100644 --- a/src/management-system-v2/components/bpmn-chatbot.tsx +++ b/src/management-system-v2/components/bpmn-chatbot.tsx @@ -10,8 +10,6 @@ import ChatbotResponseModal, { ChatbotInteraction } from './bpmn-chatbot-respons Defines the modeler functionality through JSON Scheme (see: https://json-schema.org/).*/ import tools from './bpmn-chatbot-tools.json'; -import { Point } from 'bpmn-js/lib/features/modeling/BpmnFactory'; - type ChatbotDialogProps = { show: boolean; modeler: BPMNCanvasRef | null; @@ -24,9 +22,9 @@ type FieldType = { const ChatbotDialog: React.FC = ({ show, modeler }) => { const [lastPrompts, setLastPrompts] = useState([]); const [waitForResponse, setWaitForResponse] = useState(false); - const elementFactory = modeler!.getElementFactory(); const root = modeler!.getCurrentRoot(); const modeling = modeler!.getModeling(); + const elementFactory = modeler!.getElementFactory(); const [showChatbotResponseModal, setShowChatbotResponseModal] = useState(false); const [chatbotInteraction, setChatbotInteraction] = useState(); @@ -49,53 +47,48 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { } //see tools definitions - function create_element( + function append_shape( bpmn_type: string, - created: Shape[], - source_element_id_or_name?: string, - new_element_name?: string, - connection_label?: string, - position?: Point, - ): void { - const element = elementFactory.createShape({ - type: 'bpmn:' + bpmn_type, - }); - let shape: Shape; - if (source_element_id_or_name) { - //append element to existing one - let sourceShape = modeler?.getElement(source_element_id_or_name); - if (!sourceShape) { - sourceShape = created.find((e) => e.name == source_element_id_or_name)!; - } - const point = getNewShapePosition(sourceShape as Shape, element); - shape = modeling.appendShape(sourceShape, element, point, root!, { - connection: { name: connection_label, type: 'bpmn:SequenceFlow' }, - }); - } else { - shape = modeling.createShape(element, position!, root!); - } - if (new_element_name) { - modeling.updateLabel(shape, new_element_name); + new_element_name: string, + source_element_id_or_name: string, + created: { name: string; shape: Shape }[], + label: string, + ): Shape { + let source = modeler?.getElement(source_element_id_or_name) as Shape; + if (!source) { + console.log(created); + source = created.find((e) => e.name == source_element_id_or_name)!.shape; } - created.push(shape); + let shape = elementFactory.createShape({ type: 'bpmn:' + bpmn_type }); + const position = getNewShapePosition(source, shape); + shape = modeling.createShape({ type: 'bpmn:' + bpmn_type }, position, root!); + const connection = modeling.createConnection( + source, + shape, + { type: 'bpmn:SequenceFlow' }, + root!, + ); + modeling.updateLabel(connection, label); + modeling.updateLabel(shape, new_element_name); + return shape; } function create_connection( source_element_id_or_name: string, target_element_id_or_name: string, - created: Shape[], + created: { name: string; shape: Shape }[], label?: string, ): void { - let sourceShape = modeler?.getElement(source_element_id_or_name); - if (!sourceShape) { - sourceShape = created.find((e) => e.name == source_element_id_or_name)!; + let source = modeler?.getElement(source_element_id_or_name) as Shape; + if (!source) { + source = created.find((e) => e.name == source_element_id_or_name)!.shape; } - let targetShape = modeler?.getElement(target_element_id_or_name); - if (!targetShape) { - targetShape = created.find((e) => e.name == target_element_id_or_name)!; + let target = modeler?.getElement(target_element_id_or_name) as Shape; + if (!target) { + target = created.find((e) => e.name == target_element_id_or_name)!.shape; } const connection = modeling.createConnection( - sourceShape, - targetShape, + source, + target, { type: 'bpmn:SequenceFlow' }, root!, ); @@ -105,30 +98,22 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { } function remove_elements(element_ids: string[]): void { const elements: Element[] = []; - element_ids.forEach((id) => { - let element = modeler?.getElement(id); + element_ids.forEach((e) => { + const element = modeler?.getElement(e); if (element) { elements.push(element); } }); + modeling.removeElements(elements); } //parsing tool uses listed in response function processResponse(response: any[]) { - const created: Shape[] = []; + const created: { name: string; shape: Shape }[] = []; response.forEach((res) => { if (res.type == 'tool_use') { - if (res.name == 'create_element') { - create_element( - res.input.bpmn_type, - created, - res.input.source_element_id_or_name, - res.input.new_element_name, - res.input.connection_label, - res.input.position, - ); - } else if (res.name == 'create_connection') { + if (res.name == 'create_connection') { create_connection( res.input.source_element_id_or_name, res.input.target_element_id_or_name, @@ -137,6 +122,15 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { ); } else if (res.name == 'remove_elements') { remove_elements(res.input.element_ids); + } else if (res.name == 'append_element') { + const shape = append_shape( + res.input.bpmn_type, + res.input.name, + res.input.source_element_id_or_name, + created, + res.input.label, + ); + created.push({ name: res.input.name, shape: shape }); } } }); From 3f401e2810b1d5cba3fd5aa90e4c6ac9c05d3f42 Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Sun, 29 Sep 2024 13:49:08 +0200 Subject: [PATCH 09/12] improve tool definitions change tool definitions and result processing --- .../components/bpmn-chatbot-tools.json | 14 +++--- .../components/bpmn-chatbot.tsx | 44 +++++++++---------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/management-system-v2/components/bpmn-chatbot-tools.json b/src/management-system-v2/components/bpmn-chatbot-tools.json index 4a7cfde61..8ebd1bb53 100644 --- a/src/management-system-v2/components/bpmn-chatbot-tools.json +++ b/src/management-system-v2/components/bpmn-chatbot-tools.json @@ -1,9 +1,9 @@ [ { "name": "append_element", - "description": "Create a BPMN element and connect it to an already existing element. The element can either be a task, event or gateway. This automatically places the element and creates a connection from the given source element.", - "input_schema": { + "parameters": { "type": "object", + "description": "Create a BPMN element and connect it to an already existing element. The element can either be a task, event or gateway. This automatically places the element and creates a connection from the given source element.", "properties": { "bpmn_type": { "type": "string", @@ -19,7 +19,7 @@ }, "label": { "type": "string", - "description": "The label of the connection which is automatically created. This is optional." + "description": "The label of the connection which is automatically created." } }, "required": ["bpmn_type", "name", "source_element_id_or_name"] @@ -27,9 +27,9 @@ }, { "name": "create_connection", - "description": "Create a connection between two elements. Only needed if the connection has not been created yet.", - "input_schema": { + "parameters": { "type": "object", + "description": "Create a connection between two elements. Only needed if the connection has not been created yet.", "properties": { "source_element_id_or_name": { "type": "string", @@ -49,9 +49,9 @@ }, { "name": "remove_elements", - "description": "Remove BPMN elements. Removing an element with exactly one incomming and one outgoing connection merges the two connections into one. If the deleted element has more than one incomming or outgoing connection all associated connections will be removed. Connections are also considered elements and can be seperately removed too by including their id here.", - "input_schema": { + "parameters": { "type": "object", + "description": "Remove BPMN elements. Removing an element with exactly one incomming and one outgoing connection merges the two connections into one. If the deleted element has more than one incomming or outgoing connection all associated connections will be removed. Connections are also considered elements and can be seperately removed too by including their id here.", "properties": { "element_ids": { "type": "array", diff --git a/src/management-system-v2/components/bpmn-chatbot.tsx b/src/management-system-v2/components/bpmn-chatbot.tsx index 5b68c47c1..caeccd44b 100644 --- a/src/management-system-v2/components/bpmn-chatbot.tsx +++ b/src/management-system-v2/components/bpmn-chatbot.tsx @@ -31,9 +31,9 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { function onPrompt({ prompt }: FieldType) { setWaitForResponse(true); getProcessXml().then((process) => { - fetchClaude(prompt, process) + fetchChatbot(prompt, process) .then((res) => { - processResponse(res.content); + processResponse(res); setLastPrompts( lastPrompts.concat({ userPrompt: prompt, @@ -112,26 +112,24 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { function processResponse(response: any[]) { const created: { name: string; shape: Shape }[] = []; response.forEach((res) => { - if (res.type == 'tool_use') { - if (res.name == 'create_connection') { - create_connection( - res.input.source_element_id_or_name, - res.input.target_element_id_or_name, - created, - res.input.label, - ); - } else if (res.name == 'remove_elements') { - remove_elements(res.input.element_ids); - } else if (res.name == 'append_element') { - const shape = append_shape( - res.input.bpmn_type, - res.input.name, - res.input.source_element_id_or_name, - created, - res.input.label, - ); - created.push({ name: res.input.name, shape: shape }); - } + if (res.name == 'create_connection') { + create_connection( + res.args.source_element_id_or_name, + res.args.target_element_id_or_name, + created, + res.args.label, + ); + } else if (res.name == 'remove_elements') { + remove_elements(res.args.element_ids); + } else if (res.name == 'append_element') { + const shape = append_shape( + res.args.bpmn_type, + res.args.name, + res.args.source_element_id_or_name, + created, + res.args.label, + ); + created.push({ name: res.args.name, shape: shape }); } }); } @@ -153,7 +151,7 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { } //prompting an external service for the chatbot - function fetchClaude(userPrompt: string, process: string): Promise { + function fetchChatbot(userPrompt: string, process: string): Promise { return fetch('http://localhost:2000/', { method: 'POST', headers: { From 285faa28d23341e78cc56939b9b3d6a3b2ac5993 Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Sun, 20 Oct 2024 13:44:40 +0200 Subject: [PATCH 10/12] Update bpmn-chatbot-tools.json --- src/management-system-v2/components/bpmn-chatbot-tools.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/management-system-v2/components/bpmn-chatbot-tools.json b/src/management-system-v2/components/bpmn-chatbot-tools.json index 8ebd1bb53..808722357 100644 --- a/src/management-system-v2/components/bpmn-chatbot-tools.json +++ b/src/management-system-v2/components/bpmn-chatbot-tools.json @@ -1,9 +1,9 @@ [ { "name": "append_element", + "description": "Create a BPMN element and connect it to an already existing element. The element can either be a task, event or gateway. This automatically places the element and creates a connection from the given source element.", "parameters": { "type": "object", - "description": "Create a BPMN element and connect it to an already existing element. The element can either be a task, event or gateway. This automatically places the element and creates a connection from the given source element.", "properties": { "bpmn_type": { "type": "string", @@ -27,9 +27,9 @@ }, { "name": "create_connection", + "description": "Create a connection between two elements. Only needed if the connection has not been created yet.", "parameters": { "type": "object", - "description": "Create a connection between two elements. Only needed if the connection has not been created yet.", "properties": { "source_element_id_or_name": { "type": "string", @@ -49,9 +49,9 @@ }, { "name": "remove_elements", + "description": "Remove BPMN elements. Removing an element with exactly one incomming and one outgoing connection merges the two connections into one. If the deleted element has more than one incomming or outgoing connection all associated connections will be removed. Connections are also considered elements and can be seperately removed too by including their id here.", "parameters": { "type": "object", - "description": "Remove BPMN elements. Removing an element with exactly one incomming and one outgoing connection merges the two connections into one. If the deleted element has more than one incomming or outgoing connection all associated connections will be removed. Connections are also considered elements and can be seperately removed too by including their id here.", "properties": { "element_ids": { "type": "array", From afa59b5172ad2558209f7215c788fba96119cedc Mon Sep 17 00:00:00 2001 From: bowserjuno <118111794+bowserjuno@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:11:40 +0200 Subject: [PATCH 11/12] Gemini API integration --- .../components/bpmn-chatbot-response.tsx | 3 +- .../components/bpmn-chatbot.tsx | 39 ++++------ .../lib/bpmn-chatbot/bpmn-chatbot-tools.ts | 76 +++++++++++++++++++ .../bpmnChatbotAPIcommunication.ts | 18 +++++ src/management-system-v2/package.json | 21 ++--- yarn.lock | 45 +++-------- 6 files changed, 129 insertions(+), 73 deletions(-) create mode 100644 src/management-system-v2/lib/bpmn-chatbot/bpmn-chatbot-tools.ts create mode 100644 src/management-system-v2/lib/bpmn-chatbot/bpmnChatbotAPIcommunication.ts diff --git a/src/management-system-v2/components/bpmn-chatbot-response.tsx b/src/management-system-v2/components/bpmn-chatbot-response.tsx index 1e441e04c..05d55c9b9 100644 --- a/src/management-system-v2/components/bpmn-chatbot-response.tsx +++ b/src/management-system-v2/components/bpmn-chatbot-response.tsx @@ -1,3 +1,4 @@ +import { FunctionCall } from '@google/generative-ai'; import { Modal } from 'antd'; import Title from 'antd/es/typography/Title'; import React from 'react'; @@ -11,7 +12,7 @@ type ChatbotResponseModalProps = { export type ChatbotInteraction = { userPrompt: string; bpmnProcess: string; - chatbotResponse: any[]; + chatbotResponse: FunctionCall[] | undefined; }; const ChatbotResponseModal: React.FC = ({ diff --git a/src/management-system-v2/components/bpmn-chatbot.tsx b/src/management-system-v2/components/bpmn-chatbot.tsx index caeccd44b..39d3faab9 100644 --- a/src/management-system-v2/components/bpmn-chatbot.tsx +++ b/src/management-system-v2/components/bpmn-chatbot.tsx @@ -5,10 +5,7 @@ import { getNewShapePosition } from 'bpmn-js/lib/features/auto-place/BpmnAutoPla import { Element, Shape } from 'bpmn-js/lib/model/Types'; import { MessageOutlined } from '@ant-design/icons'; import ChatbotResponseModal, { ChatbotInteraction } from './bpmn-chatbot-response'; - -/*For chatbots that have features like tool use or function calling. -Defines the modeler functionality through JSON Scheme (see: https://json-schema.org/).*/ -import tools from './bpmn-chatbot-tools.json'; +import { sendToAPI } from '@/lib/bpmn-chatbot/bpmnChatbotAPIcommunication'; type ChatbotDialogProps = { show: boolean; @@ -31,15 +28,17 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { function onPrompt({ prompt }: FieldType) { setWaitForResponse(true); getProcessXml().then((process) => { - fetchChatbot(prompt, process) + sendToAPI(prompt, process) .then((res) => { - processResponse(res); + if (res) { + processResponse(res); + } setLastPrompts( lastPrompts.concat({ userPrompt: prompt, bpmnProcess: process, - chatbotResponse: res.content, - }), + chatbotResponse: res, + }) ); }) .finally(() => setWaitForResponse(false)); @@ -52,7 +51,7 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { new_element_name: string, source_element_id_or_name: string, created: { name: string; shape: Shape }[], - label: string, + label: string ): Shape { let source = modeler?.getElement(source_element_id_or_name) as Shape; if (!source) { @@ -66,7 +65,7 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { source, shape, { type: 'bpmn:SequenceFlow' }, - root!, + root! ); modeling.updateLabel(connection, label); modeling.updateLabel(shape, new_element_name); @@ -76,7 +75,7 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { source_element_id_or_name: string, target_element_id_or_name: string, created: { name: string; shape: Shape }[], - label?: string, + label?: string ): void { let source = modeler?.getElement(source_element_id_or_name) as Shape; if (!source) { @@ -90,7 +89,7 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { source, target, { type: 'bpmn:SequenceFlow' }, - root!, + root! ); if (label) { modeling.updateLabel(connection, label); @@ -117,7 +116,7 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { res.args.source_element_id_or_name, res.args.target_element_id_or_name, created, - res.args.label, + res.args.label ); } else if (res.name == 'remove_elements') { remove_elements(res.args.element_ids); @@ -127,7 +126,7 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { res.args.name, res.args.source_element_id_or_name, created, - res.args.label, + res.args.label ); created.push({ name: res.args.name, shape: shape }); } @@ -150,17 +149,6 @@ const ChatbotDialog: React.FC = ({ show, modeler }) => { }); } - //prompting an external service for the chatbot - function fetchChatbot(userPrompt: string, process: string): Promise { - return fetch('http://localhost:2000/', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ userPrompt: userPrompt, process: process, tools: tools }), - }).then((res) => res.json()); - } - return ( <>