feat: support multimodal chat

lib/Listener/BeforeTemplateRenderedListener.php

@@ -72,6 +72,8 @@ public function handle(Event $event): void {
 			$this->initialStateService->provideInitialState('contextChatIndexingComplete', $indexingComplete);
 			$this->initialStateService->provideInitialState('contextAgentToolSources', $this->assistantService->informationSources);
 			$this->initialStateService->provideInitialState('audio_chat_available', $this->assistantService->isAudioChatAvailable());
+			$multimodalChatAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypes());
+			$this->initialStateService->provideInitialState('multimodal_chat_available', $multimodalChatAvailable);
 			$autoplayAudioChat = $this->config->getUserValue($this->userId, Application::APP_ID, 'autoplay_audio_chat', '1') === '1';
 			$this->initialStateService->provideInitialState('autoplay_audio_chat', $autoplayAudioChat);
 			$agencyAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes());

lib/Listener/ChattyLLMTaskListener.php

@@ -82,6 +82,10 @@ public function handle(Event $event): void {
 				&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\AudioToAudioChat::ID;
 			$isAgencyAudioChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentAudioInteraction')
 				&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID;
+			$isMultimodalChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools')
+				&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID;
+			$isMultimodalAgencyChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction')
+				&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID;
 
 			$taskOutput = $task->getOutput();
 
@@ -128,6 +132,16 @@ public function handle(Event $event): void {
 				// the task is not an audio one, but we might still need to Tts the answer
 				// if it is a response to a ContextAgentInteraction confirmation that was asked about an audio message
 				$this->runTtsIfNeeded($sessionId, $message, $taskTypeId, $task->getUserId());
+				if ($isMultimodalChat) {
+					$attachments = $taskOutput['output_attachments'] ?? [];
+					$attachments = array_map(function ($attachment) {
+						return [
+							'type' => 'File',
+							'file_id' => $attachment,
+						];
+					}, $attachments);
+					$message->setAttachments(json_encode($attachments));
+				}
 			}
 			try {
 				$this->messageMapper->insert($message);
@@ -138,7 +152,7 @@ public function handle(Event $event): void {
 			$session = $this->sessionMapper->getUserSession($task->getUserId(), $sessionId);
 
 			// store the conversation token and the actions if we are using the agency feature
-			if ($isAgency || $isAgencyAudioChat) {
+			if ($isAgency || $isAgencyAudioChat || $isMultimodalAgencyChat) {
 				$conversationToken = ($taskOutput['conversation_token'] ?? null) ?: null;
 				$pendingActions = ($taskOutput['actions'] ?? null) ?: null;
 				$session->setAgencyConversationToken($conversationToken);

lib/Service/AssistantService.php

@@ -784,7 +784,7 @@ private function extractFileIdsFromTask(Task $task): array {
 					/** @var int|list<int> $outputSlot */
 					$outputSlot = $task->getOutput()[$key];
 					if (is_array($outputSlot)) {
-						$ids += $outputSlot;
+						$ids = array_merge($ids, $outputSlot);
 					} else {
 						$ids[] = $outputSlot;
 					}

lib/Service/ChatService.php

@@ -40,6 +40,7 @@ public function __construct(
 		private readonly IManager $taskProcessingManager,
 		private readonly LoggerInterface $logger,
 		private readonly ITimeFactory $timeFactory,
+		private readonly AssistantService $assistantService,
 	) {
 	}
 
@@ -431,14 +432,45 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $
 				$fileId = $audioAttachment['file_id'];
 				$taskId = $this->scheduleAudioChatTask($userId, $fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId());
 			} else {
-				// for a text chat task, let's only use text in the history
-				$history = array_map(static function (Message $message) {
-					return json_encode([
-						'role' => $message->getRole(),
-						'content' => $message->getContent(),
-					]);
-				}, $history);
-				$taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $history, $sessionId);
+				if ($this->isMultimodalChatAvailable()) {
+					// for a multimodal chat also attachments need to be added to the history
+					$assistantService = $this->assistantService;
+					$historyMessages = array_map(static function (Message $message) use ($userId, $assistantService) {
+						$attachments = $message->jsonSerialize()['attachments'];
+						// Attachments that were generated need to be saved in the user's files so they are accessible to provider
+						$content = array_map(static function (array $attachment) use ($userId, $assistantService, $message) {
+							if ($message->getRole() === Message::ROLE_ASSISTANT) {
+								$info = $assistantService->saveOutputFile($userId, $message->getOcpTaskId(), $attachment['file_id']);
+								return [
+									'type' => 'file',
+									'file_id' => $info['fileId'],
+								];
+							}
+							return ['type' => 'file', 'file_id' => $attachment['file_id']];
+						}, $attachments);
+						$content[] = [
+							'type' => 'text',
+							'text' => $message->getContent(),
+						];
+						return json_encode([
+							'role' => $message->getRole(),
+							'content' => $content,
+						]);
+					}, $history);
+					$lastAttachments = array_map(static function (array $attachment) {
+						return $attachment['file_id'];
+					}, $lastAttachments);
+					$taskId = $this->scheduleMultimodalChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId, $lastAttachments);
+				} else {
+					// for a text chat task, let's only use text in the history
+					$historyMessages = array_map(static function (Message $message) {
+						return json_encode([
+							'role' => $message->getRole(),
+							'content' => $message->getContent(),
+						]);
+					}, $history);
+					$taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId);
+				}
 			}
 		}
 		return $taskId;
@@ -552,6 +584,14 @@ public function isContextAgentAudioAvailable(): bool {
 		return in_array(\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
 	}
 
+	public function isMultimodalChatAvailable(): bool {
+		if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools')) {
+			return false;
+		}
+		return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
+	}
+
+
 	private function getAudioHistory(array $history): array {
 		// history is a list of JSON strings
 		// The content is the remote audio ID (or the transcription as fallback)
@@ -667,6 +707,49 @@ private function scheduleLLMChatTask(
 		return $task->getId() ?? 0;
 	}
 
+	/**
+	 * Schedule a Multimodal Chat task
+	 *
+	 * @throws BadRequestException
+	 * @throws InternalException
+	 */
+	private function scheduleMultimodalChatTask(
+		?string $userId,
+		string $content,
+		string $systemPrompt,
+		array $history,
+		int $sessionId,
+		array $attachmentsHistory,
+	): int {
+		$customId = 'chatty-llm:' . $sessionId;
+		$this->checkIfSessionIsThinking($userId, $customId);
+		$input = [
+			'input' => $content,
+			'system_prompt' => $systemPrompt,
+			'history' => $history,
+			'input_attachments' => $attachmentsHistory,
+			'tools' => '[]', // Empty tools as there is not a non tools version
+			'tool_message' => '',
+		];
+		/** @psalm-suppress UndefinedClass */
+		$task = new Task(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $input, Application::APP_ID . ':chatty-llm', $userId, $customId);
+		/** @psalm-suppress UndefinedMethod */
+		$task->setPreferStreaming(true);
+		try {
+			$this->taskProcessingManager->scheduleTask($task);
+		} catch (PreConditionNotMetException $e) {
+			throw new BadRequestException('pre_condition_not_met', previous: $e);
+		} catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) {
+			throw new BadRequestException('unauthorized', previous: $e);
+		} catch (ValidationException $e) {
+			throw new BadRequestException('validation_failed', previous: $e);
+		} catch (\OCP\TaskProcessing\Exception\Exception $e) {
+			$this->logger->error($e->getMessage(), ['exception' => $e]);
+			throw new InternalException(previous: $e);
+		}
+		return $task->getId() ?? 0;
+	}
+
 	/**
 	 * Schedule an agency chat task
 	 *

src/components/ChattyLLM/ChattyLLMInputForm.vue

@@ -665,16 +665,20 @@ export default {
 			return session.timestamp ? (' ' + moment(session.timestamp * 1000).format('LLL')) : t('assistant', 'Untitled conversation')
 		},
 
-		async handleSubmit(event) {
+		async handleSubmit(attachedFileIds) {
 			if (this.chatContent.trim() === '') {
 				console.debug('empty message')
 				return
 			}
+			const attachments = attachedFileIds.map(fileId => ({ type: SHAPE_TYPE_NAMES.File, file_id: fileId }))
 
 			const role = Roles.HUMAN
 			const content = this.chatContent.trim()
 			const timestamp = +new Date() / 1000 | 0
 			console.debug('[Assistant] submit text', content)
+			if (attachedFileIds.length > 0) {
+				console.debug('[Assistant] submit attachments', attachments)
+			}
 
 			if (this.active === null) {
 				await this.newSession()
@@ -686,10 +690,10 @@ export default {
 				this.active.agencyAnswered = true
 			}
 
-			this.messages.push({ role, content, timestamp, session_id: this.active.id })
+			this.messages.push({ role, content, timestamp, session_id: this.active.id, attachments })
 			this.chatContent = ''
 			this.scrollToLastMessage()
-			await this.newMessage(role, content, timestamp, this.active.id)
+			await this.newMessage(role, content, timestamp, this.active.id, attachments)
 		},
 
 		async handleSubmitAudio(fileId) {

src/components/ChattyLLM/DocumentPreviews.vue

@@ -0,0 +1,82 @@
+<!--
+  - SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+  - SPDX-License-Identifier: AGPL-3.0-or-later
+-->
+<template>
+	<div class="document-previews">
+		<div class="document-previews__list">
+			<div v-for="fileId in fileIds"
+				:key="fileId"
+				class="document-previews__item">
+				<FileDisplay :file-id="fileId"
+					:task-id="null" />
+				<NcButton class="document-previews__delete"
+					variant="tertiary"
+					:aria-label="t('assistant', 'Remove this media')"
+					@click="$emit('delete', fileId)">
+					<template #icon>
+						<TrashCanOutlineIcon :size="20" />
+					</template>
+				</NcButton>
+			</div>
+		</div>
+	</div>
+</template>
+
+<script>
+import TrashCanOutlineIcon from 'vue-material-design-icons/TrashCanOutline.vue'
+
+import NcButton from '@nextcloud/vue/components/NcButton'
+
+import FileDisplay from '../fields/FileDisplay.vue'
+
+export default {
+	name: 'DocumentPreviews',
+
+	components: {
+		FileDisplay,
+		NcButton,
+		TrashCanOutlineIcon,
+	},
+
+	props: {
+		fileIds: {
+			type: Array,
+			required: true,
+		},
+	},
+
+	emits: [
+		'delete',
+	],
+}
+</script>
+
+<style lang="scss" scoped>
+.document-previews {
+	overflow-x: auto;
+
+	&__list {
+		display: flex;
+		flex-direction: row;
+		gap: 8px;
+	}
+
+	&__item {
+		position: relative;
+		padding: 8px;
+		border-radius: var(--border-radius-large);
+		background-color: var(--color-main-background);
+
+		&:hover {
+			background-color: var(--color-primary-element-light-hover);
+		}
+	}
+
+	&__delete {
+		position: absolute;
+		right: 0;
+		bottom: 0;
+	}
+}
+</style>