feat(Watermarking): Add metadata and watermarks to generated images

.github/workflows/lint-php-cs.yml

@@ -53,7 +53,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install dependencies
-        run: composer i
+        run: |
+          composer i
 
       - name: Lint
         run: composer run cs:check || ( echo 'Please run `composer run cs:fix` to format your code' && exit 1 )

.github/workflows/phpunit.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         php-versions: ['8.2']
         databases: ['sqlite']
-        server-versions: ['master', 'stable32', 'stable31', 'stable30']
+        server-versions: ['master']
 
     name: php${{ matrix.php-versions }}-${{ matrix.databases }}-${{ matrix.server-versions }}
 
@@ -61,7 +61,8 @@ jobs:
 
       - name: Set up PHPUnit
         working-directory: apps/${{ env.APP_NAME }}
-        run: composer i
+        run: |
+          composer i
 
       - name: Set up Nextcloud
         run: |

.github/workflows/psalm.yml

@@ -51,7 +51,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install dependencies
-        run: composer i
+        run: |
+          composer i
 
       - name: Run coding standards check
         run: composer run psalm

appinfo/info.xml

@@ -30,7 +30,7 @@ The rating depends on the model you select to use.
 Learn more about the Nextcloud Ethical AI Rating [in our blog](https://nextcloud.com/blog/nextcloud-ethical-ai-rating/).
 
     ]]></description>
-    <version>4.2.0</version>
+    <version>5.0.0-dev</version>
     <licence>agpl</licence>
     <author>Julien Veyssier</author>
     <namespace>Replicate</namespace>
@@ -46,7 +46,7 @@ Learn more about the Nextcloud Ethical AI Rating [in our blog](https://nextcloud
     <screenshot>https://github.com/julien-nc/integration_replicate/raw/main/img/screenshot2.jpg</screenshot>
     <screenshot>https://github.com/julien-nc/integration_replicate/raw/main/img/screenshot3.jpg</screenshot>
     <dependencies>
-        <nextcloud min-version="30" max-version="33"/>
+        <nextcloud min-version="33" max-version="33"/>
     </dependencies>
     <settings>
         <admin>OCA\Replicate\Settings\Admin</admin>

composer.json

@@ -10,21 +10,46 @@
 		"lint": "find . -name \\*.php -not -path './vendor/*' -print0 | xargs -0 -n1 php -l",
 		"cs:check": "php-cs-fixer fix --dry-run --diff",
 		"cs:fix": "php-cs-fixer fix",
-		"psalm": "psalm.phar --no-cache",
-		"test:unit": "phpunit --config tests/phpunit.xml"
+		"psalm": "psalm --no-cache",
+		"psalm:update-baseline": "psalm --threads=1 --update-baseline",
+		"psalm:update-baseline:force": "psalm --threads=1 --update-baseline --set-baseline=psalm-baseline.xml",
+		"test:unit": "phpunit --config tests/phpunit.xml",
+		"post-install-cmd": [
+			"@composer bin all update --ansi",
+			"grep -r 'OCA\\\\Replicate\\\\Vendor' ./vendor || vendor/bin/php-scoper add-prefix --prefix='OCA\\Replicate\\Vendor' --output-dir=\".\" --working-dir=\"./vendor/\" -f --config=\"../scoper.inc.php\"",
+			"composer dump-autoload",
+			"composer install --no-scripts"
+		],
+		"post-update-cmd": [
+			"@composer bin all update --ansi",
+			"grep -r 'OCA\\\\Replicate\\\\Vendor' ./vendor || vendor/bin/php-scoper add-prefix --prefix='OCA\\Replicate\\Vendor' --output-dir=\".\" --working-dir=\"./vendor/\" -f --config=\"../scoper.inc.php\"",
+			"composer dump-autoload",
+			"composer update --no-scripts"
+		]
 	},
 	"require": {
-		"php": "^8.1"
+		"php": ">= 8.2",
+		"bamarni/composer-bin-plugin": "^1.8",
+		"fileeye/pel": "^0.12.0",
+		"james-heinrich/getid3": "^1.9"
 	},
 	"require-dev": {
-		"nextcloud/coding-standard": "^1.1",
-		"psalm/phar": "6.*",
-		"nextcloud/ocp": "dev-master",
-		"phpunit/phpunit": "^9.5"
+		"nextcloud/ocp": "dev-master"
 	},
 	"config": {
+		"allow-plugins": {
+			"bamarni/composer-bin-plugin": true,
+			"composer/package-versions-deprecated": true
+		},
 		"platform": {
-			"php": "8.1.0"
+			"php": "8.2"
+		}
+	},
+	"extra": {
+		"bamarni-bin": {
+			"bin-links": true,
+			"target-directory": "vendor-bin",
+			"forward-command": true
 		}
 	}
 }

lib/AppInfo/Application.php

@@ -44,9 +44,23 @@ public function register(IRegistrationContext $context): void {
 			$context->registerSpeechToTextProvider(STTProvider::class);
 			$context->registerTextToImageProvider(TextToImageProvider::class);
 			$context->registerTextProcessingProvider(FreePromptProvider::class);
+			$context->registerTaskProcessingProvider(\OCA\Replicate\TaskProcessing\TextToImageProvider::class);
+			$context->registerTaskProcessingProvider(\OCA\Replicate\TaskProcessing\SpeechToTextProvider::class);
 		}
 	}
 
 	public function boot(IBootContext $context): void {
+		// Load PHP Exif Library for adding image metadata
+		\spl_autoload_register(function ($class) {
+			if (\substr_compare($class, 'OCA\OpenAi\Vendor\lsolesen\\pel\\', 0, 13) === 0) {
+				$classname = \str_replace('OCA\\OpenAi\\Vendor\\lsolesen\\pel\\', '', $class);
+				$load = \realpath(__DIR__ . '/../../vendor/fileeye/pel/src/' . $classname . '.php');
+				if ($load !== \false) {
+					include_once \realpath($load);
+				}
+			}
+		});
+		// Load getID3 library for adding audio metadata
+		require_once(__DIR__ . '/../../vendor/james-heinrich/getid3/getid3/getid3.php');
 	}
 }

lib/Service/ReplicateAPIService.php

@@ -17,7 +17,6 @@
 use GuzzleHttp\Exception\ServerException;
 use OCA\Replicate\AppInfo\Application;
 use OCP\Exceptions\AppConfigTypeConflictException;
-use OCP\Files\File;
 use OCP\Files\GenericFileException;
 use OCP\Files\NotPermittedException;
 use OCP\Http\Client\IClient;
@@ -65,15 +64,15 @@ public function createWhisperPrediction(string $audioFileContent, bool $translat
 	/**
 	 * Create a prediction and wait for it to complete to return the text result
 	 *
-	 * @param File $file
+	 * @param string $audio
 	 * @param bool $translate
 	 * @return string
 	 * @throws GenericFileException
 	 * @throws LockedException
 	 * @throws NotPermittedException
 	 */
-	public function transcribeFile(File $file, bool $translate = false): string {
-		$prediction = $this->createWhisperPrediction($file->getContent(), $translate);
+	public function transcribeFile(string $audio, bool $translate = false): string {
+		$prediction = $this->createWhisperPrediction($audio, $translate);
 		if (isset($prediction['id'])) {
 			$predictionId = $prediction['id'];
 
@@ -83,17 +82,17 @@ public function transcribeFile(File $file, bool $translate = false): string {
 						? $prediction['output']['translation']
 						: $prediction['output']['transcription'];
 				} elseif ($prediction['status'] === 'failed') {
-					throw new Exception('Error transcribing file "' . $file->getName() . '": remote job failed');
+					throw new Exception('Error transcribing file: remote job failed');
 				} elseif ($prediction['status'] === 'canceled') {
-					throw new Exception('Error transcribing file "' . $file->getName() . '": remote job was canceled');
+					throw new Exception('Error transcribing file: remote job was canceled');
 				} elseif ($prediction['status'] !== 'starting' && $prediction['status'] !== 'processing') {
-					throw new Exception('Error transcribing file "' . $file->getName() . '": unknown prediction status');
+					throw new Exception('Error transcribing file: unknown prediction status');
 				}
 				sleep(2);
 				$prediction = $this->getPrediction($predictionId);
 			}
 		}
-		throw new Exception('Error transcribing file "' . $file->getName() . '"');
+		throw new Exception('Error transcribing file');
 	}
 
 	/**

lib/Service/WatermarkingService.php

@@ -0,0 +1,130 @@
+<?php
+
+/**
+ * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\OpenAi\Service;
+
+use OCA\OpenAi\Vendor\getid3_lib;
+use OCA\OpenAi\Vendor\getid3_writetags;
+use OCA\OpenAi\Vendor\lsolesen\pel\PelEntryUndefined;
+use OCA\OpenAi\Vendor\lsolesen\pel\PelExif;
+use OCA\OpenAi\Vendor\lsolesen\pel\PelIfd;
+use OCA\OpenAi\Vendor\lsolesen\pel\PelJpeg;
+use OCA\OpenAi\Vendor\lsolesen\pel\PelTag;
+use OCA\OpenAi\Vendor\lsolesen\pel\PelTiff;
+use OCP\ITempManager;
+use Psr\Log\LoggerInterface;
+use const OCA\OpenAi\Vendor\GETID3_INCLUDEPATH;
+
+class WatermarkingService {
+	public const COMMENT = 'Generated with Artificial Intelligence';
+	public function __construct(
+		private ITempManager $tempManager,
+		private LoggerInterface $logger,
+	) {
+	}
+
+	public function markImage(string $image): string {
+		try {
+			$text = self::COMMENT;
+
+			$img = imagecreatefromstring($image);
+			$font = 2;// built-in font 1-5
+			$white = imagecolorallocate($img, 255, 255, 255);
+			$black = imagecolorallocate($img, 0, 0, 0);
+
+			$w = imagefontwidth($font) * strlen($text);
+			$h = imagefontheight($font);
+			$px = imagesx($img) - $w - 10;
+			$py = imagesy($img) - $h - 10;
+
+			// draw 1-pixel black outline by offsetting in 4 directions
+			for ($dx = -1; $dx <= 1; $dx++) {
+				for ($dy = -1; $dy <= 1; $dy++) {
+					if ($dx || $dy) {
+						imagestring($img, $font, $px + $dx, $py + $dy, $text, $black);
+					}
+				}
+			}
+			imagestring($img, $font, $px, $py, $text, $white);
+
+			$tempFile = $this->tempManager->getTemporaryFile('.jpg');
+			imagejpeg($img, $tempFile);
+			imagedestroy($img);
+
+			$newImage = $this->addImageExifComment($text, $tempFile);
+			return $newImage;
+		} catch (\Throwable $e) {
+			$this->logger->warning('Could not add AI watermark to AI generated image', ['exception' => $e]);
+			return $image;
+		}
+	}
+
+	private function addImageExifComment(string $text, string $filename): string {
+		$peljpeg = new PelJpeg($filename);
+		$exif = $peljpeg->getExif();
+		if (!$exif) {
+			$exif = new PelExif();
+			$peljpeg->setExif($exif);
+		}
+		$peltiff = $exif->getTiff();
+		if (!$peltiff) {
+			$peltiff = new PelTiff();
+			$exif->setTiff($peltiff);
+		}
+		$ifd = $peltiff->getIfd();
+		if (!$ifd) {
+			$peltiff->setIfd(new PelIfd(PelIfd::IFD0));
+			$ifd = $peltiff->getIfd();
+		}
+
+		$exifIfd = $ifd->getSubIfd(PelIfd::EXIF);
+		if (!$exifIfd) {
+			$exifIfd = new PelIfd(PelIfd::EXIF);
+			$ifd->addSubIfd($exifIfd);
+		}
+
+		$comment = $exifIfd->getEntry(PelTag::USER_COMMENT);
+		if (!$comment) {
+			$comment = new PelEntryUndefined(PelTag::USER_COMMENT, $text);
+			$exifIfd->addEntry($comment);
+		} else {
+			$comment->setValue($text);
+		}
+
+		return $peljpeg->getBytes();
+	}
+
+	public function markAudio(string $audio): string {
+		try {
+			$tempFile = $this->tempManager->getTemporaryFile('.mp3');
+			file_put_contents($tempFile, $audio);
+
+			$getID3 = new \OCA\OpenAi\Vendor\getID3;
+			$getID3->setOption(['encoding' => 'UTF-8']);
+			/**
+			 * @psalm-suppress UndefinedConstant
+			 */
+			getid3_lib::IncludeDependency(GETID3_INCLUDEPATH . 'write.php', __FILE__, true);
+			$tagwriter = new getid3_writetags();
+			$tagwriter->filename = $tempFile;
+			$tagwriter->tagformats = ['id3v2.4'];
+			$tagwriter->tag_encoding = 'UTF-8';
+			$tagwriter->tag_data = ['comment' => [self::COMMENT]];
+			$tagwriter->WriteTags();
+
+			$newAudio = file_get_contents($tempFile);
+			if (!$newAudio) {
+				throw new \RuntimeException('Could not read temporary audio file');
+			}
+
+			return $newAudio;
+		} catch (\Throwable $e) {
+			$this->logger->warning('Could not add AI watermark to AI generated image', ['exception' => $e]);
+			return $audio;
+		}
+	}
+}

lib/SpeechToText/STTProvider.php

@@ -32,10 +32,10 @@ public function getName(): string {
 	 */
 	public function transcribeFile(File $file): string {
 		try {
-			return $this->replicateAPIService->transcribeFile($file);
+			return $this->replicateAPIService->transcribeFile($file->getContent());
 		} catch (\Exception $e) {
-			$this->logger->warning('Replicate\'s Whisper transcription failed with: ' . $e->getMessage(), ['exception' => $e]);
-			throw new \RuntimeException('Replicate\'s Whisper transcription failed with: ' . $e->getMessage());
+			$this->logger->warning('Replicate\'s Whisper transcription of file "' . $file->getPath() .'" failed with: ' . $e->getMessage(), ['exception' => $e]);
+			throw new \RuntimeException('Replicate\'s Whisper transcription of file "' . $file->getPath() .'"  failed with: ' . $e->getMessage());
 		}
 	}
 }