Skip to content

Commit a3385e4

Browse files
marcelklehrmarcelklehr
committed
feat(Watermarking): Add metadata and watermarks to generated images
to comply with EU AI Act Signed-off-by: Marcel Klehr <mklehr@gmx.net>
1 parent ead35bd commit a3385e4

9 files changed

Lines changed: 523 additions & 16 deletions

File tree

‎composer.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414
"test:unit": "phpunit --config tests/phpunit.xml"
1515
},
1616
"require": {
17-
"php": "^8.1"
17+
"php": "^8.1",
18+
"fileeye/pel": "^0.12.0",
19+
"james-heinrich/getid3": "^1.9"
1820
},
1921
"require-dev": {
2022
"nextcloud/coding-standard": "^1.1",

‎composer.lock‎

Lines changed: 134 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/AppInfo/Application.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ public function register(IRegistrationContext $context): void {
4444
$context->registerSpeechToTextProvider(STTProvider::class);
4545
$context->registerTextToImageProvider(TextToImageProvider::class);
4646
$context->registerTextProcessingProvider(FreePromptProvider::class);
47+
$context->registerTaskProcessingProvider(\OCA\Replicate\TaskProcessing\TextToImageProvider::class);
48+
$context->registerTaskProcessingProvider(\OCA\Replicate\TaskProcessing\SpeechToTextProvider::class);
4749
}
4850
}
4951

‎lib/Service/ReplicateAPIService.php‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,15 +65,15 @@ public function createWhisperPrediction(string $audioFileContent, bool $translat
6565
/**
6666
* Create a prediction and wait for it to complete to return the text result
6767
*
68-
* @param File $file
68+
* @param string $audio
6969
* @param bool $translate
7070
* @return string
7171
* @throws GenericFileException
7272
* @throws LockedException
7373
* @throws NotPermittedException
7474
*/
75-
public function transcribeFile(File $file, bool $translate = false): string {
76-
$prediction = $this->createWhisperPrediction($file->getContent(), $translate);
75+
public function transcribeFile(string $audio, bool $translate = false): string {
76+
$prediction = $this->createWhisperPrediction($audio, $translate);
7777
if (isset($prediction['id'])) {
7878
$predictionId = $prediction['id'];
7979

@@ -83,17 +83,17 @@ public function transcribeFile(File $file, bool $translate = false): string {
8383
? $prediction['output']['translation']
8484
: $prediction['output']['transcription'];
8585
} elseif ($prediction['status'] === 'failed') {
86-
throw new Exception('Error transcribing file "' . $file->getName() . '": remote job failed');
86+
throw new Exception('Error transcribing file: remote job failed');
8787
} elseif ($prediction['status'] === 'canceled') {
88-
throw new Exception('Error transcribing file "' . $file->getName() . '": remote job was canceled');
88+
throw new Exception('Error transcribing file: remote job was canceled');
8989
} elseif ($prediction['status'] !== 'starting' && $prediction['status'] !== 'processing') {
90-
throw new Exception('Error transcribing file "' . $file->getName() . '": unknown prediction status');
90+
throw new Exception('Error transcribing file: unknown prediction status');
9191
}
9292
sleep(2);
9393
$prediction = $this->getPrediction($predictionId);
9494
}
9595
}
96-
throw new Exception('Error transcribing file "' . $file->getName() . '"');
96+
throw new Exception('Error transcribing file');
9797
}
9898

9999
/**

‎lib/Service/WatermarkingService.php‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
<?php
2+
3+
/**
4+
* SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors
5+
* SPDX-License-Identifier: AGPL-3.0-or-later
6+
*/
7+
8+
namespace OCA\OpenAi\Service;
9+
10+
use lsolesen\pel\PelEntryUndefined;
11+
use lsolesen\pel\PelExif;
12+
use lsolesen\pel\PelIfd;
13+
use lsolesen\pel\PelJpeg;
14+
use lsolesen\pel\PelTag;
15+
use lsolesen\pel\PelTiff;
16+
use OCP\ITempManager;
17+
use Psr\Log\LoggerInterface;
18+
19+
class WatermarkingService {
20+
public const COMMENT = 'Generated with Artificial Intelligence';
21+
public function __construct(
22+
private ITempManager $tempManager,
23+
private LoggerInterface $logger,
24+
) {
25+
}
26+
27+
public function markImage(string $image): string {
28+
try {
29+
$text = self::COMMENT;
30+
31+
$img = imagecreatefromstring($image);
32+
$font = 5;// built-in font 1-5
33+
$white = imagecolorallocate($img, 255, 255, 255);
34+
$black = imagecolorallocate($img, 0, 0, 0);
35+
36+
$w = imagefontwidth($font) * strlen($text);
37+
$h = imagefontheight($font);
38+
$px = imagesx($img) - $w - 10;
39+
$py = imagesy($img) - $h - 10;
40+
41+
// draw 1-pixel black outline by offsetting in 4 directions
42+
for ($dx = -1; $dx <= 1; $dx++) {
43+
for ($dy = -1; $dy <= 1; $dy++) {
44+
if ($dx || $dy) {
45+
imagestring($img, $font, $px + $dx, $py + $dy, $text, $black);
46+
}
47+
}
48+
}
49+
imagestring($img, $font, $px, $py, $text, $white);
50+
51+
$tempFile = $this->tempManager->getTemporaryFile('.jpg');
52+
imagejpeg($img, $tempFile);
53+
imagedestroy($img);
54+
55+
$newImage = $this->addImageExifComment($text, $tempFile);
56+
return $newImage;
57+
} catch (\Throwable $e) {
58+
$this->logger->warning('Could not add AI watermark to AI generated image', ['exception' => $e]);
59+
return $image;
60+
}
61+
}
62+
63+
private function addImageExifComment(string $text, string $filename): string {
64+
// Load PHP Exif Library for adding image metadata
65+
// Sadly PHPScoper was a nightmare with this dep, so here we go
66+
require_once(__DIR__ . '/../../vendor/fileeye/pel/autoload.php');
67+
68+
$peljpeg = new PelJpeg($filename);
69+
$exif = $peljpeg->getExif();
70+
if (!$exif) {
71+
$exif = new PelExif();
72+
$peljpeg->setExif($exif);
73+
}
74+
$peltiff = $exif->getTiff();
75+
if (!$peltiff) {
76+
$peltiff = new PelTiff();
77+
$exif->setTiff($peltiff);
78+
}
79+
$ifd = $peltiff->getIfd();
80+
if (!$ifd) {
81+
$peltiff->setIfd(new PelIfd(PelIfd::IFD0));
82+
$ifd = $peltiff->getIfd();
83+
}
84+
85+
$exifIfd = $ifd->getSubIfd(PelIfd::EXIF);
86+
if (!$exifIfd) {
87+
$exifIfd = new PelIfd(PelIfd::EXIF);
88+
$ifd->addSubIfd($exifIfd);
89+
}
90+
91+
$comment = $exifIfd->getEntry(PelTag::USER_COMMENT);
92+
if (!$comment) {
93+
$comment = new PelEntryUndefined(PelTag::USER_COMMENT, $text);
94+
$exifIfd->addEntry($comment);
95+
} else {
96+
$comment->setValue($text);
97+
}
98+
99+
return $peljpeg->getBytes();
100+
}
101+
102+
public function markAudio(string $audio): string {
103+
try {
104+
// Load getID3 library for adding audio metadata
105+
require_once(__DIR__ . '/../../vendor/james-heinrich/getid3/getid3/getid3.php');
106+
107+
$tempFile = $this->tempManager->getTemporaryFile('.mp3');
108+
file_put_contents($tempFile, $audio);
109+
110+
$getID3 = new \getID3;
111+
$getID3->setOption(['encoding' => 'UTF-8']);
112+
/**
113+
* @psalm-suppress UndefinedConstant
114+
*/
115+
\getid3_lib::IncludeDependency(GETID3_INCLUDEPATH . 'write.php', __FILE__, true);
116+
$tagwriter = new \getid3_writetags();
117+
$tagwriter->filename = $tempFile;
118+
$tagwriter->tagformats = ['id3v2.4'];
119+
$tagwriter->tag_encoding = 'UTF-8';
120+
$tagwriter->tag_data = ['comment' => [self::COMMENT]];
121+
$tagwriter->WriteTags();
122+
123+
$newAudio = file_get_contents($tempFile);
124+
if (!$newAudio) {
125+
throw new \RuntimeException('Could not read temporary audio file');
126+
}
127+
128+
return $newAudio;
129+
} catch (\Throwable $e) {
130+
$this->logger->warning('Could not add AI watermark to AI generated image', ['exception' => $e]);
131+
return $audio;
132+
}
133+
}
134+
}

‎lib/SpeechToText/STTProvider.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,10 @@ public function getName(): string {
3232
*/
3333
public function transcribeFile(File $file): string {
3434
try {
35-
return $this->replicateAPIService->transcribeFile($file);
35+
return $this->replicateAPIService->transcribeFile($file->getContent());
3636
} catch (\Exception $e) {
37-
$this->logger->warning('Replicate\'s Whisper transcription failed with: ' . $e->getMessage(), ['exception' => $e]);
38-
throw new \RuntimeException('Replicate\'s Whisper transcription failed with: ' . $e->getMessage());
37+
$this->logger->warning('Replicate\'s Whisper transcription of file "' . $file->getPath() .'" failed with: ' . $e->getMessage(), ['exception' => $e]);
38+
throw new \RuntimeException('Replicate\'s Whisper transcription of file "' . $file->getPath() .'" failed with: ' . $e->getMessage());
3939
}
4040
}
4141
}

0 commit comments

Comments
 (0)