Skip to content

Commit 7e6411a

Browse files
DD-ArrasDD-Arras
authored
fix: change storage location based on environmental variable
Storage adapter
2 parents a25ab96 + db3eda9 commit 7e6411a

7 files changed

Lines changed: 56 additions & 44 deletions

File tree

‎.env‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ S3_SECRET_KEY=
5454
S3_BUCKET=dev-ign-mut-validtri
5555
S3_REGION=sbg
5656

57+
# Storage type used. Currently supported: local, S3
58+
STORAGE_TYPE=local
59+
5760
### validator-worker/validator-cli postgis
5861
# DB_URL=jdbc:postgresql://${PGHOST}:${PGPORT}/${PGDATABASE}?ssl=true&sslmode=require&sslfactory=org.postgresql.ssl.NonValidatingFactory
5962
DB_URL=jdbc:postgresql://localhost:5432/validator_api

‎src/Controller/Api/HealthController.php‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
namespace App\Controller\Api;
44

5+
use App\Storage\ValidationsStorage;
56
use Exception;
67
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
78
use Symfony\Component\Routing\Annotation\Route;
@@ -17,7 +18,9 @@
1718
*/
1819
class HealthController extends AbstractController
1920
{
20-
public function __construct(private LoggerInterface $logger){
21+
public function __construct(
22+
private LoggerInterface $logger,
23+
private ValidationsStorage $storage){
2124

2225
}
2326

@@ -47,11 +50,11 @@ public function healthDB(EntityManagerInterface $entityManager)
4750
*
4851
* @Route("/s3", name="health_s3")
4952
*/
50-
public function healthS3(FilesystemOperator $dataStorage)
53+
public function healthS3()
5154
{
5255
$this->logger->info('list files from S3 bucket...');
5356
try {
54-
$files = $dataStorage->listContents('.', false);
57+
$files = $this->storage->getStorage()->listContents('.', false);
5558
$numFiles = count($files->toArray());
5659
return new JsonResponse('found '.$numFiles.' files', Response::HTTP_OK);
5760
} catch (Exception $e) {

‎src/Controller/Api/ValidationsController.php‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
use App\Storage\ValidationsStorage;
1212
use JMS\Serializer\Serializer;
1313
use JMS\Serializer\SerializerInterface;
14-
use League\Flysystem\FilesystemOperator;
1514
use Psr\Log\LoggerInterface;
1615
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
1716
use Symfony\Component\Filesystem\Filesystem;
@@ -30,7 +29,6 @@ public function __construct(
3029
private ValidationRepository $repository,
3130
private SerializerInterface $serializer,
3231
private ValidationsStorage $storage,
33-
private FilesystemOperator $dataStorage,
3432
private ValidatorArgumentsService $valArgsService,
3533
private MimeTypeGuesserService $mimeTypeGuesserService,
3634
private LoggerInterface $logger
@@ -88,7 +86,7 @@ public function readConsole($uid)
8886
$outputDirectory = $this->storage->getOutputDirectory($validation);
8987
$filepath = $outputDirectory . '/validator-debug.log';
9088

91-
$content = $this->dataStorage->read($filepath);
89+
$content = $this->storage->getStorage()->read($filepath);
9290

9391
return new Response(
9492
$content,
@@ -162,15 +160,15 @@ public function uploadDataset(Request $request)
162160

163161
// Save file to storage
164162
$uploadDirectory = $this->storage->getUploadDirectory($validation);
165-
if (!$this->dataStorage->directoryExists($uploadDirectory)) {
166-
$this->dataStorage->createDirectory($uploadDirectory);
163+
if (!$this->storage->getStorage()->directoryExists($uploadDirectory)) {
164+
$this->storage->getStorage()->createDirectory($uploadDirectory);
167165
}
168166
$fileLocation = $uploadDirectory . $validation->getDatasetName() . '.zip';
169-
if ($this->dataStorage->fileExists($fileLocation)) {
170-
$this->dataStorage->delete($fileLocation);
167+
if ($this->storage->getStorage()->fileExists($fileLocation)) {
168+
$this->storage->getStorage()->delete($fileLocation);
171169
}
172170
$stream = fopen($file->getRealPath(), 'r+');
173-
$this->dataStorage->writeStream($fileLocation, $stream);
171+
$this->storage->getStorage()->writeStream($fileLocation, $stream);
174172
fclose($stream);
175173

176174
$fs = new Filesystem;
@@ -259,12 +257,12 @@ public function deleteValidation($uid)
259257

260258
// Delete from storage
261259
$uploadDirectory = $this->storage->getUploadDirectory($validation);
262-
if ($this->dataStorage->directoryExists($uploadDirectory)) {
263-
$this->dataStorage->deleteDirectory($uploadDirectory);
260+
if ($this->storage->getStorage()->directoryExists($uploadDirectory)) {
261+
$this->storage->getStorage()->deleteDirectory($uploadDirectory);
264262
}
265263
$outputDirectory = $this->storage->getOutputDirectory($validation);
266-
if ($this->dataStorage->directoryExists($outputDirectory)) {
267-
$this->dataStorage->deleteDirectory($outputDirectory);
264+
if ($this->storage->getStorage()->directoryExists($outputDirectory)) {
265+
$this->storage->getStorage()->deleteDirectory($outputDirectory);
268266
}
269267

270268
return new JsonResponse(null, Response::HTTP_NO_CONTENT);
@@ -333,11 +331,11 @@ public function downloadSourceData($uid)
333331
*/
334332
private function getDownloadResponse($filepath, $filename)
335333
{
336-
if (!$this->dataStorage->has($filepath)) {
334+
if (!$this->storage->getStorage()->has($filepath)) {
337335
throw new ApiException("Requested files not found for this validation", Response::HTTP_FORBIDDEN);
338336
}
339337

340-
$stream = $this->dataStorage->readStream($filepath);
338+
$stream = $this->storage->getStorage()->readStream($filepath);
341339

342340
return new StreamedResponse(function () use ($stream) {
343341
fpassthru($stream);

‎src/Storage/ValidationsStorage.php‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace App\Storage ;
44

55
use App\Entity\Validation;
6+
use League\Flysystem\FilesystemOperator;
67

78
/**
89
* Manage validation files
@@ -16,9 +17,24 @@ class ValidationsStorage {
1617
*/
1718
private $path;
1819

19-
public function __construct($validationsDir)
20+
/**
21+
* Flysystem storage
22+
*
23+
* @var FilesystemOperator
24+
*/
25+
private $storageSystem;
26+
27+
public function __construct($validationsDir,
28+
FilesystemOperator $dataStorage,
29+
FilesystemOperator $defaultStorage)
2030
{
2131
$this->path = $validationsDir;
32+
// Assign storage based on env
33+
if (getenv("STORAGE_TYPE") === "S3"){
34+
$this->storageSystem = $dataStorage;
35+
} else {
36+
$this->storageSystem = $defaultStorage;
37+
}
2238
}
2339

2440
/**
@@ -28,6 +44,13 @@ public function getPath(){
2844
return $this->path;
2945
}
3046

47+
/**
48+
* @return FilesystemOperator
49+
*/
50+
public function getStorage(){
51+
return $this->storageSystem;
52+
}
53+
3154
/**
3255
* @param Validation $validation
3356
* @return string

‎src/Validation/ValidationManager.php‎

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
use Symfony\Component\Filesystem\Filesystem;
1212
use Symfony\Component\Process\Exception\ProcessFailedException;
1313
use Symfony\Component\Process\Process;
14-
use League\Flysystem\FilesystemOperator;
1514

1615
class ValidationManager
1716
{
@@ -41,11 +40,6 @@ class ValidationManager
4140
*/
4241
private $zipArchiveValidator;
4342

44-
/**
45-
* @var FilesystemOperator
46-
*/
47-
private $dataStorage;
48-
4943
/**
5044
* Current validation (in order to handle SIGTERM)
5145
* @var Validation
@@ -55,14 +49,12 @@ class ValidationManager
5549
public function __construct(
5650
EntityManagerInterface $em,
5751
ValidationsStorage $storage,
58-
FilesystemOperator $dataStorage,
5952
ValidatorCLI $validatorCli,
6053
ZipArchiveValidator $zipArchiveValidator,
6154
LoggerInterface $logger
6255
) {
6356
$this->em = $em;
6457
$this->storage = $storage;
65-
$this->dataStorage = $dataStorage;
6658
$this->validatorCli = $validatorCli;
6759
$this->zipArchiveValidator = $zipArchiveValidator;
6860
$this->logger = $logger;
@@ -94,15 +86,15 @@ public function archive(Validation $validation)
9486
'uid' => $validation->getUid(),
9587
]);
9688
$uploadDirectory = $this->storage->getUploadDirectory($validation);
97-
if ($this->dataStorage->directoryExists($uploadDirectory)) {
98-
$this->dataStorage->deleteDirectory($uploadDirectory);
89+
if ($this->storage->getStorage()->directoryExists($uploadDirectory)) {
90+
$this->storage->getStorage()->deleteDirectory($uploadDirectory);
9991
}
10092
$this->logger->info('Validation[{uid}] : remove output files', [
10193
'uid' => $validation->getUid(),
10294
]);
10395
$outputDirectory = $this->storage->getOutputDirectory($validation);
104-
if ($this->dataStorage->directoryExists($outputDirectory)) {
105-
$this->dataStorage->deleteDirectory($outputDirectory);
96+
if ($this->storage->getStorage()->directoryExists($outputDirectory)) {
97+
$this->storage->getStorage()->deleteDirectory($outputDirectory);
10698
}
10799
$this->logger->info('Validation[{uid}] : archive removing all files : completed', [
108100
'uid' => $validation->getUid(),
@@ -248,7 +240,7 @@ private function getZip(Validation $validation)
248240

249241
file_put_contents(
250242
$zipPath,
251-
$this->dataStorage->read($uploadFile)
243+
$this->storage->getStorage()->read($uploadFile)
252244
);
253245
}
254246

@@ -343,15 +335,15 @@ private function saveToStorage(Validation $validation)
343335
$validationDirectory = $this->storage->getDirectory($validation);
344336
$normDataPath = $validationDirectory . '/validation/' . $validation->getDatasetName() . '.zip';
345337
$outputDirectory = $this->storage->getOutputDirectory($validation);
346-
if (! $this->dataStorage->directoryExists($outputDirectory)){
347-
$this->dataStorage->createDirectory($outputDirectory);
338+
if (! $this->storage->getStorage()->directoryExists($outputDirectory)){
339+
$this->storage->getStorage()->createDirectory($outputDirectory);
348340
}
349341
$outputPath = $outputDirectory . $validation->getDatasetName() . '.zip';
350-
if ($this->dataStorage->fileExists($outputPath)){
351-
$this->dataStorage->delete($outputPath);
342+
if ($this->storage->getStorage()->fileExists($outputPath)){
343+
$this->storage->getStorage()->delete($outputPath);
352344
}
353345
$stream = fopen($normDataPath, 'r+');
354-
$this->dataStorage->writeStream($outputPath, $stream);
346+
$this->storage->getStorage()->writeStream($outputPath, $stream);
355347
fclose($stream);
356348

357349
// Saves validator logs to storage
@@ -363,7 +355,7 @@ private function saveToStorage(Validation $validation)
363355
$outputPath = $outputDirectory . '/validator-debug.log';
364356

365357
$stream = fopen($logPath, 'r+');
366-
$this->dataStorage->writeStream($outputPath, $stream);
358+
$this->storage->getStorage()->writeStream($outputPath, $stream);
367359
fclose($stream);
368360
}
369361

‎tests/Command/Validations/CleanupCommandTest.php‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,6 @@ public function tearDown(): void
5454
*/
5555
public function testCleanupOneSecond()
5656
{
57-
58-
$this->markTestSkipped('TODO : fix test (use local directory for dev and test?)');
59-
6057
static::ensureKernelShutdown();
6158

6259
// wait for 2 seconds

‎tests/Controller/Api/ValidationControllerTest.php‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,6 @@ public function testGetValidationNotFound()
126126
*/
127127
public function testUploadDatasetCorrectParams()
128128
{
129-
$this->markTestSkipped('TODO : fix test (use local directory for dev and test?)');
130-
131129
$filename = ValidationsFixtures::FILENAME_SUP_PM3;
132130
$dataset = $this->createFakeUpload(
133131
$filename,
@@ -229,8 +227,6 @@ public function testUploadDatasetNoFile()
229227
*/
230228
public function testDeleteValidation()
231229
{
232-
$this->markTestSkipped('TODO : fix test (use local directory for dev and test?)');
233-
234230
$validation = $this->getValidationFixture(ValidationsFixtures::VALIDATION_ARCHIVED);
235231

236232
$this->client->request(

0 commit comments

Comments
 (0)