fix: cleanup localfiles and distant storage

Author: DD-Arras

src/Controller/Api/ValidationsController.php

@@ -248,19 +248,23 @@ public function deleteValidation($uid)
             throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
         }
 
+        $this->logger->info('Validation[{uid}] : removing all saved data...', [
+            'uid' => $validation->getUid(),
+            'datasetName' => $validation->getDatasetName(),
+        ]);
+
         $em = $this->getDoctrine()->getManager();
         $em->remove($validation);
         $em->flush();
 
-        $fs = new FileSystem();
-        $validationDirectory = $this->storage->getDirectory($validation);
-        if ($fs->exists($validationDirectory)) {
-            $fs->remove($validationDirectory);
-        }
-
         // Delete from storage
-        if ($this->dataStorage->directoryExists($validationDirectory)) {
-            $this->dataStorage->deleteDirectory($validationDirectory);
+        $uploadDirectory = $this->storage->getUploadDirectory($validation);
+        if ($this->dataStorage->directoryExists($uploadDirectory)) {
+            $this->dataStorage->deleteDirectory($uploadDirectory);
+        }
+        $outputDirectory = $this->storage->getOutputDirectory($validation);
+        if ($this->dataStorage->directoryExists($outputDirectory)) {
+            $this->dataStorage->deleteDirectory($outputDirectory);
         }
 
         return new JsonResponse(null, Response::HTTP_NO_CONTENT);

src/Storage/ValidationsStorage.php

@@ -41,15 +41,15 @@ public function getDirectory(Validation $validation){
      * @return string
      */
     public function getUploadDirectory(Validation $validation){
-        return $this->getDirectory($validation) . "/upload/";
+        return $validation->getUid() . "/upload/";
     }
 
     /**
      * @param Validation $validation
      * @return string
      */
     public function getOutputDirectory(Validation $validation){
-        return $this->getDirectory($validation) . "/output/";
+        return $validation->getUid() . "/output/";
     }
 
 

src/Validation/ValidationManager.php

@@ -88,6 +88,22 @@ public function archive(Validation $validation)
             ]);
             $fs->remove($validationDirectory);
         }
+
+        // Delete from storage
+        $this->logger->info('Validation[{uid}] : remove upload files', [
+            'uid' => $validation->getUid(),
+        ]);
+        $uploadDirectory = $this->storage->getUploadDirectory($validation);
+        if ($this->dataStorage->directoryExists($uploadDirectory)) {
+            $this->dataStorage->deleteDirectory($uploadDirectory);
+        }
+        $this->logger->info('Validation[{uid}] : remove output files', [
+            'uid' => $validation->getUid(),
+        ]);
+        $outputDirectory = $this->storage->getOutputDirectory($validation);
+        if ($this->dataStorage->directoryExists($outputDirectory)) {
+            $this->dataStorage->deleteDirectory($outputDirectory);
+        }
         $this->logger->info('Validation[{uid}] : archive removing all files : completed', [
             'uid' => $validation->getUid(),
             'status' => Validation::STATUS_ARCHIVED,
@@ -216,7 +232,7 @@ private function doProcess(Validation $validation)
      */
     private function getZip(Validation $validation)
     {
-        $this->logger->info('Validation[{uid}] : Get from storage...', [
+        $this->logger->info('Validation[{uid}] : get from storage...', [
             'uid' => $validation->getUid(),
             'datasetName' => $validation->getDatasetName(),
         ]);
@@ -245,6 +261,10 @@ private function getZip(Validation $validation)
      */
     private function validateZip($validation)
     {
+        $this->logger->info('Validation[{uid}] : validate zip archive...', [
+            'uid' => $validation->getUid(),
+            'datasetName' => $validation->getDatasetName(),
+        ]);
         $validationDirectory = $this->storage->getDirectory($validation);
         $zipPath = $validationDirectory . '/' . $validation->getDatasetName() . '.zip';
         $errors = $this->zipArchiveValidator->validate($zipPath);
@@ -316,7 +336,7 @@ private function zipNormData(Validation $validation)
     private function saveToStorage(Validation $validation)
     {
         // Saves normalized data to storage
-        $this->logger->info('Validation[{uid}] : saving...', [
+        $this->logger->info('Validation[{uid}] : saving normalized data...', [
             'uid' => $validation->getUid(),
             'datasetName' => $validation->getDatasetName(),
         ]);
@@ -335,11 +355,13 @@ private function saveToStorage(Validation $validation)
         fclose($stream);
 
         // Saves validator logs to storage
+        $this->logger->info('Validation[{uid}] : saving logs...', [
+            'uid' => $validation->getUid(),
+            'datasetName' => $validation->getDatasetName(),
+        ]);
         $logPath = $validationDirectory . '/validator-debug.log';
         $outputPath = $outputDirectory . '/validator-debug.log';
-        if ($this->dataStorage->fileExists($outputPath)){
-            $this->dataStorage->delete($outputPath);
-        }
+
         $stream = fopen($logPath, 'r+');
         $this->dataStorage->writeStream($outputPath, $stream);
         fclose($stream);