Quota Cleanup - Updates to remove unused starting kits and bundles and exclude starting kit and public data from unused datasets (#2313)

* quota cleanup updated to allow users to remove unused starting kits and competition bundles. Starting kit and public data are not included in unused datasets anymore

* Merge conflicts

* Merge conflict

---------

Co-authored-by: didayolo <adrien.pavao@gmail.com>

Author: ihsaan-ullah

src/apps/api/tests/test_cleanup.py

@@ -56,8 +56,19 @@ def setUp(self):
             DataFactory(created_by=user, type=Data.INGESTION_PROGRAM),
             DataFactory(created_by=user, type=Data.SCORING_PROGRAM),
             DataFactory(created_by=user, type=Data.INPUT_DATA),
-            DataFactory(created_by=user, type=Data.REFERENCE_DATA),
-            DataFactory(created_by=user, type=Data.PUBLIC_DATA)
+            DataFactory(created_by=user, type=Data.REFERENCE_DATA)
+        ]
+
+        # Create unused starting kits
+        self.unused_starting_kits = [
+            DataFactory(created_by=user, type=Data.STARTING_KIT),
+            DataFactory(created_by=user, type=Data.STARTING_KIT)
+        ]
+
+        # Create unused competition bundles
+        self.unused_competition_bundles = [
+            DataFactory(created_by=user, type=Data.COMPETITION_BUNDLE),
+            DataFactory(created_by=user, type=Data.COMPETITION_BUNDLE)
         ]
 
         self.client.login(username='test_user', password='test_user')
@@ -72,6 +83,8 @@ def test_cleanup_stats(self):
         assert content["unused_datasets_programs"] == len(self.unused_datasets_programs)
         assert content["unused_submissions"] == len(self.unused_submissions)
         assert content["failed_submissions"] == len(self.failed_submissions)
+        assert content["unused_starting_kits"] == len(self.unused_starting_kits)
+        assert content["unused_competition_bundles"] == len(self.unused_competition_bundles)
 
     def test_delete_unused_tasks(self):
 
@@ -132,3 +145,33 @@ def test_delete_failed_submissions(self):
         assert resp.status_code == 200
         content = json.loads(resp.content)
         assert content["failed_submissions"] == 0
+
+    def test_delete_unused_starting_kits(self):
+
+        url = reverse('delete_unused_starting_kits')
+        resp = self.client.delete(url)
+        assert resp.status_code == 200
+        content = json.loads(resp.content)
+        assert content["success"]
+        assert content["message"] == "Unused starting kits deleted successfully"
+
+        url = reverse('user_quota_cleanup')
+        resp = self.client.get(url)
+        assert resp.status_code == 200
+        content = json.loads(resp.content)
+        assert content["unused_starting_kits"] == 0
+
+    def test_delete_unused_competition_bundles(self):
+
+        url = reverse('delete_unused_competition_bundles')
+        resp = self.client.delete(url)
+        assert resp.status_code == 200
+        content = json.loads(resp.content)
+        assert content["success"]
+        assert content["message"] == "Unused competition bundles deleted successfully"
+
+        url = reverse('user_quota_cleanup')
+        resp = self.client.get(url)
+        assert resp.status_code == 200
+        content = json.loads(resp.content)
+        assert content["unused_competition_bundles"] == 0

src/apps/api/urls.py

@@ -54,6 +54,8 @@
     path('delete_unused_datasets/', quota.delete_unused_datasets, name="delete_unused_datasets"),
     path('delete_unused_submissions/', quota.delete_unused_submissions, name="delete_unused_submissions"),
     path('delete_failed_submissions/', quota.delete_failed_submissions, name="delete_failed_submissions"),
+    path('delete_unused_starting_kits/', quota.delete_unused_starting_kits, name="delete_unused_starting_kits"),
+    path('delete_unused_competition_bundles/', quota.delete_unused_competition_bundles, name="delete_unused_competition_bundles"),
 
     # User account
     path('delete_account/', profiles.delete_account, name="delete_account"),

src/apps/api/views/quota.py

@@ -21,7 +21,9 @@ def user_quota_cleanup(request):
     unused_datasets_programs = Data.objects.filter(
         Q(created_by=request.user) &
         ~Q(type=Data.SUBMISSION) &
-        ~Q(type=Data.COMPETITION_BUNDLE)
+        ~Q(type=Data.COMPETITION_BUNDLE) &
+        ~Q(type=Data.PUBLIC_DATA) &
+        ~Q(type=Data.STARTING_KIT)
     ).exclude(
         Q(task_ingestion_programs__isnull=False) |
         Q(task_input_datas__isnull=False) |
@@ -42,11 +44,29 @@ def user_quota_cleanup(request):
         Q(status=Submission.FAILED)
     ).count()
 
+    # Get unused starting kits count
+    unused_starting_kits = Data.objects.filter(
+        Q(created_by=request.user) &
+        Q(type=Data.STARTING_KIT) &
+        Q(competition__isnull=True) &
+        Q(phase_starting_kit__isnull=True)
+    ).count()
+
+    # Get unused competition bundles
+    unused_competition_bundles = Data.objects.filter(
+        Q(created_by=request.user) &
+        Q(type=Data.COMPETITION_BUNDLE) &
+        Q(competition__isnull=True) &
+        Q(competition_bundles__isnull=True)
+    ).count()
+
     return Response({
         "unused_tasks": unused_tasks,
         "unused_datasets_programs": unused_datasets_programs,
         "unused_submissions": unused_submissions,
-        "failed_submissions": failed_submissions
+        "failed_submissions": failed_submissions,
+        "unused_starting_kits": unused_starting_kits,
+        "unused_competition_bundles": unused_competition_bundles
     })
 
 
@@ -84,7 +104,9 @@ def delete_unused_datasets(request):
         Data.objects.filter(
             Q(created_by=request.user) &
             ~Q(type=Data.SUBMISSION) &
-            ~Q(type=Data.COMPETITION_BUNDLE)
+            ~Q(type=Data.COMPETITION_BUNDLE) &
+            ~Q(type=Data.PUBLIC_DATA) &
+            ~Q(type=Data.STARTING_KIT)
         ).exclude(
             Q(task_ingestion_programs__isnull=False) |
             Q(task_input_datas__isnull=False) |
@@ -144,3 +166,47 @@ def delete_failed_submissions(request):
             "success": False,
             "message": f"{e}"
         })
+
+
+@api_view(['DELETE'])
+def delete_unused_starting_kits(request):
+    try:
+        Data.objects.filter(
+            Q(created_by=request.user) &
+            Q(type=Data.STARTING_KIT) &
+            Q(competition__isnull=True) &
+            Q(phase_starting_kit__isnull=True)
+        ).delete()
+
+        return Response({
+            "success": True,
+            "message": "Unused starting kits deleted successfully"
+        })
+    except Exception as e:
+        logger.error(f"UNUSED STARTING KITS DELETION --- {e}")
+        return Response({
+            "success": False,
+            "message": f"{e}"
+        })
+
+
+@api_view(['DELETE'])
+def delete_unused_competition_bundles(request):
+    try:
+        Data.objects.filter(
+            Q(created_by=request.user) &
+            Q(type=Data.COMPETITION_BUNDLE) &
+            Q(competition__isnull=True) &
+            Q(competition_bundles__isnull=True)
+        ).delete()
+
+        return Response({
+            "success": True,
+            "message": "Unused competition bundles deleted successfully"
+        })
+    except Exception as e:
+        logger.error(f"UNUSED COMPETITION BUNDLES DELETION --- {e}")
+        return Response({
+            "success": False,
+            "message": f"{e}"
+        })

src/static/js/ours/client.js

@@ -392,6 +392,12 @@ CODALAB.api = {
     delete_failed_submissions: () => {
         return CODALAB.api.request('DELETE', `${URLS.API}delete_failed_submissions/`)
     },
+    delete_unused_starting_kits: () => {
+        return CODALAB.api.request('DELETE', `${URLS.API}delete_unused_starting_kits/`)
+    },
+    delete_unused_competition_bundles: () => {
+        return CODALAB.api.request('DELETE', `${URLS.API}delete_unused_competition_bundles/`)
+    },
     /*---------------------------------------------------------------------
          User Account
     ---------------------------------------------------------------------*/

src/static/riot/quota_management.tag

@@ -54,6 +54,26 @@
                     </button>
                     </td>
                 </tr>
+                <!--  Unused Starting Kits  -->
+                <tr>
+                    <td>Unused Starting Kits <span show="{unused_starting_kits > 0}">(<b>{unused_starting_kits}</b>)</span></td>
+                    <td>
+                        <button class="ui red right floated labeled icon button {disabled: unused_starting_kits === 0}" onclick="{delete_unused_starting_kits}">
+                        <i class="icon trash"></i>
+                        Delete unused starting kits
+                    </button>
+                    </td>
+                </tr>
+                <!--  Unused Competition Bundles  -->
+                <tr>
+                    <td>Unused Competition Bundles <span show="{unused_competition_bundles > 0}">(<b>{unused_competition_bundles}</b>)</span></td>
+                    <td>
+                        <button class="ui red right floated labeled icon button {disabled: unused_competition_bundles === 0}" onclick="{delete_unused_competition_bundles}">
+                        <i class="icon trash"></i>
+                        Delete unused competition bundles
+                    </button>
+                    </td>
+                </tr>
             </tbody>
 
         </table>
@@ -66,6 +86,8 @@
         self.unused_datasets_programs = 0
         self.unused_submissions = 0
         self.failed_submissions = 0
+        self.unused_starting_kits = 0
+        self.unused_competition_bundles = 0
         self.quota = 0
         self.storage_used = 0
 
@@ -84,6 +106,8 @@
                     self.unused_datasets_programs = data.unused_datasets_programs
                     self.unused_submissions = data.unused_submissions
                     self.failed_submissions = data.failed_submissions
+                    self.unused_starting_kits = data.unused_starting_kits
+                    self.unused_competition_bundles = data.unused_competition_bundles
                     self.update()
                 })
                 .fail(function (response) {
@@ -119,7 +143,7 @@
                             self.update()
                             CODALAB.events.trigger('reload_tasks')
                             CODALAB.events.trigger('reload_datasets')
-                            self.get_cleanup()
+                            CODALAB.events.trigger('reload_quota_cleanup')
                         }else{
                             toastr.error(data.message)
                         }
@@ -141,6 +165,7 @@
                             toastr.success(data.message)
                             self.update()
                             CODALAB.events.trigger('reload_datasets')
+                            CODALAB.events.trigger('reload_quota_cleanup')
                         }else{
                             toastr.error(data.message)
                         }
@@ -162,6 +187,7 @@
                             toastr.success(data.message)
                             self.update()
                             CODALAB.events.trigger('reload_submissions')
+                            CODALAB.events.trigger('reload_quota_cleanup')
                         }else{
                             toastr.error(data.message)
                         }
@@ -183,6 +209,7 @@
                             toastr.success(data.message)
                             self.update()
                             CODALAB.events.trigger('reload_submissions')
+                            CODALAB.events.trigger('reload_quota_cleanup')
                         }else{
                             toastr.error(data.message)
                         }
@@ -193,6 +220,51 @@
             }
         }
 
+        // Delete unused starting kits
+        self.delete_unused_starting_kits = function(){
+            if (confirm(`Are you sure you want to permanently delete all unused starting kits?`)) {
+
+                CODALAB.api.delete_unused_starting_kits()
+                    .done(function (data) {
+                        if(data.success){
+                            self.unused_starting_kits = 0
+                            toastr.success(data.message)
+                            self.update()
+                            CODALAB.events.trigger('reload_datasets')
+                            CODALAB.events.trigger('reload_quota_cleanup')
+                        }else{
+                            toastr.error(data.message)
+                        }
+                    })
+                    .fail(function (response) {
+                        toastr.error("Unused starting kits deletion failed!")
+                    })
+            }
+        }
+
+        // Delete unused competition bundles
+        self.delete_unused_competition_bundles = function(){
+            if (confirm(`Are you sure you want to permanently delete all unused competition bundles?`)) {
+
+                CODALAB.api.delete_unused_competition_bundles()
+                    .done(function (data) {
+                        if(data.success){
+                            self.unused_competition_bundles = 0
+                            toastr.success(data.message)
+                            self.update()
+                            CODALAB.events.trigger('reload_competition_bundles')
+                            CODALAB.events.trigger('reload_quota_cleanup')
+                        }else{
+                            toastr.error(data.message)
+                        }
+                    })
+                    .fail(function (response) {
+                        toastr.error("Unused starting kits deletion failed!")
+                    })
+            }
+        }
+
+
         CODALAB.events.on('reload_quota_cleanup', self.get_cleanup)
 
     </script>