diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 86a9e5935a..e5f015f311 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,10 @@ Changelog v34.12.0 (unreleased) --------------------- +- Add filtering by label and pipeline in the ``flush-projects`` management command. + Also, a new ``--dry-run`` option is available to test the filters before applying + the deletion. + - Add support for using Package URL (purl) as project input. This implementation is based on ``purl2url.get_download_url``. https://github.com/aboutcode-org/scancode.io/issues/1383 diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst index 139dd5b656..044fc289f6 100644 --- a/docs/command-line-interface.rst +++ b/docs/command-line-interface.rst @@ -549,6 +549,14 @@ Optional arguments: scanpipe flush-projects --retain-days 7 +- ``--dry-run`` Do not delete any projects; just print the ones that would be flushed. + +- ``--label LABELS`` Filter projects by the provided label. + Multiple labels can be provided by using this argument multiple times. + +- ``--pipeline PIPELINES`` Filter projects by the provided pipeline name. + Multiple pipeline name can be provided by using this argument multiple times. + - ``--no-input`` Does not prompt the user for input of any kind. diff --git a/pyproject.toml b/pyproject.toml index a5a18bec17..bf554ee536 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,3 +36,5 @@ max-complexity = 10 # Allow the usage of assert in the test_spdx file. "**/test_spdx.py*" = ["S101"] "scanpipe/pipes/spdx.py" = ["UP006", "UP035"] +# Allow complexity in management commands +"scanpipe/management/commands/*" = ["C901"] diff --git a/scanpipe/management/commands/flush-projects.py b/scanpipe/management/commands/flush-projects.py index 51d3c4b41e..2e3e3cacd2 100644 --- a/scanpipe/management/commands/flush-projects.py +++ b/scanpipe/management/commands/flush-projects.py @@ -48,28 +48,72 @@ def add_arguments(self, parser): ), default=0, ) + parser.add_argument( + "--label", + action="append", + dest="labels", + default=list(), + help=( + "Filter projects by the provided label. " + "Multiple labels can be provided by using this argument multiple times." + ), + ) + parser.add_argument( + "--pipeline", + action="append", + dest="pipelines", + default=list(), + help=( + "Filter projects by the provided pipeline name. " + "Multiple pipeline name can be provided by using this argument multiple" + " times." + ), + ) parser.add_argument( "--no-input", action="store_false", dest="interactive", help="Do not prompt the user for input of any kind.", ) + parser.add_argument( + "--dry-run", + action="store_true", + help=( + "Do not delete any projects; just print the ones that would be flushed." + ), + ) def handle(self, *inputs, **options): verbosity = options["verbosity"] retain_days = options["retain_days"] - projects = Project.objects.all() + labels = options["labels"] + pipelines = options["pipelines"] + dry_run = options["dry_run"] + projects = Project.objects.order_by("-created_date").distinct() if retain_days: cutoff_date = timezone.now() - datetime.timedelta(days=retain_days) projects = projects.filter(created_date__lt=cutoff_date) + if labels: + projects = projects.filter(labels__name__in=labels) + + if pipelines: + projects = projects.filter(runs__pipeline_name__in=pipelines) + projects_count = projects.count() if projects_count == 0: if verbosity > 0: - self.stdout.write("No projects to remove.") + self.stdout.write("No projects to delete.") sys.exit(0) + if dry_run: + msg = self.style.WARNING(f"{projects_count} projects would be deleted:") + self.stdout.write(msg) + self.stdout.write("\n".join([f"- {project.name}" for project in projects])) + return + # sys.exit(0) + if options["interactive"]: confirm = input( f"You have requested the deletion of {projects_count} " diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index d40d28a43c..6dde802ff6 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -49,8 +49,23 @@ def make_project(name=None, **data): + """ + Create and return a Project instance. + Labels can be provided using the labels=["labels1", "labels2"] argument. + """ name = name or str(uuid.uuid4())[:8] - return Project.objects.create(name=name, **data) + pipelines = data.pop("pipelines", []) + labels = data.pop("labels", []) + + project = Project.objects.create(name=name, **data) + + for pipeline in pipelines: + project.add_pipeline(pipeline) + + if labels: + project.labels.add(*labels) + + return project def make_resource_file(project, path, **data): diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 1b3cf8f730..aba2920148 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -319,7 +319,7 @@ def test_scanpipe_management_command_batch_create_global_webhook( def test_scanpipe_management_command_add_input_file(self): out = StringIO() - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") parent_path = Path(__file__).parent options = [ "--input-file", @@ -346,7 +346,7 @@ def test_scanpipe_management_command_add_input_file(self): call_command("add-input", *options, stdout=out) def test_scanpipe_management_command_add_input_url(self): - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") options = [ "--input-url", "https://example.com/archive.zip", @@ -368,7 +368,7 @@ def test_scanpipe_management_command_add_input_url(self): def test_scanpipe_management_command_add_input_copy_codebase(self): out = StringIO() - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") options = ["--copy-codebase", "non-existing", "--project", project.name] expected = "non-existing not found" @@ -394,7 +394,7 @@ def test_scanpipe_management_command_add_input_copy_codebase(self): def test_scanpipe_management_command_add_pipeline(self): out = StringIO() - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") pipelines = [ self.pipeline_name, @@ -431,7 +431,7 @@ def test_scanpipe_management_command_add_pipeline(self): def test_scanpipe_management_command_add_webhook(self): out = StringIO() - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") options = ["https://example.com/webhook"] expected = "the following arguments are required: --project" @@ -486,7 +486,7 @@ def test_scanpipe_management_command_show_pipeline(self): "analyze_root_filesystem_or_vm_image", ] - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") for pipeline_name in pipeline_names: project.add_pipeline(pipeline_name) @@ -513,7 +513,7 @@ def test_scanpipe_management_command_show_pipeline(self): self.assertEqual(expected, out.getvalue()) def test_scanpipe_management_command_execute(self): - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") options = ["--project", project.name] out = StringIO() @@ -557,7 +557,7 @@ def test_scanpipe_management_command_execute(self): self.assertEqual("", run3.task_output) def test_scanpipe_management_command_execute_project_function(self): - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") expected = "No pipelines to run on project my_project" with self.assertRaisesMessage(CommandError, expected): @@ -584,7 +584,7 @@ def test_scanpipe_management_command_execute_project_function(self): self.assertIsNone(returned_value) def test_scanpipe_management_command_status(self): - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") run = project.add_pipeline(self.pipeline_name) options = ["--project", project.name, "--no-color"] @@ -657,9 +657,9 @@ def test_scanpipe_management_command_list_pipelines(self): self.assertIn("(addon)", output) def test_scanpipe_management_command_list_project(self): - project1 = Project.objects.create(name="project1") - project2 = Project.objects.create(name="project2") - project3 = Project.objects.create(name="archived", is_archived=True) + project1 = make_project(name="project1") + project2 = make_project(name="project2") + project3 = make_project(name="archived", is_archived=True) options = [] out = StringIO() @@ -686,7 +686,7 @@ def test_scanpipe_management_command_list_project(self): self.assertIn(project3.name, output) def test_scanpipe_management_command_output(self): - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") make_package(project, package_url="pkg:generic/name@1.0") out = StringIO() @@ -754,7 +754,7 @@ def test_scanpipe_management_command_output(self): self.assertIn('"specVersion": "1.5",', out_value) def test_scanpipe_management_command_delete_project(self): - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") work_path = project.work_path self.assertTrue(work_path.exists()) @@ -770,7 +770,7 @@ def test_scanpipe_management_command_delete_project(self): self.assertFalse(work_path.exists()) def test_scanpipe_management_command_archive_project(self): - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") (project.input_path / "input_file").touch() (project.codebase_path / "codebase_file").touch() self.assertEqual(1, len(Project.get_root_content(project.input_path))) @@ -797,7 +797,7 @@ def test_scanpipe_management_command_archive_project(self): self.assertEqual(0, len(Project.get_root_content(project.codebase_path))) def test_scanpipe_management_command_reset_project(self): - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") project.add_pipeline("analyze_docker_image") CodebaseResource.objects.create(project=project, path="filename.ext") DiscoveredPackage.objects.create(project=project) @@ -833,8 +833,8 @@ def test_scanpipe_management_command_reset_project(self): self.assertEqual(0, len(Project.get_root_content(project.codebase_path))) def test_scanpipe_management_command_flush_projects(self): - project1 = Project.objects.create(name="project1") - project2 = Project.objects.create(name="project2") + project1 = make_project("project1") + project2 = make_project("project2") ten_days_ago = timezone.now() - datetime.timedelta(days=10) project2.update(created_date=ten_days_ago) @@ -846,7 +846,14 @@ def test_scanpipe_management_command_flush_projects(self): self.assertEqual(expected, out_value) self.assertEqual(project1, Project.objects.get()) - Project.objects.create(name="project2") + make_project("project2") + out = StringIO() + options = ["--no-color", "--no-input", "--dry-run"] + call_command("flush-projects", *options, stdout=out) + out_value = out.getvalue().strip() + expected = "2 projects would be deleted:\n- project2\n- project1" + self.assertEqual(expected, out_value) + out = StringIO() options = ["--no-color", "--no-input"] call_command("flush-projects", *options, stdout=out) @@ -854,6 +861,43 @@ def test_scanpipe_management_command_flush_projects(self): expected = "2 projects and their related data have been removed." self.assertEqual(expected, out_value) + def test_scanpipe_management_command_flush_projects_filters(self): + label1 = "label1" + label2 = "label2" + make_project("project1", labels=[label1]) + make_project("project2", labels=[label1, label2]) + make_project("project3", pipelines=["scan_single_package"]) + + base_options = ["--no-color", "--no-input", "--dry-run"] + + out = StringIO() + options = base_options + ["--label", label1] + call_command("flush-projects", *options, stdout=out) + out_value = out.getvalue().strip() + expected = "2 projects would be deleted:\n- project2\n- project1" + self.assertEqual(expected, out_value) + + out = StringIO() + options = base_options + ["--label", label2] + call_command("flush-projects", *options, stdout=out) + out_value = out.getvalue().strip() + expected = "1 projects would be deleted:\n- project2" + self.assertEqual(expected, out_value) + + out = StringIO() + options = base_options + ["--label", label1, "--label", label2] + call_command("flush-projects", *options, stdout=out) + out_value = out.getvalue().strip() + expected = "2 projects would be deleted:\n- project2\n- project1" + self.assertEqual(expected, out_value) + + out = StringIO() + options = base_options + ["--pipeline", "scan_single_package"] + call_command("flush-projects", *options, stdout=out) + out_value = out.getvalue().strip() + expected = "1 projects would be deleted:\n- project3" + self.assertEqual(expected, out_value) + def test_scanpipe_management_command_create_user(self): out = StringIO() @@ -1123,7 +1167,7 @@ def test_scanpipe_management_command_purldb_scan_queue_worker_continue_after_fai ) def test_scanpipe_management_command_check_compliance(self): - project = Project.objects.create(name="my_project") + project = make_project(name="my_project") out = StringIO() options = ["--project", project.name] @@ -1169,9 +1213,8 @@ def test_scanpipe_management_command_check_compliance(self): self.assertEqual(expected, out_value) def test_scanpipe_management_command_report(self): - project1 = make_project("project1") label1 = "label1" - project1.labels.add(label1) + project1 = make_project("project1", labels=[label1]) make_resource_file(project1, path="file.ext", status=flag.REQUIRES_REVIEW) make_project("project2")