diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 9d60ac1dc4..fa481fa95f 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -83,6 +83,15 @@ def steps(cls): cls.find_grammar_packages, cls.map_grammar_to_class, cls.map_jar_to_grammar_source, + cls.find_groovy_packages, + cls.map_groovy_to_class, + cls.map_jar_to_groovy_source, + cls.find_aspectj_packages, + cls.map_aspectj_to_class, + cls.map_jar_to_aspectj_source, + cls.find_clojure_packages, + cls.map_clojure_to_class, + cls.map_jar_to_clojure_source, cls.find_xtend_packages, cls.map_xtend_to_class, cls.map_javascript, @@ -208,7 +217,7 @@ def find_scala_packages(self): @optional_step("Scala") def map_scala_to_class(self): - """Map a .class compiled file to its .java source.""" + """Map a .class compiled file to its .scala source.""" d2d.map_jvm_to_class( project=self.project, jvm_lang=jvm.ScalaLanguage, logger=self.log ) @@ -222,14 +231,14 @@ def map_jar_to_scala_source(self): @optional_step("Kotlin") def find_kotlin_packages(self): - """Find the java package of the .java source files.""" + """Find the java package of the kotlin source files.""" d2d.find_jvm_packages( project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log ) @optional_step("Kotlin") def map_kotlin_to_class(self): - """Map a .class compiled file to its .java source.""" + """Map a .class compiled file to its kotlin source.""" d2d.map_jvm_to_class( project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log ) @@ -262,6 +271,69 @@ def map_jar_to_grammar_source(self): project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log ) + @optional_step("Groovy") + def find_groovy_packages(self): + """Find the package of the .groovy source files.""" + d2d.find_jvm_packages( + project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log + ) + + @optional_step("Groovy") + def map_groovy_to_class(self): + """Map a .class compiled file to its .groovy source.""" + d2d.map_jvm_to_class( + project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log + ) + + @optional_step("Groovy") + def map_jar_to_groovy_source(self): + """Map .jar files to their related source directory.""" + d2d.map_jar_to_jvm_source( + project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log + ) + + @optional_step("AspectJ") + def find_aspectj_packages(self): + """Find the package of the .aj source files.""" + d2d.find_jvm_packages( + project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log + ) + + @optional_step("AspectJ") + def map_aspectj_to_class(self): + """Map a .class compiled file to its .aj source.""" + d2d.map_jvm_to_class( + project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log + ) + + @optional_step("AspectJ") + def map_jar_to_aspectj_source(self): + """Map .jar files to their related source directory.""" + d2d.map_jar_to_jvm_source( + project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log + ) + + @optional_step("Clojure") + def find_clojure_packages(self): + """Find the package of the .clj source files.""" + d2d.find_jvm_packages( + project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log + ) + + @optional_step("Clojure") + def map_clojure_to_class(self): + """Map a .class compiled file to its .clj source.""" + d2d.map_jvm_to_class( + project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log + ) + + @optional_step("Clojure") + def map_jar_to_clojure_source(self): + """Map .jar files to their related source directory.""" + d2d.map_jar_to_jvm_source( + project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log + ) + @optional_step("Xtend") def find_xtend_packages(self): """Find the java package of the xtend source files.""" diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index a428400129..c8e3b64294 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -239,8 +239,8 @@ def map_jvm_to_class(project, jvm_lang: jvm.JvmLanguage, logger=None): if logger: logger( - f"Mapping {to_resource_count:,d} .class resources to " - f"{from_resource_count:,d} {jvm_lang.source_extensions}" + f"Mapping {to_resource_count:,d} .class (or other deployed file) " + f"resources to {from_resource_count:,d} {jvm_lang.source_extensions}" ) # build an index using from-side fully qualified class file names diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index d965feae84..05ea09ca44 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -112,6 +112,14 @@ class EcosystemConfig: "*kotlin-project-structure-metadata.json", ], ), + "Groovy": EcosystemConfig( + ecosystem_option="Groovy", + matchable_package_extensions=[".jar", ".war"], + matchable_resource_extensions=[".class"], + deployed_resource_path_exclusions=[ + "*META-INF/*", + ], + ), "JavaScript": EcosystemConfig( ecosystem_option="JavaScript", matchable_resource_extensions=[ diff --git a/scanpipe/pipes/jvm.py b/scanpipe/pipes/jvm.py index b071d9dcd4..ae7b5b9203 100644 --- a/scanpipe/pipes/jvm.py +++ b/scanpipe/pipes/jvm.py @@ -192,12 +192,39 @@ class JavaLanguage(JvmLanguage): class ScalaLanguage(JvmLanguage): name = "scala" source_extensions = (".scala",) - binary_extensions = (".class",) + binary_extensions = (".class", ".tasty") source_package_attribute_name = "scala_package" package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") binary_map_type = "scala_to_class" +class GroovyLanguage(JvmLanguage): + name = "groovy" + source_extensions = (".groovy",) + binary_extensions = (".class",) + source_package_attribute_name = "groovy_package" + package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") + binary_map_type = "groovy_to_class" + + +class AspectJLanguage(JvmLanguage): + name = "aspectj" + source_extensions = (".aj",) + binary_extensions = (".class",) + source_package_attribute_name = "aspectj_package" + package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") + binary_map_type = "aspectj_to_class" + + +class ClojureLanguage(JvmLanguage): + name = "clojure" + source_extensions = (".clj",) + binary_extensions = (".class",) + source_package_attribute_name = "clojure_package" + package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") + binary_map_type = "clojure_to_class" + + class KotlinLanguage(JvmLanguage): name = "kotlin" source_extensions = (".kt", ".kts") diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 4d8433498e..73db8c0c9e 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -372,7 +372,7 @@ def test_scanpipe_pipes_d2d_map_java_to_class(self): self.project1, logger=buffer.write, jvm_lang=jvm.JavaLanguage ) - expected = "Mapping 3 .class resources to 2 ('.java',)" + expected = "Mapping 3 .class (or other deployed file) resources to 2 ('.java',)" self.assertIn(expected, buffer.getvalue()) self.assertEqual(2, self.project1.codebaserelations.count()) @@ -432,7 +432,7 @@ def test_scanpipe_pipes_d2d_map_java_to_class_with_java_in_deploy(self): d2d.map_jvm_to_class( self.project1, logger=buffer.write, jvm_lang=jvm.JavaLanguage ) - expected = "Mapping 1 .class resources to 1 ('.java',)" + expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.java',)" self.assertIn(expected, buffer.getvalue()) def test_scanpipe_pipes_d2d_map_grammar_to_class(self): @@ -452,7 +452,9 @@ def test_scanpipe_pipes_d2d_map_grammar_to_class(self): self.project1, logger=buffer.write, jvm_lang=jvm.GrammarLanguage ) - expected = "Mapping 1 .class resources to 1 ('.g', '.g4')" + expected = ( + "Mapping 1 .class (or other deployed file) resources to 1 ('.g', '.g4')" + ) self.assertIn(expected, buffer.getvalue()) self.assertEqual(1, self.project1.codebaserelations.count()) @@ -480,7 +482,9 @@ def test_scanpipe_pipes_d2d_map_xtend_to_class(self): self.project1, logger=buffer.write, jvm_lang=jvm.XtendLanguage ) - expected = "Mapping 1 .class resources to 1 ('.xtend',)" + expected = ( + "Mapping 1 .class (or other deployed file) resources to 1 ('.xtend',)" + ) self.assertIn(expected, buffer.getvalue()) self.assertEqual(1, self.project1.codebaserelations.count()) @@ -569,6 +573,124 @@ def test_scanpipe_pipes_d2d_map_jar_to_java_source(self): self.assertEqual(from2, relation.from_resource) self.assertEqual(to_jar, relation.to_resource) + def test_scanpipe_pipes_d2d_map_groovy_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/project/test.groovy", + extra_data={"groovy_package": "project"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/project/test.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.GroovyLanguage + ) + + expected = ( + "Mapping 1 .class (or other deployed file) resources to 1 ('.groovy',)" + ) + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(1, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("groovy_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + def test_scanpipe_pipes_d2d_map_aspectj_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/project/test.aj", + extra_data={"aspectj_package": "project"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/project/test.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.AspectJLanguage + ) + + expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.aj',)" + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(1, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("aspectj_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + def test_scanpipe_pipes_d2d_map_clojure_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/project/test.clj", + extra_data={"clojure_package": "project"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/project/test.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.ClojureLanguage + ) + + expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.clj',)" + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(1, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("clojure_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + def test_scanpipe_pipes_d2d_map_scala_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/tastyquery/Annotations.scala", + extra_data={"scala_package": "tastyquery"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/tastyquery/Annotations.tasty", + ) + + to2 = make_resource_file( + self.project1, + path="to/tastyquery/Annotations.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.ScalaLanguage + ) + + expected = ( + "Mapping 2 .class (or other deployed file) resources to 1 ('.scala',)" + ) + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(2, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("scala_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + r2 = self.project1.codebaserelations.get(to_resource=to2, from_resource=from1) + self.assertEqual("scala_to_class", r2.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r2.extra_data) + def test_scanpipe_pipes_d2d_map_jar_to_scala_source(self): from1 = make_resource_file( self.project1,