Merge pull request #15 from FontysVenlo/improve-comments

Improve comments

Author: Bonajo

codestripper/cli.py

@@ -13,8 +13,8 @@ def add_commandline_arguments(parser: argparse.ArgumentParser) -> None:
     # Add optional arguments
     parser.add_argument("-x", "--exclude", action="append",
                         help="files to include for code stripping (glob)", default=[])
-    parser.add_argument("-c", "--comment", action="store",
-                        help="comment symbol(s) for the given language", default="//")
+    parser.add_argument("-c", "--comment", action="append",
+                        help="comment symbol(s) for the given language, usage: <extension>:<comment> (e.g. .java://")
     parser.add_argument("-v", "--verbosity", action="count", help="increase output verbosity", default=0)
     parser.add_argument("-o", "--output", action="store",
                         help="output directory to store the stripped files", default="out")
@@ -44,4 +44,6 @@ def main() -> None:
     cwd = get_working_directory(args.working_directory)
     files = FileUtils(args.include, args.exclude, cwd, args.recursive, logger_name).get_matching_files()
     # Strip all the files
-    strip_files(files, cwd, args.comment, args.output, args.dry_run, args.fail_on_error)
+
+    strip_files(files, cwd, comments=args.comment, output=args.output, dry_run=args.dry_run,
+                fail_on_error=args.fail_on_error)

codestripper/code_stripper.py

@@ -2,19 +2,29 @@
 import os.path
 import shutil
 from pathlib import Path
-from typing import Union, Iterable, List
+from typing import Union, Iterable, List, Optional
 
 from codestripper.errors import InvalidTagError, TokenizerError
 from codestripper.tags import IgnoreFileError
 from codestripper.tags.tag import Tag, RangeTag
 from codestripper.tokenizer import Tokenizer
 from codestripper.utils import get_working_directory
+from codestripper.utils.comments import comments_mapping, Comment
 
 logger = logging.getLogger("codestripper")
 
 
-def strip_files(files: Iterable[str], working_directory: Union[str, None] = None, comment: str = "//",
+def strip_files(files: Iterable[str], working_directory: Union[str, None] = None, * ,comments: Optional[List[str]] = None,
                 output: Union[Path, str] = "out", dry_run: bool = False, fail_on_error: bool = False) -> List[str]:
+
+    if comments is not None:
+        for comment in comments:
+            parts = comment.split(":")
+            if len(parts) == 2:
+                comments_mapping[parts[0]] = Comment(parts[1])
+            else:
+                comments_mapping[parts[0]] = Comment(parts[1], parts[2])
+
     cwd = get_working_directory(working_directory)
     out = os.path.join(os.getcwd(), output)
     if os.path.isdir(out):
@@ -26,7 +36,14 @@ def strip_files(files: Iterable[str], working_directory: Union[str, None] = None
             content = handle.read()
         if content is not None:
             try:
-                stripped = CodeStripper(content, comment).strip()
+                _, file_extension = os.path.splitext(file)
+                file_extension = file_extension.lower()
+                if not file_extension in comments_mapping:
+                    logger.error(f"Unknown extension: '{file_extension}', "
+                                 f"please specify which comment to use for this file extension.")
+                    continue
+                com = comments_mapping[file_extension]
+                stripped = CodeStripper(content, com).strip()
             except IgnoreFileError:
                 logger.info(f"File '{file}' is ignored, because of ignore tag")
                 continue
@@ -50,7 +67,7 @@ def strip_files(files: Iterable[str], working_directory: Union[str, None] = None
 
 class CodeStripper:
 
-    def __init__(self, content: str, comment: str) -> None:
+    def __init__(self, content: str, comment: Comment) -> None:
         self.content = content
         self.comment = comment
 

codestripper/tags/add.py

@@ -4,7 +4,7 @@
 
 
 class AddTag(SingleTag):
-    regex = r'cs:add:(.*?)$'
+    regex = r'cs:add:(.*)?'
 
     def __init__(self, data: TagData) -> None:
         super().__init__(data)

codestripper/tags/legacy.py

@@ -4,7 +4,7 @@
 
 
 class LegacyOpenTag(RangeOpenTag):
-    regex = r'Start Solution::replacewith::(.*?)$'
+    regex = r'Start Solution::replacewith::(.*)'
 
     def __init__(self, data: TagData) -> None:
         super().__init__(LegacyRangeTag, data)
@@ -20,7 +20,7 @@ def execute(self, content: str) -> Union[str, None]:
 
 
 class LegacyCloseTag(RangeCloseTag):
-    regex = r'End Solution::replacewith::(.*?)$'
+    regex = r'End Solution::replacewith::(.*)'
 
     def __init__(self, data: TagData) -> None:
         super().__init__(LegacyRangeTag, data)

codestripper/tags/remove.py

@@ -3,7 +3,7 @@
 
 class RemoveTag(SingleTag):
 
-    regex = r'cs:remove\s*?$'
+    regex = r'cs:remove(?!:)(.*)?'
 
     def __init__(self, data: TagData):
         super().__init__(data)

codestripper/tags/replace.py

@@ -2,7 +2,7 @@
 
 
 class ReplaceTag(SingleTag):
-    regex = r'cs:replace:(.*?)$'
+    regex = r'cs:replace:(.*?)'
 
     def __init__(self, data: TagData) -> None:
         super().__init__(data)

codestripper/tags/tag.py

@@ -2,6 +2,8 @@
 from dataclasses import dataclass
 from typing import Type, Union, List, Pattern, Iterable, Optional
 
+from codestripper.utils.comments import Comment
+
 
 @dataclass
 class TagData:
@@ -13,7 +15,7 @@ class TagData:
     regex_end: int
     parameter_start: int
     parameter_end: int
-    comment: str
+    comment: Comment
 
     def __repr__(self) -> str:
         return (f"{self.line}, line ({self.line_number}): {self.line_start}:{self.line_end},"

codestripper/tags/uncomment.py

@@ -5,14 +5,14 @@
 
 
 class UncommentOpenTag(RangeOpenTag):
-    regex = r'cs:uncomment:start.*?$'
+    regex = r'cs:uncomment:start(.*)?'
 
     def __init__(self, data: TagData) -> None:
         super().__init__(UncommentRangeTag, data)
 
 
 class UncommentCloseTag(RangeCloseTag):
-    regex = 'cs:uncomment:end.*?$'
+    regex = 'cs:uncomment:end(.*)?'
 
     def __init__(self, data: TagData) -> None:
         super().__init__(UncommentRangeTag, data)
@@ -27,7 +27,7 @@ def __init__(self, open_tag: RangeOpenTag, close_tag: RangeCloseTag):
     def execute(self, content: str) -> Union[str, None]:
         if UncommentRangeTag.regex is None:
             whitespace = r"(?P<whitespace>\s*)"
-            UncommentRangeTag.regex = re.compile(f"{whitespace}{self.open_tag.data.comment}")
+            UncommentRangeTag.regex = re.compile(f"{whitespace}{self.open_tag.data.comment.open}")
         range = content[self.start:self.end]
         replacement = UncommentRangeTag.regex.sub(r"\g<whitespace>", range)
         return replacement

codestripper/tokenizer.py

@@ -5,6 +5,7 @@
 from codestripper.tags import ReplaceTag, UncommentCloseTag, IgnoreFileTag, RemoveOpenTag, RemoveCloseTag, \
     UncommentOpenTag, LegacyOpenTag, LegacyCloseTag, RemoveTag, AddTag
 from codestripper.tags.tag import SingleTag, Tag, RangeOpenTag, RangeCloseTag, RangeTag, TagData
+from codestripper.utils.comments import Comment
 
 default_tags: Set[Type[SingleTag]] = {
     IgnoreFileTag,
@@ -25,30 +26,36 @@
 CreateTagMapping = Dict[str, CreateTagLambda]
 
 
-def calculate_mappings(tags: Set[Type[SingleTag]], comment: str) -> Tuple[CreateTagMapping, Pattern]:
+def calculate_mappings(tags: Set[Type[SingleTag]], comment: Comment) -> Tuple[CreateTagMapping, Pattern]:
     strings = [r"(?P<newline>\n)"]
     mappings = {}
     for tag in tags:
         name = f"{tag.__name__}"
         mappings[name] = lambda data, constructor=tag: constructor(data)
-        strings.append(f"(?P<{name}>{comment}{tag.regex})")
+        reg = f"(?P<{name}>{re.escape(comment.open)}{tag.regex})"
+        if comment.close is not None:
+            reg += re.escape(comment.close)
+        strings.append(reg)
     regex = re.compile("|".join(strings), flags=re.MULTILINE)
     return mappings, regex  # type: ignore
 
 
 class Tokenizer:
+    mapping_cache: Dict[str, Tuple[CreateTagMapping, Pattern]] = {}
     mappings: CreateTagMapping = {}
     regex: Pattern = re.compile("")
-    comment: str = ""
+    comment: Comment
 
-    def __init__(self, content: str, comment: str) -> None:
+    def __init__(self, content: str, comment: Comment) -> None:
         self.content = content
         self.ordered_tags: List[Tag] = []
         self.open_stack: List[RangeOpenTag] = []
         self.range_stack: Dict[int, Optional[List[Tag]]] = {}
-        if len(Tokenizer.mappings) == 0 or Tokenizer.comment != comment:
-            Tokenizer.mappings, Tokenizer.regex = calculate_mappings(default_tags, comment)
-            Tokenizer.comment = comment
+        Tokenizer.comment = comment
+        if not str(comment) in Tokenizer.mapping_cache:
+            Tokenizer.mapping_cache[str(comment)] = calculate_mappings(default_tags, comment)
+        Tokenizer.mappings = Tokenizer.mapping_cache[str(comment)][0]
+        Tokenizer.regex = Tokenizer.mapping_cache[str(comment)][1]
         self.group_count = self.regex.groups
 
     def tokenize(self) -> List[Tag]:

codestripper/utils/comments.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from typing import Optional, Dict
+
+
+@dataclass(frozen=True)
+class Comment:
+    open: str
+    close: Optional[str] = None
+
+comments_mapping: Dict[str, Comment] = {
+    ".java": Comment("//"),
+    ".cs": Comment("//"),
+    ".js": Comment("//"),
+    ".php": Comment("//"),
+    ".swift": Comment("//"),
+    ".xml": Comment("<!--", "-->"),
+    ".tex": Comment("%"),
+    ".m": Comment("%"),
+    ".sql": Comment("--"),
+    ".lua": Comment("--"),
+    ".ml": Comment("(*", "*)"),
+    ".r": Comment("#"),
+    ".py": Comment("#"),
+    ".ps1": Comment("#"),
+    ".rb": Comment("#")
+}