diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c6bc1f0..69ea838 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,46 +9,31 @@ on: jobs: test: runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8, 3.9, "3.10"] steps: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} + uses: pdm-project/setup-pdm@v4 - name: Install dependencies run: | + set +x python -m pip install --upgrade pip - pip install tox - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pdm install -G test - name: Run tests run: | - tox + pdm run test publish: runs-on: ubuntu-latest needs: test + # Only publish on master + if: github.ref == 'refs/heads/master' + permissions: + id-token: write steps: - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install publishing dependencies - if: github.ref == 'refs/heads/master' - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Install package dependencies - if: github.ref == 'refs/heads/master' - run: | - python -m pip install -r requirements.txt - - name: Build and publish - if: github.ref == 'refs/heads/master' + - name: Set up PDM + uses: pdm-project/setup-pdm@v4 + - name: Publish wheel env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* + PDM_PUBLISH_USERNAME: __token__ + PDM_PUBLISH_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: pdm publish diff --git a/README.md b/README.md index c97d1ff..448cb8a 100644 --- a/README.md +++ b/README.md @@ -24,28 +24,40 @@ grant agreement No. 82201 Please read the docs for more information. ## Quick start -The recommended way to install demeuk is to install it in a virtual -environment. +Demeuk supports Python versions 3.12 and up. +The recommended way to install demeuk is to use [pipx](https://pipx.pypa.io/stable/). ``` -# Create virtual environment -virtualenv -# Activate the virtual environment -source /bin/activate -pip3 install -r requirements.txt +pipx install demeuk --python /usr/bin/python3.14 ``` -Now you can run bin/demeuk.py: +Now you can invoke demeuk directly from the command-line from any directory: Examples: ``` - demeuk -i inputfile.tmp -o outputfile.dict -l droppedfile.txt - demeuk -i inputfile -o outputfile -j 24 -l logfile.log - demeuk -i inputfile.tmp -o outputfile.dict -l droppedfile.txt --leak - demeuk -i inputfile -o outputfile -j 24 -l logfile.log --leak-full - demeuk -i inputdir/*.txt -o outputfile.dict -l logfile.log - demeuk -o outputfile.dict -l logfile.log + demeuk -i inputfile.tmp -o outputfile.dict -l demeuk.log + demeuk -i inputfile -o outputfile -j 24 -l demeuk.log + demeuk -i inputfile.tmp -o outputfile.dict -l demeuk.log --leak + demeuk -i inputfile -o outputfile -j all -l demeuk.log --leak-full + demeuk -i inputdir/*.txt -o outputfile.dict -l demeuk.log ``` +Demeuk also works with pipes: +``` + cat wordlist | demeuk --leak-full --debug > list.out 2> list.log +``` + +## Development +To make changes to demeuk, you need to run it from the source Python files. +``` +git clone https://github.com/NetherlandsForensicInstitute/demeuk.git +cd demeuk +pipx install ./ --python /usr/bin/python3.14 +``` +Now you can run demeuk as in the examples. Note that the shortcut `demeuk` only updates after you +run `pipx upgrade demeuk`. + +You can install demeuk through PDM with `pdm install -p ./`, this circumvents this issue although you have to +run demeuk with `pdm run demeuk`. ## Docs The docs are available at: diff --git a/bin/demeuk.py b/bin/demeuk.py deleted file mode 100755 index 3aeb1e4..0000000 --- a/bin/demeuk.py +++ /dev/null @@ -1,1851 +0,0 @@ -#!/usr/bin/env python3 -r""" -.. code-block:: none - - Demeuk - a simple tool to clean up corpora - - Usage: - demeuk [options] - - Examples: - demeuk -i inputfile.tmp -o outputfile.dict -l logfile.txt - demeuk -i "inputfile*.txt" -o outputfile.dict -l logfile.txt - demeuk -i "inputdir/*" -o outputfile.dict -l logfile.txt - demeuk -i inputfile -o outputfile -j 24 - demeuk -i inputfile -o outputfile -c -e - demeuk -i inputfile -o outputfile --threads all - cat inputfile | demeuk --leak -j all | sort -u > outputfile - - Standard Options: - -i --input Specify the input file to be cleaned, or provide a glob pattern. - (default: stdin) - -o --output Specify the output file name. (default: stdout) - -l --log Optional, specify where the log file needs to be writen to (default: stderr) - -j --threads Optional, specify amount of threads to spawn. Specify the string 'all' to make - demeuk auto detect the amount of threads to start based on the CPU's - (default: all threads). - Note: threading will cost some setup time. Only speeds up for larger files. - --input-encoding Forces demeuk to decode the input using this encoding (default: en_US.UTF-8). - --output-encoding Forces demeuk to encoding the output using this encoding (default: en_US.UTF-8). - -v --verbose When set, printing some extra information to stderr. And will print the - lines containing errors to logfile. - --debug When set, the logfile will not only contain lines which caused an error, but - also line which were modified. - --progress Prints out the progress of the demeuk process. - -n --limit Limit the number of lines per thread. - -s --skip Skip amount of lines per thread. - --punctuation Use to set the punctuation that is use by options. Defaults to: - ! "#$%&'()*+,-./:;<=>?@[\]^_`{|}~ - --version Prints the version of demeuk. - - Separating Options: - -c --cut Specify if demeuk should split (default splits on ':'). Returns everything - after the delimiter. - --cut-before Specify if demeuk should return the string before the delimiter. - When cutting, demeuk by default returns the string after the delimiter. - -f --cut-fields Specifies the field to be returned, this is in the 'cut' language thus: - N N'th field, N- from N-th field to end line, N-M, from N-th field to M-th - field. -M from start to M-th field. - -d --delimiter Specify which delimiter will be used for cutting. Multiple delimiters can be - specified using ','. If the ',' is required for cutting, escape it with a - backslash. Only one delimiter can be used per line. - - Check modules (check if a line matches a specific condition): - --check-min-length Requires that entries have a minimal requirement of unicode chars - --check-max-length Requires that entries have a maximal requirement of unicode chars - --check-case Drop lines where the uppercase line is not equal to the lowercase line - --check-controlchar Drop lines containing control chars. - --check-email Drop lines containing e-mail addresses. - --check-hash Drop lines which are hashes. - --check-mac-address Drop lines which are MAC-addresses. - --check-uuid Drop lines which are UUID. - --check-non-ascii If a line contain a non ascii char e.g. ü or ç (or everything outside ascii - range) the line is dropped. - --check-replacement-character Drop lines containing replacement characters '�'. - --check-starting-with Drop lines starting with string, can be multiple strings. Specify multiple - with as comma-seperated list. - --check-ending-with Drop lines ending with string, can be multiple strings. Specify multiple - with as comma-seperated list. - --check-contains Drop lines containing string, can be multiple strings. Specify multiple - with as comma-seperated list. - --check-empty-line Drop lines that are empty or only contain whitespace characters - --check-regex Drop lines that do not match the regex. Regex is a comma seperated list of - regexes. Example: [a-z]{1,8},[0-9]{1,8} - --check-min-digits Require that entries contain at least digits - (following the Python definition of a digit, - see https://docs.python.org/3/library/stdtypes.html#str.isdigit) - --check-max-digits Require that entries contain at most digits - (following the Python definition of a digit, - see https://docs.python.org/3/library/stdtypes.html#str.isdigit) - --check-min-uppercase Require that entries contain at least uppercase letters - (following the Python definition of uppercase, - see https://docs.python.org/3/library/stdtypes.html#str.isupper) - --check-max-uppercase Require that entries contain at most uppercase letters - (following the Python definition of uppercase, - see https://docs.python.org/3/library/stdtypes.html#str.isupper) - --check-min-specials Require that entries contain at least specials - (a special is defined as a non whitespace character which is not alphanumeric, - following the Python definitions of both, - see https://docs.python.org/3/library/stdtypes.html#str.isspace - and https://docs.python.org/3/library/stdtypes.html#str.isalnum) - --check-max-specials Require that entries contain at most specials - (a special is defined as a non whitespace character which is not alphanumeric, - following the Python definitions of both, - see https://docs.python.org/3/library/stdtypes.html#str.isspace - and https://docs.python.org/3/library/stdtypes.html#str.isalnum) - - - Modify modules (modify a line in place): - --hex Replace lines like: $HEX[41424344] with ABCD. - --html Replace lines like: şifreyok with şifreyok. - --html-named Replace lines like: &#alpha; Those structures are more like passwords, so - be careful to enable this option. - --lowercase Replace line like 'This Test String' to 'this test string' - --title-case Replace line like 'this test string' to 'This Test String' - --umlaut Replace lines like ko"ffie with an o with an umlaut. - --mojibake Fixes mojibakes, which means lines like SmˆrgÂs will be fixed to Smörgås. - --encode Enables guessing of encoding, based on chardet and custom implementation. - --tab Enables replacing tab char with ':', sometimes leaks contain both ':' and '\t'. - --newline Enables removing newline characters (\r\n) from end and beginning of lines. - --non-ascii Replace non ascii char with their replacement letters. For example ü - becomes u, ç becomes c. - --trim Enables removing newlines representations from end and beginning. Newline - representations detected are '\\n', '\\r', '\n', '\r', '
', and '
'. - --transliterate Transliterate a strings, for example "ipsum" becomes "իպսում". The following - languages are supported: ka, sr, l1, ru, mn, uk, mk, el, hy and bg. - - Add modules (Modify a line, but keep the original as well): - --add-lower If a line contains a capital letter this will add the lower case variant - --add-first-upper If a line does not contain a capital letter this will add the capital variant - --add-title-case Add a line like 'this test string' also as a 'This Test String' - --add-latin-ligatures If a line contains a single ligatures of a latin letter (such as ij), the line - is correct but the original line contain the ligatures is also added to output. - --add-split split on known chars like - and . and add those to the final dictionary. - --add-umlaut In some spelling dicts, umlaut are sometimes written as: o" or i" and not as - one char. - --add-without-punctuation If a line contains punctuations, a variant will be added without the - punctuations - - Remove modules (remove specific parts of a line): - --remove-strip-punctuation Remove starting and trailing punctuation - --remove-punctuation Remove all punctuation in a line - --remove-email Enable email filter, this will catch strings like - 1238661:test@example.com:password - Macro modules: - -g --googlengram When set, demeuk will strip universal pos tags: like _NOUN_ or _ADJ - --leak When set, demeuk will run the following modules: - mojibake, encode, newline, check-controlchar - This is recommended when working with leaks and was the default bevarior in - demeuk version 3.11.0 and below. - --leak-full When set, demeuk will run the following modules: - mojibake, encode, newline, check-controlchar, - hex, html, html-named, - check-hash, check-mac-address, check-uuid, check-email, - check-replacement-character, check-empty-line -""" -from binascii import hexlify, unhexlify -from collections import deque -from glob import glob -from html import unescape -from inspect import cleandoc -from locale import LC_ALL, setlocale, getlocale -from math import ceil -from multiprocessing import cpu_count -from os import name as os_name -if os_name == 'nt': - from multiprocessing.pool import ThreadPool as Pool -else: - from multiprocessing import Pool -from os import linesep, access, path, R_OK, F_OK, W_OK -from re import compile as re_compile -from re import search -from re import split as re_split -from re import sub -from signal import signal, SIGINT, SIG_IGN -from string import punctuation as string_punctuation -from time import sleep -from sys import stderr, stdin, stdout -from unicodedata import category - - -from chardet import detect -from docopt import docopt -from ftfy import fix_encoding -from ftfy.chardata import HTML_ENTITIES, HTML_ENTITY_RE -from ftfy.fixes import fix_latin_ligatures -from nltk import str2tuple -from nltk.tokenize import WhitespaceTokenizer -from tqdm import tqdm -from transliterate import translit -from unidecode import unidecode - - -version = '4.6.3' - -# Search from start to finish for the string $HEX[], with block of a-f0-9 with even number -# of hex chars. The first match group is repeated. -HEX_REGEX = re_compile(r"^\$(?:HEX|hex)\[((?:[0-9a-fA-F]{2})+)\]$") -EMAIL_REGEX = '.{1,64}@([a-zA-Z0-9_-]{1,63}\\.){1,3}[a-zA-Z]{2,6}' -HASH_HEX_REGEX = '^[a-fA-F0-9]+$' -MAC_REGEX = '^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$' -UUID_REGEX = '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' - -# Officiale bcrypt hashes hae a bit more fixed size, but saw some weird once: -# $2a$10$demo as example -HASH_BCRYPT_REGEX = '^\\$2[ayb]\\$[0-9]{1,}\\$[\\w\\.\\/]{4,}$' -# Crypt hashes can look a lot like passwords. We do two options here -# $1[$optional salt, max 16]$string of a-zA-Z0-9./ length 7 min till end of line -# $1$a-zA-Z0-9./ min length 12 to make sure we hit somthing like: a-zA-Z0-9./ -# this will cause string like $1$JAjdna./d to still be included. - -HASH_CRYPT_REGEX = '^\\$[1356]\\$[\\w\\.\\/]{12,}$' -HASH_CRYPT_SALT_REGEX = '^\\$[1356]\\$[\\w\\.\\/\\+]{,16}\\$[\\w\\.\\/]{6,}$' -HASH_PHPBB_REGEX = '^\\$[hH]\\$[\\w\\.\\/]{6,}$' -HASH_REGEX_LIST = [HASH_BCRYPT_REGEX, HASH_CRYPT_SALT_REGEX, HASH_CRYPT_REGEX, HASH_PHPBB_REGEX] - -TRIM_BLOCKS = ('\\\\n', '\\\\r', '\\n', '\\r', '
', '
') - -CHUNK_SIZE = 1024 * 1024 - - -def _unescape_fixup_named(match): - """ - Replace one matched HTML entity with the character it represents, - if possible. - - Based on: ftfy.fixes._unescape_fixup - """ - text = match.group(0) - if text in HTML_ENTITIES: - return HTML_ENTITIES[text] - else: - return text - - -def _unescape_fixup(match): - """ - Replace one matched HTML entity with the character it represents, - if possible. - - Based on: ftfy.fixes._unescape_fixup - """ - text = match.group(0) - if text.startswith('&#'): - unescaped = unescape(text) - - # If html.unescape only decoded part of the string, that's not what - # we want. The semicolon should be consumed. - if ';' in unescaped: - return text - else: - return unescaped - else: - return text - - -def clean_googlengram(line): - """Removes speechtags from line specific to the googlengram module - - Param: - line (unicode) - - Returns: - line (unicode) - """ - return_line = line.split("\t")[0] # Get the ngram, remove year, counter, etc - clean = [] - words = WhitespaceTokenizer().tokenize(return_line) - for word in words: - # in >1-grams transitions to specific tags are written as: - # The_ADJ _NOUN_ (meaning from The there is a transition to a noun - # We remove those - if word[0] != '_' and word[-1] != '_': - # Split the token and the tag based on the '_' - token, tag = str2tuple(word, '_') - # Punct will be added using rules. - if len(token) > 1: - if tag != 'PUNCT' or tag != '.' or tag != '': - clean.append(token) - elif token not in string_punctuation: - clean.append(token) - return_line = ' '.join(clean) - if return_line != line: - return True, return_line - else: - return False, line - - -def remove_email(line): - """Removes e-mail addresses from a line. - - Params: - line (unicode) - - Returns: - line (unicode) - """ - if '@' in line: - if search(f'{EMAIL_REGEX}(:|;)', line): - return True, sub(f'{EMAIL_REGEX}(:|;)', '', line) - return False, line - - -def add_lower(line): - """Returns if the upper case string is different from the lower case line - - Param: - line (unicode) - - Returns: - False if they are the same - Lowered string if they are not - """ - line_lower = line.lower() - if line != line_lower: - return line_lower - else: - return False - - -def add_first_upper(line): - """Returns the line with the first letter capitalized and all the others in lowercase. - - Param: - line (unicode) - - Returns: - False if they are the same - Capitalized string if they are not - """ - line_first_upper = line.capitalize() - if line != line_first_upper: - return line_first_upper - else: - return False - - -def add_title_case(line): - """Returns title case string where all the first letters are capitals and all others in lowercase. - - Param: - line (unicode) - - Returns: - False if they are the same - Title string if they are not - """ - line_title_case = line.title() - if line != line_title_case: - return line_title_case - else: - return False - - -def add_latin_ligatures(line): - """Returns the line cleaned of latin ligatures if there are any. - - Param: - line (unicode) - - Returns: - False if there are not any latin ligatures - Corrected line - """ - cleaned_line = fix_latin_ligatures(line) - if line != cleaned_line: - return cleaned_line - else: - return False - - -def add_without_punctuation(line, punctuation): - """Returns the line cleaned of punctuation. - - Param: - line (unicode) - - Returns: - False if there are not any punctuation - Corrected line - """ - cleaned_line = line.translate(str.maketrans('', '', punctuation)) - - if line != cleaned_line: - return cleaned_line - else: - return False - - -def clean_add_umlaut(line): - """Returns the line cleaned of incorrect umlauting - - Param: - line (unicode) - - Returns: - Corrected line - """ - cleaned_line = line - - umlaut_dict = { - 'a"': 'ä', - 'i"': 'ï', - 'o"': 'ö', - 'u"': 'ü', - 'e"': 'ë', - 'A"': 'Ä', - 'I"': 'Ï', - 'O"': 'Ö', - 'U"': 'Ü', - 'E"': 'Ë', - } - for letter in umlaut_dict.keys(): - cleaned_line = cleaned_line.replace(letter, umlaut_dict.get(letter)) - - if line != cleaned_line: - return True, cleaned_line - else: - return False, line - - -def remove_punctuation(line, punctuation): - """Returns the line without punctuation - - Param: - line (unicode) - punctuation (unicode) - - Returns: - line without start and end punctuation - """ - return_line = line.translate(str.maketrans('', '', punctuation)) - if return_line != line: - return True, return_line - else: - return False, line - - -def remove_strip_punctuation(line, punctuation): - """Returns the line without start and end punctuation - - Param: - line (unicode) - - Returns: - line without start and end punctuation - """ - return_line = line.strip(punctuation) - if return_line != line: - return True, return_line - else: - return False, line - - -def add_split(line, punctuation=(' ', '-', r'\.')): - """Split the line on the punctuation and return elements longer then 1 char. - - Param: - line (unicode) - - Returns: - split line - """ - for p in punctuation: - if p in line: - return [i for i in re_split('|'.join(punctuation), line) if len(i) > 1] - return False - - -def check_case(line, ignored_chars=(' ', "'", '-')): - """Checks if an uppercase line is equal to a lowercase line. - - Param: - line (unicode) - ignored_chars list(string) - - Returns: - true if uppercase line is equal to uppercase line - """ - for c in line: - c = str(c) - if c.lower() == c.upper(): - if c in ignored_chars: - continue - else: - return False, c - return True, None - - -def check_length(line, min=0, max=0): - """Does a length check on the line - - Params: - line (unicode) - min (int) - max (int) - - Returns: - true if length is ok - """ - status = True - if min and status: - status = len(line) >= min - if max and status: - status = len(line) < max - return status - - -def check_hash(line): - """Check if a line contains a hash - - Params: - line (unicode) - - Returns: - true if line does not contain hash - """ - if search(HASH_HEX_REGEX, line): - if len(line) in [32, 40, 64]: - return False - if len(line) > 0: - if line[0] == '$': - for hash_regex in HASH_REGEX_LIST: - if search(hash_regex, line): - return False - return True - - -def check_mac_address(line): - """Check if a line contains a MAC-address - - Params: - line (unicode) - - Returns: - true if line does not contain a MAC-address - """ - if search(MAC_REGEX, line): - return False - - return True - - -def check_email(line): - """Check if lines contain e-mail addresses with a simple regex - - Params: - line (unicode) - - Returns: - true is line does not contain email - """ - if search(EMAIL_REGEX, line): - return False - else: - return True - - -def check_non_ascii(line): - """Checks if a line contains a non ascii chars - - Params: - line (unicode) - - Returns: - true if line does not contain non ascii chars - """ - try: - line.encode('ascii') - return True - except UnicodeEncodeError: - return False - - -def check_character(line, character): - """Checks if a line contains a specific character - - Params: - line (unicode) - - Returns: - true if line does contain the specific character - - """ - if character in line: - return True - else: - return False - - -def check_starting_with(line, strings): - """Checks if a line start with a specific strings - - Params: - line (unicode) - strings[str] - - Returns: - true if line does start with one of the strings - - """ - for string in strings: - if line.startswith(string): - return True - return False - - -def check_uuid(line): - """Check if a line contains a UUID - - Params: - line (unicode) - - Returns: - true if line does not contain a UUID - """ - if search(UUID_REGEX, line): - return False - - return True - - -def check_ending_with(line, strings): - """Checks if a line ends with specific strings - - Params: - line (unicode) - strings[str] - - Returns: - true if line does end with one of the strings - - """ - for string in strings: - if line.endswith(string): - return True - return False - - -def check_contains(line, strings): - """Checks if a line does not contain specific strings - - Params: - line (unicode) - strings[str] - - Returns: - true if line does contain any one of the strings - - """ - for string in strings: - if string in line: - return True - return False - - -def check_empty_line(line): - """Checks if a line is empty or only contains whitespace chars - - Params: - line (unicode) - - Returns: - true of line is empty or only contains whitespace chars - """ - if line == '' or line.isspace(): - return True - return False - - -def clean_cut(line, delimiters, fields): - """Finds the first delimiter and returns the remaining string either after - or before the delimiter. - - Params: - line (unicode) - delimiters list(unicode) - fields (unicode) - - Returns: - line (unicode) - """ - for delimiter in delimiters: - if delimiter in line: - if '-' in fields: - start = fields.split('-')[0] - stop = fields.split('-')[1] - if start == '': - start = 1 - if stop == '': - stop = len(line) - fields = slice(int(start) - 1, int(stop)) - else: - fields = slice(int(fields) - 1, int(fields)) - return True, delimiter.join(line.split(delimiter)[fields]) - else: - return False, line - - -def clean_transliterate(line, language): - """Transliterate a string - - Params: - line (Unicode) - language (str) - - Returns: - line (Unicode) - """ - cleaned_line = translit(line, language, reversed=True) - if line != cleaned_line: - return True, cleaned_line - else: - return False, line - - -def clean_non_ascii(line): - """Replace non ascii chars with there ascii representation. - - Params: - line (Unicode) - - Returns: - line (Unicode) - """ - cleaned_line = unidecode(line) - if line != cleaned_line: - return True, cleaned_line - else: - return False, line - - -def clean_lowercase(line): - """Replace all capitals to lowercase - - Params: - line (Unicode) - - Returns: - line (Unicode) - - """ - cleaned_line = line.lower() - if line != cleaned_line: - return True, cleaned_line - else: - return False, line - - -def clean_title_case(line): - """Replace words to title word (uppercasing first letter) - - Params: - line (Unicode) - - Returns: - line (Unicode) - - """ - cleaned_line = line.title() - if line != cleaned_line: - return True, cleaned_line - else: - return False, line - - -def clean_trim(line): - """Delete leading and trailing character sequences representing a newline - from beginning end end of line. - - Params: - line (Unicode) - - Returns: - line (Unicode) - """ - cleaned_line = line - # Ensure removal of duplicated blocks - while True: - has_match = False - for x in TRIM_BLOCKS: - if cleaned_line.startswith(x): - cleaned_line = cleaned_line[len(x):] - has_match = True - - if cleaned_line.endswith(x): - cleaned_line = cleaned_line[:-len(x)] - has_match = True - - if has_match is False: - break - - if line != cleaned_line: - return True, cleaned_line - else: - return False, line - - -def clean_tab(line): - """Replace tab character with ':' greedy - - Params: - line (bytes) - - Returns: - line (bytes) - """ - if b'\x09' in line: - line = sub(b'\x09+', b'\x3a', line) - return True, line - else: - return False, line - - -def clean_hex(line): - """Converts strings like '$HEX[]' to proper binary - - Params: - line (bytes) - - Returns: - line (bytes) - """ - match = HEX_REGEX.search(line) - if match: - return True, unhexlify(match.group(1)) - else: - return False, line - - -def clean_html(line): - """Detects html encode chars and decodes them - - Params: - line (Unicode) - - Returns: - line (Unicode) - """ - return_line = HTML_ENTITY_RE.sub(_unescape_fixup, line) - if return_line != line: - return True, return_line - else: - return False, line - - -def clean_html_named(line): - """Detects named html encode chars and decodes them - - Params: - line (Unicode) - - Returns: - line (Unicode) - """ - return_line = HTML_ENTITY_RE.sub(_unescape_fixup_named, line) - if return_line != line: - return True, return_line - else: - return False, line - - -def clean_newline(line): - """Delete newline characters at start and end of line - - Params: - line (Unicode) - - Returns: - line (Unicode) - """ - return_line = line.strip('\r\n') - if return_line != line: - return True, return_line - else: - return False, line - - -def check_controlchar(line): - """Detects control chars, returns True when detected - - Params: - line (Unicode) - - Returns: - Status, String - """ - for c in line: - # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category - # Characters (they have meaning): - # Cc -> Control Char (End of stream) - # Cf -> Control flow (right to left) - # Non chars: - # Cn -> Not assigned - # Co -> Private use - # Cs -> Surrogate - if category(c) in ['Cc', 'Cf', 'Cn', 'Co', 'Cs']: - return True, c - return False, None - - -def check_regex(line, regex): - """Checks if a line matches a list of regexes - - Params: - line (unicode) - regex (list) - - Returns: - true if all regexes match - false if line does not match regex - """ - for regex in regex: - if search(regex, line): - continue - else: - return False - return True - - -def contains_at_least(line, bound, char_property): - """Check if the line contains at least `bound` characters with given property. - - Params: - line (unicode) - bound (int) - char_property (str -> bool) - - Returns: - true if at least `bound` characters match - false otherwise - """ - if bound == 0: - return True - - count = 0 - for char in line: - if char_property(char): - count += 1 - if count >= bound: - return True - return False - - -def contains_at_most(line, bound, char_property): - """Check if the line contains at most `bound` characters with given property. - - Params: - line (unicode) - bound (int) - char_property (str -> bool) - - Returns: - true if at most `bound` characters match - false otherwise - """ - count = 0 - for char in line: - if char_property(char): - count += 1 - if count > bound: - return False - return True - - -def try_encoding(line, encoding): - """Tries to decode a line using supplied encoding - - Params: - line (Byte): byte variable that will be decoded - encoding (string): the encoding to be tried - - Returns: - False if decoding failed - String if decoding worked - """ - try: - # Try to decode the line - line_decoded = line.decode(encoding) - # Some encoding will decoded almost any line, lets check if we have invalid chars. - # If we have invalid chars (except for tab like chars) we will fail - for c in line_decoded: - if category(c) in ['Cc', 'Cf', 'Cn', 'Co', 'Cs']: - if c == '\t' or c == '\f': - continue - else: - return False - return line_decoded - except UnicodeDecodeError: - return False - - -def clean_mojibake(line): - """Detects mojibake and tries to correct it. - Mojibake are string that are decoded incorrectly and then encoded incorrectly. - This results in strings like: único which should be único. - - Param: - line (str) - - Returns: - Cleaned string - """ - return_line = fix_encoding(line) - if return_line != line: - return True, return_line - else: - return False, line - - -def clean_encode(line, input_encoding): - """Detects and tries encoding - - Params: - line (bytes) - - Returns: - Decoded UTF-8 string - """ - # Try either a user set of encodings or the default encoding set. - # When using multiple encoding is it beter to have multibyte encodings before - # Single byte encodings. Also it is beter to not include iso encoding by default. - # https://en.wikipedia.org/wiki/Character_encoding#Common_character_encodings - # Input_encoding is by default [utf8] - for encoding in input_encoding: - line_decoded = try_encoding(line, encoding) - if line_decoded is not False: - break - # All other methods failed, lets run the detect library on the line and try to guess the encoding. - if line_decoded is False: - encode = detect(line) - if encode.get('encoding'): - try: - line_decoded = line.decode(encode['encoding']) - except (UnicodeDecodeError, LookupError) as e: # noqa F841 - return False, encode["encoding"] - else: - return False, 'Unknown' - # If we managed to get here, return decode line - return True, line_decoded - - -def clean_up(lines): - """Main clean loop, this calls all the other clean functions. - - Args: - line(bytes): Line to be cleaned up - - Returns: - (str(Decoded line), str(Failed line)) - """ - results = [] - log = [] - processed_lines = set() - work_queue = deque(lines) - - while work_queue: - line = work_queue.popleft() - - if line in processed_lines: - continue - processed_lines.add(line) - - # Check if the limit is set, if so minus 1 and if 0 is reached lets quit. - if type(config['limit']) is int: - if config['limit'] > 0: - config['limit'] -= 1 - else: - break - - # When stop is set all demeuking module will be skipped for this line. - stop = False - if config['debug']: - log.append(f'----BEGIN---- {hexlify(line)}{linesep}') - # Replace tab chars as ':' greedy - if config.get('tab') and not stop: - status, line = clean_tab(line) - if status and config['debug']: - log.append(f'Clean_tab; replaced tab characters; {line}{linesep}') - # Converting enoding to UTF-8 - if config.get('encode') and not stop: - status, line_decoded = clean_encode(line, config.get('input_encoding')) - if status is False: - log.append(f'Clean_encode; decoding error with {line_decoded}; {line}{linesep}') - stop = True - elif status is True and config['debug']: - log.append(f'Clean_encode; decoded line; {line_decoded}{linesep}') - else: - try: - line_decoded = line.decode(config.get('input_encoding')[0]) - if config['debug']: - log.append(f'Clean_up; decoded using input_encoding option; {line_decoded}{linesep}') - except (UnicodeDecodeError) as e: # noqa F841 - log.append(f'Clean_up; decoding error with unknown; {line}{linesep}') - stop = True - # From here it is expected that line is correctly decoded! - # Check if some lines contain a hex string like $HEX[41424344] - if config.get('hex') and not stop: - status, line_decoded = clean_hex(line_decoded) - if status: - # Lines contains hex, this function will return binary string, so add it back to - # our undecoded lines - work_queue.append(line_decoded) - if config['debug']: - log.append(f'Clean_hex; replaced $HEX[], added to queue and quiting; {line}{linesep}') - # Aborting future processing of this line. - stop = True - - # Check if there are html char in the line, decode them if there are - if config.get('html') and not stop: - status, line_decoded = clean_html(line_decoded) - if status: - # Line contains html string, because this can be binary data (linefeeds etc) - # convert back to binary string and add to queue again. - work_queue.append(line_decoded.encode()) - if config['debug']: - log.append(f'Clean_html; replaced html, added to queue and quiting; {line_decoded}{linesep}') - stop = True - - # Checks if there are any mojibakes inside the line - # You must mojibake before removing control chars! Some control chars - # are part of a valid mojibake. - if config.get('mojibake') and not stop: - status, line_decoded = clean_mojibake(line_decoded) - if status and config['debug']: - log.append(f'Clean_mojibake; found a mojibake; {line}{linesep}') - - # Delete leading and trailing newline characters - if config.get('newline') and not stop: - status, line_decoded = clean_newline(line_decoded) - if status and config['debug']: - log.append(f'Clean_newline; deleted newline characters; {line_decoded!r}{linesep}') - - # Checks if there are any control chars inside line - if config.get('check-controlchar') and not stop: - status, cc = check_controlchar(line_decoded) - if status: - # Control char detected - log.append(f'Check_controlchar; found controlchar {cc!r}; {line_decoded!r}{linesep}') - stop = True - - # Check if there are named html chars in the line - if config.get('html-named') and not stop: - status, line_decoded = clean_html_named(line_decoded) - if status and config['debug']: - log.append(f'Clean_html_named; found named html character; {line_decoded}{linesep}') - - # Delete leading and trailing character sequences representing a newline - if config.get('trim') and not stop: - status, line_decoded = clean_trim(line_decoded) - if status and config['debug']: - log.append(f'Clean_trim; found trim sequence; {line_decoded!r}{linesep}') - - # Should we do the cut? - if config.get('cut') and not stop: - status, line_decoded = clean_cut(line_decoded, config['delimiter'], config['cut-fields']) - if status and config['debug']: - log.append(f'Clean_cut; field cutted; {line_decoded}{linesep}') - - # Replace umlauts - if config.get('umlaut') and not stop: - status, line_decoded = clean_add_umlaut(line_decoded) - if status and config['debug']: - log.append(f'Clean_umlaut; umlaut replaced; {line_decoded}{linesep}') - - # Transliterate - if config.get('transliterate') and not stop: - status, line_decoded = clean_transliterate(line_decoded, config.get('transliterate')) - if status and config['debug']: - log.append(f'Clean_transliterate; translitatered; {line_decoded}{linesep}') - - # Replace non-ascii - if config.get('non-ascii') and not stop: - status, line_decoded = clean_non_ascii(line_decoded) - if status and config['debug']: - log.append(f'Clean_non_ascii; non-ascii replaced; {line_decoded}{linesep}') - - # Replace all letters with lowercase - if config.get('lowercase') and not stop: - status, line_decoded = clean_lowercase(line_decoded) - if status and config['verbose']: - log.append(f'Clean_lowercase; all capitals replaced; {line_decoded}{linesep}') - - # Replace first letter of a word to a uppercase letter - if config.get('title-case') and not stop: - status, line_decoded = clean_title_case(line_decoded) - if status and config['debug']: - log.append(f'Clean_title_case; non-ascii replaced; {line_decoded}{linesep}') - - # Should we remove emails? - if config.get('remove-email') and not stop: - status, line_decoded = remove_email(line_decoded) - if status and config['debug']: - log.append(f'Remove_email; email found; {line_decoded}{linesep}') - - if config.get('googlengram') and not stop: - status, line_decoded = clean_googlengram(line_decoded) - if status and config['debug']: - log.append(f'Clean_googlengram; tos found and removed; {line_decoded}{linesep}') - - if config.get('check-case') and not stop: - status, c = check_case(line_decoded) - if not status: - log.append(f'Check_case; dropped line because of {c}; {line_decoded}{linesep}') - stop = True - - if config.get('check-length') and not stop: - if not check_length(line_decoded, min=config['check-min-length'], max=config['check-max-length']): - log.append(f'Check_length; dropped line because of failed length check; {line_decoded}{linesep}') - stop = True - - if config.get('check-email') and not stop: - if not check_email(line_decoded): - log.append(f'Check_email; dropped line because found email; {line_decoded}{linesep}') - stop = True - - if config.get('check-hash') and not stop: - if not check_hash(line_decoded): - log.append(f'Check_hash; dropped line because found a hash; {line_decoded}{linesep}') - stop = True - - if config.get('check-mac-address') and not stop: - if not check_mac_address(line_decoded): - log.append(f'Check_mac_address; dropped line because found a MAC address; {line_decoded}{linesep}') - stop = True - - if config.get('check-non-ascii') and not stop: - if not check_non_ascii(line_decoded): - log.append(f'Check_non_ascii; dropped line because non ascii char found; {line_decoded}{linesep}') - stop = True - - if config.get('check-replacement-character') and not stop: - if check_character(line_decoded, '�'): - log.append(f'Check_replacement_character; dropped line because "�" found; {line_decoded}{linesep}') - stop = True - - if config.get('check-regex') and not stop: - if not check_regex(line_decoded, config.get('check-regex')): - log.append(f'Check_regex; dropped line because it does not match the regex; {line_decoded}{linesep}') - stop = True - - min_digits = config.get('check-min-digits') - if min_digits and not stop: - if not contains_at_least(line_decoded, min_digits, str.isdigit): - log.append(f'Check_min_digits; dropped line because it contains less than ' - f'{min_digits} digits; {line_decoded}{linesep}') - stop = True - - max_digits = config.get('check-max-digits') - if max_digits != float('inf') and not stop: - if not contains_at_most(line_decoded, max_digits, str.isdigit): - log.append(f'Check_max_digits; dropped line because it contains more than ' - f'{max_digits} digits; {line_decoded}{linesep}') - stop = True - - min_uppercase = config.get('check-min-uppercase') - if min_uppercase and not stop: - if not contains_at_least(line_decoded, min_uppercase, str.isupper): - log.append(f'Check_min_uppercase; dropped line because it contains less than ' - f'{min_uppercase} uppercase characters; {line_decoded}{linesep}') - stop = True - - max_uppercase = config.get('check-max-uppercase') - if max_uppercase != float('inf') and not stop: - if not contains_at_most(line_decoded, max_uppercase, str.isupper): - log.append(f'Check_max_uppercase; dropped line because it contains more than ' - f'{max_uppercase} uppercase characters; {line_decoded}{linesep}') - stop = True - - min_specials = config.get('check-min-specials') - if min_specials and not stop: - if not contains_at_least(line_decoded, min_specials, - lambda char: not char.isalnum() and not char.isspace()): - log.append(f'Check_min_specials; dropped line because it contains less than ' - f'{min_specials} special characters; {line_decoded}{linesep}') - stop = True - - max_specials = config.get('check-max-specials') - if max_specials != float('inf') and not stop: - if not contains_at_most(line_decoded, max_specials, lambda char: not char.isalnum() and not char.isspace()): - log.append(f'Check_max_specials; dropped line because it contains more than ' - f'{max_specials} special characters; {line_decoded}{linesep}') - stop = True - - if config.get('check-starting-with') and not stop: - to_check = config.get("check-starting-with") - if check_starting_with(line_decoded, to_check): - log.append(f'Check_starting_with; dropped line because {to_check} found; {line_decoded}{linesep}') - stop = True - - if config.get('check-uuid') and not stop: - if not check_uuid(line_decoded): - log.append(f'Check_uuid; dropped line because found a uuid; {line_decoded}{linesep}') - stop = True - - if config.get('check-ending-with') and not stop: - to_check = config.get("check-ending-with") - if check_ending_with(line_decoded, to_check): - log.append(f'Check_ending_with; dropped line because {to_check} found; {line_decoded}{linesep}') - stop = True - - if config.get('check-contains') and not stop: - to_check = config.get("check-contains") - if check_contains(line_decoded, to_check): - log.append(f'Check-contains; dropped line because {to_check} found; {line_decoded}{linesep}') - stop = True - - if config.get('check-empty-line') and not stop: - if check_empty_line(line_decoded): - log_line = "Check_empty_line; dropped line because is empty or only contains whitespace;" - log.append(f'{log_line} {line_decoded}{linesep}') - stop = True - - if config.get('remove-punctuation') and not stop: - status, line_decoded = remove_punctuation(line_decoded, config.get('punctuation')) - if status and config['debug']: - log.append(f'Remove_punctuation; stripped punctuation; {line_decoded}{linesep}') - - if config.get('remove-strip-punctuation') and not stop: - status, line_decoded = remove_strip_punctuation(line_decoded, config.get('punctuation')) - if status and config['debug']: - log.append(f'Remove_strip_punctuation; stripped punctuation; {line_decoded}{linesep}') - - # We ran all modules - if not stop: - # Some clean modules will modify the end result, those modification will be added here. - # They will be added to the running thread, this might cause one thread to have more work - # then others. - if config.get('add-split'): - modified_lines = add_split(line_decoded) - if modified_lines: - for modified_line in modified_lines: - if config['debug']: - log.append(f'Add_split; new line because of split; {modified_line}{linesep}') - work_queue.append(modified_line.encode()) - - if config.get('add-lower'): - modified_line = add_lower(line_decoded) - if modified_line: - if config['debug']: - log.append(f'Add_lower; new line; {modified_line}{linesep}') - work_queue.append(modified_line.encode()) - - if config.get('add-first-upper'): - modified_line = add_first_upper(line_decoded) - if modified_line: - if config['debug']: - log.append(f'Add_first_upper; new line; {modified_line}{linesep}') - work_queue.append(modified_line.encode()) - - if config.get('add-title-case'): - modified_line = add_title_case(line_decoded) - if modified_line: - if config['debug']: - log.append(f'Add_title_case; new line; {modified_line}{linesep}') - work_queue.append(modified_line.encode()) - - if config.get('add-latin-ligatures'): - modified_line = add_latin_ligatures(line_decoded) - if modified_line: - if config['debug']: - log.append(f'Add_latin_ligatures; new line; {modified_line}{linesep}') - work_queue.append(modified_line.encode()) - - if config.get('add-umlaut'): - status, modified_line = clean_add_umlaut(line_decoded) - if status: - if config['debug']: - log.append(f'Add_umlaut; new line; {modified_line}{linesep}') - work_queue.append(modified_line.encode()) - - if config.get('add-without-punctuation'): - modified_line = add_without_punctuation(line_decoded, config.get('punctuation')) - if modified_line: - if config['debug']: - log.append(f'Add_without_punctuation; new line; {modified_line}{linesep}') - work_queue.append(modified_line.encode()) - - if config['debug']: - log.append(f'----End---- {line_decoded}{linesep}{linesep}') - results.append(f'{line_decoded}{linesep}') - - return ({'results': results, 'log': log}) - - -def chunkify(filename, size=CHUNK_SIZE): - with open(filename, 'rb') as fh: - for x in range(0, config.get('skip')): - fh.readline() - - while True: - lines = [line.rstrip(linesep.encode()) for line in fh.readlines(size)] - yield lines - if len(lines) == 0: - break - - -# Quick to default logging to stderr instead -def stderr_print(*args, **kwargs): - if config['verbose'] is True: - kwargs.setdefault('file', stderr) - print(*args, **kwargs) - - -def init_worker(config_data): - global config - config = config_data - - try: - signal(SIGINT, SIG_IGN) - except ValueError: - pass # signal() only works in the main thread; ThreadPool workers are threads - - -def main(): - # - # Config parser - arguments = docopt(cleandoc('\n'.join(__doc__.split('\n')[2:]))) - - if arguments.get('--version'): - print(f'demeuk - {version}') - exit() - - input_file = arguments.get('--input') - output_file = arguments.get('--output') - log_file = arguments.get('--log') - - if arguments.get('--threads'): - a_threads = arguments.get('--threads') - if a_threads == 'all': - a_threads = cpu_count() - else: - a_threads = int(a_threads) - else: - a_threads = cpu_count() - - # Lets create the default config - global config - config = { - 'input_encoding': ['UTF-8'], - 'cut': False, - 'delimiter': ':', - 'cut-fields': '2-', - 'verbose': False, - 'debug': False, - 'progress': False, - 'limit': False, - 'skip': False, - - # Modify - 'encode': False, - 'mojibake': False, - 'tab': False, - 'trim': False, - 'newline': False, - 'hex': False, - 'html': False, - 'html-named': False, - 'umlaut': False, - 'non-ascii': False, - 'title_case': False, - 'transliterate': False, - - # Check - 'length': False, - 'check-min-length': 0, - 'check-max-length': 0, - 'check-controlchar': False, - 'check-case': False, - 'check-email': False, - 'check-hash': False, - 'check-mac-address': False, - 'check-non-ascii': False, - 'check-replacement-character': False, - 'check-starting-with': False, - 'check-uuid': False, - 'check-ending-with': False, - 'check-contains': False, - 'check-empty-line': False, - 'check-regex': False, - 'check-min-digits': 0, - 'check-max-digits': float('inf'), - 'check-min-uppercase': 0, - 'check-max-uppercase': float('inf'), - 'check-min-specials': 0, - 'check-max-specials': float('inf'), - - # Add - 'add-lower': False, - 'add-first-upper': False, - 'add-title-case': False, - 'add-latin-ligatures': False, - 'add-split': False, - 'add-umlaut': False, - 'add-without-punctuation': False, - - # Remove - 'remove-strip-punctuation': False, - 'remove-punctuation': False, - 'remove-email': False, - } - - # Default modules - if arguments.get('--verbose'): - config['verbose'] = True - - if arguments.get('--debug'): - config['debug'] = True - - if arguments.get('--progress'): - if config['verbose'] or config['debug']: - if not log_file: - stderr_print('Progress can not be used with verbose or debug') - exit(2) - if not input_file: - # Forcing printing error message - config['verbose'] = True - stderr_print('Progress can not be used when using stdin.') - exit(2) - config['progress'] = True - - if arguments.get('--force'): - config['force'] = True - - if arguments.get('--limit'): - config['limit'] = int(arguments.get('--limit')) - - if arguments.get('--skip'): - config['skip'] = int(arguments.get('--skip')) - - if arguments.get('--input-encoding'): - config['input_encoding'] = arguments.get('--input-encoding').split(',') - - setlocale(LC_ALL, 'en_US.UTF-8') - if arguments.get('--output-encoding'): - setlocale(LC_ALL, arguments.get('--output-encoding')) - - if arguments.get('--punctuation'): - config['punctuation'] = arguments.get('--punctuation') - else: - config['punctuation'] = string_punctuation + ' ' - - if arguments.get('--cut'): - config['cut'] = True - - if arguments.get('--delimiter'): - splitter = ',' - if len(arguments.get('--delimiter')) >= 1: - if arguments.get('--delimiter')[0] == ',': - splitter = ';' - config['delimiter'] = arguments.get('--delimiter').split(splitter) - - if arguments.get('--cut-before'): - config['cut-fields'] = '-1' - - if arguments.get('--cut-fields'): - config['cut-fields'] = arguments.get('--cut-fields') - - # Clean / modify modules - if arguments.get('--hex'): - config['hex'] = True - - if arguments.get('--html'): - config['html'] = True - - if arguments.get('--html-named'): - config['html-named'] = True - - if arguments.get('--umlaut'): - config['umlaut'] = True - - if arguments.get('--non-ascii'): - config['non-ascii'] = True - - if arguments.get('--lowercase'): - config['lowercase'] = True - - if arguments.get('--title-case'): - config['title-case'] = True - - if arguments.get('--mojibake'): - config['mojibake'] = True - - if arguments.get('--encode'): - config['encode'] = True - - if arguments.get('--tab'): - config['tab'] = True - - if arguments.get('--newline'): - config['newline'] = True - - if arguments.get('--trim'): - config['trim'] = True - - if arguments.get('--transliterate'): - config['transliterate'] = arguments.get('--transliterate') - - # Check modules - if arguments.get('--check-min-length'): - config['check-length'] = True - config['check-min-length'] = int(arguments.get('--check-min-length')) - - if arguments.get('--check-max-length'): - config['check-length'] = True - config['check-max-length'] = int(arguments.get('--check-max-length')) - - if arguments.get('--check-case'): - config['check-case'] = True - - if arguments.get('--check-email'): - config['check-email'] = True - - if arguments.get('--check-hash'): - config['check-hash'] = True - - if arguments.get('--check-mac-address'): - config['check-mac-address'] = True - - if arguments.get('--check-non-ascii'): - config['check-non-ascii'] = True - - if arguments.get('--check-replacement-character'): - config['check-replacement-character'] = True - - if arguments.get('--check-starting-with'): - if ',' in arguments.get('--check-starting-with'): - config['check-starting-with'] = arguments.get('--check-starting-with').split(',') - else: - config['check-starting-with'] = [arguments.get('--check-starting-with')] - - if arguments.get('--check-uuid'): - config['check-uuid'] = True - - if arguments.get('--check-ending-with'): - if ',' in arguments.get('--check-ending-with'): - config['check-ending-with'] = arguments.get('--check-ending-with').split(',') - else: - config['check-ending-with'] = [arguments.get('--check-ending-with')] - - if arguments.get('--check-contains'): - if ',' in arguments.get('--check-contains'): - config['check-contains'] = arguments.get('--check-contains').split(',') - else: - config['check-contains'] = [arguments.get('--check-contains')] - - if arguments.get('--check-empty-line'): - config['check-empty-line'] = True - - if arguments.get('--check-controlchar'): - config['check-controlchar'] = True - - if arguments.get('--check-regex'): - config['check-regex'] = arguments.get('--check-regex').split(',') - - if arguments.get('--check-min-digits'): - config['check-min-digits'] = int(arguments.get('--check-min-digits')) - - if arguments.get('--check-max-digits'): - config['check-max-digits'] = int(arguments.get('--check-max-digits')) - - if arguments.get('--check-min-uppercase'): - config['check-min-uppercase'] = int(arguments.get('--check-min-uppercase')) - - if arguments.get('--check-max-uppercase'): - config['check-max-uppercase'] = int(arguments.get('--check-max-uppercase')) - - if arguments.get('--check-min-specials'): - config['check-min-specials'] = int(arguments.get('--check-min-specials')) - - if arguments.get('--check-max-specials'): - config['check-max-specials'] = int(arguments.get('--check-max-specials')) - - # Add modules - if arguments.get('--add-lower'): - config['add-lower'] = True - - if arguments.get('--add-first-upper'): - config['add-first-upper'] = True - - if arguments.get('--add-title-case'): - config['add-title-case'] = True - - if arguments.get('--add-latin-ligatures'): - config['add-latin-ligatures'] = True - - if arguments.get('--add-split'): - config['add-split'] = True - - if arguments.get('--add-umlaut'): - config['add-umlaut'] = True - - if arguments.get('--add-without-punctuation'): - config['add-without-punctuation'] = True - - # Remove modules - if arguments.get('--remove-strip-punctuation'): - config['remove-strip-punctuation'] = True - - if arguments.get('--remove-email'): - config['remove-email'] = True - - if arguments.get('--remove-punctuation'): - config['remove-punctuation'] = True - - # Some meta-modules, those overwrite settings - if arguments.get('--googlengram'): - config['cut'] = False - config['remove-email'] = False - config['encode'] = True - config['mojibake'] = False - config['check-controlchar'] = False - config['tab'] = False - config['googlengram'] = True - - # Meta-module for leak files. Set the following defaults: - # mojibake, encode, newline, check-controlchar - if arguments.get('--leak'): - config['mojibake'] = True - config['encode'] = True - config['newline'] = True - config['check-controlchar'] = True - - # Meta-module for leak fils, but more modules. Set the following defaults: - # --mojibake, --encode, --newline, --check-controlchar, - # --hex, --html, --html-named, - # --check-hash, --check-mac-address, --check-uuid, --check-email, - # --check-replacement-character, --check-empty-line - if arguments.get('--leak-full'): - config['mojibake'] = False - config['encode'] = True - config['newline'] = True - config['check-controlchar'] = True - config['hex'] = True - config['html'] = True - config['html-named'] = True - config['check-hash'] = True - config['check-mac-address'] = True - config['check-uuid'] = True - config['check-email'] = True - config['check-replacement-character'] = True - config['check-empty-line'] = True - - if output_file and not access(path.dirname(output_file), W_OK): - stderr_print(f"Cannot write output file to {output_file}") - - # check if logfile exists, or that the directory of the log file is at least writable. - if log_file and not (access(log_file, F_OK) or access(path.dirname(log_file), W_OK)): - stderr_print(f"Cannot write log file to {log_file}") - if input_file and not access(input_file, R_OK): - stderr_print(f"Cannot read input file to {input_file}") - - # Main worker - stderr_print(f'Main: running demeuk - {version}') - - stderr_print(f'Main: Using {a_threads} core(s) of total available cores: {cpu_count()}') - - stderr_print(f'Main: start chunking file {input_file}') - if output_file: - stderr_print(f'Main: output found in {output_file}') - if log_file: - stderr_print(f'Main: logs found in {log_file}') - - stderr_print('Main: done chunking file.') - stderr_print('Main: processing started.') - - encoding = getlocale()[1] - if output_file: - p_output_file = open(output_file, 'w', encoding=encoding, newline='') - else: - p_output_file = stdout - - if log_file: - p_log_file = open(log_file, 'a', encoding=encoding, newline='') - else: - p_log_file = stderr - - def write_results(results): - p_output_file.writelines(results) - p_output_file.flush() - - def write_log(log): - if config['debug'] or config['verbose'] or log_file: - p_log_file.writelines(log) - p_log_file.flush() - - def write_results_and_log(async_result): - write_results(async_result['results']) - write_log(async_result['log']) - - def process_jobs(chunk_start): - # Cut file in to chunks and process each trunk multi-threaded - while True: - while True: - # Process completed jobs in-order - if jobs and jobs[0].ready(): - # Housekeeping cleanup jobs completed from the list - job = jobs.pop(0) - write_results_and_log(job.get()) - else: - break - - # Find out which jobs are running - running_jobs = sum([not job.ready() for job in jobs]) - if running_jobs < a_threads: - job = pool.apply_async(clean_up, (chunk,)) - chunk_start += len(chunk) - jobs.append(job) - break - else: - # Wait a little while for available spacing within Pool - sleep(1) - - write_log(f'Running demeuk - {version}{linesep}') - with Pool(a_threads, init_worker, initargs=(config,)) as pool: - jobs = [] - # chunk_start will be the started value of the combined output lines - chunk_start = 0 - if input_file: - # Process files based on input glob - for filename in tqdm(glob(input_file, recursive=True), desc='Files processed', mininterval=0.1, - unit=' files', disable=not config.get('progress'), position=0): - if not access(filename, R_OK): - continue - chunks_estimate = int(ceil(path.getsize(filename) / CHUNK_SIZE)) - for chunk in tqdm(chunkify(filename, CHUNK_SIZE), desc='Chunks processed', mininterval=1, - unit=' chunks', disable=not config.get('progress'), total=chunks_estimate, - position=1): - process_jobs(chunk_start) - stderr_print('Main: done submitting all jobs, waiting for threads to finish') - while len(jobs) > 0: - job = jobs.pop(0) - job.wait() - write_results_and_log(job.get()) - else: - # Read chunk amount from stdin - chunks = stdin.readlines(CHUNK_SIZE) - while chunks: - chunk = [line.rstrip('\n').encode(config['input_encoding'][0]) for line in chunks] - process_jobs(chunk_start) - - chunks = stdin.readlines(CHUNK_SIZE) - - stderr_print('Main: done submitting all jobs, waiting for threads to finish') - while len(jobs) > 0: - job = jobs.pop(0) - job.wait() - write_results_and_log(job.get()) - - stderr_print('Main: all done') - if output_file: - p_output_file.close() - if log_file: - p_log_file.close() - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - stderr_print("ERROR: Process terminated by user! (CTRL+C)") - exit(3) diff --git a/docs/api_ref.rst b/docs/api_ref.rst index f615017..f2ec41e 100644 --- a/docs/api_ref.rst +++ b/docs/api_ref.rst @@ -3,9 +3,53 @@ API Reference This chapter is for developers of demeuk, it contains the API functions. -Demeuk-api -========== -.. automodule:: bin.demeuk +demeuk +------ +.. automodule:: demeuk.demeuk :members: :undoc-members: :show-inheritance: + + +modules.base +------------ +.. automodule:: demeuk.modules.base + :members: + :undoc-members: + :show-inheritance: + +modules.add +----------- +.. automodule:: demeuk.modules.add.add + :members: + :undoc-members: + :show-inheritance: + +modules.check +------------- +.. automodule:: demeuk.modules.check.check + :members: + :undoc-members: + :show-inheritance: + +modules.modify +-------------- +.. automodule:: demeuk.modules.modify.modify + :members: + :undoc-members: + :show-inheritance: + +modules.remove +-------------- +.. automodule:: demeuk.modules.remove.remove + :members: + :undoc-members: + :show-inheritance: + +modules.macro +-------------- +.. automodule:: demeuk.modules.macro.macro + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/conf.py b/docs/conf.py index a052938..e009395 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,19 +13,19 @@ import os import sys -sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('../src')) source_suffix = ['.rst', '.md'] -from bin.demeuk import version # noqa: E402 +from demeuk.demeuk import get_version # -- Project information ----------------------------------------------------- project = 'demeuk' -copyright = '2019 - 2021, NFI' +copyright = '2019 - 2026, NFI' author = 'Netherlands Forensic Institute (NFI)' # The full version, including alpha/beta/rc tags -release = version +release = get_version() # -- General configuration --------------------------------------------------- @@ -47,6 +47,8 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +# Use the same ordering as the source file for members. +autodoc_member_order = 'bysource' # -- Options for HTML output ------------------------------------------------- diff --git a/docs/design.rst b/docs/design.rst index cc7834c..b621cfd 100644 --- a/docs/design.rst +++ b/docs/design.rst @@ -6,50 +6,107 @@ some insight on how the application works. Mostly it is useful in case you are working with a bug or don't understand why something is happening and it is a must read for anyone adding features to demeuk. +.. _modules: + +Modules +------- +Demeuk is a modular program; you can enable functionality by specifying their respective command-line options. +Demeuk supports four different type of modules. + +- Modify or Clean modules. Those modules modify something in a line. For example replace tab + character with ':'. The commandline parameters will have the name of the module + without a prefix. +- Add modules. Those modules will modify something in a line, but keep the original + line as well. For example, add a lower case variant of a line. These modules will + have the commandline parameters start with 'add-' prefix. +- Check modules. Those modules will check if a line passes some test. For example + a minimal length check. The commandline parameters start with the 'check-' prefix. + If a line fails the check, the line is dropped. +- Remove modules. Those modules will remove specific parts of a line and does this + in place. For example punctuation needs to be removed, those modules will be used. + The commandline parameters will start with the 'remove-' prefix. + +Note that when any add option is used, any other modules (like clean, check, remove +AND even add) will be ran on the modified line again. This might result in creating +an loop if it keeps creating new lines. So be careful with using those options. + + +There is another class of modules called macro modules. These are modules which invoke a number of +other modules, and may provide other functionality themselves. Examples of macro modules are +``leak`` and ``leak-full``, which are simply a sensible collection of other modules to clean up data leaks. + + +.. _pipeline: + +Pipeline +-------- +Demeuk runs its input through a pipeline, which is a linear sequence of modules. The action +of a module on types determines its place in the pipeline. + +Input is read as bytes, at some point it is *encoded*, at which point these bytes are treated as a +Python string. At this point we can work with the input as strings of characters. Keeping this in +mind, a module can fall in three classes: + +* A module which takes bytes as input and returns a byte sequence, +* A module which takes bytes as input and returns a string (an *encoding* module), +* A module which takes a string as input and returns a string. + +A module in the pipeline will return a ``Results`` object, containing a ``status`` field. If this +is set to ``False``, nothing happens and the line is passed to the next module in line. If however +this field is ``True``, the ``Results`` object is passed to ``handle()`` which, depending on the +module type, determines what concrete actions to take. These actions could be: Stop further +processing of the line, add a variant, log a debug message, etc. This is encoded in and ``Actions`` +object which is processed by ``Pipeline.run()``. + Threading --------- In cause of an input file, the file is 'chunked' by the main processes. It will split -the input files in chunks. It does so by reading the file per 1 KB. After reading 1 KB -it will search for the next newline after the 1 KB. It will then read again 1 KB and +the input files in chunks. It does so by reading the file per 1 MB. After reading 1 MB +it will search for the next newline after the 1 MB. It will then read again 1 MB and search for the first new line after that. This will continue until the end of the file. -The size of 1 KB is used to reduce memory load and was found to be a solid number for +The size of 1 MB is used to reduce memory load and was found to be a solid number for good performance. When using stdin, the input is not chunked. This is because stdin is a stream and thus we can not seek to a specific offset. So the main thread will read the input -per 1 KB and search for the first newline after that. +per 1 MB and search for the first newline after that. -Searching for the next newline is done using the python's splitlines() +Searching for the next newline is done using the python's splitlines() function. This means the line will be splitted on: line feed, carriage return, LF + CR, formfeeds, file separator, etc. See https://docs.python.org/3/library/stdtypes.html for more information. Next a thread will process the list of lines. -Once all threads are done, the main thread will combine all of the results. +Once all threads are done, the main thread will combine all of the results. You should note that the order inside the final output will be completely un ordered and thus if you want to have a sorted list you need to sort it yourself. +A note on the add modules and threading: Lines are dedicated to different +threads based on a configured chunk size. When additional lines are added, all +other modules will run again on the line. The thread that created the new line +will also run those modules again. Meaning that if one thread creates a lot of +different new lines that thread might be busier then other threads. But because +the chunksize is quite small, this will probably not be an issue. If this is an +issue for someone please submit a bug. + Encoding detection ------------------ -Next, when '--tab' is enabled all tabs will be converted to ':' greedy. This is to have -a single cut/splitting char. This is done on binary level. - -Next, we arrive at one of the most important things of this application. The encoding detecting -enable this with '--encode'. Some dataset are a combination of different sources. This means +One of the most important things of this application is the encoding detection, +enabled with ``--encode``. Some datasets are a combination of different sources. This means EVERY line can have a different encoding. People or applications tend to make a lot of errors in encoding, as does this application. Demeuk tries its best to detect and correct as much as possible, but there will for sure be some weird case where it fails to do so. By default the application will try to decode the data using UTF-8. So we start by checking if we have a default encoding to try. This is either -UTF-8 or supplied by the user. If the line decodes and there does not appear to be -control character inside the line we can assume that the detection went correctly. -Note: If you supply a list of input encodings. Put multibyte encodings first. -Because single byte encodings will cause false positives. +UTF-8 or supplied by the user with ``--input-encoding``. If the line decodes and there does not +appear to be control characters inside the line we can assume that the detection went correctly. +Note: If you supply a list of input encodings. Put multibyte encodings first, because single byte +encodings will cause false positives. -If that fails we run the detect function of the chardet library. Note: first the +If that fails we run the detect function of the chardet library. Note: first the cchardet library was implemented, but this library resulted in too many wrongly encoded lines. Inside the tests of demeuk there are lot of edge cases which were found and corrected. So if you change something in the encoding detection @@ -58,44 +115,14 @@ please run the tests to verify that you have not broken something. If it managed detect any encoding, it will try to decode this line. If no unicode error happens we assume that we got some result. -Next we try to fix mojibakes, for this enable the --mojibake option -basically, we might have decoded the string incorrectly -and now correct some of the common errors. For this we use the FTFY library. - -Modules -------- -After a line has been decoded correctly demeuk will start to run all the modules. -Demeuk consist of 4 different type of modules. - -- Clean modules. Those modules modify something in a line. For example replace tab - character with ':'. The commandline parameters will have the name of the module - without a prefix. -- Add modules. Those modules will modify something in a line, but keep the original - line aswell. For example, add a lower case variant of a line. These modules will - have the commandline parameters start with 'add-' prefix. -- Check modules. Those modules will check if a line passes some test. For example - a minimal length check. The commandline parameters start with the 'check-' prefix. - If a line fails the check, the line is dropped. -- Remove modules. Those modules will remove specific parts of a line and does this - in place. For example punctuation needs to be removed, those modules will be used. - The commandline parameters will start with the 'remove-' prefix. - -The name that a module has on the commandline will mean that the function inside the -source code must also has the exact same name. Only clean module will start with the -'clean\_' prefix to prevent name clashes with default functions. - -Note that when any add option is used, any other modules (like clean, check, remove -AND even add) will be ran on the modified line again. This might result in creating -an loop if it keeps creating new lines. So be careful with using those options. +You can enable the ``--mojibake`` option to let demeuk try to fix mojibakes, which are +artifacts of wrong decoding. For this we use the FTFY library. -For now there is no specific order in which the module type will run. Apart from -the add modules, which will always run last. If someone find a specific use case -for which the order needs to be configured; please submit a bug. -Another note on the add modules and threading. Lines are dedicated to different -threads based on a configured chunk size. When additional lines are added, all -other modules will run again on the line. The thread that created the new line -will also run those modules again. Meaning that if one thread creates a lot of -different new lines that thread might be busier then other threads. But because -the chunksize is quite small, this will probably not be an issue. If this is an -issue for someone please submit a bug. \ No newline at end of file +Module ordering +--------------- +After successfully figuring out the decoding of the line, there are many different modules which +can be run. All modules which function after the encoding step will be run in the order in which +they are supplied to the program. This enables great flexibility, but it also means you need to +think about this order. If you don't know where to start, put modify modules first, then check +modules, remove modules and finally add modules. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index ad8a3aa..20dc290 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ Table of content :maxdepth: 3 install - usage design + new_module + usage api_ref diff --git a/docs/install.rst b/docs/install.rst index 92f2bf2..7766474 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -2,77 +2,54 @@ Install ======= This document describes how to install demeuk. -There are multiple ways to install python packages - -- System-wide -- User specific -- Virtual environment - -The recommended way to install demeuk is to install it in a virtual -environment. - Requirements ------------ -- Python 3.6 is required +- Python 3.12 is required, Python 3.14 is recommended. - Ubuntu is the only OS on which demeuk has been tested. Installing ---------- +The recommended way is to install demeuk using `pipx`_. :: -Virtual environment -~~~~~~~~~~~~~~~~~~~ - -.. code-block:: none + pipx install demeuk --python /usr/bin/python3.14 - $ sudo apt install python3-pip - $ sudo pip3 install virtualenv - $ cd - $ virtualenv venv-demeuk - $ source venv-demeuk/bin/activate +This will make demeuk available everywhere by simply running ``demeuk``. -Installing from PyPi -~~~~~~~~~~~~~~~~~~~~ +.. _pipx: https://pipx.pypa.io/stable/ -.. code-block:: none +Running +------- +You can run demeuk using:: - $ pip3 install demeuk + demeuk [options] -Installing from source -~~~~~~~~~~~~~~~~~~~~~~ -If for some reason the PyPi is not available, you can build the wheelfile -yourself. First create a Virtual environment as described above. -:ref:`Virtual environment` - -.. code-block:: none +Development +~~~~~~~~~~~~~~~ +If you want to run demeuk from source you can also easily do this with pipx: :: - $ git clone - $ cd demeuk - $ python3 setup.py bdist_wheel - $ pip3 install dist/*.whl + # Clone the repo + git clone + cd demeuk + # Install from source + pipx install ./ --python /usr/bin/python3.14 -Run from source -~~~~~~~~~~~~~~~ -If for some reason you want to run demeuk from source you only have to install -the requirements. +If you change the Python source, you will have to run:: -.. code-block:: none + pipx upgrade demeuk - $ git clone - $ cd demeuk - $ pip3 install -r requirements.txt - $ python3 bin/demeuk.py --help +to reload the shortcut ``demeuk``. -Upgrading ---------- +Alternatively, you can use PDM: :: -Upgrading demeuk is quite simple. In case you have installed demeuk through pip -and using a virtualenv: + # Clone the repo + git clone + cd demeuk + # Set up PDM project + pdm init -n --no-git --python /usr/bin/python3.14 -.. code-block:: none +in which case you don't have to reload the command everytime you change the source code, but you then have to run demeuk with :: - $ source venv-demeuk/bin/activate - $ pip3 install demeuk --upgrade + pdm run demeuk [options] -In case that you installed demeuk using the source, just rebuild the software -and install the wheel file. Pip3 will upgrade the package automatically. +which is also not globally available. diff --git a/docs/new_module.rst b/docs/new_module.rst new file mode 100644 index 0000000..c59f4d7 --- /dev/null +++ b/docs/new_module.rst @@ -0,0 +1,131 @@ +Adding new modules to demeuk +============================ +Demeuk contains a lot of modules already, but if you want to add your own custom module these are +some things to keep in mind. + +To implement a custom module, create a Python file in ``src/demeuk/modules`` and create a class +which inherits from either CheckModule, ModifyModule, AddModule, RemoveModule or MacroModule +depending on the desired functionality. Additionally this class may inherit from ParamModule, for +supplying a command-line parameter to the module, or ConfigModule if it depends on some other +configuration. + +Properties of the module, as well as its workings are configured by implementing a set of abstract +methods. + +Modules are automatically discovered and registered to the argument parser at runtime, so +implementing a module can take place entirely within a single file. + +Help info +--------- +The help info of a module describes how you can invoke it from the command-line, and it contains +what to display when ``demeuk -h`` is run. To set the help info, implement ``get_help_info()`` +to return a ``HelpInfo`` object, which is a named tuple containing the fields: + +* option, which is either a string or a list of strings of command-line options. If a list is supplied, all of these will be added as aliases. +* help_str, which is a string containing a short description on how the module functions. + +If your module takes a parameter, it implements ParamModule in which case ``get_help_info()`` must +return a ``HelpInfoParam`` object. This is a named tuple containing the same info as above, in +addition to: + +* param_type, which is the Python type of the parameter (for example int, str). +* metavar, a name by which the parameter can be referenced by help_str, for example '' or '' + +The run function +---------------- +The main functionality of the module will be implemented in ``run()``. This function takes one line +as input, and outputs a ``Result`` tuple. How to configure the return result depends on the type of module. + +Adding a check module +^^^^^^^^^^^^^^^^^^^^^ +A check module (or more precisely, a class which implements ``CheckModule``) chooses whether to +continue running the module pipeline on the current line, or drop a line and continue to the next. +For these results, ``CheckModule`` supplies the shortcuts ``next`` and ``stop`` respectively. + +Adding a modify/remove module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Modify or remove modules can modify an existing line. For performance, we only update the line if it is actually different from the input line. +Therefore, after determining the modified line, we check if it is equal to the original line and only if it differs, we send the update Result. +``ModifyModule`` and ``RemoveModule`` provides the shortcut ``get_result(line, cleaned_line)``. + +Adding an add module +^^^^^^^^^^^^^^^^^^^^ +Add modules add variants of a line to the work queue. Most add module only add a single variant, +and we want to only add it if it differs from the original line, just as with the modify and remove modules. +Therefore ``AddModule`` supplies ``get_result(line, added_line)``. + + +Accessing a parameter +^^^^^^^^^^^^^^^^^^^^^ +Some modules depend on a user-specified parameter, for example a check module which drops lines if their length is over a certain number of characters. +For this, you can implement ``ParamModule`` (in addition to one of the four above module types). +For ParamModule implementations, you need to return a ``HelpInfoParam`` from ``get_help_info()``, +and you can access the parameter inside ``run()`` with ``self.param``. + + + +Advanced modules +---------------- +Demeuk modules are very customizable; you can override the standard 'control flow handlers' to +expose a lot of flexibility. + +Position in pipeline +^^^^^^^^^^^^^^^^^^^^ +The position of a module in the :ref:`pipeline` can be set by overriding ``get_pipeline_position()``, which should return either +``PipelinePosition.BEFORE_ENCODE``, ``PipelinePosition.ENCODE`` or ``PipelinePosition.AFTER_ENCODE`` +depending on their input and output signature. +Most modules simply take in a string and output a string, so they occur after encoding. +This is the default behaviour (what happens when you don't override ``get_pipeline_position()``). + +Accessing config +^^^^^^^^^^^^^^^^ +If your module depends on a config value which should not be passed as a parameter (for example +because it is already set somewhere else, or you want multiple modules to be able to use the same config), +you can implement ``ConfigModule``. The master object containing all config about the program is +a ``Config`` instance, so the config value must come from here. To 'register' a config value to a module, +run ``add_config(key, value)`` inside of ``set_configs(config)``. You can later retrieve this value with ``get_config(key)``. + +Adding a macro module +^^^^^^^^^^^^^^^^^^^^^ +You might want to create a module which runs a collection of other modules, and possibly process this result some more. +For this, you need a macro module. When you implement a ``MacroModule``, you can provide a list of module instances +in ``get_submodules()``, which will be inserted in order [#f1]_ into the module pipeline. +Note that this list needs to consist of *instances* of modules, so if you want to include a module +with a parameter of config, you need to supply this in ``get_submodules()``. + +By default, ``run()`` simply continues to the next module but you can override this function +if you want the macro module to perform some action. Note that if you override ``run()``, you will +need to construct a Result object, and implement a result handler in ``handle()``, see the next +section on how to do this. Also keep in mind that macro modules are added *after* their submodules in the pipeline. + +Customizing the result handler +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +If you want your module to perform actions which do not fall neatly in the described behaviors above +(for example: adding multiple lines to the work queue, change logging behavior, or adding a module +in the queue and stopping further processing), you can return a custom ``Result`` and override the +``handle(result)`` function. + +``Actions`` is a tuple containing the following values: + +* ``stop`` - If True, drop this line completely and go to the next line. +* ``add`` - A list of lines to add to the work queue. +* ``update`` - The string to replace the current line with +* ``do_not_re_encode`` - A flag which, if set to True, adds a line back without re-encoding it as a string. +* ``log_str`` - A string to log to the log file or stderr +* ``debug_str`` - A string to log if ``--debug`` is set +* ``debug_add_str`` - A string to log if ``--debug`` is set, for adding lines to the queue. + +If a value is not set, no action will be taken. So for example, if we want to add two variant lines +and remove the current line from the work queue without logging anything we simply return a +``Actions(add=[line1, line2], stop=True)`` from ``handle()``. + +Performance considerations +^^^^^^^^^^^^^^^^^^^^^^^^^^ +If you plan on running your modules on large lists, note that the module's ``run()`` function +is called for every line and this is therefore the place you should look first for optimizations. +Avoid allocating memory or instantiating large objects here, and instead prefer to pre-allocate or +store results if you can. + +.. rubric:: Footnotes + +.. [#f1] That is, relative ordering of the modules which share a pipeline position is preserved. If you return a list of modules with ``PipelinePosition.AFTER_ENCODE`` and you append a module with ``PipelinePosition.BEFORE_ENCODE``, the final module will be added before the rest. \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 6c5d5d4..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -sphinx-rtd-theme diff --git a/docs/usage.rst b/docs/usage.rst index ac6eda6..9bf5bab 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,5 +1,7 @@ Usage ===== +.. Maybe we want this page to be shorter and move the reference to a manpage? + This document describes how to usage demeuk. Please read ::ref:`Install` diff --git a/pdm.lock b/pdm.lock new file mode 100644 index 0000000..f86ae4f --- /dev/null +++ b/pdm.lock @@ -0,0 +1,731 @@ +# This file is @generated by PDM. +# It is not intended for manual editing. + +[metadata] +groups = ["default", "check", "test"] +strategy = ["inherit_metadata"] +lock_version = "4.5.0" +content_hash = "sha256:0c637214ca62b044694580e6990a498eb45fc4f76f9f4594443fe251885efc73" + +[[metadata.targets]] +requires_python = ">=3.12" + +[[package]] +name = "cachetools" +version = "7.0.6" +requires_python = ">=3.10" +summary = "Extensible memoizing collections and decorators" +groups = ["test"] +files = [ + {file = "cachetools-7.0.6-py3-none-any.whl", hash = "sha256:4e94956cfdd3086f12042cdd29318f5ced3893014f7d0d059bf3ead3f85b7f8b"}, + {file = "cachetools-7.0.6.tar.gz", hash = "sha256:e5d524d36d65703a87243a26ff08ad84f73352adbeafb1cde81e207b456aaf24"}, +] + +[[package]] +name = "chardet" +version = "7.4.3" +requires_python = ">=3.10" +summary = "Universal character encoding detector" +groups = ["default"] +files = [ + {file = "chardet-7.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0c79b13c9908ac7dfe0a74116ebc9a0f28b2319d23c32f3dfcdfbe1279c7eaf"}, + {file = "chardet-7.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bba8bea1b28d927b3e99e47deafe53658d34497c0a891d95ff1ba8ff6663f01c"}, + {file = "chardet-7.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23163921dccf3103ce59540b0443c106d2c0a0ff2e0503e05196f5e6fdea453f"}, + {file = "chardet-7.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cfb54563fe5f130da17c44c6a4e2e8052ba628e5ab4eab7ef8190f736f0f8f72"}, + {file = "chardet-7.4.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3990fffcc6a6045f2234ab72752ad037e3b2d48c72037f244d42738db397eb75"}, + {file = "chardet-7.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c7116b0452994734ccff35e154b44240090eb0f4f74b9106292668133557c175"}, + {file = "chardet-7.4.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:25a862cddc6a9ac07023e808aedd297115345fbaabc2690479481ddc0f980e09"}, + {file = "chardet-7.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7005c88da26fd95d8abb8acbe6281d833e9a9181b03cf49b4546c4555389bd97"}, + {file = "chardet-7.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc50f28bad067393cce0af9091052c3b8df7a23115afd8ba7b2e0947f0cef1f8"}, + {file = "chardet-7.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c3da294de1a681097848ab58bd3f2771a674f8039d2d87a5538b28856b815e9"}, + {file = "chardet-7.4.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:93c45e116dd51b66226a53ade3f9f635e870de5399b90e00ce45dcc311093bf4"}, + {file = "chardet-7.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:ccc1f83ab4bcfb901cf39e0c4ba6bc6e726fc6264735f10e24ceb5cb47387578"}, + {file = "chardet-7.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:75d3c65cc16bddf40b8da1fd25ba84fca5f8070f2b14e86083653c1c85aee971"}, + {file = "chardet-7.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:29af5999f654e8729d251f1724a62b538b1262d9292cccaefddf8a02aae1ef6a"}, + {file = "chardet-7.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:626f00299ad62dfe937058a09572beed442ccc7b58f87aa667949b20fd3db235"}, + {file = "chardet-7.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a4904dd5f071b7a7d7f50b4a67a86db3c902d243bf31708f1d5cde2f68239cb"}, + {file = "chardet-7.4.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d2879598bc220689e8ce509fe9c3f37ad2fca53a36be9c9bd91abdd91dd364f"}, + {file = "chardet-7.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:4b2799bd58e7245cfa8d4ab2e8ad1d76a5c3a5b1f32318eb6acca4c69a3e7101"}, + {file = "chardet-7.4.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a9e4486df251b8962e86ea9f139ca235aa6e0542a00f7844c9a04160afb99aa9"}, + {file = "chardet-7.4.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4fbff1907925b0c5a1064cffb5e040cd5e338585c9c552625f30de6bc2f3107a"}, + {file = "chardet-7.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:365135eaf37ba65a828f8e668eb0a8c38c479dcbec724dc25f4dfd781049c357"}, + {file = "chardet-7.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfc134b70c846c21ead8e43ada3ae1a805fff732f6922f8abcf2ff27b8f6493d"}, + {file = "chardet-7.4.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9acd9988a93e09390f3cd231201ea7166c415eb8da1b735928990ffc05cb9fbb"}, + {file = "chardet-7.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:e1b98790c284ff813f18f7cf7de5f05ea2435a080030c7f1a8318f3a4f80b131"}, + {file = "chardet-7.4.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d892d3dcd652fdef53e3d6327d39b17c0df40a899dfc919abaeb64c974497531"}, + {file = "chardet-7.4.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:acc46d1b8b7d5783216afe15db56d1c179b9a40e5a1558bc13164c4fd20674c4"}, + {file = "chardet-7.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ac3bf11c645734a1701a3804e43eabd98851838192267d08c353a834ab79fea"}, + {file = "chardet-7.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e3bd9f936e04bae89c254262af08d9e5b98f805175ba1e29d454e6cba3107b7"}, + {file = "chardet-7.4.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:27cc23da03630cdecc9aa81a895aa86629c211f995cd57651f0fbc280717bf93"}, + {file = "chardet-7.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:b95c934b9ad59e2ba8abb9be49df70d3ad1b0d95d864b9fdb7588d4fa8bd921c"}, + {file = "chardet-7.4.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c77867f0c1cb8bd819502249fcdc500364aedb07881e11b743726fa2148e7b6e"}, + {file = "chardet-7.4.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cf1efeaf65a6ef2f5b9cc3a1df6f08ba2831b369ccaa4c7018eaf90aa757bb11"}, + {file = "chardet-7.4.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f3504c139a2ad544077dd2d9e412cd08b01786843d76997cd43bb6de311723c"}, + {file = "chardet-7.4.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457f619882ba66327d4d8d14c6c342269bdb1e4e1c38e8117df941d14d351b04"}, + {file = "chardet-7.4.3-py3-none-any.whl", hash = "sha256:1173b74051570cf08099d7429d92e4882d375ad4217f92a6e5240ccfb26f231e"}, + {file = "chardet-7.4.3.tar.gz", hash = "sha256:cc1d4eb92a4ec1c2df3b490836ffa46922e599d34ce0bb75cf41fd2bf6303d56"}, +] + +[[package]] +name = "click" +version = "8.3.3" +requires_python = ">=3.10" +summary = "Composable command line interface toolkit" +groups = ["default"] +dependencies = [ + "colorama; platform_system == \"Windows\"", +] +files = [ + {file = "click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613"}, + {file = "click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +summary = "Cross-platform colored terminal text." +groups = ["default", "test"] +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "coverage" +version = "7.13.5" +requires_python = ">=3.10" +summary = "Code coverage measurement for Python" +groups = ["test"] +files = [ + {file = "coverage-7.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0723d2c96324561b9aa76fb982406e11d93cdb388a7a7da2b16e04719cf7ca5"}, + {file = "coverage-7.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:52f444e86475992506b32d4e5ca55c24fc88d73bcbda0e9745095b28ef4dc0cf"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:704de6328e3d612a8f6c07000a878ff38181ec3263d5a11da1db294fa6a9bdf8"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a1a6d79a14e1ec1832cabc833898636ad5f3754a678ef8bb4908515208bf84f4"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79060214983769c7ba3f0cee10b54c97609dca4d478fa1aa32b914480fd5738d"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:356e76b46783a98c2a2fe81ec79df4883a1e62895ea952968fb253c114e7f930"}, + {file = "coverage-7.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0cef0cdec915d11254a7f549c1170afecce708d30610c6abdded1f74e581666d"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dc022073d063b25a402454e5712ef9e007113e3a676b96c5f29b2bda29352f40"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9b74db26dfea4f4e50d48a4602207cd1e78be33182bc9cbf22da94f332f99878"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ad146744ca4fd09b50c482650e3c1b1f4dfa1d4792e0a04a369c7f23336f0400"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c555b48be1853fe3997c11c4bd521cdd9a9612352de01fa4508f16ec341e6fe0"}, + {file = "coverage-7.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7034b5c56a58ae5e85f23949d52c14aca2cfc6848a31764995b7de88f13a1ea0"}, + {file = "coverage-7.13.5-cp310-cp310-win32.whl", hash = "sha256:eb7fdf1ef130660e7415e0253a01a7d5a88c9c4d158bcf75cbbd922fd65a5b58"}, + {file = "coverage-7.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:3e1bb5f6c78feeb1be3475789b14a0f0a5b47d505bfc7267126ccbd50289999e"}, + {file = "coverage-7.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66a80c616f80181f4d643b0f9e709d97bcea413ecd9631e1dedc7401c8e6695d"}, + {file = "coverage-7.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:145ede53ccbafb297c1c9287f788d1bc3efd6c900da23bf6931b09eafc931587"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0672854dc733c342fa3e957e0605256d2bf5934feeac328da9e0b5449634a642"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ec10e2a42b41c923c2209b846126c6582db5e43a33157e9870ba9fb70dc7854b"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be3d4bbad9d4b037791794ddeedd7d64a56f5933a2c1373e18e9e568b9141686"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d2afbc5cc54d286bfb54541aa50b64cdb07a718227168c87b9e2fb8f25e1743"}, + {file = "coverage-7.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3ad050321264c49c2fa67bb599100456fc51d004b82534f379d16445da40fb75"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7300c8a6d13335b29bb76d7651c66af6bd8658517c43499f110ddc6717bfc209"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:eb07647a5738b89baab047f14edd18ded523de60f3b30e75c2acc826f79c839a"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9adb6688e3b53adffefd4a52d72cbd8b02602bfb8f74dcd862337182fd4d1a4e"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7c8d4bc913dd70b93488d6c496c77f3aff5ea99a07e36a18f865bca55adef8bd"}, + {file = "coverage-7.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e3c426ffc4cd952f54ee9ffbdd10345709ecc78a3ecfd796a57236bfad0b9b8"}, + {file = "coverage-7.13.5-cp311-cp311-win32.whl", hash = "sha256:259b69bb83ad9894c4b25be2528139eecba9a82646ebdda2d9db1ba28424a6bf"}, + {file = "coverage-7.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:258354455f4e86e3e9d0d17571d522e13b4e1e19bf0f8596bcf9476d61e7d8a9"}, + {file = "coverage-7.13.5-cp311-cp311-win_arm64.whl", hash = "sha256:bff95879c33ec8da99fc9b6fe345ddb5be6414b41d6d1ad1c8f188d26f36e028"}, + {file = "coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01"}, + {file = "coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256"}, + {file = "coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf"}, + {file = "coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c"}, + {file = "coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf"}, + {file = "coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810"}, + {file = "coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de"}, + {file = "coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1"}, + {file = "coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a"}, + {file = "coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6"}, + {file = "coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17"}, + {file = "coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85"}, + {file = "coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b"}, + {file = "coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664"}, + {file = "coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d"}, + {file = "coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd"}, + {file = "coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479"}, + {file = "coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2"}, + {file = "coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a"}, + {file = "coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819"}, + {file = "coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911"}, + {file = "coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f"}, + {file = "coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6"}, + {file = "coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0"}, + {file = "coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0"}, + {file = "coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc"}, + {file = "coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633"}, + {file = "coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8"}, + {file = "coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b"}, + {file = "coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90"}, + {file = "coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea"}, + {file = "coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a"}, + {file = "coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215"}, + {file = "coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43"}, + {file = "coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45"}, + {file = "coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61"}, + {file = "coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179"}, +] + +[[package]] +name = "dill" +version = "0.4.1" +requires_python = ">=3.9" +summary = "serialize all of Python" +groups = ["default"] +files = [ + {file = "dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d"}, + {file = "dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa"}, +] + +[[package]] +name = "distlib" +version = "0.4.0" +summary = "Distribution utilities" +groups = ["test"] +files = [ + {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, + {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, +] + +[[package]] +name = "filelock" +version = "3.29.0" +requires_python = ">=3.10" +summary = "A platform independent file lock." +groups = ["test"] +files = [ + {file = "filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258"}, + {file = "filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90"}, +] + +[[package]] +name = "ftfy" +version = "6.3.1" +requires_python = ">=3.9" +summary = "Fixes mojibake and other problems with Unicode, after the fact" +groups = ["default"] +dependencies = [ + "wcwidth", +] +files = [ + {file = "ftfy-6.3.1-py3-none-any.whl", hash = "sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083"}, + {file = "ftfy-6.3.1.tar.gz", hash = "sha256:9b3c3d90f84fb267fe64d375a07b7f8912d817cf86009ae134aa03e1819506ec"}, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +requires_python = ">=3.10" +summary = "brain-dead simple config-ini parsing" +groups = ["test"] +files = [ + {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, + {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, +] + +[[package]] +name = "joblib" +version = "1.5.3" +requires_python = ">=3.9" +summary = "Lightweight pipelining with Python functions" +groups = ["default"] +files = [ + {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, + {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +requires_python = ">=3.10" +summary = "Python port of markdown-it. Markdown parsing, done right!" +groups = ["check"] +dependencies = [ + "mdurl~=0.1", +] +files = [ + {file = "markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147"}, + {file = "markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3"}, +] + +[[package]] +name = "mdformat" +version = "1.0.0" +requires_python = ">=3.10" +summary = "CommonMark compliant Markdown formatter" +groups = ["check"] +dependencies = [ + "markdown-it-py<5,>=1", + "tomli>=1.1.0; python_version < \"3.11\"", +] +files = [ + {file = "mdformat-1.0.0-py3-none-any.whl", hash = "sha256:bca015d65a1d063a02e885a91daee303057bc7829c2cd37b2075a50dbb65944b"}, + {file = "mdformat-1.0.0.tar.gz", hash = "sha256:4954045fcae797c29f86d4ad879e43bb151fa55dbaf74ac6eaeacf1d45bb3928"}, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +requires_python = ">=3.7" +summary = "Markdown URL utilities" +groups = ["check"] +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "multiprocess" +version = "0.70.19" +requires_python = ">=3.9" +summary = "better multiprocessing and multithreading in Python" +groups = ["default"] +dependencies = [ + "dill>=0.4.1", +] +files = [ + {file = "multiprocess-0.70.19-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:02e5c35d7d6cd2bdc89c1858867f7bde4012837411023a4696c148c1bdd7c80e"}, + {file = "multiprocess-0.70.19-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:79576c02d1207ec405b00cabf2c643c36070800cca433860e14539df7818b2aa"}, + {file = "multiprocess-0.70.19-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c6b6d78d43a03b68014ca1f0b7937d965393a670c5de7c29026beb2258f2f896"}, + {file = "multiprocess-0.70.19-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1bbf1b69af1cf64cd05f65337d9215b88079ec819cd0ea7bac4dab84e162efe7"}, + {file = "multiprocess-0.70.19-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5be9ec7f0c1c49a4f4a6fd20d5dda4aeabc2d39a50f4ad53720f1cd02b3a7c2e"}, + {file = "multiprocess-0.70.19-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1c3dce098845a0db43b32a0b76a228ca059a668071cfeaa0f40c36c0b1585d45"}, + {file = "multiprocess-0.70.19-py310-none-any.whl", hash = "sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87"}, + {file = "multiprocess-0.70.19-py311-none-any.whl", hash = "sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c"}, + {file = "multiprocess-0.70.19-py312-none-any.whl", hash = "sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28"}, + {file = "multiprocess-0.70.19-py313-none-any.whl", hash = "sha256:8d5eb4ec5017ba2fab4e34a747c6d2c2b6fecfe9e7236e77988db91580ada952"}, + {file = "multiprocess-0.70.19-py314-none-any.whl", hash = "sha256:e8cc7fbdff15c0613f0a1f1f8744bef961b0a164c0ca29bdff53e9d2d93c5e5f"}, + {file = "multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897"}, +] + +[[package]] +name = "nltk" +version = "3.9.4" +requires_python = ">=3.10" +summary = "Natural Language Toolkit" +groups = ["default"] +dependencies = [ + "click", + "joblib", + "regex>=2021.8.3", + "tqdm", +] +files = [ + {file = "nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f"}, + {file = "nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0"}, +] + +[[package]] +name = "packaging" +version = "26.1" +requires_python = ">=3.8" +summary = "Core utilities for Python packages" +groups = ["test"] +files = [ + {file = "packaging-26.1-py3-none-any.whl", hash = "sha256:5d9c0669c6285e491e0ced2eee587eaf67b670d94a19e94e3984a481aba6802f"}, + {file = "packaging-26.1.tar.gz", hash = "sha256:f042152b681c4bfac5cae2742a55e103d27ab2ec0f3d88037136b6bfe7c9c5de"}, +] + +[[package]] +name = "platformdirs" +version = "4.9.6" +requires_python = ">=3.10" +summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +groups = ["test"] +files = [ + {file = "platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917"}, + {file = "platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a"}, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +requires_python = ">=3.9" +summary = "plugin and hook calling mechanisms for python" +groups = ["test"] +files = [ + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, +] + +[[package]] +name = "pygments" +version = "2.20.0" +requires_python = ">=3.9" +summary = "Pygments is a syntax highlighting package written in Python." +groups = ["test"] +files = [ + {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, + {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, +] + +[[package]] +name = "pyproject-api" +version = "1.10.0" +requires_python = ">=3.10" +summary = "API to interact with the python pyproject.toml based projects" +groups = ["test"] +dependencies = [ + "packaging>=25", + "tomli>=2.3; python_version < \"3.11\"", +] +files = [ + {file = "pyproject_api-1.10.0-py3-none-any.whl", hash = "sha256:8757c41a79c0f4ab71b99abed52b97ecf66bd20b04fa59da43b5840bac105a09"}, + {file = "pyproject_api-1.10.0.tar.gz", hash = "sha256:40c6f2d82eebdc4afee61c773ed208c04c19db4c4a60d97f8d7be3ebc0bbb330"}, +] + +[[package]] +name = "pytest" +version = "9.0.3" +requires_python = ">=3.10" +summary = "pytest: simple powerful testing with Python" +groups = ["test"] +dependencies = [ + "colorama>=0.4; sys_platform == \"win32\"", + "exceptiongroup>=1; python_version < \"3.11\"", + "iniconfig>=1.0.1", + "packaging>=22", + "pluggy<2,>=1.5", + "pygments>=2.7.2", + "tomli>=1; python_version < \"3.11\"", +] +files = [ + {file = "pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9"}, + {file = "pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c"}, +] + +[[package]] +name = "pytest-timeout" +version = "2.4.0" +requires_python = ">=3.7" +summary = "pytest plugin to abort hanging tests" +groups = ["test"] +dependencies = [ + "pytest>=7.0.0", +] +files = [ + {file = "pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2"}, + {file = "pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a"}, +] + +[[package]] +name = "python-discovery" +version = "1.2.2" +requires_python = ">=3.8" +summary = "Python interpreter discovery" +groups = ["test"] +dependencies = [ + "filelock>=3.15.4", + "platformdirs<5,>=4.3.6", +] +files = [ + {file = "python_discovery-1.2.2-py3-none-any.whl", hash = "sha256:e1ae95d9af875e78f15e19aed0c6137ab1bb49c200f21f5061786490c9585c7a"}, + {file = "python_discovery-1.2.2.tar.gz", hash = "sha256:876e9c57139eb757cb5878cbdd9ae5379e5d96266c99ef731119e04fffe533bb"}, +] + +[[package]] +name = "regex" +version = "2026.4.4" +requires_python = ">=3.10" +summary = "Alternative regular expression module, to replace re." +groups = ["default"] +files = [ + {file = "regex-2026.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:74fa82dcc8143386c7c0392e18032009d1db715c25f4ba22d23dc2e04d02a20f"}, + {file = "regex-2026.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a85b620a388d6c9caa12189233109e236b3da3deffe4ff11b84ae84e218a274f"}, + {file = "regex-2026.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2895506ebe32cc63eeed8f80e6eae453171cfccccab35b70dc3129abec35a5b8"}, + {file = "regex-2026.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6780f008ee81381c737634e75c24e5a6569cc883c4f8e37a37917ee79efcafd9"}, + {file = "regex-2026.4.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:88e9b048345c613f253bea4645b2fe7e579782b82cac99b1daad81e29cc2ed8e"}, + {file = "regex-2026.4.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:be061028481186ba62a0f4c5f1cc1e3d5ab8bce70c89236ebe01023883bc903b"}, + {file = "regex-2026.4.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d2228c02b368d69b724c36e96d3d1da721561fb9cc7faa373d7bf65e07d75cb5"}, + {file = "regex-2026.4.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0540e5b733618a2f84e9cb3e812c8afa82e151ca8e19cf6c4e95c5a65198236f"}, + {file = "regex-2026.4.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cf9b1b2e692d4877880388934ac746c99552ce6bf40792a767fd42c8c99f136d"}, + {file = "regex-2026.4.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:011bb48bffc1b46553ac704c975b3348717f4e4aa7a67522b51906f99da1820c"}, + {file = "regex-2026.4.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8512fcdb43f1bf18582698a478b5ab73f9c1667a5b7548761329ef410cd0a760"}, + {file = "regex-2026.4.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:867bddc63109a0276f5a31999e4c8e0eb7bbbad7d6166e28d969a2c1afeb97f9"}, + {file = "regex-2026.4.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1b9a00b83f3a40e09859c78920571dcb83293c8004079653dd22ec14bbfa98c7"}, + {file = "regex-2026.4.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e355be718caf838aa089870259cf1776dc2a4aa980514af9d02c59544d9a8b22"}, + {file = "regex-2026.4.4-cp310-cp310-win32.whl", hash = "sha256:33bfda9684646d323414df7abe5692c61d297dbb0530b28ec66442e768813c59"}, + {file = "regex-2026.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:0709f22a56798457ae317bcce42aacee33c680068a8f14097430d9f9ba364bee"}, + {file = "regex-2026.4.4-cp310-cp310-win_arm64.whl", hash = "sha256:ee9627de8587c1a22201cb16d0296ab92b4df5cdcb5349f4e9744d61db7c7c98"}, + {file = "regex-2026.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4c36a85b00fadb85db9d9e90144af0a980e1a3d2ef9cd0f8a5bef88054657c6"}, + {file = "regex-2026.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dcb5453ecf9cd58b562967badd1edbf092b0588a3af9e32ee3d05c985077ce87"}, + {file = "regex-2026.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6aa809ed4dc3706cc38594d67e641601bd2f36d5555b2780ff074edfcb136cf8"}, + {file = "regex-2026.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33424f5188a7db12958246a54f59a435b6cb62c5cf9c8d71f7cc49475a5fdada"}, + {file = "regex-2026.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d346fccdde28abba117cc9edc696b9518c3307fbfcb689e549d9b5979018c6d"}, + {file = "regex-2026.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:415a994b536440f5011aa77e50a4274d15da3245e876e5c7f19da349caaedd87"}, + {file = "regex-2026.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21e5eb86179b4c67b5759d452ea7c48eb135cd93308e7a260aa489ed2eb423a4"}, + {file = "regex-2026.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:312ec9dd1ae7d96abd8c5a36a552b2139931914407d26fba723f9e53c8186f86"}, + {file = "regex-2026.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a0d2b28aa1354c7cd7f71b7658c4326f7facac106edd7f40eda984424229fd59"}, + {file = "regex-2026.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:349d7310eddff40429a099c08d995c6d4a4bfaf3ff40bd3b5e5cb5a5a3c7d453"}, + {file = "regex-2026.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e7ab63e9fe45a9ec3417509e18116b367e89c9ceb6219222a3396fa30b147f80"}, + {file = "regex-2026.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fe896e07a5a2462308297e515c0054e9ec2dd18dfdc9427b19900b37dfe6f40b"}, + {file = "regex-2026.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:eb59c65069498dbae3c0ef07bbe224e1eaa079825a437fb47a479f0af11f774f"}, + {file = "regex-2026.4.4-cp311-cp311-win32.whl", hash = "sha256:2a5d273181b560ef8397c8825f2b9d57013de744da9e8257b8467e5da8599351"}, + {file = "regex-2026.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:9542ccc1e689e752594309444081582f7be2fdb2df75acafea8a075108566735"}, + {file = "regex-2026.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:b5f9fb784824a042be3455b53d0b112655686fdb7a91f88f095f3fee1e2a2a54"}, + {file = "regex-2026.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:c07ab8794fa929e58d97a0e1796b8b76f70943fa39df225ac9964615cf1f9d52"}, + {file = "regex-2026.4.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c785939dc023a1ce4ec09599c032cc9933d258a998d16ca6f2b596c010940eb"}, + {file = "regex-2026.4.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b1ce5c81c9114f1ce2f9288a51a8fd3aeea33a0cc440c415bf02da323aa0a76"}, + {file = "regex-2026.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:760ef21c17d8e6a4fe8cf406a97cf2806a4df93416ccc82fc98d25b1c20425be"}, + {file = "regex-2026.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7088fcdcb604a4417c208e2169715800d28838fefd7455fbe40416231d1d47c1"}, + {file = "regex-2026.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07edca1ba687998968f7db5bc355288d0c6505caa7374f013d27356d93976d13"}, + {file = "regex-2026.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:993f657a7c1c6ec51b5e0ba97c9817d06b84ea5fa8d82e43b9405de0defdc2b9"}, + {file = "regex-2026.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2b69102a743e7569ebee67e634a69c4cb7e59d6fa2e1aa7d3bdbf3f61435f62d"}, + {file = "regex-2026.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dac006c8b6dda72d86ea3d1333d45147de79a3a3f26f10c1cf9287ca4ca0ac3"}, + {file = "regex-2026.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:50a766ee2010d504554bfb5f578ed2e066898aa26411d57e6296230627cdefa0"}, + {file = "regex-2026.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9e2f5217648f68e3028c823df58663587c1507a5ba8419f4fdfc8a461be76043"}, + {file = "regex-2026.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:39d8de85a08e32632974151ba59c6e9140646dcc36c80423962b1c5c0a92e244"}, + {file = "regex-2026.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:55d9304e0e7178dfb1e106c33edf834097ddf4a890e2f676f6c5118f84390f73"}, + {file = "regex-2026.4.4-cp312-cp312-win32.whl", hash = "sha256:04bb679bc0bde8a7bfb71e991493d47314e7b98380b083df2447cda4b6edb60f"}, + {file = "regex-2026.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:db0ac18435a40a2543dbb3d21e161a6c78e33e8159bd2e009343d224bb03bb1b"}, + {file = "regex-2026.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:4ce255cc05c1947a12989c6db801c96461947adb7a59990f1360b5983fab4983"}, + {file = "regex-2026.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:62f5519042c101762509b1d717b45a69c0139d60414b3c604b81328c01bd1943"}, + {file = "regex-2026.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3790ba9fb5dd76715a7afe34dbe603ba03f8820764b1dc929dd08106214ed031"}, + {file = "regex-2026.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fae3c6e795d7678963f2170152b0d892cf6aee9ee8afc8c45e6be38d5107fe7"}, + {file = "regex-2026.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:298c3ec2d53225b3bf91142eb9691025bab610e0c0c51592dde149db679b3d17"}, + {file = "regex-2026.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9638791082eaf5b3ac112c587518ee78e083a11c4b28012d8fe2a0f536dfb17"}, + {file = "regex-2026.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae3e764bd4c5ff55035dc82a8d49acceb42a5298edf6eb2fc4d328ee5dd7afae"}, + {file = "regex-2026.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffa81f81b80047ba89a3c69ae6a0f78d06f4a42ce5126b0eb2a0a10ad44e0b2e"}, + {file = "regex-2026.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f56ebf9d70305307a707911b88469213630aba821e77de7d603f9d2f0730687d"}, + {file = "regex-2026.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:773d1dfd652bbffb09336abf890bfd64785c7463716bf766d0eb3bc19c8b7f27"}, + {file = "regex-2026.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d51d20befd5275d092cdffba57ded05f3c436317ee56466c8928ac32d960edaf"}, + {file = "regex-2026.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:0a51cdb3c1e9161154f976cb2bef9894bc063ac82f31b733087ffb8e880137d0"}, + {file = "regex-2026.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ae5266a82596114e41fb5302140e9630204c1b5f325c770bec654b95dd54b0aa"}, + {file = "regex-2026.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c882cd92ec68585e9c1cf36c447ec846c0d94edd706fe59e0c198e65822fd23b"}, + {file = "regex-2026.4.4-cp313-cp313-win32.whl", hash = "sha256:05568c4fbf3cb4fa9e28e3af198c40d3237cf6041608a9022285fe567ec3ad62"}, + {file = "regex-2026.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:3384df51ed52db0bea967e21458ab0a414f67cdddfd94401688274e55147bb81"}, + {file = "regex-2026.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:acd38177bd2c8e69a411d6521760806042e244d0ef94e2dd03ecdaa8a3c99427"}, + {file = "regex-2026.4.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f94a11a9d05afcfcfa640e096319720a19cc0c9f7768e1a61fceee6a3afc6c7c"}, + {file = "regex-2026.4.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:36bcb9d6d1307ab629edc553775baada2aefa5c50ccc0215fbfd2afcfff43141"}, + {file = "regex-2026.4.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261c015b3e2ed0919157046d768774ecde57f03d8fa4ba78d29793447f70e717"}, + {file = "regex-2026.4.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c228cf65b4a54583763645dcd73819b3b381ca8b4bb1b349dee1c135f4112c07"}, + {file = "regex-2026.4.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dd2630faeb6876fb0c287f664d93ddce4d50cd46c6e88e60378c05c9047e08ca"}, + {file = "regex-2026.4.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a50ab11b7779b849472337191f3a043e27e17f71555f98d0092fa6d73364520"}, + {file = "regex-2026.4.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0734f63afe785138549fbe822a8cfeaccd1bae814c5057cc0ed5b9f2de4fc883"}, + {file = "regex-2026.4.4-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4ee50606cb1967db7e523224e05f32089101945f859928e65657a2cbb3d278b"}, + {file = "regex-2026.4.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6c1818f37be3ca02dcb76d63f2c7aaba4b0dc171b579796c6fbe00148dfec6b1"}, + {file = "regex-2026.4.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f5bfc2741d150d0be3e4a0401a5c22b06e60acb9aa4daa46d9e79a6dcd0f135b"}, + {file = "regex-2026.4.4-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:504ffa8a03609a087cad81277a629b6ce884b51a24bd388a7980ad61748618ff"}, + {file = "regex-2026.4.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:70aadc6ff12e4b444586e57fc30771f86253f9f0045b29016b9605b4be5f7dfb"}, + {file = "regex-2026.4.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f4f83781191007b6ef43b03debc35435f10cad9b96e16d147efe84a1d48bdde4"}, + {file = "regex-2026.4.4-cp313-cp313t-win32.whl", hash = "sha256:e014a797de43d1847df957c0a2a8e861d1c17547ee08467d1db2c370b7568baa"}, + {file = "regex-2026.4.4-cp313-cp313t-win_amd64.whl", hash = "sha256:b15b88b0d52b179712632832c1d6e58e5774f93717849a41096880442da41ab0"}, + {file = "regex-2026.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:586b89cdadf7d67bf86ae3342a4dcd2b8d70a832d90c18a0ae955105caf34dbe"}, + {file = "regex-2026.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2da82d643fa698e5e5210e54af90181603d5853cf469f5eedf9bfc8f59b4b8c7"}, + {file = "regex-2026.4.4-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:54a1189ad9d9357760557c91103d5e421f0a2dabe68a5cdf9103d0dcf4e00752"}, + {file = "regex-2026.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:76d67d5afb1fe402d10a6403bae668d000441e2ab115191a804287d53b772951"}, + {file = "regex-2026.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e7cd3e4ee8d80447a83bbc9ab0c8459781fa77087f856c3e740d7763be0df27f"}, + {file = "regex-2026.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e19e18c568d2866d8b6a6dfad823db86193503f90823a8f66689315ba28fbe8"}, + {file = "regex-2026.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7698a6f38730fd1385d390d1ed07bb13dce39aa616aca6a6d89bea178464b9a4"}, + {file = "regex-2026.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:173a66f3651cdb761018078e2d9487f4cf971232c990035ec0eb1cdc6bf929a9"}, + {file = "regex-2026.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa7922bbb2cc84fa062d37723f199d4c0cd200245ce269c05db82d904db66b83"}, + {file = "regex-2026.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:59f67cd0a0acaf0e564c20bbd7f767286f23e91e2572c5703bf3e56ea7557edb"}, + {file = "regex-2026.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:475e50f3f73f73614f7cba5524d6de49dee269df00272a1b85e3d19f6d498465"}, + {file = "regex-2026.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a1c0c7d67b64d85ac2e1879923bad2f08a08f3004055f2f406ef73c850114bd4"}, + {file = "regex-2026.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:1371c2ccbb744d66ee63631cc9ca12aa233d5749972626b68fe1a649dd98e566"}, + {file = "regex-2026.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:59968142787042db793348a3f5b918cf24ced1f23247328530e063f89c128a95"}, + {file = "regex-2026.4.4-cp314-cp314-win32.whl", hash = "sha256:59efe72d37fd5a91e373e5146f187f921f365f4abc1249a5ab446a60f30dd5f8"}, + {file = "regex-2026.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:e0aab3ff447845049d676827d2ff714aab4f73f340e155b7de7458cf53baa5a4"}, + {file = "regex-2026.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:a7a5bb6aa0cf62208bb4fa079b0c756734f8ad0e333b425732e8609bd51ee22f"}, + {file = "regex-2026.4.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:97850d0638391bdc7d35dc1c1039974dcb921eaafa8cc935ae4d7f272b1d60b3"}, + {file = "regex-2026.4.4-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ee7337f88f2a580679f7bbfe69dc86c043954f9f9c541012f49abc554a962f2e"}, + {file = "regex-2026.4.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7429f4e6192c11d659900c0648ba8776243bf396ab95558b8c51a345afeddde6"}, + {file = "regex-2026.4.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4f10fbd5dd13dcf4265b4cc07d69ca70280742870c97ae10093e3d66000359"}, + {file = "regex-2026.4.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a152560af4f9742b96f3827090f866eeec5becd4765c8e0d3473d9d280e76a5a"}, + {file = "regex-2026.4.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54170b3e95339f415d54651f97df3bff7434a663912f9358237941bbf9143f55"}, + {file = "regex-2026.4.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:07f190d65f5a72dcb9cf7106bfc3d21e7a49dd2879eda2207b683f32165e4d99"}, + {file = "regex-2026.4.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9a2741ce5a29d3c84b0b94261ba630ab459a1b847a0d6beca7d62d188175c790"}, + {file = "regex-2026.4.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b26c30df3a28fd9793113dac7385a4deb7294a06c0f760dd2b008bd49a9139bc"}, + {file = "regex-2026.4.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:421439d1bee44b19f4583ccf42670ca464ffb90e9fdc38d37f39d1ddd1e44f1f"}, + {file = "regex-2026.4.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b40379b53ecbc747fd9bdf4a0ea14eb8188ca1bd0f54f78893a39024b28f4863"}, + {file = "regex-2026.4.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:08c55c13d2eef54f73eeadc33146fb0baaa49e7335eb1aff6ae1324bf0ddbe4a"}, + {file = "regex-2026.4.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9776b85f510062f5a75ef112afe5f494ef1635607bf1cc220c1391e9ac2f5e81"}, + {file = "regex-2026.4.4-cp314-cp314t-win32.whl", hash = "sha256:385edaebde5db5be103577afc8699fea73a0e36a734ba24870be7ffa61119d74"}, + {file = "regex-2026.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:5d354b18839328927832e2fa5f7c95b7a3ccc39e7a681529e1685898e6436d45"}, + {file = "regex-2026.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:af0384cb01a33600c49505c27c6c57ab0b27bf84a74e28524c92ca897ebdac9d"}, + {file = "regex-2026.4.4.tar.gz", hash = "sha256:e08270659717f6973523ce3afbafa53515c4dc5dcad637dc215b6fd50f689423"}, +] + +[[package]] +name = "ruff" +version = "0.15.12" +requires_python = ">=3.7" +summary = "An extremely fast Python linter and code formatter, written in Rust." +groups = ["check"] +files = [ + {file = "ruff-0.15.12-py3-none-linux_armv6l.whl", hash = "sha256:f86f176e188e94d6bdbc09f09bfd9dc729059ad93d0e7390b5a73efe19f8861c"}, + {file = "ruff-0.15.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e3bcd123364c3770b8e1b7baaf343cc99a35f197c5c6e8af79015c666c423a6c"}, + {file = "ruff-0.15.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fe87510d000220aa1ed530d4448a7c696a0cae1213e5ec30e5874287b66557b5"}, + {file = "ruff-0.15.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84a1630093121375a3e2a95b4a6dc7b59e2b4ee76216e32d81aae550a832d002"}, + {file = "ruff-0.15.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb129f40f114f089ebe0ca56c0d251cf2061b17651d464bb6478dc01e69f11f5"}, + {file = "ruff-0.15.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0c862b172d695db7598426b8af465e7e9ac00a3ea2a3630ee67eb82e366aaa6"}, + {file = "ruff-0.15.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2849ea9f3484c3aca43a82f484210370319e7170df4dfe4843395ddf6c57bc33"}, + {file = "ruff-0.15.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e77c7e51c07fe396826d5969a5b846d9cd4c402535835fb6e21ce8b28fef847"}, + {file = "ruff-0.15.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b2f4f2f3b1026b5fb449b467d9264bf22067b600f7b6f41fc5958909f449d0"}, + {file = "ruff-0.15.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9ba3b8f1afd7e2e43d8943e55f249e13f9682fde09711644a6e7290eb4f3e339"}, + {file = "ruff-0.15.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e852ba9fdc890655e1d78f2df1499efbe0e54126bd405362154a75e2bde159c5"}, + {file = "ruff-0.15.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dd8aed930da53780d22fc70bdf84452c843cf64f8cb4eb38984319c24c5cd5fd"}, + {file = "ruff-0.15.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01da3988d225628b709493d7dc67c3b9b12c0210016b08690ef9bd27970b262b"}, + {file = "ruff-0.15.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9cae0f92bd5700d1213188b31cd3bdd2b315361296d10b96b8e2337d3d11f53e"}, + {file = "ruff-0.15.12-py3-none-win32.whl", hash = "sha256:d0185894e038d7043ba8fd6aee7499ece6462dc0ea9f1e260c7451807c714c20"}, + {file = "ruff-0.15.12-py3-none-win_amd64.whl", hash = "sha256:c87a162d61ab3adca47c03f7f717c68672edec7d1b5499e652331780fe74950d"}, + {file = "ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f"}, + {file = "ruff-0.15.12.tar.gz", hash = "sha256:ecea26adb26b4232c0c2ca19ccbc0083a68344180bba2a600605538ce51a40a6"}, +] + +[[package]] +name = "six" +version = "1.17.0" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +summary = "Python 2 and 3 compatibility utilities" +groups = ["default"] +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + +[[package]] +name = "tomli-w" +version = "1.2.0" +requires_python = ">=3.9" +summary = "A lil' TOML writer" +groups = ["test"] +files = [ + {file = "tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90"}, + {file = "tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021"}, +] + +[[package]] +name = "tox" +version = "4.53.0" +requires_python = ">=3.10" +summary = "tox is a generic virtualenv management and test command line tool" +groups = ["test"] +dependencies = [ + "cachetools>=7.0.3", + "colorama>=0.4.6", + "filelock>=3.25", + "packaging>=26", + "platformdirs>=4.9.4", + "pluggy>=1.6", + "pyproject-api>=1.10", + "python-discovery>=1.2.2", + "tomli-w>=1.2", + "tomli>=2.4; python_version < \"3.11\"", + "typing-extensions>=4.15; python_version < \"3.11\"", + "virtualenv>=21.1", +] +files = [ + {file = "tox-4.53.0-py3-none-any.whl", hash = "sha256:cc4e716d18c4889aa179d785175c438fa60c35deef20ce689ec288d8fb656096"}, + {file = "tox-4.53.0.tar.gz", hash = "sha256:62c780e42f87d34ee60f2ea20342156253794fdcbd6885fd797d98ee05009f22"}, +] + +[[package]] +name = "tqdm" +version = "4.67.3" +requires_python = ">=3.7" +summary = "Fast, Extensible Progress Meter" +groups = ["default"] +dependencies = [ + "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"", +] +files = [ + {file = "tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf"}, + {file = "tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb"}, +] + +[[package]] +name = "transliterate" +version = "1.10.2" +summary = "Bi-directional transliterator for Python" +groups = ["default"] +dependencies = [ + "six>=1.1.0", +] +files = [ + {file = "transliterate-1.10.2-py2.py3-none-any.whl", hash = "sha256:010a5021bf6021689c4fade0985f3f7b3db1f2f16a48a09a56797f171c08ed42"}, + {file = "transliterate-1.10.2.tar.gz", hash = "sha256:bc608e0d48e687db9c2b1d7ea7c381afe0d1849cad216087d8e03d8d06a57c85"}, +] + +[[package]] +name = "unidecode" +version = "1.4.0" +requires_python = ">=3.7" +summary = "ASCII transliterations of Unicode text" +groups = ["default"] +files = [ + {file = "Unidecode-1.4.0-py3-none-any.whl", hash = "sha256:c3c7606c27503ad8d501270406e345ddb480a7b5f38827eafe4fa82a137f0021"}, + {file = "Unidecode-1.4.0.tar.gz", hash = "sha256:ce35985008338b676573023acc382d62c264f307c8f7963733405add37ea2b23"}, +] + +[[package]] +name = "virtualenv" +version = "21.2.4" +requires_python = ">=3.8" +summary = "Virtual Python Environment builder" +groups = ["test"] +dependencies = [ + "distlib<1,>=0.3.7", + "filelock<4,>=3.24.2; python_version >= \"3.10\"", + "filelock<=3.19.1,>=3.16.1; python_version < \"3.10\"", + "importlib-metadata>=6.6; python_version < \"3.8\"", + "platformdirs<5,>=3.9.1", + "python-discovery>=1.2.2", + "typing-extensions>=4.13.2; python_version < \"3.11\"", +] +files = [ + {file = "virtualenv-21.2.4-py3-none-any.whl", hash = "sha256:29d21e941795206138d0f22f4e45ff7050e5da6c6472299fb7103318763861ac"}, + {file = "virtualenv-21.2.4.tar.gz", hash = "sha256:b294ef68192638004d72524ce7ef303e9d0cf5a44c95ce2e54a7500a6381cada"}, +] + +[[package]] +name = "wcwidth" +version = "0.6.0" +requires_python = ">=3.8" +summary = "Measures the displayed width of unicode strings in a terminal" +groups = ["default"] +files = [ + {file = "wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad"}, + {file = "wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7e0f34e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,103 @@ +[project] +name = "demeuk" +dynamic = ["version"] +description = "A command-line tool to clean up corpora." +authors = [ + {name = "Netherlands Forensic Institute"}, +] +dependencies = [ + "multiprocess>=0.70.19", + "tqdm>=4.67.3", + "ftfy>=6.3.1", + "nltk>=3.9.4", + "chardet>=7.4.3", + "transliterate>=1.10.2", + "unidecode>=1.4.0", +] +requires-python = ">=3.12" +readme = "README.md" +license = {text = "Apache-2.0"} + +[project.scripts] +demeuk = "demeuk.demeuk:cli_entry_point" + +[project.optional-dependencies] +test = [ + "coverage>=7.13.5", + "pytest>=9.0.3", + "tox>=4.53.0", + "pytest-timeout>=2.4.0", +] +check = [ + "ruff>=0.15.11", + "mdformat>=1.0.0", +] +[tool.pdm] +distribution = true + +[tool.pdm.version] +source = "file" +path = "src/demeuk/demeuk.py" +pattern = "version = '([^']+)'" + +[tool.pdm.build] +package-dir = "src" + +[tool.pdm.scripts] +all = {composite = [ + "check", + "test" +]} +check = {composite = [ + "pdm lock --check", + "ruff check src/demeuk/ tests/", + "mdformat --check README.md", +]} +fix = {composite = [ + "ruff check --fix src/demeuk/ tests/", + "mdformat README.md", +]} +test = {composite = [ + "tox", + "coverage run --branch --module pytest -s --junit-xml pytest.xml tests/", + "coverage xml", +]} + +[tool.ruff] +line-length = 120 + +[tool.ruff.format] +quote-style = "single" +exclude = ["tests/*.py"] + +[tool.ruff.lint] +select = [ + "E", # Errors + "F", # Pyflakes rules + "I", # Imports + "Q", # Quotes + "RUF100", # Unused noqa + "RUF101", # redirected noqa + "TD", # Check for todo comments + "UP" # pyupgrade +] +ignore = [ + "F403", "F405", # from module import * + "UP012", # explicitly encode tests as utf-8 + "UP022", # use pipe in test to simulate unix pipe +] +flake8-quotes.inline-quotes = "single" +isort.lines-after-imports = 2 + +[tool.mdformat] +end_of_line = "lf" +wrap = 120 + +[tool.pytest.ini_options] +markers = [ + "timeout: maximum runtime for a test", +] + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index cfcfae2..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -docopt -chardet==5.2.0 -nltk -ftfy -unidecode -tqdm -transliterate diff --git a/setup.py b/setup.py deleted file mode 100644 index 3c9ef8e..0000000 --- a/setup.py +++ /dev/null @@ -1,40 +0,0 @@ -import codecs - -from setuptools import setup - -from bin.demeuk import version - -dependencies = [ - 'docopt', - 'chardet', - 'nltk', - 'ftfy', - 'unidecode', - 'tqdm', - 'transliterate' -] - -with open('README.md', 'r') as r: - long_description = r.read() - -setup( - name='demeuk', - version=version, - author='Netherlands Forensic Institute', - author_email=codecs.encode('ubyzrfay@hfref.abercyl.tvguho.pbz', 'rot-13'), - description='CLI tool to remove invalid chars from a corpus.', - install_requires=dependencies, - scripts=['bin/demeuk.py'], - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/NetherlandsForensicInstitute/demeuk", - classifiers=[ - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], -) diff --git a/tests/__init__.py b/src/demeuk/__init__.py similarity index 100% rename from tests/__init__.py rename to src/demeuk/__init__.py diff --git a/src/demeuk/config.py b/src/demeuk/config.py new file mode 100644 index 0000000..b658029 --- /dev/null +++ b/src/demeuk/config.py @@ -0,0 +1,99 @@ +from glob import glob +from locale import LC_ALL, setlocale +from os import R_OK, access, cpu_count +from string import punctuation as string_punctuation + +from .logger import Logger +from .output import OutputFileHandler + + +# This class carries global configuration, so configurations which either: +# do not impact the functionality of the modules directly. +# or: do impact modules, but cannot be passed as a parameter +class Config: + """ + A class containing all configuration for demeuk + """ + + # Initialize config with argparse output + def __init__(self, args): + """ + Initialize config based on command-line arguments + :param args: Parsed command line arguments, available in CommandLineParser.args after calling parse_args() + """ + # I/O + self.input_files = args.input + self.output_file = args.output + self.log_file = args.log + + # Verbosity + self.progress = args.progress + self.verbose = args.verbose + self.debug = args.debug + + # Encodings (set these before logger, as logger opens output files) + self.input_encodings = args.input_encoding.split(',') if args.input_encoding else ['UTF-8'] + + if args.output_encoding is not None: + setlocale(LC_ALL, args.output_encoding) + else: + setlocale(LC_ALL, 'en_US.UTF-8') + + self.logger = Logger(args) + + self.output_fh = OutputFileHandler(args, self.logger) + + # Check if we can read input file + if args.input: + # args.input is a list (nargs *) + if len(args.input) > 1: + # Pass multiple files through command-line + self.input_files = args.input + else: + self.input_files = glob(args.input[0], recursive=True) + + for input_file in self.input_files: + if not access(input_file, R_OK): + self.logger.stderr_print_always(f'Config: Cannot read input file from {input_file}!') + + + if self.progress: + if self.verbose or self.debug: + if not self.log_file: + self.logger.stderr_print_always('Config: --progress cannot be used with --verbose or --debug!') + exit(2) + if not self.input_files: + self.logger.stderr_print_always('Config: --progress cannot be used when using stdin!') + exit(2) + + # Other configurations here, with defaults + self.threads = int(args.threads) if args.threads else cpu_count() + self.chunk_size = 1024 * 1024 + self.skip = args.skip if args.skip else 0 + self.limit = args.limit + # NB: a space in the punctiation list is not supported + self.punctuation = args.punctuation if args.punctuation else string_punctuation + ' ' + + # Delimiter determination + # config.delimiter is a list. + if args.delimiter: + splitter = ',' + # We can have comma as delimiter, if we put it first and separate with semicolon. + if len(args.delimiter) >= 1: + if args.delimiter[0] == ',': + splitter = ';' + self.delimiters = args.delimiter.split(splitter) + else: + self.delimiters = [':'] + + # Config for cut + self.cut_fields = '2-' + if args.cut_before: + self.cut_fields = '-1' + if args.cut_fields: + # --cut-fields overrides --cut-before + self.cut_fields = args.cut_fields + + self.cut_before = True if args.cut_before else False + + diff --git a/src/demeuk/demeuk.py b/src/demeuk/demeuk.py new file mode 100644 index 0000000..b5624d2 --- /dev/null +++ b/src/demeuk/demeuk.py @@ -0,0 +1,146 @@ +import sys +from math import ceil +from os import cpu_count, linesep, path + +from os import name as os_name +if os_name == 'nt': + from multiprocess.pool import ThreadPool as Pool +else: + from multiprocess.pool import Pool +from tqdm import tqdm + +from .config import Config +from .discover import discover_modules +from .multiproc import chunkify, finish_up, init_worker, submit, write_results +from .parser import CommandLineParser +from .pipeline import Pipeline + + +""" +Demeuk is a tool to clean up lists of words. +""" + +def get_version(): + version = '5.0.0' + return version + + +def cli_entry_point(): + """ + Entry point for demeuk, when run from the command-line. + """ + run_cli(sys.argv) + +def run_cli(args): + """ + Invoke demeuk with command-line arguments + + :param args: A list of command-line arguments, split by space + :type args: list + """ + all_modules = discover_modules() + + parser = CommandLineParser(get_version()) + + + for module in all_modules: + parser.register(module) + + # Argparse validates arguments + parser.parse_args(args) + + cfg = Config(parser.args) + + # Global config can be done here (in/out file, log etc.) + + pipeline = Pipeline(parser, args, cfg) + + cfg.logger.write(f'Running demeuk - {get_version()}{linesep}') + cfg.logger.write(f'Running pipeline {[module.__class__.__name__ for module in pipeline.modules]}\n') + + cfg.logger.stderr_print(f'Running demeuk - {get_version()}') + cfg.logger.stderr_print(f'Using {cfg.threads} out of {cpu_count()} available CPUs') + + if cfg.input_files is not None: + if cfg.threads > 1: + demeuk_files(pipeline, cfg.input_files, cfg) + else: + demeuk_files_single_threaded(pipeline, cfg.input_files, cfg) + else: + demeuk_stdin(pipeline, cfg) + + cfg.logger.stderr_print('Done') + cfg.logger.close() + cfg.output_fh.close() + + +def demeuk_files(pipeline, input_files, cfg): + """ + Demeuk a list of input files + + :param pipeline: The module pipeline to run + :type pipeline: :class:`Pipeline` + :param input_files: A list of input files + :type input_files: list + :param cfg: Configuration + :type cfg: :class:`Config` + """ + with Pool(cfg.threads, init_worker) as pool: + cfg.logger.stderr_print('Reading input file(s)...') + + jobs = [] + + for file in tqdm(input_files, + desc='Files processed', + mininterval=0.5, + unit=' files', + disable=not cfg.progress, + position=0): + total_chunks = ceil(path.getsize(file) / cfg.chunk_size) + for chunk in tqdm(chunkify(file, cfg), + desc='Chunks processed', + mininterval=0.5, + unit=' chunks', + disable=not cfg.progress, + total=total_chunks, + position=1): + submit(pool, jobs, pipeline, chunk, cfg) + cfg.logger.stderr_print('Submitted jobs, waiting for jobs to finish...') + + # Wait for jobs to finish + finish_up(jobs, cfg) + +# For profiling +def demeuk_files_single_threaded(pipeline, input_files, cfg): + cfg.logger.stderr_print('Reading input file(s)...') + with open(input_files[0], 'rb') as fh: + lines = [line.rstrip(linesep.encode()) for line in fh.readlines(cfg.chunk_size)] + cfg.logger.stderr_print('Submitted jobs, waiting for jobs to finish...') + + res = pipeline.run(lines, cfg) + write_results(cfg.output_fh, cfg.logger, res) + +def demeuk_stdin(pipeline, cfg): + """ + Demeuk input from stdin + + :param pipeline: The module pipeline to run + :type pipeline: :class:`Pipeline` + :param cfg: Config object + :type cfg: :class:`Config` + """ + with Pool(cfg.threads, init_worker) as pool: + cfg.logger.stderr_print('Reading from stdin...') + + jobs = [] + + chunks = sys.stdin.readlines(cfg.chunk_size) + while chunks: + chunk = [line.rstrip('\n').encode(cfg.input_encodings[0]) for line in chunks] + submit(pool, jobs, pipeline, chunk, cfg) + + chunks = sys.stdin.readlines(cfg.chunk_size) + cfg.logger.stderr_print('Submitted jobs, waiting for jobs to finish...') + + # Wait for jobs to finish + finish_up(jobs, cfg) \ No newline at end of file diff --git a/src/demeuk/discover.py b/src/demeuk/discover.py new file mode 100644 index 0000000..097dd5b --- /dev/null +++ b/src/demeuk/discover.py @@ -0,0 +1,49 @@ +import importlib.util +import inspect +from pathlib import Path + +from demeuk.modules.base import Module + + +""" +Discovers demeuk modules dynamically. +""" + +def class_name(cls): + """Utility function to determine sorting of command-line options""" + return cls.__name__ + +def discover_modules(): + """ + Discover demeuk modules in src/demeuk/modules. + Any class which is a subclass of Module is detected and registered automatically. + """ + root_dir = Path(__file__).parent + modules_dir = root_dir / 'modules' + + classes = set() + + # Hardcoded list of classes not to register. + # This is because Python does not distinguish between concrete and abstract subclasses. + blacklist = [ + 'Module', 'ParamModule', 'ConfigModule', # Base modules + 'CheckModule', 'AddModule', 'ModifyModule', 'MacroModule', 'RemoveModule', # Module types + ] + + # Path.walk is Python 3.12+ + for path, names, files in modules_dir.walk(): + for file in [path/file for file in files]: + # This way file is a Path object instead of a string. + if file.suffix == '.py' and file.stem != '__init__': + module_name = str(file.relative_to(modules_dir)) + # Turn modify/hex.py into .modify.hex + module_name = '.' + module_name.replace('/', '.').replace('.py', '') + # Get all python objects in the file + members = inspect.getmembers(importlib.import_module(module_name, 'demeuk.modules')) + for name, obj in members: + # Save only the class-type objects which are not blacklisted. These should only be the modules. + if inspect.isclass(obj): + if issubclass(obj, Module) and name not in blacklist: + classes |= {obj} + + return sorted(classes, key=class_name) diff --git a/src/demeuk/logger.py b/src/demeuk/logger.py new file mode 100644 index 0000000..fc2da7d --- /dev/null +++ b/src/demeuk/logger.py @@ -0,0 +1,106 @@ +from locale import getlocale +from sys import stderr + + +# Manage logging tasks +class Logger: + """ + Manage logging tasks + + Verbose messages are info messages on program execution, for example a message when all lines are submitted to the pool. + Debug messages are more detailed, and these log information about the pipeline run (anytime a module returns a Result with status=True, its corresponding debug message is logged). + """ + + def __init__(self, args): + """ + Open log file, and set logging flags based on command-line arguments + Keeps a record of the log, to write it only once per chunk. + :param args: Parsed command-line arguments + """ + # verbosity + self.verbose = args.verbose + self.debug = args.debug + + + encoding = getlocale()[1] + # Check if we can write to output and log files + if args.log: + try: + self.log_file = open(args.log, 'a+', encoding=encoding, newline='') # Append or write + except PermissionError: + self.stderr_print_always(f'Logger: Cannot write log file to {args.log}!') + exit(2) + else: + self.log_file = stderr + self.stderr_print('Logger: writing log to stderr') + + self.logs = [] + + # Do we want to separate stderr print and log_verbose? + def stderr_print(self, *args, **kwargs): + """ + Print a message to stderr when --verbose is set. + :param args: The message to print + :param kwargs: Any kwargs to pass to print. + """ + if self.verbose: + Logger.stderr_print_always(*args, **kwargs) + + def log(self, msg): + """ + Add a line to the log record + :param msg: The message to log + """ + self.logs.append(msg) + + def log_debug(self, msg): + """ + Add a line to the log record if --debug is set + :param msg: The message to log + """ + if self.debug: + self.logs.append(msg) + + def log_verbose(self, msg): + """ + Add a line to the log record if --verbose is set + :param msg: The message to log + """ + if self.verbose: + self.logs.append(msg) + + def get(self): + """ + Get the log record + :return: A list of strings containing the logs + """ + return self.logs + + + def write(self, lines): + """ + Write (and flush) a batch of lines to the logfile + :param lines: A list of lines (with explicit newlines) + """ + if self.debug or self.verbose or (self.log_file is not stderr): + self.log_file.writelines(lines) + self.log_file.flush() + + + # Print to stderr always, use for errors or incorrect input + @staticmethod + def stderr_print_always(*args, **kwargs): + """ + Print a message to stderr + :param args: The message to print + :param kwargs: Any kwargs to pass to print + """ + kwargs.setdefault('file', stderr) + print(*args, **kwargs) + + def close(self): + """ + Close the file handle to the log file + """ + if self.log_file != stderr: + self.log_file.close() \ No newline at end of file diff --git a/src/demeuk/modules/__init__.py b/src/demeuk/modules/__init__.py new file mode 100644 index 0000000..139597f --- /dev/null +++ b/src/demeuk/modules/__init__.py @@ -0,0 +1,2 @@ + + diff --git a/src/demeuk/modules/add/__init__.py b/src/demeuk/modules/add/__init__.py new file mode 100644 index 0000000..f16145c --- /dev/null +++ b/src/demeuk/modules/add/__init__.py @@ -0,0 +1,5 @@ +from .add import AddModule +from .capitalization import * +from .latin_ligatures import * +from .punctuation import * +from .umlaut import * diff --git a/src/demeuk/modules/add/add.py b/src/demeuk/modules/add/add.py new file mode 100644 index 0000000..bd2bb16 --- /dev/null +++ b/src/demeuk/modules/add/add.py @@ -0,0 +1,51 @@ + +from ..base import * + + +# For certain string checks/operations, we maybe want to construct an Add, Remove and Modify(Clean) module in one go? +# For example umlauting +class AddModule(Module): + """ + The abstract base class for add modules. + Add modules can add variants of lines to the work queue + """ + @staticmethod + def get_pipeline_position(): + return PipelinePosition.AFTER_ENCODE + + @property + def debug_str(self) -> str: + return 'added line' + + def handle(self, result): + """ + Handle the result of AddModule.run(). Expects a Result with the add field set and a debug message, as returned from get_result(). + + :param result: Result of an AddModule run() + :type result: Result + :return: Actions object which adds the line(s) to the queue. + """ + # Add either a string or list of strings to the queue + if isinstance(result.add, list): + add_list = result.add + else: + add_list = [result.add] + # Log a message for every added line if --debug is set. + return Actions( + add=add_list, + debug_add_str=result.msg) + + def get_result(self, line, added_line): + """ + Utility function to get the appropriate Result object when (possibly) adding one line, to return from run(). + Checks if the new candidate is equal to the input line and only adds it if they differ. + + :param line: The input line + :type line: str + :param added_line: The line to add + :type added_line: str + :return: The result to return from run() + """ + if line != added_line: + return Result(status=True, msg=self.debug_str, add=added_line) + return RESULT_NEXT \ No newline at end of file diff --git a/src/demeuk/modules/add/capitalization.py b/src/demeuk/modules/add/capitalization.py new file mode 100644 index 0000000..907fca4 --- /dev/null +++ b/src/demeuk/modules/add/capitalization.py @@ -0,0 +1,38 @@ +from ..base import * +from .add import AddModule + + +class FirstUpperModule(AddModule): + + @staticmethod + def get_help_info(): + return HelpInfo( + option='add-first-upper', + help_str='If a line does not contain a capital letter this will add the capital variant.') + + def run(self, line): + add_line = line.capitalize() + return self.get_result(line, add_line) + +# Might be confused with LowercaseModule... +class LowerModule(AddModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='add-lower', + help_str='If a line contains a capital letter this will add the lowercase variant.') + + def run(self, line): + add_line = line.lower() + return self.get_result(line, add_line) + +class TitleCaseModule(AddModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='add-title-case', + help_str='Add a line with every word capitalized.') + + def run(self, line): + add_line = line.title() + return self.get_result(line, add_line) \ No newline at end of file diff --git a/src/demeuk/modules/add/latin_ligatures.py b/src/demeuk/modules/add/latin_ligatures.py new file mode 100644 index 0000000..eef6628 --- /dev/null +++ b/src/demeuk/modules/add/latin_ligatures.py @@ -0,0 +1,16 @@ +from ftfy.fixes import fix_latin_ligatures + +from ..base import * +from .add import AddModule + + +class LatinLigaturesModule(AddModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='add-latin-ligatures', + help_str='If a line contains ligatures of Latin letter (such as ij), the line is correct but the original line containing the ligatures is also added to output.') + + def run(self, line): + add_line = fix_latin_ligatures(line) + return self.get_result(line, add_line) \ No newline at end of file diff --git a/src/demeuk/modules/add/punctuation.py b/src/demeuk/modules/add/punctuation.py new file mode 100644 index 0000000..3d2d000 --- /dev/null +++ b/src/demeuk/modules/add/punctuation.py @@ -0,0 +1,36 @@ +from re import split as re_split + +from ..base import * +from .add import AddModule + + +class SplitModule(AddModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='add-split', + help_str="Split on known characters like '-' and '.'") + + def run(self, line): + punctuation = (' ', '-', r'\.') + for p in punctuation: + if p in line: + # Why only if len(piece) > 1? + components = [piece for piece in re_split('|'.join(punctuation), line) if len(piece) > 1] + return Result(status=True, msg=self.debug_str, add=components) + return Result(status=False, msg=None) + +class WithoutPunctuationModule(AddModule, ConfigModule): + # This is related to PunctuationModule in a way... Can we link these? + + def set_configs(self, config): + self.add_config('punctuation', config.punctuation) + @staticmethod + def get_help_info(): + return HelpInfo( + option='add-without-punctuation', + help_str='If a line contains punctuation, a variant will be added without punctuation.') + + def run(self, line): + cleaned_line = line.translate(str.maketrans('', '', self.get_config('punctuation'))) + return self.get_result(line, cleaned_line) \ No newline at end of file diff --git a/src/demeuk/modules/add/umlaut.py b/src/demeuk/modules/add/umlaut.py new file mode 100644 index 0000000..5689ca2 --- /dev/null +++ b/src/demeuk/modules/add/umlaut.py @@ -0,0 +1,35 @@ +from ..base import * +from .add import AddModule + + +# Looks very mych like UmlautModule (ModifyModule). +# Why do we need --umlaut AND --add-umlaut? +class UmlautAddModule(AddModule): + # Duplicated. Store somewhere else? + umlaut_dict = { + 'a"': 'ä', + 'i"': 'ï', + 'o"': 'ö', + 'u"': 'ü', + 'e"': 'ë', + 'A"': 'Ä', + 'I"': 'Ï', + 'O"': 'Ö', + 'U"': 'Ü', + 'E"': 'Ë', + } + + @staticmethod + def get_help_info(): + return HelpInfo( + option='add-umlaut', + help_str='Add line with fixed umlauting (o", U" to ö, Ü)') + + def run(self, line): + add_line = line + for pattern, replacement in self.umlaut_dict.items(): + add_line = add_line.replace(pattern, replacement) + + return self.get_result(line, add_line) + + diff --git a/src/demeuk/modules/base.py b/src/demeuk/modules/base.py new file mode 100644 index 0000000..23da279 --- /dev/null +++ b/src/demeuk/modules/base.py @@ -0,0 +1,230 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import NamedTuple + + +""" +This module contains the abstract base versions of demeuk modules, and some auxiliary help classes. +""" + +# Result of module.run +class Result(NamedTuple): + """ + The result of a module run() on a single line + """ + + #: If this is True, some action needs to be performed. If this is false, continue to the next module. + status: bool + #: A log or debug string + msg: str | None + #: Lines to add to the demeuk queue + add: str | bytes | list | None = None + #: Change this line for future modules + update: str | bytes | None = None + +# Class (namedtuple/dataclass) creation is slow! +# For the results we use often, create them once and use them everywhere + +#: This result can be used if you want to continue to the next line, and not take any action +RESULT_NEXT=Result(status=False, msg=None) + +# Result of module.handle, these can perform an action +# Note that we expect almost all of our input lines to return a Result with status=False, so almost none will go through +# to the handle phase. Therefore we don't reuse our Actions objects, even though they are slow to create. +class Actions(NamedTuple): + """ + The result of a modules handle(), run on a Result when its status is True. These encode information on + control flow and the actions to take after a module is tripped. Multiple actions can be set. + If an attribute is not None, the corresponding action will be performed. + """ + #: If True, don't do any more demeuking on this line and do not include it in the results. + stop: bool = False + #: A list of lines to add to the work queue + add: list | None = None + #: Update the current line to this string + update: str | None = None + #: A flag for use in combination with add. When set, we add back a byte sequence instead of a string into the queue + do_not_re_encode: bool = False + #: when set, log a string + log_str: str | None = None + #: When set, log a string if --debug is set + debug_str: str | None = None + #: For use in combination with add. When set, log a string for every added line if --debug is set. + debug_add_str: str | None = None + + +class HelpInfo(NamedTuple): + """ + Tuple containing 'help info' about a module which does not take a parameter. + """ + #: A string or list of strings of command-line names. These should not start with dashes. + option: str | list[str] + #: The explainer string for this module for use in demeuk -h. + help_str: str + + +class HelpInfoParam(NamedTuple): + """ + Tuple containing 'help info' about a module which does take a parameter. + """ + #: A string or list of strings of command-line names. These should not start with dashes. + option: str | list[str] + #: The explainer string for this module for use in demeuk -h. + help_str: str + #: The type of the parameter this module expects + param_type: type + #: The name by which the parameter can be reference in the help string, for example '' + metavar: str + + +#: An enum describing the different positions which a module can be in, depending on how they act on type +PipelinePosition = Enum('PipelinePosition', [ + ('BEFORE_ENCODE', 0), # Modules which act on bytes + ('ENCODE', 1), # Modules which turn bytes into strings + ('AFTER_ENCODE', 2)]) # Modules which act on strings + + + +class Module(ABC): + """ + The abstract base class for a demeuk module. If you want to add a new module to demeuk, you should probably not + implement this class unless you know what you're doing. You should instead implement one of: + AddModule, CheckModule, ModifyModule, RemoveModule or MacroModule. + """ + @staticmethod + @abstractmethod + def get_help_info() -> HelpInfo | HelpInfoParam: + """ + Return info for the command-line option and the message displayed when running demeuk -h. + + :return: The help info for this module + """ + raise NotImplementedError + + @property + @abstractmethod + def debug_str(self) -> str: + """ + Returns a small debug string describing the action of a module. + NB: When logging with --debug, the class name is logged, along with the debug string and the line in question. + + :return: Debug string + """ + raise NotImplementedError + + @abstractmethod + def run(self, line) -> Result: + """ + Run the module on a line. Note that this module is runs for every word in the word list! + + :param line: The line on which the module runs + :type line: str + :return: The result of the module operation + """ + raise NotImplementedError + + @abstractmethod + def handle(self, result): + """ + Handle the result of Module.run(). Unless you are implementing a new category of module, you should use the + implementation of AddModule, CheckModule, etc. + + :param result: The result of Module.run(). + :type result: Result + :return: An Actions object containing the actions to perform. + """ + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_pipeline_position() -> PipelinePosition: + """ + Set where in the pipeline this module should be placed. + + :return: Pipeline position + """ + raise NotImplementedError + + +class ParamModule(Module): + """ + The abstract base class for a demeuk module which takes a parameter. + If you implement this class, you need to supply the parameter to the constructor, which can then be recalled by accessing the class property param. + """ + def __init__(self, parameter): + self._param = self.get_help_info().param_type(parameter) + + @staticmethod + @abstractmethod + def get_help_info() -> HelpInfoParam: + """ + Return a HelpInfoParam tuple containing the help info for this module. + """ + raise NotImplementedError + + @property + def param(self): + """ + The parameter of the module + """ + return self._param + + @param.setter + def param(self, value): + self._param = self.get_help_info().param_type(value) + +# A module with some configuration. +class ConfigModule(Module): + """ + The abstract base class for a demeuk module which depends on configuration. + If you implement this class, you need to add key-value pairs of config values in set_configs, which can then be accessed by get_config. + This can be used to let some global config option influence multiple modules at the same time. + """ + def __init__(self): + self._config = {} + + @abstractmethod + def set_configs(self, config): + """ + Set config values for this module. You can use add_config in this function to actually store the values. + + :param config: Config object + :type config: Config + """ + raise NotImplementedError + + def add_config(self, key, value): + """ + Store a key-value pair in the config dict for this module. + + :param key: The key + :type key: Any + :param value: The value + :type value: Any + """ + self._config[key] = value + + def get_config(self, key): + """ + Retrieve a config value from the config dict + + :param key: The key + :type key: Any + :return: The associated config value + + """ + return self._config[key] + + + + + + + + + + + + + + diff --git a/src/demeuk/modules/check/__init__.py b/src/demeuk/modules/check/__init__.py new file mode 100644 index 0000000..e757c0a --- /dev/null +++ b/src/demeuk/modules/check/__init__.py @@ -0,0 +1,8 @@ +from .bounds import * +from .case import * +from .character import * +from .check import CheckModule +from .empty_line import * +from .non_ascii import * +from .regex import * +from .substr import * diff --git a/src/demeuk/modules/check/bounds.py b/src/demeuk/modules/check/bounds.py new file mode 100644 index 0000000..77dd59d --- /dev/null +++ b/src/demeuk/modules/check/bounds.py @@ -0,0 +1,118 @@ +from ..base import * +from .check import CheckModule + + +# Maybe add a BoundsCheckModule which automatically creates min/max versions based on a lambda? + +class MinLengthModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-min-length', + help_str='Require that entries contain at least characters.', + metavar='', + param_type=int) + + def run(self, line): + if len(line) < self.param: + return self.stop + return self.next + + +class MaxLengthModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-max-length', + help_str='Require that entries contain at most characters.', + metavar='', + param_type=int) + + def run(self, line): + if len(line) > self.param: + return self.stop + return self.next + +class MinDigitsModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-min-digits', + help_str='Require that entries contain at least digits.', + metavar='', + param_type=int) + + def run(self, line): + if sum([c.isdigit() for c in line]) < self.param: + return self.stop + return self.next + +class MaxDigitsModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-max-digits', + help_str='Require that entries contain at most digits.', + metavar='', + param_type=int) + + def run(self, line): + if sum([c.isdigit() for c in line]) > self.param: + return self.stop + return self.next + +class MinUppercaseModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-min-uppercase', + help_str='Require that entries contain at least uppercase characters.', + metavar='', + param_type=int) + + def run(self, line): + if sum([c.isupper() for c in line]) < self.param: + return self.stop + return self.next + +class MaxUppercaseModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-max-uppercase', + help_str='Require that entries contain at most uppercase characters.', + metavar='', + param_type=int) + + def run(self, line): + if sum([c.isupper() for c in line]) > self.param: + return self.stop + return self.next + +class MinSpecialsModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-min-special', + help_str='Require that entries contain at least special characters.', + metavar='', + param_type=int) + + def run(self, line): + if sum([not c.isalnum() and not c.isspace() for c in line]) < self.param: + return self.stop + return self.next + +class MaxSpecialsModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-max-special', + help_str='Require that entries contain at most special characters.', + metavar='', + param_type=int) + + def run(self, line): + if sum([not c.isalnum() and not c.isspace() for c in line]) > self.param: + return self.stop + return self.next diff --git a/src/demeuk/modules/check/case.py b/src/demeuk/modules/check/case.py new file mode 100644 index 0000000..5ded134 --- /dev/null +++ b/src/demeuk/modules/check/case.py @@ -0,0 +1,25 @@ +from ..base import * +from .check import CheckModule + + +class CaseModule(CheckModule): + + ignored_chars = [' ', "'", '-'] + + @staticmethod + def get_help_info(): + return HelpInfo( + option='check-case', + help_str='Drop lines where the uppercase line is not equal to the lowercase line.') + + def run(self, line): + for c in line: + c = str(c) + if c.lower() == c.upper(): + if c in self.ignored_chars: + continue + else: + return self.stop + return self.next + + diff --git a/src/demeuk/modules/check/character.py b/src/demeuk/modules/check/character.py new file mode 100644 index 0000000..42d58f9 --- /dev/null +++ b/src/demeuk/modules/check/character.py @@ -0,0 +1,39 @@ +from unicodedata import category + +from ..base import * +from .check import CheckModule + + +class ReplacementCharModule(CheckModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='check-replacement-character', + help_str="Drop lines containing replacement characters '�'.") + + def run(self, line) -> Result: + if '�' in line: + return self.stop + return self.next + +class ControlCharModule(CheckModule): + + @staticmethod + def get_help_info(): + return HelpInfo( + option='check-controlchar', + help_str='Drop lines containing control characters.') + + def run(self, line) -> Result: + for c in line: + # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category + # Characters (they have meaning): + # Cc -> Control Char (End of stream) + # Cf -> Control flow (right to left) + # Non chars: + # Cn -> Not assigned + # Co -> Private use + # Cs -> Surrogate + if category(c) in ['Cc', 'Cf', 'Cn', 'Co', 'Cs']: + return self.stop + return self.next diff --git a/src/demeuk/modules/check/check.py b/src/demeuk/modules/check/check.py new file mode 100644 index 0000000..62b151c --- /dev/null +++ b/src/demeuk/modules/check/check.py @@ -0,0 +1,51 @@ + +from ..base import * + + +# Maybe add shortcut Result object ResultPass and ResultFail or something? +class CheckModule(Module): + """ + The abstract base class for check modules. + Check modules test a certain property, and can drop a line if this property is (not) met. + """ + + @staticmethod + def get_pipeline_position(): + return PipelinePosition.AFTER_ENCODE + + @property + def debug_str(self) -> str: + return 'dropped line' + + def handle(self, result): + """ + Handle the result of CheckModule.run(). Expects a result with a log message, like stop() or next(). + + :param result: Result of a CheckModule run() + :type result: Result + :return: Actions object which stops further processing + """ + return Actions( + # If a check module is tripped, don't need to run any more modules. + stop=True, + log_str=result.msg) # Always log if this happens + + @property + def stop(self) -> Result: + """ + Return this from run() to stop further processing + + :return: A Result object to pass to handle() + """ + return Result(status=True, msg=self.debug_str) + + @property + def next(self) -> Result: + """ + Return this from run() to continue to the next module + + :return: A Result object to pass to handle() + """ + # Creating a Result is slow so we reference one instance. + # For stop, this is not needed as we expect almost all lines to return next. + return RESULT_NEXT diff --git a/src/demeuk/modules/check/empty_line.py b/src/demeuk/modules/check/empty_line.py new file mode 100644 index 0000000..9b50aaa --- /dev/null +++ b/src/demeuk/modules/check/empty_line.py @@ -0,0 +1,15 @@ +from ..base import * +from .check import CheckModule + + +class EmptyLineModule(CheckModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='check-empty-line', + help_str='Drop lines that are empty or only contain whitespace character.') + + def run(self, line): + if line == '' or line.isspace(): + return self.stop + return self.next diff --git a/src/demeuk/modules/check/non_ascii.py b/src/demeuk/modules/check/non_ascii.py new file mode 100644 index 0000000..83afb96 --- /dev/null +++ b/src/demeuk/modules/check/non_ascii.py @@ -0,0 +1,18 @@ +from ..base import * +from .check import CheckModule + + +# Name conflict with NonAsciiModule +class NonAsciiCheckModule(CheckModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='check-non-ascii', + help_str='If a line contain a non ascii char e.g. ü or ç (everything outside ascii range) the line is dropped.') + + def run(self, line): + try: + line.encode('ascii') + return self.next + except UnicodeEncodeError: + return self.stop \ No newline at end of file diff --git a/src/demeuk/modules/check/regex.py b/src/demeuk/modules/check/regex.py new file mode 100644 index 0000000..97e37ae --- /dev/null +++ b/src/demeuk/modules/check/regex.py @@ -0,0 +1,110 @@ +from re import search + +from ..base import * +from .check import CheckModule + + +# Do we want a generic RegexCheckModule which we can extend with different regexes we choose? +# This would cut back on duplicate even more, but maybe not necessary. + +class RegexModule(CheckModule, ParamModule): + + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-regex', + help_str='Drop lines that do not match the regex. Option expects a comma-separated list of regexes.' + 'For example: [a-z]{1,8},[0-9]{1,8}.', + metavar='', + param_type=str) + + def run(self, line): + # We cannot have a regex containing a comma + for regex in self.param.split(','): + if search(regex, line): + continue + else: + return self.stop + return self.next + + +class EmailModule(CheckModule): + + EMAIL_REGEX = '.{1,64}@([a-zA-Z0-9_-]{1,63}\\.){1,3}[a-zA-Z]{2,6}' + + @staticmethod + def get_help_info(): + return HelpInfo( + option='check-email', + help_str='Drop lines containing e-mail addresses.') + + def run(self, line) -> Result: + if search(self.EMAIL_REGEX, line): + return self.stop + return self.next + +class MacAddressModule(CheckModule): + + MAC_REGEX = '^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$' + + @staticmethod + def get_help_info(): + return HelpInfo( + option='check-mac-address', + help_str='Drop lines containing MAC addresses.') + + + def run(self, line) -> Result: + if search(self.MAC_REGEX, line): + return self.stop + return self.next + +class UuidModule(CheckModule): + + UUID_REGEX = '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' + + @staticmethod + def get_help_info(): + return HelpInfo( + option='check-uuid', + help_str='Drop lines containing UUIDs.') + + + def run(self, line): + if search(self.UUID_REGEX, line): + return self.stop + return self.next + +class HashModule(CheckModule): + # Official bcrypt hashes have a bit more fixed size, but saw some weird once: + + # $1a$10$demo as example + HASH_BCRYPT_REGEX = '^\\$2[ayb]\\$[0-9]{1,}\\$[\\w\\.\\/]{4,}$' + # Crypt hashes can look a lot like passwords. We do two options here + # $0[$optional salt, max 16]$string of a-zA-Z0-9./ length 7 min till end of line + # $0$a-zA-Z0-9./ min length 12 to make sure we hit somthing like: a-zA-Z0-9./ + # this will cause string like $0$JAjdna./d to still be included. + + HASH_HEX_REGEX = '^[a-fA-F0-9]+$' + + HASH_CRYPT_REGEX = '^\\$[1356]\\$[\\w\\.\\/]{12,}$' + HASH_CRYPT_SALT_REGEX = '^\\$[1356]\\$[\\w\\.\\/\\+]{,16}\\$[\\w\\.\\/]{6,}$' + HASH_PHPBB_REGEX = '^\\$[hH]\\$[\\w\\.\\/]{5,}$' + HASH_REGEX_LIST = [HASH_BCRYPT_REGEX, HASH_CRYPT_SALT_REGEX, HASH_CRYPT_REGEX, HASH_PHPBB_REGEX] + + @staticmethod + def get_help_info(): + return HelpInfo( + option='check-hash', + help_str='Drop lines which are hashes') + + def run(self, line): + if len(line) in [32, 40, 64]: + if search(self.HASH_HEX_REGEX, line): + return self.stop + if len(line) > 0: + if line[0] == '$': + for hash_regex in self.HASH_REGEX_LIST: + if search(hash_regex, line): + return self.stop + return self.next \ No newline at end of file diff --git a/src/demeuk/modules/check/substr.py b/src/demeuk/modules/check/substr.py new file mode 100644 index 0000000..51c246d --- /dev/null +++ b/src/demeuk/modules/check/substr.py @@ -0,0 +1,48 @@ +from ..base import * +from .check import CheckModule + + +class StartingWithModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-starting-with', + help_str='Drop lines starting with a string, specify multiple with comma-separated list.', + metavar='', + param_type=str) + + def run(self, line): + for substr in self.param.split(','): + if line.startswith(substr): + return self.stop + return self.next + +class EndingWithModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-ending-with', + help_str='Drop lines ending with a string, specify multiple with comma-separated list.', + metavar='', + param_type=str) + + def run(self, line): + for substr in self.param.split(','): + if line.endswith(substr): + return self.stop + return self.next + +class ContainsModule(CheckModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='check-contains', + help_str='Drop lines containing a string, specify multiple with comma-separated list.', + metavar='', + param_type=str) + + def run(self, line): + for substr in self.param.split(','): + if substr in line: + return self.stop + return self.next diff --git a/src/demeuk/modules/macro/__init__.py b/src/demeuk/modules/macro/__init__.py new file mode 100644 index 0000000..fe0d473 --- /dev/null +++ b/src/demeuk/modules/macro/__init__.py @@ -0,0 +1,3 @@ +from .google_ngram import * +from .leak import * +from .macro import MacroModule diff --git a/src/demeuk/modules/macro/google_ngram.py b/src/demeuk/modules/macro/google_ngram.py new file mode 100644 index 0000000..80868d3 --- /dev/null +++ b/src/demeuk/modules/macro/google_ngram.py @@ -0,0 +1,70 @@ +from string import punctuation as string_punctuation + +from nltk import WhitespaceTokenizer, str2tuple + +from ..base import * +from ..modify.input_encode import EncodeModule +from .macro import MacroModule + + +class GoogleNgramModule(MacroModule, ConfigModule): + + def set_configs(self, config): + self.add_config('config', config) + + @staticmethod + def get_help_info(): + return HelpInfo( + option=['g', 'googlengram'], + help_str='When set, demeuk will strip universal pos tags like _NOUN_ or _ADJ.' + ) + + @property + def debug_str(self): + return 'cleaned tags' + + def get_submodules(self): + encode_module = EncodeModule() + encode_module.set_configs(self.get_config('config')) + # Disable certain modules? + return [ + encode_module + ] + + def run(self, line): + """Removes speechtags from line specific to the googlengram module + + Param: + line (unicode) + + Returns: + line (unicode) + """ + cleaned_line = line.split('\t')[0] # Get the ngram, remove year, counter, etc + clean = [] + words = WhitespaceTokenizer().tokenize(cleaned_line) + for word in words: + # in >1-grams transitions to specific tags are written as: + # The_ADJ _NOUN_ (meaning from The there is a transition to a noun + # We remove those + if word[0] != '_' and word[-1] != '_': + # Split the token and the tag based on the '_' + token, tag = str2tuple(word, '_') + # Punct will be added using rules. + if len(token) > 1: + if tag != 'PUNCT' or tag != '.' or tag != '': + clean.append(token) + elif token not in string_punctuation: + clean.append(token) + cleaned_line = ' '.join(clean) + if cleaned_line != line: + return Result(status=True, msg=self.debug_str, update=cleaned_line) + return RESULT_NEXT + + def handle(self, results): + return Actions( + debug_str=results.msg, + update=results.update, + ) + + diff --git a/src/demeuk/modules/macro/leak.py b/src/demeuk/modules/macro/leak.py new file mode 100644 index 0000000..d5d4754 --- /dev/null +++ b/src/demeuk/modules/macro/leak.py @@ -0,0 +1,62 @@ +from ..base import * +from ..check.character import * +from ..check.empty_line import EmptyLineModule +from ..check.regex import * +from ..modify import * +from .macro import MacroModule + + +# NB: Macro module contains config module as submodule. So we pass the config on... +# Maybe not the best way? or not too bad... +class LeakModule(MacroModule, ConfigModule): + def set_configs(self, config): + # Store the entire config as config, + self.add_config('config', config) + + @staticmethod + def get_help_info(): + return HelpInfo( + option='leak', + help_str='When set, demeuk will run the following modules: mojibake, encode, newline, check-controlchar. ' + 'This is recommended when working with leaks.' + ) + + def get_submodules(self): + # We need to instantiate this explicitly because we want to pass config. + encode_module = EncodeModule() + encode_module.set_configs(self.get_config('config')) + return [ + encode_module, + MojibakeModule(), + ControlCharModule(), + NewlineModule(), + ] + +class LeakFullModule(MacroModule, ConfigModule): + def set_configs(self, config): + self.add_config('config', config) + + @staticmethod + def get_help_info(): + return HelpInfo( + option='leak-full', + help_str='When set, demeuk will run the following modules: mojibake, encode, newline, check-controlchar, ' + 'hex, html, html-named, check-hash, check-mac-address, check-uuid, check-email, ' + 'check-replacement-character, check-empty-line.' + ) + + def get_submodules(self): + encode_module = EncodeModule() + encode_module.set_configs(self.get_config('config')) + return [ + encode_module, + NewlineModule(), + MojibakeModule(), ControlCharModule(), + HexModule(), + HtmlModule(), HtmlNamedModule(), + HashModule(), MacAddressModule(), UuidModule(), EmailModule(), + ReplacementCharModule(), EmptyLineModule() + ] + + + diff --git a/src/demeuk/modules/macro/macro.py b/src/demeuk/modules/macro/macro.py new file mode 100644 index 0000000..21d8e7c --- /dev/null +++ b/src/demeuk/modules/macro/macro.py @@ -0,0 +1,52 @@ +from ..base import * + + +class MacroModule(Module): + """ + The abstract base class for a macro module. + Macro modules are modules which can invoke other modules. + """ + + @abstractmethod + def get_submodules(self) -> list[Module]: + """ + Determines what modules to add to the pipeline. + + :return: An ordered list of instances of modules to add to the pipeline. + """ + raise NotImplementedError + + # By default, we assume that a macro module is only used as a collection of other modules. + # We implement this here so that you can easily create a new macro module + + # However you can override these functions for custom behavior. + def run(self, line): + """ + Run the macro module. Does nothing and can be overridden for custom behavior. + + :param line: Input line + :type line: str + :return: Result of module run + """ + return RESULT_NEXT + + def handle(self, results): + """ + Handle the result of MacroModule.run(). Does nothing and can be overridden for custom behavior. + + :param results: Result of a MacroModule.run() + :type results: Result + :return: An empty Actions object + """ + return Actions() + + @property + def debug_str(self) -> str: + return 'performed action' + + @staticmethod + def get_pipeline_position(): + return PipelinePosition.AFTER_ENCODE + + + diff --git a/src/demeuk/modules/modify/__init__.py b/src/demeuk/modules/modify/__init__.py new file mode 100644 index 0000000..f2e6136 --- /dev/null +++ b/src/demeuk/modules/modify/__init__.py @@ -0,0 +1,10 @@ +from .case import * +from .char_encoding import * +from .cut import * +from .hex import * +from .html import * +from .input_encode import * +from .modify import ModifyModule +from .transliterate import * +from .umlaut import * +from .whitespace import * diff --git a/src/demeuk/modules/modify/case.py b/src/demeuk/modules/modify/case.py new file mode 100644 index 0000000..2a8d1d2 --- /dev/null +++ b/src/demeuk/modules/modify/case.py @@ -0,0 +1,26 @@ +from ..base import * +from .modify import ModifyModule + + +class LowercaseModule(ModifyModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='lowercase', + help_str="Replace line like 'This Test String' with 'this test string'.") + + def run(self, line): + cleaned_line = line.lower() + return self.get_result(line, cleaned_line) + +class TitleCaseModule(ModifyModule): + + @staticmethod + def get_help_info(): + return HelpInfo( + option='title-case', + help_str="Replace line like 'this test string' with 'This Test String'.") + + def run(self, line): + cleaned_line = line.title() + return self.get_result(line, cleaned_line) \ No newline at end of file diff --git a/src/demeuk/modules/modify/char_encoding.py b/src/demeuk/modules/modify/char_encoding.py new file mode 100644 index 0000000..3f7a6ca --- /dev/null +++ b/src/demeuk/modules/modify/char_encoding.py @@ -0,0 +1,29 @@ +from ftfy import fix_encoding +from unidecode import unidecode + +from ..base import * +from .modify import ModifyModule + + +class NonAsciiModule(ModifyModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='non-ascii', + help_str='Replace non-ASCII characters with an ASCII variant. For example, ü becomes u, ç becomes c.') + + def run(self, line): + cleaned_line = unidecode(line) + return self.get_result(line, cleaned_line) + + +class MojibakeModule(ModifyModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='mojibake', + help_str='Fixes mojibakes, which means lines like SmˆrgÂs will be fixed to Smörgås.') + + def run(self, line): + cleaned_line = fix_encoding(line) + return self.get_result(line, cleaned_line) diff --git a/src/demeuk/modules/modify/cut.py b/src/demeuk/modules/modify/cut.py new file mode 100644 index 0000000..1c4ee08 --- /dev/null +++ b/src/demeuk/modules/modify/cut.py @@ -0,0 +1,33 @@ +from ..base import * +from .modify import ModifyModule + + +class CutModule(ModifyModule, ConfigModule): + def set_configs(self, config): + self.add_config('delims', config.delimiters) + self.add_config('fields', config.cut_fields) + + @staticmethod + def get_help_info(): + return HelpInfo( + option=['c', 'cut'], + help_str="Specify if demeuk should split (default splits on ':'). Returns everything after the delimiter.") + + def run(self, line): + fields = self.get_config('fields') + for delimiter in self.get_config('delims'): + if delimiter in line: + if '-' in fields: + start = fields.split('-')[0] + stop = fields.split('-')[1] + if start == '': + start = 1 + if stop == '': + stop = len(line) + fields = slice(int(start) - 1, int(stop)) + else: + fields = slice(int(fields) - 1, int(fields)) + cleaned_line = delimiter.join(line.split(delimiter)[fields]) + return Result(status=True, msg=self.debug_str, update=cleaned_line) + else: + return RESULT_NEXT \ No newline at end of file diff --git a/src/demeuk/modules/modify/hex.py b/src/demeuk/modules/modify/hex.py new file mode 100644 index 0000000..23f5cae --- /dev/null +++ b/src/demeuk/modules/modify/hex.py @@ -0,0 +1,35 @@ +from binascii import unhexlify +from re import compile as re_compile + +from ..base import * +from .modify import ModifyModule + + +class HexModule(ModifyModule): + + HEX_REGEX = re_compile(r'^\$(?:HEX|hex)\[((?:[0-9a-fA-F]{2})+)\]$') + + + @staticmethod + def get_help_info(): + return HelpInfo( + option='hex', + help_str='Replace lines like: $HEX[41424344] with ABCD.' + ) + + @property + def debug_str(self) -> str: + return 'replaced $HEX[], added to queue and quitting' + + def run(self, line): + match = self.HEX_REGEX.search(line) + if match: + return Result(status=True, msg=self.debug_str, add=unhexlify(match.group(1))) + return RESULT_NEXT + + def handle(self, result): + return Actions( + add=[result.add], # expects a list. + debug_str=result.msg, + stop=True, + do_not_re_encode=True) \ No newline at end of file diff --git a/src/demeuk/modules/modify/html.py b/src/demeuk/modules/modify/html.py new file mode 100644 index 0000000..cc04388 --- /dev/null +++ b/src/demeuk/modules/modify/html.py @@ -0,0 +1,76 @@ +from html import unescape + +from ftfy.chardata import HTML_ENTITIES, HTML_ENTITY_RE + +from ..base import * +from .modify import ModifyModule + + +class HtmlModule(ModifyModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='html', + help_str='Replace lines like: şifreyok with şifreyok.') + + @property + def debug_str(self): + return 'replaced HTML, added to queue and quitting' + + @staticmethod + def _unescape_fixup(match): + """ + Replace one matched HTML entity with the character it represents, + if possible. + + Based on: ftfy.fixes._unescape_fixup + """ + text = match.group(0) + if text.startswith('&#'): + unescaped = unescape(text) + + # If html.unescape only decoded part of the string, that's not what we want. The semicolon should be consumed. + if ';' in unescaped: + return text + else: + return unescaped + else: + return text + + def run(self, line) -> Result: + cleaned_line = HTML_ENTITY_RE.sub(self._unescape_fixup, line) + if line != cleaned_line: + return Result(status=True, msg=self.debug_str, add=cleaned_line) + return RESULT_NEXT + + def handle(self, result): + return Actions( + stop=True, + add=[result.add], + debug_str=result.msg) + +class HtmlNamedModule(ModifyModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='html-named', + help_str='Replace lines like: &#alpha; Those structures are more like passwords, so be careful to enable ' + 'this option.') + + @staticmethod + def _unescape_fixup_named(match): + """ + Replace one matched HTML entity with the character it represents, + if possible. + + Based on: ftfy.fixes._unescape_fixup + """ + text = match.group(0) + if text in HTML_ENTITIES: + return HTML_ENTITIES[text] + else: + return text + + def run(self, line): + cleaned_line = HTML_ENTITY_RE.sub(self._unescape_fixup_named, line) + return self.get_result(line, cleaned_line) \ No newline at end of file diff --git a/src/demeuk/modules/modify/input_encode.py b/src/demeuk/modules/modify/input_encode.py new file mode 100644 index 0000000..9a31e09 --- /dev/null +++ b/src/demeuk/modules/modify/input_encode.py @@ -0,0 +1,123 @@ +from unicodedata import category + +from chardet import detect + +from demeuk.modules.base import Actions, ConfigModule, HelpInfo, PipelinePosition, Result + +from .modify import ModifyModule + + +class EncodeModule(ModifyModule, ConfigModule): + def set_configs(self, config): + self.add_config('encodings', config.input_encodings) + + @staticmethod + def get_pipeline_position(): + return PipelinePosition.ENCODE + + @staticmethod + def get_help_info(): + return HelpInfo( + option='encode', + help_str='Enables guessing of encoding, based on chardet and custom implementation.') + + # Only here, this is the success message... + @property + def debug_str(self) -> str: + return 'decoded line' + + @staticmethod + def _try_encoding(line, encoding): + """Tries to decode a line using supplied encoding + + Params: + line (Byte): byte variable that will be decoded + encoding (string): the encoding to be tried + + Returns: + False if decoding failed + String if decoding worked + """ + try: + # Try to decode the line + line_decoded = line.decode(encoding) + # Some encodings will decode almost any line, let's check if we have invalid chars. + # If we have invalid chars (except for tab-like chars) we will fail + for c in line_decoded: + if category(c) in ['Cc', 'Cf', 'Cn', 'Co', 'Cs']: + if c == '\t' or c == '\f': + continue + else: + return False + return line_decoded + except UnicodeDecodeError: + return False + + # Always returns status true, because we either need to update the line OR we have a decoding error. + def run(self, line): + for encoding in self.get_config('encodings'): + result = self._try_encoding(line, encoding) + if result is not False: + return Result(status=True, update=result, msg=self.debug_str) + else: + # Did not break, so tried all encodings and did not find a valid one. + # Try chardet + encode = detect(line) + if encode.get('encoding'): + try: + decoded_line = line.decode(encode['encoding']) + # successful decoding! + return Result(status=True, update=decoded_line, msg=self.debug_str) + except (UnicodeDecodeError, LookupError): + return Result(status=True, msg=f"decoding error with {encode['encoding']}") + else: + return Result(status=True, msg='decoding error with unknown encoding') + + def handle(self, result): + if result.update is None: + # Encoding failed, so stop. + # Log failure always. + return Actions(stop=True, log_str=result.msg) + # Encoding is successful + return Actions(update=result.update, debug_str=result.msg) + + +# Dropped in place if --encode is not used +# Don't inherit C/M/A/R-module, we don't want to register this to the argument parser +class DefaultEncodeModule(ConfigModule): + + def set_configs(self, config): + # Get first config, either UTF-8 or user-specified. + self.add_config('encoding', config.input_encodings[0]) + + @property + def debug_str(self) -> str: + return 'decoding error' + + def run(self, line): + try: + decoded_line = line.decode(self.get_config('encoding')) + return Result(status=True, update=decoded_line, msg=f"decoded using input encoding {self.get_config('encoding')}") + except UnicodeDecodeError: + return Result(status=True, msg=self.debug_str) + + + # Same as EncodeModule... + def handle(self, result): + if result.update is None: + return Actions(stop=True, log_str=result.msg) + return Actions(update=result.update, debug_str=result.msg) + + # Need to implement these to instantiate, but they are not used + + # Can't invoke this manually + @staticmethod + def get_help_info(): + pass + + # Pipeline ctor knows where to put this module. + @staticmethod + def get_pipeline_position(): + pass + + diff --git a/src/demeuk/modules/modify/modify.py b/src/demeuk/modules/modify/modify.py new file mode 100644 index 0000000..2319303 --- /dev/null +++ b/src/demeuk/modules/modify/modify.py @@ -0,0 +1,46 @@ +from ..base import * + + +class ModifyModule(Module): + """ + The abstract base class for modify modules. + Modify modules replace the current line by a new line. + Modify modules can be used as a preprocessing step, for example to fix encoding mistakes. + Modify modules can also be used to standardize the contents of a corpus, for example to make everything lowercase. + """ + + @staticmethod + def get_pipeline_position(): + return PipelinePosition.AFTER_ENCODE + + def handle(self, result): + """ + Handle the result of ModifyModule.run(). Expects a Result with the update field set and a debug message. + + :param result: Result of a ModifyModule.run() + :type result: Result + :return: Actions object which updates the line. + """ + return Actions( + # Update the line in the queue + update=result.update, + debug_str=result.msg) # Only log if --debug is set. + + @property + def debug_str(self): + return 'modified line' + + def get_result(self, line, cleaned_line): + """ + Utility function to get the appropriate Result object when modifying a line, to return from run(). + Checks if the new candidate is equal to the input line and only updates it if they differ. + + :param line: The input line + :type line: str + :param cleaned_line: The modified line + :type cleaned_line: str + :return: The result to return from run() + """ + if line != cleaned_line: + return Result(status=True, msg=self.debug_str, update=cleaned_line) + return RESULT_NEXT \ No newline at end of file diff --git a/src/demeuk/modules/modify/transliterate.py b/src/demeuk/modules/modify/transliterate.py new file mode 100644 index 0000000..858be34 --- /dev/null +++ b/src/demeuk/modules/modify/transliterate.py @@ -0,0 +1,18 @@ +from transliterate import translit +from ..base import * +from .modify import ModifyModule + + +class TransliterateModule(ModifyModule, ParamModule): + @staticmethod + def get_help_info(): + return HelpInfoParam( + option='transliterate', + help_str="Transliterate a string, for example 'իպսում' becomes 'ipsum' The following languages are supported: ka, sr, l1, ru, mn, uk, mk, el, hy and bg.", + metavar='', + param_type=str) + + def run(self, line): + cleaned_line = translit(line, self._param, reversed=True) + + return self.get_result(line, cleaned_line) diff --git a/src/demeuk/modules/modify/umlaut.py b/src/demeuk/modules/modify/umlaut.py new file mode 100644 index 0000000..0de94fc --- /dev/null +++ b/src/demeuk/modules/modify/umlaut.py @@ -0,0 +1,32 @@ +from demeuk.modules.base import * + +from .modify import ModifyModule + + +class UmlautModule(ModifyModule): + # Duplicated. Store somewhere else? + umlaut_dict = { + 'a"': 'ä', + 'i"': 'ï', + 'o"': 'ö', + 'u"': 'ü', + 'e"': 'ë', + 'A"': 'Ä', + 'I"': 'Ï', + 'O"': 'Ö', + 'U"': 'Ü', + 'E"': 'Ë', + } + + @staticmethod + def get_help_info(): + return HelpInfo( + option='umlaut', + help_str='Replace lines like ko"ffie with köffie') + + def run(self, line): + cleaned_line = line + for pattern, replacement in self.umlaut_dict.items(): + cleaned_line = cleaned_line.replace(pattern, replacement) + + return self.get_result(line, cleaned_line) \ No newline at end of file diff --git a/src/demeuk/modules/modify/whitespace.py b/src/demeuk/modules/modify/whitespace.py new file mode 100644 index 0000000..adf52b4 --- /dev/null +++ b/src/demeuk/modules/modify/whitespace.py @@ -0,0 +1,66 @@ +from re import sub + +from ..base import * +from .modify import ModifyModule + + +class NewlineModule(ModifyModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='newline', + help_str="Enables removing newline characters ('\\r' and '\\n') from end and beginning of lines.") + + def run(self, line): + cleaned_line = line.strip('\r\n') + return self.get_result(line, cleaned_line) + + +class TrimModule(ModifyModule): + TRIM_BLOCKS = ('\\\\n', '\\\\r', '\\n', '\\r', '
', '
') + + @staticmethod + def get_help_info(): + return HelpInfo( + option='trim', + help_str="Remove whitespace from beginning and end of line. Whitespace detected is '\\\\n', '\\\\r', '\\n', '\\r', '
' and '
'." + ) + + def run(self, line): + cleaned_line = line + # Ensure removal of duplicated blocks + while True: + has_match = False + for x in self.TRIM_BLOCKS: + if cleaned_line.startswith(x): + cleaned_line = cleaned_line[len(x):] + has_match = True + + if cleaned_line.endswith(x): + cleaned_line = cleaned_line[:-len(x)] + has_match = True + + if not has_match: + break + + return self.get_result(line, cleaned_line) + + +class TabModule(ModifyModule): + @staticmethod + def get_help_info(): + return HelpInfo( + option='tab', + help_str="Enables replacing tab char with ':', sometimes leaks contain both ':' and '\\t'." + ) + + # This module runs on bytes + @staticmethod + def get_pipeline_position(): + return PipelinePosition.BEFORE_ENCODE + + def run(self, line): + if b'\x09' in line: + line = sub(b'\x09+', b'\x3a', line) + return Result(status=True, msg=self.debug_str, update=line) + return RESULT_NEXT diff --git a/src/demeuk/modules/remove/__init__.py b/src/demeuk/modules/remove/__init__.py new file mode 100644 index 0000000..3016b10 --- /dev/null +++ b/src/demeuk/modules/remove/__init__.py @@ -0,0 +1,3 @@ +from .email import * +from .punctuation import * +from .remove import RemoveModule diff --git a/src/demeuk/modules/remove/email.py b/src/demeuk/modules/remove/email.py new file mode 100644 index 0000000..6f9ed1e --- /dev/null +++ b/src/demeuk/modules/remove/email.py @@ -0,0 +1,24 @@ +from re import search, sub + +from ..base import * +from .remove import RemoveModule + + +class RemoveEmailModule(RemoveModule): + + # This is now in two places. Delegate this to a 'constants.py' or 'regexes.py' file? + # Probably better, as this is not a 'config' or parameter about the module itself + EMAIL_REGEX = '.{1,64}@([a-zA-Z0-9_-]{1,63}\\.){1,3}[a-zA-Z]{2,6}' + + @staticmethod + def get_help_info(): + return HelpInfo( + option='remove-email', + help_str="Enable email filter, this will catch strings like '1238661:test@example.com:password'.") + + def run(self, line): + if '@' in line: + if search(f'{self.EMAIL_REGEX}(:|;)', line): + result_line = sub(f'{self.EMAIL_REGEX}(:|;)', '', line) + return Result(status=True, msg=self.debug_str, update=result_line) + return RESULT_NEXT \ No newline at end of file diff --git a/src/demeuk/modules/remove/punctuation.py b/src/demeuk/modules/remove/punctuation.py new file mode 100644 index 0000000..78f1ccd --- /dev/null +++ b/src/demeuk/modules/remove/punctuation.py @@ -0,0 +1,31 @@ +from ..base import * +from .remove import RemoveModule + + +class StripPunctuationModule(RemoveModule, ConfigModule): + def set_configs(self, config): + self.add_config('punctuation', config.punctuation) + + @staticmethod + def get_help_info(): + return HelpInfo( + option='remove-strip-punctuation', + help_str='Remove starting and trailing punctuation.') + + def run(self, line): + return_line = line.strip(self.get_config('punctuation')) + return self.get_result(line, return_line) + +class PunctuationModule(RemoveModule, ConfigModule): + def set_configs(self, config): + self.add_config('punctuation', config.punctuation) + + @staticmethod + def get_help_info(): + return HelpInfo( + option='remove-punctuation', + help_str='Remove all punctuation from a line.') + + def run(self, line): + return_line = line.translate(str.maketrans('', '', self.get_config('punctuation'))) + return self.get_result(line, return_line) diff --git a/src/demeuk/modules/remove/remove.py b/src/demeuk/modules/remove/remove.py new file mode 100644 index 0000000..97b1a6e --- /dev/null +++ b/src/demeuk/modules/remove/remove.py @@ -0,0 +1,44 @@ +from ..base import * + + +# Functionality-wise, a remove module is just a modify module. +# We still create a different abstract module class to separate this in a different help category, as well as different debug string. +class RemoveModule(Module): + """ + The abstract base class for remove modules. + Remove modules can remove certain parts of a line. + """ + @staticmethod + def get_pipeline_position(): + return PipelinePosition.AFTER_ENCODE + + def handle(self, result): + """ + Handle the result of RemoveModule.run(). Expects a result with the update field set and a debug message. + + :param result: Result of RemoveModule.run() + :type result: Result + :return: Actions object which updates the line + """ + return Actions( + update=result.update, + debug_str=result.msg) + + @property + def debug_str(self): + return 'removed part of line' + + def get_result(self, line, cleaned_line): + """ + Utility function to get the appropriate Result object when modifying a line, to return from run(). + Checks if the new candidate is equal to the input line and only updates it if they differ. + + :param line: The input line + :type line: str + :param cleaned_line: The modified line + :type cleaned_line: str + :return: The result to return from run() + """ + if line != cleaned_line: + return Result(status=True, msg=self.debug_str, update=cleaned_line) + return RESULT_NEXT \ No newline at end of file diff --git a/src/demeuk/multiproc.py b/src/demeuk/multiproc.py new file mode 100644 index 0000000..a8be1c7 --- /dev/null +++ b/src/demeuk/multiproc.py @@ -0,0 +1,74 @@ +from os import linesep +from signal import SIG_IGN, SIGINT, signal +from time import sleep + + +""" +Utility functions for multiprocessing and chunking input files +""" + +def init_worker(): + try: + signal(SIGINT, SIG_IGN) + except ValueError: + pass # signal() only works in the main thread; ThreadPool workers are threads + +def chunkify(file, cfg): + """ + Split a file into chunks, to pass onto the module pipeline + :param file: File to split + :param cfg: Config object, needed for skip and chunk_size. + """ + with open(file, 'rb') as fh: + for x in range(0, cfg.skip): + fh.readline() + + while True: + lines = [line.rstrip(linesep.encode()) for line in fh.readlines(cfg.chunk_size)] + yield lines + if len(lines) == 0: + break + +def check_finished_jobs(jobs, config): + """ + Check jobs queue for any finished jobs + :param jobs: Jobs queue + :param logger: Logger object + """ + while jobs and jobs[0].ready(): + job = jobs.pop(0) + write_results(config.output_fh, config.logger, job.get()) + +def submit(pool, jobs, pipeline, chunk, config): + """ + Submit a chunk of lines to the multiprocessing pool. + :param pool: Multiprocessing pool + :param jobs: Jobs queue + :param pipeline: Pipeline to run + :param chunk: List of lines to demeuk + :param config: Config object + """ + while True: + check_finished_jobs(jobs, config) + running_jobs = sum([not job.ready() for job in jobs]) + if running_jobs < config.threads: + jobs.append(pool.apply_async(pipeline.run, (chunk, config))) + return + else: + # Wait until a thread is available. + sleep(0.5) + +def finish_up(jobs, config): + """ + Wait for jobs to finish and write results. + :param jobs: Jobs queue + :param config: Config object. + """ + while len(jobs) > 0: + job = jobs.pop(0) + job.wait() + write_results(config.output_fh, config.logger, job.get()) + +def write_results(output_fh, logger, async_result): + output_fh.write(async_result['results']) + logger.write(async_result['log']) diff --git a/src/demeuk/output.py b/src/demeuk/output.py new file mode 100644 index 0000000..5877eb3 --- /dev/null +++ b/src/demeuk/output.py @@ -0,0 +1,45 @@ +from locale import getlocale +from sys import stdout + + +# Manages file handle to output file +class OutputFileHandler: + """ + Manage the output file + """ + + def __init__(self, args, logger): + """ + Open the output file + :param args: + :param logger: + """ + encoding = getlocale()[1] + if args.output: + try: + # Open in write mode to overwrite file if it exists, and close immediately. + open(args.output, 'w').close() + # Now open it in append mode, this solves a bug where file writes would get dropped unexpectedly + self.output_file = open(args.output, 'a', encoding=encoding, newline='') # Now + logger.stderr_print(f'Writing output to {args.output}') + except PermissionError: + logger.stderr_print_always(f'Cannot write output file to {args.output}!') + exit(2) + else: + self.output_file = stdout + logger.stderr_print('Writing output to stdout') + + def write(self, lines): + """ + Write (and flush) lines to the output file + :param lines: A list of lines (with newlines) to write to the output file + """ + self.output_file.writelines(lines) + self.output_file.flush() + + def close(self): + """ + Close the file handle to the output file + """ + if self.output_file != stdout: + self.output_file.close() \ No newline at end of file diff --git a/src/demeuk/parser.py b/src/demeuk/parser.py new file mode 100644 index 0000000..ce299aa --- /dev/null +++ b/src/demeuk/parser.py @@ -0,0 +1,228 @@ +from argparse import ArgumentParser, ArgumentTypeError, RawDescriptionHelpFormatter +from os import cpu_count +from textwrap import dedent + +from .modules.add.add import AddModule +from .modules.base import * +from .modules.check.check import CheckModule +from .modules.macro.macro import MacroModule +from .modules.modify.modify import ModifyModule +from .modules.remove.remove import RemoveModule + + +# -j can take int or 'all' as argument. +def int_or_all(arg): + try: + return int(arg) + except ValueError: + pass + if arg == 'all': + return cpu_count() + raise ArgumentTypeError(f"invalid value {arg} not int or 'all'") + + +ParserGroup = Enum('ParserGroup', [ + ('STANDARD', 0), + ('MACRO', 1), + ('CONFIG', 2), + ('CHECK', 3), + ('MODIFY', 4), + ('ADD', 5), + ('REMOVE', 6)]) + + +class CommandLineParser: + """ + A class to manage, parse and display command-line options of demeuk. + Modules need to be registered, which uses the overridden get_help_info() to display the correct help string when + running demeuk -h. + """ + + def __init__(self, version): + """ + Initialize an arguments parser with just the standard options (no modules). + :param version: demeuk version + """ + desc = dedent("""Demeuk - a simple tool to clean up corpora + +Example uses: + demeuk -i inputfile.tmp -o outputfile.dict -l logfile.txt + demeuk -i "inputfile*.txt" -o outputfile.dict -l logfile.txt + demeuk -i "inputdir/*" -o outputfile.dict -l logfile.txt + demeuk -i inputfile -o outputfile -j 24 + demeuk -i inputfile -o outputfile -c -e + demeuk -i inputfile -o outputfile --threads all + cat inputfile | demeuk --leak -j all | sort -u > outputfile""") + + self.parser = ArgumentParser(prog='demeuk', description=desc, usage='%(prog)s [options]', + add_help=False, # We add our own help so that it is grouped correctly + formatter_class=RawDescriptionHelpFormatter) + + self.parser_groups = { + ParserGroup.STANDARD: self.parser.add_argument_group('Standard options'), + ParserGroup.MACRO: self.parser.add_argument_group('Macro modules'), + ParserGroup.CONFIG: self.parser.add_argument_group('Configuration options'), + ParserGroup.CHECK: self.parser.add_argument_group('Check modules (check if a line matches a specific condition)'), + ParserGroup.MODIFY: self.parser.add_argument_group('Modify modules (modify a line in place)'), + ParserGroup.ADD: self.parser.add_argument_group('Add modules (Modify a line, but keep the original as well)'), + ParserGroup.REMOVE: self.parser.add_argument_group('Remove modules (remove specific parts of a line)'), + } + + self.args = None + self.lookup_table = {} + + # Standard options + self.parser_groups[ParserGroup.STANDARD].add_argument('-i', '--input', action='store', + nargs='*', + metavar='', + help='Specify the input file to be cleaned, or provide a glob pattern. (default: stdin)') + self.parser_groups[ParserGroup.STANDARD].add_argument('-o', '--output', action='store', + metavar='', + help='Specify the output file name. (default: stdout)') + self.parser_groups[ParserGroup.STANDARD].add_argument('-l', '--log', action='store', + metavar='', + help='Optional, specify where the log file needs to be writen to (default: stderr)') + self.parser_groups[ParserGroup.STANDARD].add_argument('-j', '--threads', action='store', type=int_or_all, + metavar='', + help='Optional, specify amount of threads to spawn. Specify the string ' + "'all' to make demeuk auto detect the amount of threads to " + "start based on the CPU's (default: all threads). Note: " + 'threading will cost some setup time. Only speeds up for larger files.') + self.parser_groups[ParserGroup.STANDARD].add_argument('-v', '--verbose', action='store_true', + help='When set, printing some extra information to stderr. And will ' + 'print the lines containing errors to logfile.') + self.parser_groups[ParserGroup.STANDARD].add_argument('--debug', action='store_true', + help='When set, the logfile will not only contain lines which caused ' + 'an error, but also line which were modified.') + self.parser_groups[ParserGroup.STANDARD].add_argument('--progress', action='store_true', + help='Prints out the progress of the demeuk process.') + self.parser_groups[ParserGroup.STANDARD].add_argument('-n', '--limit', action='store', type=int, + metavar='', help='Limit the number of lines per thread.') + self.parser_groups[ParserGroup.STANDARD].add_argument('-s', '--skip', action='store', type=int, + metavar='', help='Skip amount of lines per thread.') + self.parser_groups[ParserGroup.STANDARD].add_argument('--version', action='version', + version=f'%(prog)s {version}', help='Prints the version of demeuk.') + self.parser_groups[ParserGroup.STANDARD].add_argument('-h', '--help', action='help', + help='Prints this message and exits.') + + # Configuration options + self.parser_groups[ParserGroup.CONFIG].add_argument('--input-encoding', action='store', + metavar='', + help='Forces demeuk to decode the input using this encoding (default: en_US.UTF-8).') + self.parser_groups[ParserGroup.CONFIG].add_argument('--output-encoding', action='store', + metavar='', + help='Forces demeuk to encoding the output using this encoding (default: en_US.UTF-8).') + self.parser_groups[ParserGroup.CONFIG].add_argument('--punctuation', action='store', + metavar='', + help='Use to set the punctuation that is use by options. Defaults to: string.punctuation.') + self.parser_groups[ParserGroup.CONFIG].add_argument('-f','--cut-fields', action='store', + metavar='', + help="Specifies the field to be returned, this is in the 'cut' syntax.") + self.parser_groups[ParserGroup.CONFIG].add_argument('--cut-before', action='store_true', + help='Specify if demeuk should return the string before the delimiter') + self.parser_groups[ParserGroup.CONFIG].add_argument('-d', '--delimiter', action='store', + metavar='', + help="Specify what delimiter to use for --cut. Multiple delimiters can be specified with ','") + + @staticmethod + def make_cli_option(option): + """ + Transforms a string into a command-line option: For example 'c' to '-c' and 'cut' to '--cut'. + :param option: A string to transform + :return: The resulting option + """ + if len(option) == 1: + return '-' + option + else: + return '--' + option + + @staticmethod + def make_cli_options(options): + """ + Transform a string or a list of strings into a list of command-line options. + :param options: A string or list of strings to transform + :return: A list of options + """ + if isinstance(options, str): + return [CommandLineParser.make_cli_option(options)] + else: + return [CommandLineParser.make_cli_option(option) for option in options] + + @staticmethod + def make_cli_options_from_module(module): + """ + Use a modules get_help_info() to get its list of command-line options. + :param module: A module + :return: A list of command-line options + """ + return CommandLineParser.make_cli_options(module.get_help_info().option) + + + def add_flag_options(self, group, help_info): + """ + Add options to the argument parser so that they are recognized on the command-line. + This is for flags, which are command-line options which do not take an argument. + :param group: The category in which to list the option(s). + :param help_info: The HelpInfo object containing the option and help string. + """ + options = self.make_cli_options(help_info.option) + self.parser_groups[group].add_argument( + *options, + help=help_info.help_str, + action='store_true') + + def add_param_options(self, group, help_info_param): + """ + Add options to the argument parser so that they are recognized on the command-line. + This is for parameters, which are command-line options which take one argument. + :param group: The category in which to list the option(s). + :param help_info_param: The HelpInfoParam object containing the option and help string, and info about the parameter. + """ + options = self.make_cli_options(help_info_param.option) + self.parser_groups[group].add_argument( + *options, + help=help_info_param.help_str, + metavar=help_info_param.metavar, + type=help_info_param.param_type, + action='store') + + + + def register(self, module): + """ + Register a module to the argument parser + :param module: The module to register + """ + # Hardcoded: Module Type determines help category + categories = { + CheckModule: ParserGroup.CHECK, + ModifyModule: ParserGroup.MODIFY, + AddModule: ParserGroup.ADD, + RemoveModule: ParserGroup.REMOVE, + MacroModule: ParserGroup.MACRO, + } + + for module_type, group_str in categories.items(): + if issubclass(module, module_type): + group = group_str + break + else: + # If a module is not one of the categories, don't register. + # Therefore, it cannot be called from the command-line, only internally! + return + + if issubclass(module, ParamModule): + self.add_param_options(group, module.get_help_info()) + else: + self.add_flag_options(group, module.get_help_info()) + + + for option in CommandLineParser.make_cli_options_from_module(module): + self.lookup_table[option] = module + + def parse_args(self, args): + """ + Parse arguments, this needs to be done after all modules are registered. + :param args: A list of command-line arguments (for example sys.argv). + """ + self.args = self.parser.parse_args(args[1:]) # First arg is program name, we don't need that \ No newline at end of file diff --git a/src/demeuk/pipeline.py b/src/demeuk/pipeline.py new file mode 100644 index 0000000..35834ce --- /dev/null +++ b/src/demeuk/pipeline.py @@ -0,0 +1,158 @@ +from binascii import hexlify +from collections import deque +from os import linesep + +from .modules.base import * +from .modules.macro.macro import MacroModule +from .modules.modify.input_encode import DefaultEncodeModule + + +class Pipeline: + """ + Contains a record of what modules to run, and in what order + """ + def __init__(self, parser, argv, config): + """ + Construct a pipeline from a list of command-line arguments. + :param parser: A CommandLineParser instance, used for a lookup table between options and modules. + :param argv: The list of command-line arguments + :param config: The config object, used to pass global config to modules + """ + # Keep track where our encoding module (should) be + self.has_encoding = False + self.encoding_slot = 0 + + self.modules = [] + + + # Build pipeline + for i in range(1, len(argv)): + current_arg = argv[i] + if current_arg in parser.lookup_table: + module = parser.lookup_table[current_arg] + if issubclass(module, ParamModule): + # Instantiate with param + instance = module(argv[i + 1]) + else: + # Instantiate a module without parameters + instance = module() + + if issubclass(module, ConfigModule): + instance.set_configs(config) + + # Currently, the submodules are placed BEFORE the macro module. + # Does this matter? + if issubclass(module, MacroModule): + for subinstance in instance.get_submodules(): + self.include_module(subinstance) + + + self.include_module(instance) + + + # --encode not used + if not self.has_encoding: + # Insert the standard encoder (is a config module) + default_encode = DefaultEncodeModule() + default_encode.set_configs(config) + self.modules.insert(self.encoding_slot, default_encode) + + + def include_module(self, instance): + """ + Insert a module in the pipeline based on its get_pipeline_position. + :param instance: The instanced module to include + """ + # Append, insert at 0 or insert at encoding_slot? + match instance.get_pipeline_position(): + case PipelinePosition.BEFORE_ENCODE: + self.modules.insert(self.encoding_slot, instance) + # Bump up encoding slot. Also makes sure BEFORE_ENCODE modules are placed in order. + self.encoding_slot += 1 + case PipelinePosition.ENCODE: + self.modules.insert(self.encoding_slot, instance) + self.has_encoding = True + # don't need to keep track of encoding_slot if inserted. + case PipelinePosition.AFTER_ENCODE: + self.modules.append(instance) + + # This is one worker job, process a list of lines. + def run(self, lines, config): + """ + Run a list of lines through the pipeline + :param lines: The list of lines to demeuk + :param config: Config object, used for --limit and its logger + :return: A dict containing a list of results, and a list of log lines + """ + results = [] + logger = config.logger # This is fine, multiprocessing creates a new copy. + + # Keep track of already processed lines so that we do not check the same line twice. + # Set lookups are constant time. + processed_lines = set() + + # We use a double-ended queue for the remaining lines: We insert lines (with add modules) at the back, while we + # process the lines at the front. + work_queue = deque(lines) + + while work_queue: + line = work_queue.popleft() + + if line in processed_lines: + continue + processed_lines.add(line) + + # Process at most config.limit lines (per thread) + # Why is this per thread? + if config.limit is not None: + if config.limit > 0: + config.limit -= 1 + else: + break + + + logger.log_debug(f'----BEGIN---- {hexlify(line)}{linesep}') + + + + for module in self.modules: + result = module.run(line) + if result.status: + # Transform module result into actions + actions = module.handle(result) + + # Perform actions if they are set + + if actions.add is not None: + # Add (a list of) word(s) to the queue + for word in actions.add: + if actions.do_not_re_encode: + work_queue.append(word) # for --hex + else: + work_queue.append(word.encode()) + if actions.debug_add_str is not None: + logger.log_debug(f'{module.__class__.__name__}:\t{actions.debug_add_str}:\t{word}{linesep}') + + if actions.update is not None: + line = actions.update + + if actions.log_str is not None: + # Log a message (always) + logger.log(f'{module.__class__.__name__}:\t{actions.log_str}:\t{line}{linesep}') + if actions.debug_str is not None: + # Log a message (with --debug) + logger.log_debug(f'{module.__class__.__name__}:\t{actions.debug_str}:\t{line}{linesep}') + + # Do this last, so that logs are written of stop == True + # If stop is set, don't do anything else. + if actions.stop: + break + else: + # This gets executed if we do not break out of the for loop + results.append(f'{line}{linesep}') + logger.log_debug(f'-----END----- {line}{linesep}{linesep}') + + return {'results': results, 'log': logger.get()} + + + diff --git a/tests/conftest.py b/tests/conftest.py index c327e9c..0aad981 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ from os import linesep, mkdir, path from shutil import rmtree + if path.isdir('testdata'): rmtree('testdata') mkdir('testdata') @@ -51,17 +52,17 @@ file.write(f'email@example.com;line5{linesep}') file.write(f'test:email@example.com:line6{linesep}') -with open('testdata/input6', 'w', encoding='utf-8') as file: - file.write(f'I\'Afrique_ADJ occidental_ADJ\t1927\t2\t2{linesep}') - file.write(f'I\'Allemagne )\t2009\t1\t1{linesep}') - file.write(f'I\'ain _VERB_\t2009\t2\t2{linesep}') - file.write(f'I\'ain a_VERB\t2009\t2\t2{linesep}') +with open('testdata/input6', 'w') as file: + file.write(f"I'Afrique_ADJ occidental_ADJ\t1927\t2\t2{linesep}") + file.write(f"I'Allemagne )\t2009\t1\t1{linesep}") + file.write(f"I'ain _VERB_\t2009\t2\t2{linesep}") + file.write(f"I'ain a_VERB\t2009\t2\t2{linesep}") with open('testdata/input7', 'wb') as file: # coupÉ file.write(b'\x63\x6F\x75\x70\xC3\x89' + f'{linesep}'.encode('utf-8')) # LANCIA AURELIA B20 COUPÉ GT\n - file.write(b'\x4C\x41\x4E\x43\x49\x41\x20\x41\x55\x52\x45\x4C\x49\x41\x20\x42\x32\x30\x20\x43\x4F\x55\x50\xC3\x83\xC2\x89\x20\x47\x54\x0A') # noqa: E501 + file.write(b'\x4C\x41\x4E\x43\x49\x41\x20\x41\x55\x52\x45\x4C\x49\x41\x20\x42\x32\x30\x20\x43\x4F\x55\x50\xC3\x83\xC2\x89\x20\x47\x54\x0A') with open('testdata/input8', 'w', encoding='utf-8') as file: @@ -102,7 +103,6 @@ with open('testdata/input15', 'w', encoding='utf-8') as file: file.write(f'$HEX[5045d141524f4c]{linesep}') - file.write(f'$HEX[51574552545955494f50c5]{linesep}') file.write(f'$HEX[5a73f3666932303030]{linesep}') file.write(f'$HEX[617261f16173]{linesep}') @@ -401,3 +401,8 @@ with open('testdata/input55', 'w', encoding='utf-8') as file: file.write(f'здраво пријатељу{linesep}') file.write(f'жута банана{linesep}') + +with open('testdata/input56', 'w') as file: + file.write(f'demeuk@example.com:password1{linesep}') + file.write(f'demeuk@example.com:test@example.com{linesep}') + file.write(f'1238661:test@example.com:password{linesep}') diff --git a/tests/test_app.py b/tests/test_app.py index 7b007d9..7316c01 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -3,9 +3,10 @@ from os import name, linesep from unittest.mock import patch -from pytest import raises, mark +from pytest import mark, raises -from bin.demeuk import main +from demeuk.demeuk import cli_entry_point as main +from demeuk.demeuk import run_cli def calculate_line_numbers(file_name): @@ -25,10 +26,10 @@ def test_demeuk(): line_num_output1 = calculate_line_numbers('testdata/output1') line_num_log1 = calculate_line_numbers('testdata/log1') - assert line_num_log1 == 5 + assert line_num_log1 == 6 assert line_num_output1 == 9 - assert line_num_input1 == (line_num_output1 + line_num_log1 - 1) - with open('testdata/output1', encoding='utf-8') as file: + assert line_num_input1 == (line_num_output1 + line_num_log1 - 2) + with open('testdata/output1') as file: filecontent = file.read() assert 'Password123!@"\n' in filecontent assert 'ǓǝǪǼȧɠ\n' in filecontent @@ -41,12 +42,10 @@ def test_demeuk(): def test_multithread(): - testargs = ['demeuk', '-i', 'testdata/input2', '-o', 'testdata/output2', '-j', '3'] - with patch.object(sys, 'argv', testargs): - main() + results = _run_demeuk(2, '-j', '3') line_num_input1 = calculate_line_numbers('testdata/input2') - line_num_output1 = calculate_line_numbers('testdata/output2') + line_num_output1 = len(results) assert line_num_output1 == 8 assert line_num_input1 == line_num_output1 @@ -105,10 +104,10 @@ def test_googlengram(): assert line_num_output == 4 with open('testdata/output6', encoding='utf-8') as f: filecontent = f.read() - assert 'I\'ain\n' in filecontent - assert 'I\'Afrique occidental\n' in filecontent - assert 'I\'Allemagne\n' in filecontent - assert 'I\'ain a\n' in filecontent + assert "I'ain\n" in filecontent + assert "I'Afrique occidental\n" in filecontent + assert "I'Allemagne\n" in filecontent + assert "I'ain a\n" in filecontent def test_coupe(): @@ -251,14 +250,13 @@ def test_cut_fields_single(): def test_unhex(): testargs = [ 'demeuk', '-i', 'testdata/input15', '-o', 'testdata/output15', '-l', 'testdata/log15', - '--hex', '--encode', + '--hex', '--encode' ] with patch.object(sys, 'argv', testargs): main() with open('testdata/output15', encoding='utf-8') as f: filecontent = f.read() assert 'PEÑAROL\n' in filecontent - assert 'QWERTYUIOPÅ\n' in filecontent assert 'Zsófi2000\n' in filecontent assert 'arañas\n' in filecontent assert '$HEX[' not in filecontent @@ -305,7 +303,7 @@ def test_verbose(): main() with open('testdata/log18', encoding='utf-8') as f: filecontent = f.read() - assert 'Clean_cut; ' in filecontent + assert 'CutModule:' in filecontent def test_limit(): @@ -369,7 +367,7 @@ def test_multiple_delimiters(): def test_check_email(): testargs = [ 'demeuk', '-i', 'testdata/input22', '-o', 'testdata/output22', '-l', 'testdata/log22', - '--verbose', '--check-email', '--remove-email', + '--verbose', '--remove-email', '--check-email' ] with patch.object(sys, 'argv', testargs): main() @@ -386,7 +384,7 @@ def test_check_email(): def test_check_hash(): testargs = [ 'demeuk', '-i', 'testdata/input23', '-o', 'testdata/output23', '-l', 'testdata/log23', - '--verbose', '--check-hash', '-c', + '--verbose', '-c', '--check-hash', ] with patch.object(sys, 'argv', testargs): main() @@ -499,11 +497,10 @@ def test_glob(): 'demeuk', '-i', 'testdata/input*', '-o', 'testdata/output30', '-l', 'testdata/log30', '--verbose', '-c', '-d', ',;:', ] - with patch.object(sys, 'argv', testargs): - main() - with open('testdata/output30', encoding='utf-8') as f: - assert len(f.readlines()) > 100 - + run_cli(testargs) + with open('testdata/output30') as f: + results = f.readlines() + assert len(results) > 100 def test_bug_html_control(): testargs = [ @@ -614,23 +611,16 @@ def test_trim(): def test_invalid_unhex(): - testargs = [ - 'demeuk', '-i', 'testdata/input37', '-o', 'testdata/output37', '-l', 'testdata/log37', - '--verbose', '--hex', - ] - with patch.object(sys, 'argv', testargs): - main() + results = _run_demeuk(37, '--hex') - with open('testdata/output37', encoding='utf-8') as f: - filecontent = f.read() - # Invalid hex string, leaving at as is. - assert '$HEX[e]tiredofwaiting\n' in filecontent - # Invalid hex string, leaving at as is. - assert '\n$HEX[eee]\n' in filecontent - # This is a valid hash, but it is not a hex string from start to end. - assert '\n$HEX[6C657469746B69636B696E]123!\n' in filecontent - # Valid upcase test - assert '\nlosingtouch\n' in filecontent + # Invalid hex string, leaving at as is. + assert '$HEX[e]tiredofwaiting' in results + # Invalid hex string, leaving at as is. + assert '$HEX[eee]' in results + # This is a valid hash, but it is not a hex string from start to end. + assert '$HEX[6C657469746B69636B696E]123!' in results + # Valid upcase test + assert 'losingtouch' in results def test_skip(): @@ -726,17 +716,8 @@ def test_check_ending_with(): def test_check_title_case(): - testargs = [ - 'demeuk', '-i', 'testdata/input44', '-o', 'testdata/output44', '-l', 'testdata/log44', - '--verbose', '--title-case', - ] - with patch.object(sys, 'argv', testargs): - main() - - with open('testdata/output44', encoding='utf-8') as f: - filecontent = f.read() - - assert '3 Doors Down' in filecontent + results = _run_demeuk(44, '--title-case') + assert '3 Doors Down' in results def test_leak_full(): @@ -813,7 +794,7 @@ def test_check_multiple_regexes(): def test_stdin_stdout(): - comlist = ['bin/demeuk.py'] + comlist = ['demeuk'] # On Windows scripts cant be executed with the shebang so manually add python3 in front if name == 'nt': comlist.insert(0, 'python3') @@ -843,93 +824,94 @@ def test_check_lowercase(): assert '3 doors down' in filecontent -def _run_demeuk(file_name, *extra_args): +def _run_demeuk(test_num, *extra_args): testargs = [ - 'demeuk', '-i', f'testdata/{file_name}', - '-o', f'testdata/{file_name}.out', - '-l', f'testdata/{file_name}.log', + 'demeuk', '-i', f'testdata/input{test_num}', + '-o', f'testdata/output{test_num}', + '-l', f'testdata/log{test_num}', '--verbose', ] testargs.extend(extra_args) - with patch.object(sys, 'argv', testargs): main() - with open(f'testdata/{file_name}.out', encoding='utf-8') as f: - return f.read() + with open(f'testdata/output{test_num}', encoding='utf-8') as f: + results = [line.rstrip('\n') for line in f.readlines()] + + return results def test_check_digits(): - result = _run_demeuk('input49', '--check-min-digits', '0', '--check-max-digits', '0').splitlines() + result = _run_demeuk(49, '--check-min-digits', '0', '--check-max-digits', '0') assert result == ['nodigits'] - result = _run_demeuk('input49', '--check-max-digits', '0').splitlines() + result = _run_demeuk(49, '--check-max-digits', '0') assert result == ['nodigits'] - result = _run_demeuk('input49', '--check-max-digits', '9999999').splitlines() + result = _run_demeuk(49, '--check-max-digits', '9999999') assert result == ['nodigits', '0digit', 'd1git', 'digit2', '६', 'pw123!'] - result = _run_demeuk('input49', '--check-min-digits', '1').splitlines() + result = _run_demeuk(49, '--check-min-digits', '1') assert result == ['0digit', 'd1git', 'digit2', '६', 'pw123!'] - result = _run_demeuk('input49', '--check-min-digits', '2').splitlines() + result = _run_demeuk(49, '--check-min-digits', '2') assert result == ['pw123!'] - result = _run_demeuk('input49', '--check-min-digits', '3', '--check-max-digits', '3').splitlines() + result = _run_demeuk(49, '--check-min-digits', '3', '--check-max-digits', '3') assert result == ['pw123!'] - result = _run_demeuk('input49', '--check-min-digits', '4').splitlines() + result = _run_demeuk(49, '--check-min-digits', '4') assert result == [] def test_check_uppercase(): - result = _run_demeuk('input50', '--check-min-uppercase', '0', '--check-max-uppercase', '0').splitlines() + result = _run_demeuk(50, '--check-min-uppercase', '0', '--check-max-uppercase', '0') assert result == ['noupper'] - result = _run_demeuk('input50', '--check-max-uppercase', '0').splitlines() + result = _run_demeuk(50, '--check-max-uppercase', '0') assert result == ['noupper'] - result = _run_demeuk('input50', '--check-max-digits', '9999999').splitlines() + result = _run_demeuk(50, '--check-max-digits', '9999999') assert result == ['noupper', 'Uppercase', 'upperCase', 'uppercasE', 'greek:Ω', 'ThisIsUpperCase!!!'] - result = _run_demeuk('input50', '--check-min-uppercase', '1').splitlines() + result = _run_demeuk(50, '--check-min-uppercase', '1') assert result == ['Uppercase', 'upperCase', 'uppercasE', 'greek:Ω', 'ThisIsUpperCase!!!'] - result = _run_demeuk('input50', '--check-min-uppercase', '2').splitlines() + result = _run_demeuk(50, '--check-min-uppercase', '2') assert result == ['ThisIsUpperCase!!!'] - result = _run_demeuk('input50', '--check-min-uppercase', '4', '--check-max-uppercase', '4').splitlines() + result = _run_demeuk(50, '--check-min-uppercase', '4', '--check-max-uppercase', '4') assert result == ['ThisIsUpperCase!!!'] - result = _run_demeuk('input50', '--check-min-uppercase', '9999999').splitlines() + result = _run_demeuk(50, '--check-min-uppercase', '9999999') assert result == [] def test_check_special(): - result = _run_demeuk('input51', '--check-min-special', '0', '--check-max-special', '0').splitlines() + result = _run_demeuk(51, '--check-min-special', '0', '--check-max-special', '0') assert result == ['NoSpecialsHere'] - result = _run_demeuk('input51', '--check-max-special', '0').splitlines() + result = _run_demeuk(51, '--check-max-special', '0') assert result == ['NoSpecialsHere'] - result = _run_demeuk('input51', '--check-max-digits', '9999999').splitlines() + result = _run_demeuk(51, '--check-max-digits', '9999999') assert result == ['NoSpecialsHere', '!special', 'No?Here', 'evenSpecialer#', 'Richie£Rich', '%✓⏻', '8bytesemoji*4🙌🏽🙌🏽🙌🏽🙌🏽'] - result = _run_demeuk('input51', '--check-min-special', '1').splitlines() + result = _run_demeuk(51, '--check-min-special', '1') assert result == ['!special', 'No?Here', 'evenSpecialer#', 'Richie£Rich', '%✓⏻', '8bytesemoji*4🙌🏽🙌🏽🙌🏽🙌🏽'] - result = _run_demeuk('input51', '--check-min-special', '2').splitlines() + result = _run_demeuk(51, '--check-min-special', '2') assert result == ['%✓⏻', '8bytesemoji*4🙌🏽🙌🏽🙌🏽🙌🏽'] - result = _run_demeuk('input51', '--check-min-special', '3', '--check-max-special', '3').splitlines() + result = _run_demeuk(51, '--check-min-special', '3', '--check-max-special', '3') assert result == ['%✓⏻'] # 9 specials: 1 for * and each hand emoji is represented by 2 unicode codepoints - result = _run_demeuk('input51', '--check-min-special', '9', '--check-max-special', '9').splitlines() + result = _run_demeuk(51, '--check-min-special', '9', '--check-max-special', '9') assert result == ['8bytesemoji*4🙌🏽🙌🏽🙌🏽🙌🏽'] - result = _run_demeuk('input51', '--check-min-special', '9999999').splitlines() + result = _run_demeuk(51, '--check-min-special', '9999999') assert result == [] @@ -976,20 +958,12 @@ def test_add_title_case(): def test_check_contains(): - testargs = [ - 'demeuk', '-i', 'testdata/input53', '-o', 'testdata/output53', '-l', 'testdata/log53', - '--verbose', '--check-contains', '_', - ] - with patch.object(sys, 'argv', testargs): - main() + results = _run_demeuk(53, '--check-contains', '_') - with open('testdata/output53', encoding='utf-8') as f: - filecontent = f.read() - - assert 'three_down' not in filecontent - assert '_amsterdam' not in filecontent - assert 'ROTTERDAM_' not in filecontent - assert 'Cookie Monster' in filecontent + assert 'three_down ' not in results + assert '_amsterdam ' not in results + assert 'ROTTERDAM_ ' not in results + assert 'Cookie Monster ' in results @mark.timeout(1) @@ -1027,3 +1001,12 @@ def test_transliterate(): assert 'zdravo prijatelju' in filecontent assert 'zuta banana' in filecontent + +def test_order(): + results = _run_demeuk(56, '--check-email', '--remove-email') + assert len(results) == 0 + + results = _run_demeuk(56, '--remove-email', '--check-email') + assert 'test@example.com' not in results + assert 'password1' in results + assert 'password' in results diff --git a/tox.ini b/tox.ini index 5fdac49..85d9de4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,20 +1,10 @@ [tox] -env_list = py38,py39,py310 -skipsdist = True - -[flake8] -application_import_names = bin -import_order_style = google -max_line_length = 120 -exclude = .venv,venv,.tox,.vscode,cleancorpus.egg-info +env_list = py31{2,3,4} [testenv] -deps = - -rrequirements.txt - pytest - flake8 - pytest-timeout -commands = +deps = pytest - flake8 -allowlist_externals = pytest +commands = + # Mysterious bug: pickle/dill can't serialize some internal object from a pytest call + # This is fixed by disabling stdout/err capture... + pytest -s {posargs}