filter-github-orgs

#!/usr/bin/env python3
# =============================================================================
# @file    filter-github-orgs
# @brief   Filter GitHub organization names to produce a higher quality list
# @author  Michael Hucka <mhucka@caltech.edu>
# @license MIT license; please see the file LICENSE in the repo
# @repo    https://github.com/caltechlibrary/iga
#
# Approach: this program applies various heuristics and NLP tools to filter
# the raw list of GitHub accounts produced by the companion program
# list-github-orgs. The output is (hopefully) a higher quality list of names.
# Note this means the resulting list is not a complete list of all org
# accounts in GitHub -- the filtering applied is designed to make the results
# useful for the specific goal of distinguishing people names from org names,
# not to produce a comprehensive list of all org accounts in GitHub.
# =============================================================================

from   base64 import b64encode
from   commonpy.data_structures import CaseFoldSet
import os
import re
import requests.exceptions
import rich_click as click
from   rich_click import File
from   sidetrack import set_debug, log
import string
import sys


# Internal constants.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# We load spaCy at run time when it's finally needed, because it takes so long.
_SPACY = None

# Set of common ~1000 words, used as part of our name filtering.  (The
# original set came from lists of words found on various places on the web.
# I removed words that could be people's names, like "chase", "black", etc.,
# and added some words that show up often in GitHub, like "cloud" and "dev".)
_COMMON_NONNAME_WORDS = {
    'à', 'a', 'ability', 'able', 'aboard', 'about', 'above',
    'absolutely', 'accept', 'according', 'account', 'across',
    'act', 'action', 'activity', 'actually', 'add',
    'administration', 'affect', 'after', 'again', 'against',
    'age', 'agency', 'ago', 'agree', 'air', 'all', 'allow',
    'almost', 'along', 'already', 'also', 'although', 'always',
    'american', 'amid', 'amidst', 'among', 'amongst', 'amount',
    'an', 'analysis', 'and', 'animal', 'another', 'answer',
    'any', 'anyone', 'anything', 'anyway', 'app', 'appear', 'apply',
    'approach', 'area', 'argue', 'argument', 'arm', 'army',
    'around', 'arrive', 'article', 'artist', 'as', 'ask', 'at',
    'atop', 'attack', 'attention', 'author', 'authority',
    'available', 'avoid', 'away', 'baby', 'back', 'bad', 'ball',
    'bank', 'base', 'be', 'beautiful', 'because', 'become',
    'bed', 'before', 'begin', 'behavior', 'behind', 'believe',
    'below', 'beneath', 'benefit', 'beside', 'besides', 'best',
    'better', 'between', 'beyond', 'big', 'bit',
    'blood', 'blue', 'board', 'body', 'book', 'both', 'box',
    'boy', 'brain', 'break', 'bring', 'brother', 'build',
    'building', 'business', 'but', 'buy', 'by', 'call',
    'camera', 'campaign', 'can', 'candidate', 'car', 'card',
    'care', 'career', 'carry', 'case', 'catch', 'cause', 'cell',
    'center', 'central', 'century', 'certain', 'certainly',
    'challenge', 'chance', 'change', 'character', 'charge',
    'check', 'chef', 'child', 'choice', 'choose', 'church', 'circa',
    'city', 'claim', 'class', 'clear', 'clearly', 'close', 'cloud',
    'club', 'code', 'college', 'color', 'come', 'comment', 'committee',
    'common', 'community', 'company', 'compare', 'completely',
    'computer', 'concern', 'condition', 'congress', 'consider',
    'continue', 'control', 'conversation', 'cost', 'could',
    'country', 'county', 'couple', 'course', 'court', 'cover',
    'create', 'crime', 'culture', 'cup', 'current', 'cut', 'cyber',
    'dad', 'dark', 'data', 'date', 'daughter', 'day', 'dead',
    'deal', 'death', 'decade', 'decide', 'decision', 'defense',
    'degree', 'democrat', 'department', 'describe', 'design',
    'despite', 'detail', 'determine', 'dev', 'develop', 'developer',
    'development', 'die', 'difference', 'different', 'difficult',
    'digital', 'director', 'discuss', 'discussion', 'disease', 'district',
    'do', 'doctor', 'dog', 'dollar', 'door', 'down', 'draw', 'dream',
    'drive', 'drop', 'drug', 'during', 'each', 'early', 'earth',
    'east', 'easy', 'eat', 'economic', 'economy', 'education',
    'effect', 'effort', 'eight', 'either', 'election', 'else',
    'employee', 'end', 'energy', 'enjoy', 'enough', 'enter',
    'entire', 'environment', 'especially', 'et', 'even',
    'event', 'ever', 'every', 'everybody', 'everyone',
    'everything', 'evidence', 'exactly', 'example', 'exist',
    'expect', 'experience', 'explain', 'eye', 'face', 'fact',
    'factor', 'fail', 'fall', 'family', 'far', 'father',
    'federal', 'feel', 'feeling', 'few', 'field', 'fight',
    'figure', 'fill', 'film', 'final', 'finally', 'financial',
    'find', 'fine', 'finish', 'fire', 'first', 'five', 'floor',
    'focus', 'follow', 'food', 'foot', 'for', 'force',
    'foreign', 'forget', 'form', 'former', 'forward', 'four',
    'free', 'friend', 'from', 'full', 'fund', 'future', 'game',
    'general', 'get', 'girl', 'give', 'global', 'go', 'goal',
    'good', 'government', 'great', 'ground', 'group', 'grow',
    'growth', 'guess', 'gun', 'hair', 'half', 'hand', 'hang',
    'happen', 'happy', 'hard', 'hate', 'have', 'he', 'head',
    'health', 'hear', 'heart', 'hell', 'help', 'her', 'here',
    'herself', 'hey', 'hi', 'high', 'him', 'himself', 'his',
    'history', 'hit', 'hold', 'home', 'hospital', 'hot',
    'hour', 'house', 'how', 'however', 'huge', 'human',
    'husband', 'idea', 'identify', 'if', 'image',
    'imagine', 'impact', 'important', 'in', 'include',
    'including', 'increase', 'indeed', 'indicate', 'individual',
    'industry', 'information', 'inside', 'instead', 'interest',
    'interesting', 'international', 'interview', 'into',
    'involve', 'island', 'issue', 'it', 'its', 'itself', 'job',
    'join', 'just', 'justice', 'keep', 'kid', 'kill', 'kind',
    'king', 'know', 'knowledge', 'la', 'lab', 'lady', 'land',
    'language', 'large', 'last', 'late', 'later', 'laugh',
    'law', 'lay', 'lead', 'leader', 'learn', 'least', 'leave',
    'left', 'legal', 'less', 'let', 'letter', 'level', 'lie',
    'life', 'light', 'like', 'likely', 'line', 'list', 'listen',
    'little', 'live', 'local', 'long', 'look', 'lose', 'loss',
    'lot', 'love', 'low', 'main', 'major', 'make', 'man',
    'manage', 'many', 'market', 'material', 'matter', 'maybe',
    'me', 'mean', 'media', 'medical', 'meet', 'meeting',
    'member', 'memory', 'mention', 'message', 'method', 'might',
    'mile', 'military', 'million', 'mind', 'minus', 'minute',
    'miss', 'model', 'mom', 'moment', 'money', 'month', 'more',
    'morning', 'most', 'mother', 'move', 'movement', 'movie',
    'much', 'music', 'must', 'my', 'myself', 'name', 'nation',
    'national', 'natural', 'nature', 'near', 'nearly', 'need',
    'network', 'never', 'new', 'news', 'next', 'nice', 'night',
    'no', 'nobody', 'north', 'not', 'note', 'nothing', 'notice',
    'now', 'number', 'occur', 'of', 'off', 'offer',
    'office', 'officer', 'official', 'often', 'oh', 'oil', 'ok',
    'okay', 'old', 'on', 'once', 'one', 'only', 'onto', 'open',
    'opinion', 'opportunity', 'opposite', 'option', 'or',
    'order', 'organization', 'other', 'our', 'out', 'outside',
    'over', 'own', 'page', 'pain', 'paper', 'parent', 'park',
    'part', 'particularly', 'party', 'pass', 'past', 'patient',
    'pay', 'people', 'per', 'percent', 'perfect', 'performance',
    'perhaps', 'period', 'person', 'personal', 'phone', 'photo',
    'physical', 'pick', 'picture', 'piece', 'place', 'plan',
    'plant', 'play', 'player', 'please', 'plus', 'point',
    'police', 'policy', 'political', 'poor', 'population',
    'position', 'possible', 'post', 'power', 'practice',
    'prepare', 'present', 'president', 'press', 'pressure',
    'pretty', 'price', 'private', 'pro', 'probably', 'problem',
    'process', 'produce', 'product', 'professor', 'program',
    'project', 'property', 'protect', 'prove', 'provide',
    'public', 'pull', 'purpose', 'push', 'put', 'quality',
    'question', 'quickly', 'quite', 'race', 'raise', 'rate',
    'rather', 'reach', 'read', 'ready', 'real', 'reality',
    'realize', 'really', 'reason', 'receive', 'recent',
    'recently', 'recognize', 'record', 'reduce', 'region',
    'relationship', 'release', 'remain', 'remember', 'remove',
    'report', 'reports', 'represent', 'republican', 'require', 'research',
    'resource', 'respond', 'response', 'rest', 'result',
    'return', 'right', 'rise', 'risk', 'road', 'robot', 'role', 'room',
    'round', 'rule', 'run', 'safe', 'same', 'save', 'say',
    'scene', 'school', 'science', 'season', 'seat', 'second',
    'section', 'security', 'see', 'seek', 'seem', 'sell',
    'send', 'sense', 'september', 'series', 'serious', 'serve',
    'service', 'set', 'seven', 'several', 'sex', 'share', 'she',
    'shoot', 'short', 'shot', 'should', 'show', 'side',
    'sign', 'significant', 'similar', 'simple', 'simply',
    'since', 'single', 'sir', 'sister', 'sit', 'site',
    'situation', 'six', 'size', 'skill', 'small', 'so',
    'social', 'society', 'some', 'somebody', 'someone',
    'something', 'sometimes', 'son', 'soon', 'sorry',
    'sort', 'sound', 'source', 'south', 'space', 'speak',
    'special', 'specific', 'spend', 'sport', 'staff', 'stage',
    'stand', 'standard', 'star', 'start', 'state', 'statement',
    'station', 'stay', 'step', 'still', 'stop', 'store',
    'story', 'strategy', 'street', 'strong', 'student', 'study',
    'stuff', 'subject', 'success', 'such', 'suggest', 'sun',
    'support', 'supposed', 'sure', 'system', 'table', 'take',
    'talk', 'tax', 'teach', 'teacher', 'team', 'technology',
    'tell', 'ten', 'term', 'test', 'than', 'thank', 'thanks',
    'that', 'the', 'their', 'them', 'themselves', 'then',
    'theory', 'there', 'these', 'they', 'thing', 'think',
    'third', 'this', 'those', 'though', 'thought', 'three',
    'through', 'throughout', 'throw', 'thus', 'till', 'time',
    'to', 'today', 'together', 'tonight', 'too', 'top',
    'toward', 'towards', 'town', 'training', 'treat',
    'treatment', 'tree', 'trouble', 'truth', 'try', 'turn',
    'tv', 'two', 'type', 'uh', 'under', 'underneath',
    'understand', 'union', 'university', 'unlike', 'until',
    'up', 'upon', 'us', 'use', 'usually', 'value', 'various',
    'version', 'versus', 'very', 'via', 'video', 'view',
    'visit', 'voice', 'vote', 'wait', 'walk', 'wall', 'want',
    'war', 'watch', 'water', 'way', 'we', 'weapon', 'wear', 'web',
    'week', 'well', 'west', 'what', 'whatever', 'when', 'where',
    'whether', 'which', 'while', 'who', 'whole',
    'whose', 'why', 'wife', 'win', 'window', 'wish', 'with',
    'within', 'without', 'woman', 'wonder', 'word', 'work',
    'worker', 'world', 'worry', 'would', 'write', 'wrong',
    'yeah', 'year', 'yes', 'yet', 'you', 'your', 'yourself',
}

_NON_PERSON_ELEMENTS = {
    # Possessive expressions are almost never part of a person's name.
    "'s",
    # People who hyphenate their names don't put spaces around the hyphen.
    ' - ',
    ' – ',                              # en-dash
    ' — ',                              # em-dash
}

_PUNCTUATION = set(string.punctuation)

# This next list is originally from the tldextract Python package.
_PUBLIC_SUFFIX_LIST_URLS = (
    "https://publicsuffix.org/list/public_suffix_list.dat",
    "https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat",
)

# Characters allowed in a domain name: letters, digits, and hyphen.
_DOMAIN_NAME_CHARTS = set(string.ascii_letters) ^ set(['-'])

# List of known top level domains. Populated at run time.
_DOMAIN_SUFFIXES = CaseFoldSet()

# Style preferences for rich_click.
click.rich_click.STYLE_HELPTEXT = ""
click.rich_click.USE_MARKDOWN = True
click.rich_click.STYLE_ERRORS_SUGGESTION = "bold italic"
click.rich_click.ERRORS_EPILOGUE = "Suggestion: use the --help flag to get help."


# Callback functions used in the click CLI definition.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

def _config_debug(ctx, param, debug_dest):
    '''Handle the --debug option and configure debug settings as needed.'''
    if debug_dest:
        if debug_dest.name == '<stdout>':
            set_debug(True, '-')
        else:
            set_debug(True, debug_dest.name)
        import faulthandler
        faulthandler.enable()
        if os.name != 'nt':                 # Can't use next part on Windows.
            import signal
            from boltons.debugutils import pdb_on_signal
            pdb_on_signal(signal.SIGUSR1)
            log(f'installed signal handler on {signal.SIGUSR1}')
    return debug_dest


def _read_token(ctx, param, file_):
    '''Read the file and set the environment variable GITHUB_TOKEN.'''
    if ctx.params.get('args', None) == 'help':
        _print_help_and_exit(ctx)
    elif file_:
        log('reading token from file')
        os.environ['GITHUB_TOKEN'] = file_.read()
    elif 'GITHUB_TOKEN' in os.environ:
        log('GITHUB_TOKEN found in environment')
    else:
        _alert(ctx, 'Cannot proceed without an access token. (Tip: provide the'
               ' `--token` option or set the environment variable **GITHUB_TOKEN**.)')
        sys.exit(1)
    return os.environ['GITHUB_TOKEN']


def _print_help_and_exit(ctx):
    '''Print the help text and exit with a success code.'''
    click.echo(ctx.get_help())
    sys.exit()


# Main function.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@click.command(add_help_option=False)
@click.help_option('--help', '-h', help='Show this message and exit')
#
@click.option('--input', '-i', metavar='SRC', type=File('r'),
              help='Read input from SRC (default: stdin)')
#
@click.option('--output', '-o', metavar='DEST', type=File('w'),
              help='Write the output to file DEST (default: stdout)')
#
@click.option('--debug', '-@', metavar='OUT', type=File('w', lazy=False),
              callback=_config_debug, help='Write debug output to destination "OUT"')
#
@click.pass_context
def main(ctx, input=None, output=None, debug=False):
    '''Filter a list of (putative) GitHub organization names.
\r
This program reads the input line-by-line and outputs a subset of the input
after applying heuristics to try to filter out names that do not _appear_ to
be actual organization names. It applies various heuristics and NLP tools to
detect and ignore text strings that look like the names of people, are too
short (1-2 chars long or a single word), are only numbers, look like
mistakes, or a variety of other things that don't look like organization
names. Note that this means the resulting list is _not_ a complete list of
all org accounts in GitHub – the filtering applied is designed to make the
results useful for the specific goal of IGA's name discrimination functions,
not to generate a comprehensive list of all org accounts in GitHub. (Some
organizations are truly named after people, but to reduce false positives in
the code in IGA, this program rejects such org names.)
\r
Simple example of usage:
```
    filter-github-orgs --input raw-github-orgs-list.txt
```
\r
If given the `--debug` argument, this program will output details about what it
is doing. The debug trace will be written to the given destination path; to
write to standard output instead, use a dash (`-`).
\r
Running IGA with the option `--help` will make it print help text and exit
without doing anything else.
'''
    # Process arguments & handle early exits ..................................

    input = input or sys.stdin
    log('writing output to ' + input.name)

    output = output or sys.stdout
    log('writing output to ' + output.name)

    # Do the main work ........................................................

    exit_code = 0
    try:
        for line in input:
            name = line.strip()
            if _acceptable(name):
                output.write(name + '\n')
                output.flush()
                log('accepted: ' + name)
            else:
                log('rejected: ' + name)
    except KeyboardInterrupt:
        # Catch it, but don't treat it as an error; just stop execution.
        log('keyboard interrupt received')
    except Exception as ex:             # noqa: PIE786
        exit_code = 2
        import traceback
        exception = sys.exc_info()
        details = ''.join(traceback.format_exception(*exception))
        log('exception: ' + str(ex) + '\n\n' + details)
        _alert(ctx, 'Experienced an unrecoverable error: ' + str(ex), False)
    sys.exit(exit_code)


# Miscellaneous helpers.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

def _load_spacy():
    global _SPACY
    log('loading spaCy en_core_web_trf -- this may take a long time')
    import spacy
    _SPACY = spacy.load('en_core_web_trf')
    log('finished loading spaCy pipeline')


def _load_suffixes():
    global _DOMAIN_SUFFIXES
    from commonpy.network_utils import net
    for url in _PUBLIC_SUFFIX_LIST_URLS:
        log(f'getting suffix list {url}')
        (response, error) = net('get', url)
        if error:
            log(f'error trying to get {url}: ' + str(error))
            break
        for line in filter(None, response.text.splitlines()):
            if line.startswith('/') or line.startswith(' '):
                continue
            _DOMAIN_SUFFIXES.add(line)
    log(f'collected {len(_DOMAIN_SUFFIXES)} known domain suffixes')


def _acceptable(name):
    '''Return True if the name passes some basic acceptability criteria.'''
    global _DOMAIN_SUFFIXES
    global _SPACY
    if not _SPACY:
        _load_spacy()
        _load_suffixes()

    name_tokens = re.split(r'[-\s_+:^~¡¿]+', name)
    split_name  = [part.lower() for part in re.split(r'[\s]+', name)]

    def likely_person():
        # If there are special characters in the name, it's not a person.
        if any(c in name for c in '#@[]<>/!|&%$=+:_^~¡¿'):
            return False
        # If a word in the name is a common word, it's probably not a person.
        # (Not necessarily true, but good enough for the filtering done here.)
        if any(word in _COMMON_NONNAME_WORDS for word in split_name):
            return False
        # A name like "Joe's Foobar" is sometimes labeled as a person by spaCy.
        # We try to catch them before we finally invoke spaCy.
        if any(item in name for item in _NON_PERSON_ELEMENTS):
            return False
        try:
            parsed = _SPACY(name)
            if parsed.ents:
                entity_type = parsed.ents[0].label_
                log(f'spaCy entity type for {name}: {entity_type}')
                return entity_type == 'PERSON'
            else:
                # If spaCy doesn't label it, don't assume it's a person.
                log(f'spaCy did not return entity labels for {name}')
                return False
        except Exception as ex:             # noqa: PIE786
            log('unable to use spaCy due to error: ' + str(ex))
        return False

    def _likely_domain():
        if '.' not in name:
            return False
        parts = name.split('.')
        return (len(parts) >= 2
                and all(c in _DOMAIN_NAME_CHARTS for c in parts[0])
                and ''.join(parts[1:]) in _DOMAIN_SUFFIXES)

    return (
        # Reject names shorter than 2 chars long.
        not len(name) <= 2
        # Reject names that don't have at least 2 items separated by whitespace
        # unless they seem to be domain names (e.g., companyname.com).
        and not (len(split_name) == 1 and not _likely_domain())
        # Reject names whose parts are only numbers.
        and not all(token.isdigit() for token in name_tokens)
        # Reject names whose parts are all in entirely lower case.
        and not all(token.islower() for token in name_tokens)
        # Reject names whose parts are only punctuation characters.
        and not all(part in _PUNCTUATION for part in split_name)
        # Reject names that look like they contain HTML tags.
        and not any(('<' in part and '>' in part) for part in split_name)
        # Reject names that start with unlikely characters.
        and not any(name.startswith(c) for c in '-=,!¿')
        # Reject what looks like a person's name.
        and not likely_person()
    )


def _alert(ctx, msg, print_usage=True):
    '''Print an error message in the style of rich_click.'''
    # The following code tries to emulate what rich_click does. It doesn't use
    # private methods or properties, but it might break if rich_click changes.
    log('error: ' + msg)
    from rich.console import Console
    from rich.markdown import Markdown
    from rich.padding import Padding
    from rich.panel import Panel
    from rich.theme import Theme
    from rich_click.rich_click import (
        ALIGN_ERRORS_PANEL,
        ERRORS_PANEL_TITLE,
        STYLE_ERRORS_PANEL_BORDER,
        STYLE_USAGE,
        STYLE_OPTION,
        STYLE_ARGUMENT,
        STYLE_SWITCH,
    )
    from rich.highlighter import RegexHighlighter
    class OptionHighlighter(RegexHighlighter):
            """Highlights our special options."""

            highlights = [
                r"(^|[^\w\-])(?P<switch>-([^\W0-9][\w\-]*\w|[^\W0-9]))",
                r"(^|[^\w\-])(?P<option>--([^\W0-9][\w\-]*\w|[^\W0-9]))",
                r"(?P<metavar><[^>]+>)",
            ]
    highlighter = OptionHighlighter()
    console = Console(theme=Theme({
        "option": STYLE_OPTION,
        "argument": STYLE_ARGUMENT,
        "switch": STYLE_SWITCH,
        "usage": STYLE_USAGE,
    }), highlighter=highlighter)
    if print_usage:
        console.print(Padding(highlighter(ctx.get_usage()), 1))
    console.print(
        Panel(
            Markdown(msg),
            border_style=STYLE_ERRORS_PANEL_BORDER,
            title=ERRORS_PANEL_TITLE,
            title_align=ALIGN_ERRORS_PANEL,
        )
    )


# Main invocation.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

main()