ToxicComments/DataExploratory.py at master · glapointe7/ToxicComments · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import numpy as np
import pandas
import re
import string


## Faster with Series than Counter.
def GetWordFrequency(train_comments):
    return pandas.Series(' '.join(train_comments).lower().split()).value_counts()

def ascii_chars_from_text(text, ascii_chars):
    for char in text:
        if char in ascii_chars:
            yield char


def clean_text(comments):
    train_comments_cleaned = []
    ascii_chars = set(string.printable)

    for comment in comments:
        ## Convert to lower case , so that Hi and hi are the same.
        comment = comment.lower().strip(' ')
        ## Remove carriage returns.
        comment = re.sub("\\n", " ", comment)
        ## Remove leaky elements like IP addresses.
        #comment = re.sub("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}", "", comment)
        ## Remove usernames.
        comment = re.sub("\[\[.*\]", "", comment)
        comment = re.sub("[\$\*&%#@\"]", " ", comment)
        ## Remove punctuation.
        comment = re.sub('\W', ' ', comment)
        ## Remove non ascii characters.
        comment = ''.join([char for char in ascii_chars_from_text(comment, ascii_chars)])
        comment = re.sub("fck", "fuck", comment)
        comment = re.sub("f ck", "fuck", comment)
        comment = re.sub("fagget", "faggot", comment)
        comment = re.sub("you re", "you are", comment)
        comment = re.sub("\d", " ", comment)

        train_comments_cleaned.append(comment)

    return pandas.Series(train_comments_cleaned).astype(str)

def clean(train_perspective):
    classes = ['comment', 'is_toxic']
    train = train_perspective.loc[:, classes]

    train.comment = clean_text(train.comment)
    train.is_toxic = train.is_toxic.astype(np.int64)

    return train