smartclide-smart-assistant/smartclide-dle-models/serviceclassification/servclassify/PreProcessTextData.py at 3ed53c0c69056a536670bff5895c3e201e9741a2 · eclipse-opensmartclide/smartclide-smart-assistant · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
#!/usr/bin/python3
#*******************************************************************************
# Copyright (C) 2021-2022 AIR Institute
#
# This program and the accompanying materials are made
# available under the terms of the Eclipse Public License 2.0
# which is available at https://www.eclipse.org/legal/epl-2.0/
#
# SPDX-License-Identifier: EPL-2.0
#*******************************************************************************


import re
import pandas as pd
import numpy as np
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer
from nltk.stem.snowball import SnowballStemmer
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.tokenize import WhitespaceTokenizer
from nltk import sent_tokenize, word_tokenize
from nltk.stem.wordnet import WordNetLemmatizer
# nltk.download('stopwords')
# nltk.download('wordnet')
# nltk.download('punkt')
# nltk.download('omw-1.4')


class TextDataPreProcess:

    def __init__(self):
        self.level=1


    def cleanPunc(self, sentence):
        import re
        cleaned = re.sub(r'[?|!|\[|\]|\'|"|#]', r'', sentence)
        cleaned = re.sub(r'[.|)|(|\|/]', r' ', cleaned)
        cleaned = re.sub(r'[\d]', r' ', cleaned)  # remove digite
        cleaned = cleaned.strip()
        cleaned = cleaned.replace("\n", "")
        return cleaned

    def removeStopWords(self, sentence):
        try:
            from nltk.corpus import stopwords
            global re_stop_words
            stop_words = set(stopwords.words('english'))
            stop_words.update(['monthly', "google", "api", "apis", 'json','Json',"service", "provide","including", "data", "REST", "RESTFUL", "website", "site"])
        except:
            nltk.download('stopwords')
            from nltk.corpus import stopwords
            global re_stop_words
            stop_words = set(stopwords.words('english'))
            stop_words.update(['monthly', "google", "api", "apis", 'json','Json',"service", "provide","including", "data", "REST", "RESTFUL", "website", "site"])

        re_stop_words = re.compile(r"\b(" + "|".join(stop_words) + ")\\W", re.I)
        return re_stop_words.sub(" ", sentence)

    def wordLemmatizer(self, sentence):
        #nltk.download('punkt')
        #nltk.download('wordnet')
        try:
            from nltk.stem.wordnet import WordNetLemmatizer
            lemmatizer = WordNetLemmatizer()
            lem_sentence = " ".join([lemmatizer.lemmatize(i) for i in sentence])
        except:
            nltk.download('wordnet')
            nltk.download('punkt')
            nltk.download('omw-1.4')
            from nltk.stem.wordnet import WordNetLemmatizer
            lemmatizer = WordNetLemmatizer()
            lem_sentence = " ".join([lemmatizer.lemmatize(i) for i in sentence])
        return lem_sentence

    def getTopNwords(self, corpus, n=None):
        from sklearn.feature_extraction.text import CountVectorizer
        vec = CountVectorizer.fit(corpus)
        bagofWords = vec.transform(corpus)
        sumWords = bagofWords.sum(axis=o)
        wordFreq = [(word, sumWords[0, indx]) for word, idx in vec.vocabulary_items()]
        wordFreq = sorted(wordFreq, key=lambda x: x[1], reverse=True)
        return wordsFreq[:n]

    def getCommonWords(self, count_data, countVectorizerOBJ, n):
#         words = countVectorizerOBJ.get_feature_names_out()
        try:
              words = countVectorizerOBJ.get_feature_names_out()
        except:
              words = countVectorizerOBJ.get_feature_names()


        total_counts = np.zeros(len(words))
        for t in count_data:
            total_counts += t.toarray()[0]
        count_dict = (zip(words, total_counts))
        count_dict = sorted(count_dict, key=lambda x: x[1], reverse=True)[0:n]
        words = [w[0] for w in count_dict]
        counts = [w[1] for w in count_dict]
        x_pos = np.arange(len(words))
        return words

    def removeHTMLTags(self, string):
        result = re.sub('<.*?>', '', string)
        return result

    def uniqueList(self, l):
        ulist = []
        [ulist.append(x) for x in l if x not in ulist]
        return ulist