nltk_utils.py

import nltk
from nltk.stem.porter import PorterStemmer

import numpy as np


stemmer = PorterStemmer()


def tokenize ( sentence ):
    """
    split sentence into array of words/tokens
    a token can be a word or punctuation character, or number
    """
    return nltk.word_tokenize( sentence )


def stem ( word ):
    """
    stemming = find the root form of the word
    examples:
    words = ["organize", "organizes", "organizing"]
    words = [stem(w) for w in words]
    -> ["organ", "organ", "organ"]
    """
    return stemmer.stem(word.lower())



def bag_of_words ( tokenise_sentences ,  all_words ):
    """
    return bag of words array:
    1 for each known word that exists in the sentence, 0 otherwise
    example:
    sentence = ["hello", "how", "are", "you"]
    words = ["hi", "hello", "I", "you", "bye", "thank", "cool"]
    bag   = [  0 ,    1 ,    0 ,   1 ,    0 ,    0 ,      0]
    """
    
    #stem each tokenize sentences , and all_words
    stemize_words = [ stem(w) for w in tokenise_sentences]

    stemize_all_words = [ stem(w) for w in all_words]

    # initate bag with 0 for each words of intention.json [0 , 0 , 0]
    bag = np.zeros( len(stemize_all_words) , dtype=np.float32)

    for id , word in enumerate(stemize_all_words):
        if word in stemize_words:
            bag[id] = 1.0

    return bag