nltk_utils.py

import nltk
import numpy as np
from nltk.stem.lancaster import LancasterStemmer
nltk.download('punkt_tab')

stemmer = LancasterStemmer()

def tokenize(sentence):
    """
    Tokenize a sentence into words
    """
    return nltk.word_tokenize(sentence)

def stem(word):
    """
    Stem a word to its root form
    """
    return stemmer.stem(word.lower())

def bag_of_words(tokenized_sentence, words):
    """
    Create a bag of words representation
    """
    sentence_words = [stem(word) for word in tokenized_sentence]
    bag = np.zeros(len(words), dtype=np.float32)
    for idx, w in enumerate(words):
        if w in sentence_words:
            bag[idx] = 1.0
    return bag