Introduction: Predicting The Next Word In A Sentence

import string
#Provides a set of diverse algorithms for an LP and for Computational Linguistics.
import nltk
import re
import string
from nltk.corpus import stopwords
import pkg_resources
import pickle
import nltk
import re, string, json
from tqdm.notebook import tqdm
from nltk.tokenize import word_tokenize

from numpy import array
from pickle import dump
from keras.preprocessing.text import Tokenizer
#keras's library function that is embedded into the TensorFlow Library
#TensorFlow provides both high level and low level APIs for working with deep learning models
#Using packages required to handle string data. 
from tensorflow.keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Embedding

from random import randint
from pickle import load
from keras.models import load_model
from keras.preprocessing.sequence import pad_sequences

# load doc into memory
def load_doc(filename):
    # open the file as read only
    file = open(filename, 'r')
    # read all text
    text = file.read()
    # close the file
    file.close()
    return text

# turn a doc into clean tokens
def clean_doc(doc):
    # replace '--' with a space ' '
    doc = doc.replace('--', ' ')
    # split into tokens by white space
    tokens = doc.split()
    # remove punctuation from each token
    table = str.maketrans('', '', string.punctuation)
    tokens = [w.translate(table) for w in tokens]
    # remove remaining tokens that are not alphabetic
    tokens = [word for word in tokens if word.isalpha()]
    # make lower case
    tokens = [word.lower() for word in tokens]
    return tokens

# save tokens to file, one dialog per line
def save_doc(lines, filename):
    data = '\n'.join(lines)
    file = open(filename, 'w')
    file.write(data)
    file.close()

def tokenize_word(sentences):
    """
    Tokenize sentences into tokens (words)
    
    Args:
        sentences: List of strings
    
    Returns:
        List of lists of tokens
    """
    print("Starting Cleaning Process")
    tokenized_sentences = []
    for sentence in tqdm(sentences):
        
        # Convert to lowercase letters
        sentence = cleanhtml(sentence)
        sentence = _replace_urls(sentence)
        sentence = remove_email(sentence)
        #Anything that is not a lower or upper case word remove from our sentence repalce with a empty string
        sentence = re.sub(r'[^a-zA-Z]', ' ', sentence)
        sentence = sentence.lower()
        sentence = misc(sentence)
        

        # tokenized = nltk.word_tokenize(sentence)
        
        # append the list of words to the list of lists
        # tokenized_sentences.append(tokenized)
        tokenized_sentences.append(sentence)
    
    return tokenized_sentences

#We use particular functions below to clean the string list.
def cleanhtml(raw_html):
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, '', raw_html)
    return cleantext

#Function replaces and removes URLS from the dataset.
def _replace_urls(data):
    #Removing URLs with a regular expression
    url_pattern = re.compile(r'https?://\S+|www\.\S+')
    data = url_pattern.sub(r'', data)
    return data

#Function removes emails 
def remove_email(data):
    # Remove Emails
    data = re.sub('\S*@\S*\s?', '', data)
    return data

#Function removes other random data not needed
def misc(data):
    # Remove new line characters
    data = re.sub('\s+', ' ', data)
    # Remove distracting single quotes
    data = re.sub("\'", "", data)
    data = re.sub("ww+", "", data)
    # Removing roman-case:
    MAYBE_ROMAN = re.compile(r'(\b[MDCLXVI]+\b)(\.)?', re.I)
    data = re.sub(MAYBE_ROMAN, "", data)
    return data

#This function is looking for sentences that the number of tokens are less than 5 remove then sentence. 
#Sentences with less than 5 tokens are to short for our LSTM model to be useful.
def littleCleaning(sentences):
    print("Starting cleaning Process")
    ret_list = []
    for sentence in sentences:
        words = sentence.split(" ")
        if len(words) > 5:
            ret_list.append(sentence)
        else:
            continue
    return ret_list

#Download the Natural Language Processing with Python package that is needed
nltk.download('wordnet')
nltk.download('punkt')

path = 'Dataset/republic.txt'
text = open(path).read().lower()
print('length of the corpus is: :', len(text))

# Converting the data into lists.
# Creating sentences into strings using the split fuction

data_list = text.split(".")
data_list[:20]

pro_sentences = []
# Function to clean up data
def normalization_pipeline(sentences):
    print("Starting Normalization Process")
    sentences = tokenize_word(sentences)
    sentences = littleCleaning(sentences)
    print("Normalization Process Finished")
    return sentences

pro_sentences = normalization_pipeline(data_list)
pro_sentences[: 5]

#After the clean up procress we have 6,309 sentences compaired to 7,012 setences before the cleaning process
len(pro_sentences)

# Structuring th etext into a paragraph:
# We are only using 700 sentences out of 6,309 sentences. 
dataText = "".join(pro_sentences[: 700])
dataText[: 200]

# turn a doc into clean tokens
def clean_doc(doc):
    # replace '--' with a space ' '
    doc = doc.replace('--', ' ')
    # split into tokens by white space
    tokens = doc.split()
    # remove punctuation from each token
    table = str.maketrans('', '', string.punctuation)
    tokens = [w.translate(table) for w in tokens]
    # remove remaining tokens that are not alphabetic
    tokens = [word for word in tokens if word.isalpha()]
    # make lower case
    tokens = [word.lower() for word in tokens]
    return tokens

# clean document
tokens = clean_doc(dataText)
print(tokens[:200])
print('Total Tokens: %d' % len(tokens))
print('Unique Tokens: %d' % len(set(tokens)))

# organize into sequences of tokens to create our input data
#Creating a list that has 51 words
#the words will be input and the 51 word will be the independent variable
#The 51 word will be the word we use to predict
length = 50 + 1
sequences = list()
for i in range(length, len(tokens)):
    # select sequence of tokens
    seq = tokens[i-length:i]
    # convert into a line
    line = ' '.join(seq)
    # store
    sequences.append(line)
print('Total Sequences: %d' % len(sequences))

# save sequences to file

# save tokens to file, one dialog per line
def save_doc(lines, filename):
    data = '\n'.join(lines)
    file = open(filename, 'w')
    file.write(data)
    file.close()
    
out_filename = 'Dataset/republic_sequences.txt'
save_doc(sequences, out_filename)

# load
in_filename = 'Dataset/republic_sequences.txt'
doc = load_doc(in_filename)
lines = doc.split('\n')

# integer encode sequences of words
tokenizer = Tokenizer()
tokenizer.fit_on_texts(lines)
# Transforming my regular texts into numberical ones like the list of vectors
sequences = tokenizer.texts_to_sequences(lines)
# vocabulary size meaning the number of unqunie words
vocab_size = len(tokenizer.word_index) + 1

# Separate into input and output
# Convert the sequences into a array to bulid our X and Y variables
sequences = array(sequences)
X, y = sequences[:,:-1], sequences[:,-1]
y = to_categorical(y, num_classes=vocab_size)
seq_length = X.shape[1]

# define model
model = Sequential()
#Embedding the model use vocab_size is to create vector and vector for each word
model.add(Embedding(vocab_size, 50, input_length=seq_length))
#Using the LSTM function with input to 50 tokens or words
model.add(LSTM(50, return_sequences=True))
model.add(LSTM(50))
#In our model, we are classified by taking the input variable and classifying unique words used for the prediction 
#of our model. Then we use the words to make the prediction. 
#Dense model understanding our input for the 50 words
model.add(Dense(50, activation='relu'))
#Desse model to understand the vocab_size and use the activation function softmax which is used for classification
model.add(Dense(vocab_size, activation='softmax'))
print(model.summary())
# compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# fit model
batch_size=128
epochs=50
model.fit(X, y, batch_size=batch_size, epochs=epochs)

# save the model to file
model.save("Dataset/DataScienceModels/nexWordPredict/nextWord.h5")
# save the tokenizer
dump(tokenizer, open('Dataset/DataScienceModels/tokenizer.pkl', 'wb'))

# generate a sequence from a language model to test our model
import numpy as np

def generate_seq(model, tokenizer, seq_length, seed_text, n_words):
    result = list()
    in_text = seed_text
    # generate a fixed number of words
    for _ in range(n_words):
        # encode the text as integer
        encoded = tokenizer.texts_to_sequences([in_text])[0]
        # truncate sequences to a fixed length
        #padding function for to make word size
        encoded = pad_sequences([encoded], maxlen=seq_length, truncating='pre')
        # predict probabilities for each word
        # yhat = model.predict_classes(encoded, verbose=0)
        predict_x=model.predict(encoded) 
        yhat=np.argmax(predict_x,axis=1)
        # map predicted word index to word
        out_word = ''
        for word, index in tokenizer.word_index.items():
            if index == yhat:
                out_word = word
                break
        # append to input
        in_text += ' ' + out_word
        result.append(out_word)
    return ' '.join(result)

# load cleaned text sequences
in_filename = 'Dataset/republic_sequences.txt'
doc = load_doc(in_filename)
lines = doc.split('\n')
seq_length = len(lines[0].split()) - 1

print(len(lines))
print(lines[0])

# load the model
model = load_model("Dataset/DataScienceModels/nexWordPredict/nextWord.h5")

# load the tokenizer
tokenizer = load(open('Dataset/DataScienceModels/tokenizer.pkl', 'rb'))

# select a seed text
seed_text = lines[randint(0,len(lines))]
print(seed_text + '\n')

# generate new text
generated = generate_seq(model, tokenizer, seq_length, seed_text, 12)
print(generated)