Using the classifier about nlp-tutorial HOT 10 OPEN

# save model architecture 
model_json = model.to_json()
with open("model.json", "w") as json_file : 
    json_file.write(model_json)

# save weights 
model.save_weights("model.h5")

# load model architecture
from keras.models import model_from_json
with open("model.json", "r") as json_file:
    loaded_model_json = json_file.read()
loaded_model = model_from_json(loaded_model_json)

# load weights
loaded_model.load_weights("model.h5")

from keras.models import load_model
import keras.preprocessing.text as kpt
import numpy as np
import pandas as pd
import pickle

# loading
model = load_model('model_20190519141343.h5')

handle = open('tokenizer.pickle', 'rb')
tk = pickle.load(handle)

data = pd.read_json('News_Category_Dataset_v2.json', lines=True).drop(['authors', 'date', 'link'], axis=1)

while 1:
	text = input("Say something: ")
	
	if len(text) == 0:
		break

	#tk.fit_on_texts(text)
	#converts the texts to the index equivalents in our dictionary
	pred = tk.texts_to_sequences([text])
	print(pred)
	
	#onehot representation of all words in the evaluation text, and how they appear in our dictionary
	#input = tk.sequences_to_matrix(pred, mode='binary')
	
	arr = np.zeros(50).reshape(1, 50)
	print(arr)
	for i, word in enumerate(pred[0]):
		arr[0][i] = word
	print(arr)

	prediction = model.predict(arr)
	print(prediction)
	cls = np.argmax(prediction)
	
	print(cls)
	print(data['category'][cls])

>>> m = MosesTokenizer()
>>> m.tokenize('2016, pp.')
    [u'2016', u',', u'pp', u'.']

    # save model
    model.save('model_'+current+'.h5')
    print('MODEL SAVED')
	
	# saving
    with open('tokenizer.pickle', 'wb') as handle:
        pickle.dump(loader.tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)

# loading
model = load_model('model_20190519141343.h5')

handle = open('tokenizer.pickle', 'rb')
#tk = pickle.load(handle)
tk = MosesTokenizer()

data = pd.read_json('News_Category_Dataset_v2.json', lines=True).drop(['authors', 'date', 'link'], axis=1)

while 1:
	text = input("Say something: ")
	
	if len(text) == 0:
		break

	#tk.fit_on_texts(text)
	#converts the texts to the index equivalents in our dictionary
	tokens = tk.tokenize(text.strip(), escape=False)
	print(tokens)

	arr = np.zeros(50).reshape(1, 50)
	print(arr)
	for i, word in enumerate(tokens):
		arr[0][i] = word
	print(arr)

	prediction = model.predict(arr)
	print(prediction)
	cls = np.argmax(prediction)
	
	print(cls)
	print(data['category'][cls])

tokenizer = Tokenizer(num_words = 50000+1, oov_token='UNK')
tokenizer.texts_to_sequences(tokens)

# token_to_index
tokens = tokenized_corpus.apply(lambda i: i.split())
tokenizer.fit_on_texts(tokens)
tokenizer.word_index = {word:index for word, index in tokenizer.word_index.items() if index <= 50000}

def token_to_index(self, tokenized_corpus, maximum_word_num):
   tokenizer = Tokenizer(num_words = maximum_word_num+1, oov_token='UNK')
        
   # tokenizer fitting (token to index number)
   tokens = tokenized_corpus.apply(lambda i: i.split())
   tokenizer.fit_on_texts(tokens)

   # build vocabulary
   tokenizer.word_index = {word:index for word, index in tokenizer.word_index.items() if index <= maximum_word_num}
   vocabulary = tokenizer.word_index

   # texts_to_sequences changes words into indexes
   return vocabulary, tokenizer.texts_to_sequences(tokens)

from keras.models import load_model
from nltk.tokenize.moses import MosesTokenizer
import keras.preprocessing.text as kpt
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer, text_to_word_sequence
import numpy as np
import pandas as pd
import pickle
import data_loader

# load model architecture
from keras.models import model_from_json

# loading
model = load_model('model_20190519141343.h5')

handle = open('tokenizer.pickle', 'rb')
tk = pickle.load(handle)

loader = data_loader.DataLoader("corpus.tk.txt", "corpus.tk.vec.txt", "News_Category_Dataset_v2.json")
loader.load_cat()
print(loader.category_dict)
while 1:
	text = input("Say something: ")
	
	if len(text) == 0:
		break

	seq = tk.texts_to_sequences([text.strip()])
	x = pad_sequences(seq, 50)
	print(x)
	prediction = model.predict(x)
	print(prediction)
	cls = np.argmax(prediction)
	
	print(cls)
	print(loader.category_dict[cls])