text classification using neural network in keras - model is weak

Question

i'm trying to classify verses to book in the bible, the problem is that my model is not good and i can't find a way to improve it.

this is my code:

import tensorflow.keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation
from tensorflow.keras.layers import MaxPooling2D,Conv2D
from tensorflow.keras.optimizers import SGD
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from tensorflow.keras.layers import LSTM
from tensorflow.keras.layers import Embedding
from tensorflow.keras.layers import SpatialDropout1D
from sklearn.model_selection import train_test_split
from tensorflow.keras import regularizers

import pandas as pd              
import numpy as np 

data = pd.read_csv("bible_data_set (with count and testament).csv")
data

import nltk
from nltk.stem import PorterStemmer

ps = PorterStemmer() 

vocabulary_size = 0
word2location = {}

def prepare_vocabulary(data):
    index = 0
    for sentance in data['text']:
        #sentance = sentance.lower()
        words = nltk.word_tokenize(sentance)
        for word in words:
            stemed_word = ps.stem(word)
            if stemed_word not in word2location:
                word2location[stemed_word] = index
                index += 1
    return index

def convert2vec(sentance):
    #sentance = sentance.lower()
    res_vec = np.zeros(vocabulary_size)
    words = nltk.word_tokenize(sentance)
    for word in words:
        stemed_word = ps.stem(word)
        if stemed_word in word2location:
            res_vec[word2location[stemed_word]]+=1
    return res_vec

books = ['Genesis', 'Exodus', 'Leviticus', 'Numbers', 'Deuteronomy', 'Joshua', 'Judges',
            'Ruth', '1 Samuel', '2 Samuel', '1 Kings', '2 Kings', '1 Chronicles', '2 Chronicles',
            'Ezra', 'Nehemiah', 'Esther', 'Job', 'Psalms', 'Proverbs', 'Ecclesiastes',
            'Song of Solomon', 'Isaiah', 'Jeremiah', 'Lamentations', 'Ezekiel', 'Daniel',
            'Hosea', 'Joel', 'Amos', 'Obadiah', 'Jonah', 'Micah', 'Nahum', 'Habakkuk',
            'Zephaniah', 'Haggai',    'Zechariah',    'Malachi', 'Matthew', 'Mark', 'Luke', 'John', 'Acts', 'Romans', '1 Corinthians',
            '2 Corinthians', 'Galatians', 'Ephesians', 'Philippians', 'Colossians',
            '1 Thessalonians', '2 Thessalonians', '1 Timothy', '2 Timothy', 'Titus', 'Philemon',
            'Hebrews', 'James', '1 Peter', '2 Peter', '1 John', '2 John', '3 John', 'Jude',
            'Revelation']

def encode(line):
    res_vec = np.zeros(66)
    idx = books.index(data.iloc[line]['book'])
    res_vec[idx] = 1
    return res_vec

vocabulary_size = prepare_vocabulary(data)
print("the size of the vocabulary is: ", vocabulary_size)
word2location

import random

rand = []
for r in range (4500):
    ra = random.randrange(0, 31101)
    if(ra not in rand):
        rand.append(ra)
            
train_x = []
train_y = []
test_x = []
test_y = []
for i in range(len(data['text'])):
    if(i not in rand):
        train_x.append(i)
        train_y.append(i)
        
    elif(i in rand):
        test_x.append(i)
        test_y.append(i)

data_x = np.array([convert2vec(data.iloc[i]['text']) for i in train_x])
np.random.shuffle(data_x)
data_y = np.array([encode(i) for i in train_y])
np.random.shuffle(data_y)
test_data_x = np.array([convert2vec(data.iloc[i]['text']) for i in test_x])
np.random.shuffle(test_data_x)
test_data_y = np.array([encode(i) for i in test_y])
np.random.shuffle(test_data_y)

model = Sequential()
model.add(Dense(128, activation = 'sigmoid', input_dim = vocabulary_size))
model.add(Dropout(0.1))
model.add(Dense(128, activation = 'sigmoid'))
model.add(Dropout(0.1))
model.add(Dense(66, activation = 'softmax'))

opt = SGD(lr=0.01)

model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
history = model.fit(data_x, data_y, epochs=50, batch_size=16,validation_data=(test_data_x,test_data_y),callbacks=[EarlyStopping(monitor='val_loss', patience=5, min_delta=0.00001)])

i keep getting overfitting or under fitting. i have tried relu activation for the dense, and changed the loss function and optimizer, but nothing helped. is there anything i'm missing?

try decreasing your learning rate to 0.001 - does it improve the model at all? — Stephen Mylabathula

Gaussian Prior Gaussian Prior · Accepted Answer · 2020-12-23T15:54:50

Here

data_x = np.array([convert2vec(data.iloc[i]['text']) for i in train_x])
np.random.shuffle(data_x)
data_y = np.array([encode(i) for i in train_y])
np.random.shuffle(data_y)
test_data_x = np.array([convert2vec(data.iloc[i]['text']) for i in test_x])
np.random.shuffle(test_data_x)
test_data_y = np.array([encode(i) for i in test_y])
np.random.shuffle(test_data_y)

You called np.random.shuffle for your train data (data_x) and np.random.shuffle for your train labels (data_y). This should not be correct as your features should remain paired to your label. Just pair these together and random shuffle once and do the same for testing.

text classification using neural network in keras - model is weak

1 Answers