# Importing the required the libraries

# To read and manipulate the data
import pandas as pd
pd.set_option('max_colwidth', None)

# To visualise the graphs
import matplotlib.pyplot as plt
import seaborn as sns

# Helps to display the images
from PIL import Image

# Helps to extract the data using regular expressions
import re

# Helps to remove the punctuation
import string

# It helps to remove the accented characters 

import unidecode

# Importing the NLTK library
import nltk

nltk.download('stopwords')    # Loading the stopwords
nltk.download('punkt')        # Loading the punkt module, used in Tokenization
nltk.download('omw-1.4')      # Dependency for Tokenization
nltk.download('wordnet')      # Loading the wordnet module, used in stemming and lemmatization

# downloading vader lexicon
nltk.download('vader_lexicon')

from nltk.corpus import stopwords

# Helps to visualize the wordcloud
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

# Used in Stemming
from nltk.stem.porter import PorterStemmer


# Used in Lemmatization
from nltk.stem import WordNetLemmatizer


from sklearn.feature_extraction.text import CountVectorizer          #For Bag of words
from sklearn.feature_extraction.text import TfidfVectorizer          #For TF-IDF

# Helped to create train and test data
from sklearn.model_selection import train_test_split

# Importing the Random Forest model
from sklearn.ensemble import RandomForestClassifier

# Metrics to evaluate the model
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix

# Unsupervised learning models

# Install vader sentiment package
!pip install vaderSentiment 

# Install textblob package
!pip install textblob   

# View the first and last 5 rows of the dataset
data.head(5)

data.tail()

# Understand the shape of the dataset
data.shape               

# Check the data types of the columns for the dataset
data.info()

# checking for duplicate values
data.duplicated().sum()

# dropping the duplicates
data = data.drop_duplicates(keep = 'first')

# checking for duplicate values
data.duplicated().sum()

plt.figure(1, figsize = (12, 12))

plt.imshow(wordcloud)

plt.axis('off')

plt.show()

plt.figure(1, figsize = (12, 12))

plt.imshow(wordcloud)

plt.axis('off')

plt.show()

# Check the labels
data['sentiment'].unique()       

# check the count of each labels
data['sentiment'].value_counts()    

# Plot the distribution of the class label
def bar_plot(data, feature):
    # Creating the countplot
    plot = sns.countplot(x = feature, data = data)
    
    # Finding the length the whole data
    total = len(data)
    
    # Creating the percentages to each label in the data
    for p in plot.patches:
        percentage = '{:.1f}%'.format(100 * p.get_height()/total)
        x = p.get_x() + p.get_width() / 2 - 0.05
        y = p.get_y() + p.get_height()
        plot.annotate(percentage, (x, y),ha="center",
            va = "center",
            size = 12,
            xytext = (0, 5),
            textcoords = "offset points")
    plt.show()
    
bar_plot(data,'sentiment')

# let's have look at the cleaned text
final_corpus[0]

final_corpus[1]

data.head(5)

# Shape of the feature vector
data_features.shape       

# Training the best model and calculating accuracy on test data 

clf = RandomForestClassifier(n_estimators = 100)

clf.fit(X_train, y_train)

y_pred_test = clf.predict(X_test)

metrics_score(y_test, y_pred_test)

 #Instantiate the feature from the vectorizer
    
features = Count_vec.get_feature_names()   

get_top40_words(clf,features)

# Using TfidfVectorizer to convert text data to numbers.

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf_vect = TfidfVectorizer(max_features = 1000)

data_features = tfidf_vect.fit_transform(data['review'])

data_features = data_features.toarray()

# Feature shape
data_features.shape     

# Training the best model and calculating accuracy on test data 

clf1 = RandomForestClassifier(n_estimators = 100)

clf1.fit(X_train, y_train)

y_pred_test1 = clf1.predict(X_test)

metrics_score(y_test, y_pred_test1)

#Instantiate the feature from the vectorizer    
features = tfidf_vect.get_feature_names()  

get_top40_words(clf1,features)

#convert the test samples into a dataframe where the columns are

#the y_test(ground truth labels),tf-idf model predicted labels(tf_idf_predicted),
#Count Vectorizer model predicted labels(count_vectorizer_predicted)

df = pd.DataFrame(y_test.tolist(), columns = ['y_test'])

df['count_vectorizer_predicted'] = y_pred_test

df['tf_idf_predicted'] = y_pred_test1

df.head()

# Calculate the polarity score of the reviews
data_new['scores'] = data_new['review'].apply(lambda text: sentiment.polarity_scores(text))

data_new.head()

# Calculate the compund score of the reviews
data_new['compound']  = data_new['scores'].apply(lambda score_dict: score_dict['compound'])

data_new.head()

# Classify the class of the review by keeping threshold on the compund score

data_new['comp_score'] = data_new['compound'].apply(lambda c: '1' if c >=0 else '0')

data_new.head()

data["VADER_pred"] = data_new['comp_score'].tolist()
data.head()

# Calculate the accuracy of the Vader Sentiment Analysis
data["sentiment"] = data["sentiment"].astype(int)   #convert the sentiment column values into int data type

data["VADER_pred"] = data["VADER_pred"].astype(int) #convert the vader_predicted column values into int data type

metrics_score(data["sentiment"], data["VADER_pred"])

# Calculate the polarity score of the reviews
data_new['polarity']  = data_new['review'].apply(lambda text: TextBlob(text).sentiment.polarity)

data_new.head()

# Classify the class of the review by keeping threshold on the polarity score
data_new['polarity_score'] = data_new['polarity'].apply(lambda c: '1' if c >=0 else '0')

data_new.head()

data["Text_Blob_pred"] = data_new['polarity_score'].tolist()
data.head()

# Calculate the accuracy of the Vader Sentiment Analysis
data["sentiment"] = data["sentiment"].astype(int)  #convert the sentiment column values into int data type

data["Text_Blob_pred"] = data["Text_Blob_pred"].astype(int)  #convert the textblob predicted column values into int data type

metrics_score(data["sentiment"], data["Text_Blob_pred"])