# import all the libraries

import numpy as np
import re
import nltk
from sklearn.datasets import load_files
nltk.download('stopwords')
import pickle
import pandas as pd
from sklearn.multioutput import MultiOutputClassifier
from sklearn.preprocessing import LabelEncoder
from nltk.corpus import stopwords
from collections import defaultdict
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline
import joblib 


pd.options.display.float_format = "{:,.2f}".format

df = pd.read_csv(r'/Users/tabetibrahim/Desktop/Vona/vef/facture_mobile_mars_avril_2021.csv', sep=";")

data_set = df[['Tem section 1', 'Tem section 2','Tem section 3','Description facture operateur']]
data_set.head()

tem_section= data_set[['Tem section 1', 'Tem section 2','Tem section 3']]

tem_en= data_set[['Tem section 1', 'Tem section 2','Tem section 3']]

labelencoder = LabelEncoder()
le = LabelEncoder()
encoder_dict = defaultdict(LabelEncoder)

df_encode = tem_en.apply(lambda x: encoder_dict[x.name].fit_transform(x))

inverse_transform_lambda = df_encode.apply(lambda x: encoder_dict[x.name].inverse_transform(x))

X, y =  data_set[['Description facture operateur']], tem_section


import nltk
import pandas as pd
import numpy as np
from nltk.stem import PorterStemmer
from nltk.corpus import stopwords

# If not previously performed:
# nltk.download('stopwords')

stemming = PorterStemmer()
stops = set(stopwords.words("french"))

def apply_cleaning_function_to_list(X):
    cleaned_X = []
    for element in X:
        cleaned_X.append(clean_text(element))
    return cleaned_X


def clean_text(raw_text):
    """This function works on a raw text string, and:
        1) changes to lower case
        2) tokenizes (breaks down into words
        3) removes punctuation and non-word text
        4) finds word stems
        5) removes stop words
        6) rejoins meaningful stem words"""

    # Convert to lower case
    text = raw_text.lower()

    # Tokenize
    tokens = nltk.word_tokenize(text)

    # Keep only words (removes punctuation + numbers)
    # use .isalnum to keep also numbers
    token_words = [w for w in tokens if w.isalpha()]

    # Stemming
    stemmed_words = [stemming.stem(w) for w in token_words]

    # Remove stop words
    meaningful_words = [w for w in stemmed_words if not w in stops]

    # Rejoin meaningful stemmed words
    joined_words = ( " ".join(meaningful_words))

    # Return cleaned data
    return joined_words




# Get text to clean
text_to_clean = list(data_set['Description facture operateur'])

# Clean text
cleaned_text = apply_cleaning_function_to_list(text_to_clean)

# Add cleaned data back into DataFrame
data_set['cleaned_review'] = cleaned_text

# Remove temporary cleaned_text list (after transfer to DataFrame)
del cleaned_text


from sklearn.model_selection import train_test_split
X = list(data_set['cleaned_review'])
y = df_encode
X_train, X_test, y_train, y_test = train_test_split(
    X,y, test_size = 0.25)

def create_bag_of_words(X):
    from sklearn.feature_extraction.text import CountVectorizer

    print ('Creating bag of words...')
    # Initialize the "CountVectorizer" object, which is scikit-learn's
    # bag of words tool.  

    # In this example features may be single words or two consecutive words
    # (as shown by ngram_range = 1,2)
    vectorizer = CountVectorizer(analyzer = "word",   \
                                 tokenizer = None,    \
                                 preprocessor = None, \
                                 stop_words = None,   \
                                 ngram_range = (1,2), \
                                 max_features = 10000
                                ) 

    # fit_transform() does two functions: First, it fits the model
    # and learns the vocabulary; second, it transforms our training data
    # into feature vectors. The input to fit_transform should be a list of 
    # strings. The output is a sparse array
    train_data_features = vectorizer.fit_transform(X)

    # Convert to a NumPy array for easy of handling
    train_data_features = train_data_features.toarray()

    # tfidf transform
    from sklearn.feature_extraction.text import TfidfTransformer
    tfidf = TfidfTransformer()
    tfidf_features = tfidf.fit_transform(train_data_features).toarray()

    # Get words in the vocabulary
    vocab = vectorizer.get_feature_names()

    return vectorizer, vocab, train_data_features, tfidf_features, tfidf


vectorizer, vocab, train_data_features, tfidf_features, tfidf  = \
    create_bag_of_words(X_train)

def train_logistic_regression(features, label):
    print ("Training the logistic regression model...")
    from sklearn.linear_model import LogisticRegression

    ml_model =  MultiOutputClassifier(LogisticRegression(C = 0.1, solver = 'newton-cg', penalty = 'l2',multi_class = 'multinomial'))


    ml_model.fit(features, label)

    print ('Finished')
    return  ml_model 

ml_model = train_logistic_regression(tfidf_features, y_train)


test_data_features = vectorizer.transform(X_test)
# Convert to numpy array
test_data_features = test_data_features.toarray()


test_data_tfidf_features = tfidf.fit_transform(test_data_features)
# Convert to numpy array
test_data_tfidf_features = test_data_tfidf_features.toarray()


predicted_y = ml_model.predict(test_data_tfidf_features)
correctly_identified_y = predicted_y == y_test
accuracy = np.mean(correctly_identified_y) * 100