
import shap
import matplotlib.pylab as plt
from sklearn.preprocessing import StandardScaler, LabelEncoder , LabelBinarizer
import seaborn as sns
import pandas as pd
import pyodbc
import numpy as np
import pyodbc
import missingno as msno
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import TruncatedSVD

from data_science_helper import helper_dataframe as hd

import string
from sklearn.metrics import silhouette_score
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans
from sklearn.cluster import DBSCAN
from sklearn.cluster import SpectralClustering
from scipy.stats import itemfreq
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.model_selection import train_test_split
import lightgbm as lgb
import pandas as pd

from sklearn import preprocessing, decomposition, svm #, cross_validation


class ShapFeatureSelector( BaseEstimator, TransformerMixin ):
    #Class Constructor 
    def __init__( self,  objective="binary",min_shap=0,test_size=0.25,verbose=1,categorical_columns_name=[] ):
        #self._feature_names = feature_names 
        self._objective = objective
        self._min_shap = min_shap
        self._test_size=test_size
        self._model = None
        self._verbose = verbose
        self._categorical_columns_name = categorical_columns_name
    
    #Return self nothing else to do here    
    def fit( self, X, y = None ):
        
        print("inicio del proceso: feature_sel_shap con  min_shap:"+str(self._min_shap))
        train_cols = X.columns.tolist()

#         _cols = X.columns
#         _num_cols = X._get_numeric_data().columns
#         _categorical_columns_name = list(set(_cols) - set(_num_cols))
        
#         le = preprocessing.LabelEncoder()
#         for c in _categorical_columns_name:
#             col_type = X[c].dtype
#             if col_type == 'object' or col_type.name == 'category':
#                 #print("column : "+str(c))
#                 X[c] = le.fit_transform(X[c])
#                 #X[c] = X[c].astype('category') 
        
        
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=self._test_size, stratify=y, random_state=1)

        
        train_data = lgb.Dataset(X_train, label=y_train.astype(int), feature_name=train_cols)
        test_data = lgb.Dataset(X_test, label=y_test.astype(int), feature_name=train_cols, reference=train_data)
        
        # LGB parameters:
        params = {
                'metric': 'binary_error',
                'learning_rate': 0.01,
                'boosting': 'gbdt', 
                'objective': self._objective,
                'num_leaves': 2000,
                'min_data_in_leaf': 200,
                'max_bin': 200,
                'max_depth': -1,
                'seed': 2018,
                'nthread': 10,
                "lambda_l1": 0.1,            
                }
        
        self._model = lgb.train(params, train_data, 
                    categorical_feature=self._categorical_columns_name,
                    num_boost_round=1000, 
                    valid_sets=(test_data,), 
                    valid_names=('valid',), 
                    verbose_eval=False, 
                    early_stopping_rounds=20)
        
        return self 
    
    #Method that describes what we need this transformer to do
    def transform( self, X, y = None ):
        
        if(self._model is not None):
            explainer = shap.TreeExplainer(self._model)
            shap_values = explainer.shap_values(X)

            if(self._objective=="multiclass"):
                shap_sum = np.abs(shap_values).mean(axis=0).mean(axis=0)
            else:  
                shap_sum = np.abs(shap_values).mean(axis=0).mean(axis=0)

            #print("shap_sum...done")
            importance_df = pd.DataFrame([X.columns.tolist(), shap_sum.tolist()]).T
            importance_df.columns = ['column_name', 'shap_importance']
            importance_df = importance_df.sort_values('shap_importance', ascending=False)

            importance_df['percentage'] = (importance_df['shap_importance']/importance_df['shap_importance'].sum())*100
            importance_df.reset_index(inplace=True,drop=True)
            if(self._verbose==1):
                print(importance_df)
            
            
            zero_features = importance_df[importance_df['shap_importance']<=self._min_shap].column_name.values
            return X.drop(columns = zero_features)     
        else:
            print("No se ha ejecutado el metodo FIT")
            
    def importances_df(self):
        return self._importance_df


class CorrTransformer( BaseEstimator, TransformerMixin ):
    #Class Constructor 
    def __init__( self, umbral  = 0.95 ):
        self.to_drop  = []
        self.umbral = umbral
    
    #Return self nothing else to do here    
    def fit( self, X, y = None ):   
        
        c_c = hd.get_cat_columns(X)
        
        # Create correlation matrix
        corr_matrix = X.drop(c_c,axis=1).corr().abs()
        # Select upper triangle of correlation matrix
        upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))
        # Find index of feature columns with correlation greater than 0.95
        to_drop = [column for column in upper.columns if any(upper[column] > self.umbral)]
        self.to_drop = to_drop
        return self 
    
    #Method that describes what we need this transformer to do
    def transform( self, X, y = None ):
        
        new_df = X.drop(self.to_drop,axis=1,inplace=False)  
        return new_df
        
     
       
    
class CatTransformer( BaseEstimator, TransformerMixin ):
    #Class Constructor 
    def __init__( self, pp  = "le" , cl_list=None, console= False ):
        
        #columns_cat
        self.cat_encoders  = []
        self.encoded_df = []
        self.pp = pp
        self.console = console
        self.cl_list = cl_list
    
    #Return self nothing else to do here    
    def fit( self, X, y = None ):
        if self.cl_list is None:
            self.catcolumns = hd.get_cat_columns(X)
        else:
            self.catcolumns = self.cl_list 

        for name in self.catcolumns:
            
            if name in X.columns:
            
                if(self.pp=="le"):
                    le = LabelEncoder()  
                elif(self.pp=="lb"):
                    le = LabelBinarizer()  
                else:
                    msg = "Error: pp INVALIDO"
                    raise Exception(msg)
                    break;
                if self.console==True:
                    print("NEW Binarios {}".format(name))
                if X[name].isna().any():
                    msg = "ERROR: La columna "+name+" tiene valores MISSING, corregir eso para continuar"
                    raise Exception(msg)
                    #print("ERROR: La columna ",name," tiene valores MISSING, corregir eso para continuar")
                    break
                le.fit(X[name])
                self.cat_encoders.append((name, le))
            else:
                print(f"no se transformara la columna {name} porque no existe en el dataframe" )
        return self 
    
    #Method that describes what we need this transformer to do
    def transform( self, X, y = None ):
        
        #print(self.cat_encoders)
        for name, encs in self.cat_encoders :
            
            df_c = encs.transform(X[name])
            
            prefix = str(name) + "_"

            if(self.pp=="le"):
                self.encoded_df.append(pd.DataFrame(df_c,columns=[name],index=X.index))
            elif(self.pp=="lb"):
                #self.encoded_df.append(pd.DataFrame(df_c,columns=encs.classes_,index=X.index).add_prefix(str(prefix).upper()))
                if(df_c.shape[1]==1):                    
                    if (len(encs.classes_)==2):  
                        classe_1 = encs.classes_[1]+"_pos_"+encs.classes_[0]+"_neg"
                        self.encoded_df.append(pd.DataFrame(df_c,columns=[classe_1],index=X.index).add_prefix(str(prefix).upper()))
                    else:
                        self.encoded_df.append(pd.DataFrame(df_c,columns=[name],index=X.index))                    
                else:                
                    self.encoded_df.append(pd.DataFrame(df_c,columns=encs.classes_,index=X.index).add_prefix(str(prefix).upper()))          
            
            else:
                print("Error: pp INVALIDO")
                break;            
            
        encoded_df_c = pd.concat(self.encoded_df, axis = 1, ignore_index  = False)   
        #print(list(encoded_df_c.columns))
        #print(self.encoded_df.columns)
       
        self.df_num = X.drop(self.catcolumns, axis = 1) 
        #print(self.df_num.shape)
        #print(list(self.df_num.columns))
        new_df = pd.concat([self.df_num, encoded_df_c], axis = 1, ignore_index = False)
        
        #para que las columnas categoricas codificadas con numeros no se confundan con valores numericos, le asignamos tipo category
        if(self.pp=="le"):
            for c in self.catcolumns:
                new_df[c] = new_df[c].astype('category') 
        
        #print(new_df.shape)
        #print(list(y.columns))
        return new_df
        
     
        
        
        
class FeatureSelector( BaseEstimator, TransformerMixin ):
    #Class Constructor 
    def __init__( self, feature_names ):
        self._feature_names = feature_names 
    
    #Return self nothing else to do here    
    def fit( self, X, y = None ):
        return self 
    
    #Method that describes what we need this transformer to do
    def transform( self, X, y = None ):
        return X[ self._feature_names ] 