Análisis del Sistema Colaborativo :
"Stack Overflow"

from bs4 import BeautifulSoup
                  import pandas as pd

                  df=pd.read_csv("datos.csv", quotechar='"')
                  #Archivo con los tags más populares
                  df_tags=pd.read_csv("top_tags.csv", quotechar='"')
                  #Se elimina registros con valores nulos
                  df.dropna() 
                  df['post_creation_date']=pd.to_datetime(df['post_creation_date'])
                  df['users_creation_date']=pd.to_datetime(df['users_creation_date'])
                  

df.head()
df.describe()

#Se considera las features que se utilizarán en el clasificador. 
df2 = pd.DataFrame(data=df, index=df.index, columns=['class',
                            'post_comment_count',
                            'post_favorite_count', 'post_score',
                            'post_view_count',
                            'users_reputation','users_up_votes','users_down_votes',
                            'score_prev_acceptans' ,'score_prev_ans',
                            'score_prev_comment','score_prev_question',
                            'score_prev_favquestion','user_numpost','user_numpost_answered', 'post_answer_count']
                  )
df2.head()

from nltk.corpus import stopwords


#Edad de creacion de la cuenta en relacion al post en dias.
df2['age_user'] = (df['post_creation_date'] - df['users_creation_date']).fillna(0).astype('timedelta64[D]')
df2['title_length'] = df['post_title'].apply(lambda x: len(x))
df2['num_block_code'] = 0
df2["num_i_sentences"]=0
df2["num_wh_words"]=0
df2["num_y_sentences"]=0
df2["tags_popularity"]=0
df2["num_tags"]=0

whwords=['what','how', 'which', 'when', 'why', 'where']
for index, row in df.iterrows():
    sbody=row["post_body"]
    soup = BeautifulSoup(sbody, "html5lib")
    sentences =  soup.find_all(name="p")
    urls =  soup.find_all('a', href=True)
    #Questions words
    count_wh=0
    #Oraciones que tienen el pronombre I
    count_is=0
    #Oraciones que tienen el pronombre You
    count_ys = 0
    palabras=[]
    filtered_words=[]
    for sentence in sentences:
        try:
            palabras=sentence.contents[0].split()
        except:
            palabras=str(sentence.contents).split() 
        if(len(palabras)==0):
            children = soup.find_all("li", { "class" : "expandable" }, recursive=False)
            for child in children:
                palabras.extend(child.getText().split())
        filtered_words.extend([word for word in palabras if word not in stopwords.words('english')])
        count_is=count_is+len([x for x in palabras if x == "I"])
        count_is=count_is+len([x for x in palabras if len(x.split("I'"))>1])    
        count_ys=count_ys+len([x for x in palabras if x == "You"])
        count_ys=count_ys+len([x for x in palabras if len(x.split("You'"))>1])
        for word in whwords:
            count_wh=count_wh+len([x for x in palabras if x == word])
    df2.loc[index, "num_i_sentences"] = count_is
    df2.loc[index, "num_wh_words"] = count_wh
    df2.loc[index, "num_y_sentences"] = count_ys    
    df2.loc[index, "num_words"] = len(filtered_words)
    df2.loc[index, "num_url"]=len(urls)

df2.head()

#Caracteristicas del codigo y tags
import string

for index, row in df.iterrows():
    body=row["post_body"]
    tags_column=row["post_tags"]
    tags=tags_column.split("|")
    if(body.find("error")>=0):        
        df2.loc[index, "post_error"] = 1
    else:
        df2.loc[index, "post_error"] = 0
    soup = BeautifulSoup(body, "html5lib")
    precode = soup.find_all("pre")
    df2.loc[index, "num_block_code"]=len(precode)
    content=""
    countError = 0
    for codeline in precode:
        contentPre = codeline.contents
        for contentCode in contentPre:
            try:
                content=content+contentCode.contents[0]
            except :
                try:
                    content = content + str(contentCode)
                except:
                    print(contentCode)
    wordCodeCount =len(content)
    df2.loc[index,"code_length"]=wordCodeCount    
    counttag=len(tags)
    pop_tag=0
    #numero de tags y popularidad del tag
    for tag in tags:
        if tag in df_tags['tag_name'].values:
            pop_tag+=1;
    df2.loc[index, "num_tags"] = counttag
    df2.loc[index, "tags_popularity"] = pop_tag
df2.head()

#class 0: preguntas sin respuesta
#class 1: preguntas con respuesta

class0= df2.loc[df2['class'] ==0,:]
class1= df2.loc[df2['class'] ==1,:]

# Se almacena el archivo resultado del preprocesamiento para el posterior análisis.
df2.to_csv("features3.csv", sep=',', encoding='utf-8')
df2.describe()

class0.describe()

class1.describe()

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline

dataTag = df2[["tags_popularity", "num_tags"]]
totalTags = len(dataTag['num_tags'])

ar = np.array(dataTag['num_tags'])

g1 = ar[np.where(ar == 1)]
g2 = ar[np.where(ar == 2)]
g3 = ar[np.where(ar == 3)]
g4 = ar[np.where(ar == 4)]
g5 = ar[np.where(ar == 5)]

labels = '1 Tag', '2 Tags', '3 Tags', '4 Tags','5 Tags'
sizes = [len(g1) , len(g2), len(g3), len(g4) ,len(g5)]
explode = (0.1, 0, 0, 0, 0) 

fig1, ax1 = plt.subplots()
ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',shadow=True, startangle=90)
ax1.axis('equal') 
ax1.set_title(" Cantidad de ''TAGS'' registrados en el Post ")
plt.show()

dataUsers = df2[['users_reputation','age_user','users_up_votes','users_down_votes']]

bins = np.linspace(-20,1000,10)
nums = np.vstack([dataUsers['age_user'],dataUsers['users_reputation']]).T
plt.hist(nums, bins, alpha=0.7, label=['Antiguedad Usuario', 'Reputación Usuario'])
plt.legend(loc='upper right')
plt.xlabel("Usuarios")
plt.ylabel("Frecuencia")
plt.show()

n_groups = 1
upVotes = np.array(dataUsers['users_up_votes'])
downVotes = np.array(dataUsers['users_down_votes'])

avgUp = np.average(upVotes)
avgDown = np.average(downVotes)

means_up = (avgUp)
means_down = (avgDown)
 
# create plot
fig, ax = plt.subplots()
index = np.arange(n_groups)
bar_width = 0.35
opacity = 0.8
 
rects1 = plt.bar(index, means_up, bar_width,
                 alpha=opacity,
                 color='b',
                 label='User Up Votes')
 
rects2 = plt.bar(index + bar_width, means_down, bar_width,
                 alpha=opacity,
                 color='g',
                 label='User Down Votes')
 
plt.ylabel('Votes')
plt.title('User Votes')
plt.xticks(index + bar_width, ())
plt.legend()
 
plt.tight_layout()
plt.show()

dataPosts = df2[['title_length','num_block_code','num_i_sentences','num_wh_words','num_y_sentences','code_length']]

res = dataPosts.describe().as_matrix()
promedios =[]
tipoPost = []

mayorPromedio = []
menorPromedio = []
for i in range(len(dataPosts.columns)):
    promedios.append(res[1][i])

    
def titleLength(a):
    for i in np.nditer(a):
        
        if( float(i) > promedios[0]):
            mayorPromedio.append(i)
        else:             
            menorPromedio.append(i)
            
    labels = 'Post Title >'+str(promedios[0]), 'Post Title <= '+str(promedios[0]) 
    sizes = [len(mayorPromedio), len(menorPromedio)]
    explode = (0, 0.1)
    fig1, ax1 = plt.subplots()
    ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',shadow=True, startangle=90)
    ax1.set_title(" Cantidad de Carácteres Promedio de los Títulos de un Post ")
    ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.

    return plt.show()
                
titleLength(dataPosts['title_length'])

x = np.linspace(dataPosts['num_block_code'].values.min(),dataPosts['num_block_code'].values.max(),num=100)
x1 = np.linspace(dataPosts['code_length'].values.min(),dataPosts['code_length'].values.max(),num=100)

fig, ax = plt.subplots()

ax.plot(x, dataPosts['num_block_code'].head(100))
ax.set_ylabel('Frecuencia')
ax.set_xlabel('Bloques de Código')
ax.set_title(" Bloques de Código por Post ")

plt.show()


with plt.style.context('fivethirtyeight'):
    plt.plot(x, dataPosts['num_block_code'].head(100))
    plt.plot(x1, dataPosts['code_length'].head(100))
    
    # Number of accent colors in the color scheme
    plt.title(' Tamanio de Código por Post')
    plt.xlabel('Tam. Código')
    plt.ylabel('Frecuencia')

plt.show()

from bs4 import BeautifulSoup
import pandas as pd

df=pd.read_csv("res-1.csv", usecols=['ans_scoreclass','ans_score','ans_body','ans_comment_count',
    'ans_creation_date','post_comment_count','post_creation_date','post_answer_count',
    'post_view_count','post_favorite_count','post_score','user_numans', 'user_numans_accepted'], quotechar='"')
df.dropna() 
df['post_creation_date']=pd.to_datetime(df['post_creation_date'])
df['ans_creation_date']=pd.to_datetime(df['ans_creation_date'])

df.head()
df.describe()

df3 = pd.DataFrame(data=df, index=df.index, columns=['ans_scoreclass','ans_comment_count'
    ,'post_comment_count','post_answer_count','post_view_count',
    'post_favorite_count','post_score','user_numans','user_numans_accepted'])
df3.head()

from nltk.corpus import stopwords


#Edad de creacion de la cuenta en relacion al post en dias.
df3['time_answer'] = (df['ans_creation_date'] - df['post_creation_date']).fillna(0).astype('timedelta64[D]')
df3['num_block_code'] = 0
df3["num_i_sentences"]=0
df3["num_wh_words"]=0
df3["num_y_sentences"]=0

whwords=['what','how', 'which', 'when', 'why', 'where']
for index, row in df.iterrows():
    sbody=row["ans_body"]
    soup = BeautifulSoup(sbody, "html5lib")
    sentences =  soup.find_all(name="p")
    urls =  soup.find_all('a', href=True)
    #Questions words
    count_wh=0
    #Oraciones que tienen el pronombre I
    count_is=0
    #Oraciones que tienen el pronombre You
    count_ys = 0
    palabras=[]
    filtered_words=[]
    for sentence in sentences:
        try:
            palabras=sentence.contents[0].split()
        except:
            palabras=str(sentence.contents).split() 
        if(len(palabras)==0):
            children = soup.find_all("li", { "class" : "expandable" }, recursive=False)
            for child in children:
                palabras.extend(child.getText().split())
        filtered_words.extend([word for word in palabras if word not in stopwords.words('english')])
        count_is=count_is+len([x for x in palabras if x == "I"])
        count_is=count_is+len([x for x in palabras if len(x.split("I'"))>1])    
        count_ys=count_ys+len([x for x in palabras if x == "You"])
        count_ys=count_ys+len([x for x in palabras if len(x.split("You'"))>1])
        for word in whwords:
            count_wh=count_wh+len([x for x in palabras if x == word])
    df3.loc[index, "num_i_sentences"] = count_is
    df3.loc[index, "num_wh_words"] = count_wh
    df3.loc[index, "num_y_sentences"] = count_ys    
    df3.loc[index, "num_words"] = len(filtered_words)
    df3.loc[index, "num_url"]=len(urls)

df3.head()

#Caracteristicas del codigo 
import string

for index, row in df.iterrows():
    body=row["ans_body"]
    soup = BeautifulSoup(body, "html5lib")
    precode = soup.find_all("pre")
    df3.loc[index, "num_block_code"]=len(precode)
    content=""
    for codeline in precode:
        contentPre = codeline.contents
        for contentCode in contentPre:
            try:
                content=content+contentCode.contents[0]
            except :
                try:
                    content = content + str(contentCode)
                except:
                    print(contentCode)
    wordCodeCount =len(content)
    df3.loc[index,"code_length"]=wordCodeCount    
df3.head()

df3.to_csv("features4.csv", sep=',', encoding='utf-8')

import pandas as pd
import numpy as np
df2 = pd.read_csv("features3.csv", quotechar='"',
               usecols=['class','post_comment_count',
                'post_favorite_count','post_score',
                'post_view_count','users_reputation','users_up_votes',
                'users_down_votes','score_prev_acceptans',
                'score_prev_ans','score_prev_comment',
                'score_prev_question','score_prev_favquestion',
                'age_user','title_length','num_block_code','num_i_sentences',
                'num_wh_words','num_y_sentences','tags_popularity','num_tags','num_words','code_length', 'num_url','user_numpost','user_numpost_answered','post_error']
                   )
df2.head()

num_features = ['post_comment_count',
                'post_favorite_count','post_score',
                'post_view_count','users_reputation','users_up_votes',
                'users_down_votes','score_prev_acceptans',
                'score_prev_ans','score_prev_comment',
                'score_prev_question','score_prev_favquestion',
                'age_user','title_length','num_block_code','num_i_sentences',
                'num_wh_words','num_y_sentences','tags_popularity','num_tags','num_words','code_length', 'num_url','user_numpost','user_numpost_answered']

df_statistics=pd.DataFrame(index=num_features)
for index, row in df_statistics.iterrows():
    df_statistics.loc[index,"min"]=df2[index].min()
    df_statistics.loc[index,"max"] = df2[index].max()
    df_statistics.loc[index,"mean"] = df2[index].mean()
    df_statistics.loc[index,"std"] = df2[index].std()
    df_statistics.loc[index,"var"] = df2[index].var()
df_statistics

df2= df2.loc[df2['num_words'] >0,:]
df2= df2.loc[df2['age_user'] >=0, :]
df2.describe()

import matplotlib.pyplot as plt
plt.hist(df2['post_view_count'], bins=100, color="blue")
plt.xlim(xmin=0, xmax=df2['post_view_count'].max())
plt.title("Post View")
plt.ylabel("Frequencia")
plt.xlabel("post_view_count")
plt.show()

plt.hist(df2['users_reputation'], bins=100, color="blue")
plt.xlim(xmin=0, xmax=df2['users_reputation'].max())
plt.title("Users reputation")
plt.ylabel("Frequencia")
plt.xlabel("users_reputation")
plt.show()

plt.hist(df2['users_up_votes'], bins=100, color="blue")
plt.xlim(xmin=0, xmax=df2['users_up_votes'].max())
plt.title("Users up-votes")
plt.ylabel("Frequencia")
plt.xlabel("users_up_votes")
plt.show()

plt.hist(df2['score_prev_acceptans'], bins=100, color="blue")
plt.xlim(xmin=0, xmax=df2['score_prev_acceptans'].max())
plt.title("Score de respuestas aceptadas")
plt.ylabel("Frequencia")
plt.xlabel("score_prev_acceptans")
plt.show()

plt.hist(df2['score_prev_ans'], bins=100, color="blue")
plt.xlim(xmin=0, xmax=df2['score_prev_ans'].max())
plt.title("Score de respuestas no aceptadas")
plt.ylabel("Frequencia")
plt.xlabel("score_prev_ans")
plt.show()

plt.hist(df2['score_prev_ans'], bins=100, color="blue")
plt.xlim(xmin=0, xmax=df2['score_prev_ans'].max())
plt.title("Score de respuestas no aceptadas")
plt.ylabel("Frequencia")
plt.xlabel("score_prev_ans")
plt.show()

import seaborn as sns
df_rel=pd.DataFrame(data=df2, columns=['users_reputation', 'users_up_votes',
                'users_down_votes', 'score_prev_acceptans',
                'score_prev_ans', 'score_prev_comment',
                'score_prev_question', 'score_prev_favquestion',
                'age_user','user_numpost','user_numpost_answered'], index=df2.index)

f, ax = plt.subplots(figsize=(10, 8))
corr = df_rel.corr()
sns.heatmap(corr, mask=np.zeros_like(corr, dtype=np.bool), cmap=sns.diverging_palette(220, 10, as_cmap=True),
            square=True, ax=ax)
plt.show()

df2= df2.loc[df2['users_reputation'] <10000, :]

plt.hist(df2['users_reputation'], bins=100, color="blue")
plt.xlim(xmin=0, xmax=df2['users_reputation'].max())
plt.title("Users reputation")
plt.ylabel("Frequencia")
plt.xlabel("users_reputation")
plt.show()

df2.describe()

df3 = pd.DataFrame(data=df2, index=df2.index, columns=['class','post_comment_count',
                'post_favorite_count','post_score',
                'post_view_count','users_reputation','users_up_votes',
                'users_down_votes','score_prev_question','score_prev_favquestion',
                'age_user','title_length','num_block_code','num_i_sentences',
                'num_wh_words','num_y_sentences','tags_popularity','num_tags',
                'num_words','code_length', 'num_url','user_numpost','user_numpost_answered'])
df3.head()

num_features=['post_comment_count', 'post_favorite_count','post_score',
                'post_view_count','users_reputation','users_up_votes',
                'users_down_votes','score_prev_question','score_prev_favquestion',
                'age_user','title_length','num_block_code','num_i_sentences',
                'num_wh_words','num_y_sentences','tags_popularity','num_tags',
                'num_words','code_length', 'num_url','user_numpost','user_numpost_answered']

for each in num_features:
    mean, std = df3[each].mean(), df3[each].std()
    df3.loc[:, each] = (df3[each] - mean)/std
df3.describe()

from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn import svm
from sklearn.model_selection import cross_val_predict
from sklearn.metrics import classification_report

X = df3.iloc[:,1:]
y = df3.iloc[:,0]

clf=svm.SVC(kernel="linear",random_state=10)

print("Clasificador SVM")
print()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=3000,random_state=10)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))

import pandas as pd
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.dummy import DummyClassifier

c1 = ("Decision Tree", DecisionTreeClassifier())
c2 = ("Gaussian NB", GaussianNB())
# Para definir el numero de vecinos para KNN se aplica la raiz cuadrada de n 
c3 = ("KNeighbors", KNeighborsClassifier(n_neighbors=100))
c4 =  ("SVC",svm.SVC(kernel="linear",random_state=10))
c5 = ("Dummy: stratified", DummyClassifier(strategy="stratified", random_state=10))

clasificadores=[c1, c2, c3, c4, c5]

summary=pd.DataFrame(index=['precision','recall', 'f1', 'accuracy'], columns=['Decision Tree','Gaussian NB','KNeighbors','SVC',
                                                                              'Dummy: stratified'])

for name, clf in clasificadores:
    print(name)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=5000,random_state=10)
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    print(classification_report(y_test, y_pred))
    summary.loc['precision',name]=np.mean(metrics.precision_score(y_test, y_pred,average='weighted', labels=[0,1]))
    summary.loc['recall', name] = np.mean(metrics.recall_score(y_test, y_pred,average='weighted', labels=[0,1]))
    summary.loc['f1', name] = np.mean(metrics.f1_score(y_test, y_pred,average='weighted', labels=[0,1]))
    summary.loc['accuracy', name]=np.mean(metrics.accuracy_score(y_test, y_pred))

summary

clf=DecisionTreeClassifier()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=5000,random_state=10)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)

import itertools

from sklearn.metrics import confusion_matrix

class_names=[0,1]
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# Compute confusion matrix

cnf_matrix = confusion_matrix(y_test, y_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=class_names,
                      title='Confusion matrix, without normalization')
print("DecisionTreeClassifier")
plt.show()

from sklearn.ensemble import ExtraTreesClassifier

forest = ExtraTreesClassifier(n_estimators=250,
                              random_state=0)

forest.fit(X, y)
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],
             axis=0)
indices = np.argsort(importances)

plt.figure()
plt.title("Feature importances")
plt.barh(range(X.shape[1]), importances[indices],
       color="r", xerr=std[indices], align="center")

plt.yticks(range(X.shape[1]), indices)
plt.ylim([-1, X.shape[1]])
plt.show()

list(df3)

df_wpost=pd.DataFrame(data=df3, columns=['class','title_length',
 'num_block_code',
 'num_i_sentences',
 'num_wh_words',
 'num_tags',
 'code_length','users_reputation',
 'users_up_votes',
 'age_user'], index=df2.index)

X = df_wpost.iloc[:,1:]
y = df_wpost.iloc[:,0]
clf=DecisionTreeClassifier()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=5000,random_state=10)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))

df0= df2.loc[df3['class'] ==0, :]
df0.describe()
plt.hist(df0['num_block_code'], bins=10, color="blue")
plt.xlim(xmin=0, xmax=df0['num_block_code'].max())
plt.ylabel("Frequencia")
plt.xlabel("Block code")


df1= df2.loc[df3['class'] ==1, :]
plt.hist(df1['num_block_code'], bins=10, color="red")
plt.xlim(xmin=0, xmax=df1['num_block_code'].max())
plt.ylabel("Frequencia")
plt.xlabel("Block code")
plt.show()