In [1]:
import sqlite3
import pandas as pd
import datetime
import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import matplotlib.pyplot as plt
from datetime import datetime
from nltk.corpus import stopwords
import string
import numpy as np
import statsmodels.api as sm
from dateutil.parser import parse
from statsmodels.tsa.arima_model import ARIMA
from statsmodels.tsa.stattools import adfuller
import seaborn as sns
import sklearn.preprocessing as skp
import sklearn.decomposition as skd
import sklearn.cluster as skc
import scipy.spatial.distance as spd
import sklearn.metrics as skm
import sklearn.cross_validation as skcv
import sklearn.pipeline as skpipe
import sklearn.feature_extraction.text as skft
import sklearn.naive_bayes as sknb
import sklearn.metrics as skmetrics
import wordcloud
import statsmodels.graphics as smg
First, we query chat.db, create a dataframe of its contents, send all text messages through nltk's sentiment intensity analyzer (sia), and assign all values return by sia to new columns in the dataframe.¶
In [2]:
# # This notebook cell is intended to be run once.
# # The structure that follows is designed to pull resulting data from csv files
# # rather than running time- and resource-consuming operations that need not be repeated.
# # Establish a connection to the chat.db database.
# conn = sqlite3.connect('/Users/johnglennvoorhess/Library/Messages/chat.db')
# c = conn.cursor()
# # Store and execute a SQL query to grab all text message data from chat.db.
# cmd = 'SELECT datetime(date + strftime(\'%s\',\'2001-01-01\'), \'unixepoch\') as date_utc, ROWID, text, handle_id, is_from_me FROM message;'
# c.execute(cmd)
# # Store the query result in a dataframe.
# df_all = pd.DataFrame(c.fetchall(), columns=['time', 'id', 'text', 'sender', 'is_from_me'])
# # Create an instance of the nltk sentiment analyzer.
# sia = SentimentIntensityAnalyzer()
# # Instantiate dictionaries to store sentiment values.
# comp_dict = {}
# neu_dict = {}
# pos_dict = {}
# neg_dict = {}
# # Send all message text through the sentiment analyzer.
# for i in range(len(df_all)):
# try:
# ss = sia.polarity_scores(df_all.loc[i]['text'])
# comp_dict[i] = ss['compound']
# pos_dict[i] = ss['pos']
# neg_dict[i] = ss['neg']
# neu_dict[i] = ss['neu']
# except:
# comp_dict[i] = 0
# pos_dict[i] = 0
# neg_dict[i] = 0
# neu_dict[i] = 0
# # Convert the dictionaries to Series and add them to the dataframe.
# df_all['compound_polarity_score'] = pd.Series(comp_dict)
# df_all['positivity_score'] = pd.Series(pos_dict)
# df_all['negativity_score'] = pd.Series(neg_dict)
# df_all['neutrality_score'] = pd.Series(neu_dict)
# # Set the dataframe index to the 'time' column.
# df_all.set_index('time')
# # Save the dataframe to a csv file.
# df_all.to_csv('df_all.csv', encoding='utf-8')
In [3]:
sia = SentimentIntensityAnalyzer()
In [4]:
print(sia.polarity_scores('This is the worst movie.'))
In [5]:
print(sia.polarity_scores('This is really the worst movie.'))
Read the csv and make a dataframe with sentiment scores.¶
In [6]:
# Read the csv that we have saved.
df_all = pd.read_csv('df_all.csv',parse_dates=True, index_col=0)
# Convert the time column to Pandas datetime.
df_all['time'] = pd.to_datetime(df_all['time'])
# Set the index of the dataframe to the time column for timeseries analysis
df_all.set_index('time')
# Fill all NaN values with zero
df_all.fillna(0)
# Store a list of stopwords in a variable.
# Call it 'swords' because Wu-Tang forever.
swords = stopwords.words('english')
Process and tokenize the text for further analysis.¶
In [7]:
# Create a dictionary to store processed text
content_dict = {}
# Iterate through text column in dataframe
for index,value in df_all.text.iteritems():
try:
# Make the text lowercase and tokenize it.
words = [w.lower() for w in nltk.tokenize.wordpunct_tokenize(value)]
# Eliminate the punctuation
words = [w for w in words if w not in string.punctuation]
# Take out the stopwords
words = [w for w in words if w not in swords]
# Send the processed text to content_dict
content_dict[index] = words
# send an empty list to content_dict if there's a bad value in the text.
except TypeError:
content_dict[index] = []
# Turn conten_dict into a series
s_processed = pd.Series(content_dict)
# Assign that series to a new column in the dataframe
# representing processed text.
df_all['p_text'] = s_processed
In [8]:
df_all
Out[8]:
To find the senders with whom I most frequently communicate, we get value counts for the top 20 most frequnt communicators and plot the top ten to get a sense of scale.¶
In [9]:
df_all.sender.value_counts()[:10].plot(kind='barh')
df_all.sender.value_counts()[:20]
Out[9]:
Contact IDs 7 and 47 have significant quantities of text messages. ID 35 is an order of magnitude greater, but expected since that is my partner of four years, Emmalee.¶
Now, we create dataframes for each of the top three contacts.¶
In [10]:
df_7 = df_all[df_all['sender']==7].copy()
df_47 = df_all[df_all['sender']==47].copy()
df_emmalee = df_all[df_all['sender']==35].copy()
As it turns out, contact 7 and 47 are the same person. So we'll concatenate the dataframes and go back to our list to get the next most frequent message sender.¶
(I printed the dataframes and examined the text but chose not to present them here since it was a subjective decision.)
In [11]:
sender_747 = [df_7,df_47]
df_747 = pd.concat(sender_747)
In [12]:
df_14 = df_all[df_all['sender']==14].copy()
For each of the dataframes we will forward fill missing sentiment data to smooth the plot. The idea being that the sentiment of one text will be close to that of the next.¶
At the same time, we will also set the index of each dataframe to be the datetime of the message.
In [13]:
cols = ['compound_polarity_score','positivity_score','negativity_score','neutrality_score']
dfs = [df_emmalee,df_14,df_747]
for df in dfs:
df['time'] = pd.to_datetime(df['time'])
df.set_index('time', inplace=True)
df[cols] = df[cols].replace({0:np.nan})
df.fillna(method='ffill', inplace=True)
df_emmalee.fillna(0, inplace=True)
df_14.fillna(0, inplace=True)
df_747.fillna(0, inplace=True)
In [14]:
weekly_rolling_emmalee = df_emmalee['compound_polarity_score'].rolling(window=7, center=True)
data_smooth = pd.DataFrame({'input': df_emmalee['compound_polarity_score'], 'weekly rolling_mean': weekly_rolling_emmalee.mean()})
ax = data_smooth.plot()
ax.lines[0].set_alpha(0.3)
In [15]:
print(type(df_emmalee['compound_polarity_score']))
# remove duplicate timestamped rows (cant have duplicate indexes)
df_emmalee = df_emmalee.loc[~df_emmalee.index.duplicated(keep='first')]
df_14 = df_14.loc[~df_14.index.duplicated(keep='first')]
df_747 = df_747.loc[~df_747.index.duplicated(keep='first')]
# new dataframe with datetime index and all compound polarity scores as columns
df_cps = pd.concat([df_emmalee['compound_polarity_score'], df_14['compound_polarity_score'], df_747['compound_polarity_score']], axis=1)
df_pos = pd.concat([df_emmalee['positivity_score'], df_14['positivity_score'], df_747['positivity_score']], axis=1)
df_neg = pd.concat([df_emmalee['negativity_score'], df_14['negativity_score'], df_747['negativity_score']], axis=1)
In [16]:
headers = ['emmalee', '14', '747']
df_cps.columns = headers
df_pos.columns = headers
df_neg.columns = headers
In [17]:
# Group all values by week
df4 = df_cps['2014'].groupby(df_cps['2014'].index.week).mean()
df5 = df_cps['2015'].groupby(df_cps['2015'].index.week).mean()
df6 = df_cps['2016'].groupby(df_cps['2016'].index.week).mean()
df7 = df_cps['2017'].groupby(df_cps['2017'].index.week).mean()
df8 = df_cps['2018'].groupby(df_cps['2018'].index.week).mean()
In [18]:
# Concatenate all grouped frames
df4=pd.concat([df4,df5,df6,df7,df8]).reset_index(drop=True)
In [19]:
# Check for correlation between these three series
df4.fillna(0,inplace=True)
np.corrcoef(df4['emmalee'], df4['14'])
Out[19]:
In [20]:
np.corrcoef(df4['emmalee'], df4['747'])
Out[20]:
In [21]:
np.corrcoef(df4['14'], df4['747'])
Out[21]:
Contact 14's average sentiment does not seem to correlate to 747 or Emmalee. 747 and Emmalee display minimal correlation in average sentiment by week.¶
Let's try time series decomposition.¶
First, Emmalee.
In [22]:
df_cps.fillna(0,inplace=True)
# Emmalee compound polarity score time series decomposition.
ts_emmalee_cps = df_cps.loc['2017':'2018']['emmalee']
decompose_result = sm.tsa.seasonal_decompose(ts_emmalee_cps, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
In [23]:
df_neg.fillna(0,inplace=True)
# Emmalee negativity score time series decomposition.
ts_emmalee_neg = df_neg.loc['2017':'2018']['emmalee']
decompose_result = sm.tsa.seasonal_decompose(ts_emmalee_neg, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
In [24]:
df_pos.fillna(0,inplace=True)
# Emmalee positivity score time series decomposition.
ts_emmalee_pos = df_pos.loc['2018':'2018']['emmalee']
decompose_result = sm.tsa.seasonal_decompose(ts_emmalee_pos, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
In [25]:
df_cps.fillna(0,inplace=True)
# 14 compound polarity score time series decomposition.
ts_14_cps = df_cps.loc['2014':'2018']['14']
decompose_result = sm.tsa.seasonal_decompose(ts_14_cps, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
In [26]:
df_pos.fillna(0,inplace=True)
# 14 positivity score time series decomposition.
ts_14_pos = df_pos.loc['2014':'2018']['14']
decompose_result = sm.tsa.seasonal_decompose(ts_14_pos, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
In [27]:
df_pos.fillna(0,inplace=True)
# 14 negativity score time series decomposition.
ts_14_neg = df_pos.loc['2014':'2018']['14']
decompose_result = sm.tsa.seasonal_decompose(ts_14_neg, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
In [28]:
df_cps.fillna(0,inplace=True)
# 747 compound polarity score time series decomposition.
ts_747_cps = df_cps.loc['2014':'2018']['747']
decompose_result = sm.tsa.seasonal_decompose(ts_747_cps, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
In [29]:
df_neg.fillna(0,inplace=True)
# 747 negativity score time series decomposition.
ts_747_neg = df_neg.loc['2014':'2018']['747']
decompose_result = sm.tsa.seasonal_decompose(ts_747_neg, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
In [30]:
df_pos.fillna(0,inplace=True)
# 747 positivity score time series decomposition.
ts_747_pos = df_pos.loc['2014':'2018']['747']
decompose_result = sm.tsa.seasonal_decompose(ts_747_pos, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
Prediction - Can we predict sentiment moving forward?¶
In [31]:
def plotTS(timeseries):
timeseries.plot(label = 'original series ')
ts_rolling = timeseries.rolling(window=12)
rollmean = ts_rolling.mean().plot(label = 'rolling_mean')
plt.legend()
plt.show()
rollstd = ts_rolling.std().plot(label = 'rolling standard deviation')
plt.legend()
def testDF(timeseries):
print('Results of Dickey-Fuller Test:')
dftest = adfuller(timeseries, autolag='AIC')
dfoutput = pd.Series(dftest[0:4], index=['Test Statistic','p-value','#Lags Used','Number of Observations Used'])
for key,value in dftest[4].items():
dfoutput['Critical Value (%s)'%key] = value
print(dfoutput)
def plotRSS(ts1,ts2):
plt.plot(ts1)
plt.plot(ts2, color='red')
plt.title('RSS: %.4f'% sum((ts2-ts1)**2))
In [32]:
testDF(df_pos['emmalee'])
P-Value is super small and Test Statistic is less than 1% critical value, so this time series is already stationary.¶
Does that mean that we will not be able to do any forecasting?¶
A guy on Quora said to go for it - https://www.quora.com/If-I-have-a-stationary-time-series-with-no-trend-or-seasonality-does-the-ARIMA-model-still-give-me-a-sensible-result¶
In [33]:
# min_error = float('inf')
# best_i = 0
# best_j = 0
# for i in range (10):
# for j in range(10):
# model = ARIMA(df_pos['emmalee'], order=(i, 1, j))
# try:
# results_ARIMA = model.fit()
# except:
# continue
# predictions_ARIMA_TA_log_diff = pd.Series(results_ARIMA.fittedvalues, copy=True)
# predictions_ARIMA_TA_log_diff_cumsum = predictions_ARIMA_TA_log_diff.cumsum()
# predictions_ARIMA_TA_log_first_term = pd.Series(df_pos['emmalee'].iloc[0], index=df_pos['emmalee'].index)
# predictions_ARIMA_TA_log = predictions_ARIMA_TA_log_first_term.add(predictions_ARIMA_TA_log_diff_cumsum,fill_value=0)
# predictions_ARIMA_TA = np.exp(predictions_ARIMA_TA_log)
# MAE = sum(abs(predictions_ARIMA_TA-df_pos['emmalee']))/len(df_pos['emmalee'])
# if MAE < min_error:
# min_error = MAE
# best_i = i
# best_j = j
# print (best_i, best_j, min_error)
Since the previous attempt at fitting the best possible model never finished and I believe that it is the result of having too many observations, we'll try the same operation on a smaller time frame- just 2018.¶
In [34]:
# emmalee positivity score 2018 time series decomposition.
ts_em_2018_pos = df_pos.loc['2018']['emmalee']
decompose_result = sm.tsa.seasonal_decompose(ts_em_2018_pos, freq=52)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 10.5)
In [35]:
testDF(df_pos.loc['2018']['emmalee'])
Again, it's already stationary.¶
In [36]:
em_2018_log = np.log(df_pos.loc['2018']['emmalee']);
plotTS(em_2018_log);
In [37]:
# min_error = float('inf')
# best_i = 0
# best_j = 0
# for i in range (10):
# for j in range(10):
# model = ARIMA(df_pos.loc['2018']['emmalee'], order=(i, 1, j))
# try:
# results_ARIMA = model.fit()
# except:
# continue
# predictions_ARIMA_TA_log_diff = pd.Series(results_ARIMA.fittedvalues, copy=True)
# predictions_ARIMA_TA_log_diff_cumsum = predictions_ARIMA_TA_log_diff.cumsum()
# predictions_ARIMA_TA_log_first_term = pd.Series(df_pos.loc['2018']['emmalee'].iloc[0], index=df_pos.loc['2018']['emmalee'].index)
# predictions_ARIMA_TA_log = predictions_ARIMA_TA_log_first_term.add(predictions_ARIMA_TA_log_diff_cumsum,fill_value=0)
# predictions_ARIMA_TA = np.exp(predictions_ARIMA_TA_log)
# MAE = sum(abs(predictions_ARIMA_TA-df_pos.loc['2018']['emmalee']))/len(df_pos.loc['2018']['emmalee'])
# if MAE < min_error:
# min_error = MAE
# best_i = i
# best_j = j
# print (best_i, best_j, min_error)
Seems like this is too many observations. Try a smaller subset?¶
In [38]:
df_pos.loc['2018']['emmalee'].count()
Out[38]:
In [39]:
em_2018_log = np.log(df_pos.loc['2018']['emmalee']);
plotTS(em_2018_log);
In [40]:
# em_2018_log.dropna(inplace=True)
# testDF(em_2018_log)
I thought that the issue causing the 'SVD did not converge' error was NaN values in em_2018_log, but after dropping nulls nothing changed. Nevertheless, we move on.¶
In [41]:
min_error = float('inf')
best_i = 0
best_j = 0
for i in range (10):
for j in range(10):
model = ARIMA(em_2018_log, order=(i, 1, j))
try:
results_ARIMA = model.fit()
except:
continue
predictions_ARIMA_TA_log_diff = pd.Series(results_ARIMA.fittedvalues, copy=True)
predictions_ARIMA_TA_log_diff_cumsum = predictions_ARIMA_TA_log_diff.cumsum()
predictions_ARIMA_TA_log_first_term = pd.Series(em_2018_log.iloc[0], index=em_2018_log.index)
predictions_ARIMA_TA_log = predictions_ARIMA_TA_log_first_term.add(predictions_ARIMA_TA_log_diff_cumsum,fill_value=0)
predictions_ARIMA_TA = np.exp(predictions_ARIMA_TA_log)
MAE = sum(abs(predictions_ARIMA_TA-df_pos.loc['2018']['emmalee']))/len(df_pos.loc['2018']['emmalee'])
if MAE < min_error:
min_error = MAE
best_i = i
best_j = j
print (best_i, best_j, min_error)
In [42]:
em_log_diff = em_2018_log - em_2018_log.shift(1)
em_log_diff.dropna(inplace=True)
In [43]:
model = ARIMA(em_2018_log, order=(0, 0, 0))
results_ARIMA = model.fit()
predictions_ARIMA_TA_log_diff = pd.Series(results_ARIMA.fittedvalues, copy=True)
predictions_ARIMA_TA_log_diff_cumsum = predictions_ARIMA_TA_log_diff.cumsum()
predictions_ARIMA_TA_log_first_term = pd.Series(em_2018_log.iloc[0], index=em_2018_log.index)
predictions_ARIMA_TA_log = predictions_ARIMA_TA_log_first_term.add(predictions_ARIMA_TA_log_diff_cumsum,fill_value=0)
predictions_ARIMA = np.exp(predictions_ARIMA_TA_log)
plt.plot(df_pos.loc['2018']['emmalee'])
plt.plot(predictions_ARIMA)
plt.title('Mean Abs error: '+str(sum(abs(predictions_ARIMA-df_pos.loc['2018']['emmalee']))/len(df_pos.loc['2018']['emmalee'])))
Out[43]:
Those predictions are so flat it's not funny. This is a deadend.¶
I grouped by month, year and ran the same operation below.¶
In their 2018 paper, In an Absolute State: Elevated Use of Absolutist Words Is a Marker Specific to Anxiety, Depression, and Suicidal Ideation, Mohammed Al-Mosaiwi and Tom Johnstone propose a list of 'absolutist' words present in greater proportions of communication by those who are depressed versus those who do not identify as being depressed.¶
Can we characterize senders by the texts that they send?¶
In [44]:
abs_words = ['absolutely','all','always','complete','completely','constant','constantly','definitely','entire','ever','every','everyone','everything','full','must','never','nothing','totally','whole']
In [45]:
abs_count = {}
for index,value in df_all['text'].iteritems():
text_split = str(value).split()
abs_count[index] = 0
for word in text_split:
if word in abs_words:
abs_count[index] += 1
abs_count
Out[45]:
In [46]:
# Turn abs_count into a series and insert into the dataframe
df_all['abs_count'] = pd.Series(abs_count)
In [47]:
# is there correlation between absolute words and negativity?
np.corrcoef(df_all['abs_count'], df_all['negativity_score'])
Out[47]:
The abs_count and negativity_score do not seem to be correlated, but maybe we can create a new value by which we can characterize these texts. neg_abs will be a weighted average of abs_proportion and negativity_score.¶
In [48]:
word_count = {}
for index,value in df_all['text'].iteritems():
text_split = str(value).split()
word_count[index] = len(text_split)
df_all['word_count'] = pd.Series(word_count)
df_all['abs_proportion'] = df_all['abs_count']/df_all['word_count']
df_all['neg_abs'] = (.2*df_all['negativity_score'])+(.8*df_all['abs_proportion'])
In [49]:
df_all
Out[49]:
In [50]:
#group by sender where is_from_me=0 and average other values
df_quant = pd.concat([df_all['sender'],df_all['is_from_me'],df_all['compound_polarity_score'], df_all['positivity_score'], df_all['negativity_score'], df_all['neutrality_score'], df_all['abs_count'], df_all['word_count'], df_all['abs_proportion'], df_all['neg_abs']], axis=1)
df_sender_grouped = df_quant[df_quant["is_from_me"] == 0].groupby(['sender']).mean().copy()
df_sender_grouped.reset_index()
Out[50]:
What are the most common words in positive text messages?¶
I have defined positive text messages as those with a positivity score of greater than or equal to .5.
In [51]:
df_pos = df_all[df_all["positivity_score"] >= .5].copy()
positive_word_dict = {}
for index,value in df_pos['text'].iteritems():
for word in value.split():
word = word.lower()
if word in swords:
pass
if word in positive_word_dict.keys():
positive_word_dict[word] += 1
else:
positive_word_dict[word] = 1
positive_word_dict_sorted = sorted(positive_word_dict.items(), key=lambda x: x[1], reverse=True)
top_10_positive = []
for item in positive_word_dict_sorted[:10]:
top_10_positive.append(item[0])
print(top_10_positive)
What are the most common words in negative text messages?¶
I have defined negative text messages as those with a negativity score of greater than or equal to .5.
In [52]:
df_neg = df_all[df_all["negativity_score"] >= .5].copy()
negative_word_dict = {}
for index,value in df_neg['text'].iteritems():
for word in value.split():
word = word.lower()
if word in swords:
pass
if word in negative_word_dict.keys():
negative_word_dict[word] += 1
else:
negative_word_dict[word] = 1
negative_word_dict_sorted = sorted(negative_word_dict.items(), key=lambda x: x[1], reverse=True)
top_10_negative = []
for item in negative_word_dict_sorted[:10]:
top_10_negative.append(item[0])
print(top_10_negative)
What are the most common words in text messages with absolute words?¶
In [53]:
abs_word_dict = {}
for index,value in df_all['text'].iteritems():
value = str(value)
for word in value.split():
word = word.lower()
word = str(word)
if word in swords:
pass
if word in abs_words:
if word in abs_word_dict.keys():
abs_word_dict[word] += 1
else:
abs_word_dict[word] = 1
abs_word_dict_sorted = sorted(abs_word_dict.items(), key=lambda x: x[1], reverse=True)
top_10_abs = []
for item in abs_word_dict_sorted[:10]:
top_10_abs.append(item[0])
print(top_10_abs)
Is there correlation between any unexpected pairs of values in the dataframe?¶
In [54]:
sns.pairplot(df_sender_grouped.loc[:, 'compound_polarity_score':'neg_abs'],size=3);
In [55]:
sns.clustermap(df_sender_grouped.loc[:, 'compound_polarity_score':'neg_abs'].corr(),cmap=plt.cm.OrRd)
Out[55]:
There aren't any surprising correlations.¶
PCA Analysis¶
In [56]:
df_norm = df_sender_grouped.copy()
df_norm.drop(['is_from_me'], axis=1)
Out[56]:
In [57]:
df_norm['compound_polarity_score'] = skp.scale(df_norm['compound_polarity_score'].astype(np.float))
df_norm['positivity_score'] = skp.scale(df_norm['positivity_score'].astype(np.float))
df_norm['negativity_score'] = skp.scale(df_norm['negativity_score'].astype(np.float))
df_norm['neutrality_score'] = skp.scale(df_norm['neutrality_score'].astype(np.float))
df_norm['abs_count'] = skp.scale(df_norm['abs_count'].astype(np.float))
df_norm['word_count'] = skp.scale(df_norm['word_count'].astype(np.float))
df_norm['abs_proportion'] = skp.scale(df_norm['abs_proportion'].astype(np.float))
df_norm['neg_abs'] = skp.scale(df_norm['neg_abs'].astype(np.float))
In [58]:
pca_model = skd.PCA().fit(df_norm)
In [59]:
pca_model.components_.shape
Out[59]:
In [60]:
pca_model.explained_variance_
Out[60]:
Now, we generate a scree plot to determine the number of principle components that we will use.¶
In [61]:
plt.plot(range(1,10),pca_model.explained_variance_,'b-o')
Out[61]:
Based on the scree plot, we will need 3 principle components because it is accepted to drop principle components whose explained variance is less than 1. However, the "elbow point" is at principle component 6.¶
In [62]:
X = pca_model.transform(df_norm) #Applies dimensionality reduction
plt.figure(figsize=(20,20))
plt.scatter(X[:,0], X[:,1])
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.ylim(-4, 4)
#Add variable unit vector projections
V = pca_model.transform(np.identity(X.shape[1]))
for i, v in enumerate(V):
plt.annotate(df_norm.columns[i],
xy=(0,0), xytext=v[:2]*6,
fontsize=13, color='orange',
arrowprops=dict(
arrowstyle='<-', linewidth=2, color='orange'))
We create a three-factor model.¶
In [63]:
# Create a three-factor model
fa_model = skd.FactorAnalysis(n_components=3).fit(df_norm)
# Show the loadings
df_loadings = pd.DataFrame(fa_model.components_[:3,:].T,
index=df_norm.columns,
columns=['Factor1', 'Factor2', 'Factor3'])
df_loadings
Out[63]:
Then we can visualize the factor loadings.¶
In [64]:
sns.clustermap(df_loadings)
Out[64]:
Cluster analysis with PCA.¶
In [65]:
kmeans_model = skc.KMeans(2).fit(df_norm)
pca_model = skd.PCA().fit(df_norm)
X = pca_model.transform(df_norm)
df_norm['cluster_label'] = kmeans_model.labels_
df_norm['PC1'] = X[:,0]
df_norm['PC2'] = X[:,1]
df_norm
Out[65]:
In [66]:
K = range(1,11)
kmeans_models = [skc.KMeans(k).fit(df_norm) for k in K]
centroids = [m.cluster_centers_ for m in kmeans_models]
D_k = [spd.cdist(df_norm,cent,'euclidean') for cent in centroids]
dist = [np.min(D,axis=1) for D in D_k]
dist_sq = [d**2 for d in dist]
dist_sum = [sum(d) for d in dist_sq]
plt.plot(K, dist_sum, '-o')
plt.xlabel('Number of clusters');
plt.ylabel('Average within-cluster sum of squares');
plt.title('Elbow for K-Means clustering');
In [67]:
K = range(2,11)
KM = [skc.KMeans(n_clusters=k).fit(df_norm) for k in K]
silh_scores = [skm.silhouette_score(df_norm,km.labels_) for km in KM]
kIdx = np.argmax(silh_scores)
kIdx + 2
plt.plot(K, silh_scores, 'b*-')
plt.plot(K[kIdx], silh_scores[kIdx], marker='o', markersize=12,
markeredgewidth=2, markeredgecolor='r', markerfacecolor='None')
plt.xlim(1, plt.xlim()[1])
plt.xlabel('Number of clusters');
plt.ylabel('Silhouette Coefficient');
plt.title('Silhouette Scores for k-means clustering');
8 clusters? Ok, then.¶
In [162]:
kmeans_model = skc.KMeans(8).fit(df_norm)
pca_model = skd.PCA().fit(df_norm)
X = pca_model.transform(df_norm)
df_norm['cluster_label'] = kmeans_model.labels_
df_norm['PC1'] = X[:,0]
df_norm['PC2'] = X[:,1]
In [163]:
sender = [x[0] for x in df_norm]
f = sns.lmplot(x='PC1', y='PC2', data=df_norm,
hue='cluster_label',
fit_reg=False)
plt.title('Trait k-means (k=10) Displayed with PCA',
fontsize=15);
# Annotate each individual contact number
# for i, name in enumerate(sender):
# plt.annotate(name, (X[i,0]+0.1, X[i,1]-0.1),
# fontsize=10)
Who are the points at the top right?¶
In [70]:
df_norm[df_norm['cluster_label']==3]
Out[70]:
In [71]:
df_all[df_all['sender']==23].text
Out[71]:
In [72]:
df_all[df_all['sender']==98].text
Out[72]:
Those two are Emmalee's friend, Jessica, and my aunt Christie- both sent only a couple very positive messages.¶
Who is at the bottom left?¶
In [73]:
df_norm[df_norm['cluster_label']==0]
Out[73]:
In [74]:
df_all[df_all['sender']==133].text
Out[74]:
Those are null or empty sets of messages.¶
Which sender has the highest average negativity_score?¶
In [75]:
df_all[df_all['sender']==df_sender_grouped['negativity_score'].argmax()].text
Out[75]:
It's my old building manager from my last apartment. She wasn't a very happy person.¶
Which sender has the highest average positivity_score?¶
In [76]:
df_all[df_all['sender']==df_sender_grouped['positivity_score'].argmax()].text
Out[76]:
In [77]:
df_sender_grouped.sort_values(by=['positivity_score'])
Out[77]:
In [78]:
df_all[df_all['sender']==94].text
Out[78]:
The top two were people interested in buying my old van, so I went with the next one. Weird, it's the building manager from my previous apartment!¶
Now we'll try to make a classifier to classify text in terms of a sender.¶
First, I make a dataframe without my own text
In [79]:
df_not_me = df_all[df_all['is_from_me']==0].copy()
df_not_me.dropna(0, inplace=True)
In [80]:
df_all_train, df_all_test = skcv.train_test_split(df_not_me, test_size=0.3, random_state=0)
pipeline = skpipe.Pipeline(
steps = [('vect', skft.CountVectorizer(max_df=0.001, min_df=.0001)),
('tfidf', skft.TfidfTransformer()),
('clf', sknb.MultinomialNB())])
pipeline.fit(df_all_train.text, df_all_train.sender)
Out[80]:
In [81]:
test_predicted = pipeline.predict(df_all_test.text)
print(skmetrics.accuracy_score(df_all_test.sender, test_predicted))
I can't seem to come up with an accuracy level much higher than .70.¶
In [82]:
print('Clasification Result:', pipeline.predict(['The unfortunate reality of living above the']))
Also, I think that the imbalance in the size of the corpus for sender 35 (Emmalee) is skewing the prediction results.¶
I'll try classifying whether the text is or isn't from me.¶
In [83]:
# max_score = 0
# best_i = 0
# best_j = 0
# for i in range(100):
# for j in range(100):
# try:
# df_all_train, df_all_test = skcv.train_test_split(df_all, test_size=0.3, random_state=0)
# pipeline = skpipe.Pipeline(
# steps = [('vect', skft.CountVectorizer(max_df=i/10, min_df= j/10, stop_words='english')),
# ('tfidf', skft.TfidfTransformer()),
# ('clf', sknb.MultinomialNB())])
# pipeline.fit(df_all_train.text, df_all_train.is_from_me)
# print(i,j)
# test_predicted = pipeline.predict(df_all_test.text)
# x = skmetrics.accuracy_score(df_all_test.is_from_me, test_predicted)
# if x>max_score:
# max_score=x
# best_i = i
# best_j = j
# except:
# print('nope')
# print(max_score, best_i, best_j)
In [84]:
test_predicted = pipeline.predict(df_all_test.text)
print(skmetrics.accuracy_score(df_all_test.is_from_me, test_predicted))
group sentiment by month and plot frequency distribution of my own words topic modeling use topic modeling as features to predict labels
Group Emmalee's sentiment by month and year and examine for trends.¶
In [85]:
df_emmalee
# df_em_month = df_emmalee.groupby(df_emmalee.index.month).mean()
df_em_month = df_emmalee.groupby(pd.Grouper(freq='M')).mean()
df_em_week = df_emmalee.groupby(pd.Grouper(freq='W')).mean()
In [86]:
df_em_week['compound_polarity_score'].plot()
Out[86]:
In [87]:
df_em_month['compound_polarity_score'].plot()
Out[87]:
Definitely looks like a trend there.¶
In [88]:
df_em_month['positivity_score'].plot()
Out[88]:
In [89]:
df_em_month['negativity_score'].plot()
Out[89]:
Time series decomposition of Em's compound polarity score¶
In [90]:
plotTS(df_em_month['compound_polarity_score'])
In [91]:
ts_em_comp = df_em_month['compound_polarity_score']
# decompose_result = sm.tsa.seasonal_decompose(ts_em_comp, freq=52)
decompose_result = sm.tsa.seasonal_decompose(ts_em_comp)
fig = decompose_result.plot()
fig.set_size_inches(18.5, 9)
In [92]:
fig, ax = plt.subplots()
smg.tsaplots.plot_acf(df_emmalee['compound_polarity_score'],ax=ax, alpha = None, use_vlines=True, lw = .5)
plt.title("Autocorrelation for df_em['compound_polarity_score']")
Out[92]:
In [93]:
fig, ax = plt.subplots()
smg.tsaplots.plot_acf(df_em_week['compound_polarity_score'],ax=ax, alpha = None, use_vlines=True, lw = .5)
plt.title("Autocorrelation for df_em_week['compound_polarity_score']")
Out[93]:
In [94]:
fig, ax = plt.subplots()
smg.tsaplots.plot_acf(df_em_month['compound_polarity_score'],ax=ax, alpha = None, use_vlines=True, lw = .5)
plt.title("Autocorrelation for df_em_month['compound_polarity_score']")
Out[94]:
In [95]:
testDF(df_em_month['compound_polarity_score'])
ADF shows that the time series is not stationary.¶
Now we will take steps to make it stationary
In [96]:
# Take the log.
em_log = np.log(df_em_month['compound_polarity_score']);
plotTS(em_log);
Definitley still not stationary.¶
We can take more steps toward stationarity.
In [97]:
# Differencing
em_log_diff = em_log - em_log.shift(1)
em_log_diff.dropna(inplace=True)
plotTS(em_log_diff)
Looks closer. Let's test with ADF.¶
In [98]:
testDF(em_log_diff)
In [99]:
min_error = float('inf')
best_i = 0
best_j = 0
for i in range (10):
for j in range(10):
model = ARIMA(em_log, order=(i, 1, j))
try:
results_ARIMA = model.fit()
except:
continue
predictions_ARIMA_TA_log_diff = pd.Series(results_ARIMA.fittedvalues, copy=True)
predictions_ARIMA_TA_log_diff_cumsum = predictions_ARIMA_TA_log_diff.cumsum()
predictions_ARIMA_TA_log_first_term = pd.Series(em_log.iloc[0], index=em_log.index)
predictions_ARIMA_TA_log = predictions_ARIMA_TA_log_first_term.add(predictions_ARIMA_TA_log_diff_cumsum,fill_value=0)
predictions_ARIMA_TA = np.exp(predictions_ARIMA_TA_log)
MAE = sum(abs(predictions_ARIMA_TA-df_em_month['compound_polarity_score']))/len(df_em_month['compound_polarity_score'])
if MAE < min_error:
min_error = MAE
best_i = i
best_j = j
print (best_i, best_j, min_error)
In [100]:
model = ARIMA(em_log, order=(best_i,1,best_j))
results_ARIMA = model.fit()
predictions_ARIMA_TA_log_diff = pd.Series(results_ARIMA.fittedvalues, copy=True)
predictions_ARIMA_TA_log_diff_cumsum = predictions_ARIMA_TA_log_diff.cumsum()
predictions_ARIMA_TA_log_first_term = pd.Series(em_log.iloc[0], index=em_log.index)
predictions_ARIMA_TA_log = predictions_ARIMA_TA_log_first_term.add(predictions_ARIMA_TA_log_diff_cumsum,fill_value=0)
predictions_ARIMA_TA = np.exp(predictions_ARIMA_TA_log)
plt.plot(df_em_month['compound_polarity_score'])
plt.plot(predictions_ARIMA_TA)
Out[100]:
That model doesn't look all that close. Let's try another method.¶
In [101]:
decomposition = sm.tsa.seasonal_decompose(em_log)
residual = decomposition.resid
em_log_residual = residual
em_log_residual.dropna(inplace=True)
testDF(em_log_residual)
In [102]:
min_error = float('inf')
best_i = 0
best_j = 0
for i in range (10):
for j in range(10):
model = ARIMA(em_log_residual, order=(i, 1, j))
try:
results_ARIMA = model.fit()
except:
continue
predictions_ARIMA_TA_log_diff = pd.Series(results_ARIMA.fittedvalues, copy=True)
predictions_ARIMA_TA_log_diff_cumsum = predictions_ARIMA_TA_log_diff.cumsum()
predictions_ARIMA_TA_log_first_term = pd.Series(em_log_residual.iloc[0], index=em_log_residual.index)
predictions_ARIMA_TA_log = predictions_ARIMA_TA_log_first_term.add(predictions_ARIMA_TA_log_diff_cumsum,fill_value=0)
predictions_ARIMA_TA = np.exp(predictions_ARIMA_TA_log)
MAE = sum(abs(predictions_ARIMA_TA-df_em_month['compound_polarity_score']))/len(df_em_month['compound_polarity_score'])
if MAE < min_error:
min_error = MAE
best_i = i
best_j = j
print (best_i, best_j, min_error)
That is not any better¶
Plotting my own compound polarity score grouped by month.¶
In [103]:
df_me = df_all[df_all['is_from_me']==1].copy()
df_me.set_index('time',inplace=True)
In [104]:
df_me_month = df_me.groupby(pd.Grouper(freq='M')).mean()
In [105]:
df_me_month['compound_polarity_score'].plot()
Out[105]:
In [106]:
plotTS(df_me_month['compound_polarity_score'])
Creating a plot of the words that I use most frequently.¶
In [107]:
df_me = df_all[df_all['is_from_me']==1].copy()
In [108]:
df_me['text'] = df_me['text'].astype(str)
all_john_text = '\n'.join(df_me.text)
tokens_all = nltk.tokenize.wordpunct_tokenize(all_john_text)
fd = nltk.probability.FreqDist(tokens_all)
fd.most_common(20)
Out[108]:
Too many stopwords. I'll take those out.¶
In [109]:
all_john_text = []
for index, value in df_me['text'].iteritems():
for word in value.split():
if word not in swords:
all_john_text.append(word)
john_string = ' '.join(all_john_text)
tokens_all = nltk.tokenize.wordpunct_tokenize(john_string)
fd = nltk.probability.FreqDist(tokens_all)
fd.most_common(20)
Out[109]:
In [110]:
plt.figure(figsize=(20,8))
fd.plot(50)
In [111]:
wc = wordcloud.WordCloud(max_words=1000,stopwords=swords,
margin=10,random_state=2).generate(john_string)
fig,ax = plt.subplots(figsize=(20,20))
ax.imshow(wc) #Display an image on the axes.
Out[111]:
Now we can plot Emmalee's most frequent words.¶
In [112]:
df_em_text = df_emmalee[df_emmalee['is_from_me']==0].copy()
all_em_text = []
for index, value in df_em_text['text'].iteritems():
for word in value.split():
if word not in swords:
all_em_text.append(word)
em_string = ' '.join(all_em_text)
tokens_all = nltk.tokenize.wordpunct_tokenize(em_string)
fd = nltk.probability.FreqDist(tokens_all)
fd.most_common(20)
Out[112]:
In [113]:
plt.figure(figsize=(20,8))
fd.plot(50)
In [114]:
wc = wordcloud.WordCloud(max_words=1000,stopwords=swords,
margin=10,random_state=2).generate(em_string)
fig,ax = plt.subplots(figsize=(20,20))
ax.imshow(wc) #Display an image on the axes.
Out[114]:
Now we will attempt to resample the dataframe to build a model from a set of equal-sized samples.¶
In [115]:
# Create dataframe of messages not from me
df_all_2 = df_all[df_all['is_from_me']==0].copy()
df_not_me.dropna(0, inplace=True)
In [116]:
senders = df_all_2['sender'].groupby(df_all_2['sender']).mean()
senders = senders.values
Here I am looping through the list of senders and sampling a single row 100 times to create a dataframe of 100 texts from each sender.¶
In [121]:
frames = []
for i in senders:
for j in range(1000):
temp_df = df_all_2[df_all_2['sender']==i]
dfi = temp_df.sample(n=1)
frames.append(dfi)
In [122]:
sampled_df = pd.concat(frames)
sampled_df.dropna(0, inplace=True)
In [123]:
df_all_train, df_all_test = skcv.train_test_split(sampled_df, test_size=0.3, random_state=0)
pipeline = skpipe.Pipeline(
steps = [('vect', skft.CountVectorizer(max_df=0.001, min_df=.0001)),
('tfidf', skft.TfidfTransformer()),
('clf', sknb.MultinomialNB())])
pipeline.fit(df_all_train.text, df_all_train.sender)
test_predicted = pipeline.predict(df_all_test.text)
print(skmetrics.accuracy_score(df_all_test.sender, test_predicted))
.28 is pretty bad but probably better than the previous classifier with 70% accuracy since it was predicting the same sender every time.¶
In [124]:
max_score = 0
best_i = 0
best_j = 0
for i in range(100):
for j in range(100):
try:
df_all_train, df_all_test = skcv.train_test_split(sampled_df, test_size=0.3, random_state=0)
pipeline = skpipe.Pipeline(
steps = [('vect', skft.CountVectorizer(max_df=i/10, min_df= j/10, stop_words='english')),
('tfidf', skft.TfidfTransformer()),
('clf', sknb.MultinomialNB())])
pipeline.fit(df_all_train.text, df_all_train.is_from_me)
print(i,j)
test_predicted = pipeline.predict(df_all_test.text)
x = skmetrics.accuracy_score(df_all_test.is_from_me, test_predicted)
if x>max_score:
max_score=x
best_i = i
best_j = j
except:
pass
print(max_score, best_i, best_j)
When run, the previous code shows a max score of 1.0 with max_df = .1 and min_df = 0.¶
In [152]:
df_all_train, df_all_test = skcv.train_test_split(sampled_df, test_size=0.3, random_state=0)
pipeline = skpipe.Pipeline(
steps = [('vect', skft.CountVectorizer(max_df=0.124, min_df=0)),
('tfidf', skft.TfidfTransformer()),
('clf', sknb.MultinomialNB())])
pipeline.fit(df_all_train.text, df_all_train.sender)
test_predicted = pipeline.predict(df_all_test.text)
print(skmetrics.accuracy_score(df_all_test.sender, test_predicted))
85% is pretty accurate.¶
In [160]:
print('Clasification Result:', pipeline.predict(['I like data']))
In [161]:
df_120 = df_all[df_all['sender']==120].copy()
df_120
Out[161]: