pods_capstone.py

# -*- coding: utf-8 -*-
"""PODS_Capstone.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1bHWFhK-8iz3PRzluKCIdku8f8rlwM9Yk
"""

import csv
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans, DBSCAN
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from scipy.special import expit
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, silhouette_samples, roc_auc_score
from sklearn.model_selection import train_test_split
seed = 12121616

df = pd.read_csv('spotify52kData.csv')
df.head()

df.describe()

# Checking for any songs with duration of 0
df[df['duration'] == 0]

df[df['popularity'] == 0]

df = df.drop(columns=['artists','album_name','track_name'])
df.head()

df.isna().any()

corr = df[['popularity', 'duration', 'danceability', 'energy', 'loudness', 'speechiness', 'acousticness','instrumentalness',
            'liveness', 'valence','tempo','time_signature']].corr()
corr.style.background_gradient(cmap='coolwarm')

# lets just use a sample of the dataset
df_sample = df.sample(n=20000, random_state=seed)

'''
Q1
Consider the 10 song features duration, danceability, energy, loudness, speechiness,
acousticness, instrumentalness, liveness, valence and tempo. Is any of these features
reasonably distributed normally? If so, which one? [Suggestion: Include a 2x5 figure with
histograms for each feature)
duration, danceability, energy, loudness, speechiness,
acousticness, instrumentalness, liveness, valence and tempo

'''
features = ['duration', 'danceability', 'energy', 'loudness', 'speechiness', 'acousticness','instrumentalness',
            'liveness', 'valence','tempo']

numBins = 21
plt.figure(figsize=(20, 8))
for i, feature in enumerate(features):
  plt.subplot(2, 5, i + 1)
  plt.hist(df_sample[feature],bins=numBins, color='purple', alpha=0.7)
  plt.title(feature)

plt.show()

'''
Q2
Is there a relationship between song length and popularity of a song? If so, if the relationship
positive or negative? [Suggestion: Include a scatterplot]
'''
plt.scatter(df_sample['duration'], df_sample['popularity'], color='purple')
plt.title('Duration vs. Popularity')
plt.xlabel('Duration')
plt.ylabel('Popularity')
plt.show()
corr = df[['duration','popularity']].corr()
corr

'''
Q3
Are explicitly rated songs more popular than songs that are not explicit? [Suggestion: Do a
suitable significance test, be it parametric, non-parametric or permutation]

Null hypothesis: Explicit songs are as popular as non-explicit songs
'''
# determine popularity distribution
plt.hist(df_sample['popularity'],bins=numBins, color='purple', alpha=0.7)
plt.title('Popularity')
plt.show()
# not normally distributed

explicit = df_sample[df_sample['explicit'] == True]['popularity']
not_explicit = df_sample[df_sample['explicit'] == False]['popularity']

plt.hist(explicit,bins=numBins, color='purple', alpha=0.7)
plt.title('Popularity of Explicit Songs')
plt.show()

plt.hist(not_explicit,bins=numBins, color='purple', alpha=0.7)
plt.title('Popularity of Non-explicit Songs')
plt.show()

# Perform the Mann-Whitney U Test
stat, p_value = stats.mannwhitneyu(explicit, not_explicit)
stat, p_value

explicit.mean(), explicit.median()

not_explicit.mean(), not_explicit.median()

df_test = df_sample.sample(n=2000, random_state=seed)

explicit = df_test[df_test['explicit'] == True]['popularity']
not_explicit = df_test[df_test['explicit'] == False]['popularity']
stat, p_value = stats.mannwhitneyu(explicit, not_explicit)
stat, p_value
#

'''
Q4
Are songs in major key more popular than songs in minor key? [Suggestion: Do a suitable
significance test, be it parametric, non-parametric or permutation]

Null hypothesis: Songs in major key are as popular as songs in minor key
'''
# We know popularity distribution is not normal, so use Mann-Whitney again
major = df_sample[df_sample['mode'] == 1]['popularity']
minor = df_sample[df_sample['mode'] == 0]['popularity']

plt.hist(major,bins=numBins, color='purple', alpha=0.7)
plt.title('Popularity of Major key Songs')
plt.show()

plt.hist(minor,bins=numBins, color='purple', alpha=0.7)
plt.title('Popularity of Minor key Songs')
plt.show()

stat, p_value = stats.mannwhitneyu(major, minor)
stat, p_value

major.mean(), major.median()

minor.mean(), minor.median()

'''
Q5
Energy is believed to largely reflect the “loudness” of a song. Can you substantiate (or refute)
that this is the case? [Suggestion: Include a scatterplot]
'''
plt.scatter(df['energy'], df['loudness'])
plt.title('Energy vs. Loudness')
plt.xlabel('Energy')
plt.ylabel('Loudness')
plt.show()

df_sample[['energy','loudness']].corr(method='spearman')

'''
Q6
Which of the 10 individual (single) song features from question 1 predicts popularity best?
How good is this “best” model?
'''
plt.figure(figsize=(20, 10))
for i, feature in enumerate(features):
  regressContainer = np.empty([len(df_sample),5])
  regressContainer[:] = np.NaN
  varCorrs = np.corrcoef(df_sample[feature],df_sample['popularity'])
  for j in range(len(df_sample)):
    regressContainer[j,0] = df_sample[feature][j]
    regressContainer[j,1] = df_sample['popularity'][j]
    regressContainer[j,2] = df_sample[feature][j]*df_sample['popularity'][j]
    regressContainer[j,3] = df_sample[feature][j]**2
    regressContainer[j,4] = df_sample['popularity'][j]**2

  n = len(df_sample)

  mNumerator = n*sum(regressContainer[:,2]) - sum(regressContainer[:,0])*sum(regressContainer[:,1])
  mDenominator = n*sum(regressContainer[:,3]) - (sum(regressContainer[:,0]))**2
  m = mNumerator/mDenominator

  bNumerator = sum(regressContainer[:,1]) - m * sum(regressContainer[:,0])
  bDenominator = n
  b = bNumerator/bDenominator
  rSquared = varCorrs[0,1]**2
  plt.subplot(2, 5, i + 1)
  plt.plot(df_sample[feature],df_sample['popularity'],'o',markersize=3, color='purple')
  plt.xlabel(feature)
  plt.ylabel('Popularity')
  yHat = m*df_sample[feature] + b
  plt.plot(df_sample[feature],yHat,color='orange',linewidth=3)
  plt.title('R^2 = {:.3f}'.format(rSquared))

plt.show()

'''
Q7
Building a model that uses *all* of the song features from question 1, how well can you
predict popularity now? How much (if at all) is this model improved compared to the best
model in question 6). How do you account for this?
'''
X = df_sample[features]
y = df_sample['popularity']

fullModel = LinearRegression().fit(X,y)
rSqrFull = fullModel.score(X,y)
print(rSqrFull)

'''
Q8

'''
plt.figure(figsize=(30, 10))
data = df_sample[features]
zScoredData = stats.zscore(data).to_numpy().reshape(len(data), 10)
pca = PCA().fit(zScoredData)
eigVals = pca.explained_variance_
loadings = pca.components_*-1
rotatedData = pca.fit_transform(zScoredData)
varExplained = eigVals/sum(eigVals)*100
for ii in range(len(varExplained)):
    print(varExplained[ii].round(3))
x = np.linspace(1,10,10)
plt.subplot(2, 5, i + 1)
plt.bar(x, eigVals, color='gray')
plt.plot([0,20],[1,1],color='orange') # Orange Kaiser criterion line
plt.xlabel('Principal component')
plt.ylabel('Eigenvalue')

plt.show()

whichPrincipalComponent = 1 # Select and look at one factor at a time, in Python indexing
y = loadings[whichPrincipalComponent,:]
plt.bar(x,y)
def addlabels(x,y):
    for i in range(len(x)):
        y[i] = round(y[i], 3)
        plt.text(i,y[i],y[i])
addlabels(x,y)
plt.xlabel('Song Attribute')
plt.ylabel('Loading')
plt.show()
# features = ['duration', 'danceability', 'energy', 'loudness', 'speechiness', 'acousticness','instrumentalness',
#            'liveness', 'valence','tempo']
# loudness, danceability, liveness

'''
Q9
Can you predict whether a song is in major or minor key from valence? If so, how good is this
prediction? If not, is there a better predictor? [Suggestion: It might be nice to show the logistic
regression once you are done building the model]

'''
# major key descriptives
numMajor = len(df_sample[df_sample['mode'] == 1])
avgMajor = np.mean(df_sample[df_sample['mode'] == 1]['valence'])
stdMajor = np.std(df_sample[df_sample['mode'] == 1]['valence'])
# minor key descriptives
numMinor = len(df_sample[df_sample['mode'] == 0])
avgMinor = np.mean(df_sample[df_sample['mode'] == 0]['valence'])
stdMinor = np.std(df_sample[df_sample['mode'] == 0]['valence'])

print(numMajor, avgMajor, stdMajor)
print(numMinor, avgMinor, stdMinor)

# plotting valence and major/minor
plt.scatter(df_sample['valence'],df_sample['mode'],color='black')
plt.xlabel('Valence')
plt.xlim([0,1])
plt.ylabel('Major or Minor')
plt.yticks(np.array([0,1]))
plt.show()

X = df_sample['valence'].to_numpy().reshape(len(df_sample),1)
y = df_sample['mode']

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=seed)

model = LogisticRegression().fit(X_train,y_train)

x1 = np.linspace(0,1,50)
y1 = x1 * model.coef_ + model.intercept_
sigmoid = expit(y1)

plt.plot(x1,sigmoid.ravel(),color='red',linewidth=3) # the ravel function returns a flattened array
plt.scatter(df_sample['valence'],df_sample['mode'],color='black')
plt.hlines(0.5,0,1,colors='gray',linestyles='dotted')
plt.xlabel('Valence')
plt.xlim([0,1])
plt.ylabel('Major or Minor')
plt.yticks(np.array([0,1]))
plt.show()

major_valence = df_sample[df_sample['mode'] == 1]['loudness']
plt.hist(major_valence,bins=numBins, color='purple', alpha=0.7)

minor_valence = df_sample[df_sample['mode'] == 0]['loudness']
plt.hist(minor_valence,bins=numBins, color='purple', alpha=0.7)

valence = 0
probKey = sigmoid[0,np.abs(valence).argmin()]
print('Probability of Major',probKey.round(3))

y_pred_p = model.predict_proba(X_test)[:, 1]
y_pred = model.predict(X_test)

auc = roc_auc_score(y_test, y_pred_p)
auc

'''
Q10
Which is a better predictor of whether a song is classical music – duration or the principal
components you extracted in question 8? [Suggestion: You might have to convert the
qualitative genre label to a binary numerical label (classical or not)]
'''

classical =
not_classical =
# I would also do a logistic regression here using the three variables I found through PCA- liveness, danceability, and loudness
# im sorry i ran out of time. i really wish i couldve finished this.

'''
Extra Credit:
Tell us something interesting about this dataset that is not trivial and not already part of
an answer (implied or explicitly) to these enumerated questions [Suggestion: Do something with the
number of beats per measure, something with the key, or something with the song or album titles]
'''