musicrecommendation.py

# -*- coding: utf-8 -*-
"""musicRecommendation.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1whhMuFJjtp3jjIVblKMREOraymRMQwa8
"""

#new code
from google.colab import drive
drive.mount('/content/gdrive')

import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix
from sklearn.neighbors import NearestNeighbors
from sklearn.model_selection import train_test_split
#!pip install fuzzywuzzy
#from fuzzywuzzy import fuzz
import tensorflow as tf
from tensorflow import keras
from keras.optimizers import Adam
import matplotlib.pyplot as plt
import random


"""
Importing the dataset. The dataset contains artists data, users-artist interaction data
and other csv files as well containing other information like genres, timestamps etc.
"""
artistsDataDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/artistsData.csv')
userArtistDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/userArtistsData.csv')
userFriendsDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/userFriendsData.csv')
tagDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/tagData.csv')
userTaggedArtistsDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/userTaggedArtistsData.csv')
userTaggedAritstsTimeStampDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/userTaggedArtistsTimeStampData.csv')

"""
Starting data pre-processing. This includes removing duplicate entries and merging
our artist data and user-artist interaction data into a single dataframe.
"""
artistsDataDF.drop_duplicates(['id'], inplace = True)
artistsDataDF.columns = ['artistID', 'name', 'url', 'pictureURL']
artists = pd.merge(userArtistDF, artistsDataDF, on="artistID", how = "left")

# Converting the user and artist IDs to be sequential and starting from 0.
artists.userID = artists.userID.astype('category').cat.codes.values
artists.artistID = artists.artistID.astype('category').cat.codes.values

# min-max normalization of the weights.
max_value = artists["weight"].max()
min_value = artists["weight"].min()

for index, row in artists.iterrows():
    weight = row['weight']
    weight = (weight - min_value) / (max_value - min_value)
    row['weight'] = weight

    artists.iloc[index] = row

"""
Finished data pre-processing. Now we will start splitting our dataset into 
training and testing datasets with an 80-20 split. We will also select our number
of latent factors for matrix factorization to be 50.
"""
train, test = train_test_split(artists, test_size = 0.2) 
n_users, n_artists = len(artists.userID.unique()), len(artists.artistID.unique())
n_latent_factors = 50 #20

# Creating artist input and embedding layers.
artist_input = keras.layers.Input(shape=[1], name = 'artist')
artist_embedding = keras.layers.Embedding(n_artists + 1, n_latent_factors, name='Artist-Embedding')(artist_input)
# Flattening the embedding layer to find the dot product.
artist_vec = keras.layers.Flatten(name='FlattenArtists')(artist_embedding)

# Creating user input and embedding layers.
user_input = keras.layers.Input(shape=[1], name = 'user')
user_embedding = keras.layers.Embedding(n_users +1, n_latent_factors, name='User-Embedding')(user_input)
# Flattening the embedding layer to find the dot product.
user_vec = keras.layers.Flatten(name='FlattenUsers')(user_embedding)

# Dot product layer that will be used to fit and evaluate the model.
product = keras.layers.dot([artist_vec, user_vec], axes = 1, name='DotProduct')

"""
Model 1 - Applying matrix factorization using the embedding layers dot product
to find the expected number of times a user would listen to an artist. Using adam optimizer
"""
print("Model 1 - training and testing:")
model = tf.keras.Model(inputs = [user_input, artist_input], outputs = product)
opt = Adam(lr = 0.001)

model.compile(loss='mean_squared_error', optimizer = opt, metrics = ['mae', 'mse', 'accuracy'])
model.summary()
tf.keras.utils.plot_model(model, to_file='model.png')

# fitting the model to the training data.
history = model.fit([train.userID, train.artistID], train.weight, validation_split= 0.33, epochs=10, verbose=1, batch_size = 64)

plot1 = plt.figure(1)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

# evaluating the model on the test data.
results = model.evaluate((test.userID, test.artistID), test.weight, batch_size=64)

# obtaining the updated embedding layers with the weights.
artist_embedding_learnt = model.get_layer(name='Artist-Embedding').get_weights()[0]
pd.DataFrame(artist_embedding_learnt).describe()
user_embedding_learnt = model.get_layer(name='User-Embedding').get_weights()[0]

"""
def model_v2(user_input, artist_input, user_vec, artist_vec, dotProduct): 

    nn_inp = tf.keras.layers.Concatenate()([user_vec, artist_vec])
    nn_inp= tf.keras.layers.Dense(96,activation='relu')(dotProduct)
    nn_inp= tf.keras.layers.Dropout(0.4)(nn_inp)
    nn_inp= tf.keras.layers.Dense(1,activation='relu')(nn_inp)
    nn_model =keras.models.Model([user_input, artist_input],nn_inp)
    nn_model.summary()

    return nn_model

model_v2 = model_v2(user_input, artist_input, user_vec, artist_vec, product)
model_v2.compile(optimizer=Adam(lr=1e-3),loss= "root_mean_squared_error")

history = model.fit([train.userID, train.artistID], train.weight, epochs=10, verbose=0, batch_size = 64)
pd.Series(history.history['loss']).plot(logy=True)
plt.xlabel("Epoch")
plt.ylabel("Training Error")

results = model.evaluate((test.userID, test.artistID), test.weight, batch_size=64) #1

artist_embedding_learnt = model.get_layer(name='Artist-Embedding').get_weights()[0]
pd.DataFrame(artist_embedding_learnt).describe()
user_embedding_learnt = model.get_layer(name='User-Embedding').get_weights()[0]
"""

def recommend(user_id, number_of_artists=10):
  artists = user_embedding_learnt[user_id]@artist_embedding_learnt.T
  mids = np.argpartition(artists, -number_of_artists)[-number_of_artists:]
  return mids

def map_indeces_to_artist_title(recommendation_ids):
    recommendation = []

    for id in recommendation_ids:
        row = artists[artists['artistID'] == id]
        artist_name = row.iloc[0]['name']
        recommendation.append(artist_name)

    return recommendation

if __name__ == "__main__":
  print("")
  for i in range(10):
    recommendUser = recommend(i)
    mapRecommendation = map_indeces_to_artist_title(recommendUser)
    print("Artist IDs:", recommendUser)
    print("Artists:", mapRecommendation)

#new code
from google.colab import drive
drive.mount('/content/gdrive')

import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix
from sklearn.neighbors import NearestNeighbors
from sklearn.model_selection import train_test_split
#!pip install fuzzywuzzy
#from fuzzywuzzy import fuzz
import tensorflow as tf
from tensorflow import keras
from keras.optimizers import Adam
import matplotlib.pyplot as plt
import random


"""
Importing the dataset. The dataset contains artists data, users-artist interaction data
and other csv files as well containing other information like genres, timestamps etc.
"""
artistsDataDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/artistsData.csv')
userArtistDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/userArtistsData.csv')
userFriendsDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/userFriendsData.csv')
tagDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/tagData.csv')
userTaggedArtistsDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/userTaggedArtistsData.csv')
userTaggedAritstsTimeStampDF = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/userTaggedArtistsTimeStampData.csv')

"""
Starting data pre-processing. This includes removing duplicate entries and merging
our artist data and user-artist interaction data into a single dataframe.
"""
artistsDataDF.drop_duplicates(['id'], inplace = True)
artistsDataDF.columns = ['artistID', 'name', 'url', 'pictureURL']
artists = pd.merge(userArtistDF, artistsDataDF, on="artistID", how = "left")

# Converting the user and artist IDs to be sequential and starting from 0.
artists.userID = artists.userID.astype('category').cat.codes.values
artists.artistID = artists.artistID.astype('category').cat.codes.values

# min-max normalization of the weights.
max_value = artists["weight"].max()
min_value = artists["weight"].min()

for index, row in artists.iterrows():
    weight = row['weight']
    weight = (weight - min_value) / (max_value - min_value)
    row['weight'] = weight

    artists.iloc[index] = row

"""
Finished data pre-processing. Now we will start splitting our dataset into 
training and testing datasets with an 80-20 split. We will also select our number
of latent factors for matrix factorization to be 50.
"""
train, test = train_test_split(artists, test_size = 0.2) 
n_users, n_artists = len(artists.userID.unique()), len(artists.artistID.unique())
n_latent_factors = 50 #20

# Creating artist input and embedding layers.
artist_input = keras.layers.Input(shape=[1], name = 'artist')
artist_embedding = keras.layers.Embedding(n_artists + 1, n_latent_factors, name='Artist-Embedding')(artist_input)
# Flattening the embedding layer to find the dot product.
artist_vec = keras.layers.Flatten(name='FlattenArtists')(artist_embedding)

# Creating user input and embedding layers.
user_input = keras.layers.Input(shape=[1], name = 'user')
user_embedding = keras.layers.Embedding(n_users +1, n_latent_factors, name='User-Embedding')(user_input)
# Flattening the embedding layer to find the dot product.
user_vec = keras.layers.Flatten(name='FlattenUsers')(user_embedding)

# Dot product layer that will be used to fit and evaluate the model.
product = keras.layers.dot([artist_vec, user_vec], axes = 1, name='DotProduct')

"""
Model 1 - Applying matrix factorization using the embedding layers dot product
to find the expected number of times a user would listen to an artist. Using adam optimizer
"""
"""
print("Model 1 - training and testing:")
model = tf.keras.Model(inputs = [user_input, artist_input], outputs = product)
opt = Adam(lr = 0.001)

model.compile(loss='mean_squared_error', optimizer = opt, metrics = ['mae', 'mse', 'accuracy'])
model.summary()
tf.keras.utils.plot_model(model, to_file='model.png')

# fitting the model to the training data.
history = model.fit([train.userID, train.artistID], train.weight, validation_split= 0.33, epochs=10, verbose=1, batch_size = 64)

plot1 = plt.figure(1)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

# evaluating the model on the test data.
results = model.evaluate((test.userID, test.artistID), test.weight, batch_size=64)

# obtaining the updated embedding layers with the weights.
artist_embedding_learnt = model.get_layer(name='Artist-Embedding').get_weights()[0]
pd.DataFrame(artist_embedding_learnt).describe()
user_embedding_learnt = model.get_layer(name='User-Embedding').get_weights()[0]
"""

"""
Model 2 - Collaborative filtering using a small neural network. We concatenate the
user and artist embedding layers and pass that as an input to a neural network. The model that is
returned is then fit and evaluated on train and test data.
"""
# function that produces the neural network model
def model_v2(user_input, artist_input, user_vec, artist_vec, dotProduct): 

    nn_inp = tf.keras.layers.Concatenate()([user_vec, artist_vec])
    nn_inp= tf.keras.layers.Dense(96,activation='relu')(dotProduct)
    nn_inp= tf.keras.layers.Dropout(0.4)(nn_inp)
    nn_inp= tf.keras.layers.Dense(1,activation='relu')(nn_inp)
    nn_model =keras.models.Model([user_input, artist_input],nn_inp)
    nn_model.summary()

    return nn_model

model_v2 = model_v2(user_input, artist_input, user_vec, artist_vec, product)
model_v2.compile(optimizer=Adam(lr=1e-3),loss= "mean_squared_error")
# fitting the model to our training data
history = model_v2.fit([train.userID, train.artistID], train.weight, validation_split = 0.33, epochs=10, verbose=1, batch_size = 64)

plot1 = plt.figure(1)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

results = model_v2.evaluate((test.userID, test.artistID), test.weight, batch_size=64) #1

artist_embedding_learnt = model_v2.get_layer(name='Artist-Embedding').get_weights()[0]
pd.DataFrame(artist_embedding_learnt).describe()
user_embedding_learnt = model_v2.get_layer(name='User-Embedding').get_weights()[0]


def recommend(user_id, number_of_artists=10):
  artists = user_embedding_learnt[user_id]@artist_embedding_learnt.T
  mids = np.argpartition(artists, -number_of_artists)[-number_of_artists:]
  return mids

def map_indeces_to_artist_title(recommendation_ids):
    recommendation = []

    for id in recommendation_ids:
        row = artists[artists['artistID'] == id]
        artist_name = row.iloc[0]['name']
        recommendation.append(artist_name)

    return recommendation

if __name__ == "__main__":
  print("")
  for i in range(10):
    recommendUser = recommend(i)
    mapRecommendation = map_indeces_to_artist_title(recommendUser)
    print("Artist IDs:", recommendUser)
    print("Artists:", mapRecommendation)