Chapter 147 · Advanced & Applied Topics
Recommendation Systems · Challenge Solutions
Worked solutions: build the matrix, two baselines, item-item similarity, matrix factorization, and a top-N recommendation list.
In [1]:
import numpy as np, pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics.pairwise import cosine_similarity
plt.rcParams.update({"figure.dpi":110,"axes.grid":True,"grid.alpha":0.25,"font.size":11})
FU, BL, GR = "#a21caf", "#2563eb", "#16a34a"
BASE = "https://raw.githubusercontent.com/johnfisher-ai/Statistics-Data-Science-AI-Visual-Book/main/data/"
fn = "recommendation-systems--movie-ratings.xlsx"
try:
df = pd.read_excel("../../data/" + fn); movies = pd.read_excel("../../data/" + fn, sheet_name="Movies")
except FileNotFoundError:
df = pd.read_excel(BASE + fn); movies = pd.read_excel(BASE + fn, sheet_name="Movies")
genre = dict(zip(movies.movie_id, movies.genre))
print(df.shape); print(df.head())
def rmse(pred, actual): return float(np.sqrt(np.mean((np.asarray(pred, float) - np.asarray(actual, float))**2)))
rng = np.random.default_rng(0); mask = rng.random(len(df)) < 0.8
tr, te = df[mask], df[~mask]; mu = tr.rating.mean()
(4383, 3) user_id movie_id rating 0 1 52 4 1 1 51 4 2 1 44 4 3 1 28 3 4 1 14 4
Challenge 1 · Build the matrix¶
Pivot and report sparsity.
In [2]:
R = df.pivot_table(index="user_id", columns="movie_id", values="rating")
print(f"shape {R.shape}, {R.notna().mean().mean():.1%} filled ({R.notna().sum().sum()} of {R.size} cells)")
shape (300, 60), 24.3% filled (4383 of 18000 cells)
Challenge 2 · Two baselines¶
Global mean vs user+item bias.
In [3]:
rg = rmse([mu]*len(te), te.rating)
um = tr.groupby("user_id").rating.mean(); mm = tr.groupby("movie_id").rating.mean()
rb = rmse([(um.get(r.user_id,mu)+mm.get(r.movie_id,mu))/2 for r in te.itertuples()], te.rating)
print(f"global mean RMSE = {rg:.3f} | user+item bias RMSE = {rb:.3f}")
global mean RMSE = 0.952 | user+item bias RMSE = 0.845
Challenge 3 · Item-item similarity¶
Cosine similarity between movie columns; check genre agreement.
In [4]:
R = df.pivot_table(index="user_id", columns="movie_id", values="rating").fillna(0)
mids = list(R.columns); sim = cosine_similarity(R.T.values)
genre = dict(zip(movies.movie_id, movies.genre)); t = mids[0]
top = [i for i in np.argsort(-sim[0]) if mids[i]!=t][:5]
print(f"most similar to movie {t} ({genre[t]}):", [(mids[i], genre[mids[i]]) for i in top])
most similar to movie 1 (Action): [(26, 'Drama'), (43, 'Action'), (16, 'SciFi'), (45, 'Comedy'), (21, 'Comedy')]
Challenge 4 · Train matrix factorization¶
Biased MF with SGD, K=3.
In [5]:
nu, ni, K = df.user_id.max(), df.movie_id.max(), 3
rs = np.random.RandomState(0); P = rs.normal(0,0.1,(nu,K)); Q = rs.normal(0,0.1,(ni,K))
bu = np.zeros(nu); bi = np.zeros(ni); g = tr.rating.mean(); lr, reg = 0.02, 0.05
data = tr[["user_id","movie_id","rating"]].values.astype(float).copy()
for _ in range(40):
rs.shuffle(data)
for u,m,r in data:
u,m=int(u)-1,int(m)-1; e=r-(g+bu[u]+bi[m]+P[u]@Q[m])
bu[u]+=lr*(e-reg*bu[u]); bi[m]+=lr*(e-reg*bi[m])
Pu=P[u].copy(); P[u]+=lr*(e*Q[m]-reg*P[u]); Q[m]+=lr*(e*Pu-reg*Q[m])
mf = rmse([g+bu[int(r.user_id)-1]+bi[int(r.movie_id)-1]+P[int(r.user_id)-1]@Q[int(r.movie_id)-1] for r in te.itertuples()], te.rating)
print(f"matrix factorization RMSE = {mf:.3f} (beats both baselines)")
matrix factorization RMSE = 0.740 (beats both baselines)
Challenge 5 · Recommend for a user¶
Predict unseen movies, exclude seen, take the top 5.
In [6]:
genre = dict(zip(movies.movie_id, movies.genre)); user = 7; u = user-1
seen = set(df[df.user_id==user].movie_id)
recs = sorted([(m+1, g+bu[u]+bi[m]+P[u]@Q[m]) for m in range(ni) if (m+1) not in seen], key=lambda t:-t[1])[:5]
for mid, sc in recs: print(f" movie {mid} ({genre[mid]}): {sc:.2f} stars")
movie 26 (Drama): 4.33 stars movie 7 (Action): 4.19 stars movie 27 (SciFi): 3.98 stars movie 1 (Action): 3.96 stars movie 9 (Action): 3.94 stars