Chapter 148 · Advanced & Applied Topics
NLP & LLMs · Challenge Solutions
Worked solutions: bag-of-words vs TF-IDF, reading the model's words, cosine neighbors, attention by hand, and perplexity plus BLEU and ROUGE.
In [1]:
import numpy as np, pandas as pd, re, math
import matplotlib.pyplot as plt
from collections import Counter
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, f1_score
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics.pairwise import cosine_similarity
plt.rcParams.update({"figure.dpi":110,"axes.grid":True,"grid.alpha":0.25,"font.size":11})
FU, BL, GR, RD = "#a21caf", "#2563eb", "#16a34a", "#dc2626"
BASE = "https://raw.githubusercontent.com/johnfisher-ai/Statistics-Data-Science-AI-Visual-Book/main/data/"
fn = "nlp-and-large-language-models--product-reviews.xlsx"
try:
df = pd.read_excel("../../data/" + fn)
except FileNotFoundError:
df = pd.read_excel(BASE + fn)
print(df.shape); print(df.head(3).to_string())
X = df.review_text.values; y = (df.sentiment == "positive").astype(int).values
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3, random_state=1, stratify=y)
(800, 3) review_id review_text sentiment 0 R1000 for the price, arrived on time, fantastic build quality. positive 1 R1001 would not recommend, shipping was quick, used it for a few days. negative 2 R1002 exceeded my expectations, used it for a few days, the packaging was fine. positive
Challenge 1 · Bag-of-words vs TF-IDF¶
Same classifier, two representations.
In [2]:
P = {}
for name, vec in [("bag-of-words", CountVectorizer()), ("TF-IDF", TfidfVectorizer())]:
A = vec.fit_transform(Xtr); clf = LogisticRegression(max_iter=1000).fit(A, ytr)
P[name] = clf.predict(vec.transform(Xte))
print(f"{name:14} accuracy = {accuracy_score(yte, P[name]):.3f}")
print(f"Tie: the two predict the same label for {(P['bag-of-words'] == P['TF-IDF']).mean():.0%} of the test reviews.")
bag-of-words accuracy = 0.929 TF-IDF accuracy = 0.929 Tie: the two predict the same label for 100% of the test reviews.
Challenge 2 · Read the model¶
Largest coefficients either way.
In [3]:
vec = TfidfVectorizer(); A = vec.fit_transform(Xtr); clf = LogisticRegression(max_iter=1000).fit(A, ytr)
names = np.array(vec.get_feature_names_out()); coef = clf.coef_[0]
print("POSITIVE:", ", ".join(names[np.argsort(-coef)][:8]))
print("NEGATIVE:", ", ".join(names[np.argsort(coef)][:8]))
POSITIVE: great, highly, works, this, excellent, fantastic, build, exceeded NEGATIVE: poor, not, would, of, waste, experience, terrible, away
Challenge 3 · Similar words by cosine¶
SVD the term-document matrix, then cosine similarity.
In [4]:
cv = CountVectorizer(); M = cv.fit_transform(df.review_text).T
emb = TruncatedSVD(n_components=8, random_state=0).fit_transform(M)
vocab = np.array(cv.get_feature_names_out()); S = cosine_similarity(emb)
for w in ["great", "poor"]:
i = int(np.where(vocab == w)[0][0])
print(w, "->", [(vocab[j], round(float(S[i][j]),2)) for j in np.argsort(-S[i]) if j != i][:4])
great -> [('value', 0.98), ('works', 0.93), ('money', 0.91), ('quickly', 0.84)]
poor -> [('recommend', 0.93), ('cheaply', 0.92), ('made', 0.92), ('broke', 0.91)]
Challenge 4 · Attention by hand¶
softmax(QK^T/sqrt(d))V, and the weights must sum to 1.
In [5]:
rng = np.random.default_rng(0); n, d = 4, 4
Q, K, V = (rng.normal(size=(n,d)) for _ in range(3))
sc = Q @ K.T / np.sqrt(d); sc -= sc.max(axis=1, keepdims=True)
W = np.exp(sc); W /= W.sum(axis=1, keepdims=True)
print(W.round(3)); print("row sums:", W.sum(axis=1).round(6)); print("output shape:", (W @ V).shape)
[[0.349 0.219 0.232 0.201] [0.554 0.24 0.075 0.131] [0.323 0.133 0.206 0.338] [0.158 0.205 0.119 0.518]] row sums: [1. 1. 1. 1.] output shape: (4, 4)
Challenge 5 · Perplexity, BLEU, ROUGE¶
Implemented directly.
In [6]:
def toks(s): return ["<s>"] + re.findall(r"[a-z']+", s.lower()) + ["</s>"]
tr_t=[toks(s) for s in Xtr]; te_t=[toks(s) for s in Xte]
uni=Counter(); bi=Counter()
for t in tr_t: uni.update(t); bi.update(zip(t[:-1],t[1:]))
V_=len(uni); k=0.5; tot=sum(uni.values())
pu=math.exp(-sum(math.log((uni[w]+k)/(tot+k*V_)) for t in te_t for w in t[1:])/sum(len(t)-1 for t in te_t))
lp=0.0; n=0
for t in te_t:
for a,b in zip(t[:-1],t[1:]): lp+=math.log((bi[(a,b)]+k)/(uni[a]+k*V_)); n+=1
print(f"unigram perplexity {pu:.1f} -> bigram perplexity {math.exp(-lp/n):.1f}")
ref="the cat sat on the mat".split(); cand="the cat sat on a mat".split()
ng=lambda s,n: Counter(tuple(s[i:i+n]) for i in range(len(s)-n+1))
p1=sum(min(v,ng(ref,1)[g]) for g,v in ng(cand,1).items())/len(cand)
p2=sum(min(v,ng(ref,2)[g]) for g,v in ng(cand,2).items())/(len(cand)-1)
r1=sum(min(v,ng(cand,1)[g]) for g,v in ng(ref,1).items())/len(ref)
print(f"BLEU-style {math.sqrt(p1*p2):.2f} ROUGE-1 recall {r1:.2f}")
unigram perplexity 64.7 -> bigram perplexity 5.1 BLEU-style 0.71 ROUGE-1 recall 0.83