import numpy as np, pandas as pd, re, math
import matplotlib.pyplot as plt
from collections import Counter
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, f1_score
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics.pairwise import cosine_similarity
plt.rcParams.update({"figure.dpi":110,"axes.grid":True,"grid.alpha":0.25,"font.size":11})
FU, BL, GR, RD = "#a21caf", "#2563eb", "#16a34a", "#dc2626"
BASE = "https://raw.githubusercontent.com/johnfisher-ai/Statistics-Data-Science-AI-Visual-Book/main/data/"
fn = "nlp-and-large-language-models--product-reviews.xlsx"
try:
    df = pd.read_excel("../../data/" + fn)
except FileNotFoundError:
    df = pd.read_excel(BASE + fn)
print(df.shape); print(df.head(3).to_string())
X = df.review_text.values; y = (df.sentiment == "positive").astype(int).values
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3, random_state=1, stratify=y)

(800, 3)
  review_id                                                                review_text sentiment
0     R1000                   for the price, arrived on time, fantastic build quality.  positive
1     R1001           would not recommend, shipping was quick, used it for a few days.  negative
2     R1002  exceeded my expectations, used it for a few days, the packaging was fine.  positive

for name, vec in [("bag-of-words", CountVectorizer()), ("TF-IDF", TfidfVectorizer())]:
    A = vec.fit_transform(Xtr); clf = LogisticRegression(max_iter=1000).fit(A, ytr)
    print(f"{name:14} accuracy = {accuracy_score(yte, clf.predict(vec.transform(Xte))):.3f}")
print("Tie: the reviews are short with an 80-word vocabulary, so there is nothing for IDF to down-weight.")

bag-of-words   accuracy = 0.929
TF-IDF         accuracy = 0.929
Tie: the reviews are short with an 80-word vocabulary, so there is nothing for IDF to down-weight.

vec = TfidfVectorizer(); A = vec.fit_transform(Xtr); clf = LogisticRegression(max_iter=1000).fit(A, ytr)
names = np.array(vec.get_feature_names_out()); coef = clf.coef_[0]
print("POSITIVE:", ", ".join(names[np.argsort(-coef)][:8]))
print("NEGATIVE:", ", ".join(names[np.argsort(coef)][:8]))

POSITIVE: great, highly, works, this, excellent, build, fantastic, expectations
NEGATIVE: poor, not, would, waste, of, terrible, experience, right

cv = CountVectorizer(); M = cv.fit_transform(df.review_text).T
emb = TruncatedSVD(n_components=8, random_state=0).fit_transform(M)
vocab = np.array(cv.get_feature_names_out()); S = cosine_similarity(emb)
for w in ["great", "poor"]:
    i = int(np.where(vocab == w)[0][0])
    print(w, "->", [(vocab[j], round(float(S[i][j]),2)) for j in np.argsort(-S[i]) if j != i][:4])

great -> [('value', 0.98), ('works', 0.93), ('money', 0.91), ('and', 0.84)]
poor -> [('recommend', 0.93), ('made', 0.92), ('cheaply', 0.92), ('broke', 0.91)]

rng = np.random.default_rng(0); n, d = 4, 4
Q, K, V = (rng.normal(size=(n,d)) for _ in range(3))
sc = Q @ K.T / np.sqrt(d); sc -= sc.max(axis=1, keepdims=True)
W = np.exp(sc); W /= W.sum(axis=1, keepdims=True)
print(W.round(3)); print("row sums:", W.sum(axis=1).round(6)); print("output shape:", (W @ V).shape)

[[0.349 0.219 0.232 0.201]
 [0.554 0.24  0.075 0.131]
 [0.323 0.133 0.206 0.338]
 [0.158 0.205 0.119 0.518]]
row sums: [1. 1. 1. 1.]
output shape: (4, 4)

def toks(s): return ["<s>"] + re.findall(r"[a-z']+", s.lower()) + ["</s>"]
tr_t=[toks(s) for s in Xtr]; te_t=[toks(s) for s in Xte]
uni=Counter(); bi=Counter()
for t in tr_t: uni.update(t); bi.update(zip(t[:-1],t[1:]))
V_=len(uni); k=0.5; tot=sum(uni.values())
pu=math.exp(-sum(math.log((uni[w]+k)/(tot+k*V_)) for t in te_t for w in t[1:])/sum(len(t)-1 for t in te_t))
lp=0.0; n=0
for t in te_t:
    for a,b in zip(t[:-1],t[1:]): lp+=math.log((bi[(a,b)]+k)/(uni[a]+k*V_)); n+=1
print(f"unigram perplexity {pu:.1f}  ->  bigram perplexity {math.exp(-lp/n):.1f}")
ref="the cat sat on the mat".split(); cand="the cat sat on a mat".split()
ng=lambda s,n: Counter(tuple(s[i:i+n]) for i in range(len(s)-n+1))
p1=sum(min(v,ng(ref,1)[g]) for g,v in ng(cand,1).items())/len(cand)
p2=sum(min(v,ng(ref,2)[g]) for g,v in ng(cand,2).items())/(len(cand)-1)
r1=sum(min(v,ng(cand,1)[g]) for g,v in ng(ref,1).items())/len(ref)
print(f"BLEU-style {math.sqrt(p1*p2):.2f}   ROUGE-1 recall {r1:.2f}")

unigram perplexity 64.7  ->  bigram perplexity 5.1
BLEU-style 0.71   ROUGE-1 recall 0.83

Challenge 1 · Bag-of-words vs TF-IDF¶

Challenge 2 · Read the model¶

Challenge 3 · Similar words by cosine¶

Challenge 4 · Attention by hand¶

Challenge 5 · Perplexity, BLEU, ROUGE¶