EXECUTED, WITH ASSERTIONS
This program was run during verification and its results asserted. The runner that does it is tools/run_nlp_labs.py.
Straight from labs/course-15a-nlp/10_sentiment_rnn.py, unchanged.
"""Experiments 10 and 11 -- a sentiment classifier with scikit-learn, and a
character-level RNN that generates text.
Real NLTK movie_reviews (2,000 hand-labelled documents), real scikit-learn,
real PyTorch. Nothing here is simulated.
"""
import time
import numpy as np
import fixtures as f
def experiment_10():
print("\n --- experiment 10: a sentiment classifier, with baselines")
from nltk.corpus import movie_reviews, stopwords
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.dummy import DummyClassifier
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import (classification_report, confusion_matrix,
accuracy_score)
from sklearn.pipeline import make_pipeline
docs, labels = [], []
for cat in movie_reviews.categories():
for fid in movie_reviews.fileids(cat):
docs.append(movie_reviews.raw(fid))
labels.append(1 if cat == "pos" else 0)
labels = np.array(labels)
Xtr, Xte, ytr, yte = train_test_split(
docs, labels, test_size=0.25, random_state=f.SEED, stratify=labels)
print(f"\n NLTK movie_reviews: {len(docs):,} documents, "
f"{len(Xtr):,} train / {len(Xte):,} test")
print(f" class balance: {labels.mean():.3f} positive "
f"(perfectly balanced by construction)")
print(f" mean document length: "
f"{np.mean([len(d.split()) for d in docs]):.0f} words")
sw = stopwords.words("english")
configs = [
("majority class", DummyClassifier(strategy="most_frequent"), None),
("Naive Bayes, counts", MultinomialNB(), CountVectorizer()),
("Naive Bayes, TF-IDF", MultinomialNB(), TfidfVectorizer()),
("LogReg, TF-IDF", LogisticRegression(max_iter=2000),
TfidfVectorizer()),
("LogReg, TF-IDF, no stopwords", LogisticRegression(max_iter=2000),
TfidfVectorizer(stop_words=sw)),
("LogReg, TF-IDF 1-2 grams", LogisticRegression(max_iter=2000),
TfidfVectorizer(ngram_range=(1, 2), min_df=2)),
]
print(f"\n {'model':<32}{'features':>10}{'accuracy':>10}{'secs':>7}")
scores = {}
for name, clf, vec in configs:
t0 = time.perf_counter()
if vec is None:
pipe = make_pipeline(CountVectorizer(max_features=10), clf)
else:
pipe = make_pipeline(vec, clf)
pipe.fit(Xtr, ytr)
acc = accuracy_score(yte, pipe.predict(Xte))
nfeat = (pipe.steps[0][1].transform(Xte[:1]).shape[1])
scores[name] = (acc, pipe)
print(f" {name:<32}{nfeat:>10,}{acc:>10.4f}"
f"{time.perf_counter() - t0:>7.1f}")
base = scores["majority class"][0]
best_name = max((k for k in scores if k != "majority class"),
key=lambda k: scores[k][0])
best = scores[best_name][0]
print(f"\n best: {best_name} at {best:.4f}, against a "
f"majority-class baseline of {base:.4f}")
print(""" REPORT THE BASELINE ROW. On a balanced two-class problem
it is 0.5 and everyone knows it; on an imbalanced one it can
be 0.9 and an unwary report claims a triumph. Printing it
costs one line and makes every other number interpretable""")
# ---- the stopword question, answered rather than assumed -------------
with_sw = scores["LogReg, TF-IDF"][0]
without_sw = scores["LogReg, TF-IDF, no stopwords"][0]
print(f"\n the stopword question: {with_sw:.4f} with them, "
f"{without_sw:.4f} without")
if with_sw >= without_sw:
print(""" KEEPING THE STOPWORDS WAS AT LEAST AS GOOD, which is
what experiment 5 predicted for a sentiment task: 'not' is a
stopword, and negation is exactly what sentiment turns on.
The habit of stripping stopwords 'because that is what you
do' costs accuracy here""")
else:
print(f""" REMOVING THEM SCORED {without_sw - with_sw:+.4f} HIGHER,
and before reading anything into that, hold the number until
the cross-validation table below -- it turns out to be
smaller than the variation between folds.
The mechanism, either way, is that TF-IDF has ALREADY
down-weighted the function words (experiment 9's IDF table),
so an explicit stopword list has little left to remove""")
# ---- the confusion matrix and the errors -----------------------------
pipe = scores[best_name][1]
pred = pipe.predict(Xte)
cm = confusion_matrix(yte, pred)
print(f"\n confusion matrix for {best_name}:")
print(f" {'':<14}{'pred neg':>10}{'pred pos':>10}")
print(f" {'true neg':<14}{cm[0, 0]:>10}{cm[0, 1]:>10}")
print(f" {'true pos':<14}{cm[1, 0]:>10}{cm[1, 1]:>10}")
print("\n" + "\n".join(" " + l for l in
classification_report(yte, pred,
target_names=["neg", "pos"]
).splitlines()))
# ---- what the model learned ------------------------------------------
if hasattr(pipe.steps[-1][1], "coef_"):
vec = pipe.steps[0][1]
coef = pipe.steps[-1][1].coef_[0]
names = vec.get_feature_names_out()
top_pos = names[np.argsort(coef)[-10:]][::-1]
top_neg = names[np.argsort(coef)[:10]]
print(f"\n the 10 most POSITIVE features: {list(top_pos)}")
print(f" the 10 most NEGATIVE features: {list(top_neg)}")
print(""" READ THAT LIST -- it is the cheapest sanity check in
all of NLP. If the strongest features are sentiment words,
the model learned the task. If they are punctuation, a film
title or a reviewer's name, it learned an artefact of the
corpus and will not transfer.
A LINEAR MODEL HANDS YOU THIS FOR FREE. The deep models in
Course 14 A do not, which is why that course had to build a
dataset with a known answer to get the same check""")
# ---- cross-validation, because one split is one number ---------------
cv_pipe = make_pipeline(TfidfVectorizer(), LogisticRegression(max_iter=2000))
cv = cross_val_score(cv_pipe, docs, labels, cv=5)
print(f"\n 5-fold cross-validation: {cv.round(4).tolist()}")
print(f" mean {cv.mean():.4f}, std {cv.std():.4f}, "
f"range {cv.max() - cv.min():.4f}")
spread = cv.max() - cv.min()
gaps = {
"stopwords on vs off": abs(with_sw - without_sw),
"best vs second-best": abs(
best - sorted((v[0] for k, v in scores.items()
if k != "majority class"), reverse=True)[1]),
"best vs the baseline": best - base,
}
print(f"\n {'comparison':<26}{'gap':>8} verdict against a "
f"fold spread of {spread:.4f}")
for label, g in gaps.items():
verdict = "REAL" if g > spread else "NOISE -- do not report it"
print(f" {label:<26}{g:>8.4f} {verdict}")
print(f""" NOW GO BACK AND RE-READ THE FIRST TABLE. The spread
across folds is {spread:.4f}, and every difference between
the real models is smaller than that. Naive Bayes 'beating'
logistic regression, and stopword removal 'helping', are both
inside the noise -- so the honest conclusion from that table
is THESE MODELS ARE INDISTINGUISHABLE ON THIS DATA.
That is a legitimate finding and it is the one most lab
reports get wrong: they rank six models on a single split and
declare a winner that a different random seed would reverse.
Only the gap to the baseline, {gaps['best vs the baseline']:.4f},
is far outside the spread and therefore real""")
assert best > base + 0.25, f"{best:.4f} should beat the {base:.4f} baseline"
assert cv.mean() > 0.75
return scores, cv
def experiment_11():
print("\n --- experiment 11: a character-level RNN that generates text")
import torch
import torch.nn as nn
torch.manual_seed(f.SEED)
torch.set_num_threads(2)
from nltk.corpus import gutenberg
raw = gutenberg.raw("austen-emma.txt")[:120000]
text = " ".join(raw.split()).lower()
chars = sorted(set(text))
stoi = {c: i for i, c in enumerate(chars)}
itos = {i: c for c, i in stoi.items()}
data = np.array([stoi[c] for c in text], dtype="int64")
print(f"\n corpus: Jane Austen's Emma, from NLTK's gutenberg")
print(f" {len(text):,} characters, {len(chars)} distinct")
seq_len = 60
step = 3
X = np.stack([data[i:i + seq_len]
for i in range(0, len(data) - seq_len - 1, step)])
Y = np.stack([data[i + 1:i + seq_len + 1]
for i in range(0, len(data) - seq_len - 1, step)])
print(f" {len(X):,} training sequences of {seq_len} characters")
class CharRNN(nn.Module):
def __init__(self, n, emb=48, hid=192):
super().__init__()
self.emb = nn.Embedding(n, emb)
self.lstm = nn.LSTM(emb, hid, batch_first=True)
self.out = nn.Linear(hid, n)
def forward(self, x, h=None):
e = self.emb(x)
o, h = self.lstm(e, h)
return self.out(o), h
model = CharRNN(len(chars))
n_params = sum(p.numel() for p in model.parameters())
print(f" model: embedding + LSTM + linear, {n_params:,} parameters")
opt = torch.optim.Adam(model.parameters(), lr=3e-3)
lossf = nn.CrossEntropyLoss()
Xt, Yt = torch.tensor(X), torch.tensor(Y)
batch = 128
print(f"\n {'epoch':>7}{'loss':>10}{'perplexity':>13}")
losses = []
for ep in range(6):
model.train()
perm = torch.randperm(len(Xt))
tot, nb = 0.0, 0
for i in range(0, len(Xt), batch):
idx = perm[i:i + batch]
opt.zero_grad()
logits, _ = model(Xt[idx])
loss = lossf(logits.reshape(-1, len(chars)), Yt[idx].reshape(-1))
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
opt.step()
tot += loss.item()
nb += 1
losses.append(tot / nb)
print(f" {ep:>7}{losses[-1]:>10.4f}{np.exp(losses[-1]):>13.2f}")
print(f""" PERPLEXITY IS exp(cross-entropy), and it has a reading:
'the model is as uncertain as if it were choosing uniformly
among this many characters'. It started at
{np.exp(losses[0]):.1f} and ended at {np.exp(losses[-1]):.1f},
against {len(chars)} characters in the alphabet.
Uniform guessing would give {len(chars):.1f}. That is the
baseline, and it is why perplexity is quoted instead of loss:
the number means something on its own""")
def generate(seed_text, n=180, temp=1.0):
model.eval()
ctx = [stoi.get(c, 0) for c in seed_text.lower()][-seq_len:]
out = list(seed_text)
h = None
with torch.no_grad():
for _ in range(n):
x = torch.tensor([ctx[-seq_len:]])
logits, _ = model(x)
p = torch.softmax(logits[0, -1] / temp, dim=-1).numpy()
nxt = int(np.random.default_rng().choice(len(chars), p=p))
out.append(itos[nxt])
ctx.append(nxt)
return "".join(out)
seed = "she was the "
print(f"\n generated from the seed {seed!r}:")
for t in (0.2, 0.6, 1.0, 1.6):
np.random.seed(f.SEED)
g = generate(seed, 150, t)[len(seed):]
print(f"\n T = {t}")
for i in range(0, len(g), 68):
print(f" {g[i:i + 68]!r}")
print(""" LOW TEMPERATURE REPEATS ITSELF; high temperature
invents words. In between, it produces English-shaped text
with correct spelling, plausible word boundaries and no
meaning whatever.
THAT IS EXACTLY WHAT A CHARACTER MODEL CAN LEARN. Spelling
and local grammar are decided within a few characters, so the
LSTM's memory reaches them. What a paragraph is ABOUT is
decided over hundreds of characters, and it does not.
Experiment 12's transformer is the architecture built to
close that gap""")
assert losses[-1] < losses[0], "the model should learn something"
assert np.exp(losses[-1]) < len(chars), (
"perplexity should beat uniform guessing over the alphabet")
return losses
def main():
print(" Experiments 10 and 11 -- sentiment classification, character RNN")
experiment_10()
experiment_11()
print("\n all assertions passed")
if __name__ == "__main__":
main()
One experiment from the Natural Language Processing lab. The rest of them, and the theory behind this one, are on the lab page.