🎯 What you'll build in this notebook¶
| # | Demo | Idea it builds |
|---|---|---|
| 1 | The bin trap | bin width changes a histogram's apparent shape |
| 2 | What the box hides | box plots compare groups but conceal modality |
| 3 | Taming spaghetti | highlight one line, gray the rest |
| 4 | Reading a scatter | direction, form, strength, and overplotting |
| 5 | Anscombe's quartet | identical stats, four different realities |
⚙️ Setup, imports & the book's plotting style¶
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.stats import gaussian_kde
rng = np.random.default_rng(15)
NAVY="#0a1230"; INK="#1a2138"; INK_SOFT="#4a5578"
CYAN="#0891b2"; PURPLE="#7c3aed"; AMBER="#d97706"; GREEN="#059669"; PINK="#db2777"; BLUE="#2563eb"; GRID="#e6e9f2"
plt.rcParams.update({
"figure.facecolor":"white","axes.facecolor":"white","figure.dpi":110,"font.size":11,
"axes.edgecolor":GRID,"axes.linewidth":1.2,"axes.grid":True,"grid.color":GRID,"axes.axisbelow":True,
"axes.spines.top":False,"axes.spines.right":False,"axes.titlesize":12.5,"axes.titleweight":"bold","axes.titlecolor":INK,
"axes.labelcolor":INK_SOFT,"axes.labelsize":11,"xtick.color":INK_SOFT,"ytick.color":INK_SOFT,"legend.frameon":False,
})
print("✅ Environment ready.")
✅ Environment ready.
data = np.concatenate([rng.normal(35, 6, 1200), rng.normal(62, 7, 800)]) # secretly bimodal
fig, axes = plt.subplots(1, 4, figsize=(15, 3.8))
for ax, b in zip(axes[:3], [4, 18, 120]):
ax.hist(data, bins=b, color=CYAN, edgecolor="white", linewidth=0.3)
ax.set_title(f"{b} bins"); ax.set_yticks([])
# Freedman-Diaconis principled default, with a KDE overlay (density scale)
iqr = np.subtract(*np.percentile(data, [75, 25]))
fd_w = 2 * iqr / len(data) ** (1/3)
fd_bins = int((data.max() - data.min()) / fd_w)
axes[3].hist(data, bins=fd_bins, density=True, color=AMBER, edgecolor="white", linewidth=0.3)
xs = np.linspace(data.min(), data.max(), 300)
axes[3].plot(xs, gaussian_kde(data)(xs), color=PURPLE, lw=2.2)
axes[3].set_title(f"Freedman-Diaconis\n({fd_bins} bins) + KDE"); axes[3].set_yticks([])
plt.tight_layout(); plt.show()
print("4 bins call it one hump; 120 bins is static. The principled binning reveals the truth: two modes.")
4 bins call it one hump; 120 bins is static. The principled binning reveals the truth: two modes.
# ECDF: a bin-free alternative to the histogram. Sort the values, plot the running
# fraction <= x. No bin width to bias the picture, and the two rises reveal the two modes.
xs = np.sort(data); ecdf = np.arange(1, len(xs) + 1) / len(xs)
fig, ax = plt.subplots(figsize=(7, 3.4))
ax.step(xs, ecdf, where="post", color=CYAN, lw=1.8)
ax.axhline(0.5, color=GRID, ls="--")
ax.set(title="ECDF: same data, no bins to choose", xlabel="value", ylabel="fraction of data <= x")
ax.margins(x=0.01); plt.tight_layout(); plt.show()
A bin-free alternative: the ECDF. A histogram's shape depends on the bin width you happen to pick. The empirical cumulative distribution function sidesteps that entirely: sort the values and plot the running fraction at or below each x. There is nothing to tune, every point is shown exactly, and you can read any percentile straight off the y-axis (the median is where the curve crosses 0.5). The two steep climbs here, near 35 and 62, are the same two modes the box plot hides, surfaced without a single binning decision.
normal_g = rng.normal(50, 12, 600)
bimodal_g = np.concatenate([rng.normal(32, 5, 300), rng.normal(68, 5, 300)]) # same rough center/spread
uniform_g = rng.uniform(25, 75, 600)
groups = [normal_g, bimodal_g, uniform_g]; names = ["Normal", "Bimodal", "Uniform"]
fig, (top, bot) = plt.subplots(2, 1, figsize=(9, 7))
bp = top.boxplot(groups, showfliers=True, patch_artist=True, widths=0.5)
for patch in bp["boxes"]: patch.set_facecolor("#dfe7f7")
for med in bp["medians"]: med.set_color(PINK); med.set_linewidth(2)
top.set_xticks([1,2,3]); top.set_xticklabels(names)
top.set_title("Three box plots that look almost the same"); top.set_ylabel("value")
for i, (g, c) in enumerate(zip(groups, [CYAN, AMBER, GREEN])):
bot.hist(g, bins=30, density=True, alpha=0.55, color=c, label=names[i])
bot.set_title("...but the histograms tell completely different stories")
bot.set_yticks([]); bot.legend()
plt.tight_layout(); plt.show()
for g, n in zip(groups, names):
q1, med, q3 = np.percentile(g, [25, 50, 75])
print(f"{n:8} median={med:5.1f} Q1={q1:5.1f} Q3={q3:5.1f} (similar five-number summaries)")
Normal median= 50.0 Q1= 41.5 Q3= 58.2 (similar five-number summaries) Bimodal median= 49.7 Q1= 31.6 Q3= 68.1 (similar five-number summaries) Uniform median= 51.0 Q1= 37.8 Q3= 63.1 (similar five-number summaries)
months = np.arange(1, 13)
regions = {"North": 100, "South": 90, "East": 95, "West": 88, "Central": 92}
series = {r: base + np.cumsum(rng.normal(2 if r=="West" else 0.4, 3, 12)) for r, base in regions.items()}
fig, ax = plt.subplots(figsize=(9.5, 4.8))
for r, y in series.items():
if r == "West":
ax.plot(months, y, color=AMBER, lw=3, zorder=5)
ax.text(months[-1]+0.1, y[-1], " West", color=AMBER, fontweight="bold", va="center")
else:
ax.plot(months, y, color="#c7ccda", lw=1.4)
ax.text(months[-1]+0.1, y[-1], f" {r}", color="#aab0c0", fontsize=8.5, va="center")
ax.set_title("Sales by region: West is the story"); ax.set_xlabel("month"); ax.set_ylabel("sales")
ax.set_xlim(1, 13.6); ax.set_xticks(months)
plt.tight_layout(); plt.show()
print("One color does the talking. Note: line charts need NOT start at zero (position, not length, encodes value).")
One color does the talking. Note: line charts need NOT start at zero (position, not length, encodes value).
# a clear positive relationship with a trend line
x = rng.uniform(0, 10, 120)
y = 2.2 * x + 5 + rng.normal(0, 4, 120)
m, b = np.polyfit(x, y, 1)
r = np.corrcoef(x, y)[0, 1]
# a heavily overplotted cloud
bigx = rng.normal(0, 1, 12000); bigy = bigx * 0.6 + rng.normal(0, 1, 12000)
fig, (a1, a2) = plt.subplots(1, 2, figsize=(13, 4.6))
a1.scatter(x, y, color=CYAN, s=28, edgecolor="white", linewidth=0.4)
a1.plot(np.sort(x), m*np.sort(x)+b, color=PINK, lw=2.2, label=f"trend (r={r:.2f})")
a1.set_title("Direction +, form linear, strength strong"); a1.set_xlabel("x"); a1.set_ylabel("y"); a1.legend()
hb = a2.hexbin(bigx, bigy, gridsize=35, cmap="Purples", mincnt=1)
a2.set_title("12,000 points: hexbin shows true density"); a2.set_xlabel("x"); a2.set_ylabel("y")
a2.grid(False)
plt.tight_layout(); plt.show()
print(f"trend line: y = {m:.2f}x + {b:.2f}, r = {r:.2f}. A line summarizes, but always look at the cloud first.")
trend line: y = 2.02x + 5.33, r = 0.82. A line summarizes, but always look at the cloud first.
x123 = np.array([10,8,13,9,11,14,6,4,12,7,5], float)
x4 = np.array([8,8,8,8,8,8,8,19,8,8,8], float)
Y = {
"I": np.array([8.04,6.95,7.58,8.81,8.33,9.96,7.24,4.26,10.84,4.82,5.68]),
"II": np.array([9.14,8.14,8.74,8.77,9.26,8.10,6.13,3.10,9.13,7.26,4.74]),
"III": np.array([7.46,6.77,12.74,7.11,7.81,8.84,6.08,5.39,8.15,6.42,5.73]),
"IV": np.array([6.58,5.76,7.71,8.84,8.47,7.04,5.25,12.50,5.56,7.91,6.89]),
}
xs = {"I":x123,"II":x123,"III":x123,"IV":x4}
fig, axes = plt.subplots(2, 2, figsize=(11, 8))
line = np.array([2, 20])
for ax, key in zip(axes.ravel(), ["I","II","III","IV"]):
xv, yv = xs[key], Y[key]
ax.scatter(xv, yv, color=CYAN, s=55, edgecolor="white", zorder=5)
ax.plot(line, 3 + 0.5*line, color=PINK, lw=2) # the SAME line for all four
ax.set_title(f"Dataset {key}: mean y={yv.mean():.2f}, r={np.corrcoef(xv,yv)[0,1]:.2f}")
ax.set_xlim(2,20); ax.set_ylim(2,14)
fig.suptitle("Anscombe's quartet: all share y = 3 + 0.5x, r = 0.82", fontsize=13, fontweight="bold")
plt.tight_layout(); plt.show()
print("Same regression line through all four. Only the picture reveals the curve, the outlier, and the leverage point.")
Same regression line through all four. Only the picture reveals the curve, the outlier, and the leverage point.
✈️ Real-World Example: Flight Arrival Delays¶
A single numeric variable with real character: 500 arrival delays. The histogram shows the shape (most flights near on-time, a long right tail of bad delays), while the box plot puts numbers on it, the median, the quartiles, and the cloud of long-delay outliers past the upper whisker. Two charts, two complementary reads of the same column.
# --- Real-World beat: histogram and box plot of a skewed numeric variable ---
BASE = "https://raw.githubusercontent.com/johnfisher-ai/Statistics-Data-Science-AI-Visual-Book/main/data/"
try: flights = pd.read_excel("../../data/charts-for-numerical-data--flight_delays.xlsx", sheet_name="Data")
except FileNotFoundError: flights = pd.read_excel(BASE+"charts-for-numerical-data--flight_delays.xlsx", sheet_name="Data")
d = flights.delay_minutes
q1, q3 = d.quantile([.25, .75]); fence = q3 + 1.5*(q3-q1)
print(f"mean {d.mean():.1f} min median {d.median():.0f} min skew {d.skew():.2f}")
print(f"Q1 {q1:.0f} Q3 {q3:.0f} upper fence {fence:.0f} long-delay outliers: {(d > fence).sum()}")
fig, ax = plt.subplots(1, 2, figsize=(11,4))
ax[0].hist(d, bins=40, color=CYAN, alpha=0.85, edgecolor="white")
ax[0].axvline(d.mean(), color=PURPLE, ls="--", lw=2, label=f"mean {d.mean():.0f}")
ax[0].axvline(d.median(), color=AMBER, ls="--", lw=2, label=f"median {d.median():.0f}")
ax[0].set_title("Histogram: the shape"); ax[0].set_xlabel("delay (minutes)"); ax[0].set_ylabel("flights"); ax[0].legend()
ax[1].boxplot(d, patch_artist=True, boxprops=dict(facecolor=CYAN, alpha=0.6),
medianprops=dict(color=AMBER, linewidth=2), flierprops=dict(marker="o", markersize=4, alpha=0.4))
ax[1].set_title("Box plot: quartiles and outliers"); ax[1].set_ylabel("delay (minutes)"); ax[1].set_xticks([])
plt.tight_layout(); plt.show()
mean 13.5 min median 8 min skew 2.60 Q1 1 Q3 17 upper fence 41 long-delay outliers: 46
- The histogram shows one variable's shape, but bin width is editorial: try several, lean on Freedman-Diaconis.
- Box plots compare groups compactly yet hide modality; a violin or strip plot shows what the box conceals.
- Line charts are for ordered/time data; declutter spaghetti by highlighting one series. They need not start at zero.
- Scatter plots show direction, form, and strength of a relationship; fight overplotting with alpha or hexbin.
- Anscombe's quartet: identical summary stats, four different shapes. Always plot your data.