←Back to the bookChapter 9 · Measures of Dispersion

Part II · Describing Data  |  Chapter 9

Measures of Dispersion 🐍 Notebook

Five demos on how spread out the data is: range and IQR, variance and standard deviation built from scratch, the 68-95-99.7 rule, the box-plot outlier test, and the coefficient of variation.

Author: John Fisher  ·  Statistics, Data Science and AI: A Visual Handbook  ·  2026

🎯 What you'll build in this notebook

#	Demo	Idea it builds
1	Same mean, different spread	why the average alone can mislead
2	Variance & SD from scratch	deviations, squaring, and n vs n−1
3	The 68-95-99.7 rule	reading the standard deviation
4	Box plot & the outlier test	SD vs IQR when an outlier appears
5	Coefficient of variation	comparing spread across different scales

---

⚙️ Setup, imports & the book's plotting style

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

rng = np.random.default_rng(9)

NAVY="#0a1230"; INK="#1a2138"; INK_SOFT="#4a5578"
CYAN="#0891b2"; PURPLE="#7c3aed"; AMBER="#d97706"; GREEN="#059669"; PINK="#db2777"; BLUE="#2563eb"; GRID="#e6e9f2"
plt.rcParams.update({
    "figure.facecolor":"white","axes.facecolor":"white","figure.dpi":110,"font.size":11,
    "axes.edgecolor":GRID,"axes.linewidth":1.2,"axes.grid":True,"grid.color":GRID,"axes.axisbelow":True,
    "axes.spines.top":False,"axes.spines.right":False,"axes.titlesize":15,"axes.titleweight":"bold","axes.titlecolor":INK,
    "axes.labelcolor":INK_SOFT,"axes.labelsize":11.5,"xtick.color":INK_SOFT,"ytick.color":INK_SOFT,"legend.frameon":False,
})
def titlecard(ax,t,sub=None):
    ax.set_title(t, loc="left", pad=18)
    if sub: ax.text(0,1.02,sub,transform=ax.transAxes,fontsize=10.5,color=INK_SOFT,va="bottom")
print("✅ Environment ready.")

✅ Environment ready.

DEMO 1 · WHY SPREAD MATTERS

⚖️ Same average, very different stories

Two classes can share an identical mean score yet be worlds apart. Dispersion is what the mean hides.

A = pd.Series([70, 72, 75, 78, 80])    # steady class
B = pd.Series([50, 60, 75, 90, 100])   # all over the place
print(f"Class A: mean {A.mean():.0f}, range {A.max()-A.min()}, SD {A.std(ddof=1):.1f}")
print(f"Class B: mean {B.mean():.0f}, range {B.max()-B.min()}, SD {B.std(ddof=1):.1f}")
print("Same mean (75), but B is far more spread out.")

Class A: mean 75, range 10, SD 4.1
Class B: mean 75, range 50, SD 20.6
Same mean (75), but B is far more spread out.

fig,(a1,a2)=plt.subplots(1,2,figsize=(11,3.4),sharex=True,constrained_layout=True)
for ax,data,name,c in [(a1,A,"Class A (consistent)",CYAN),(a2,B,"Class B (erratic)",AMBER)]:
    ax.scatter(data, [1]*len(data), s=120, color=c, alpha=0.8, edgecolor="white", zorder=3)
    ax.axvline(75, color=PURPLE, ls="--", lw=2)
    ax.set_title(name, loc="left", fontsize=12, fontweight="bold", color=INK)
    ax.set_yticks([]); ax.set_ylim(0.5, 1.6); ax.set_xlabel("Score")
a1.text(75, 1.42, "mean 75", color=PURPLE, ha="center", fontsize=9, fontweight="bold")
plt.show()

DEMO 2 · VARIANCE & SD FROM SCRATCH

📐 Deviations, squared, averaged, rooted

Variance averages the squared distances from the mean. Standard deviation square-roots it back into the data's own units.

x = np.array([4, 8, 6, 5, 7])
mean = x.mean()
dev  = x - mean                 # deviations from the mean (they always sum to 0)
sq   = dev**2                   # square so they do not cancel, and big gaps count more

print("values     :", x)
print("deviations :", dev, " (sum =", dev.sum(), ")")
print("squared    :", sq)
print()
var_pop  = sq.mean()            # divide by n   (whole population)
var_samp = sq.sum()/(len(x)-1)  # divide by n-1 (sample estimate, Bessel's correction)
print(f"Population variance (÷n)   = {var_pop:.2f}   -> SD = {np.sqrt(var_pop):.2f}")
print(f"Sample variance     (÷n-1) = {var_samp:.2f}   -> SD = {np.sqrt(var_samp):.2f}")
print(f"NumPy check: np.std(x, ddof=1) = {np.std(x, ddof=1):.2f}")

# --- added visual ---
import matplotlib.pyplot as plt, numpy as np
_CY,_PU,_AM,_GR,_PK,_INK,_GRY,_BL = "#0891b2","#7c3aed","#d97706","#059669","#db2777","#1a2138","#c7ccda","#2563eb"
idxs = np.arange(len(x))
fig, ax = plt.subplots(figsize=(8, 3.8))
ax.axhline(mean, color=_INK, lw=1.5, ls="--", label=f"mean {mean:.1f}")
for i, v in zip(idxs, x):
    ax.plot([i, i], [mean, v], color=_PK if v >= mean else _AM, lw=2.4, zorder=1)
ax.scatter(idxs, x, s=70, color=_CY, zorder=3)
ax.set_xticks(idxs); ax.set_xlabel("data point"); ax.set_ylabel("value"); ax.legend()
ax.set_title("Variance = average of these squared deviations from the mean")
plt.tight_layout(); plt.show()

values     : [4 8 6 5 7]
deviations : [-2.  2.  0. -1.  1.]  (sum = 0.0 )
squared    : [4. 4. 0. 1. 1.]

Population variance (÷n)   = 2.00   -> SD = 1.41
Sample variance     (÷n-1) = 2.50   -> SD = 1.58
NumPy check: np.std(x, ddof=1) = 1.58

Why n−1? The sample mean sits at the center of its own sample, so deviations from it run a touch too small. Dividing by the slightly smaller n−1 corrects that downward bias. For large samples the difference is tiny; it matters most when n is small.

DEMO 3 · THE 68-95-99.7 RULE

🔔 What the standard deviation means

For roughly bell-shaped data, about 68% of values fall within 1 SD of the mean, 95% within 2, and 99.7% within 3. The SD becomes a ruler for "how unusual is this?"

data = rng.normal(100, 15, 100_000)   # mean 100, SD 15 (like IQ scores)
mu, sd = data.mean(), data.std()
for k in (1, 2, 3):
    pct = np.mean(np.abs(data-mu) < k*sd) * 100
    cheb = (1 - 1/k**2) * 100 if k>1 else 0
    print(f"within {k} SD: {pct:4.1f}%  (empirical-rule target ~{[68,95,99.7][k-1]}%, Chebyshev floor ≥{cheb:.0f}%)")

within 1 SD: 68.2%  (empirical-rule target ~68%, Chebyshev floor ≥0%)
within 2 SD: 95.4%  (empirical-rule target ~95%, Chebyshev floor ≥75%)
within 3 SD: 99.7%  (empirical-rule target ~99.7%, Chebyshev floor ≥89%)

fig,ax=plt.subplots(figsize=(9,4.4))
ax.hist(data, bins=80, color=AMBER, alpha=0.55, edgecolor="white")
for k,c in [(1,PURPLE),(2,CYAN),(3,GREEN)]:
    ax.axvline(mu-k*sd, color=c, ls="--", lw=1.8)
    ax.axvline(mu+k*sd, color=c, ls="--", lw=1.8)
ax.axvline(mu, color=NAVY, lw=2.2)
titlecard(ax,"The Empirical Rule","±1 SD ≈ 68% · ±2 SD ≈ 95% · ±3 SD ≈ 99.7%  (bell-shaped data)")
ax.set_xlabel("value"); ax.set_yticks([])
plt.tight_layout(); plt.show()

Chebyshev's bonus: even for any shape of data (not just bell curves), at least 75% of values lie within 2 SD and at least 89% within 3 SD. Weaker guarantees, but they always hold. And z = (x − mean) / SD tells you how many SDs from the mean a value sits, the idea behind the z-score we use later.

DEMO 4 · THE OUTLIER TEST

📦 Box plots, and SD vs IQR

The box plot draws the five-number summary and flags outliers beyond 1.5×IQR. Watch how one outlier balloons the SD while the IQR barely moves.

salaries = pd.Series([42, 45, 48, 50, 47, 44, 46])      # $k
with_ceo = pd.concat([salaries, pd.Series([1000])], ignore_index=True)

def iqr(s): return s.quantile(.75) - s.quantile(.25)
print(f"Team only : SD {salaries.std(ddof=1):6.1f},  IQR {iqr(salaries):.1f}")
print(f"+ the CEO : SD {with_ceo.std(ddof=1):6.1f},  IQR {iqr(with_ceo):.1f}")
print("The SD exploded; the IQR hardly noticed. IQR is the robust choice.")

Team only : SD    2.6,  IQR 3.0
+ the CEO : SD  337.3,  IQR 3.8
The SD exploded; the IQR hardly noticed. IQR is the robust choice.

# Two views: zoomed in (so the boxes are readable) and full range (so the outlier shows)
fig,(a1,a2)=plt.subplots(1,2,figsize=(10,4.6))
for ax in (a1,a2):
    bp=ax.boxplot([salaries, with_ceo], patch_artist=True, widths=0.55,
                  medianprops=dict(color=NAVY, linewidth=2),
                  flierprops=dict(marker="o", markerfacecolor=PINK, markersize=8, markeredgecolor="white"))
    for patch,c in zip(bp["boxes"],[CYAN,AMBER]): patch.set_facecolor(c); patch.set_alpha(0.75)
    ax.set_xticks([1,2]); ax.set_xticklabels(["team only","+ the CEO"]); ax.set_ylabel("Salary ($k)")
a1.set_ylim(38, 56)
a1.set_title("Zoomed: the boxes barely differ", loc="left", fontsize=11.5, fontweight="bold", color=INK)
a2.set_ylim(0, 1060)
a2.set_title("Full range: one lone outlier", loc="left", fontsize=11.5, fontweight="bold", color=INK)
plt.tight_layout(); plt.show()

DEMO 5 · COEFFICIENT OF VARIATION

📊 Comparing spread across different scales

Is height more variable than weight? They use different units, so raw SD can't be compared. The coefficient of variation (CV = SD / mean) is unitless, so it can.

heights = rng.normal(170, 7,  500)    # cm
weights = rng.normal(70, 12,  500)    # kg

for name, s in [("Height (cm)", heights), ("Weight (kg)", weights)]:
    cv = s.std(ddof=1) / s.mean() * 100
    print(f"{name:<12}: mean {s.mean():6.1f}, SD {s.std(ddof=1):5.1f}, CV {cv:5.1f}%")
print("\nWeight has the larger CV, so it is RELATIVELY more variable, even though the numbers look smaller.")
print("Caution: CV only makes sense for positive data on a ratio scale (a true zero).")

Height (cm) : mean  169.7, SD   6.9, CV   4.1%
Weight (kg) : mean   70.1, SD  12.0, CV  17.1%

Weight has the larger CV, so it is RELATIVELY more variable, even though the numbers look smaller.
Caution: CV only makes sense for positive data on a ratio scale (a true zero).

🚚 Real-World Example: Two Carriers, Same Average

The measures so far used small arrays. Here they settle a real question: two shipping carriers average about the same delivery time, so which should you trust? The mean cannot tell them apart, but the spread can. Watch the standard deviation, IQR, range, and coefficient of variation separate the dependable carrier from the erratic one.

# --- Real-World beat: dispersion tells two same-mean carriers apart ---
BASE = "https://raw.githubusercontent.com/johnfisher-ai/Statistics-Data-Science-AI-Visual-Book/main/data/"
try:    deliv = pd.read_excel("../../data/measures-of-dispersion--delivery_times.xlsx", sheet_name="Data")
except FileNotFoundError: deliv = pd.read_excel(BASE+"measures-of-dispersion--delivery_times.xlsx", sheet_name="Data")
g = deliv.groupby("carrier").delivery_days
summary = g.agg(mean="mean", SD="std", IQR=lambda s: s.quantile(.75)-s.quantile(.25), rng=lambda s: s.max()-s.min())
summary["CV"] = g.std() / g.mean()
print(summary.round(2).to_string())
print("Same average, very different spread: SwiftShip is consistent, ValuePost is a gamble.")
fig, ax = plt.subplots(figsize=(9,4.2))
for c, col in [("SwiftShip", CYAN), ("ValuePost", AMBER)]:
    s = deliv[deliv.carrier==c].delivery_days
    ax.hist(s, bins=24, alpha=0.55, color=col, edgecolor="white", label=f"{c}  (mean {s.mean():.1f}, SD {s.std():.1f})")
ax.axvline(deliv.delivery_days.mean(), color=INK_SOFT, ls="--", lw=1.5)
ax.set_title("Same center, different spread: the mean hides reliability")
ax.set_xlabel("delivery days"); ax.set_ylabel("orders"); ax.legend()
plt.tight_layout(); plt.show()

           mean    SD   IQR  rng    CV
carrier                               
SwiftShip  4.94  0.81  1.17  3.9  0.16
ValuePost  5.03  2.32  3.40  9.6  0.46
Same average, very different spread: SwiftShip is consistent, ValuePost is a gamble.

---

🎓 Recap

• Dispersion tells you what the mean hides: how spread out the data is.
• Range and IQR are quick; IQR (and the box plot) resist outliers.
• Variance averages squared deviations; SD roots it back into real units. Use n−1 for a sample.
• For bell-shaped data the 68-95-99.7 rule turns SD into a ruler; Chebyshev gives looser limits for any shape.
• CV compares relative spread across different units, when the data is positive with a true zero.

Statistics, Data Science and AI: A Visual Handbook · © 2026 John Fisher

←Back to the bookChapter 9 · Measures of Dispersion