import numpy as np, pandas as pd
from scipy import stats
from scipy.stats import norm
from statsmodels.stats.proportion import proportions_ztest, confint_proportions_2indep
BASE="https://raw.githubusercontent.com/johnfisher-ai/Statistics-Data-Science-AI-Visual-Book/main/data/"
rng=np.random.default_rng(820)

def n_per_arm(p1,p2,alpha=0.05,power=0.80):
    za=norm.ppf(1-alpha/2); zb=norm.ppf(power); pbar=(p1+p2)/2
    return ((za*np.sqrt(2*pbar*(1-pbar))+zb*np.sqrt(p1*(1-p1)+p2*(1-p2)))**2)/(p2-p1)**2
print(f"~{int(np.ceil(n_per_arm(0.12,0.14))):,} per arm")

~4,438 per arm

z,p=proportions_ztest([572,472],[4000,4000])
lo,hi=confint_proportions_2indep(572,4000,472,4000,method="wald")
print(f"z={z:.2f}, p={p:.4f}, 95% CI for lift [{lo*100:+.3f}, {hi*100:+.3f}] pts")

z=3.32, p=0.0009, 95% CI for lift [+1.025, +3.975] pts

def peek(checks=10,nmax=6000):
    a=rng.random(nmax)<0.12; b=rng.random(nmax)<0.12
    for n in np.linspace(nmax//checks,nmax,checks).astype(int):
        if 0 in (a[:n].sum(),b[:n].sum()): continue
        if proportions_ztest([b[:n].sum(),a[:n].sum()],[n,n])[1] < 0.05: return True
    return False
print(f"false-positive rate with peeking = {np.mean([peek() for _ in range(1200)]):.2f}")

false-positive rate with peeking = 0.19

a=np.where(rng.random(2000)<0.12, rng.normal(54,18,2000),0)
b=np.where(rng.random(2000)<0.14, rng.normal(57,18,2000),0)
print(f"Welch p={stats.ttest_ind(b,a,equal_var=False).pvalue:.4f}")

Welch p=0.1049

try:    e = pd.read_excel("../../data/a-b-testing-and-online-experiments--online_experiment.xlsx", sheet_name="Sessions")
except FileNotFoundError: e = pd.read_excel(BASE+"a-b-testing-and-online-experiments--online_experiment.xlsx", sheet_name="Sessions")
A=e[e.variant=="A"]; B=e[e.variant=="B"]; cA,nA,cB,nB=A.converted.sum(),len(A),B.converted.sum(),len(B)
z,p=proportions_ztest([cB,cA],[nB,nA]); lo,hi=confint_proportions_2indep(cB,nB,cA,nA,method="wald")
print(f"lift={(B.converted.mean()-A.converted.mean())*100:+.2f}pts, z={z:.2f}, p={p:.4f}, CI [{lo*100:+.2f},{hi*100:+.2f}]pts")

lift=+2.25pts, z=2.23, p=0.0256, CI [+0.28,+4.22]pts

⚙️ Setup¶