# Display Settings
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
from IPython.display import display, HTML

# Set Seed 
my_seed=1995

# Libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import json
from scipy import stats
from scipy.stats import (
    ttest_ind,
    ttest_rel,
    chi2_contingency,
    mannwhitneyu,
    levene,
    shapiro
)
import statsmodels.api as sm
from statsmodels.stats.power import (
    TTestIndPower,
    TTestPower,
    FTestAnovaPower,
    NormalIndPower
)
from statsmodels.stats.multitest import multipletests
from sklearn.model_selection import train_test_split

from ab_utils_01_data_setup import *
from ab_utils_02_power_analysis import *
from ab_utils_03_randomization import *
from ab_utils_04_aa_testing import *
from ab_utils_05_ab_testing import *
from ab_utils_06_post_hoc import *
import sys
sys.path.insert(0, '..')
from Hypothesis_Testing.ht_utils import print_config_summary

# 1. Main outcome variable you're testing
outcome_metric_col = 'engagement_score'

# 2. Metric type: 'binary', 'continuous', or 'categorical'
outcome_metric_datatype = 'continuous'

# 3. Group assignment (to be generated)
group_labels = ('control', 'treatment')

# 3b. Number of groups in the experiment (e.g., 2 for A/B test, 3 for A/B/C test)
group_count = len(group_labels)

# 4. Experimental design variant: independent or paired
variant = 'independent'  # Options: 'independent' (supported), 'paired' (not supported yet, todo)

# 5. Optional: Unique identifier for each observation (can be user_id, session_id, etc.)
observation_id_col = 'user_id'

# 6. Optional: Pre-experiment metric for CUPED, if used
pre_experiment_metric = 'past_purchase_count'  # Can be None

# Column name used to store assigned group after randomization
group_col = 'group'

# Randomization method to assign users to groups
# Options: 'simple', 'stratified', 'block', 'matched_pair', 'cluster', 'cuped'
randomization_method = "simple"

# Optional: guardrail metric column for simulated outcome data. Set to None to omit.
guardrail_metric_col = 'bounce_rate'

test_config = {
    # Core experiment setup
    'outcome_metric_col'     : outcome_metric_col,         # Main metric to analyze (e.g., 'engagement_score')
    'observation_id_col'     : observation_id_col,         # Unique identifier for each observation
    'pre_experiment_metric'  : pre_experiment_metric,      # Used for CUPED adjustment (if any)
    'outcome_metric_datatype': outcome_metric_datatype,    # One of: 'binary', 'continuous', 'categorical'
    'group_labels'           : group_labels,               # Tuple of (control, treatment) group names
    'group_count'            : group_count,                # Number of groups (usually 2 for A/B tests)
    'variant'                : variant,                    # 'independent' or 'paired'
    'guardrail_metric_col'   : guardrail_metric_col,       # Optional: e.g. 'bounce_rate'; None to omit

    # Diagnostic results — filled after EDA/assumptions check
    'normality'              : None,  # Will be set based on Shapiro-Wilk or visual tests
    'equal_variance'         : None,  # Will be set using Levene’s/Bartlett’s test
    'family'                 : None   # Test family → 'z_test', 't_test', 'anova', 'chi_square', etc.
}

print_config_summary(test_config)

📋 Hypothesis Test Configuration Summary

🔸 Outcome Metric Col        : engagement_score
🔸 Observation Id Col        : user_id
🔸 Pre Experiment Metric     : past_purchase_count
🔸 Outcome Metric Datatype   : continuous
🔸 Group Labels              : ('control', 'treatment')
🔸 Group Count               : 2
🔸 Variant                   : independent
🔸 Guardrail Metric Col      : bounce_rate
🔸 Normality                 : None
🔸 Equal Variance            : None
🔸 Family                    : None

observations_count = 1000
df = create_dummy_ab_data(observations_count, seed=my_seed, outcome_metric_col=outcome_metric_col, guardrail_metric_col=guardrail_metric_col)
historical_df = create_historical_df(df, outcome_metric_col, guardrail_metric_col, seed=my_seed)
df
historical_df

# Define Core Inputs

# Use values from your config or plug in manually
alpha = 0.05  # False positive tolerance (Type I error)
power = 0.80  # Statistical power (1 - Type II error)

# 🧮 Data-Driven Baseline Metric from historical data (stored in test_config; only relevant keys are set per test family)
_b = compute_baseline_from_data(historical_df, test_config)
test_config['baseline_rate'] = _b['baseline_rate']
test_config['baseline_mean'] = _b['baseline_mean']
test_config['std_dev'] = _b['std_dev']

📊 Baseline mean (historical): 49.16
📏 Baseline std dev (historical): 14.64

# Minimum Detectable Effect (MDE)
# This is NOT data-driven — it reflects the minimum improvement you care about detecting.
# It should be small enough to catch valuable changes, but large enough to avoid inflating sample size.

# Examples by Metric Type:
# - Binary       : 0.02 → detect a 2% lift in conversion rate (e.g., from 10% to 12%)
# - Categorical  : 0.05 → detect a 5% shift in plan preference (e.g., more users choosing 'premium' over 'basic')
# - Continuous   : 3.0  → detect a 3-point gain in engagement score (e.g., from 50 to 53 avg. score)

mde = 5  # TODO: Change this based on business relevance

test_config['family'] = determine_test_family(test_config)
# test_config
print_config_summary(test_config)

print(f"✅ Selected test family: {test_config['family']}")

📋 Hypothesis Test Configuration Summary

🔸 Outcome Metric Col        : engagement_score
🔸 Observation Id Col        : user_id
🔸 Pre Experiment Metric     : past_purchase_count
🔸 Outcome Metric Datatype   : continuous
🔸 Group Labels              : ('control', 'treatment')
🔸 Group Count               : 2
🔸 Variant                   : independent
🔸 Guardrail Metric Col      : bounce_rate
🔸 Normality                 : None
🔸 Equal Variance            : None
🔸 Family                    : mann_whitney_u_test
🔸 Baseline Rate             : None
🔸 Baseline Mean             : 49.16337003981812
🔸 Std Dev                   : 14.640142733622508
✅ Selected test family: mann_whitney_u_test

required_sample_size = calculate_power_sample_size(
    test_family=test_config['family'],
    variant=test_config.get('variant'),
    alpha=alpha,
    power=power,
    baseline_rate=test_config.get('baseline_rate'),
    mde=mde,
    std_dev=test_config.get('std_dev'),
    effect_size=None,  # Let it compute internally via mde/std
    num_groups=2
)

test_config['required_sample_size'] = required_sample_size
print(f"✅ Required sample size per group: {required_sample_size}")
print(f"👥 Total sample size: {required_sample_size * 2}")

✅ Required sample size per group: 136
👥 Total sample size: 272

print_power_summary(
    test_family=test_config['family'],
    variant=test_config.get('variant'),
    alpha=alpha,
    power=power,
    baseline_rate=test_config.get('baseline_rate'),
    mde=mde,
    std_dev=test_config.get('std_dev'),
    required_sample_size=required_sample_size
)

📈 Power Analysis Summary
- Test: MANN_WHITNEY_U_TEST (independent)
- Significance level (α): 0.05
- Statistical power (1 - β): 0.8
- Std Dev (baseline): 14.64
- MDE (mean difference): 5
- Cohen's d: 0.34

✅ To detect a 5-unit lift in mean outcome,
you need 136 users per group → total 272 users.

n_required = test_config['required_sample_size'] * test_config['group_count']
df = df.sample(n=n_required, random_state=42)

df.head()

# Apply randomization method
if randomization_method == "simple":
    df = apply_simple_randomization(df, group_col=group_col, seed=my_seed)

elif randomization_method == "stratified":
    df = apply_stratified_randomization(df, stratify_col='platform', group_col=group_col, seed=my_seed)

elif randomization_method == "block":
    df = apply_block_randomization(df, observation_id_col='user_id', group_col=group_col, block_size=10, seed=my_seed)

elif randomization_method == "matched_pair":
    df = apply_matched_pair_randomization(df, sort_col=pre_experiment_metric, group_col=group_col, group_labels=test_config['group_labels'])

elif randomization_method == "cluster":
    df = apply_cluster_randomization(df, cluster_col='city', group_col=group_col, seed=my_seed)

else:
    raise ValueError(f"❌ Unsupported randomization method: {randomization_method}")

# Randomization only assigns group. Outcome data is collected in the next section (Outcome data).
df

check_sample_ratio_mismatch(df, group_col=group_col, group_labels=test_config['group_labels'], expected_ratios=[0.5, 0.5], alpha=0.05)

🔍 Sample Ratio Mismatch (SRM) Check
Group control: 133 users (48.90%) — Expected: 136.0
Group treatment: 139 users (51.10%) — Expected: 136.0

Chi2 Statistic: 0.1324
P-value       : 0.7160
✅ No SRM — group sizes look balanced.

# Experiment has run; outcome and guardrail data come in (simulated via add_outcome_metrics). A/A: no treatment effect.
df = add_outcome_metrics(df, group_col=group_col, group_labels=test_config['group_labels'], outcome_metric_col=test_config['outcome_metric_col'], guardrail_metric_col=test_config.get('guardrail_metric_col') or guardrail_metric_col, treatment_effect=False, seed=my_seed)
df.head()

df.groupby("group")[test_config['outcome_metric_col']].mean() # TODO, move this into function

# Outcome similarity test only.
_ = run_outcome_similarity_test(
    df=df,
    group_col='group',
    metric_col=test_config['outcome_metric_col'],
    test_family=test_config['family'],
    variant=test_config.get('variant'),
    group_labels=test_config['group_labels'],
    alpha=0.05,
    verbose=True
)

group
control      46.639234
treatment    49.714740
Name: engagement_score, dtype: float64

📏 Outcome Similarity Check


🧠 Interpretation:
Used a Mann-Whitney U test to compare medians of 'engagement_score' across groups (non-parametric).
Null Hypothesis: Distributions are identical across groups.

We use α = 0.05
✅ p = 0.1748 ≥ α → Failed to reject null. No significant difference.

visualize_aa_distribution(
    df,
    group_col='group',
    metric_col=test_config['outcome_metric_col'],
    test_family=test_config['family'],
    group_labels=test_config['group_labels'],
    variant=test_config.get('variant')
)

_ = simulate_aa_type1_error_rate(
    df=df,
    metric_col=test_config['outcome_metric_col'],
    group_labels=test_config['group_labels'],
    test_family=test_config['family'],
    variant=test_config.get('variant'),
    runs=100,
    alpha=0.05
)

📈 Type I Error Rate Estimate: 1/100 = 1.00%

            🧠 Summary Interpretation:
            We simulated 100 A/A experiments using random group assignment (no actual treatment).

            Test: MANN_WHITNEY_U_TEST (independent)
            Metric: engagement_score
            Alpha: 0.05

            False positives (p < α): 1 / 100
            → Estimated Type I Error Rate: 1.00%

            This is within expected range for α = 0.05.
            → ✅ Test framework is behaving correctly — no bias or sensitivity inflation.

# Simulate experiment data (outcome + guardrail) with treatment effect for A/B analysis.
df = add_outcome_metrics(df, group_col=group_col, group_labels=test_config['group_labels'], outcome_metric_col=test_config['outcome_metric_col'], guardrail_metric_col=test_config.get('guardrail_metric_col') or guardrail_metric_col, treatment_effect=True, seed=my_seed)
df.head()

result = run_ab_test(
    df=df,
    group_col='group',
    metric_col=test_config['outcome_metric_col'],
    group_labels=test_config['group_labels'],
    test_family=test_config['family'],
    variant=test_config.get('variant'),
    alpha=0.05
)
print_config_summary(result) # TODO: Print nested json properly
# result

📋 Hypothesis Test Configuration Summary

🔸 Test Family    : mann_whitney_u_test
🔸 Variant        : independent
🔸 Group Labels   : ('control', 'treatment')
🔸 Alpha          : 0.05
🔸 Summary        : {'control': {'n': 133, 'mean': np.float64(46.639233969426925), 'std': 14.203174151273695, 'sum': None}, 'treatment': {'n': 139, 'mean': np.float64(54.66250808310677), 'std': 16.47955206608026, 'sum': None}}
🔸 Test           : Mann-Whitney U test
🔸 U Stat         : 6750.0
🔸 P Value        : 0.00012097959260162445

summarize_ab_test_result(result)

=============================================
🧪 A/B Test Result Summary [MANN_WHITNEY_U_TEST]
=============================================

📊 Hypothesis Test Result
Test used: Mann-Whitney U test
U-statistic: 6750.0000
P-value    : 0.0001
✅ Statistically significant difference detected.

📋 Group Summary:

📈 Lift Analysis
- Absolute Lift   : 8.0233
- Percentage Lift : 17.20%
- 95% CI for Lift : [4.3719, 11.6746]
=============================================

plot_ab_test_results(result)

📊 Visualization:

plot_confidence_intervals(result)

compute_lift_confidence_interval(result)

=============================================
📈 95% CI for Difference in Outcome [mann_whitney_u_test]
=============================================
- Absolute Lift (diff in means): 8.0233
- 95% CI for difference        : [4.3719, 11.6746]
✅ Likely positive impact (CI > 0)
=============================================

print_final_ab_test_summary(result)

========================================
          📊 FINAL A/B TEST SUMMARY
========================================
👥  Control Avg outcome         :  46.6392
🧪  Treatment Avg outcome         :  54.6625
📈  Absolute lift              :  8.0233
📊  Percentage lift            :  17.20%
🧪  P-value (from Mann-Whitney U test):  0.0001
----------------------------------------
✅ RESULT: Statistically significant difference detected.
========================================

daily_eligible_users = 1000
allocation_ratios = (0.5, 0.5)
buffer_days = 2

test_duration_result = estimate_test_duration(
    required_sample_size_per_group=test_config['required_sample_size'],
    daily_eligible_users=daily_eligible_users,
    allocation_ratios=allocation_ratios,
    buffer_days=buffer_days,
    test_family=test_config['family']
)
test_duration_result

🧮 Estimated Test Duration
- Test family               : mann_whitney_u_test
- Required sample per group : 136
- Daily eligible traffic    : 1000
- Allocation ratio          : (0.5, 0.5)
- Longest group runtime     : 1 days
- Buffer days               : 2
✅ Total estimated duration : 3 days

{'test_family': 'mann_whitney_u_test',
 'per_group_days': [np.float64(1.0), np.float64(1.0)],
 'longest_group_runtime': 1,
 'recommended_total_duration': 3}

analyze_segment_lift(
    df=df,
    test_config=test_config,
    segment_cols=['platform', 'device_type'], # , 'user_tier', 'region'
    min_count_per_group=30,
    visualize=True
)

🔎 Segmenting by: platform

🔎 Segmenting by: device_type

run_guardrail_analysis(df, test_config, group_col='group', alpha=0.05)

🚦 Guardrail Metric Check → 'bounce_rate'
Hypothesis (two-sided t-test): H₀ — no difference in mean vs H₁ — means differ.
- control   : 0.5534
- treatment : 0.5733
- Difference   : +0.0198
- P-value (t-test): 0.2849
🟡 No statistically significant change — guardrail looks stable.

df = apply_cuped(
    df=df,
    pre_metric='past_purchase_count',
    outcome_metric_col=test_config['outcome_metric_col'],
    group_col='group',
    group_labels=test_config['group_labels']
)

df.head()

# TODO: move these into apply_cuped()
original_std = df[test_config['outcome_metric_col']].std()
cuped_std = df[f"{test_config['outcome_metric_col']}_cuped_adjusted"].std()

print("Variance Reduction from CUPED")
print("--------------------------------")
print(f"Original std dev : {original_std:.3f}")
print(f"CUPED std dev    : {cuped_std:.3f}")
print(f"Reduction        : {(1 - cuped_std/original_std)*100:.2f}%")

Variance Reduction from CUPED
--------------------------------
Original std dev : 15.896
CUPED std dev    : 15.859
Reduction        : 0.24%

result_cuped = run_ab_test(
    df=df,
    group_col='group',
    metric_col=f"{test_config['outcome_metric_col']}_cuped_adjusted",
    group_labels=test_config['group_labels'],
    test_family=test_config['family'],
    variant=test_config.get('variant')
)

summarize_ab_test_result(result_cuped)

=============================================
🧪 A/B Test Result Summary [MANN_WHITNEY_U_TEST]
=============================================

📊 Hypothesis Test Result
Test used: Mann-Whitney U test
U-statistic: 6790.0000
P-value    : 0.0002
✅ Statistically significant difference detected.

📋 Group Summary:

📈 Lift Analysis
- Absolute Lift   : 7.8776
- Percentage Lift : 18.46%
- 95% CI for Lift : [4.2310, 11.5242]
=============================================

# Original inputs
segment_names = ['North', 'South', 'East', 'West']
p_vals = [0.03, 0.06, 0.02, 0.10]

# Create DataFrame and sort by raw p-values BEFORE correction
df_pvalues = pd.DataFrame({
    'Segment': segment_names,
    'Raw_pValue': p_vals
}).sort_values('Raw_pValue').reset_index(drop=True)

# Apply corrections to the sorted p-values
_, bonf, _, _ = multipletests(df_pvalues['Raw_pValue'], alpha=0.05, method='bonferroni')
_, bh, _, _ = multipletests(df_pvalues['Raw_pValue'], alpha=0.05, method='fdr_bh')

# Add to DataFrame
df_pvalues['Bonferroni_Adj_pValue'] = bonf
df_pvalues['BH_Adj_pValue'] = bh
df_pvalues

# TODO: decision from p-value?

#TODO: club with earlier cell, function possible?

# Plot p values - raw and adjusted
plt.figure(figsize=(8, 5))

# Plot lines
plt.plot(df_pvalues.index + 1, df_pvalues['Raw_pValue'], marker='o', label='Raw p-value')
plt.plot(df_pvalues.index + 1, df_pvalues['Bonferroni_Adj_pValue'], marker='^', label='Bonferroni Adj p-value')
plt.plot(df_pvalues.index + 1, df_pvalues['BH_Adj_pValue'], marker='s', label='BH Adj p-value')

# Add value labels next to each point
for i in range(len(df_pvalues)):
    x = i + 1
    plt.text(x + 0.05, df_pvalues['Raw_pValue'][i], f"{df_pvalues['Raw_pValue'][i]:.2f}", va='center')
    plt.text(x + 0.05, df_pvalues['Bonferroni_Adj_pValue'][i], f"{df_pvalues['Bonferroni_Adj_pValue'][i]:.2f}", va='center')
    plt.text(x + 0.05, df_pvalues['BH_Adj_pValue'][i], f"{df_pvalues['BH_Adj_pValue'][i]:.2f}", va='center')

# Axis & labels
plt.xticks(df_pvalues.index + 1, df_pvalues['Segment']);
plt.axhline(0.05, color='gray', linestyle='--', label='α = 0.05');
plt.xlabel("Segment (Ranked by Significance)");
plt.ylabel("p-value");
plt.title("p-value Correction: Bonferroni vs Benjamini Hochberg (FDR)");
plt.legend();
plt.tight_layout();
plt.show();

# Derive daily volume from actual data
daily_traffic_estimate = df.shape[0]  # Assuming full traffic per day

simulate_rollout_impact(
    experiment_result=result,                         # Output from run_ab_test()
    daily_eligible_observations=daily_traffic_estimate,
    metric_unit=test_config['outcome_metric_col']     # Dynamic label like 'engagement_score' or 'revenue'
)

📦 Rollout Simulation
- Outcome Metric      : engagement_score
- Observed Lift       : 8.0233 per unit
- Daily Eligible Units: 272
- Estimated Daily Impact   : 2,182 engagement_score/day
- Estimated Monthly Impact : 65,470 engagement_score/month

	user_id	engagement_score	bounce_rate	past_purchase_count	platform	device_type
0	1	NaN	NaN	37.593666	Android	mobile
1	2	NaN	NaN	35.294211	Android	mobile
2	3	NaN	NaN	71.011905	iOS	desktop
3	4	NaN	NaN	35.351784	Android	mobile
4	5	NaN	NaN	58.179221	Android	mobile
...	...	...	...	...	...	...
995	996	NaN	NaN	56.227450	iOS	mobile
996	997	NaN	NaN	57.966997	iOS	mobile
997	998	NaN	NaN	50.674093	Android	mobile
998	999	NaN	NaN	51.941265	iOS	mobile
999	1000	NaN	NaN	38.515887	iOS	desktop

	user_id	engagement_score	bounce_rate	past_purchase_count	platform	device_type
0	1	31.390500	0.560153	37.593666	Android	mobile
1	2	27.941317	0.547708	35.294211	Android	mobile
2	3	81.517858	0.452955	71.011905	iOS	desktop
3	4	28.027676	0.565883	35.351784	Android	mobile
4	5	62.268831	0.403212	58.179221	Android	mobile
...	...	...	...	...	...	...
995	996	59.341175	0.622611	56.227450	iOS	mobile
996	997	61.950496	0.422644	57.966997	iOS	mobile
997	998	51.011140	0.544316	50.674093	Android	mobile
998	999	52.911898	0.440618	51.941265	iOS	mobile
999	1000	32.773830	0.442153	38.515887	iOS	desktop

Parameter	Meaning
alpha (α)	Significance level (probability of false positive), e.g. 0.05
Power (1 - β)	Probability of detecting a true effect, e.g. 0.80 or 0.90
Baseline	Current outcome (e.g., 10% conversion, $50 revenue)
MDE	Minimum detectable effect — the smallest meaningful lift (e.g., +2% or +$5)
Std Dev	Standard deviation of the metric (for continuous outcomes)
Effect Size	Optional: Cohen's d (for t-tests) or f (for ANOVA)
Groups	Number of groups (relevant for ANOVA)

Outcome Metric	Normality	Group Count	Selected Test Family
binary	—	2	`z_test`
binary	—	3+	`chi_square`
continuous	✅	2	`t_test`
continuous	✅	3+	`anova`
continuous	❌	2	`non_parametric` (Mann-Whitney U)
continuous	❌	3+	`non_parametric` (Kruskal-Wallis)
categorical	—	2	`chi_square`
categorical	—	3+	`chi_square`

	user_id	engagement_score	bounce_rate	past_purchase_count	platform	device_type
521	522	NaN	NaN	25.729102	Android	mobile
737	738	NaN	NaN	51.861463	iOS	mobile
740	741	NaN	NaN	42.022993	Android	desktop
660	661	NaN	NaN	49.573649	Android	mobile
411	412	NaN	NaN	43.824528	iOS	mobile

	group	user_id	engagement_score	bounce_rate	past_purchase_count	platform	device_type
521	control	522	NaN	NaN	25.729102	Android	mobile
737	control	738	NaN	NaN	51.861463	iOS	mobile
740	control	741	NaN	NaN	42.022993	Android	desktop
660	treatment	661	NaN	NaN	49.573649	Android	mobile
411	treatment	412	NaN	NaN	43.824528	iOS	mobile
...	...	...	...	...	...	...	...
380	treatment	381	NaN	NaN	56.269061	iOS	mobile
631	control	632	NaN	NaN	61.199801	iOS	desktop
381	control	382	NaN	NaN	48.515995	iOS	mobile
490	control	491	NaN	NaN	55.112986	Android	mobile
118	control	119	NaN	NaN	61.520038	iOS	desktop

Strategy	Description
Cluster Randomization	Randomize at group level (e.g., friend group, region, org ID)
Isolation Experiments	Only roll out to fully disconnected segments (e.g., one region only)
Network-Based Metrics	Include network centrality / exposure as covariates
Post-Experiment Checks	Monitor if control group was exposed indirectly (e.g., referrals, shared UIs)
Simulation-Based Designs	Use agent-based or graph simulations to estimate contamination risk

Scenario	Split	Why
Default A/B	50 / 50	Maximizes power and ensures fairness
Risky feature	10 / 90 or 20 / 80	Limits user exposure to minimize risk
Ramp-up	Step-wise (1-5-25-50…)	Gradual rollout to catch issues early
A/B/C Test	33 / 33 / 33 or weighted	Compare multiple variants fairly or with bias
High control confidence needed	70 / 30 or 60 / 40	More stability in baseline comparisons

Result	Interpretation
No significant difference	✅ Randomization looks good. Test setup is sound.
Statistically significant difference	⚠️ Something’s off — check assignment logic, instrumentation, or sample leakage

	group	user_id	engagement_score	bounce_rate	past_purchase_count	platform	device_type
521	control	522	31.390500	0.572531	25.729102	Android	mobile
737	control	738	27.941317	0.484713	51.861463	iOS	mobile
740	control	741	81.517858	0.710801	42.022993	Android	desktop
660	treatment	661	28.027676	0.389262	49.573649	Android	mobile
411	treatment	412	62.268831	0.610558	43.824528	iOS	mobile

	group	user_id	engagement_score	bounce_rate	past_purchase_count	platform	device_type
521	control	522	31.390500	0.490186	25.729102	Android	mobile
737	control	738	27.941317	0.094631	51.861463	iOS	mobile
740	control	741	81.517858	0.531660	42.022993	Android	desktop
660	treatment	661	31.301273	0.564170	49.573649	Android	mobile
411	treatment	412	65.362110	0.572919	43.824528	iOS	mobile

	n	mean	std	sum
control	133.0	46.639234	14.203174	NaN
treatment	139.0	54.662508	16.479552	NaN

Area	What to Show
Segment Analysis	Table or chart showing lift per segment, sorted by effect size or risk
Guardrail Metrics	Summary table of guardrails vs baseline, with thresholds or annotations
Revenue Simulation	Projected uplift × traffic volume × conversion = business impact
Confidence Range	95% CI for key metrics per segment (wherever possible)
Rollout Readiness	Any blockers, mitigations, or next steps if full rollout is considered

	platform	count_control	count_treatment	mean_control	mean_treatment	std_control	std_treatment	lift	p_value_lift
0	Android	56	48	44.881922	53.659185	13.900282	16.633431	8.777262	None
1	iOS	77	91	47.917279	55.191734	14.374087	16.465483	7.274455	None

	device_type	count_control	count_treatment	mean_control	mean_treatment	std_control	std_treatment	lift	p_value_lift
0	mobile	87	106	46.029603	54.846641	13.205630	16.346295	8.817038	None
1	desktop	46	33	47.792232	54.071050	16.012127	17.144749	6.278819	None

Type	Examples
UX Health	Bounce Rate, Session Length, Engagement
Performance	Page Load Time, API Latency, CPU Usage
Reliability	Error Rate, Crash Rate, Timeout Errors
Behavioral	Scroll Depth, Page Views per Session

Method	Use Case	Risk
Bonferroni	Very strict, controls Family-Wise Error Rate (FWER)	❄️ Conservative
Benjamini-Hochberg	Controls False Discovery Rate (FDR)	🔥 Balanced

	Segment	Raw_pValue	Bonferroni_Adj_pValue	BH_Adj_pValue
0	East	0.02	0.08	0.06
1	North	0.03	0.12	0.06
2	South	0.06	0.24	0.08
3	West	0.10	0.40	0.10

	n	mean	std	sum
control	133.0	42.684917	14.179971	NaN
treatment	139.0	50.562556	16.462328	NaN

📖 AB Testing

🗂️ Data Setup

⚙️ Environment Setup

🛠️ Experiment Setup

🔧 Central Control Panel

📥 Read/Generate Data

⚡ Power Analysis

Why It Matters:

Key Inputs:

⚙️ Setup Inputs + Config Values

📈 Baseline Estimation (Pre-Experiment)

📈 Minimum Detectable Effect

🔍 Test Family

🧪 Experiment Type → Test Family Mapping

📐 Required Sample Size

🔀 Randomization

When to Use

How It Works

When to Use

How It Works

When to Use

How It Works

When to Use

How It Works

🎲 Apply Randomization

🕸️ Network Effects & SUTVA Violations

🧪 Why It Matters¶

⚠️ Common Violation Scenarios¶

🧩 Solutions (If You Suspect Interference)¶

⚖️ Sample Ratio Mismatch

🧪 AA Testing

🧬 Outcome Similarity Test

📊 AA Test Visualization

🎲 Type I Error Simulation

🔁 Repeated A/A Tests

📊 What this tells you:

🧪 A/B Testing

🧪 A/B Testing - Outcome Comparison

📌 Metric Tracked:

🔬 Outcome Analysis Steps:

🧾 Summaries

📊 Visualization

🎯 95% Confidence Intervalsfor outcome in groups

📈 Lift AnalysisAKA 95% Confidence Intervals for (difference in outcomes)

✅ Final Conclusion

⏱️ How Long

✅ Key Inputs

🧮 Formula

💡 Planning Tips

🧠 Real-World Considerations

🗣️ Common Practitioner Advice

🔍 Post Hoc Analysis

🧩 Segmented Lift

🚦 Guardrail Metrics

🧪 How We Check

📊 Common Guardrail Metrics

✅ When to Act

🧠 Why Guardrails Matter

🔄 CUPED

When to Use

How It Works

🧠 Correcting for Multiple Comparisons

📉 Example Scenario:¶

✅ Correction Methods¶

🧠 In Practice:¶

❄️ Bonferroni Correction¶

🔬 Benjamini-Hochberg (BH) Procedure¶

🪄 Novelty Effects & Behavioral Decay

🪄 Novelty Effects & Behavioral Decay¶

📉 Common Signs of Novelty Effects¶

🧭 What We Do About It¶

🎯 Primacy Effect & Order Bias

🚩 Common Indicators¶

🧭 What We Do About It¶

🎲 Rollout Simulation

🧪 A/B Test Holdouts

🧪 A/B Test Holdouts¶

🏢 Industry Practice¶

⚠️ When We Skip Holdouts¶

🚫 Limits & Alteratives

🎯 95% Confidence Intervals
for `outcome in groups`

📈 Lift Analysis
AKA 95% Confidence Intervals for (difference in outcomes)