Packages Installation#
First, install the holisticai package if you haven’t already:
!pip install holisticai[all]
Then, import the necessary libraries.
[2]:
import warnings
import pandas as pd
from holisticai.bias.metrics import multiclass_bias_metrics
from holisticai.datasets import load_dataset
from holisticai.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
warnings.filterwarnings("ignore")
Data Loading#
[4]:
dataset = load_dataset('us_crime_multiclass', protected_attribute='race')
train_test = dataset.train_test_split(test_size=0.2, random_state=42)
train = train_test['train']
test = train_test['test']
dataset
[4]:
[Dataset]
Instances: 123
Features: X , y , p_attrs , group_a , group_b
Metadata: race: {'group_a': 'racePctWhite>0.5', 'group_b': 'racePctWhite<=0.5'}
[8]:
model = LogisticRegression()
# set scaler
scaler = StandardScaler()
X_train = scaler.fit_transform(train['X'])
# apply mitigation and fit model
model.fit(X_train, train['y'])
# predict on test set
X_test = scaler.transform(test['X'])
y_pred = model.predict(X_test)
# compute bias metrics
metrics = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics
[8]:
| Value | Reference | |
|---|---|---|
| Metric | ||
| Max Multiclass Statistical Parity | 0.857143 | 0 |
| Mean Multiclass Statistical Parity | 0.857143 | 0 |
| Max Multiclass Equality of Opportunity | 0.689394 | 0 |
| Max Multiclass Average Odds | 0.608586 | 0 |
| Max Multiclass True Positive Difference | 0.578283 | 0 |
| Mean Multiclass Equality of Opportunity | 0.689394 | 0 |
| Mean Multiclass Average Odds | 0.608586 | 0 |
| Mean Multiclass True Positive Difference | 0.578283 | 0 |
1. Correlation Remover#
Traditional implementation#
[9]:
from holisticai.bias.mitigation import CorrelationRemover
preprocessing_mitigator = CorrelationRemover()
model = LogisticRegression()
# set scaler
scaler = StandardScaler()
X_train = scaler.fit_transform(train['X'])
# apply mitigation and fit model
X_p_train = preprocessing_mitigator.fit_transform(X_train, train['group_a'], train['group_b'])
model.fit(X_p_train, train['y'])
# predict on test set
X_test = scaler.transform(test['X'])
X_p_test = preprocessing_mitigator.transform(X_test, test['group_a'], test['group_b'])
y_pred = model.predict(X_p_test)
# compute bias metrics
metrics = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics
[9]:
| Value | Reference | |
|---|---|---|
| Metric | ||
| Max Multiclass Statistical Parity | 0.511905 | 0 |
| Mean Multiclass Statistical Parity | 0.511905 | 0 |
| Max Multiclass Equality of Opportunity | 0.378788 | 0 |
| Max Multiclass Average Odds | 0.358586 | 0 |
| Max Multiclass True Positive Difference | 0.378788 | 0 |
| Mean Multiclass Equality of Opportunity | 0.378788 | 0 |
| Mean Multiclass Average Odds | 0.358586 | 0 |
| Mean Multiclass True Positive Difference | 0.378788 | 0 |
Pipeline implementation#
[10]:
mitigator = CorrelationRemover()
# set pipeline
pipeline = Pipeline(steps=[('scalar', StandardScaler()),("bm_preprocessing", mitigator), ("model", LogisticRegression())])
pipeline.fit(train['X'], train['y'], bm__group_a=train['group_a'], bm__group_b=train['group_b'])
# predict on test set
y_pred = pipeline.predict(test['X'], bm__group_a=test['group_a'], bm__group_b=test['group_b'])
# compute bias metrics
metrics_pipeline = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics_pipeline
[10]:
| Value | Reference | |
|---|---|---|
| Metric | ||
| Max Multiclass Statistical Parity | 0.511905 | 0 |
| Mean Multiclass Statistical Parity | 0.511905 | 0 |
| Max Multiclass Equality of Opportunity | 0.378788 | 0 |
| Max Multiclass Average Odds | 0.358586 | 0 |
| Max Multiclass True Positive Difference | 0.378788 | 0 |
| Mean Multiclass Equality of Opportunity | 0.378788 | 0 |
| Mean Multiclass Average Odds | 0.358586 | 0 |
| Mean Multiclass True Positive Difference | 0.378788 | 0 |
Comparison#
[11]:
pd.concat([metrics['Value'], metrics_pipeline], axis=1, keys=['Traditional', 'Pipeline'])
[11]:
| Traditional | Pipeline | ||
|---|---|---|---|
| Value | Value | Reference | |
| Metric | |||
| Max Multiclass Statistical Parity | 0.511905 | 0.511905 | 0 |
| Mean Multiclass Statistical Parity | 0.511905 | 0.511905 | 0 |
| Max Multiclass Equality of Opportunity | 0.378788 | 0.378788 | 0 |
| Max Multiclass Average Odds | 0.358586 | 0.358586 | 0 |
| Max Multiclass True Positive Difference | 0.378788 | 0.378788 | 0 |
| Mean Multiclass Equality of Opportunity | 0.378788 | 0.378788 | 0 |
| Mean Multiclass Average Odds | 0.358586 | 0.358586 | 0 |
| Mean Multiclass True Positive Difference | 0.378788 | 0.378788 | 0 |
2. Disparate Impact Remover#
Traditional Implementation#
[13]:
from holisticai.bias.mitigation import DisparateImpactRemover
preprocessing_mitigator = DisparateImpactRemover()
model = LogisticRegression()
# set scaler
scaler = StandardScaler()
X_train = scaler.fit_transform(train['X'])
# fit mitgator and model
X_p_train = preprocessing_mitigator.fit_transform(X_train, group_a=train['group_a'], group_b=train['group_b'])
model.fit(X_p_train, train['y'])
# predict on test set
X_test = scaler.transform(test['X'])
X_p_test = preprocessing_mitigator.transform(X_test, group_a=test['group_a'], group_b=test['group_b'])
y_pred = model.predict(X_p_test)
# compute bias metrics
metrics = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics
[13]:
| Value | Reference | |
|---|---|---|
| Metric | ||
| Max Multiclass Statistical Parity | 0.416667 | 0 |
| Mean Multiclass Statistical Parity | 0.416667 | 0 |
| Max Multiclass Equality of Opportunity | 0.406566 | 0 |
| Max Multiclass Average Odds | 0.270202 | 0 |
| Max Multiclass True Positive Difference | 0.406566 | 0 |
| Mean Multiclass Equality of Opportunity | 0.406566 | 0 |
| Mean Multiclass Average Odds | 0.270202 | 0 |
| Mean Multiclass True Positive Difference | 0.406566 | 0 |
Pipeline Implementation#
[14]:
mitigator = DisparateImpactRemover()
# set pipeline
pipeline = Pipeline(steps=[('scalar', StandardScaler()),("bm_preprocessing", mitigator), ("model", LogisticRegression())])
pipeline.fit(train['X'], train['y'], bm__group_a=train['group_a'], bm__group_b=train['group_b'])
# predict on test set
y_pred = pipeline.predict(test['X'], bm__group_a=test['group_a'], bm__group_b=test['group_b'])
# compute bias metrics
metrics_pipeline = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics_pipeline
[14]:
| Value | Reference | |
|---|---|---|
| Metric | ||
| Max Multiclass Statistical Parity | 0.416667 | 0 |
| Mean Multiclass Statistical Parity | 0.416667 | 0 |
| Max Multiclass Equality of Opportunity | 0.406566 | 0 |
| Max Multiclass Average Odds | 0.270202 | 0 |
| Max Multiclass True Positive Difference | 0.406566 | 0 |
| Mean Multiclass Equality of Opportunity | 0.406566 | 0 |
| Mean Multiclass Average Odds | 0.270202 | 0 |
| Mean Multiclass True Positive Difference | 0.406566 | 0 |
Comparison#
[15]:
pd.concat([metrics['Value'], metrics_pipeline], axis=1, keys=['Traditional', 'Pipeline'])
[15]:
| Traditional | Pipeline | ||
|---|---|---|---|
| Value | Value | Reference | |
| Metric | |||
| Max Multiclass Statistical Parity | 0.416667 | 0.416667 | 0 |
| Mean Multiclass Statistical Parity | 0.416667 | 0.416667 | 0 |
| Max Multiclass Equality of Opportunity | 0.406566 | 0.406566 | 0 |
| Max Multiclass Average Odds | 0.270202 | 0.270202 | 0 |
| Max Multiclass True Positive Difference | 0.406566 | 0.406566 | 0 |
| Mean Multiclass Equality of Opportunity | 0.406566 | 0.406566 | 0 |
| Mean Multiclass Average Odds | 0.270202 | 0.270202 | 0 |
| Mean Multiclass True Positive Difference | 0.406566 | 0.406566 | 0 |
3. Reweighing#
Traditional Implementation#
[16]:
from holisticai.bias.mitigation import Reweighing
mitigator = Reweighing()
model = LogisticRegression()
# set scaler
scaler = StandardScaler()
X_train = scaler.fit_transform(train['X'])
# fit mitigator and model
mitigator.fit(train['y'], group_a=train['group_a'], group_b=train['group_b'])
sw = mitigator.estimator_params["sample_weight"]
model.fit(X_train, train['y'], sw)
# predict on test set
X_test = scaler.transform(test['X'])
X_p_test = mitigator.transform(X_test, group_a=test['group_a'], group_b=test['group_b'])
y_pred = model.predict(X_p_test)
# Compute metrics
metrics = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics
[16]:
| Value | Reference | |
|---|---|---|
| Metric | ||
| Max Multiclass Statistical Parity | 0.809524 | 0 |
| Mean Multiclass Statistical Parity | 0.809524 | 0 |
| Max Multiclass Equality of Opportunity | 0.633838 | 0 |
| Max Multiclass Average Odds | 0.553030 | 0 |
| Max Multiclass True Positive Difference | 0.522727 | 0 |
| Mean Multiclass Equality of Opportunity | 0.633838 | 0 |
| Mean Multiclass Average Odds | 0.553030 | 0 |
| Mean Multiclass True Positive Difference | 0.522727 | 0 |
Pipeline Implementation#
[17]:
mitigator = Reweighing()
# set pipeline
pipeline = Pipeline(steps=[('scalar', StandardScaler()),("bm_preprocessing", mitigator), ("model", LogisticRegression())])
pipeline.fit(train['X'], train['y'], bm__group_a=train['group_a'], bm__group_b=train['group_b'])
# predict on test set
y_pred = pipeline.predict(test['X'], bm__group_a=test['group_a'], bm__group_b=test['group_b'])
# compute bias metrics
metrics_pipeline = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics_pipeline
[17]:
| Value | Reference | |
|---|---|---|
| Metric | ||
| Max Multiclass Statistical Parity | 0.809524 | 0 |
| Mean Multiclass Statistical Parity | 0.809524 | 0 |
| Max Multiclass Equality of Opportunity | 0.633838 | 0 |
| Max Multiclass Average Odds | 0.553030 | 0 |
| Max Multiclass True Positive Difference | 0.522727 | 0 |
| Mean Multiclass Equality of Opportunity | 0.633838 | 0 |
| Mean Multiclass Average Odds | 0.553030 | 0 |
| Mean Multiclass True Positive Difference | 0.522727 | 0 |
Comparison#
[18]:
pd.concat([metrics['Value'], metrics_pipeline], axis=1, keys=['Traditional', 'Pipeline'])
[18]:
| Traditional | Pipeline | ||
|---|---|---|---|
| Value | Value | Reference | |
| Metric | |||
| Max Multiclass Statistical Parity | 0.809524 | 0.809524 | 0 |
| Mean Multiclass Statistical Parity | 0.809524 | 0.809524 | 0 |
| Max Multiclass Equality of Opportunity | 0.633838 | 0.633838 | 0 |
| Max Multiclass Average Odds | 0.553030 | 0.553030 | 0 |
| Max Multiclass True Positive Difference | 0.522727 | 0.522727 | 0 |
| Mean Multiclass Equality of Opportunity | 0.633838 | 0.633838 | 0 |
| Mean Multiclass Average Odds | 0.553030 | 0.553030 | 0 |
| Mean Multiclass True Positive Difference | 0.522727 | 0.522727 | 0 |