Packages Installation

First, install the holisticai package if you haven’t already:

!pip install holisticai[all]

Then, import the necessary libraries.

import warnings

import pandas as pd
import numpy as np
from holisticai.bias.metrics import multiclass_bias_metrics
from holisticai.datasets import load_dataset
from holisticai.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression

warnings.filterwarnings("ignore")

Data Loading

dataset = load_dataset('student_multiclass', protected_attribute='sex')
train_test = dataset.train_test_split(test_size=0.2, random_state=100, stratify=dataset['y'])

train = train_test['train']
test = train_test['test']

dataset

[-] [Dataset]

Instances: 395

Features: X , y , p_attrs , group_a , group_b

Metadata: sex: {'group_a': 'F', 'group_b': 'M'}

from holisticai.bias.mitigation import LPDebiaserMulticlass

model = LogisticRegression()

# set scaler
scaler = StandardScaler()
X_train = scaler.fit_transform(train['X'])

# fit model and predict
model.fit(X_train, train['y'])
y_pred = model.predict(X_train)

# fit mitigator
X_test = scaler.transform(test['X'])

# predict and mitigate
y_pred = model.predict(X_test)

# compute bias metrics
metrics = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics

	Value	Reference
Metric
Max Multiclass Statistical Parity	0.147619	0
Mean Multiclass Statistical Parity	0.147619	0
Max Multiclass Equality of Opportunity	0.232407	0
Max Multiclass Average Odds	0.199074	0
Max Multiclass True Positive Difference	0.190741	0
Mean Multiclass Equality of Opportunity	0.232407	0
Mean Multiclass Average Odds	0.199074	0
Mean Multiclass True Positive Difference	0.190741	0

1. LP Debiaser Multiclass

Traditional implementation

from holisticai.bias.mitigation import LPDebiaserMulticlass

mitigator = LPDebiaserMulticlass(constraint="EqualizedOpportunity")
mitigator

[-] [LPDebiaserMulticlass]

LPDebiaserMulticlass(constraint=EqualizedOpportunity, loss=macro)

---

Type: Bias Mitigation Postprocessing

model = LogisticRegression()

# set scaler
scaler = StandardScaler()
X_train = scaler.fit_transform(train['X'])

# fit model and predict
model.fit(X_train, train['y'])
y_pred = model.predict(X_train)

# fit mitigator
mitigator.fit(train['y'], y_pred, group_a=train['group_a'], group_b=train['group_b'])

X_test = scaler.transform(test['X'])

# predict and mitigate
y_pred = model.predict(X_test)
y_pred = mitigator.transform(y_pred, group_a=test['group_a'], group_b=test['group_b'])['y_pred']

# compute bias metrics
metrics = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics

	Value	Reference
Metric
Max Multiclass Statistical Parity	0.123129	0
Mean Multiclass Statistical Parity	0.123129	0
Max Multiclass Equality of Opportunity	0.225463	0
Max Multiclass Average Odds	0.179167	0
Max Multiclass True Positive Difference	0.136574	0
Mean Multiclass Equality of Opportunity	0.225463	0
Mean Multiclass Average Odds	0.179167	0
Mean Multiclass True Positive Difference	0.136574	0

from holisticai.bias.mitigation import LPDebiaserMulticlass

mitigator = LPDebiaserMulticlass(constraint="EqualizedOpportunity")
model = LogisticRegression()

# set scaler
scaler = StandardScaler()
X_train = scaler.fit_transform(train['X'])

# fit model and predict
model.fit(X_train, train['y'])
y_pred = model.predict(X_train)

# fit mitigator
mitigator.fit(train['y'], y_pred, group_a=train['group_a'], group_b=train['group_b'])
X_test = scaler.transform(test['X'])

# predict and mitigate
y_pred = model.predict(X_test)
y_pred = mitigator.transform(y_pred, group_a=test['group_a'], group_b=test['group_b'])['y_pred']

# compute bias metrics
metrics = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics

	Value	Reference
Metric
Max Multiclass Statistical Parity	0.123129	0
Mean Multiclass Statistical Parity	0.123129	0
Max Multiclass Equality of Opportunity	0.225463	0
Max Multiclass Average Odds	0.179167	0
Max Multiclass True Positive Difference	0.136574	0
Mean Multiclass Equality of Opportunity	0.225463	0
Mean Multiclass Average Odds	0.179167	0
Mean Multiclass True Positive Difference	0.136574	0

Pipeline Implementation

mitigator = LPDebiaserMulticlass(constraint="EqualizedOpportunity", loss="macro")
model = LogisticRegression()

# set pipeline
pipeline = Pipeline(steps=[('scalar', StandardScaler()), ("model", model), ("bm_postprocessing", mitigator)])
pipeline.fit(train['X'], train['y'], bm__group_a=train['group_a'], bm__group_b=train['group_b'])
pipeline

[-] [Pipeline]

[-] scalar [StandardScaler]

StandardScaler(copy=True, with_mean=True, with_std=True)

[-] model [LogisticRegression]

LogisticRegression(penalty=l2, dual=False, tol=0.0001, C=1.0, ...)

[-] bm_postprocessing [LPDebiaserMulticlass]

LPDebiaserMulticlass(constraint=EqualizedOpportunity, loss=macro)

# predict on test set
y_pred = pipeline.predict(test['X'], bm__group_a=test['group_a'], bm__group_b=test['group_b'])

# compute bias metrics
metrics_pipeline = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics_pipeline

	Value	Reference
Metric
Max Multiclass Statistical Parity	0.123129	0
Mean Multiclass Statistical Parity	0.123129	0
Max Multiclass Equality of Opportunity	0.225463	0
Max Multiclass Average Odds	0.179167	0
Max Multiclass True Positive Difference	0.136574	0
Mean Multiclass Equality of Opportunity	0.225463	0
Mean Multiclass Average Odds	0.179167	0
Mean Multiclass True Positive Difference	0.136574	0

Comparison

pd.concat([metrics['Value'], metrics_pipeline], axis=1, keys=['Traditional', 'Pipeline'])

	Traditional	Pipeline
	Value	Value	Reference
Metric
Max Multiclass Statistical Parity	0.123129	0.123129	0
Mean Multiclass Statistical Parity	0.123129	0.123129	0
Max Multiclass Equality of Opportunity	0.225463	0.225463	0
Max Multiclass Average Odds	0.179167	0.179167	0
Max Multiclass True Positive Difference	0.136574	0.136574	0
Mean Multiclass Equality of Opportunity	0.225463	0.225463	0
Mean Multiclass Average Odds	0.179167	0.179167	0
Mean Multiclass True Positive Difference	0.136574	0.136574	0

2. ML Debiser Multiclass

Traditional Implementation

from holisticai.bias.mitigation import MLDebiaser

mitigator = MLDebiaser(gamma=0.1)
model = LogisticRegression()

# set scaler
scaler = StandardScaler()
X_train = scaler.fit_transform(train['X'])

# fit model and predict probabilities
model.fit(X_train, train['y'])
y_pred = model.predict_proba(X_train)

# fit mitigator
mitigator.fit(y_pred, group_a=train['group_a'], group_b=train['group_b'])
X_test = scaler.transform(test['X'])

# model predict and mitigator transform
y_pred = model.predict_proba(X_test)
y_pred = mitigator.transform(y_pred, group_a=test['group_a'], group_b=test['group_b'])['y_pred']

# compute bias metrics
metrics = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics

[elapsed time: 00:00:07 | iter:5/5 | primal_residual::3.0970 | dual_residual::0.2351]

	Value	Reference
Metric
Max Multiclass Statistical Parity	0.083673	0
Mean Multiclass Statistical Parity	0.083673	0
Max Multiclass Equality of Opportunity	0.188426	0
Max Multiclass Average Odds	0.100463	0
Max Multiclass True Positive Difference	0.188426	0
Mean Multiclass Equality of Opportunity	0.188426	0
Mean Multiclass Average Odds	0.100463	0
Mean Multiclass True Positive Difference	0.188426	0

Pipeline Implementation

mitigator = MLDebiaser(gamma=0.1)
model = LogisticRegression()

# set pipeline
pipeline = Pipeline(steps=[('scalar', StandardScaler()), ("model", model), ("bm_postprocessing", mitigator)])
pipeline.fit(train['X'], train['y'], bm__group_a=train['group_a'], bm__group_b=train['group_b'])

# predict on test set
y_pred = pipeline.predict(test['X'], bm__group_a=test['group_a'], bm__group_b=test['group_b'])

# compute bias metrics
metrics_pipeline = multiclass_bias_metrics(test['group_a'], y_pred, test['y'], metric_type='both')
metrics_pipeline

[elapsed time: 00:00:07 | iter:5/5 | primal_residual::3.0970 | dual_residual::0.2351]

	Value	Reference
Metric
Max Multiclass Statistical Parity	0.083673	0
Mean Multiclass Statistical Parity	0.083673	0
Max Multiclass Equality of Opportunity	0.188426	0
Max Multiclass Average Odds	0.100463	0
Max Multiclass True Positive Difference	0.188426	0
Mean Multiclass Equality of Opportunity	0.188426	0
Mean Multiclass Average Odds	0.100463	0
Mean Multiclass True Positive Difference	0.188426	0

Comparison

pd.concat([metrics['Value'], metrics_pipeline], axis=1, keys=['Traditional', 'Pipeline'])

	Traditional	Pipeline
	Value	Value	Reference
Metric
Max Multiclass Statistical Parity	0.083673	0.083673	0
Mean Multiclass Statistical Parity	0.083673	0.083673	0
Max Multiclass Equality of Opportunity	0.188426	0.188426	0
Max Multiclass Average Odds	0.100463	0.100463	0
Max Multiclass True Positive Difference	0.188426	0.188426	0
Mean Multiclass Equality of Opportunity	0.188426	0.188426	0
Mean Multiclass Average Odds	0.100463	0.100463	0
Mean Multiclass True Positive Difference	0.188426	0.188426	0