Using ML anonymization to defend against attribute inference attacks

Load data

import pandas as pd
import numpy as np
from sklearn.metrics import accuracy_score

import warnings
# Filter out all warnings
warnings.filterwarnings("ignore")

First of all, we need to import the required packages to perform our privacy analysis and mitigation. You will need to have the holisticai package installed on your system, remember that you can install it by running:

!pip install holisticai[all]

from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from holisticai.utils import BinaryClassificationProxy
from holisticai.security.commons import DataMinimizer
from holisticai.security.metrics import classification_privacy_metrics

# Identify categorical and numerical features
def create_preprocessor(X):
    categorical_features = X.select_dtypes(include=['category']).columns
    numerical_fatures = X.select_dtypes(exclude=['category']).columns

    # Create transformers for numerical and categorical features
    numeric_transformer = Pipeline(steps=[('scaler', StandardScaler())])
    categorical_transformer = Pipeline(steps=[('onehot', OneHotEncoder(handle_unknown='ignore'))])

    # Combine transformers into a preprocessor using ColumnTransformer
    return ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numerical_fatures),
            ('cat', categorical_transformer, categorical_features)
    ])

def evaluate_privacy(model, train, test):
    proxy = BinaryClassificationProxy(predict=model.predict, predict_proba=model.predict_proba, classes=[0, 1])
    dmin = DataMinimizer(proxy=proxy)
    dmin.fit(train['X'], train['y'])

    y_pred_train = proxy.predict(train['X'])
    y_pred_test = proxy.predict(test['X'])
    y_pred_test_dm = dmin.predict(test['X'])

    return classification_privacy_metrics(train['X'], test['X'], train['y'], test['y'], y_pred_train, y_pred_test, y_pred_test_dm, attribute_attack='education')

Train decision tree model

from sklearn.tree import DecisionTreeClassifier
from holisticai.datasets import load_dataset
from sklearn.pipeline import Pipeline

dataset = load_dataset('adult', preprocessed=False)
dataset['X'] = dataset["X"].drop('fnlwgt', axis=1)
train_test = dataset.train_test_split(0.2, random_state=42)
train = train_test['train']
test = train_test['test']

preprocessor = create_preprocessor(train['X'])
model = Pipeline(steps=[('preprocessor', preprocessor),
                        ('classifier', DecisionTreeClassifier())])

model.fit(train['X'], train['y'])

y_pred = model.predict(test['X'])
y_proba = model.predict_proba(test['X'])

print('Base model accuracy: ', model.score(test['X'], test['y']))

Base model accuracy:  0.8199004975124378

evaluate_privacy(model, train, test)

An NVIDIA GPU may be present on this machine, but a CUDA-enabled jaxlib is not installed. Falling back to cpu.

	metric	value	reference
0	SHAPr	0.819901	0.0
1	Data Minimization Accuracy Ratio	1.012700	inf
2	Attribute Attack Score	0.405970	0.0

Anonymized data. Improving privacy

from holisticai.security.mitigation import Anonymize

X_train = train['X']
y_train = train['y']

feature_names = X_train.columns
categorical_features = list(X_train.select_dtypes(include=['category']).columns)
QI = ['education', 'marital-status', 'age']

anonymizer = Anonymize(500, QI, categorical_features=categorical_features, features_names=feature_names)
anon = anonymizer.anonymize(X_train, y_train)

from holisticai.datasets import Dataset

new_train = Dataset(X=anon, y=y_train)

Train decision tree model on anonymized data

from sklearn.tree import DecisionTreeClassifier

preprocessor = create_preprocessor(new_train['X'])
model = Pipeline(steps=[('preprocessor', preprocessor),
                        ('classifier', DecisionTreeClassifier())])

model.fit(anon, new_train['y'])

y_pred = model.predict(test['X'])
y_proba = model.predict_proba(test['X'])

print('Base model accuracy: ', model.score(test['X'], test['y']))

Base model accuracy:  0.802653399668325

evaluate_privacy(model, new_train, test)

	metric	value	reference
0	SHAPr	0.802653	0.0
1	Data Minimization Accuracy Ratio	0.998762	inf
2	Attribute Attack Score	0.352902	0.0