Merge pull request #58 from Renumics/feature/huggingface-integration

Feature/huggingface integration

Author: dani2112

pyproject.toml

@@ -21,8 +21,9 @@ dependencies = [
   "scikit-learn >= 1.2.2",
   "umap-learn >= 0.5.3",
   "tqdm >= 4.65.0",
-  "renumics-spotlight == 1.4.0rc2",
+  "renumics-spotlight >= 1.5.3",
   "datasets >= 2.13.1",
+  "puremagic >= 1.15"
 ]
 
 [project.optional-dependencies]

sliceguard/data.py

@@ -1,9 +1,11 @@
-from typing import List, Optional
+from os import rename
+from typing import List
 from pathlib import Path
 import pandas as pd
 import datasets
-from datasets import Image, ClassLabel, Value, Sequence
-
+from datasets import Image, Audio, ClassLabel, Value, Sequence
+import uuid
+import puremagic
 
 def _get_tutorial_imports():
     try:
@@ -15,43 +17,94 @@ def _get_tutorial_imports():
     return downloader
 
 
-def from_huggingface(dataset_identifier: str):
+def write_file(data: dict, suffix: str, data_dir: str):
+    with open(f"{data_dir}/{uuid.uuid4().hex}{suffix}", "wb") as tmp:
+        tmp.write(data["bytes"])
+        return tmp.name
+
+
+def convert_data(data: dict, data_dir: str):
+    """
+    Prefer raw data over path
+    """
+    if "bytes" in data and data['bytes'] is not None:
+        if len(data['bytes']) > 0:
+            suffix = puremagic.from_string(data['bytes'])
+            return write_file(data, suffix, data_dir)
+
+    if "path" in data and data['path'] is not None:
+        if data['path'] != "":
+            suffix = puremagic.from_file(data['path'])
+            new_path = f"{data['path']}{suffix}"
+
+            # In case of missing file extension
+            rename(data['path'], new_path)
+
+            return new_path
+
+
+# Tested with the following datasets:
+# Image:
+# "mnist"
+# "ceyda/smithsonian_butterflies"
+# "GabrielVidal/dead-by-daylight-perks"
+
+# Audio:
+# "437aewuh/dog-dataset"
+# "Gae8J/modeling"
+# "ccmusic-database/piano_sound_quality"
+
+# Text:
+# "xtreme", "XNLI"
+# "indonlp/indonlu", "smsa"
+# "tweet_eval", "emoji"
+
+
+def from_huggingface(dataset_identifier: str, name=None, split=None, extract_dir="./sliceguard_tmp"):
     # Simple utility method to support loading of huggingface datasets
-    # Currently only supports image data. Use custom load function if you need something else.
-    dataset = datasets.load_dataset(dataset_identifier)
+    dataset = datasets.load_dataset(dataset_identifier, name, split)
     overall_df = None
+
+    # Create missing directories if non-existent
+    Path(extract_dir).mkdir(parents=True, exist_ok=True)
+
+    # Iterate splits in dataset.
     for split in dataset.keys():
         cur_split = dataset[split]
 
         split_df = dataset[split].to_pandas()
         split_df["split"] = split
 
+        # Create a dataframe from each split.
         for fname, ftype in cur_split.features.items():
             if (
                 not isinstance(ftype, Image)
+                and not isinstance(ftype, Audio)
                 and not isinstance(ftype, ClassLabel)
                 and not isinstance(ftype, Value)
+                and not isinstance(ftype, list)
                 and not isinstance(ftype, Sequence)
             ):
                 raise RuntimeError(
                     f"Found unsupported datatype {ftype}. Use custom load function."
                 )
+
+            if isinstance(ftype, list):
+                split_df = split_df.drop(columns=fname)
+                print(
+                    f"Column {fname} with type {ftype} dropped. Lists are currently not supported."
+                )
+
             # Run transformations for specific data types if needed.
             if isinstance(ftype, ClassLabel):
                 class_label_lookup = {i: l for i, l in enumerate(ftype.names)}
                 split_df[fname] = split_df[fname].map(lambda x: class_label_lookup[x])
 
-            if isinstance(ftype, Image):
-                all_has_paths = all(
-                    x is not None and "path" in x for x in split_df[fname].values
-                )
-                if not all_has_paths:
-                    print(
-                        f"Column {fname} dropped. Images are not extracted onto harddrive. Currently this is not supported."
-                    )
-                    split_df = split_df.drop(columns=fname)
+            if isinstance(ftype, Image) or isinstance(ftype, Audio):
+                if any(x is None for x in split_df[fname].values):
+                    print("Column {fname} dropped due to None-type entries.")
                 else:
-                    split_df[fname] = split_df[fname].map(lambda x: x["path"])
+                    split_df[fname] = split_df[fname].map(lambda x: convert_data(x, extract_dir))
 
         if overall_df is None:
             overall_df = split_df

sliceguard/detection.py

@@ -322,7 +322,7 @@ def detect_issues(
     marked_issue_idx = 0
     for idx, row in all_groups_df.iterrows():
         if row["issue"] == True:
-            group_dfs[int(row["level"])].loc[idx] = True
+            group_dfs[int(row["level"])].loc[idx, "issue"] = True
 
             marked_issue_idx += 1
             if n_slices is not None and marked_issue_idx >= n_slices:

sliceguard/explanation.py

@@ -103,7 +103,13 @@ def explain_clusters(features, feature_types, issues, df, prereduced_embeddings)
                 predicate["maximum"] = val.max()
             elif feature_type in ["nominal", "ordinal"]:
                 val = df[f].iloc[issue_rows]
-                predicate["mode"] = val.mode()[0]
+                feature_mode = val.mode()
+                if len(feature_mode) == 0:
+                    predicate["mode"] = "no mode"
+                elif len(feature_mode == 1):
+                    predicate["mode"] = feature_mode[0]
+                else:
+                    raise RuntimeError("Invalid value encountered when calculating feature mode.")
             predicates_list.append(predicate)
 
         issue["explanation"] = predicates_list

sliceguard/sliceguard.py

@@ -2,8 +2,11 @@
 import warnings
 from numba.core.errors import NumbaDeprecationWarning, NumbaPendingDeprecationWarning
 
+# Ignore warnings caused by dependency umap-learn
 warnings.simplefilter("ignore", category=NumbaDeprecationWarning)
 warnings.simplefilter("ignore", category=NumbaPendingDeprecationWarning)
+# For now ignore warnings caused by dependency fairlearn. Remove once they address Pandas 2.0
+warnings.simplefilter(action='ignore', category=FutureWarning)
 
 # Real imports
 from uuid import uuid4

sliceguard/utils.py

@@ -282,7 +282,6 @@ def encode_normalize_features(
                     num_dimensions,
                 ),  # TODO: Do not hardcode this, probably determine based on embedding size and variance. Also, check implications on normalization.
                 # min_dist=0.0,
-                random_state=42,
                 set_op_mix_ratio=op_mix_ratio_prereduction,
             ).fit_transform(embeddings)
 

tests/test_huggingface.py

@@ -0,0 +1,202 @@
+import os
+import uuid
+import shutil
+from pathlib import Path
+from urllib.parse import urlparse
+
+from sklearn.metrics import accuracy_score
+import requests
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+from jiwer import wer
+import datasets
+from renumics.spotlight import Image, Audio
+from sliceguard import data
+
+from sliceguard import SliceGuard
+
+
+def wer_metric(y_true, y_pred):
+    return np.mean([wer(s_y, s_pred) for s_y, s_pred in zip(y_true, y_pred)])
+
+
+def test_huggingface_mnist():
+    df = data.from_huggingface("mnist")
+
+    sg = SliceGuard()
+    issue_df = sg.find_issues(
+        df.sample(100),
+        ["image"],
+        y="label",
+        metric=accuracy_score,
+        metric_mode="max",
+        min_support=10,
+        min_drop=0.08,
+    )
+
+    sg.report(spotlight_dtype={"image_path": Image})
+
+
+def test_huggingface_butterflies():
+    df = data.from_huggingface("ceyda/smithsonian_butterflies")
+
+    sg = SliceGuard()
+    issue_df = sg.find_issues(
+        df,
+        ["image"],
+        y="scientific_name",
+        metric=accuracy_score,
+        metric_mode="max",
+        min_support=10,
+        min_drop=0.08,
+        automl_train_split="train",
+        automl_task="classification",
+        automl_time_budget=40.0,
+    )
+
+    sg.report(spotlight_dtype={"image_path": Image})
+
+
+def test_huggingface_dead_by_daylight_perks():
+    df = data.from_huggingface("GabrielVidal/dead-by-daylight-perks")
+
+    sg = SliceGuard()
+    issue_df = sg.find_issues(
+        df,
+        ["image"],
+        y="type",
+        metric=accuracy_score,
+        metric_mode="max",
+        min_support=10,
+        min_drop=0.08,
+        automl_train_split="train",
+        automl_task="classification",
+        # automl_use_full_embeddings=True,
+        automl_time_budget=40.0,
+    )
+
+    sg.report(spotlight_dtype={"image_path": Image})
+
+
+def test_huggingface_dog_dataset():
+    df = data.from_huggingface("437aewuh/dog-dataset")
+
+    sg = SliceGuard()
+    issue_df = sg.find_issues(
+        df.sample(200),
+        ["audio"],
+        "label",
+        metric=accuracy_score,
+        metric_mode="max",
+        embedding_models={"path": "superb/wav2vec2-base-superb-sid"},
+        min_support=5,
+        min_drop=0.1,
+    )
+    sg.report(spotlight_dtype={"path": Audio})
+
+
+def test_huggingface_modeling():
+    df = data.from_huggingface("Gae8J/modeling")
+
+    sg = SliceGuard()
+    issue_df = sg.find_issues(
+        df.sample(200),
+        ["audio"],
+        "label",
+        metric=accuracy_score,
+        metric_mode="max",
+        automl_train_split="train",
+        automl_task="classification",
+        automl_time_budget=40.0,
+    )
+    sg.report(spotlight_dtype={"path": Audio})
+
+
+def test_huggingface_piano():
+    df = data.from_huggingface("ccmusic-database/piano_sound_quality")
+
+    sg = SliceGuard()
+    issue_df = sg.find_issues(
+        df.sample(200),
+        ["audio"],
+        "label",
+        metric=accuracy_score,
+        metric_mode="max",
+        automl_train_split="train",
+        automl_task="classification",
+        # automl_use_full_embeddings=True,
+        automl_time_budget=40.0,
+    )
+    sg.report(spotlight_dtype={"path": Audio})
+
+
+def test_huggingface_xtreme():
+    df = data.from_huggingface("xtreme", "XNLI")
+    sg = SliceGuard()
+    issue_df = sg.find_issues(
+            df.sample(1000),
+            ['language'],
+            "gold_label",
+            metric=accuracy_score,
+            min_drop=0.05,
+            min_support=10,
+            automl_task="classification",
+            automl_time_budget=40.0,
+        )
+    sg.report()
+
+
+def test_huggingface_indonlu():
+    df = data.from_huggingface("indonlp/indonlu", "smsa")
+    sg = SliceGuard()
+    issue_df = sg.find_issues(
+            df.sample(1000),
+            ['text'],
+            "label",
+            metric=accuracy_score,
+            min_drop=0.05,
+            min_support=10,
+            automl_train_split="train",
+            automl_task="classification",
+            automl_time_budget=40.0,
+        )
+    sg.report()
+
+
+def test_huggingface_tweet_eval():
+    df = data.from_huggingface("tweet_eval", "emoji")
+    sg = SliceGuard()
+    issue_df = sg.find_issues(
+            df.sample(1000),
+            ['text'],
+            "label",
+            metric=accuracy_score,
+            # metric_mode="max",
+            # wer_metric,
+            # metric_mode="min",
+            min_drop=0.05,
+            min_support=10,
+            # automl_split_key="",
+            automl_train_split="train",
+            automl_task="classification",
+            # automl_use_full_embeddings=True,
+            automl_time_budget=40.0,
+        )
+    sg.report()
+
+
+# Image:
+test_huggingface_mnist()
+# test_huggingface_butterflies()
+# test_huggingface_dead_by_daylight_perks()
+
+# Audio:
+# test_huggingface_dog_dataset()
+# test_huggingface_modeling()
+# test_huggingface_piano()
+
+# Text:
+# test_huggingface_xtreme()
+# test_huggingface_indonlu()
+# test_huggingface_tweet_eval()