{ style="display:block; max-width:100%; height:auto; margin:0 auto;" }

Generation Demonstration¶

This notebook show case what can be done with interpreto for generation.

author: Antonin Poché

0. Imports and model loading¶

In [ ]:

import os

import datasets
import transformers

import interpreto
from interpreto import KernelShap, plot_attributions
from interpreto.attributions.metrics import Deletion
from interpreto.concepts import LLMLabels, SemiNMFConcepts, TopKInputs
from interpreto.concepts.metrics import MSE, Sparsity

import os import datasets import transformers import interpreto from interpreto import KernelShap, plot_attributions from interpreto.attributions.metrics import Deletion from interpreto.concepts import LLMLabels, SemiNMFConcepts, TopKInputs from interpreto.concepts.metrics import MSE, Sparsity

In [3]:

# load dataset examples
dataset = datasets.load_dataset("dair-ai/emotion", "split")["train"]["text"][:1000]

# load dataset examples dataset = datasets.load_dataset("dair-ai/emotion", "split")["train"]["text"][:1000]

In [4]:

# load the model and tokenizer
tokenizer = transformers.AutoTokenizer.from_pretrained("gpt2")
model = transformers.AutoModelForCausalLM.from_pretrained("gpt2").cuda()

# load the model and tokenizer tokenizer = transformers.AutoTokenizer.from_pretrained("gpt2") model = transformers.AutoModelForCausalLM.from_pretrained("gpt2").cuda()

1. Attribution Demonstration¶

1.1 Obtain attribution¶

In [ ]:

# instantiate Lime
attribution_explainer = KernelShap(model, tokenizer)

# compute attributions
attributions = attribution_explainer(
    model_inputs="Alice and Bob enter the bar,",
    targets="then Alice offers a drink to Bob.",
)

# visualize attributions
plot_attributions(attributions[0])

# instantiate Lime attribution_explainer = KernelShap(model, tokenizer) # compute attributions attributions = attribution_explainer( model_inputs="Alice and Bob enter the bar,", targets="then Alice offers a drink to Bob.", ) # visualize attributions plot_attributions(attributions[0])

The new embeddings will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`

Inputs

Outputs

1.2 Evaluate attributions¶

In [6]:

# use the same model as for the explainer
metric = Deletion(model, tokenizer)

# compute scores on attributions
auc, detailed_scores = metric.evaluate(attributions)

print(f"Deletion AUC: {round(auc, 3)} (higher is better)")

# use the same model as for the explainer metric = Deletion(model, tokenizer) # compute scores on attributions auc, detailed_scores = metric.evaluate(attributions) print(f"Deletion AUC: {round(auc, 3)} (higher is better)")

Deletion AUC: 0.044 (higher is better)

2. Concepts Demonstration¶

2.1 Split model and get a dataset of activations¶

In [7]:

# split the model
splitter = interpreto.SplitterForGeneration(model, tokenizer=tokenizer, split_point=8)

# compute the token activations
activations, _ = splitter.get_activations(
    inputs=dataset,
)

# split the model splitter = interpreto.SplitterForGeneration(model, tokenizer=tokenizer, split_point=8) # compute the token activations activations, _ = splitter.get_activations( inputs=dataset, )

2.2 Learn concepts as patterns in the activations¶

In [ ]:

# create an explainer around the split model
concept_explainer = SemiNMFConcepts(splitter, nb_concepts=20, device="cuda")

# train the concept model of the explainer
concept_explainer.fit(activations)

# create an explainer around the split model concept_explainer = SemiNMFConcepts(splitter, nb_concepts=20, device="cuda") # train the concept model of the explainer concept_explainer.fit(activations)

2.3 Interpret the concepts¶

In [ ]:

try:
    from interpreto.commons.llm_interface import OpenAILLM

    llm_interface = OpenAILLM(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4.1-nano")

    # instantiate the interpretation method with the concept explainer
    interpretation_method = LLMLabels(
        concept_explainer=concept_explainer,
        llm_interface=llm_interface,
        k_examples=20,
    )
except ImportError:
    interpretation_method = TopKInputs(
        concept_explainer=concept_explainer,
        k=10,
    )

# interpret the concepts via top-k words
interpretations = interpretation_method.interpret(
    inputs=dataset,
    latent_activations=activations,
    concepts_indices="all",
)
print("\n".join(interpretations.values()))

try: from interpreto.commons.llm_interface import OpenAILLM llm_interface = OpenAILLM(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4.1-nano") # instantiate the interpretation method with the concept explainer interpretation_method = LLMLabels( concept_explainer=concept_explainer, llm_interface=llm_interface, k_examples=20, ) except ImportError: interpretation_method = TopKInputs( concept_explainer=concept_explainer, k=10, ) # interpret the concepts via top-k words interpretations = interpretation_method.interpret( inputs=dataset, latent_activations=activations, concepts_indices="all", ) print("\n".join(interpretations.values()))

2.4 Evaluate concepts¶

In [11]:

test_activations = activations  # in practice, these should be different

mse = MSE(concept_explainer).compute(test_activations)
sparsity = Sparsity(concept_explainer).compute(test_activations)

print(f"MSE: {round(mse, 3)}, Sparsity: {round(sparsity, 3)}")

test_activations = activations # in practice, these should be different mse = MSE(concept_explainer).compute(test_activations) sparsity = Sparsity(concept_explainer).compute(test_activations) print(f"MSE: {round(mse, 3)}, Sparsity: {round(sparsity, 3)}")

MSE: 11245.277, Sparsity: 1.0