Examples of how to build concept-based explanations

Load the model and list modules to find where to split it.

from transformers import AutoModelForMaskedLM, AutoTokenizer

model = AutoModelForMaskedLM.from_pretrained("EuroBERT/EuroBERT-210m", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained("EuroBERT/EuroBERT-210m")
split_point = "model.layers.10.mlp"

print(list(model.named_children()))

[('model', EuroBertModel(
  (embed_tokens): Embedding(128256, 768, padding_idx=128001)
  (layers): ModuleList(
    (0-11): 12 x EuroBertDecoderLayer(
      (self_attn): EuroBertAttention(
        (q_proj): Linear(in_features=768, out_features=768, bias=False)
        (k_proj): Linear(in_features=768, out_features=768, bias=False)
        (v_proj): Linear(in_features=768, out_features=768, bias=False)
        (o_proj): Linear(in_features=768, out_features=768, bias=False)
      )
      (mlp): EuroBertMLP(
        (gate_proj): Linear(in_features=768, out_features=3072, bias=False)
        (up_proj): Linear(in_features=768, out_features=3072, bias=False)
        (down_proj): Linear(in_features=3072, out_features=768, bias=False)
        (act_fn): SiLU()
      )
      (input_layernorm): EuroBertRMSNorm((768,), eps=1e-05)
      (post_attention_layernorm): EuroBertRMSNorm((768,), eps=1e-05)
    )
  )
  (norm): EuroBertRMSNorm((768,), eps=1e-05)
  (rotary_emb): EuroBertRotaryEmbedding()
)), ('lm_head', Linear(in_features=768, out_features=128256, bias=False))]

Split the model using the ModelWithSplitPoints class

from interpreto import ModelWithSplitPoints

splitted_model = ModelWithSplitPoints(
    model_or_repo_id=model,
    tokenizer=tokenizer,
    split_points=split_point,
    device_map="cuda",
    batch_size=64,
)

Load the dataset and compute activations

from datasets import load_dataset

rotten_tomatoes = load_dataset("cornell-movie-review-data/rotten_tomatoes")["train"]["text"]

activations = splitted_model.get_activations(
    rotten_tomatoes,
    activation_granularity=ModelWithSplitPoints.activation_granularities.WORD,
)

print(activations[split_point].shape)

Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.

torch.Size([187890, 768])

Create and fit the concept explainer

from interpreto.concepts import ICAConcepts

concept_explainer = ICAConcepts(splitted_model, nb_concepts=50)

concept_explainer.fit(activations)

/home/antonin.poche/interpreto/.venv/lib/python3.12/site-packages/sklearn/decomposition/_fastica.py:127: ConvergenceWarning: FastICA did not converge. Consider increasing tolerance or the maximum number of iterations.
  warnings.warn(

Interpret the concepts

from interpreto.concepts.interpretations import TopKInputs

interpretations = concept_explainer.interpret(
    TopKInputs,
    concepts_indices="all",
    source=TopKInputs.sources.LATENT_ACTIVATIONS,
    granularity=TopKInputs.granularities.WORD,
    inputs=rotten_tomatoes,
    latent_activations=activations,
    k=10,
)

for concept_id, words_importance in interpretations.items():
    print(f"Concept {concept_id}: {list(words_importance.keys()) if words_importance is not None else 'None'}")

Concept 0: [' all', ' everything', ' two', ' under', ' beyond', ' below', ' above', ' self', ' sibling', ' southern']
Concept 1: [' see', ' watching', ' watch', ' view', ' seen', ' find', ' seeing', ' admire', ' sees', ' look']
Concept 2: [' to', ' about', ' will', ' in', ' and', ' with', ' of', ' that', ' out', ' feels']
Concept 3: [' doesn', ' does', ' believes', ' thing', ' to', ' into', ' of', ' claims', ' promises', ' cannot']
Concept 4: [' in', '-in', 'in', ' en', ' into', ' em', ' inside', ' within', ' on', ' at']
Concept 5: [' of', 'of', ' de', ' or', ' del', ' da', ' to', ' dos', '-of', ' do']
Concept 6: [' like', ' as', ' be', ' apparently', ' corrupt', ' cult', ' supernatural', ' shot', ' towering', ' skin']
Concept 7: [' one', ' on', ' end', ' upon', ' photographed', '-on', ' next', ' start', ' may', ' first']
Concept 8: [' reason', ' admirable', ' reasons', ' genesis', ' love', ' thing', ' success', ' passion', ' wonderful', ' emerged']
Concept 9: [' on', ' through', ' into', ' up', ' mind', ' to', ' forward', ' may', ' minds', ' along']
Concept 10: [' a', ' an', 'a', ' another', ' one', 'an', ' some', 'one', ' um', '-a']
Concept 11: [' to', ' be', ' note', 'not', ' takes', ' not', ' didn', ' testimony', ' give', ' done']
Concept 12: [' with', 'with', ' com', ' con', '\x85with', ' by', ' appreciate', ' using', ' to', ' like']
Concept 13: [' i', 'i', ' me', ' my', ' am', 'my', "'m", ' eu', ' myself', ' mine']
Concept 14: ['.', ',', ' )', ' *', ' -', ' t', ' (', '?', ' s', '+']
Concept 15: ['.', '!', ',', '?', ' )', "'", " '", ' "', ' ;', '-']
Concept 16: [' surprising', ' breakthrough', ' believe', ' superb', ' most', ' wait', ' astonishing', ' time', ' impressive', ' complete']
Concept 17: [' life', ' say', ' sweet', ' on', ' when', ' mature', ' of', ' love', ' ends', ' do']
Concept 18: [' from', ' behind', 'from', ' than', ' beneath', ' front', ' moving', ' stuck', ' here', ' turned']
Concept 19: [' and', 'and', '-and', ' y', ' &', ' e', ' or', ' as', ' nor', ',']
Concept 20: [' hit', ' hitting', ' relentlessly', ' killed', ' doesn', ' don', ' extreme', ' have', ' won', ' derive']
Concept 21: [' may', ' can', ' could', ' might', ' would', ' will', ' must', ' wouldn', ' won', 'may']
Concept 22: [' still', ' simply', ' nevertheless', ' certainly', ' also', ' ultimately', ' even', ' definitely', ' equally', ' similarly']
Concept 23: [' text', ' filled', ' if', ' as', ' clear', ' time', ' line', ' impossible', ' adapted', ' title']
Concept 24: [' maze', ' swamp', ' yarn', ' bowl', ' pony', ' mess', ' truck', ' bag', ' wreck', ' whirl']
Concept 25: [' continues', ' has', ' further', ' only', ' later', ' ongoing', '-in', ' update', ' summer', ' new']
Concept 26: [' rip', ' sink', ' shines', ' thump', ' collapses', ' sting', ' butt', ' bites', ' hums', ' suck']
Concept 27: [' between', ' separates', ' typical', ' comes', ' need', ' from', ' prove', ' usual', ' fit', ' much']
Concept 28: [' of', ' kind', ' intelligible', ' like', ' any', ' from', ' seem', ' treatises', ' devoid', ' story']
Concept 29: ['it', 'a', 'if', 'at', 'the', 'in', 'as', 'not', 'there', 'no']
Concept 30: [' see', ' all', ' every', ' missing', ' through', ' catch', ' displaying', ' gets', ' watch', ' use']
Concept 31: [' little', "'", ' into', ' and', ' path', ' with', ' through', ' between', '-school', ' of']
Concept 32: [' makes', ' seen', ' seem', ' thoughtful', ' often', ' were', ' like', ' two', ' many', ' kind']
Concept 33: ['.', ' and', ' has', 'odd', ' poo', 'a', 'cinematic', ' very', ' ride', ' sometimes']
Concept 34: [',', ' )', ' ;', ' :', ' --', '.', '?', ' -', '!', ' "']
Concept 35: [' on', ' about', ' upon', ' at', 'about', ' over', ' into', ' sobre', ' what', ' how']
Concept 36: [' get', ' subtle', ' like', ' what', ' "', ' four', ' three', "'", ' stark', ' don']
Concept 37: [' makes', ' make', ' made', ' lifts', ' leaves', ' making', ' render', ' sent', ' leave', ' keep']
Concept 38: [' rare', ' little', ' greatly', ' desperate', ' fierce', ' big', ' mountain', ' distant', ' grand', ' struggle']
Concept 39: [' joan', 'payne', ' huston', 'cho', ' jonah', ' genet', ' chan', ' eyre', 'ne', ' jonze']
Concept 40: [' by', ' from', ' por', 'by', ' pelo', 'from', ' since', ' da', ' through', ' at']
Concept 41: [' that', ' which', ' who', ' que', ' whose', ' whom', ' new', 'what', ' qui', ' what']
Concept 42: [' is', ' isn', ' was', ' are', ' wasn', ' aren', ' doesn', ' were', ' be', 'is']
Concept 43: [' the', 'this', 'the', ' this', 'these', ' these', 'that', ' those', ' that', ' his']
Concept 44: [' observations', ' insight', ' life', ' error', ' vision', ' investigation', ' experience', ' environment', ' effort', ' into']
Concept 45: [' most', ' mainstream', ' mainly', ' mediocre', ' general', ' mostly', 'most', ' certain', ' typical', ' completely']
Concept 46: [' lesser', '-fashioned', ' had', ' simplistic', ' primitive', ' inferior', ' just', ' disparate', '-made', ' mundane']
Concept 47: [' off', ' death', ' out', ' chair', ' on', ' skin', ' seconds', ' time', ' door', ' way']
Concept 48: [' inhabit', ' below', ' expressed', ' reflect', ' haunting', ' live', ' beneath', ' indulge', ' deferred', ' distancing']
Concept 49: [' sur', ' twists', ' on', ' hours', ' hour', ' weeks', ' includes', ' onto', ' first', ' charm']