fixed phi-3

Author: bryce13950

transformer_lens/model_bridge/bridge.py

@@ -107,7 +107,16 @@ def __init__(
 
         # Infer vocab size from tokenizer (similar to HookedTransformer)
         if self.cfg.d_vocab == -1:
-            self.cfg.d_vocab = max(self.tokenizer.vocab.values()) + 1
+            # Use get_vocab() method which works across different tokenizer types
+            # Some tokenizers (like CodeGenTokenizer) don't support direct .vocab access
+            if hasattr(self.tokenizer, 'get_vocab'):
+                vocab = self.tokenizer.get_vocab()
+                self.cfg.d_vocab = max(vocab.values()) + 1
+            elif hasattr(self.tokenizer, 'vocab'):
+                self.cfg.d_vocab = max(self.tokenizer.vocab.values()) + 1
+            else:
+                # Fallback: use vocab_size attribute if available
+                self.cfg.d_vocab = getattr(self.tokenizer, 'vocab_size', 50257)
         if self.cfg.d_vocab_out == -1:
             self.cfg.d_vocab_out = self.cfg.d_vocab
 

transformer_lens/model_bridge/supported_architectures/phi3.py

@@ -74,7 +74,17 @@ def __init__(self, cfg: Any) -> None:
                 submodules={
                     "ln1": NormalizationBridge(name="input_layernorm", config=self.cfg),
                     "ln2": NormalizationBridge(name="post_attention_layernorm", config=self.cfg),
-                    "attn": AttentionBridge(name="self_attn", config=self.cfg),
+                    "attn": AttentionBridge(
+                        name="self_attn",
+                        config=self.cfg,
+                        submodules={
+                            # Phi-3 uses combined qkv_proj, but we still need submodules for hooks
+                            "q": LinearBridge(name="qkv_proj"),
+                            "k": LinearBridge(name="qkv_proj"),
+                            "v": LinearBridge(name="qkv_proj"),
+                            "o": LinearBridge(name="o_proj"),
+                        },
+                    ),
                     "mlp": MLPBridge(name="mlp"),
                 },
             ),