Skip to content

Commit c2bf5f0

Browse files
ko3n1gyaoyu-33
andauthored
Update Mllama cross attn signature to match update MCore (#13048) (#13122)
* update Cross Attn signature * update TransformerLayer signature * update SelfAttention signature --------- Signed-off-by: yaoyu-33 <yaoyu.094@gmail.com> Co-authored-by: Yu Yao <54727607+yaoyu-33@users.noreply.github.com>
1 parent 4823514 commit c2bf5f0

File tree

2 files changed

+6
-0
lines changed

2 files changed

+6
-0
lines changed

nemo/collections/vlm/mllama/model/language.py

+2
Original file line numberDiff line numberDiff line change
@@ -553,13 +553,15 @@ def __init__(
553553
submodules: MLlamaCrossAttentionSubmodules,
554554
layer_number: int,
555555
attn_mask_type=AttnMaskType.padding,
556+
**kwargs,
556557
):
557558
super().__init__(
558559
config=config,
559560
submodules=submodules,
560561
layer_number=layer_number,
561562
attn_mask_type=attn_mask_type,
562563
attention_type="cross",
564+
**kwargs,
563565
)
564566

565567
# TODO might need special care when TP>8

nemo/collections/vlm/mllama/model/vision.py

+4
Original file line numberDiff line numberDiff line change
@@ -355,12 +355,14 @@ def __init__(
355355
submodules: SelfAttentionSubmodules,
356356
layer_number: int,
357357
attn_mask_type=AttnMaskType.padding,
358+
**kwargs,
358359
):
359360
super().__init__(
360361
config=config,
361362
submodules=submodules,
362363
layer_number=layer_number,
363364
attn_mask_type=attn_mask_type,
365+
**kwargs,
364366
)
365367

366368
# Override to remove bias since we don't have a good config for this.
@@ -408,12 +410,14 @@ def __init__(
408410
submodules: TransformerLayerSubmodules,
409411
layer_number: int = 1,
410412
hidden_dropout: float = None,
413+
**kwargs,
411414
):
412415
super().__init__(
413416
config=config,
414417
submodules=submodules,
415418
layer_number=layer_number,
416419
hidden_dropout=hidden_dropout,
420+
**kwargs,
417421
)
418422
self.gated = self.config.gated
419423
if self.gated:

0 commit comments

Comments
 (0)