Skip to content

Commit 5147c84

Browse files
committed
fix
1 parent c21377c commit 5147c84

File tree

1 file changed

+3
-0
lines changed

1 file changed

+3
-0
lines changed

mergekit/scripts/tokensurgeon.py

+3
Original file line numberDiff line numberDiff line change
@@ -976,6 +976,8 @@ def main(
976976
if (tok not in well_trained_donor_tokens)
977977
and (tok not in well_trained_orig_tokens)
978978
]
979+
else:
980+
junk_tokens = []
979981

980982
if orig_embed is not None:
981983
if donor_embed is None:
@@ -1012,6 +1014,7 @@ def main(
10121014
donor_lm_head,
10131015
orig_vocab=orig_vocab,
10141016
donor_vocab=donor_vocab,
1017+
junk_tokens=junk_tokens,
10151018
allow_prefix=prefix_match in (AllowMatch.YES, AllowMatch.EMBED_ONLY),
10161019
allow_byte=byte_match in (AllowMatch.YES, AllowMatch.EMBED_ONLY),
10171020
is_lm_head=True,

0 commit comments

Comments
 (0)