Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Multi-Module Architecture Handling #534

Merged
merged 15 commits into from
Mar 14, 2025
Merged

Multi-Module Architecture Handling #534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
922441e
Refactor architecture handling to support multiple modules
cg123 Mar 13, 2025
19cfef9
Try adding Gemma3
cg123 Mar 13, 2025
eb6044c
Various fixes
cg123 Mar 13, 2025
e5996f1
Speed up scheduling by avoiding task reserialization
cg123 Mar 13, 2025
6125c51
Fix scripts
cg123 Mar 13, 2025
5e1888e
Bring along preprocessor/processor config
cg123 Mar 13, 2025
6804807
Handle AutoModel better
cg123 Mar 13, 2025
0cc3f96
Formatting
cg123 Mar 13, 2025
3489ba1
Fix name
cg123 Mar 13, 2025
f8e4ec6
Add T5 support
cg123 Mar 14, 2025
c0419f0
Unbreak mergekit-extract-lora
cg123 Mar 14, 2025
8f2c524
Formatting
cg123 Mar 14, 2025
1043093
Oh everything's tied huh
cg123 Mar 14, 2025
e19916c
Better dummy model creation (no more warning spam)
cg123 Mar 14, 2025
e4e6da8
oops
cg123 Mar 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions mergekit/_data/architectures/gemma3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
{
"model_type": "gemma3_text",
"architectures": [
"Gemma3ForCausalLM"
],
"pre_weights": [
{
"name": "model.embed_tokens.weight",
"is_embed": true
}
],
"num_layers_config_key": "num_hidden_layers",
"layer_templates": {
"weights": [
{
"name": "model.layers.${layer_index}.input_layernorm.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.q_proj.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.q_norm.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.k_proj.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.k_norm.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.v_proj.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.o_proj.weight"
},
{
"name": "model.layers.${layer_index}.post_attention_layernorm.weight"
},
{
"name": "model.layers.${layer_index}.pre_feedforward_layernorm.weight"
},
{
"name": "model.layers.${layer_index}.mlp.up_proj.weight"
},
{
"name": "model.layers.${layer_index}.mlp.gate_proj.weight"
},
{
"name": "model.layers.${layer_index}.mlp.down_proj.weight"
},
{
"name": "model.layers.${layer_index}.post_feedforward_layernorm.weight"
}
]
},
"post_weights": [
{
"name": "model.norm.weight"
},
{
"name": "lm_head.weight",
"is_embed": true,
"optional": true,
"tied_names": [
"model.embed_tokens.weight"
]
}
]
}
184 changes: 184 additions & 0 deletions mergekit/_data/architectures/gemma3vl.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
{
"kind": "modular",
"architectures": [
"Gemma3ForConditionalGeneration"
],
"model_type": "gemma3",
"tagalong_files": [
"preprocessor_config.json",
"processor_config.json"
],
"modules": {
"text_decoder": {
"weight_prefix": "language_model.",
"architecture": {
"model_type": "gemma3_text",
"architectures": [
"Gemma3ForCausalLM"
],
"pre_weights": [
{
"name": "model.embed_tokens.weight",
"is_embed": true
}
],
"num_layers_config_key": "text_config.num_hidden_layers",
"layer_templates": {
"weights": [
{
"name": "model.layers.${layer_index}.input_layernorm.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.q_proj.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.q_norm.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.k_proj.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.k_norm.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.v_proj.weight"
},
{
"name": "model.layers.${layer_index}.self_attn.o_proj.weight"
},
{
"name": "model.layers.${layer_index}.post_attention_layernorm.weight"
},
{
"name": "model.layers.${layer_index}.pre_feedforward_layernorm.weight"
},
{
"name": "model.layers.${layer_index}.mlp.up_proj.weight"
},
{
"name": "model.layers.${layer_index}.mlp.gate_proj.weight"
},
{
"name": "model.layers.${layer_index}.mlp.down_proj.weight"
},
{
"name": "model.layers.${layer_index}.post_feedforward_layernorm.weight"
}
]
},
"post_weights": [
{
"name": "model.norm.weight"
},
{
"name": "lm_head.weight",
"is_embed": true,
"optional": true,
"tied_names": [
"model.embed_tokens.weight"
]
}
]
}
},
"multi_modal_projector": {
"weight_prefix": "multi_modal_projector.",
"architecture": {
"model_type": "gemma3_mmproj",
"architectures": [],
"pre_weights": [
{
"name": "mm_input_projection_weight"
},
{
"name": "mm_soft_emb_norm.weight"
}
],
"post_weights": [],
"layer_templates": {
"weights": []
},
"override_num_layers": 0
}
},
"vision_tower": {
"weight_prefix": "vision_tower.vision_model.",
"architecture": {
"model_type": "siglip_vision_model",
"architectures": [],
"pre_weights": [
{
"name": "embeddings.patch_embedding.bias"
},
{
"name": "embeddings.patch_embedding.weight"
},
{
"name": "embeddings.position_embedding.weight"
}
],
"post_weights": [
{
"name": "post_layernorm.bias"
},
{
"name": "post_layernorm.weight"
}
],
"layer_templates": {
"weights": [
{
"name": "encoder.layers.${layer_index}.layer_norm1.bias"
},
{
"name": "encoder.layers.${layer_index}.layer_norm1.weight"
},
{
"name": "encoder.layers.${layer_index}.layer_norm2.bias"
},
{
"name": "encoder.layers.${layer_index}.layer_norm2.weight"
},
{
"name": "encoder.layers.${layer_index}.mlp.fc1.bias"
},
{
"name": "encoder.layers.${layer_index}.mlp.fc1.weight"
},
{
"name": "encoder.layers.${layer_index}.mlp.fc2.bias"
},
{
"name": "encoder.layers.${layer_index}.mlp.fc2.weight"
},
{
"name": "encoder.layers.${layer_index}.self_attn.k_proj.bias"
},
{
"name": "encoder.layers.${layer_index}.self_attn.k_proj.weight"
},
{
"name": "encoder.layers.${layer_index}.self_attn.out_proj.bias"
},
{
"name": "encoder.layers.${layer_index}.self_attn.out_proj.weight"
},
{
"name": "encoder.layers.${layer_index}.self_attn.q_proj.bias"
},
{
"name": "encoder.layers.${layer_index}.self_attn.q_proj.weight"
},
{
"name": "encoder.layers.${layer_index}.self_attn.v_proj.bias"
},
{
"name": "encoder.layers.${layer_index}.self_attn.v_proj.weight"
}
]
},
"num_layers_config_key": "vision_config.num_hidden_layers"
}
}
}
}
Loading
Loading