fix typo

tmm1 · tmm1 · commit 2eda9e02a9d1 · 2023-08-03T21:04:12.000Z
diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py
@@ -333,7 +333,7 @@ def load_model(
             model, use_gradient_checkpointing=cfg.gradient_checkpointing
         )
 
-        # LlamaRMSNorm layers are in fp32 after kit call, so we need to
+        # LlamaRMSNorm layers are in fp32 after kbit_training, so we need to
         # convert them back to fp16/bf16 for flash-attn compatibility.
         if cfg.flash_attention and cfg.is_llama_derived_model:
             for name, module in model.named_modules():

Original file line number	Diff line number	Diff line change
`@@ -333,7 +333,7 @@ def load_model(`
`333`	`333`	`model, use_gradient_checkpointing=cfg.gradient_checkpointing`
`334`	`334`	`)`
`335`	`335`
`336`		`- # LlamaRMSNorm layers are in fp32 after kit call, so we need to`
	`336`	`+ # LlamaRMSNorm layers are in fp32 after kbit_training, so we need to`
`337`	`337`	`# convert them back to fp16/bf16 for flash-attn compatibility.`
`338`	`338`	`if cfg.flash_attention and cfg.is_llama_derived_model:`
`339`	`339`	`for name, module in model.named_modules():`