Skip to content

Commit 06c2b15

Browse files
authored
convert : fix Norway problem when parsing YAML (ggml-org#12114)
* convert : fix Norway problem when parsing YAML * Update gguf-py/gguf/metadata.py * add newline at correct place
1 parent 70680c4 commit 06c2b15

File tree

1 file changed

+28
-8
lines changed

1 file changed

+28
-8
lines changed

gguf-py/gguf/metadata.py

+28-8
Original file line numberDiff line numberDiff line change
@@ -121,19 +121,39 @@ def load_model_card(model_path: Optional[Path] = None) -> dict[str, Any]:
121121
if not model_card_path.is_file():
122122
return {}
123123

124-
# The model card metadata is assumed to always be in YAML
124+
# The model card metadata is assumed to always be in YAML (frontmatter)
125125
# ref: https://github.com/huggingface/transformers/blob/a5c642fe7a1f25d3bdcd76991443ba6ff7ee34b2/src/transformers/modelcard.py#L468-L473
126+
yaml_content: str = ""
126127
with open(model_card_path, "r", encoding="utf-8") as f:
127-
if f.readline() == "---\n":
128-
raw = f.read().partition("---\n")[0]
129-
data = yaml.safe_load(raw)
130-
if isinstance(data, dict):
131-
return data
128+
content = f.read()
129+
lines = content.splitlines()
130+
lines_yaml = []
131+
if len(lines) == 0:
132+
# Empty file
133+
return {}
134+
if len(lines) > 0 and lines[0] != "---":
135+
# No frontmatter
136+
return {}
137+
for line in lines[1:]:
138+
if line == "---":
139+
break # End of frontmatter
132140
else:
133-
logger.error(f"while reading YAML model card frontmatter, data is {type(data)} instead of dict")
134-
return {}
141+
lines_yaml.append(line)
142+
yaml_content = "\n".join(lines_yaml) + "\n"
143+
144+
# Quick hack to fix the Norway problem
145+
# https://hitchdev.com/strictyaml/why/implicit-typing-removed/
146+
yaml_content = yaml_content.replace("- no\n", "- \"no\"\n")
147+
148+
if yaml_content:
149+
data = yaml.safe_load(yaml_content)
150+
if isinstance(data, dict):
151+
return data
135152
else:
153+
logger.error(f"while reading YAML model card frontmatter, data is {type(data)} instead of dict")
136154
return {}
155+
else:
156+
return {}
137157

138158
@staticmethod
139159
def load_hf_parameters(model_path: Optional[Path] = None) -> dict[str, Any]:

0 commit comments

Comments
 (0)