Skip to content

Commit 0085b83

Browse files
authored
Merge pull request #44 from Peyman-N/pipeline_namespace
Adoption python pipeline to new namespace
2 parents f0ed26f + 5ff2fc1 commit 0085b83

File tree

6 files changed

+92
-34
lines changed

6 files changed

+92
-34
lines changed

build.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
instances[version][instance_data["@type"]].append(instance_data)
4848

4949
python_modules = defaultdict(list)
50+
5051
for schema_version in schema_loader.get_schema_versions():
5152

5253
# Step 3 - find all involved schemas for the current version
@@ -55,8 +56,10 @@
5556
# Step 4a - figure out which schemas are embedded and which are linked
5657
embedded = set()
5758
linked = set()
59+
class_to_module_map = {}
5860
for schema_file_path in schemas_file_paths:
5961
emb, lnk = PythonBuilder(schema_file_path, schema_loader.schemas_sources).get_edges()
62+
class_to_module_map=PythonBuilder(schema_file_path, schema_loader.schemas_sources).update_class_to_module_map(class_to_module_map)
6063
embedded.update(emb)
6164
linked.update(lnk)
6265
conflicts = linked.intersection(embedded)
@@ -76,7 +79,7 @@
7679
schema_loader.schemas_sources,
7780
instances=instances.get(schema_version, None),
7881
additional_methods=additional_methods,
79-
).build(embedded=embedded)
82+
).build(embedded=embedded,class_to_module_map=class_to_module_map)
8083

8184
parts = module_path.split(".")
8285
parent_path = ".".join(parts[:-1])

pipeline/src/base.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,10 @@ def to_jsonld(
7373

7474
data = {"@type": self.type_}
7575
if with_context:
76-
data["@context"] = {"@vocab": "https://openminds.ebrains.eu/vocab/"}
76+
if self.type_.startswith("https://openminds.ebrains.eu/"):
77+
data["@context"] = {"@vocab": "https://openminds.ebrains.eu/vocab/"}
78+
else:
79+
data["@context"] = {"@vocab": "https://openminds.om-i.org/props/"}
7780
if hasattr(self, "id") and self.id:
7881
data["@id"] = self.id
7982
for property in self.__class__.properties:

pipeline/src/collection.py

+12-2
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,12 @@ def save(self, path, individual_files=False, include_empty_properties=False):
8585
# we first re-add all child nodes to the collection.
8686
# This is probably not the most elegant or fast way to do this, but it is simple and robust.
8787
for node in tuple(self.nodes.values()):
88+
89+
if node.type_.startswith("https://openminds.ebrains.eu/"):
90+
data_context = {"@vocab": "https://openminds.ebrains.eu/vocab/"}
91+
else:
92+
data_context = {"@vocab": "https://openminds.om-i.org/props/"}
93+
8894
for linked_node in node.links:
8995
self._add_node(linked_node)
9096
# Now we can actually save the nodes
@@ -97,7 +103,7 @@ def save(self, path, individual_files=False, include_empty_properties=False):
97103
if parent_dir:
98104
os.makedirs(parent_dir, exist_ok=True)
99105
data = {
100-
"@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
106+
"@context": data_context,
101107
"@graph": [
102108
node.to_jsonld(
103109
embed_linked_nodes=False, include_empty_properties=include_empty_properties, with_context=False
@@ -154,9 +160,13 @@ def load(self, *paths):
154160
with open(path, "r") as fp:
155161
data = json.load(fp)
156162
if "@graph" in data:
163+
if data["@context"]["@vocab"].startswith("https://openminds.ebrains.eu/"):
164+
version="v3"
165+
else:
166+
version="latest"
157167
for item in data["@graph"]:
158168
if "@type" in item:
159-
cls = lookup_type(item["@type"])
169+
cls = lookup_type(item["@type"],version=version)
160170
node = cls.from_jsonld(item)
161171
else:
162172
# allow links to metadata instances outside this collection

pipeline/src/module_template.py.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ class {{ class_name }}({{ base_class }}):
1616
"""
1717
type_ = "{{ openminds_type }}"
1818
context = {
19-
"@vocab": "https://openminds.ebrains.eu/vocab/"
19+
"@vocab": "{{ context_vocab }}"
2020
}
2121
schema_version = "{{ schema_version }}"
2222

@@ -61,4 +61,4 @@ class {{ class_name }}({{ base_class }}):
6161
{{key}}={{value}},
6262
{%- endif %}
6363
{% endfor -%}
64-
){% endfor %}
64+
){% endfor %}

pipeline/tests/test_regressions.py

+20-19
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ def test_issue_0002():
1212

1313
node = build_fake_node(omcore.Person)
1414
data = node.to_jsonld()
15-
assert data["@type"] == "https://openminds.ebrains.eu/core/Person"
15+
assert data["@type"] == "https://openminds.om-i.org/types/Person"
1616

1717

1818
def test_issue_0003():
@@ -38,21 +38,22 @@ def test_issue_0003():
3838
)
3939
# on export, a single item should be wrapped in a list, where the property expects an array
4040
expected = {
41-
"@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
42-
"@type": "https://openminds.ebrains.eu/core/FileArchive",
41+
"@context": {"@vocab": "https://openminds.om-i.org/props/"},
42+
"@type": "https://openminds.om-i.org/types/FileArchive",
4343
"IRI": "http://example.com/archive.zip",
4444
"format": {
45-
"@type": "https://openminds.ebrains.eu/core/ContentType",
45+
"@type": "https://openminds.om-i.org/types/ContentType",
4646
"name": "application/zip",
47-
},
47+
},
4848
"sourceData": [
4949
{
50-
"@type": "https://openminds.ebrains.eu/core/File",
50+
"@type": "https://openminds.om-i.org/types/File",
5151
"IRI": "http://example.com/some_file.txt",
5252
"name": "some_file.txt",
5353
}
5454
],
5555
}
56+
5657
assert (
5758
node1.to_jsonld(include_empty_properties=False) == node2.to_jsonld(include_empty_properties=False) == expected
5859
)
@@ -89,20 +90,20 @@ def test_issue0007():
8990

9091
actual = person.to_jsonld(include_empty_properties=False, embed_linked_nodes=False, with_context=True)
9192
expected = {
92-
"@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
93+
"@context": {"@vocab": "https://openminds.om-i.org/props/"},
9394
"@id": "_:001",
94-
"@type": "https://openminds.ebrains.eu/core/Person",
95+
"@type": "https://openminds.om-i.org/types/Person",
9596
"familyName": "Professor",
9697
"givenName": "A",
9798
"affiliation": [
9899
{
99-
"@type": "https://openminds.ebrains.eu/core/Affiliation",
100+
"@type": "https://openminds.om-i.org/types/Affiliation",
100101
"memberOf": {
101102
"@id": "_:002"
102103
},
103104
},
104105
{
105-
"@type": "https://openminds.ebrains.eu/core/Affiliation",
106+
"@type": "https://openminds.om-i.org/types/Affiliation",
106107
"memberOf": {
107108
"@id": "_:003"
108109
},
@@ -119,20 +120,20 @@ def test_issue0007():
119120
saved_data = json.load(fp)
120121
os.remove("issue0007.jsonld")
121122
expected_saved_data = {
122-
"@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
123+
"@context": {"@vocab": "https://openminds.om-i.org/props/"},
123124
"@graph": [
124125
{
125126
"@id": "_:001",
126-
"@type": "https://openminds.ebrains.eu/core/Person",
127+
"@type": "https://openminds.om-i.org/types/Person",
127128
"affiliation": [
128129
{
129-
"@type": "https://openminds.ebrains.eu/core/Affiliation",
130+
"@type": "https://openminds.om-i.org/types/Affiliation",
130131
"memberOf": {
131132
"@id": "_:002"
132133
},
133134
},
134135
{
135-
"@type": "https://openminds.ebrains.eu/core/Affiliation",
136+
"@type": "https://openminds.om-i.org/types/Affiliation",
136137
"memberOf": {
137138
"@id": "_:003"
138139
},
@@ -143,12 +144,12 @@ def test_issue0007():
143144
},
144145
{
145146
"@id": "_:002",
146-
"@type": "https://openminds.ebrains.eu/core/Organization",
147+
"@type": "https://openminds.om-i.org/types/Organization",
147148
"fullName": "University of This Place",
148149
},
149150
{
150151
"@id": "_:003",
151-
"@type": "https://openminds.ebrains.eu/core/Organization",
152+
"@type": "https://openminds.om-i.org/types/Organization",
152153
"fullName": "University of That Place",
153154
},
154155
],
@@ -170,12 +171,12 @@ def test_issue0008():
170171
)
171172
actual = person.to_jsonld(include_empty_properties=False, embed_linked_nodes=False, with_context=True)
172173
expected = {
173-
"@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
174+
"@context": {"@vocab": "https://openminds.om-i.org/props/"},
174175
"@id": "_:002",
175-
"@type": "https://openminds.ebrains.eu/core/Person",
176+
"@type": "https://openminds.om-i.org/types/Person",
176177
"affiliation": [
177178
{
178-
"@type": "https://openminds.ebrains.eu/core/Affiliation",
179+
"@type": "https://openminds.om-i.org/types/Affiliation",
179180
"endDate": "2023-09-30",
180181
"memberOf": {
181182
"@id": "_:001"

pipeline/translator.py

+50-9
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def generate_python_name(json_name, allow_multiple=False):
2525
python_name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", json_name.strip())
2626
python_name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", python_name).lower()
2727
replacements = [
28-
("-", "_"), (".", "_"), ("+", "plus"), ("#", "sharp"), (",", "comma"), ("(", ""), (")", "")
28+
("-", "_"), (".", "_"),("'","_prime_"), ("+", "plus"), ("#", "sharp"), (",", "comma"), ("(", ""), (")", "")
2929
]
3030
for before, after in replacements:
3131
python_name = python_name.replace(before, after)
@@ -57,14 +57,19 @@ class PythonBuilder(object):
5757

5858
def __init__(self, schema_file_path: str, root_path: str, instances: Optional[dict] = None,
5959
additional_methods: Optional[dict] = None):
60-
self.template_name = "src/module_template.py.txt"
6160
self.env = Environment(
6261
loader=FileSystemLoader(os.path.dirname(os.path.realpath(__file__))), autoescape=select_autoescape()
6362
)
6463
_relative_path_without_extension = (
6564
schema_file_path[len(root_path) + 1 :].replace(".schema.omi.json", "").split("/")
6665
)
6766
self.version = _relative_path_without_extension[0]
67+
self.template_name = "src/module_template.py.txt"
68+
if self.version in ["v3.0" , "v2.0" , "v1.0"]:
69+
self.context_vocab = "https://openminds.ebrains.eu/vocab/"
70+
else:
71+
self.context_vocab = "https://openminds.om-i.org/props/"
72+
6873
self.relative_path_without_extension = [
6974
generate_python_name(part) for part in _relative_path_without_extension[1:]
7075
]
@@ -83,7 +88,7 @@ def _version_module(self):
8388
def _target_file_without_extension(self) -> str:
8489
return os.path.join(self._version_module, "/".join(self.relative_path_without_extension))
8590

86-
def translate(self, embedded=None):
91+
def translate(self, embedded=None, class_to_module_map=None):
8792
def get_type(property):
8893
type_map = {
8994
"string": "str",
@@ -100,17 +105,23 @@ def get_type(property):
100105
if "_linkedTypes" in property:
101106
types = []
102107
for item in property["_linkedTypes"]:
103-
openminds_module, class_name = item.split("/")[-2:]
104-
openminds_module = generate_python_name(openminds_module)
108+
openminds_module_from_type, class_name = item.split("/")[-2:]
109+
if isinstance(class_to_module_map,dict) and (class_name in class_to_module_map):
110+
openminds_module = generate_python_name(class_to_module_map[class_name])
111+
else:
112+
openminds_module = generate_python_name(openminds_module_from_type)
105113
types.append(f"openminds.{self._version_module}.{openminds_module}.{class_name}")
106114
if len(types) == 1:
107115
types = f'"{types[0]}"'
108116
return types
109117
elif "_embeddedTypes" in property:
110118
types = []
111119
for item in property["_embeddedTypes"]:
112-
openminds_module, class_name = item.split("/")[-2:]
113-
openminds_module = generate_python_name(openminds_module)
120+
openminds_module_from_type, class_name = item.split("/")[-2:]
121+
if isinstance(class_to_module_map,dict) and (class_name in class_to_module_map):
122+
openminds_module = generate_python_name(class_to_module_map[class_name])
123+
else:
124+
openminds_module = generate_python_name(openminds_module_from_type)
114125
types.append(f"openminds.{self._version_module}.{openminds_module}.{class_name}")
115126
if len(types) == 1:
116127
types = f'"{types[0]}"'
@@ -201,6 +212,7 @@ def filter_instance(instance):
201212
"class_name": class_name,
202213
"openminds_type": openminds_type,
203214
"schema_version": self.version,
215+
"context_vocab": self.context_vocab,
204216
"properties": properties,
205217
"additional_methods": "",
206218
"instances": instances
@@ -233,11 +245,11 @@ def filter_instance(instance):
233245
if extra_imports:
234246
self.context["preamble"] = "\n".join(sorted(extra_imports))
235247

236-
def build(self, embedded=None):
248+
def build(self, embedded=None, class_to_module_map=None):
237249
target_file_path = os.path.join("target", "openminds", f"{self._target_file_without_extension()}.py")
238250
os.makedirs(os.path.dirname(target_file_path), exist_ok=True)
239251

240-
self.translate(embedded=embedded)
252+
self.translate(embedded=embedded, class_to_module_map=class_to_module_map)
241253

242254
with open(target_file_path, "w") as target_file:
243255
contents = self.env.get_template(self.template_name).render(self.context)
@@ -252,3 +264,32 @@ def get_edges(self):
252264
embedded.update(property.get("_embeddedTypes", []))
253265
linked.update(property.get("_linkedTypes", []))
254266
return embedded, linked
267+
268+
def update_class_to_module_map(self,class_to_module_map):
269+
"""
270+
Updates a dictionary with the class name and its corresponding module based on the schemas.
271+
272+
This method extracts the class name and module from the `_schema_payload` attribute
273+
and updates the provided dictionary (`class_to_module_map`) with a mapping of
274+
the class name to its module. If the `_module` key exists in `_schema_payload`
275+
(which was introduced in version 4 of openMINDS), its value is used as the module.
276+
Otherwise, the module is derived from the second-to-last component of the `_type`
277+
field in `_schema_payload`.
278+
279+
Args:
280+
class_to_module_map (dict): A dictionary where keys are class names and values
281+
are their corresponding modules.
282+
283+
Returns:
284+
dict: The updated dictionary with the class name and module mapping.
285+
"""
286+
schema_type=self._schema_payload["_type"]
287+
class_name=schema_type.split("/")[-1]
288+
if "_module" in self._schema_payload:
289+
module=self._schema_payload["_module"]
290+
else:
291+
module=schema_type.split("/")[-2]
292+
293+
class_to_module_map[class_name]=module
294+
295+
return class_to_module_map

0 commit comments

Comments
 (0)