From 9cabb20b3610a705c17fa4f5b846030f0e565751 Mon Sep 17 00:00:00 2001 From: Alejandra Gonzalez-Beltran Date: Fri, 8 Mar 2019 11:21:35 +0000 Subject: [PATCH 01/11] Adding titles to subsections --- source/semDiff/semanticComparator.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/source/semDiff/semanticComparator.rst b/source/semDiff/semanticComparator.rst index 52e2c8c..c7b4b2f 100644 --- a/source/semDiff/semanticComparator.rst +++ b/source/semDiff/semanticComparator.rst @@ -2,6 +2,10 @@ How to compare and merge set of JSON-LD-schemas: code documentation ==================================================================== + +Compare entities +------------------------------------------- + .. image:: ../_static/classes_semDiff_compareEntities.png .. automodule:: compareEntities @@ -9,6 +13,11 @@ How to compare and merge set of JSON-LD-schemas: code documentation :private-members: :special-members: + +Compare network +------------------------------------------- + + .. image:: ../_static/classes_semDiff_compareNetwork.png .. automodule:: compareNetwork @@ -17,6 +26,10 @@ How to compare and merge set of JSON-LD-schemas: code documentation :special-members: +Full difference +------------------------------------------- + + .. image:: ../_static/classes_semDiff_fullDiff.png .. automodule:: fullDiff @@ -24,6 +37,10 @@ How to compare and merge set of JSON-LD-schemas: code documentation :private-members: :special-members: + +Merge entities +------------------------------------------- + .. image:: ../_static/classes_semDiff_mergeEntities.png .. automodule:: mergeEntities From 07b0a46122fb3029f308deef0ba81547b5a0d7b9 Mon Sep 17 00:00:00 2001 From: Alejandra Gonzalez-Beltran Date: Mon, 18 Mar 2019 12:52:34 +0000 Subject: [PATCH 02/11] Update to documentation --- source/API/apiClient.rst | 11 ++++++++++- source/index.rst | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source/API/apiClient.rst b/source/API/apiClient.rst index 3b19405..d154a35 100644 --- a/source/API/apiClient.rst +++ b/source/API/apiClient.rst @@ -7,13 +7,22 @@ The JSONLDschema package offers and API client that provides most of the library Here we include the documentation of each of the Python classes and properties for the API client. +API client +------------------------------------------- + + +.. image:: ../_static/classes_api_client.png + .. automodule:: client :members: :noindex: ---------------- +API utilities +------------------------------------------- + +.. image:: ../_static/classes_api_utility.png .. automodule:: utility :members: diff --git a/source/index.rst b/source/index.rst index ece701a..2237231 100644 --- a/source/index.rst +++ b/source/index.rst @@ -6,7 +6,7 @@ Welcome to jsonldschema's documentation! ======================================== -The JSONLDschema python package offers functionality to support the creation and use of machine-actionable and FAIR (Findable, Accessible, Interoperable and Reusable) metadata models expressed as `JSON-schemas `_ for `JSON-LD `_ data. +The JSONLDschema python package offers functionality to support the creation and use of machine-actionable and FAIR (Findable, Accessible, Interoperable and Reusable) metadata models expressed as `JSON-schemas `_ to generate `JSON-LD `_ data. The approach relies on representing the metadata models as JSON-schemas with additional JSON-LD context files to provide semantic annotations to the model. From 4ba641e93090d1c681d074b9379799376bd8b6c5 Mon Sep 17 00:00:00 2001 From: Alejandra Gonzalez-Beltran Date: Mon, 18 Mar 2019 12:55:54 +0000 Subject: [PATCH 03/11] Updated documentation --- source/cedar/cedarIndex.rst | 1 + source/cedar/cedarUtilities.rst | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/source/cedar/cedarIndex.rst b/source/cedar/cedarIndex.rst index 97615ed..1c5fdc7 100644 --- a/source/cedar/cedarIndex.rst +++ b/source/cedar/cedarIndex.rst @@ -5,6 +5,7 @@ CEDAR Templating Tools Cedar utilities ------------------------------------------- + .. toctree:: cedarUtilities diff --git a/source/cedar/cedarUtilities.rst b/source/cedar/cedarUtilities.rst index 536cd6d..1c42c39 100644 --- a/source/cedar/cedarUtilities.rst +++ b/source/cedar/cedarUtilities.rst @@ -3,16 +3,22 @@ Code documentation for the CEDAR utilities ============================================= -The CEDAR client will provide all the links the the CEDAR API functionality such as get, post and updates on templates, -template elements, folders, instances ect ... +The CEDAR client provides all the links the the CEDAR API functionality such as GET, POST and updates on templates, +template elements, folders, instances, and so on. + +.. image:: ../_static/classes_cedar_client.png + .. automodule:: client :members: --------------- -The schema2cedar classes will help you transform your JSON schemas draft 04 into compatible CEDAR schemas + +The schema2cedar classes will help you transform your JSON schemas draft-04 into compatible CEDAR schemas, which can be visualised on the CEDAR Workbench tool. + +.. image:: ../_static/classes_cedar_schema2cedar.png .. automodule:: schema2cedar :members: From 6932ae3b0efcad8381a0c212b6b67d5f712f0a7c Mon Sep 17 00:00:00 2001 From: Alejandra Gonzalez-Beltran Date: Thu, 21 Mar 2019 11:08:56 +0000 Subject: [PATCH 04/11] Extend integration test for generating labels --- tests/integration/test_schema2context.py | 45 +++++++++++++++++------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_schema2context.py b/tests/integration/test_schema2context.py index b5c496a..fd8e8ef 100644 --- a/tests/integration/test_schema2context.py +++ b/tests/integration/test_schema2context.py @@ -5,22 +5,22 @@ prepare_input, create_and_save_contexts, generate_context_mapping_dict, + generate_labels_from_contexts, json ) class TestSchema2Context(unittest.TestCase): - input = dict({ - "MIACME": { - "schema_url": "https://w3id.org/mircat/miacme/schema/miacme_schema.json" - }, - "MIACA": { - "schema_url": "https://w3id.org/mircat/miaca/schema/miaca_schema.json" - }, - "MIFlowCyt": { - "schema_url": "https://w3id.org/mircat/miflowcyt/schema/miflowcyt_schema.json" - } + "MIACME": { + "schema_url": "https://w3id.org/mircat/miacme/schema/miacme_schema.json" + }, + "MIACA": { + "schema_url": "https://w3id.org/mircat/miaca/schema/miaca_schema.json" + }, + "MIFlowCyt": { + "schema_url": "https://w3id.org/mircat/miflowcyt/schema/miflowcyt_schema.json" + } }) base = { @@ -49,8 +49,29 @@ def test_create_and_save_contexts(self): def test_generate_mapping_dict(self): for key in self.input: mapping, errors = generate_context_mapping_dict( - self.input[key]["schema_url"], - self.regexes, key) + self.input[key]["schema_url"], + self.regexes, key) print(json.dumps(mapping, indent=4)) print(errors) self.assertTrue(mapping["networkName"] == key) + + def test_generate_labels_from_contexts(self): + labels = {} + contexts = { + "cell_line_obo_context.jsonld": { + "@context": { + "obo": "http://purl.obolibrary.org/obo/", + "CellLine": "obo:CLO_0000031", + "@language": "en", + "ID": "obo:IAO_0000578", + "lineQC": "obo:ERO_0001219", + "validation": "obo:ERO_0002025", + "cellLineName": "obo:IAO_0000590", + "modifications": "obo:CLO_0000004", + "passageNo": "obo:EFO_0007061", + "cellSource": "obo:CL_0000000" + } + } + } + generate_labels_from_contexts(contexts, labels) + print(labels) From 83005c63f44e1caf84c18d66f5a8d6194792209e Mon Sep 17 00:00:00 2001 From: Alejandra Gonzalez-Beltran Date: Thu, 21 Mar 2019 12:15:36 +0000 Subject: [PATCH 05/11] Added test --- tests/integration/test_schema2context.py | 25 +++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_schema2context.py b/tests/integration/test_schema2context.py index fd8e8ef..0cf88c5 100644 --- a/tests/integration/test_schema2context.py +++ b/tests/integration/test_schema2context.py @@ -55,7 +55,7 @@ def test_generate_mapping_dict(self): print(errors) self.assertTrue(mapping["networkName"] == key) - def test_generate_labels_from_contexts(self): + def test_generate_labels_from_contexts_1(self): labels = {} contexts = { "cell_line_obo_context.jsonld": { @@ -75,3 +75,26 @@ def test_generate_labels_from_contexts(self): } generate_labels_from_contexts(contexts, labels) print(labels) + + def test_generate_labels_from_contexts_2(self): + labels = {} + contexts={ + "@context": { + "obo": "http://purl.obolibrary.org/obo/", + "bao": "http://www.bioassayontology.org/bao#", + "ImagingCondition": "obo:FBbi_root_00000000", + "@language": "en", + "timeSeriesInterval": "obo:IAO_0000584", + "imageSequenceType": "obo:NCIT_C69268", + "pixels": "obo:NCIT_C48367", + "observationPeriod": "obo:NCIT_C25616", + "numericalAperture": "obo:NCIT_C44175", + "observationPeriodUnit": "obo:UO_0000003", + "objectiveLens": "obo:ERO_0000453", + "modality": "obo:FBbi_00000222", + "channelDefinition": "bao:BAO_0000184", + "timeSeriesIntervalUnit": "obo:UO_0000003" + } + } + generate_labels_from_contexts(contexts, labels) + print(labels) From b9787f84fe999d45227fdd69c0b31c8485972b07 Mon Sep 17 00:00:00 2001 From: Alejandra Gonzalez-Beltran Date: Thu, 21 Mar 2019 12:21:21 +0000 Subject: [PATCH 06/11] Added image to doc --- source/validation/validationUtilities.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/validation/validationUtilities.rst b/source/validation/validationUtilities.rst index 5f81f45..c08ef91 100644 --- a/source/validation/validationUtilities.rst +++ b/source/validation/validationUtilities.rst @@ -8,5 +8,8 @@ How to validate JSON-schemas and instances: code documentation .. automodule:: miflowcyt_validate :members: + +.. image:: ../_static/classes_validate_miflowcyt_validate.png + .. autoclass:: FlowRepoClient :members: From 32c185dd372425b1d11c792ad96c5c10ff48ddcd Mon Sep 17 00:00:00 2001 From: Terazus Date: Wed, 27 Mar 2019 10:03:04 -0700 Subject: [PATCH 07/11] Added an example exploiting multiprocessing for make_diff integration test --- tests/integration/make_fulldiff.py | 31 ++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/integration/make_fulldiff.py b/tests/integration/make_fulldiff.py index 0248926..04de836 100644 --- a/tests/integration/make_fulldiff.py +++ b/tests/integration/make_fulldiff.py @@ -1,6 +1,7 @@ import json import os from semDiff.fullDiff import FullDiffGenerator +import multiprocessing output_base_path = os.path.join(os.path.dirname(__file__), "../fullDiffOutput/") @@ -55,24 +56,34 @@ def make_diff(network1, network2): "regex": regex, "url": MyFlowCyt_schema_url } - MIACA_MIACME_merge_network = { "name": "MIACA_MIACME_merge", "regex": regex2, "url": MIACA_MIACME_merge_schema_url } - make_diff(MIACA_network, MIACA_network) - make_diff(MIACA_network, MyFlowCyt_network) - make_diff(MIACA_network, MIACME_network) + cpu_count = multiprocessing.cpu_count() + + processes = [ + (MIACA_network, MIACME_network), + (MIACA_network, MyFlowCyt_network), + (MIACA_network, MIACA_network), + + (MIACME_network, MIACME_network), + (MIACME_network, MyFlowCyt_network), + (MIACME_network, MIACA_network), + + (MyFlowCyt_network, MIACME_network), + (MyFlowCyt_network, MyFlowCyt_network), + (MyFlowCyt_network, MIACA_network) + ] - make_diff(MIACME_network, MIACME_network) - make_diff(MIACME_network, MyFlowCyt_network) - make_diff(MIACME_network, MIACA_network) + p = multiprocessing.Pool(processes=cpu_count) + result = [p.apply_async(make_diff, args=(x, y)) for (x, y) in processes] - make_diff(MyFlowCyt_network, MyFlowCyt_network) - make_diff(MyFlowCyt_network, MIACME_network) - make_diff(MyFlowCyt_network, MIACA_network) + output = [pp.get() for pp in result] + """ make_diff(MIACA_MIACME_merge_network, MIACA_network) make_diff(MIACA_network, MIACA_MIACME_merge_network) + """ From d644c62d7e89741a06a09546ed28cd594a4ceda7 Mon Sep 17 00:00:00 2001 From: Alejandra Gonzalez-Beltran Date: Thu, 28 Mar 2019 14:06:39 +0000 Subject: [PATCH 08/11] Fixing URLs for merge test --- tests/integration/make_fulldiff.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/integration/make_fulldiff.py b/tests/integration/make_fulldiff.py index 0248926..a23dea6 100644 --- a/tests/integration/make_fulldiff.py +++ b/tests/integration/make_fulldiff.py @@ -25,9 +25,9 @@ def make_diff(network1, network2): MIACA_schema_url = "https://w3id.org/mircat/miaca/schema/miaca_schema.json" MyFlowCyt_schema_url = "https://w3id.org/mircat/miflowcyt/schema/miflowcyt_schema.json" MIACA_MIACME_merge_schema_url = \ - "https://w3id.org/mircat/miaca_miacme_merge/schema/miaca_schema.json" + "https://w3id.org/mircat/miaca_miacme_merge/schema/miaca_miacme_merged_schema.json" MIACME_MIACA_merge_schem_url = \ - "https://w3id.org/mircat/miacme_miaca_merge/schema/miacme_schema.json" + "https://w3id.org/mircat/miacme_miaca_merge/schema/miacme_miaca_merged_schema.json" regex = { "/schema": "/context/obo", @@ -62,17 +62,17 @@ def make_diff(network1, network2): "url": MIACA_MIACME_merge_schema_url } - make_diff(MIACA_network, MIACA_network) - make_diff(MIACA_network, MyFlowCyt_network) - make_diff(MIACA_network, MIACME_network) - - make_diff(MIACME_network, MIACME_network) - make_diff(MIACME_network, MyFlowCyt_network) - make_diff(MIACME_network, MIACA_network) - - make_diff(MyFlowCyt_network, MyFlowCyt_network) - make_diff(MyFlowCyt_network, MIACME_network) - make_diff(MyFlowCyt_network, MIACA_network) + # make_diff(MIACA_network, MIACA_network) + # make_diff(MIACA_network, MyFlowCyt_network) + # make_diff(MIACA_network, MIACME_network) + # + # make_diff(MIACME_network, MIACME_network) + # make_diff(MIACME_network, MyFlowCyt_network) + # make_diff(MIACME_network, MIACA_network) + # + # make_diff(MyFlowCyt_network, MyFlowCyt_network) + # make_diff(MyFlowCyt_network, MIACME_network) + # make_diff(MyFlowCyt_network, MIACA_network) make_diff(MIACA_MIACME_merge_network, MIACA_network) make_diff(MIACA_network, MIACA_MIACME_merge_network) From 4317bb94dd258be5b3ccb89a6724d621cbe6fa4d Mon Sep 17 00:00:00 2001 From: Dominique Date: Thu, 28 Mar 2019 14:42:41 +0000 Subject: [PATCH 09/11] fixed oneOf/anyOf/allOf issues upon merging --- semDiff/mergeEntities.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/semDiff/mergeEntities.py b/semDiff/mergeEntities.py index 6e4c6a5..32aaf9f 100644 --- a/semDiff/mergeEntities.py +++ b/semDiff/mergeEntities.py @@ -266,13 +266,17 @@ def modify_references(self): for reference in look_for: if reference in field['items']: + sub_item_iterator = 0 for sub_item in field['items'][reference]: if '$ref' in sub_item: - field_ref = sub_item['$ref'] + field_ref = sub_item['$ref'].replace('#', '') + if field_ref in self.name_mapping: + print(field_ref) + print(self.output['schemas'][schema]['properties']) self.output['schemas'][schema]['properties'][ - reference]['items'][ + item]['items'][reference][ sub_item_iterator]['$ref'] = \ self.name_mapping[field_ref] + "#" sub_item_iterator += 1 From f82f4de56e495329034480617b866ef7c6566b90 Mon Sep 17 00:00:00 2001 From: Dominique Date: Thu, 28 Mar 2019 15:34:13 +0000 Subject: [PATCH 10/11] fixed pep8 --- tests/integration/make_fulldiff.py | 6 +----- tests/integration/test_schema2context.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integration/make_fulldiff.py b/tests/integration/make_fulldiff.py index 6047aa2..3d665aa 100644 --- a/tests/integration/make_fulldiff.py +++ b/tests/integration/make_fulldiff.py @@ -62,9 +62,6 @@ def make_diff(network1, network2): "url": MIACA_MIACME_merge_schema_url } - - cpu_count = multiprocessing.cpu_count() - processes = [ (MIACA_network, MIACME_network), (MIACA_network, MyFlowCyt_network), @@ -79,12 +76,11 @@ def make_diff(network1, network2): (MyFlowCyt_network, MIACA_network) ] + cpu_count = multiprocessing.cpu_count() p = multiprocessing.Pool(processes=cpu_count) result = [p.apply_async(make_diff, args=(x, y)) for (x, y) in processes] output = [pp.get() for pp in result] - make_diff(MIACA_MIACME_merge_network, MIACA_network) make_diff(MIACA_network, MIACA_MIACME_merge_network) - diff --git a/tests/integration/test_schema2context.py b/tests/integration/test_schema2context.py index 0cf88c5..ed84222 100644 --- a/tests/integration/test_schema2context.py +++ b/tests/integration/test_schema2context.py @@ -78,7 +78,7 @@ def test_generate_labels_from_contexts_1(self): def test_generate_labels_from_contexts_2(self): labels = {} - contexts={ + contexts = { "@context": { "obo": "http://purl.obolibrary.org/obo/", "bao": "http://www.bioassayontology.org/bao#", From 66da5557ec0041fb01fcf0d47bdee93ea735b07c Mon Sep 17 00:00:00 2001 From: Dominique Date: Thu, 28 Mar 2019 15:51:40 +0000 Subject: [PATCH 11/11] updated test output for merges --- tests/fullDiffOutput/merges/example_merge.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fullDiffOutput/merges/example_merge.json b/tests/fullDiffOutput/merges/example_merge.json index 288421a..f1e71db 100644 --- a/tests/fullDiffOutput/merges/example_merge.json +++ b/tests/fullDiffOutput/merges/example_merge.json @@ -42,7 +42,7 @@ "items": { "anyOf": [ { - "$ref": "test1_second_schema.json#" + "$ref": "test1_second_test2_second_merged_schema.json#" }, { "$ref": "test1_third_schema.json#" @@ -67,10 +67,10 @@ "items": { "anyOf": [ { - "$ref": "test2_second_schema.json#" + "$ref": "test1_second_test2_second_merged_schema.json#" }, { - "$ref": "third_schema.json#" + "$ref": "third_merged_schema.json#" } ] }