Cleanup (#5)

* Gif for data import tab updated + importing tab only displays json files now

* Datetime in properties fix + relationship mappings added to data-import json generation

Author: jalakoo

README.md

@@ -1,6 +1,8 @@
 # MOCK GRAPH DATA GENERATOR
 This is a prototype app for generating mock graph data for [Neo4j](https://neo4j.com/) database instances.
 
+<!-- TODO: Add animated gif: 1. Design in Arrows, 2. Mapping options, 3. Uploading to data-importer -->
+
 ## Requirements
 [Poetry](https://python-poetry.org/) should be installed. Code in this repo uses Poetry for managing dependencies and running a virtual environment.
 

mock_generators/constants.py

@@ -27,24 +27,3 @@
 
 # TODO: Can Streamlit's st.session hold all the data we'll be generating?
 MAPPINGS = "mappings"
-
-# TODO: Add page titles
-# OVERVIEW_PAGE_TITLE = "🏠 Overview"
-# DESIGN_PAGE_TITLE = "✍️ Design"
-# IMPORT_PAGE_TITLE = "🔼 Import"
-# SETTINGS_PAGE_TITLE = "⚙️ Settings"
-# MAPPINGS_PAGE_TITLE = "🔀 Mapping"
-# GENERATORS_PAGE_TITLE = "👨‍👩‍👧‍👦 Generators"
-# GENERATE_PAGE_TITLE = "🏃‍♂️ Generate"
-# EXPORT_PAGE_TITLE = "🔽 Export"
-# PROPERTIES_PAGE_TITLE = "🧩 Properties"
-
-# OVERVIEW_SHORT_TITLE = "Overview"
-# DESIGN_SHORT_TITLE = "Design"
-# IMPORT_SHORT_TITLE = "Import"
-# PROPERTIES_SHORT_TITLE = "Properties"
-# MAPPINGS_SHORT_TITLE = "Mapping"
-# GENERATE_SHORT_TITLE = "Generate"
-# EXPORT_SHORT_TITLE = "Export"
-# SETTINGS_SHORT_TITLE = "Settings"
-# GENERATORS_SHORT_TITLE = "Generators"

mock_generators/generators/ecdff22c.py

@@ -1,12 +1,3 @@
-# from base_generators.bool_generator import BoolGenerator
-
-# class ConstantBoolGenerator(BoolGenerator):
-
-#     def generate(self, *args)->bool:
-#         value = bool(args[0])
-#         return value
-
-
 # Do not change function name or arguments
 def generate(args: list[any]):
     value = args[0]

mock_generators/list_utils.py

@@ -0,0 +1,10 @@
+import datetime
+
+def clean_list(list: list[any]) -> list[any]:
+    # Remove None values from a list
+    result = [item for item in list if item is not None]
+
+    # Convert datetime to string
+    result = [item.isoformat() if isinstance(item, datetime.date) else item for item in result]
+
+    return result

mock_generators/models/data_import.py

@@ -13,10 +13,10 @@ def file_schema_for_property(property: PropertyMapping)-> dict:
     # Why is it so tough to catch datetime objects?
     if isinstance(sample_value, datetime.date):
         sample_value = sample_value.isoformat()
-    if type(sample_value) is datetime:
-        sample_value = sample_value.isoformat()
-    if property.type == GeneratorType.DATETIME:
-        sample_value = sample_value.isoformat()
+    # if type(sample_value) is datetime:
+    #     sample_value = sample_value.isoformat()
+    # if property.type == GeneratorType.DATETIME:
+    #     sample_value = sample_value.isoformat()
 
     result = {
         "name": property.name,
@@ -60,6 +60,14 @@ def mapping_model_node_mappings(node:NodeMapping)->list[dict[str,str]]:
         })
     return result
 
+def mapping_model_relationship_mappings(rel:RelationshipMapping)->list[dict[str,str]]:
+    result = []
+    for property in rel.properties.values():
+        result.append({
+            "field": property.name,
+        })
+    return result
+
 class DataImporterJson():
     # Object for converting mapping info to a JSON file that can be imported into the Data Importer
 
@@ -289,7 +297,7 @@ def add_relationship(
             {
                     f"{relationship.rid}":{
                         "relationshipSchema": f'{relationship.rid}',
-                        "mappings":[],
+                        "mappings":mapping_model_node_mappings(relationship),
                         # NOTE: The prefixing
                         "sourceMappings": [{
                             "field":f"_from_{relationship.from_node.key_property.name}"

mock_generators/models/node_mapping.py

@@ -2,6 +2,7 @@
 from models.generator import Generator
 import sys
 import logging
+from list_utils import clean_list
 
 # TODO: Should have made these dataclasses
 class NodeMapping():
@@ -54,7 +55,7 @@ def to_dict(self):
                 "labels": self.labels,
                 "properties": {key: property.to_dict() for (key, property) in self.properties.items() if property.type is not None},
                 "count_generator": self.count_generator.to_dict() if self.count_generator is not None else None,
-                "count_args": self.count_args,
+                "count_args": clean_list(self.count_args),
                 "key_property" : self.key_property.to_dict() if self.key_property is not None else None
             }
 

mock_generators/models/property_mapping.py

@@ -1,4 +1,5 @@
 from models.generator import Generator, GeneratorType
+from list_utils import clean_list
 import logging
 
 class PropertyMapping():
@@ -28,7 +29,9 @@ def __init__(
         self.args = args
 
     def __str__(self):
-        return f"PropertyMapping(pid={self.pid}, name={self.name}, type={self.type}, generator={self.generator.name}, generator_id={self.generator.id}, args={self.args})"
+        name = self.name if self.name is not None else "<unnamed>"
+        generator = self.generator if self.generator is not None else "<no_generator_assigned>"
+        return f"PropertyMapping(pid={self.pid}, name={name}, type={self.type}, generator={generator}, args={self.args}"
 
     def __repr__(self):
         return self.__str__()
@@ -40,9 +43,9 @@ def to_dict(self):
         return {
             "pid": self.pid,
             "name": self.name,
-            "type": self.type.to_string(),
-            "generator": self.generator.to_dict(),
-            "args": self.args
+            "type": self.type.to_string() if self.type is not None else None,
+            "generator": self.generator.to_dict() if self.generator is not None else None,
+            "args": clean_list(self.args)
         }
 
     def ready_to_generate(self):

mock_generators/models/relationship_mapping.py

@@ -6,6 +6,7 @@
 import random
 import sys
 from copy import deepcopy
+from list_utils import clean_list
 
 class RelationshipMapping():
 
@@ -50,12 +51,12 @@ def __init__(
         self.to_node = to_node
         self.properties = properties
         self.count_generator = count_generator
-        self.count_args = count_args
+        self.count_args = clean_list(count_args)
         self.generated_values = None
         self.filter_generator = filter_generator
-        self.filter_generator_args = filter_args
+        self.filter_generator_args = clean_list(filter_args)
         self.assignment_generator = assignment_generator
-        self.assignment_args = assignment_args
+        self.assignment_args = clean_list(assignment_args)
 
     def __str__(self):
         return f"RelationshipMapping(rid={self.rid}, type={self.type}, from_node={self.from_node}, to_node={self.to_node}, properties={self.properties}, count_generator={self.count_generator}, count_args={self.count_args})"

mock_generators/tabs/data_importer.py

@@ -7,7 +7,7 @@ def data_importer_tab():
 
     col1, col2 = st.columns([1,11])
     with col1:
-        st.image("mock_generators/media/abstract.gif")
+        st.image("mock_generators/media/signpost.gif")
     with col2:
         st.write(f"Data Importer App.\n\nUse the [Data Importer Tool](https://data-importer.graphapp.io/) to upload generated .zip file to for review and ingesetion to a Neo4j database instance.")
     st.markdown("--------")

mock_generators/tabs/importing_tab.py

@@ -77,7 +77,9 @@ def import_tab():
             st.write("Select an import file:")
 
             folder_files_expander(
-                folder_path=st.session_state[IMPORTS_PATH], file_selected=file_selected, file_selection_button_text="Load this file")
+                folder_path=st.session_state[IMPORTS_PATH], file_selected=file_selected, 
+                specific_extension=".json",
+                file_selection_button_text="Load this file")
 
         elif import_option == "Upload":
             # Upload a new file

mock_generators/widgets/property_row.py

@@ -31,9 +31,11 @@ def property_row(
             if recommended_generator is None:
                 logging.info(f'No recommended generator for property {existing_name}')
         name = st.text_input("Property Name",value=existing_name, key=f"{type}_{pid}_property_{index}_name")
-        if name != "" and name[0] == "_":
-            st.error("Property names cannot start with an underscore")
-            name = None
+        if name is None or name == "":
+            st.error("Property name cannot be empty")
+        # if name[0] == "_":
+        #     st.error("Property names cannot start with an underscore")
+        #     name = None
 
     # Property type
     with pc2:
@@ -140,6 +142,7 @@ def sort_by_name(generator: Generator):
         st.write("Options")
         should_ignore = st.checkbox("Exclude/ignore", value=False,  key=f"{type}_{pid}_property_{index}__ignore")
 
+    # TODO: Update to fail validation if any property items failed (ie empty name)
     if should_ignore == True:
         return PropertyMapping.empty()