diff --git a/mock_generators/__init__.py b/mock_generators/__init__.py
index e69de29..b31ecc4 100644
--- a/mock_generators/__init__.py
+++ b/mock_generators/__init__.py
@@ -0,0 +1,2 @@
+
+__version__ = "0.1.0"
\ No newline at end of file
diff --git a/mock_generators/app.py b/mock_generators/app.py
index 6caebff..8ec4a87 100644
--- a/mock_generators/app.py
+++ b/mock_generators/app.py
@@ -1,16 +1,9 @@
import streamlit as st
from constants import *
-from tabs.config_tab import config_tab
-from tabs.generators_tab import generators_tab
-from tabs.new_generator_tab import create_tab
-from tabs.mapping_tab import mapping_tab
-from tabs.generate_tab import generate_tab
-from tabs.export_tab import export_tab
from tabs.importing_tab import import_tab
from tabs.design_tab import design_tab
from tabs.data_importer import data_importer_tab
-from models.mapping import Mapping
-
+from config import load_generators
# SETUP
st.set_page_config(layout="wide")
@@ -42,9 +35,9 @@
if CODE_TEMPLATE_FILE not in st.session_state:
st.session_state[CODE_TEMPLATE_FILE] = DEFAULT_CODE_TEMPLATES_FILE
if MAPPINGS not in st.session_state:
- st.session_state[MAPPINGS] = Mapping(
- nodes={},
- relationships={})
+ st.session_state[MAPPINGS] = None
+
+load_generators()
# UI
st.title("Mock Graph Data Generator")
@@ -54,31 +47,15 @@
imported_file = None
# Streamlit runs from top-to-bottom from tabs 1 through 8. This is essentially one giant single page app. Earlier attempt to use Streamlit's multi-page app functionality resulted in an inconsistent state between pages.
-tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8, tab9 = st.tabs(["Config >", "Design >", "Import >", "Mapping >", "Search Generators >", "Add New Generator >", "Generate >", "Export >", "Data Importer"])
-
-with tab1:
- config_tab()
-with tab2:
+t1, t2, t5 = st.tabs([
+ "① Design",
+ "② Generate",
+ "③ Data Importer"
+])
+with t1:
design_tab()
-
-with tab3:
+with t2:
import_tab()
-
-with tab4:
- mapping_tab()
-
-with tab5:
- generators_tab()
-
-with tab6:
- create_tab()
-
-with tab7:
- generate_tab()
-
-with tab8:
- export_tab()
-
-with tab9:
+with t5:
data_importer_tab()
\ No newline at end of file
diff --git a/mock_generators/config.py b/mock_generators/config.py
new file mode 100644
index 0000000..d814cb4
--- /dev/null
+++ b/mock_generators/config.py
@@ -0,0 +1,23 @@
+import streamlit as st
+from constants import *
+from file_utils import load_json, load_string
+from models.generator import Generator, generators_from_json
+import os
+import sys
+import logging
+from widgets.folder_files import folder_files_expander
+
+def load_generators():
+
+ spec_filepath = st.session_state[SPEC_FILE]
+ generators = st.session_state[GENERATORS]
+ try:
+ with open(spec_filepath) as input:
+ # generators_file = input.read()
+ generators_json = load_json(spec_filepath)
+ new_generators = generators_from_json(generators_json)
+ if generators != new_generators:
+ st.session_state[GENERATORS] = new_generators
+
+ except FileNotFoundError:
+ st.error('File not found.')
\ No newline at end of file
diff --git a/mock_generators/constants.py b/mock_generators/constants.py
index 1085b0c..28ba3d3 100644
--- a/mock_generators/constants.py
+++ b/mock_generators/constants.py
@@ -1,7 +1,7 @@
import streamlit as st
# Default local filepaths
-DEFAULT_GENERATORS_SPEC_FILE = "mock_generators/generators.json"
+DEFAULT_GENERATORS_SPEC_FILE = "mock_generators/named_generators.json"
DEFAULT_GENERATORS_CODE_PATH = "mock_generators/generators"
DEFAULT_ARROWS_SAMPLE_PATH = "mock_generators/samples/arrows.json"
DEFAULT_IMPORTS_PATH = "mock_generators/imports"
diff --git a/mock_generators/generate.py b/mock_generators/generate.py
new file mode 100644
index 0000000..e4aea24
--- /dev/null
+++ b/mock_generators/generate.py
@@ -0,0 +1,94 @@
+import streamlit as st
+from constants import *
+from models.mapping import Mapping
+from logic.generate_csv import generate_csv
+from logic.generate_data_import import generate_data_importer_json
+import os
+import logging
+import sys
+import zipfile
+from datetime import datetime
+
+def generate_data(mapping: Mapping):
+
+ export_folder = st.session_state[EXPORTS_PATH]
+ zips_folder = st.session_state[ZIPS_PATH]
+ imported_filename = st.session_state[IMPORTED_FILENAME]
+
+ # TODO: Implement better filename cleaning
+ # TODO: Breaks when using a copy and pasted file
+ export_zip_filename = f'{imported_filename}'.lower()
+ export_zip_filename = export_zip_filename.replace(".json", "")
+ export_zip_filename.replace(" ", "_")
+ export_zip_filename.replace(".", "_")
+
+ # Stop if no mapping data available
+ if len(mapping.nodes) == 0:
+ st.error('No nodes to generate data for. Map at least one noded.')
+ st.stop()
+ return
+
+ # Generate values from mappings
+ for _, node in mapping.nodes.items():
+ # logging.info(f'Generating data for node: {node}')
+ if len(node.properties) == 0:
+ st.error(f'Node {node.caption} has no properties. Add at least one property to generate data.')
+ st.stop()
+ return
+ node.generate_values()
+
+ for _, rel in mapping.relationships.items():
+ rel.generate_values()
+
+ # Delete all files in export folder first
+ dir = export_folder
+ for f in os.listdir(dir):
+ os.remove(os.path.join(dir, f))
+
+ # Data Importer Options
+ success = generate_csv(
+ mapping,
+ export_folder=export_folder)
+
+ # Check that data was generated
+ if success == False:
+ st.error('Error generating data. Check console for details.')
+ # st.stop()
+ # return
+
+ success = generate_data_importer_json(
+ mapping,
+ export_folder=export_folder,
+ export_filename=DEFAULT_DATA_IMPORTER_FILENAME)
+
+ # Check that data-import data was generated
+ if success == False:
+ st.error('Error generating data-import json. Check console for details.')
+ # st.stop()
+ # return
+
+ # Only attempt to zip files if data generation was successful
+ if success:
+ try:
+ # Create zip file, appended with time created
+ # now = str(datetime.now().isoformat())
+ zip_path = f'{zips_folder}/{export_zip_filename}.zip'
+ logging.info(f'generate_tab: Creating zip file: {zip_path}')
+ with zipfile.ZipFile(f'{zip_path}', 'w', zipfile.ZIP_DEFLATED) as zipf:
+ # zipdir(export_folder, zipf)
+ path = export_folder
+ for root, dirs, files in os.walk(path):
+ for file in files:
+ if file[0] =='.':
+ # Skip hidden files
+ continue
+ zipf.write(os.path.join(root, file),
+ os.path.relpath(os.path.join(root, file),
+ os.path.join(path, '..')))
+ except:
+ st.error(f'Error creating zip file: {sys.exc_info()[0]}')
+ # st.stop()
+ return
+
+ if success == True:
+ st.success('Data generated successfully.')
diff --git a/mock_generators/generate_mapping.py b/mock_generators/generate_mapping.py
new file mode 100644
index 0000000..9d5b738
--- /dev/null
+++ b/mock_generators/generate_mapping.py
@@ -0,0 +1,272 @@
+# Builds mapping file from specially formatted arrows.app JSON file
+
+import json
+from models.mapping import Mapping
+from models.node_mapping import NodeMapping
+from models.relationship_mapping import RelationshipMapping
+from models.property_mapping import PropertyMapping
+from models.generator import Generator
+import logging
+import uuid
+
+def generator_for_raw_property(
+ property_value: str,
+ generators: dict[str, Generator]
+ ) -> tuple[Generator, list[any]]:
+ """Returns a generator and args for a property"""
+ # Sample expected string: "{\"company_name\":[]}"
+
+ # Check that the property info is notated for mock data generation use
+ # leading_bracket = property_value[0]
+ # trailing_bracket = property_value[-1]
+ # if leading_bracket != "{" or trailing_bracket != "}":
+ # logging.warning(f'generate_mapping.py: generator_for_raw_property: property_value not wrapped in {{ and }}. Skipping generator assignment for property_value: {property_value}')
+ # return (None, None)
+
+ # # The property value should be a JSON string. Convert to a dict obj
+ # json_string = property_value[1:-1]
+
+ try:
+ obj = json.loads(property_value)
+ except Exception as e:
+ logging.info(f'generate_mapping.py: generator_for_raw_property: Could not parse JSON string: {property_value}. Skipping generator assignment for property_value: {property_value}')
+ return (None, None)
+
+ # Should only ever be one
+ for key, value in obj.items():
+ generator_id = key
+ generator = generators.get(generator_id, None)
+ if generator is None:
+ logging.error(f'generate_mapping.py: generator_for_raw_property: generator_id {generator_id} not found in generators. Skipping generator assignment for property_value: {property_value}')
+ return None
+
+ args = value
+ return (generator, args)
+
+def propertymappings_for_raw_properties(
+ raw_properties: dict[str, str],
+ generators: dict[str, Generator]
+ ) -> dict[str, PropertyMapping]:
+ """Returns a list of property mappings for a node or relationship"""
+ property_mappings = {}
+ for key, value in raw_properties.items():
+ generator, args = generator_for_raw_property(value, generators)
+ if generator is None:
+ # TODO: Insert PropertyMapping with no generator? Use literal value?
+ continue
+
+ pid = str(uuid.uuid4())[:8]
+ property_mapping = PropertyMapping(
+ pid = pid,
+ name=key,
+ generator=generator,
+ args=args
+ )
+ property_mappings[pid] = property_mapping
+ return property_mappings
+
+def node_mappings_from(
+ node_dicts: list,
+ generators: dict[str, Generator]
+ ) -> dict[str, NodeMapping]:
+ """Converts node information from JSON file to mapping objects"""
+ # Sample node_dict
+ # {
+ # "id": "n1",
+ # "position": {
+ # "x": 284.5,
+ # "y": -204
+ # },
+ # "caption": "Company",
+ # "labels": [],
+ # "properties": {
+ # "name": "{{\"company_name\":[]}}",
+ # "uuid": "{{\"uuid\":[8]}}",
+ # "{{count}}": "{{\"int\":[1]}}",
+ # "{{key}}": "uuid"
+ # },
+ # "style": {}
+ # }
+
+ node_mappings = {}
+ for node_dict in node_dicts:
+
+ # Check for required data in raw node dict from arrows.app json
+ position = node_dict.get("position", None)
+ if position is None:
+ logging.warning(f"generate_mappings: node_mappings_from: node properties is missing position key from: {node_dict}: Skipping {node_dict}")
+ continue
+
+ caption = node_dict.get("caption", None)
+ if caption is None:
+ logging.warning(f"generate_mappings: node_mappings_from: node properties is missing caption key from: {node_dict}: Skipping {node_dict}")
+ continue
+
+
+ # Check for required properties dict
+ properties = node_dict.get("properties", None)
+ if properties is None:
+ logging.warning(f"generate_mappings: node_mappings_from: dict is missing properties: {node_dict}. Can not configure for data generation. Skipping node.")
+ continue
+
+ # Determine count generator to use
+ count_generator_config = properties.get("{count}", None)
+ if count_generator_config is None:
+ logging.warning(f"generate_mappings: node_mappings_from: node properties is missing {{count}} key from properties: {properties}: Skipping {node_dict}")
+ continue
+
+ # Get string name for key property. Value should be an unformatted string
+ key = properties.get("{key}", None)
+ if key is None:
+ logging.warning(f"generate_mappings: node_mappings_from: node properties is missing {{key}}: Skipping {node_dict}")
+ continue
+
+ # Get proper generators for count generator
+ count_generator, count_args = generator_for_raw_property(count_generator_config, generators)
+
+ # Create property mappings for properties
+ property_mappings = propertymappings_for_raw_properties(properties, generators)
+
+ # Assign correct property mapping as key property
+ # logging.info(f'generate_mappings: node_mappings_from: key_property: {key}, property_mappings: {property_mappings}')
+ key_property = next((v for k,v in property_mappings.items() if v.name == key), None)
+ if key_property is None:
+ logging.warning(f"generate_mappings: node_mappings_from: No key property mapping found for node: {node_dict} - key name: {key}. Skipping node.")
+ continue
+
+ # Create node mapping
+ node_mapping = NodeMapping(
+ nid = node_dict["id"],
+ labels = node_dict["labels"],
+ properties = property_mappings,
+ count_generator=count_generator,
+ count_args=count_args,
+ key_property=key_property,
+ position = position,
+ caption=caption
+ )
+ # TODO:
+ # Run generators
+ node_mappings[node_mapping.nid] = node_mapping
+ return node_mappings
+
+def relationshipmappings_from(
+ relationship_dicts: list[dict],
+ nodes: dict[str, NodeMapping],
+ generators: dict[str, Generator]
+ ) -> dict[str,RelationshipMapping]:
+ # Sample relationship_dict
+ # {
+ # "id": "n0",
+ # "fromId": "n1",
+ # "toId": "n0",
+ # "type": "EMPLOYS",
+ # "properties": {
+ # "{count}": "{\"int\":[10]}",
+ # "{assignment}": "{\"exhaustive_random\":[]}",
+ # "{filter}": "{string_from_list:[]}"
+ # },
+ # "style": {}
+ # },
+ relationshipmappings = {}
+ for relationship_dict in relationship_dicts:
+ # Check for required data in raw node dict from arrows.app json
+
+ rid = relationship_dict.get("id", None)
+ if rid is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing 'id' key from: {relationship_dict}: Skipping {relationship_dict}")
+ continue
+ type = relationship_dict.get("type", None)
+ if type is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing 'type' key from: {relationship_dict}: Skipping {relationship_dict}")
+ continue
+ from_id = relationship_dict.get("fromId", None)
+ if from_id is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing 'fromId' key from: {relationship_dict}: Skipping {relationship_dict}")
+ continue
+
+ to_id = relationship_dict.get("toId", None)
+ if to_id is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing 'toId' key from: {relationship_dict}: Skipping {relationship_dict}")
+ continue
+
+ # Check for required properties dict
+ properties = relationship_dict.get("properties", None)
+ if properties is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: dict is missing properties: {relationship_dict}. Can not configure for data generation. Skipping relationship.")
+ continue
+
+ # Determine count generator to use
+ count_generator_config = properties.get("{count}", None)
+ if count_generator_config is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing '{{count}}' key from properties: {properties}: Skipping {relationship_dict}")
+ continue
+
+ assignment_generator_config = properties.get("{assignment}", None)
+ # If missing, use ExhaustiveRandom
+ if assignment_generator_config is None:
+ assignment_generator_config = "{\"exhaustive_random\":[]}"
+
+ # Get proper generators for count generator
+ count_generator, count_args = generator_for_raw_property(count_generator_config, generators)
+
+ # Create property mappings for properties
+ property_mappings = propertymappings_for_raw_properties(properties, generators)
+
+ assignment_generator, assignment_args = generator_for_raw_property(assignment_generator_config, generators)
+
+ from_node = nodes.get(from_id, None)
+ if from_node is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: No node mapping found for relationship: {relationship_dict} - fromId: {from_id}. Skipping relationship.")
+ continue
+
+ to_node = nodes.get(to_id, None)
+ if to_node is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: No node mapping found for relationship: {relationship_dict} - toId: {to_id}. Skipping relationship.")
+ continue
+
+ # Create relationship mapping
+ relationship_mapping = RelationshipMapping(
+ rid = rid,
+ type = type,
+ from_node = from_node,
+ to_node = to_node ,
+ count_generator=count_generator,
+ count_args=count_args,
+ properties=property_mappings,
+ assignment_generator= assignment_generator,
+ assignment_args=assignment_args
+ )
+ relationshipmappings[relationship_mapping.rid] = relationship_mapping
+
+ return relationshipmappings
+
+
+def mapping_from_json(
+ json_file: str,
+ generators: list[Generator]) -> Mapping:
+
+ try:
+ # Validate json file
+ loaded_json = json.loads(json_file)
+ except Exception as e:
+ raise Exception(f"generate_mappings: mapping_from_json: Error loading JSON file: {e}")
+
+ # Extract and process nodes
+ node_dicts = loaded_json.get("nodes", None)
+ if node_dicts is None:
+ raise Exception(f"generate_mappings: mapping_from_json: No nodes found in JSON file: {json}")
+ relationship_dicts = loaded_json.get("relationships", None)
+ if relationship_dicts is None:
+ raise Exception(f"generate_mappings: mapping_from_json: No relationships found in JSON file: {json}")
+
+ # Convert source information to mapping objects
+ nodes = node_mappings_from(node_dicts, generators)
+ relationships = relationshipmappings_from(relationship_dicts, nodes, generators)
+
+ # Create mapping object
+ mapping = Mapping(
+ nodes=nodes,
+ relationships=relationships
+ )
+ return mapping
diff --git a/mock_generators/generators/05add148.py b/mock_generators/generators/05add148.py
index 24a7933..9125da6 100644
--- a/mock_generators/generators/05add148.py
+++ b/mock_generators/generators/05add148.py
@@ -3,9 +3,12 @@
# Do not change function name or arguments
def generate(args: list[any]):
- domain = args[0]
- if domain == "" or domain is None:
+ result = None
+ if len(args) != 0:
+ domain = args[0]
+ if domain != "" and domain is not None:
+ result = fake.email(domain=domain)
+ if result is None:
result = fake.email()
- else:
- result = fake.email(domain=domain)
+
return result
\ No newline at end of file
diff --git a/mock_generators/imports/simple_org_chart.json b/mock_generators/imports/simple_org_chart.json
deleted file mode 100644
index 10ab77a..0000000
--- a/mock_generators/imports/simple_org_chart.json
+++ /dev/null
@@ -1,126 +0,0 @@
-{
- "nodes": [
- {
- "id": "n0",
- "position": {
- "x": -135,
- "y": -74
- },
- "caption": "Person",
- "labels": [],
- "properties": {
- "first name": "string",
- "email": "string",
- "uuid": "string"
- },
- "style": {}
- },
- {
- "id": "n1",
- "position": {
- "x": 163.93435707302254,
- "y": -64.06262614269966
- },
- "caption": "Roles",
- "labels": [],
- "properties": {
- "title": "string",
- "uuid": "string"
- },
- "style": {}
- },
- {
- "id": "n2",
- "position": {
- "x": 304.44455275328244,
- "y": -307.43342404235483
- },
- "caption": "Company",
- "labels": [],
- "properties": {
- "name": "string",
- "uuid": "string"
- },
- "style": {}
- }
- ],
- "relationships": [
- {
- "id": "n0",
- "fromId": "n0",
- "toId": "n1",
- "type": "WORKS_AS",
- "properties": {},
- "style": {}
- },
- {
- "id": "n1",
- "fromId": "n2",
- "toId": "n1",
- "type": "EMPLOYS",
- "properties": {},
- "style": {}
- },
- {
- "id": "n2",
- "fromId": "n1",
- "toId": "n1",
- "type": "WORKS_FOR",
- "properties": {},
- "style": {}
- }
- ],
- "style": {
- "font-family": "sans-serif",
- "background-color": "#ffffff",
- "background-image": "",
- "background-size": "100%",
- "node-color": "#ffffff",
- "border-width": 4,
- "border-color": "#000000",
- "radius": 50,
- "node-padding": 5,
- "node-margin": 2,
- "outside-position": "auto",
- "node-icon-image": "",
- "node-background-image": "",
- "icon-position": "inside",
- "icon-size": 64,
- "caption-position": "inside",
- "caption-max-width": 200,
- "caption-color": "#000000",
- "caption-font-size": 50,
- "caption-font-weight": "normal",
- "label-position": "inside",
- "label-display": "pill",
- "label-color": "#000000",
- "label-background-color": "#ffffff",
- "label-border-color": "#000000",
- "label-border-width": 4,
- "label-font-size": 40,
- "label-padding": 5,
- "label-margin": 4,
- "directionality": "directed",
- "detail-position": "inline",
- "detail-orientation": "parallel",
- "arrow-width": 5,
- "arrow-color": "#000000",
- "margin-start": 5,
- "margin-end": 5,
- "margin-peer": 20,
- "attachment-start": "normal",
- "attachment-end": "normal",
- "relationship-icon-image": "",
- "type-color": "#000000",
- "type-background-color": "#ffffff",
- "type-border-color": "#000000",
- "type-border-width": 0,
- "type-font-size": 16,
- "type-padding": 5,
- "property-position": "outside",
- "property-alignment": "colon",
- "property-color": "#000000",
- "property-font-size": 16,
- "property-font-weight": "normal"
- }
-}
\ No newline at end of file
diff --git a/mock_generators/logic/generate_mapping.py b/mock_generators/logic/generate_mapping.py
new file mode 100644
index 0000000..7025b22
--- /dev/null
+++ b/mock_generators/logic/generate_mapping.py
@@ -0,0 +1,315 @@
+# Builds mapping file from specially formatted arrows.app JSON file
+
+import json
+from models.mapping import Mapping
+from models.node_mapping import NodeMapping
+from models.relationship_mapping import RelationshipMapping
+from models.property_mapping import PropertyMapping
+from models.generator import Generator
+import logging
+import uuid
+
+def generator_for_raw_property(
+ property_value: str,
+ generators: dict[str, Generator]
+ ) -> tuple[Generator, list[any]]:
+ """Returns a generator and args for specially formatted property values from the arrows.app JSON file"""
+ # Sample expected string: "{\"company_name\":[]}"
+
+ # Throws an error if a generator can not be found
+
+ obj = json.loads(property_value)
+
+ if len(obj) == 0:
+ raise Exception(f'generate_mapping.py: generator_for_raw_property: Expected dictionary object from json string not found: {property_value}')
+
+ generator = None
+ args = None
+ # Should only be one item, if not take the last
+ for key, value in obj.items():
+ generator_id = key
+ generator = generators.get(generator_id, None)
+ if generator is None:
+ raise Exception(f'generate_mapping.py: generator_for_raw_property: generator_id {generator_id} not found in generators.')
+ args = value
+ return (generator, args)
+
+def propertymappings_for_raw_properties(
+ raw_properties: dict[str, str],
+ generators: dict[str, Generator]
+ ) -> dict[str, PropertyMapping]:
+ """Returns a list of property mappings for a node or relationship"""
+
+ # raw_properties is a dict of key value pairs from properties value from an entry from the arrows.app JSON file. Example:
+ # {
+ # "name": "{\"company_name\":[]}",
+ # "uuid": "{\"uuid\":[8]}",
+ # "{count}": "{\"int\":[1]}",
+ # "{key}": "uuid"
+ # },
+
+ property_mappings = {}
+
+ if raw_properties is None or len(raw_properties) == 0:
+ raise Exception(f'generate_mapping.py: propertymappings_for_raw_properties: No raw_properties assignment received.')
+
+ if generators is None or len(generators) == 0:
+ raise Exception(f'generate_mapping.py: propertymappings_for_raw_properties: No generators assignment received.')
+
+ for key, value in raw_properties.items():
+ # Skip any keys with { } (brackets) as these are special cases for defining count/assignment/filter generators
+ if key.startswith('{') and key.endswith('}'):
+ continue
+
+ # Only process values with string { } (brackets)
+ if not isinstance(value, str) or not value.startswith('{') or not value.endswith('}'):
+ property_mappings[key] = value
+ continue
+ try:
+ generator, args = generator_for_raw_property(value, generators)
+ if generator is None:
+ # TODO: Insert PropertyMapping with no generator? Use literal value?
+ logging.warning(f'generate_mapping.py: propertymappings_for_raw_properties: could not find generator for key: {key}, property_value: {value}')
+ continue
+
+ pid = str(uuid.uuid4())[:8]
+ property_mapping = PropertyMapping(
+ pid = pid,
+ name=key,
+ generator=generator,
+ args=args
+ )
+ property_mappings[pid] = property_mapping
+ except Exception as e:
+ logging.warning(f'generate_mapping.py: propertymappings_for_raw_properties: could not create property mapping for key: {key}, property_value: {value}: {e}')
+ continue
+ return property_mappings
+
+def node_mappings_from(
+ node_dicts: list,
+ generators: dict[str, Generator]
+ ) -> dict[str, NodeMapping]:
+ """Converts node information from JSON file to mapping objects"""
+ # Sample node_dict
+ # {
+ # "id": "n1",
+ # "position": {
+ # "x": 284.5,
+ # "y": -204
+ # },
+ # "caption": "Company",
+ # "labels": [],
+ # "properties": {
+ # "name": "{\"company_name\":[]}",
+ # "uuid": "{\"uuid\":[8]}}",
+ # "{count}": "{{"int\":[1]}",
+ # "{key}": "uuid"
+ # },
+ # "style": {}
+ # }
+
+ node_mappings = {}
+ for node_dict in node_dicts:
+
+ # Check for required data in raw node dict from arrows.app json
+ position = node_dict.get("position", None)
+ if position is None:
+ logging.warning(f"generate_mappings: node_mappings_from: node properties is missing position key from: {node_dict}: Skipping {node_dict}")
+ continue
+
+ caption = node_dict.get("caption", None)
+ if caption is None:
+ logging.warning(f"generate_mappings: node_mappings_from: node properties is missing caption key from: {node_dict}: Skipping {node_dict}")
+ continue
+
+
+ # Check for required properties dict
+ properties = node_dict.get("properties", None)
+ if properties is None:
+ logging.warning(f"generate_mappings: node_mappings_from: dict is missing properties: {node_dict}. Can not configure for data generation. Skipping node.")
+ continue
+
+ # Determine count generator to use
+ count_generator_config = properties.get("{count}", None)
+ if count_generator_config is None:
+ logging.warning(f"generate_mappings: node_mappings_from: node properties is missing {{count}} key from properties: {properties}: Skipping {node_dict}")
+ continue
+
+ # Get string name for key property. Value should be an unformatted string
+ key = properties.get("{key}", None)
+ if key is None:
+ logging.warning(f"generate_mappings: node_mappings_from: node properties is missing {{key}}: Skipping {node_dict}")
+ continue
+
+ # Get proper generators for count generator
+ try:
+ count_generator, count_args = generator_for_raw_property(count_generator_config, generators)
+ except Exception as e:
+ logging.warning(f"generate_mappings: node_mappings_from: could not find count generator for node: {node_dict}: {e}")
+ continue
+
+ # Create property mappings for properties
+ try:
+ property_mappings = propertymappings_for_raw_properties(properties, generators)
+ except Exception as e:
+ logging.warning(f"generate_mappings: node_mappings_from: could not create property mappings for node: {node_dict}: {e}")
+ continue
+
+ # Assign correct property mapping as key property
+ # logging.info(f'generate_mappings: node_mappings_from: key_property: {key}, property_mappings: {property_mappings}')
+ key_property = next((v for k,v in property_mappings.items() if v.name == key), None)
+ if key_property is None:
+ logging.warning(f"generate_mappings: node_mappings_from: No key property mapping found for node: {node_dict} - key name: {key}. Skipping node.")
+ continue
+
+ # Create node mapping
+ node_mapping = NodeMapping(
+ nid = node_dict["id"],
+ labels = node_dict["labels"],
+ properties = property_mappings,
+ count_generator=count_generator,
+ count_args=count_args,
+ key_property=key_property,
+ position = position,
+ caption=caption
+ )
+ # TODO:
+ # Run generators
+ node_mappings[node_mapping.nid] = node_mapping
+ return node_mappings
+
+def relationshipmappings_from(
+ relationship_dicts: list[dict],
+ nodes: dict[str, NodeMapping],
+ generators: dict[str, Generator]
+ ) -> dict[str,RelationshipMapping]:
+ # Sample relationship_dict
+ # {
+ # "id": "n0",
+ # "fromId": "n1",
+ # "toId": "n0",
+ # "type": "EMPLOYS",
+ # "properties": {
+ # "{count}": "{\"int\":[10]}",
+ # "{assignment}": "{\"exhaustive_random\":[]}",
+ # "{filter}": "{string_from_list:[]}"
+ # },
+ # "style": {}
+ # },
+ relationshipmappings = {}
+ for relationship_dict in relationship_dicts:
+ # Check for required data in raw node dict from arrows.app json
+
+ rid = relationship_dict.get("id", None)
+ if rid is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing 'id' key from: {relationship_dict}: Skipping {relationship_dict}")
+ continue
+ type = relationship_dict.get("type", None)
+ if type is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing 'type' key from: {relationship_dict}: Skipping {relationship_dict}")
+ continue
+ from_id = relationship_dict.get("fromId", None)
+ if from_id is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing 'fromId' key from: {relationship_dict}: Skipping {relationship_dict}")
+ continue
+
+ to_id = relationship_dict.get("toId", None)
+ if to_id is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing 'toId' key from: {relationship_dict}: Skipping {relationship_dict}")
+ continue
+
+ # Check for required properties dict
+ properties = relationship_dict.get("properties", None)
+ if properties is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: dict is missing properties: {relationship_dict}. Can not configure for data generation. Skipping relationship.")
+ continue
+
+ # Determine count generator to use
+ count_generator_config = properties.get("{count}", None)
+ if count_generator_config is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: relationship properties is missing '{{count}}' key from properties: {properties}: Skipping {relationship_dict}")
+ continue
+
+ assignment_generator_config = properties.get("{assignment}", None)
+ # If missing, use ExhaustiveRandom
+ if assignment_generator_config is None:
+ assignment_generator_config = "{\"exhaustive_random\":[]}"
+
+ # Get proper generators for count generator
+ try:
+ count_generator, count_args = generator_for_raw_property(count_generator_config, generators)
+ except Exception as e:
+ logging.warning(f"generate_mappings: relationshipmappings_from: could not find count generator for relationship: {relationship_dict}: {e}")
+ continue
+
+ # Create property mappings for properties
+ try:
+ property_mappings = propertymappings_for_raw_properties(properties, generators)
+ except Exception as e:
+ logging.warning(f"generate_mappings: relationshipmappings_from: could not create property mappings for relationship: {relationship_dict}: {e}")
+ continue
+
+ try:
+ assignment_generator, assignment_args = generator_for_raw_property(assignment_generator_config, generators)
+ except Exception as e:
+ logging.warning(f"generate_mappings: relationshipmappings_from: could not create assignment generator for relationship: {relationship_dict}: {e}")
+ continue
+
+ from_node = nodes.get(from_id, None)
+ if from_node is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: No node mapping found for relationship: {relationship_dict} - fromId: {from_id}. Skipping relationship.")
+ continue
+
+ to_node = nodes.get(to_id, None)
+ if to_node is None:
+ logging.warning(f"generate_mappings: relationshipmappings_from: No node mapping found for relationship: {relationship_dict} - toId: {to_id}. Skipping relationship.")
+ continue
+
+ # Create relationship mapping
+ relationship_mapping = RelationshipMapping(
+ rid = rid,
+ type = type,
+ from_node = from_node,
+ to_node = to_node ,
+ count_generator=count_generator,
+ count_args=count_args,
+ properties=property_mappings,
+ assignment_generator= assignment_generator,
+ assignment_args=assignment_args
+ )
+ relationshipmappings[relationship_mapping.rid] = relationship_mapping
+
+ return relationshipmappings
+
+
+def mapping_from_json(
+ json_file: str,
+ generators: list[Generator]) -> Mapping:
+
+ try:
+ # Validate json file
+ loaded_json = json.loads(json_file)
+ except Exception as e:
+ raise Exception(f"generate_mappings: mapping_from_json: Error loading JSON file: {e}")
+
+ # Extract and process nodes
+ node_dicts = loaded_json.get("nodes", None)
+ if node_dicts is None:
+ raise Exception(f"generate_mappings: mapping_from_json: No nodes found in JSON file: {json}")
+ relationship_dicts = loaded_json.get("relationships", None)
+ if relationship_dicts is None:
+ raise Exception(f"generate_mappings: mapping_from_json: No relationships found in JSON file: {json}")
+
+ # TODO:
+ # Purge orphaned nodes
+
+ # Convert source information to mapping objects
+ nodes = node_mappings_from(node_dicts, generators)
+ relationships = relationshipmappings_from(relationship_dicts, nodes, generators)
+
+ # Create mapping object
+ mapping = Mapping(
+ nodes=nodes,
+ relationships=relationships
+ )
+ return mapping
diff --git a/mock_generators/media/sample_generator.png b/mock_generators/media/sample_generator.png
new file mode 100644
index 0000000..b89abf8
Binary files /dev/null and b/mock_generators/media/sample_generator.png differ
diff --git a/mock_generators/media/sample_node_properties.png b/mock_generators/media/sample_node_properties.png
new file mode 100644
index 0000000..420d62c
Binary files /dev/null and b/mock_generators/media/sample_node_properties.png differ
diff --git a/mock_generators/media/sample_relationship_properties.png b/mock_generators/media/sample_relationship_properties.png
new file mode 100644
index 0000000..3aad420
Binary files /dev/null and b/mock_generators/media/sample_relationship_properties.png differ
diff --git a/mock_generators/models/data_import.py b/mock_generators/models/data_import.py
index 40d93fc..dfe3b93 100644
--- a/mock_generators/models/data_import.py
+++ b/mock_generators/models/data_import.py
@@ -18,9 +18,10 @@ def file_schema_for_property(property: PropertyMapping)-> dict:
# if property.type == GeneratorType.DATETIME:
# sample_value = sample_value.isoformat()
+ type = property.generator.type.to_string()
result = {
"name": property.name,
- "type": property.type.to_string().lower(),
+ "type": type,
"sample": sample_value,
"include": True
}
@@ -47,7 +48,7 @@ def file_schema_for_relationship(relationship: RelationshipMapping)-> list[dict]
def graph_model_property(property: PropertyMapping)-> dict:
result = {
"property": property.name,
- "type": property.type.to_string().lower(),
+ "type": property.generator.type.to_string().lower(),
"identifier": property.pid
}
return result
@@ -198,6 +199,7 @@ def add_relationship(
):
# In case relationshipMapping has not yet generated data
+ # TODO: Need this here anymore?
if relationship.generated_values is None:
relationship.generate_values()
diff --git a/mock_generators/models/generator.py b/mock_generators/models/generator.py
index 55ded5c..f65b676 100644
--- a/mock_generators/models/generator.py
+++ b/mock_generators/models/generator.py
@@ -104,6 +104,21 @@ def __str__(self):
def __repr__(self):
return self.__str__()
+ def __eq__(self, other):
+ if isinstance(other, GeneratorArg) == False:
+ return False
+ if self.type != other.type:
+ return False
+ if self.label != other.label:
+ return False
+ if self.default != other.default:
+ return False
+ if self.hint != other.hint:
+ return False
+ if self.description != other.description:
+ return False
+ return True
+
def to_dict(self):
return {
"type": self.type.to_string(),
@@ -171,6 +186,8 @@ def __init__(
name: str,
description: str,
code_url: str,
+ # Information on arguments the generator CAN take.
+ # Argument values to use during generate are passed in the generate call
args: list[GeneratorArg],
tags: list[str]
):
@@ -202,6 +219,25 @@ def __str__(self):
def __repr__(self):
return self.__str__()
+ def __eq__(self, other):
+ if isinstance(other, Generator) == False:
+ return False
+ if self.id != other.id:
+ return False
+ if self.name != other.name:
+ return False
+ if self.description != other.description:
+ return False
+ if self.code_url != other.code_url:
+ return False
+ if self.args != other.args:
+ return False
+ if self.type != other.type:
+ return False
+ if self.tags != other.tags:
+ return False
+ return True
+
def generate(self, args):
# Args are not the same as the generator args, these are the arg inputs from user
module = __import__(self.import_url(), fromlist=['generate'])
diff --git a/mock_generators/models/node_mapping.py b/mock_generators/models/node_mapping.py
index 95aae35..b8876f2 100644
--- a/mock_generators/models/node_mapping.py
+++ b/mock_generators/models/node_mapping.py
@@ -46,18 +46,42 @@ def __str__(self):
def __repr__(self):
return self.__str__()
+ def __eq__(self, other):
+ if not isinstance(other, NodeMapping):
+ return False
+ if self.nid != other.nid:
+ return False
+ if self.caption != other.caption:
+ return False
+ if self.labels != other.labels:
+ return False
+ if self.properties != other.properties:
+ return False
+ if self.count_generator != other.count_generator:
+ return False
+ if self.count_args != other.count_args:
+ return False
+ if self.key_property != other.key_property:
+ return False
+ return True
def to_dict(self):
- return {
- "nid": self.nid,
- "caption": self.caption,
- "position": self.position,
- "labels": self.labels,
- "properties": {key: property.to_dict() for (key, property) in self.properties.items() if property.type is not None},
- "count_generator": self.count_generator.to_dict() if self.count_generator is not None else None,
- "count_args": clean_list(self.count_args),
- "key_property" : self.key_property.to_dict() if self.key_property is not None else None
- }
+ properties = {}
+ for key, property in self.properties.items():
+ if isinstance(property, PropertyMapping):
+ properties[key] = property.to_dict()
+ continue
+ properties[key] = property
+ return {
+ "nid": self.nid,
+ "caption": self.caption,
+ "position": self.position,
+ "labels": self.labels,
+ "properties": properties,
+ "count_generator": self.count_generator.to_dict() if self.count_generator is not None else None,
+ "count_args": clean_list(self.count_args),
+ "key_property" : self.key_property.to_dict() if self.key_property is not None else None
+ }
def filename(self):
return f"{self.caption.lower()}_{self.nid.lower()}"
@@ -98,9 +122,15 @@ def generate_values(self) -> list[dict]:
try:
for _ in range(count):
node_result = {}
- for property_name, property in self.properties.items():
+ # logging.info(f'node_mapping.py: NodeMapping.generate_values() generating values for node mapping \'{self.caption}\' with properties {self.properties}')
+ for property_id, property in self.properties.items():
+ # Pass literal values
+ if isinstance(property, PropertyMapping) == False:
+ node_result[property_id] = property
+ continue
+ # Have PropertyMapping generate a value
value = property.generate_value()
- node_result[property_name] = value
+ node_result[property.name] = value
# node_result["_uid"] = f"{self.id}_{str(uuid.uuid4())[:8]}"
all_results.append(node_result)
except:
@@ -108,5 +138,5 @@ def generate_values(self) -> list[dict]:
# Store and return all_results
self.generated_values = all_results
- logging.info(f'node_mapping.py: NodeMapping.generate_values() generated {len(self.generated_values)} values for node mapping {self.caption}')
+ # logging.info(f'node_mapping.py: NodeMapping.generate_values() generated {len(self.generated_values)} values for node mapping {self.caption}')
return self.generated_values
\ No newline at end of file
diff --git a/mock_generators/models/property_mapping.py b/mock_generators/models/property_mapping.py
index 8d5849e..e98354a 100644
--- a/mock_generators/models/property_mapping.py
+++ b/mock_generators/models/property_mapping.py
@@ -9,7 +9,7 @@ def empty():
return PropertyMapping(
pid = None,
name = None,
- type = None,
+ # type = None,
generator = None,
args = None
)
@@ -18,20 +18,20 @@ def __init__(
self,
pid: str,
name: str = None,
- type: GeneratorType = None,
+ # type: GeneratorType = None,
generator: Generator = None,
# Args to pass into generator during running
args: list[any] = []):
self.pid = pid
self.name = name
- self.type = type
+ # self.type = type
self.generator = generator
self.args = args
def __str__(self):
name = self.name if self.name is not None else "<unnamed>"
generator = self.generator if self.generator is not None else "<no_generator_assigned>"
- return f"PropertyMapping(pid={self.pid}, name={name}, type={self.type}, generator={generator}, args={self.args}"
+ return f"PropertyMapping(pid={self.pid}, name={name}, generator={generator}, args={self.args}"
def __repr__(self):
return self.__str__()
@@ -43,7 +43,7 @@ def to_dict(self):
return {
"pid": self.pid,
"name": self.name,
- "type": self.type.to_string() if self.type is not None else None,
+ # "type": self.type.to_string() if self.type is not None else None,
"generator": self.generator.to_dict() if self.generator is not None else None,
"args": clean_list(self.args)
}
@@ -51,15 +51,17 @@ def to_dict(self):
def ready_to_generate(self):
if self.name is None:
return False
- if self.type is None:
- return False
+ # if self.type is None:
+ # return False
if self.generator is None:
return False
return True
def generate_value(self):
if self.generator == None:
- logging.error(f'Generator is not set for property {self.name}')
+ logging.error(f'property_mapping.py: generate_value: Generator is not set for property {self.name}')
+ if isinstance(self.args, list) == False:
+ logging.error(f'property_mapping.py: generate_value: Args for generator is not a list for property {self.name}')
result = self.generator.generate(self.args)
return result
diff --git a/mock_generators/models/relationship_mapping.py b/mock_generators/models/relationship_mapping.py
index 0771e13..45c8e3b 100644
--- a/mock_generators/models/relationship_mapping.py
+++ b/mock_generators/models/relationship_mapping.py
@@ -98,15 +98,6 @@ def generate_values(
self,
)-> list[dict]:
- # Sample incoming all_node_values:
- # {
- # "<node_mapping>":[
- # {
- # "first_name":"John",
- # "last_name":"Doe"
- # }
- # ]
- # }
# Make sure from and to nodes have generated values already
if self.from_node.generated_values == None:
@@ -151,9 +142,11 @@ def generate_values(
# Get the key property name and value for the source node record
from_node_key_property_name = self.from_node.key_property.name
- from_node_key_property_value = value_dict[from_node_key_property_name]
+ from_node_key_property_value = value_dict.get(from_node_key_property_name, None)
+ if from_node_key_property_value is None:
+ raise Exception(f"Key property '{from_node_key_property_name}' not found in node: {value_dict}")
- # If count is zero - no relationship to generate for the curent source node
+ # If count is zero - no relationship to generate for the current source node
# Generate a new relationship for each count
for i in range(count):
@@ -197,5 +190,6 @@ def generate_values(
# Store results for reference
self.generated_values = all_results
+ # logging.info(f'relationship_mapping.py: 1 value: {self.generated_values[0]}')
return self.generated_values
\ No newline at end of file
diff --git a/mock_generators/named_generators.json b/mock_generators/named_generators.json
new file mode 100644
index 0000000..4023d76
--- /dev/null
+++ b/mock_generators/named_generators.json
@@ -0,0 +1,454 @@
+{
+ "README":{
+ "content": "This is the default list of all generators used by the app. If you add new generators they will be added to this file. The default_generators.json file contains a copy of this from the repo maintainer(s)"
+ },
+ "uri": {
+ "args": [],
+ "code_url": "mock_generators/generators/05711cac.py",
+ "description": "Random URI with Faker library.",
+ "name": "URL",
+ "tags": [
+ "uri",
+ "url"
+ ],
+ "type": "String"
+ },
+ "email": {
+ "args": [
+ {
+ "default": "",
+ "label": "Optional Domain (ie: company.com)",
+ "type": "String"
+ }
+ ],
+ "code_url": "mock_generators/generators/05add148.py",
+ "description": "Random email with Faker library.",
+ "name": "Email",
+ "tags": [
+ "email"
+ ],
+ "type": "String"
+ },
+ "float_from_list": {
+ "args": [
+ {
+ "default": "",
+ "label": "List of float (ie: 1.0, 2.2, 3.3)",
+ "type": "String"
+ }
+ ],
+ "code_url": "mock_generators/generators/111d38e0.py",
+ "description": "Randomly selected float from a comma-seperated list of options.",
+ "name": "Float from list",
+ "tags": [
+ "float",
+ "list"
+ ],
+ "type": "Float"
+ },
+ "lorem_paragraphs": {
+ "args": [
+ {
+ "default": 1,
+ "label": "Minimum Number",
+ "type": "Integer"
+ },
+ {
+ "default": 10,
+ "label": "Maximum Number",
+ "type": "Integer"
+ }
+ ],
+ "code_url": "mock_generators/generators/338d576e.py",
+ "description": "String generator using the lorem-text package",
+ "name": "Paragraphs",
+ "tags": [
+ "string",
+ "lorem",
+ "ipsum",
+ "paragraph",
+ "paragraphs"
+ ],
+ "type": "String"
+ },
+ "int_range": {
+ "args": [
+ {
+ "default": 1,
+ "label": "Min",
+ "type": "Integer"
+ },
+ {
+ "default": 10,
+ "label": "Max",
+ "type": "Integer"
+ }
+ ],
+ "code_url": "mock_generators/generators/469b37c7.py",
+ "description": "Random integer from a min and max value argument. Argument values are inclusive.",
+ "name": "Int Range",
+ "tags": [
+ "int",
+ "integer",
+ "number",
+ "num",
+ "count"
+ ],
+ "type": "Integer"
+ },
+ "country": {
+ "args": [],
+ "code_url": "mock_generators/generators/470ff56f.py",
+ "description": "Country name generator using the Faker library.",
+ "name": "Country",
+ "tags": [
+ "country",
+ "from"
+ ],
+ "type": "String"
+ },
+ "pure_random": {
+ "args": [],
+ "code_url": "mock_generators/generators/4b0db60a.py",
+ "description": "Randomly assigns to a target node. Duplicates and orphan nodes possible.",
+ "name": "Pure Random",
+ "tags": [
+ "random"
+ ],
+ "type": "Assignment"
+ },
+ "bool": {
+ "args": [
+ {
+ "default": 50,
+ "label": "Percent chance of true (out of 100)",
+ "type": "Integer"
+ }
+ ],
+ "code_url": "mock_generators/generators/57f2df99.py",
+ "description": "Bool generator using the Faker library.",
+ "name": "Bool",
+ "tags": [
+ "bool",
+ "boolean"
+ ],
+ "type": "Bool"
+ },
+ "first_name": {
+ "args": [],
+ "code_url": "mock_generators/generators/58e9ddbb.py",
+ "description": "First name generator using the Faker library",
+ "name": "First Name",
+ "tags": [
+ "first",
+ "name"
+ ],
+ "type": "String"
+ },
+ "last_name": {
+ "args": [],
+ "code_url": "mock_generators/generators/5929c11b.py",
+ "description": "Last name generator using the Faker library.",
+ "name": "Last Name",
+ "tags": [
+ "last",
+ "name"
+ ],
+ "type": "String"
+ },
+ "string_from_list": {
+ "args": [
+ {
+ "default": "",
+ "label": "List of words (ie: alpha, brave, charlie)",
+ "type": "String"
+ }
+ ],
+ "code_url": "mock_generators/generators/5bf1fbd6.py",
+ "description": "Randomly selected string from a comma-seperated list of options.",
+ "name": "String from list",
+ "tags": [
+ "string",
+ "list",
+ "word",
+ "words",
+ "status",
+ "type"
+ ],
+ "type": "String"
+ },
+ "company_name": {
+ "args": [],
+ "code_url": "mock_generators/generators/5e30c30b.py",
+ "description": "Company name generator using the Faker library.",
+ "name": "Company Name",
+ "tags": [
+ "company",
+ "name"
+ ],
+ "type": "String"
+ },
+ "exhaustive_random": {
+ "args": [],
+ "code_url": "mock_generators/generators/73853311.py",
+ "description": "Assigns each source node to a random target node, until target node records are exhausted. No duplicates, no orphan to nodes.",
+ "name": "Exhaustive Random",
+ "tags": [
+ "exhaustive"
+ ],
+ "type": "Assignment"
+ },
+ "uuid": {
+ "args": [
+ {
+ "default": 37,
+ "label": "Limit character length",
+ "type": "Integer"
+ }
+ ],
+ "code_url": "mock_generators/generators/78bc0765.py",
+ "description": "Random UUID 4 hash using Faker library. 37 Characters Max.",
+ "name": "UUID",
+ "tags": [
+ "uuid",
+ "hash",
+ "unique",
+ "uid"
+ ],
+ "type": "String"
+ },
+ "city": {
+ "args": [],
+ "code_url": "mock_generators/generators/92eeddbb.py",
+ "description": "City name generator using the Faker library.",
+ "name": "City",
+ "tags": [
+ "city",
+ "name"
+ ],
+ "type": "String"
+ },
+ "date": {
+ "args": [
+ {
+ "default": "1970-01-01",
+ "label": "Oldest Date",
+ "type": "Datetime"
+ },
+ {
+ "default": "2022-11-24",
+ "label": "Newest Date",
+ "type": "Datetime"
+ }
+ ],
+ "code_url": "mock_generators/generators/ab64469b.py",
+ "description": "Generate a random date between 2 specified dates. Exclusive of days specified.",
+ "name": "Date",
+ "tags": [
+ "date",
+ "datetime",
+ "created",
+ "updated",
+ "at"
+ ],
+ "type": "Datetime"
+ },
+ "technical_phrase": {
+ "args": [],
+ "code_url": "mock_generators/generators/c9a071b5.py",
+ "description": "Technobabble words all lower-cased. Faker Library",
+ "name": "Technical BS Phrase",
+ "tags": [
+ "phrase",
+ "phrases",
+ "technical",
+ "jargon",
+ "task",
+ "description"
+ ],
+ "type": "String"
+ },
+ "catch_phrase": {
+ "args": [],
+ "code_url": "mock_generators/generators/d1ebdc1a.py",
+ "description": "Phrase with first letter capitalized. Faker Library",
+ "name": "Catch Phrase",
+ "tags": [
+ "phrase",
+ "phrases",
+ "catch",
+ "project",
+ "description"
+ ],
+ "type": "String"
+ },
+ "string_from_csv": {
+ "args": [
+ {
+ "default": "",
+ "label": "CSV Filepath",
+ "type": "String",
+ "hint": "mock_generators/datasets/tech_companies.csv",
+ "description":""
+ },
+ {
+ "default": "",
+ "label": "Header Column Field",
+ "type": "String",
+ "hint": "Company Name",
+ "description":""
+ }
+ ],
+ "code_url": "mock_generators/generators/df2bbd43.py",
+ "description": "Random string row value from a specified csv file. Be certain field contains string values.",
+ "name": "String from CSV",
+ "tags": [
+ "csv",
+ " string",
+ " random"
+ ],
+ "type": "String"
+ },
+ "md5": {
+ "args": [
+ {
+ "default": 33,
+ "label": "Limit Character Length",
+ "type": "Integer"
+ }
+ ],
+ "code_url": "mock_generators/generators/e0eb78b0.py",
+ "description": "Random MD5 hash using Faker library. 33 Characters max",
+ "name": "MD5",
+ "tags": [
+ "md5",
+ "hash",
+ "unique"
+ ],
+ "type": "String"
+ },
+ "int_from_list": {
+ "args": [
+ {
+ "default": "",
+ "label": "List of integers (ie: 1, 2, 3)",
+ "type": "String"
+ }
+ ],
+ "code_url": "mock_generators/generators/e56d87a3.py",
+ "description": "Randomly selected int from a comma-seperated list of options. If no list provided, will return 0",
+ "name": "Int from list",
+ "tags": [
+ "int",
+ "integer",
+ "number",
+ "num",
+ "count",
+ "list",
+ "salary",
+ "cost"
+ ],
+ "type": "Integer"
+ },
+ "float_range": {
+ "args": [
+ {
+ "default": 0.0,
+ "label": "Min",
+ "type": "Float"
+ },
+ {
+ "default": 1.0,
+ "label": "Max",
+ "type": "Float"
+ },
+ {
+ "default": 2,
+ "label": "Decimal Places",
+ "type": "Integer"
+ }
+ ],
+ "code_url": "mock_generators/generators/e8cff8c1.py",
+ "description": "Random float between a range. Inclusive.",
+ "name": "Float",
+ "tags": [
+ "float",
+ "decimal",
+ "number",
+ "num",
+ "count",
+ "cost",
+ "price"
+ ],
+ "type": "Float"
+ },
+ "int": {
+ "args": [
+ {
+ "default": 1,
+ "label": "Value",
+ "type": "Integer"
+ }
+ ],
+ "code_url": "mock_generators/generators/ecdff22c.py",
+ "description": "Constant integer value",
+ "name": "Int",
+ "tags": [
+ "int",
+ "integer",
+ "num",
+ "number",
+ "count"
+ ],
+ "type": "Integer"
+ },
+ "lorem_words": {
+ "args": [
+ {
+ "default": 1,
+ "label": "Minimum Number",
+ "type": "Integer"
+ },
+ {
+ "default": 10,
+ "label": "Maximum Number",
+ "type": "Integer"
+ }
+ ],
+ "code_url": "mock_generators/generators/loremtext_words.py",
+ "description": "String generator using the lorem-text package",
+ "name": "Words",
+ "tags": [
+ "words",
+ "lorem",
+ "text",
+ "description"
+ ],
+ "type": "String"
+ },
+ "lorem_sentences": {
+ "args": [
+ {
+ "default": 1,
+ "label": "Minimum Number",
+ "type": "Integer"
+ },
+ {
+ "default": 10,
+ "label": "Maximum Number",
+ "type": "Integer"
+ }
+ ],
+ "code_url": "mock_generators/generators/loremtext_sentence.py",
+ "description": "String generator using the lorem-text package",
+ "name": "Sentences",
+ "tags": [
+ "sentence",
+ "sentences",
+ "lorem",
+ "text",
+ "description"
+ ],
+ "type": "String"
+ }
+}
\ No newline at end of file
diff --git a/mock_generators/tabs/data_importer.py b/mock_generators/tabs/data_importer.py
index ecda82d..a901101 100644
--- a/mock_generators/tabs/data_importer.py
+++ b/mock_generators/tabs/data_importer.py
@@ -5,11 +5,25 @@
def data_importer_tab():
- col1, col2 = st.columns([1,11])
- with col1:
- st.image("mock_generators/media/signpost.gif")
- with col2:
- st.write(f"Data Importer App.\n\nUse the [Data Importer Tool](https://data-importer.graphapp.io/) to upload generated .zip file to for review and ingesetion to a Neo4j database instance.")
+ # col1, col2 = st.columns([1,11])
+ # with col1:
+ # st.image("mock_generators/media/export.gif")
+ # # st.image("mock_generators/media/signpost.gif")
+ # with col2:
+ # st.write(f"Data Importer App.\n\nUse the [Data Importer Tool](https://data-importer.graphapp.io/) to upload generated .zip file to for review and ingesetion to a Neo4j database instance.")
+ with st.expander('Instructions'):
+ st.write("""
+ 1. Connect to your Neo4j instance
+ 2. Click on the '...' options button in the Data Importer header
+ 3. Select 'Open model (with data)'
+ 4. Select the .zip file with the generated data
+ 5. Click the 'Run import' button
+ """)
+ with st.expander("Options"):
+ is_local = st.checkbox("Use HTTP", value=False, help="Select Use HTTP if connecting with a local Neo4j instance.")
st.markdown("--------")
- components.iframe("https://data-importer.graphapp.io/", height=1000, scrolling=False)
\ No newline at end of file
+ if is_local == True:
+ components.iframe("http://data-importer.graphapp.io/", height=1000, scrolling=False)
+ else:
+ components.iframe("https://data-importer.graphapp.io/", height=1000, scrolling=False)
\ No newline at end of file
diff --git a/mock_generators/tabs/design_tab.py b/mock_generators/tabs/design_tab.py
index f390db8..7940a17 100644
--- a/mock_generators/tabs/design_tab.py
+++ b/mock_generators/tabs/design_tab.py
@@ -2,14 +2,128 @@
import streamlit.components.v1 as components
from streamlit_extras.colored_header import colored_header
from constants import *
+from file_utils import load_string
+import sys
+import logging
+import datetime
+from models.generator import GeneratorType
+import pyperclip
+import json
def design_tab():
- col1, col2 = st.columns([1,11])
- with col1:
- st.image("mock_generators/media/abstract.gif")
- with col2:
- st.write(f"Design Data Model.\n\nUse the [arrows.app](https://arrows.app) then download the .json file to the Import tab.")
- st.markdown("--------")
+ # col1, col2 = st.columns([1,11])
+ # with col1:
+ # st.image("mock_generators/media/abstract.gif")
+ # with col2:
+ # st.write(f"Design Data Model.\n\nUse the [arrows.app](https://arrows.app) then download the .json file to the Import tab.")
+ # st.markdown("--------")
+ with st.expander("Instructions"):
+ st.write("""
+ 1. Connect to arrows.app. Optionally login via Google to save your model designs
+ 2. Select a Node or Relationship to display a properties inspector on the right
+ 3. Configure the mock graph generator by adding properties with specially formatted keys and value strings. See additional details on how to do this in the dropdowns below
+ 4. Once completed, click on the 'Download/Export' button in the arrows.app. Make sure to use the 'JSON' export option
+ 5. Proceed to the '② Generate' tab
+ """)
+ d1, d2, d3 = st.columns(3)
+ with d1:
+ with st.expander("NODE requirements"):
+ st.write("Nodes require 2 keys to have mock data generated for it: {count} and {key}. The {count} key identifies which data generator (and args) to use for creating the number of nodes.\n\nExample of node properties that would create mock data:")
+ st.image("mock_generators/media/sample_node_properties.png")
+ with d2:
+ with st.expander("RELATIONSHIP requirements"):
+ st.write("Relationships require 1 key: {count} and can take 2 optional keys: {assignment} and {filter}. The {count} key identifies which data generator (and args) to use for creating the number of relationships between a source node and a target node. EXAMPLE: If every source node should connect to exactly one target node, then the target generator value should be 1.\n\nThe {assignement} key identifies which data generator (and args) to use when deciding which target nodes this relationship should connect with.\n\nThe {filter} key identifies a data generator (and args) to use for deciding what, if any, target nodes to ignore.\n\nExample of node properties that would create mock data:")
+ st.image("mock_generators/media/sample_relationship_properties.png")
+ with d3:
+ with st.expander("PROPERTY requirements"):
+ st.write("Properties needing mock generated data should be a stringified JSON object. The unique generator name should be used as a key, followed by a list/array or argument values.\n\nSee the NODE and RELATIONSHIP properties dropdown for examples.\n\nThe right hand Generators preview lists all the available mock data generators. Arguments can be set and example output data can be previewed by clicking on the 'Generate Example Output' button. Use the 'Copy for Arrows' button to copy the required formatted JSON string to your clipboard, to paste into the arrows.app")
+ st.image("mock_generators/media/sample_generator.png")
- components.iframe("https://arrows.app", height=1000, scrolling=False)
\ No newline at end of file
+ c1, c2 = st.columns([8,2])
+ with c1:
+ components.iframe("https://arrows.app", height=1000, scrolling=False)
+ with c2:
+ # st.write("Generators")
+ # st.markdown("--------")
+
+ generators = st.session_state[GENERATORS]
+ if generators is None:
+ st.write("No generators loaded")
+ st.stop()
+
+ # Search by name or description
+ search_term = st.text_input("Search Generators by keyword", "", help="Generators are functions for creating mock data.")
+
+ # Filter by type
+ st.write('<style>div.row-widget.stRadio > div{flex-direction:row;}</style>', unsafe_allow_html=True)
+ type_filter = st.radio("Filter Generator outputs by type", ["All", "String", "Bool", "Integer", "Float","Datetime", "Assignment"])
+ for _, generator in sorted(generators.items(), key=lambda gen:(gen[1].name)):
+ # generator = generators[key]
+ # Filtering
+ if type_filter != "All" and type_filter != generator.type.to_string():
+ continue
+
+ if search_term != "" and search_term.lower() not in generator.name.lower() and search_term.lower() not in generator.description.lower():
+ continue
+
+ try:
+ # Check that we can load code first
+ code_filepath = generator.code_url
+ code_file = load_string(code_filepath)
+ except:
+ logging.error(f"Could not load generator code from {code_filepath}: {sys.exc_info()[0]}")
+ continue
+
+ with st.expander(generator.name):
+ st.write(f"\n {generator.description}")
+ # st.write(f'Generator Code:\n')
+ # st.markdown(f'```python\n{code_file}\n```')
+ args = generator.args
+ arg_inputs = []
+ for arg in args:
+ if arg.type == GeneratorType.STRING:
+ arg_inputs.append(st.text_input(
+ label=arg.label,
+ value = arg.default,
+ key = f'{generator.id}_{arg.label}',
+ placeholder = f'{arg.hint}',
+ help = f'{arg.description}'
+ ))
+ if arg.type == GeneratorType.INT or arg.type == GeneratorType.FLOAT:
+ arg_inputs.append(st.number_input(
+ label= arg.label,
+ value= arg.default,
+ key = f'{generator.id}_{arg.label}'
+ ))
+ if arg.type == GeneratorType.BOOL:
+ arg_inputs.append(st.radio(
+ label=arg.label,
+ index=arg.default,
+ key = f'{generator.id}_{arg.label}'
+ ))
+ if arg.type == GeneratorType.DATETIME:
+ arg_inputs.append(st.date_input(
+ label=arg.label,
+ value=datetime.datetime.fromisoformat(arg.default),
+ key = f'{generator.id}_{arg.label}'
+ ))
+ if st.button("Generate Example Output", key=f"run_{generator.id}"):
+ try:
+ module = __import__(generator.import_url(), fromlist=['generate'])
+ # logging.info(f'arg_inputs: {arg_inputs}')
+
+ # TODO: Need a fake list of Node Mappings to test out assignment generators
+ result = module.generate(arg_inputs)
+ st.write(f'Output: {result}')
+ except:
+ st.error(f"Problem running generator {generator.name}: {sys.exc_info()[0]}")
+ if st.button("Copy for Arrows", key=f"copy_{generator.id}"):
+ name = generator.name
+ args = arg_inputs
+ obj = {
+ generator.id: args
+ }
+ json_string = json.dumps(obj)
+ pyperclip.copy(json_string)
+ st.success(f'Copied to clipboard: {json_string}. Paste into Arrows.app to use.')
\ No newline at end of file
diff --git a/mock_generators/tabs/generate_tab.py b/mock_generators/tabs/generate_tab.py
index e1f2df4..4d041d0 100644
--- a/mock_generators/tabs/generate_tab.py
+++ b/mock_generators/tabs/generate_tab.py
@@ -1,6 +1,8 @@
+# Replaced with expanded Import Tab
+
import streamlit as st
from constants import *
-# from models.mapping import Mapping
+from models.mapping import Mapping
from logic.generate_csv import generate_csv
from logic.generate_data_import import generate_data_importer_json
import os
@@ -20,6 +22,8 @@ def generate_tab():
st.markdown("--------")
mapping = st.session_state[MAPPINGS]
+ if mapping is None:
+ mapping = Mapping.empty()
export_folder = st.session_state[EXPORTS_PATH]
zips_folder = st.session_state[ZIPS_PATH]
imported_filename = st.session_state[IMPORTED_FILENAME]
@@ -31,27 +35,27 @@ def generate_tab():
export_zip_filename.replace(" ", "_")
export_zip_filename.replace(".", "_")
- g1, g2, g3, g4 = st.columns(4)
+ g2, g3, g4 = st.columns(3)
- with g1:
- st.write('MAPPING DATA:')
- if MAPPINGS not in st.session_state:
- # Why hasn't mappings been preloaded by now?
- st.error(f'Mappings data was not preloaded')
- elif st.session_state[MAPPINGS] is None:
- st.error(f'Mappping option not valid for generation. Please configure mapping options below.')
- elif st.session_state[MAPPINGS].is_empty() == True:
- st.error(f'No data currently mapped. Please configure in Mapping tab.')
- elif st.session_state[MAPPINGS].is_valid() == False:
- st.error(f'Mappping option not valid for generation. Please configure in Mapping tab.')
- else:
- st.success(f'Mappping options valid for generation.')
+ # with g1:
+ # st.write('MAPPING DATA:')
+ # if MAPPINGS not in st.session_state:
+ # # Why hasn't mappings been preloaded by now?
+ # st.error(f'Mappings data was not preloaded')
+ # elif st.session_state[MAPPINGS] is None:
+ # st.error(f'Mappping option not valid for generation. Please configure mapping options below.')
+ # elif st.session_state[MAPPINGS].is_empty() == True:
+ # st.error(f'No data currently mapped. Please configure in Mapping tab.')
+ # elif st.session_state[MAPPINGS].is_valid() == False:
+ # st.error(f'Mappping option not valid for generation. Please configure in Mapping tab.')
+ # else:
+ # st.success(f'Mappping options valid for generation.')
- # For the curious
- with st.expander("Raw Mapping Data"):
- st.json(mapping.to_dict())
+ # # For the curious
+ # with st.expander("Raw Mapping Data"):
+ # st.json(mapping.to_dict())
- # should_clear = st.checkbox("Delete all files in export folder with each Generate run", value=True)
+ # # should_clear = st.checkbox("Delete all files in export folder with each Generate run", value=True)
with g2:
st.write(f'READY FOR GENERATION:')
@@ -67,12 +71,17 @@ def generate_tab():
st.stop()
return
- for key, node in mapping.nodes.items():
+ # Generate values from mappings
+ for _, node in mapping.nodes.items():
# logging.info(f'Generating data for node: {node}')
if len(node.properties) == 0:
st.error(f'Node {node.caption} has no properties. Add at least one property to generate data.')
st.stop()
return
+ node.generate_values()
+
+ for _, rel in mapping.relationships.items():
+ rel.generate_values()
# Delete all files in export folder first
dir = export_folder
@@ -130,7 +139,7 @@ def generate_tab():
with g4:
st.write(f'GENERATED DATA SUMMARY:')
# TODO: Why do these disappear when returning to tab?
- nodes = st.session_state[MAPPINGS].nodes
+ nodes = mapping.nodes
for _, node in nodes.items():
values = node.generated_values
if values is not None:
diff --git a/mock_generators/tabs/importing_tab.py b/mock_generators/tabs/importing_tab.py
index 59be758..dc2d6d3 100644
--- a/mock_generators/tabs/importing_tab.py
+++ b/mock_generators/tabs/importing_tab.py
@@ -1,3 +1,5 @@
+# Now the new Generate Tab
+
import streamlit as st
import json
from constants import *
@@ -7,26 +9,27 @@
from file_utils import load_string, save_file
from models.mapping import Mapping
import sys
-import datetime
import os
+from logic.generate_mapping import mapping_from_json
+from generate import generate_data
+from datetime import datetime
# Convenience functions
def import_file(file) -> bool:
if file is None:
raise Exception(f'import.py: import_file function called with no file')
+ # Check if file is a valid JSON file
try:
json_file = json.loads(file)
# Bring in the raw import data for nodes and relationships
node_dicts = json_file["nodes"]
relationship_dicts = json_file["relationships"]
if node_dicts is None:
- node_dicts = []
+ return False
if relationship_dicts is None:
- relationship_dicts = []
-
- st.session_state[IMPORTED_NODES] = node_dicts
- st.session_state[IMPORTED_RELATIONSHIPS] = relationship_dicts
+ return False
+
return True
except json.decoder.JSONDecodeError:
st.error(f'JSON file {file} is not valid.')
@@ -50,28 +53,45 @@ def file_selected(path):
# Import
sucessful_import = import_file(selected_file)
if sucessful_import:
- st.success(f"Import Complete. Proceed to the Mapping page")
+ st.success(f"Import Complete.")
else:
- st.error(f"Import Failed. Please try again.")
+ st.error(f"Import Failed. Check file format.")
def import_tab():
- col1, col2 = st.columns([1,11])
- with col1:
- st.image("mock_generators/media/import.gif")
- with col2:
- st.markdown("Import JSON files from an [arrows.app](https://arrows.app/#/local/id=A330UT1VEBAjNH1Ykuss) data model. \n\nProceed to the Mapping Tab when complete.")
+ # col1, col2 = st.columns([1,11])
+ # with col1:
+ # # st.image("mock_generators/media/import.gif")
+ # st.image("mock_generators/media/fireworks.gif")
+ # with col2:
+ # st.markdown("Import JSON files from an [arrows.app](https://arrows.app/#/local/id=A330UT1VEBAjNH1Ykuss) data model. \n\nProceed to the Mapping Tab when complete.")
+
+ with st.expander("Instructions"):
+ st.write(
+ """
+ 1. Import or select a previously imported JSON file from an arrows.app export
+ 2. The mock graph data generator will automatically generate a .csv and .zip files
+ 3. Download the .zip file
+ 4. Proceed to the '③ Data Importer' tab
+ """
+ )
+
st.markdown("--------")
- i1, i2 = st.columns([3,1])
+ i1, i2, i3 = st.columns(3)
+
+ start_generated = datetime.now()
+ last_generated = start_generated
with i1:
+ # File Selection
+ st.write('SELECT ARROWS FILE:')
+ # st.markdown("--------")
+
selected_file = None
import_option = st.radio("Select Import Source", ["An Existing File", "Upload"], horizontal=True)
- st.markdown("--------")
-
if import_option == "An Existing File":
# Saved options
st.write("Select an import file:")
@@ -103,43 +123,40 @@ def import_tab():
file_selected(selected_filepath)
else:
logging.info(f'Copy & Paste Option disabled')
- # else:
- # # Copy & Paste
- # pasted_json = st.text_area("Paste an arrows JSON file here", height=300)
- # # logging.info(f'pasted_json: {pasted_json}')
- # if pasted_json is not None and pasted_json != "":
- # temp_filename = f'pasted_file.json'
- # selected_filepath = f"{st.session_state[IMPORTS_PATH]}/{temp_filename}"
- # # data = json.dumps(pasted_json, indent=4)
- # try:
- # save_file(
- # filepath=selected_filepath,
- # data=pasted_json)
- # except:
- # st.error(f"Error saving file to {st.session_state[IMPORTS_PATH]}")
- # logging.error(f'Error saving file: {sys.exc_info()[0]}')
-
- # file_selected(selected_filepath)
- with i2:
+
# Process uploaded / selected file
current_file = st.session_state[IMPORTED_FILE]
if current_file is not None:
# Verfiy file is valid arrows JSON
try:
- json_file = json.loads(current_file)
- nodes = json_file["nodes"]
- relationships = json_file["relationships"]
- if nodes is None:
- nodes = []
- if relationships is None:
- relationships = []
- st.session_state[IMPORTED_NODES] = nodes
- st.session_state[IMPORTED_RELATIONSHIPS] = relationships
-
- st.write(f'Imported Data:')
- st.write(f' - {len(nodes)} Nodes')
- st.write(f' - {len(relationships)} Relationships')
+ generators = st.session_state[GENERATORS]
+ mapping = mapping_from_json(
+ current_file,
+ generators)
+ # st.session_state[MAPPINGS] = mapping
+ generate_data(mapping)
+
+ last_generated = datetime.now()
+ # st.success(f"New data generated from import file.")
except json.decoder.JSONDecodeError:
- st.error('JSON file is not valid.')
\ No newline at end of file
+ st.error('Import JSON file is not valid.')
+ except Exception as e:
+ st.error(f'Uncaught Error: {e}')
+
+ with i2:
+ export_folder = st.session_state[EXPORTS_PATH]
+ st.write(f"RECENTLY GENERATED FILES:")
+ if start_generated != last_generated:
+ st.success(f"New data generated from import file.")
+ folder_files_expander(export_folder, widget_id="export_tab", enable_download=True)
+
+ with i3:
+
+ st.write('GENERATED ZIP FILES:')
+ if start_generated != last_generated:
+ st.success(f"New zip files generated from import file.")
+ zips_folder = st.session_state[ZIPS_PATH]
+
+ folder_files_expander(zips_folder, widget_id="export_tab", enable_download=True, enable_delete_button=True)
\ No newline at end of file
diff --git a/poetry.lock b/poetry.lock
index 7d6be89..69a7fae 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -604,6 +604,14 @@ python-versions = ">=3.6.8"
[package.extras]
diagrams = ["jinja2", "railroad-diagrams"]
+[[package]]
+name = "pyperclip"
+version = "1.8.2"
+description = "A cross-platform clipboard module for Python. (Only handles plain text for now.)"
+category = "main"
+optional = false
+python-versions = "*"
+
[[package]]
name = "pyrsistent"
version = "0.19.2"
@@ -614,7 +622,7 @@ python-versions = ">=3.7"
[[package]]
name = "pytest"
-version = "7.2.0"
+version = "7.2.1"
description = "pytest: simple powerful testing with Python"
category = "main"
optional = false
@@ -1034,7 +1042,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools"
[metadata]
lock-version = "1.1"
python-versions = "^3.11"
-content-hash = "bc8d3f2168c9cbf29966625ee8b2b50c6deb273211819f880877a985f9b30fb7"
+content-hash = "139ef946899dcf744a4d7b18c235d88178911d8cf9842f91893fca672cccce67"
[metadata.files]
altair = [
@@ -1669,6 +1677,9 @@ pyparsing = [
{file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"},
{file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"},
]
+pyperclip = [
+ {file = "pyperclip-1.8.2.tar.gz", hash = "sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57"},
+]
pyrsistent = [
{file = "pyrsistent-0.19.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d6982b5a0237e1b7d876b60265564648a69b14017f3b5f908c5be2de3f9abb7a"},
{file = "pyrsistent-0.19.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:187d5730b0507d9285a96fca9716310d572e5464cadd19f22b63a6976254d77a"},
@@ -1694,8 +1705,8 @@ pyrsistent = [
{file = "pyrsistent-0.19.2.tar.gz", hash = "sha256:bfa0351be89c9fcbcb8c9879b826f4353be10f58f8a677efab0c017bf7137ec2"},
]
pytest = [
- {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"},
- {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"},
+ {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"},
+ {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"},
]
python-dateutil = [
{file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
diff --git a/pyproject.toml b/pyproject.toml
index 9215d63..dbd72ef 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,8 +16,17 @@ streamlit-toggle-switch = "^1.0.2"
hydralit-components = "^1.0.10"
pytest = "^7.2.0"
dataclasses-json = "^0.5.7"
+pyperclip = "^1.8.2"
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.2.1"
+
+[tool.pytest.ini_options]
+pythonpath = [
+ ".", "mock_generators"
+]
+
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
diff --git a/tests/__init__.py b/tests/__init__.py
index e69de29..b7be57f 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -0,0 +1,2 @@
+import sys
+sys.path.append('.')
\ No newline at end of file
diff --git a/tests/test_generate_mappings.py b/tests/test_generate_mappings.py
new file mode 100644
index 0000000..7a3b427
--- /dev/null
+++ b/tests/test_generate_mappings.py
@@ -0,0 +1,188 @@
+import pytest
+# from mock_generators import __version__
+
+from mock_generators.models.generator import Generator, GeneratorType, GeneratorArg
+
+from mock_generators.models.node_mapping import NodeMapping
+
+from mock_generators.generate_mapping import generator_for_raw_property, mapping_from_json, propertymappings_for_raw_properties, node_mappings_from
+
+import json
+# For pytest to properly find modules, add following to pyproject.toml:
+# [tool.pytest.ini_options]
+# pythonpath = [
+# ".", "mock_generators"
+# ]
+
+int_args_test = GeneratorArg(
+ type=GeneratorType.INT,
+ label="test_arg",
+ default=1,
+ )
+
+int_generator_test = Generator(
+ id="test_int",
+ name="test_int",
+ type=GeneratorType.INT,
+ description="test",
+ args=[int_args_test],
+ code_url="mock_generators/generators/ecdff22c.py",
+ tags=["int, integer, number"]
+ )
+
+float_args_test = [
+ GeneratorArg(
+ type=GeneratorType.FLOAT,
+ label="min",
+ default=1.0,
+ ),
+ GeneratorArg(
+ type=GeneratorType.FLOAT,
+ label="max",
+ default=2.0,
+ ),
+ ]
+
+float_generator_test = Generator(
+ id="test_float",
+ name = "test_float",
+ type=GeneratorType.FLOAT,
+ description="test",
+ args=float_args_test,
+ code_url="mock_generators/generators/e8cff8c1.py",
+ tags=["float", "number"]
+ )
+
+test_generators = {
+ "test_int" : int_generator_test,
+ "test_float" : float_generator_test
+}
+
+class TestClass:
+
+ # def test_version(self):
+ # assert __version__ == "0.1.0"
+
+ def test_generator(self):
+ generator = Generator(
+ id="test",
+ name="test",
+ type=GeneratorType.INT,
+ description="test",
+ args=[],
+ code_url="",
+ tags = ["test"]
+ )
+ assert generator.id == "test"
+ assert generator.name == "test"
+ assert generator.type == GeneratorType.INT
+ assert generator.description == "test"
+ assert generator.args == []
+ assert generator.code_url == ""
+ # TODO: Why is this not working
+ # tag_check = generator.tags("test")
+ # assert tag_check > 0
+
+ def test_jsonification(self):
+ # Parsing json strings critical to this class
+ json_string = "{\"test_int\":[1]}"
+ try:
+ obj = json.loads(json_string)
+ items = obj.items()
+ assert items is not None
+ assert len(items) == 1
+ for key, value in items:
+ assert key == "test_int"
+ assert value == [1]
+
+ value = obj.get('test_int', None)
+ assert value is not None
+ assert value == [1]
+ except Exception as e:
+ print(f'Exception: {e}')
+ assert False
+
+
+ def test_generator_for_raw_property(self):
+ print(f'int_generator_test: {int_generator_test}')
+ print(f'int_args_test: {int_args_test}')
+ print(f'test_generators: {test_generators}')
+ try:
+ test_string = "{\"test_int\":[1]}"
+ generator, args = generator_for_raw_property(test_string, test_generators)
+ except Exception as e:
+ print(f'Exception: {e}')
+ print(f'generator: {generator}, args: {args}')
+ # if status_message is not None:
+ # print(f'status: {status_message}')
+ assert int_generator_test is not None
+ assert int_args_test is not None
+ assert generator == int_generator_test
+ assert args == [1]
+ # Test generator returned creates acceptable value
+ value = generator.generate(args)
+ assert value == 1
+
+ def test_failed_generator_for_raw_property(self):
+ try:
+ generator, args = generator_for_raw_property("{\'test_doesnt_exist\':[1]}", test_generators)
+ except Exception as e:
+ print(f'Exception: {e}')
+ assert generator is None
+ assert args is None
+
+ def test_propertymappings_for_raw_properties(self):
+ mappings = propertymappings_for_raw_properties(
+ raw_properties={
+ "alpha": "{\'test_int\':[1]",
+ "bravo": "{\'test_float\':[1.0, 2.0]"
+ },
+ generators= test_generators)
+ print(f'mappings: {mappings}')
+ assert len(mappings) == 2
+ assert mappings['alpha'].generator == self.int_generator_test()
+ assert mappings['alpha'].args == self.int_args_test()
+ assert mappings['bravo'].generator == self.float_generator_test()
+ assert mappings['bravo'].args == self.float_args_test()
+
+ # def test_node_mappings_from(self):
+ # nodes = node_mappings_from(
+ # node_dicts=[{
+ # "id": "n1",
+ # "position": {
+ # "x": 284.5,
+ # "y": -204
+ # },
+ # "caption": "Company",
+ # "labels": [],
+ # "properties": {
+ # "uuid": "{\"test_float\":[8]}",
+ # "{count}": "{\"test_int\":[1]}",
+ # "{key}": "uuid"
+ # },
+ # "style": {}
+ # }],
+ # generators= test_generators
+ # )
+ # expectedNodeMapping = NodeMapping(
+ # nid="n1",
+ # position={
+ # "x": 284.5,
+ # "y": -204
+ # },
+ # caption="Company",
+ # labels=[],
+ # properties={
+ # "uuid": "{{\"test_float\":[8]}}",
+ # "{count}": "{{\"test_int\":[1]}}",
+ # "{key}": "uuid"
+ # },
+ # count_generator=int_generator_test,
+ # count_args=int_args_test,
+ # key_property=int_generator_test
+ # )
+ # assert len(nodes) == 1
+ # assert nodes.get("n1") == expectedNodeMapping
+
+ # def test_mapping_from_json(self):
+