Arrows Config Support (#6)

* WIP Simplification update to use arrows only for generator assignment & config

* Mappings tabbed replaced with auto generated mappings from specially formatted arrows.app json files

* Generator search and code copy in design tab added

* Config tab removed

* Import, Generate, and Export tabs combined into new Generate tab

* Design tab updated with new instructions

* Clean up and instructions added

Author: jalakoo

mock_generators/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = "0.1.0"

mock_generators/app.py

@@ -1,16 +1,9 @@
 import streamlit as st
 from constants import *
-from tabs.config_tab import config_tab
-from tabs.generators_tab import generators_tab
-from tabs.new_generator_tab import create_tab
-from tabs.mapping_tab import mapping_tab
-from tabs.generate_tab import generate_tab
-from tabs.export_tab import export_tab
 from tabs.importing_tab import import_tab
 from tabs.design_tab import design_tab
 from tabs.data_importer import data_importer_tab
-from models.mapping import Mapping
-
+from config import load_generators
 
 # SETUP
 st.set_page_config(layout="wide")
@@ -42,9 +35,9 @@
 if CODE_TEMPLATE_FILE not in st.session_state:
     st.session_state[CODE_TEMPLATE_FILE] = DEFAULT_CODE_TEMPLATES_FILE
 if MAPPINGS not in st.session_state:
-    st.session_state[MAPPINGS] = Mapping(
-        nodes={}, 
-        relationships={})
+    st.session_state[MAPPINGS] = None
+
+load_generators()
 
 # UI
 st.title("Mock Graph Data Generator")
@@ -54,31 +47,15 @@
 imported_file = None
 
 # Streamlit runs from top-to-bottom from tabs 1 through 8. This is essentially one giant single page app.  Earlier attempt to use Streamlit's multi-page app functionality resulted in an inconsistent state between pages.
-tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8, tab9 = st.tabs(["Config >", "Design >", "Import >",  "Mapping >", "Search Generators >", "Add New Generator >", "Generate >", "Export >", "Data Importer"])
-
-with tab1:
-    config_tab()
 
-with tab2:
+t1, t2, t5 = st.tabs([
+    "① Design",
+    "② Generate",
+    "③ Data Importer"
+])
+with t1:
     design_tab()
-
-with tab3:
+with t2:
     import_tab()
-
-with tab4:
-    mapping_tab()
-
-with tab5:
-    generators_tab()
-
-with tab6:
-    create_tab()
-
-with tab7:
-    generate_tab()
-
-with tab8:
-    export_tab()
-
-with tab9:
+with t5:
     data_importer_tab()

mock_generators/config.py

@@ -0,0 +1,23 @@
+import streamlit as st
+from constants import *
+from file_utils import load_json, load_string
+from models.generator import Generator, generators_from_json
+import os
+import sys
+import logging
+from widgets.folder_files import folder_files_expander
+
+def load_generators():
+
+    spec_filepath = st.session_state[SPEC_FILE]
+    generators = st.session_state[GENERATORS]
+    try:
+        with open(spec_filepath) as input:
+            # generators_file = input.read()
+            generators_json = load_json(spec_filepath)
+            new_generators = generators_from_json(generators_json)
+            if generators != new_generators:
+                st.session_state[GENERATORS] = new_generators
+
+    except FileNotFoundError:
+        st.error('File not found.')

mock_generators/constants.py

@@ -1,7 +1,7 @@
 import streamlit as st
 
 # Default local filepaths
-DEFAULT_GENERATORS_SPEC_FILE = "mock_generators/generators.json"
+DEFAULT_GENERATORS_SPEC_FILE = "mock_generators/named_generators.json"
 DEFAULT_GENERATORS_CODE_PATH = "mock_generators/generators"
 DEFAULT_ARROWS_SAMPLE_PATH = "mock_generators/samples/arrows.json"
 DEFAULT_IMPORTS_PATH = "mock_generators/imports"

mock_generators/generate.py

@@ -0,0 +1,94 @@
+import streamlit as st
+from constants import *
+from models.mapping import Mapping
+from logic.generate_csv import generate_csv
+from logic.generate_data_import import generate_data_importer_json
+import os
+import logging
+import sys
+import zipfile
+from datetime import datetime
+
+def generate_data(mapping: Mapping):
+
+    export_folder = st.session_state[EXPORTS_PATH]
+    zips_folder = st.session_state[ZIPS_PATH]
+    imported_filename = st.session_state[IMPORTED_FILENAME]
+
+    # TODO: Implement better filename cleaning
+    # TODO: Breaks when using a copy and pasted file
+    export_zip_filename = f'{imported_filename}'.lower()
+    export_zip_filename = export_zip_filename.replace(".json", "")
+    export_zip_filename.replace(" ", "_")
+    export_zip_filename.replace(".", "_")
+
+    # Stop if no mapping data available
+    if len(mapping.nodes) == 0:
+        st.error('No nodes to generate data for. Map at least one noded.')
+        st.stop()
+        return
+
+    # Generate values from mappings
+    for _, node in mapping.nodes.items():
+        # logging.info(f'Generating data for node: {node}')
+        if len(node.properties) == 0:
+            st.error(f'Node {node.caption} has no properties. Add at least one property to generate data.')
+            st.stop()
+            return
+        node.generate_values()
+
+    for _, rel in mapping.relationships.items():
+        rel.generate_values()
+
+    # Delete all files in export folder first
+    dir = export_folder
+    for f in os.listdir(dir):
+        os.remove(os.path.join(dir, f))
+
+    # Data Importer Options
+    success = generate_csv(
+        mapping, 
+        export_folder=export_folder)
+
+    # Check that data was generated
+    if success == False:
+        st.error('Error generating data. Check console for details.')
+        # st.stop()
+        # return
+
+    success = generate_data_importer_json(
+        mapping,
+        export_folder=export_folder,
+        export_filename=DEFAULT_DATA_IMPORTER_FILENAME)
+
+    # Check that data-import data was generated
+    if success == False:
+        st.error('Error generating data-import json. Check console for details.')
+        # st.stop()
+        # return
+
+    # Only attempt to zip files if data generation was successful
+    if success:
+        try:
+            # Create zip file, appended with time created
+            # now = str(datetime.now().isoformat())
+            zip_path = f'{zips_folder}/{export_zip_filename}.zip'
+            logging.info(f'generate_tab: Creating zip file: {zip_path}')
+            with zipfile.ZipFile(f'{zip_path}', 'w', zipfile.ZIP_DEFLATED) as zipf:
+                # zipdir(export_folder, zipf)
+                path = export_folder
+                for root, dirs, files in os.walk(path):
+                    for file in files:
+                        if file[0] =='.':
+                            # Skip hidden files
+                            continue
+                        zipf.write(os.path.join(root, file), 
+                                os.path.relpath(os.path.join(root, file), 
+                                                os.path.join(path, '..')))
+        except:
+            st.error(f'Error creating zip file: {sys.exc_info()[0]}')
+            # st.stop()
+            return
+
+    if success == True:
+        st.success('Data generated successfully.')