Graphgpt (#11)

* Basic literal support added - ints, floats, ranges, lists

* Datetime generator fix + keywords generator support added

* Default count and key assignments added + missing generator updates

* Random image url added and string_from_csv fixed

* ADDRESS keyword literal and mock address data generation added

* graphGPT integration

* Cache openai responses to prevent behind the scenes rerun of ideation tab

* Video tutorial url made remote configurable

Author: jalakoo

Pipfile

@@ -15,6 +15,8 @@ streamlit-player = "*"
 requests = "*"
 pandas = "*"
 random-address = "*"
+openai = "*"
+numpy = "*"
 
 [dev-packages]
 mock-generators = {editable = true, path = "."}

Pipfile.lock

@@ -1,9 +1,12 @@
 import streamlit as st
 from constants import *
+from tabs.ideate_tab import ideate_tab
 from tabs.importing_tab import import_tab
 from tabs.design_tab import design_tab
 from tabs.data_importer import data_importer_tab
 from tabs.tutorial import tutorial_tab
+from tabs.getting_help import get_help_tab
+
 from config import setup_logging, preload_state, load_generators_to_streamlit
 
 # SETUP
@@ -22,18 +25,24 @@
 
 # Streamlit runs from top-to-bottom from tabs 1 through 8. This is essentially one giant single page app.  Earlier attempt to use Streamlit's multi-page app functionality resulted in an inconsistent state between pages.
 
-t0, t1, t2, t5 = st.tabs([
-    "⓪ Tutorial",
-    "① Design",
-    "② Generate",
-    "③ Data Importer"
+t0, t1, t2, t3, t4, t5 = st.tabs([
+    "⓪ Getting Started",
+    "① Ideate",
+    "② Design",
+    "③ Generate",
+    "④ Data Importer",
+    "Ⓘ Info"
 ])
 
 with t0:
     tutorial_tab()
 with t1:
-    design_tab()
+    ideate_tab()
 with t2:
+    design_tab()
+with t3:
     import_tab()
+with t4:
+    data_importer_tab()
 with t5:
-    data_importer_tab()
+    get_help_tab()

mock_generators/app.py

@@ -1,10 +1,11 @@
-from random_address import real_random_address_by_state
+from random_address import real_random_address
 import random
 
 def generate(args: list[any]):
     # Generate a dictionary with valid random address information
-    states = [
-        "AL", "AR", "CA", "CO", "CT", "DC", "FL", "GA", "HI", "KY", "MA" "MD", "TN", "TX", "OK", "VT"
-    ]
-    state_code = random.choice(states)
-    return real_random_address_by_state(state_code)
+    # states = [
+    #     "AL", "AR", "CA", "CO", "CT", "DC", "FL", "GA", "HI", "KY", "MA" "MD", "TN", "TX", "OK", "VT"
+    # ]
+    # state_code = random.choice(states)
+    # return real_random_address_by_state(state_code)
+    return real_random_address()

mock_generators/generators/address_usa.py

@@ -0,0 +1,201 @@
+# For converting data from various forms to other forms
+from streamlit_agraph import agraph, Node, Edge, Config
+import logging
+import json
+
+def random_coordinates_for(
+        number: int):
+    import numpy as np
+
+    # Generate random coordinates
+    min_range = 0
+    max_width = number * 300
+    x_values = np.random.uniform(min_range, max_width, number)
+    y_values = np.random.uniform(min_range, max_width, number)
+
+    # Create list of (x, y) coordinates
+    coordinates = list(zip(x_values, y_values))
+    return coordinates
+
+
+def agraph_data_from_response(response: str)->tuple[any, any, any]:
+    # Returns agraph compatible nodes, edges, and config
+    logging.debug(f'Response: {response}')
+    # Response will be a list of 3 item tuples
+    try:
+        answers = json.loads(response)
+    except Exception as e:
+        logging.error(e)
+        return None, None, None
+    if isinstance(answers, list) == False:
+        logging.error(f'Response could not be converted to list, got {type(answers)} instead.')
+        return None, None, None
+    nodes = []
+    # node_labels = []
+    edges = []
+    for idx, item in enumerate(answers):
+        if item is None or len(item) != 3:
+            continue
+        n1_label = item[0]
+        r = item[1]
+        n2_label = item[2]
+
+        # We only want to add agraph nodes with the same label once in our return
+
+        # Gross but works
+        add_n1 = True
+        for node in nodes:
+            if node.label == n1_label:
+                add_n1 = False
+        if add_n1:
+            nodes.append(Node(id=n1_label, label=n1_label))
+
+        add_n2 = True
+        for node in nodes:
+            if node.label == n2_label:
+                add_n2 = False
+        if add_n2:
+            nodes.append(Node(id=n2_label, label=n2_label))
+
+        # agraph requires source and target ids to use what we consider labels
+        edge = Edge(source=n1_label, target=n2_label, label=r)
+        edges.append(edge)
+    config = Config(
+        width=800, 
+        height=800, 
+        backgroundColor="#000000",
+        directed=True)
+    return (nodes, edges, config)
+
+
+def convert_agraph_nodes_to_arrows_nodes(
+        agraph_nodes: list
+)-> list[dict]:
+    # Convert agraph nodes to arrows nodes
+    arrows_nodes = []
+
+    # Generate random coordinates to init new arrows nodes with - since we can't extract the location data from agraph
+    coordinates = random_coordinates_for(len(agraph_nodes))
+
+    for nidx, node in enumerate(agraph_nodes):
+        new_node = convert_agraph_node_to_arrows_node(
+            nidx, node, coordinates[nidx][0], coordinates[nidx][1])
+        arrows_nodes.append(new_node)
+    return arrows_nodes
+
+def convert_agraph_node_to_arrows_node(
+        idx, 
+        node,
+        x,
+        y):
+    # Convert agraph node to arrows node
+    arrows_node = {
+        "id": f'n{idx+1}',
+        "caption": node.label,
+        "position": {
+            "x": x,
+            "y": y,
+        },
+        "labels":[],
+        "style": {},
+        "properties": {}
+    }
+    return arrows_node
+
+def convert_agraph_edge_to_arrows_relationship(
+        idx, 
+        edge, 
+        arrows_nodes: list):
+    # Example: {'source': 'People', 'from': 'People', 'to': 'Cars', 'color': '#F7A7A6', 'label': 'DRIVE'}
+    source_node_label = edge.source
+    target_node_label = edge.to
+    source_node_id = None
+    target_node_id = None
+
+    for node in arrows_nodes:
+        if node['caption'] == source_node_label:
+            source_node_id = node['id']
+        if node['caption'] == target_node_label:
+            target_node_id = node['id']
+
+    if source_node_id is None or target_node_id is None:
+        node_info = [node.__dict__ for node in arrows_nodes]
+        logging.error(f'Could not find source or target node for edge {edge.__dict__} from nodes: {node_info}')
+        return None
+    edge_type = edge.label
+    arrows_relationship = {
+        "id": f'n{idx+1}',
+        "fromId": source_node_id,
+        "toId": target_node_id,
+        "type": edge_type,
+        "properties": {},
+        "style": {}
+    }
+    return arrows_relationship
+
+def convert_agraph_to_arrows(agraph_nodes, agraph_edges):
+    arrows_nodes = convert_agraph_nodes_to_arrows_nodes(agraph_nodes)
+
+    arrows_relationships = []
+    for eidx, edge in enumerate(agraph_edges):
+        new_relationship = convert_agraph_edge_to_arrows_relationship(eidx, edge, arrows_nodes=arrows_nodes)
+        arrows_relationships.append(new_relationship)
+    arrows_json = {
+        "nodes": arrows_nodes,
+        "relationships": arrows_relationships,
+          "style": {
+            "font-family": "sans-serif",
+            "background-color": "#ffffff",
+            "background-image": "",
+            "background-size": "100%",
+            "node-color": "#ffffff",
+            "border-width": 4,
+            "border-color": "#000000",
+            "radius": 50,
+            "node-padding": 5,
+            "node-margin": 2,
+            "outside-position": "auto",
+            "node-icon-image": "",
+            "node-background-image": "",
+            "icon-position": "inside",
+            "icon-size": 64,
+            "caption-position": "inside",
+            "caption-max-width": 200,
+            "caption-color": "#000000",
+            "caption-font-size": 50,
+            "caption-font-weight": "normal",
+            "label-position": "inside",
+            "label-display": "pill",
+            "label-color": "#000000",
+            "label-background-color": "#ffffff",
+            "label-border-color": "#000000",
+            "label-border-width": 4,
+            "label-font-size": 40,
+            "label-padding": 5,
+            "label-margin": 4,
+            "directionality": "directed",
+            "detail-position": "inline",
+            "detail-orientation": "parallel",
+            "arrow-width": 5,
+            "arrow-color": "#000000",
+            "margin-start": 5,
+            "margin-end": 5,
+            "margin-peer": 20,
+            "attachment-start": "normal",
+            "attachment-end": "normal",
+            "relationship-icon-image": "",
+            "type-color": "#000000",
+            "type-background-color": "#ffffff",
+            "type-border-color": "#000000",
+            "type-border-width": 0,
+            "type-font-size": 16,
+            "type-padding": 5,
+            "property-position": "outside",
+            "property-alignment": "colon",
+            "property-color": "#000000",
+            "property-font-size": 16,
+            "property-font-weight": "normal"
+        }
+    }
+    return arrows_json
+    

mock_generators/logic/agraph_conversions.py

@@ -18,21 +18,21 @@ def design_tab():
         2. Select a Node or Relationship to display a properties inspector on the right
         3. Configure the mock graph generator by adding properties with specially formatted keys and value strings. See additional details on how to do this in the dropdowns below
         4. Once completed, click on the 'Download/Export' button in the arrows.app. Make sure to use the 'JSON' export option
-        5. Proceed to the '② Generate' tab
+        5. Proceed to the '③ Generate' tab
         """)
     d1, d2, d3 = st.columns(3)
     with d1:
-        with st.expander("NODE requirements"):
-            st.write("Nodes require 2 keys to have mock data generated for it:  {count} and {key}. The {count} key identifies which data generator (and args) to use for creating the number of nodes.\n\nExample of node properties that would create mock data:")
-            st.image("mock_generators/media/sample_node_properties.png")
+        with st.expander("NODE Options"):
+            st.write("Add a COUNT key to identify how many of a given node you'd like created. Use a KEY key to notate which other property field you'd like use a unique identifier for a node type.\n\nExample of node properties for specifying specific number of nodes with particular properties:")
+            st.image("mock_generators/media/sample_node_0-5-0.png")
     with d2:
-        with st.expander("RELATIONSHIP requirements"):
-            st.write("Relationships require 1 key: {count} and can take 2 optional keys: {assignment} and {filter}. The {count} key identifies which data generator (and args) to use for creating the number of relationships between a source node and a target node. EXAMPLE: If every source node should connect to exactly one target node, then the target generator value should be 1.\n\nThe {assignement} key identifies which data generator (and args) to use when deciding which target nodes this relationship should connect with.\n\nThe {filter} key identifies a data generator (and args) to use for deciding what, if any, target nodes to ignore.\n\nExample of node properties that would create mock data:")
-            st.image("mock_generators/media/sample_relationship_properties.png")
+        with st.expander("RELATIONSHIP Options"):
+            st.write("Relationships can take 2 optional keys: COUNT and ASSIGNMENT to specify how many of a given relationship should be created and what assignment generator should be used")
+            st.image("mock_generators/media/sample_relationship_0-5-0.png")
     with d3:
-        with st.expander("PROPERTY requirements"):
-            st.write("Properties needing mock generated data should be a stringified JSON object. The unique generator name should be used as a key, followed by a list/array or argument values.\n\nSee the NODE and RELATIONSHIP properties dropdown for examples.\n\nThe right hand Generators preview lists all the available mock data generators. Arguments can be set and example output data can be previewed by clicking on the 'Generate Example Output' button. Use the 'Copy for Arrows' button to copy the required formatted JSON string to your clipboard, to paste into the arrows.app")
-            st.image("mock_generators/media/sample_generator.png")
+        with st.expander("PROPERTY Options"):
+            st.write("Properties needing mock generated data can be a stringified JSON object representing a generator specification. Literals such as numbers, number ranges, and list of options can also be used. For example: 1, 1.0, 1-10, 3.3-4.5, [1,2,3], [1.0, 2.2, 3.3], a_word, [word1, word2, word3] are valid values. General types can also be used, such as string, integer, float, boolean, and datetime.")
+            st.image("mock_generators/media/sample_properties_0-5-0.png")
 
 
     c1, c2 = st.columns([8,2])

mock_generators/media/sample_node_0-5-0.png

@@ -0,0 +1,5 @@
+import streamlit as st
+
+def get_help_tab():
+    st.write("Version 0.5.0")
+    st.write("Email jason.koo@neo4j.com for help.")