Visualization updates: new stitch release & offline rendering for guidance frontend.

client/graphpaper-inline/src/StitchHandler.svelte

@@ -1,29 +1,41 @@
 <!-- Handles stitch client and kernel messages. -->
 <script lang="ts">
     import type { Unsubscriber } from 'svelte/store';
-    import { kernelmsg, clientmsg, type StitchMessage } from './stitch';
+    import { kernelmsg, clientmsg, type StitchMessage, state } from './stitch';
     import { onMount, onDestroy } from 'svelte';
 
     const handleMessage = (event: MessageEvent<any>) => {
-        if (event.source === window.parent && 'type' in event.data && event.data.type === 'kernelmsg') {
-            let stitchMessage: StitchMessage = event.data;
-
-            // Notify kernel message has been received to subscribers
-            kernelmsg.set(stitchMessage);
+        if (event.source === window.parent && 'type' in event.data) {
+            if (event.data.type === 'kernelmsg') {
+                let stitchMessage: StitchMessage = event.data;
+                kernelmsg.set(stitchMessage);
+            } else if (event.data.type === 'init_state') {
+                let stitchMessage: StitchMessage = event.data;
+                state.set(stitchMessage);
+            }
         }
     };
 
-    let unsubscribe: Unsubscriber | null = null;
+    let unsubscribeClient: Unsubscriber | null = null;
+    let unsubscribeState: Unsubscriber | null = null;
     onMount(() => {
-        unsubscribe = clientmsg.subscribe((msg) => {
+        unsubscribeClient = clientmsg.subscribe((msg) => {
+            if (msg !== undefined) {
+                window.parent.postMessage(msg, "*");
+            }
+        });
+        unsubscribeState = state.subscribe((msg) => {
             if (msg !== undefined) {
                 window.parent.postMessage(msg, "*");
             }
         });
     });
     onDestroy(() => {
-        if (unsubscribe) {
-            unsubscribe();
+        if (unsubscribeClient) {
+            unsubscribeClient();
+        }
+        if (unsubscribeState) {
+            unsubscribeState();
         }
     });
 </script>

client/graphpaper-inline/src/metrics.ts

@@ -0,0 +1,69 @@
+// Metrics and their definitions.
+
+import type { MetricDef } from './interfaces';
+
+export const metricDefs: Record<string, MetricDef> = {
+  'status': {
+    name: 'status',
+    units: '',
+    description: 'Determines whether engine is running, completed or in error.',
+    isScalar: true,
+    precision: 0
+  },
+  'cpu': {
+    name: 'cpu',
+    units: '%',
+    description: 'Average utilization across CPU cores.',
+    isScalar: false,
+    precision: 1
+  },
+  'gpu': {
+    name: 'gpu',
+    units: '%',
+    description: 'Average utilization across GPUs.',
+    isScalar: false,
+    precision: 1
+  },
+  'ram': {
+    name: 'ram',
+    units: 'GB',
+    description: 'Utilization of RAM.',
+    isScalar: true,
+    precision: 1
+  },
+  'vram': {
+    name: 'vram',
+    units: 'GB',
+    description: 'Utilization of video RAM.',
+    isScalar: true,
+    precision: 1
+  },
+  'wall time': {
+    name: 'wall time',
+    units: 's',
+    description: 'Time taken from initial display to engine completion.',
+    isScalar: true,
+    precision: 1
+  },
+  'avg latency': {
+    name: 'avg latency',
+    units: 'ms',
+    description: 'Average roundtrip latency per token',
+    isScalar: true,
+    precision: 0
+  },
+  'consumed': {
+    name: 'consumed',
+    units: 'tkn',
+    description: 'Total tokens consumed by language model.',
+    isScalar: true,
+    precision: 0
+  },
+  'token reduction': {
+    name: 'token reduction',
+    units: '%',
+    description: 'Total tokens consumed by language model divided by total tokens.',
+    isScalar: true,
+    precision: 0
+  }
+};

client/graphpaper-inline/src/stitch.ts

@@ -96,7 +96,7 @@ export interface MetricMessage extends GuidanceMessage {
 }
 
 export interface StitchMessage {
-    type: "resize" | "clientmsg" | "kernelmsg",
+    type: "resize" | "clientmsg" | "kernelmsg" | "state" | "init_state",
     content: any
 }
 
@@ -157,4 +157,5 @@ export function isTokensMessage(o: GuidanceMessage | undefined | null): o is Tok
 
 export const kernelmsg = writable<StitchMessage | undefined>(undefined);
 export const clientmsg = writable<StitchMessage | undefined>(undefined);
+export const state = writable<StitchMessage | undefined>(undefined);
 

packages/python/stitch/examples/introduction.ipynb

@@ -9,7 +9,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -18,18 +18,39 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b05706653924451ab8c6464f439f4a4e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "StitchWidget(initial_height='auto', initial_width='100%', srcdoc='\\n<html style=\"\">\\n<script>\\nwindow.addEvent…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "w = stitch.StitchWidget()\n",
     "w.srcdoc = \"\"\"\n",
     "<html style=\"\">\n",
     "<script>\n",
     "window.addEventListener(\"message\", function(event) {\n",
-    "    if (event.source === window.parent && event.data.type === \"kernelmsg\") {\n",
-    "        document.getElementById(\"msgview\").innerHTML = event.data.content;\n",
-    "        window.parent.postMessage({type: \"clientmsg\", content: event.data.content}, \"*\");\n",
+    "    if (event.source === window.parent) {\n",
+    "        if (event.data.type === \"kernelmsg\") {\n",
+    "            document.getElementById(\"msgview\").innerHTML = event.data.content;\n",
+    "            window.parent.postMessage({type: \"clientmsg\", content: event.data.content}, \"*\");\n",
+    "            // Save state for offline render\n",
+    "            window.parent.postMessage({type: \"state\", content: event.data.content}, \"*\");\n",
+    "        } else if (event.data.type === \"init_state\") {\n",
+    "            document.getElementById(\"msgview\").innerHTML = event.data.content;\n",
+    "        }\n",
     "    }\n",
     "});\n",
     "\n",
@@ -68,7 +89,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -83,9 +104,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Wow, a change!\n"
+     ]
+    }
+   ],
    "source": [
     "w.observe(lambda x: print(x['new']), 'kernelmsg')\n",
     "w.kernelmsg = \"Wow, a change!\""
@@ -116,6 +145,36 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.12.5"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {
+     "b05706653924451ab8c6464f439f4a4e": {
+      "model_module": "@guidance-ai/stitch",
+      "model_module_version": "^0.1.1",
+      "model_name": "StitchModel",
+      "state": {
+       "_model_module_version": "^0.1.1",
+       "_view_module_version": "^0.1.1",
+       "clientmsg": "Wow, a change!",
+       "initial_height": "auto",
+       "initial_width": "100%",
+       "kernelmsg": "Wow, a change!",
+       "layout": "IPY_MODEL_c8d8410bc496453882680ff0fb8f6d0d",
+       "srcdoc": "\n<html style=\"\">\n<script>\nwindow.addEventListener(\"message\", function(event) {\n    if (event.source === window.parent) {\n        if (event.data.type === \"kernelmsg\") {\n            document.getElementById(\"msgview\").innerHTML = event.data.content;\n            window.parent.postMessage({type: \"clientmsg\", content: event.data.content}, \"*\");\n            // Save state for offline render\n            window.parent.postMessage({type: \"state\", content: event.data.content}, \"*\");\n        } else if (event.data.type === \"init_state\") {\n            document.getElementById(\"msgview\").innerHTML = event.data.content;\n        }\n    }\n});\n\nwindow.addEventListener(\"load\", function(){\n    var prevHeight = 0;\n    setInterval(function() {\n        var body = document.body;\n        var html = document.documentElement;\n        var height = html.getBoundingClientRect().height\n        if (height !== prevHeight && html.checkVisibility()) {\n            msg = {\n                type: 'resize',\n                content: {\n                    height: height + 'px',\n                    width: '100%'\n                }\n            };\n            window.parent.postMessage(msg, \"*\");\n            prevHeight = height;\n        }\n    }, 100);\n});\n\n</script>\n<body style=\"\">\n<div>\nMESSAGE: <span id=\"msgview\" style=\"background-color: #90ee90;\"></span>\n</div>\n</body>\n</html>\n",
+       "state": "Wow, a change!"
+      }
+     },
+     "c8d8410bc496453882680ff0fb8f6d0d": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
+     }
+    },
+    "version_major": 2,
+    "version_minor": 0
+   }
   }
  },
  "nbformat": 4,

packages/python/stitch/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@guidance-ai/stitch",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Bidirectional comms for Jupyter and JavaScript.",
   "keywords": [
     "jupyter",

packages/python/stitch/pyproject.toml

@@ -37,7 +37,7 @@ classifiers = [
 dependencies = [
     "ipywidgets>=8.0.0",
 ]
-version = "0.1.2"
+version = "0.1.4"
 
 [project.optional-dependencies]
 docs = [

packages/python/stitch/src/widget.ts

@@ -28,6 +28,7 @@ export class StitchModel extends DOMWidgetModel {
       initial_height: '1px',
       initial_width: '1px',
       initial_border: '0',
+      state: '',
     };
   }
 
@@ -48,6 +49,8 @@ export class StitchView extends DOMWidgetView {
   private _iframe: HTMLIFrameElement;
 
   render() {
+    // console.log('stitch:render');
+
     // Create sandboxed frame
     const iframe = document.createElement('iframe');
     iframe.sandbox.add('allow-scripts');
@@ -58,16 +61,20 @@ export class StitchView extends DOMWidgetView {
     iframe.style.display = 'block';
     this._iframe = iframe;
 
-    // Send first kernelmsg on load.
+    // Initialization steps when model is ready.
     const refreshTimeMs = 100;
-    const sendKernelMsgOnReady = () => {
+    const initOnReady = () => {
       if (this.model.isNew()) {
-        window.setTimeout(sendKernelMsgOnReady, refreshTimeMs);
+        window.setTimeout(initOnReady, refreshTimeMs);
       } else {
+        // Send init state if needed.
+        this.emit_init_state();
+        // Send first kernelmsg on load.
         this.kernelmsg_changed();
+        // console.log('stitch:is_new');
       }
     };
-    window.setTimeout(sendKernelMsgOnReady, refreshTimeMs);
+    window.setTimeout(initOnReady, refreshTimeMs);
 
     // Add callback for forwarding messages from client to kernel
     const model = this.model;
@@ -79,6 +86,9 @@ export class StitchView extends DOMWidgetView {
       } else if (win === event.source && event.data.type === 'resize') {
         iframe.style.height = event.data.content.height;
         iframe.style.width = event.data.content.width;
+      } else if (win === event.source && event.data.type === 'state') {
+        model.set('state', event.data.content);
+        model.save_changes();
       }
     };
     window.addEventListener('message', recvFromClient);
@@ -93,6 +103,21 @@ export class StitchView extends DOMWidgetView {
     this.model.on('change:srcdoc', this.srcdoc_changed, this);
   }
 
+  emit_init_state() {
+    const state = this.model.get('state');
+    if (state === '') {
+      // console.log('stitch:empty init state');
+      return;
+    }
+    const winmsg = {
+      type: 'init_state',
+      content: state,
+    };
+
+    // console.log('stitch:state');
+    this._iframe.contentWindow?.postMessage(winmsg, '*');
+  }
+
   kernelmsg_changed() {
     // Forward message from kernel to client
     const kernelmsg = this.model.get('kernelmsg');

packages/python/stitch/stitch/_frontend.py

@@ -9,4 +9,4 @@
 """
 
 module_name = "@guidance-ai/stitch"
-module_version = "^0.1.1"
+module_version = "^0.1.4"

packages/python/stitch/stitch/_version.py

@@ -4,5 +4,5 @@
 # Copyright (c) Guidance Contributors.
 # Distributed under the terms of the Modified BSD License.
 
-version_info = (0, 1, 2)
+version_info = (0, 1, 4)
 __version__ = ".".join(map(str, version_info))

packages/python/stitch/stitch/stitch.py

@@ -29,3 +29,6 @@ class StitchWidget(DOMWidget):
     initial_height = Unicode("1px").tag(sync=True)
     initial_width = Unicode("1px").tag(sync=True)
     initial_border = Unicode("0").tag(sync=True)
+
+    # NOTE(nopdive): Should we sync or not? There are overheads when we deal with bandwidth on real time applications.
+    state = Unicode("").tag(sync=True)