Merged in zprocess_changes (pull request labscript-suite#28)

Update to use new zprocess functionality

Author: chrisjbillington

__init__.py

@@ -16,6 +16,11 @@
 if PY2:
     str = unicode
 
+try:
+    from labscript_utils import check_version
+except ImportError:
+    raise ImportError('Require labscript_utils > 2.1.0')
+
 import itertools
 import os
 import sys
@@ -30,7 +35,9 @@
 import h5py
 import numpy as np
 
-import zprocess
+check_version('labscript_utils', '2.11.0', '3')
+from labscript_utils.ls_zprocess import ProcessTree, zmq_push_multipart
+process_tree = ProcessTree.instance()
 
 __version__ = '2.2.0'
 
@@ -681,13 +688,20 @@ def compile_labscript_with_globals_files(labscript_file, globals_files, output_p
 
 
 def compile_labscript_async(labscript_file, run_file, stream_port, done_callback):
-    """Compiles labscript_file with run_file. This function is designed
-    to be called in a thread.  The stdout and stderr from the compilation
-    will be shoveled into stream_port via zmq push as it spews forth, and
-    when compilation is complete, done_callback will be called with a
-    boolean argument indicating success."""
+    """Compiles labscript_file with run_file. This function is designed to be called in
+    a thread.  The stdout and stderr from the compilation will be shovelled into
+    stream_port via zmq push as it spews forth, and when compilation is complete,
+    done_callback will be called with a boolean argument indicating success. Note that
+    the zmq communication will be encrypted, or not, according to security settings in
+    labconfig. If you want to receive the data on a zmq socket, do so using a PULL
+    socket created from a labscript_utils.ls_zprocess.Context, or using a
+    labscript_utils.ls_zprocess.ZMQServer. These subclasses will also be configured
+    with the appropriate security settings and will be able to receive the messages.
+    """
     compiler_path = os.path.join(os.path.dirname(__file__), 'batch_compiler.py')
-    to_child, from_child, child = zprocess.subprocess_with_queues(compiler_path, stream_port)
+    to_child, from_child, child = process_tree.subprocess(
+        compiler_path, output_redirection_port=stream_port
+    )
     to_child.put(['compile', [labscript_file, run_file]])
     while True:
         signal, data = from_child.get()
@@ -702,14 +716,19 @@ def compile_labscript_async(labscript_file, run_file, stream_port, done_callback
 
 
 def compile_multishot_async(labscript_file, run_files, stream_port, done_callback):
-    """Compiles labscript_file with run_files. This function is designed
-    to be called in a thread.  The stdout and stderr from the compilation
-    will be shoveled into stream_port via zmq push as it spews forth,
-    and when each compilation is complete, done_callback will be called
-    with a boolean argument indicating success. Compilation will stop
-    after the first failure."""
+    """Compiles labscript_file with run_files. This function is designed to be called in
+    a thread.  The stdout and stderr from the compilation will be shovelled into
+    stream_port via zmq push as it spews forth, and when each compilation is complete,
+    done_callback will be called with a boolean argument indicating success. Compilation
+    will stop after the first failure.  If you want to receive the data on a zmq socket,
+    do so using a PULL socket created from a labscript_utils.ls_zprocess.Context, or
+    using a labscript_utils.ls_zprocess.ZMQServer. These subclasses will also be
+    configured with the appropriate security settings and will be able to receive the
+    messages."""
     compiler_path = os.path.join(os.path.dirname(__file__), 'batch_compiler.py')
-    to_child, from_child, child = zprocess.subprocess_with_queues(compiler_path, stream_port)
+    to_child, from_child, child = process_tree.subprocess(
+        compiler_path, output_redirection_port=stream_port
+    )
     try:
         for run_file in run_files:
             to_child.put(['compile', [labscript_file, run_file]])
@@ -723,7 +742,7 @@ def compile_multishot_async(labscript_file, run_files, stream_port, done_callbac
                 break
     except Exception:
         error = traceback.format_exc()
-        zprocess.zmq_push_multipart(stream_port, data=[b'stderr', error.encode('utf-8')])
+        zmq_push_multipart(stream_port, data=[b'stderr', error.encode('utf-8')])
         to_child.put(['quit', None])
         child.communicate()
         raise
@@ -732,12 +751,15 @@ def compile_multishot_async(labscript_file, run_files, stream_port, done_callbac
 
 
 def compile_labscript_with_globals_files_async(labscript_file, globals_files, output_path, stream_port, done_callback):
-    """Same as compile_labscript_with_globals_files, except it launches
-    a thread to do the work and does not return anything. Instead,
-    stderr and stdout will be put to stream_port via zmq push in
-    the multipart message format ['stdout','hello, world\n'] etc. When
-    compilation is finished, the function done_callback will be called
-    a boolean argument indicating success or failure."""
+    """Same as compile_labscript_with_globals_files, except it launches a thread to do
+    the work and does not return anything. Instead, stderr and stdout will be put to
+    stream_port via zmq push in the multipart message format ['stdout','hello, world\n']
+    etc. When compilation is finished, the function done_callback will be called a
+    boolean argument indicating success or failure.  If you want to receive the data on
+    a zmq socket, do so using a PULL socket created from a
+    labscript_utils.ls_zprocess.Context, or using a
+    labscript_utils.ls_zprocess.ZMQServer. These subclasses will also be configured with
+    the appropriate security settings and will be able to receive the messages."""
     try:
         make_run_file_from_globals_files(labscript_file, globals_files, output_path)
         thread = threading.Thread(
@@ -746,7 +768,7 @@ def compile_labscript_with_globals_files_async(labscript_file, globals_files, ou
         thread.start()
     except Exception:
         error = traceback.format_exc()
-        zprocess.zmq_push_multipart(stream_port, data=[b'stderr', error.encode('utf-8')])
+        zmq_push_multipart(stream_port, data=[b'stderr', error.encode('utf-8')])
         t = threading.Thread(target=done_callback, args=(False,))
         t.daemon = True
         t.start()

__main__.py

@@ -58,19 +58,17 @@
 splash.update_text('importing Qt')
 check_version('qtutils', '2.0.0', '3.0.0')
 
-splash.update_text('importing zprocess')
-check_version('zprocess', '1.1.5', '3.0')
-
 splash.update_text('importing pandas')
 check_version('pandas', '0.13', '2')
 
 from qtutils.qt import QtCore, QtGui, QtWidgets
 from qtutils.qt.QtCore import pyqtSignal as Signal
 
-import zprocess.locking
 from zmq import ZMQError
 
 splash.update_text('importing labscript suite modules')
+check_version('labscript_utils', '2.11.0', '3')
+from labscript_utils.ls_zprocess import zmq_get, ProcessTree
 from labscript_utils.labconfig import LabConfig, config_prefix
 from labscript_utils.setup_logging import setup_logging
 import labscript_utils.shared_drive as shared_drive
@@ -84,8 +82,10 @@
 runmanager_dir = os.path.dirname(os.path.realpath(__file__))
 os.chdir(runmanager_dir)
 
+process_tree = ProcessTree.instance()
+
 # Set a meaningful name for zprocess.locking's client id:
-zprocess.locking.set_client_process_name('runmanager')
+process_tree.zlock_client.set_process_name('runmanager')
 
 
 def log_if_global(g, g_list, message):
@@ -1331,8 +1331,9 @@ def __init__(self):
 
         splash.update_text('starting compiler subprocess')
         # Start the compiler subprocess:
-        self.to_child, self.from_child, self.child = zprocess.subprocess_with_queues(
-            'batch_compiler.py', self.output_box.port)
+        self.to_child, self.from_child, self.child = process_tree.subprocess(
+            'batch_compiler.py', output_redirection_port=self.output_box.port
+        )
 
         # Start a thread to monitor the time of day and create new shot output
         # folders for each day:
@@ -1748,8 +1749,9 @@ def check_child_exited(self, timeout_time, kill=False):
         else:
             self.output_box.output('done.\n')
         self.output_box.output('Spawning new compiler subprocess...')
-        self.to_child, self.from_child, self.child = zprocess.subprocess_with_queues(
-            'batch_compiler.py', self.output_box.port)
+        self.to_child, self.from_child, self.child = process_tree.subprocess(
+            'batch_compiler.py', output_redirection_port=self.output_box.port
+        )
         self.output_box.output('done.\n')
         self.output_box.output('Ready.\n\n')
 
@@ -3347,7 +3349,7 @@ def send_to_BLACS(self, run_file, BLACS_hostname):
         agnostic_path = shared_drive.path_to_agnostic(run_file)
         self.output_box.output('Submitting run file %s.\n' % os.path.basename(run_file))
         try:
-            response = zprocess.zmq_get(port, BLACS_hostname, data=agnostic_path)
+            response = zmq_get(port, BLACS_hostname, data=agnostic_path)
             if 'added successfully' in response:
                 self.output_box.output(response)
             else:
@@ -3360,7 +3362,7 @@ def send_to_runviewer(self, run_file):
         runviewer_port = int(self.exp_config.get('ports', 'runviewer'))
         agnostic_path = shared_drive.path_to_agnostic(run_file)
         try:
-            response = zprocess.zmq_get(runviewer_port, 'localhost', data='hello', timeout=1)
+            response = zmq_get(runviewer_port, 'localhost', data='hello', timeout=1)
             if 'hello' not in response:
                 raise Exception(response)
         except Exception as e:
@@ -3379,12 +3381,12 @@ def send_to_runviewer(self, run_file):
                                      stdin=devnull, stdout=devnull, stderr=devnull, close_fds=True)
                     os._exit(0)
             try:
-                zprocess.zmq_get(runviewer_port, 'localhost', data='hello', timeout=15)
+                zmq_get(runviewer_port, 'localhost', data='hello', timeout=15)
             except Exception as e:
                 self.output_box.output('Couldn\'t submit shot to runviewer: %s\n\n' % str(e), red=True)
 
         try:
-            response = zprocess.zmq_get(runviewer_port, 'localhost', data=agnostic_path, timeout=0.5)
+            response = zmq_get(runviewer_port, 'localhost', data=agnostic_path, timeout=0.5)
             if 'ok' not in response:
                 raise Exception(response)
             else:

batch_compiler.py

@@ -15,14 +15,20 @@
 if PY2:
     str = unicode
 
+from labscript_utils.ls_zprocess import ProcessTree
+process_tree = ProcessTree.connect_to_parent()
+to_parent = process_tree.to_parent
+from_parent = process_tree.from_parent
+kill_lock = process_tree.kill_lock
+
+# Set a meaningful name for zprocess.locking's client id:
+process_tree.zlock_client.set_process_name('runmanager.batch_compiler')
+
 import os
 import sys
 import traceback
-from zprocess import setup_connection_with_parent
-to_parent, from_parent, kill_lock = setup_connection_with_parent(lock = True)
 
 import labscript
-import labscript_utils.excepthook
 from labscript_utils.modulewatcher import ModuleWatcher
 
 class BatchProcessor(object):