Merged in cbillington/lyse/feature (pull request labscript-suite#22)

Unicode strings and other Python 3 forward-compatible changes.

Approved-by: Jan Werkmann <jan.wrk.fb@gmail.com>

Author: chrisjbillington

__init__.py

@@ -11,11 +11,10 @@
 #                                                                   #
 #####################################################################
 
-from dataframe_utilities import get_series_from_shot as _get_singleshot
-from dataframe_utilities import dict_diff
+from __future__ import division, unicode_literals, print_function, absolute_import
+    
+from lyse.dataframe_utilities import get_series_from_shot as _get_singleshot, dict_diff
 import os
-import urllib
-import urllib2
 import socket
 import pickle as pickle
 import inspect
@@ -38,7 +37,11 @@
 
 # require pandas v0.15.0 up to the next major version
 check_version('pandas', '0.15.0', '1.0')
-check_version('zprocess', '2.2', '3.0')
+check_version('zprocess', '2.2.0', '3.0')
+check_version('labscript_utils', '2.4', '3.0')
+from labscript_utils import PY2
+if PY2:
+    str = unicode
 
 # If running stand-alone, and not from within lyse, the below two variables
 # will be as follows. Otherwise lyse will override them with spinning_top =
@@ -114,6 +117,9 @@ def __init__(self,h5_path,no_write=False):
                 # this Run object:
                 frame = inspect.currentframe()
                 __file__ = frame.f_back.f_locals['__file__']
+                if PY2:
+                    __file__ = __file__.decode(sys.getfilesystemencoding())
+                    print(repr(__file__))
                 self.group = os.path.basename(__file__).split('.py')[0]
                 with h5py.File(h5_path) as h5_file:
                     if not self.group in h5_file['results']:
@@ -224,7 +230,7 @@ def save_results(self, *args):
         names = args[::2]
         values = args[1::2]
         for name, value in zip(names, values):
-            print 'saving %s ='%name, value
+            print('saving %s =' % name, value)
             self.save_result(name, value)
 
     def save_results_dict(self, results_dict, uncertainties=False, **kwargs):
@@ -372,6 +378,8 @@ def __init__(self,h5_path,run_paths):
         frame = inspect.currentframe()
         try:
             __file__ = frame.f_back.f_locals['__file__']
+            if PY2:
+                __file__ = __file__.decode(sys.getfilesystemencoding())
             self.group = os.path.basename(__file__).split('.py')[0]
             with h5py.File(h5_path) as h5_file:
                 if not self.group in h5_file['results']:

__main__.py

@@ -5,7 +5,6 @@
 import os
 import sys
 import socket
-import Queue
 import logging
 import threading
 import signal
@@ -31,9 +30,7 @@
 except ImportError:
     raise ImportError('Require labscript_utils > 2.1.0')
 
-check_version('labscript_utils', '2.1', '3.0')
 check_version('qtutils', '2.0.0', '3.0.0')
-check_version('zprocess', '1.1.7', '3.0')
 
 import zprocess.locking
 from zprocess import ZMQServer
@@ -54,6 +51,13 @@
 from qtutils.auto_scroll_to_end import set_auto_scroll_to_end
 import qtutils.icons
 
+from labscript_utils import PY2
+if PY2:
+    str = unicode
+    import Queue as queue
+else:
+    import queue
+
 # Set working directory to lyse folder, resolving symlinks
 lyse_dir = os.path.dirname(os.path.realpath(__file__))
 os.chdir(lyse_dir)
@@ -878,10 +882,14 @@ def populate_model(self, column_names, columns_visible):
         self.ui.treeView.resizeColumnToContents(self.COL_VISIBLE)
         # Which indices in self.columns_visible the row numbers correspond to
         self.column_indices = {}
-        for column_index, name in sorted(column_names.items(), key=lambda s: s[1]):
-            if not isinstance(name, tuple):
-                # one of our special columns, ignore:
-                continue
+
+        # Remove our special columns from the dict of column names by keeping only tuples:
+        column_names = {i: name for i, name in column_names.items() if isinstance(name, tuple)}
+
+        # Sort the column names as comma separated values, converting to lower case:
+        sortkey = lambda item: ', '.join(item[1]).lower().strip(', ')
+
+        for column_index, name in sorted(column_names.items(), key=sortkey):
             visible = columns_visible[column_index]
             visible_item = QtGui.QStandardItem()
             visible_item.setCheckable(True)
@@ -893,7 +901,7 @@ def populate_model(self, column_names, columns_visible):
                 visible_item.setData(QtCore.Qt.Unchecked, self.ROLE_SORT_DATA)
             name_as_string = ', '.join(name).strip(', ')
             name_item = QtGui.QStandardItem(name_as_string)
-            name_item.setData(name_as_string, self.ROLE_SORT_DATA)
+            name_item.setData(sortkey((column_index, name)), self.ROLE_SORT_DATA)
             self.model.appendRow([visible_item, name_item])
             self.column_indices[self.model.rowCount() - 1] = column_index
 
@@ -1578,7 +1586,7 @@ def __init__(self, container, exp_config, to_singleshot, from_singleshot, to_mul
         # A queue for storing incoming files from the ZMQ server so
         # the server can keep receiving files even if analysis is slow
         # or paused:
-        self.incoming_queue = Queue.Queue()
+        self.incoming_queue = queue.Queue()
 
         # Start the thread to handle incoming files, and store them in
         # a buffer if processing is paused:
@@ -1686,7 +1694,7 @@ def incoming_buffer_loop(self):
                 while True:
                     try:
                         filepath = self.incoming_queue.get(False)
-                    except Queue.Empty:
+                    except queue.Empty:
                         break
                     else:
                         filepaths.append(filepath)
@@ -1851,12 +1859,12 @@ def __init__(self):
 
         # The singleshot routinebox will be connected to the filebox
         # by queues:
-        to_singleshot = Queue.Queue()
-        from_singleshot = Queue.Queue()
+        to_singleshot = queue.Queue()
+        from_singleshot = queue.Queue()
 
         # So will the multishot routinebox:
-        to_multishot = Queue.Queue()
-        from_multishot = Queue.Queue()
+        to_multishot = queue.Queue()
+        from_multishot = queue.Queue()
 
         self.output_box = OutputBox(self.ui.verticalLayout_output_box)
         self.singleshot_routinebox = RoutineBox(self.ui.verticalLayout_singleshot_routinebox, self.exp_config,

analysis_subprocess.py

@@ -11,6 +11,11 @@
 #                                                                   #
 #####################################################################
 
+from __future__ import division, unicode_literals, print_function, absolute_import
+from labscript_utils import PY2
+if PY2:
+    str = unicode
+
 import labscript_utils.excepthook
 import zprocess
 to_parent, from_parent, kill_lock = zprocess.setup_connection_with_parent(lock = True)
@@ -221,9 +226,18 @@ def __init__(self, filepath, to_parent, from_parent):
         self.to_parent = to_parent
         self.from_parent = from_parent
         self.filepath = filepath
+
+        # Filepath as a unicode string on py3 and a bytestring on py2,
+        # so that the right string type can be passed to functions that
+        # require the 'native' string type for that python version. On
+        # Python 2, encode it with the filesystem encoding.
+        if PY2:
+            self.filepath_native_string = self.filepath.encode(sys.getfilesystemencoding())
+        else:
+            self.filepath_native_string = self.filepath
 
         # Add user script directory to the pythonpath:
-        sys.path.insert(0, os.path.dirname(self.filepath))
+        sys.path.insert(0, os.path.dirname(self.filepath_native_string))
 
         # Plot objects, keyed by matplotlib Figure object:
         self.plots = {}
@@ -272,7 +286,7 @@ def do_analysis(self, path):
         # The namespace the routine will run in:
         sandbox = _DeprecationDict(path=path,
                                    __name__='__main__',
-                                   __file__= self.filepath)
+                                   __file__= os.path.basename(self.filepath_native_string))
         # path global variable is deprecated:
         deprecation_message = ("use of 'path' global variable is deprecated and will be removed " +
                                "in a future version of lyse.  Please use lyse.path, which defaults " +
@@ -281,20 +295,43 @@ def do_analysis(self, path):
         # Use lyse.path instead:
         lyse.path = path
         lyse._updated_data = {}
+
+        # Save the current working directory before changing it to the
+        # location of the user's script:
+        cwd = os.getcwd()
+        os.chdir(os.path.dirname(self.filepath))
+
         # Do not let the modulewatcher unload any modules whilst we're working:
         try:
             with self.modulewatcher.lock:
                 # Actually run the user's analysis!
-                execfile(self.filepath, sandbox, sandbox)
+                with open(self.filepath) as f:
+                    code = compile(f.read(), os.path.basename(self.filepath_native_string),
+                                   'exec', dont_inherit=True)
+                    exec(code, sandbox)
         except:
             traceback_lines = traceback.format_exception(*sys.exc_info())
             del traceback_lines[1]
-            message = ''.join(traceback_lines)
+            # Avoiding a list comprehension here so as to avoid this
+            # python bug in earlier versions of 2.7 (fixed in 2.7.9):
+            # https://bugs.python.org/issue21591
+            message = ''
+            for line in traceback_lines:
+                if PY2:
+                    # errors='replace' is for Windows filenames present in the
+                    # traceback that are not UTF8. They will not display
+                    # correctly, but that's the best we can do - the traceback
+                    # may contain code from the file in a different encoding,
+                    # so we could have a mixed encoding string. This is only
+                    # a problem for Python 2.
+                    line = line.decode('utf8', errors='replace')
+                message += line
             sys.stderr.write(message)
             return False
         else:
             return True
         finally:
+            os.chdir(cwd)
             print('')
             self.post_analysis_plot_actions()
 

dataframe_utilities.py

@@ -11,6 +11,11 @@
 #                                                                   #
 #####################################################################
 
+from __future__ import division, unicode_literals, print_function, absolute_import
+from labscript_utils import PY2
+if PY2:
+    str = unicode
+    
 import labscript_utils.h5_lock, h5py
 import pandas
 import os
@@ -20,7 +25,53 @@
 
 import runmanager
 
+# Monkey patch a bugfix onto older versions of pandas on Python 2. This code
+# can be removed once lyse otherwise depends on pandas >= 0.21.0.
+# https://github.com/pandas-dev/pandas/pull/17099
+if PY2:
+    try:
+        from labscript_utils import check_version, VersionException
+        check_version('pandas', '0.21.0', '2.0')
+    except VersionException:
+        
+        import numpy as np
+        from pandas import Series, Index
+        from pandas.core.indexing import maybe_droplevels
+        def _getitem_multilevel(self, key):
+            loc = self.columns.get_loc(key)
+            if isinstance(loc, (slice, Series, np.ndarray, Index)):
+                new_columns = self.columns[loc]
+                result_columns = maybe_droplevels(new_columns, key)
+                if self._is_mixed_type:
+                    result = self.reindex(columns=new_columns)
+                    result.columns = result_columns
+                else:
+                    new_values = self.values[:, loc]
+                    result = self._constructor(new_values, index=self.index,
+                                               columns=result_columns)
+                    result = result.__finalize__(self)
+                if len(result.columns) == 1:
+                    top = result.columns[0]
+                    if isinstance(top, tuple):
+                        top = top[0]
+                    if top == '':
+                        result = result['']
+                        if isinstance(result, Series):
+                            result = self._constructor_sliced(result,
+                                                              index=self.index,
+                                                              name=key)
+
+                result._set_is_copy(self)
+                return result
+            else:
+                return self._get_item_cache(key)
+
+        pandas.DataFrame._getitem_multilevel = _getitem_multilevel
+
+
 def asdatetime(timestr):
+    if isinstance(timestr, bytes):
+        timestr = timestr.decode('utf-8')
     tz = tzlocal.get_localzone().zone
     return pandas.Timestamp(timestr, tz=tz)
 
@@ -78,10 +129,10 @@ def flatten_dict(dictionary, keys=tuple()):
     result = {}
     for name in dictionary:
         if isinstance(dictionary[name],dict):
-            flat = flatten_dict(dictionary[name],keys=keys + (str(name),))
+            flat = flatten_dict(dictionary[name],keys=keys + (name,))
             result.update(flat)
         else:
-            result[keys + (str(name),)] = dictionary[name]
+            result[keys + (name,)] = dictionary[name]
     return result
 
 def flat_dict_to_hierarchical_dataframe(dictionary):

figure_manager.py

@@ -11,6 +11,11 @@
 #                                                                   #
 #####################################################################
 
+from __future__ import division, unicode_literals, print_function, absolute_import
+from labscript_utils import PY2
+if PY2:
+    str = unicode
+    
 import lyse
 import sys
 

tempfile2clipboard.py

@@ -11,6 +11,7 @@
 #                                                                   #
 #####################################################################
 
+from __future__ import division, unicode_literals, print_function, absolute_import
 import sys
 import os