Merged in PhyNerd/lyse/UpdateDataframeNoRead (pull request labscript-suite#17)

Update Dataframe without Readoperations

Approved-by: Chris Billington <chrisjbillington@gmail.com>
Approved-by: Shaun Johnstone <shaun.johnstone@monash.edu>

Author: shjohnst

__init__.py

@@ -43,6 +43,8 @@
 # will be as follows. Otherwise lyse will override them with spinning_top =
 # True and path <name of hdf5 file being analysed>:
 spinning_top = False
+# data to be sent back to the lyse GUI if running within lyse
+_updated_data = {}
 
 if len(sys.argv) > 1:
     path = sys.argv[1]
@@ -159,7 +161,12 @@ def save_result(self, name, value, group=None, overwrite=True):
                 raise Exception('Attribute %s exists in group %s. ' \
                                 'Use overwrite=True to overwrite.' % (name, group))                   
             h5_file[group].attrs.modify(name, value)
-        
+            
+        if spinning_top:
+            if self.h5_path not in _updated_data:
+                _updated_data[self.h5_path] = {}
+            _updated_data[self.h5_path][str(self.group), name] = value
+
     def save_result_array(self, name, data, group=None, overwrite=True, keep_attrs=False):
         if self.no_write:
             raise Exception('This run is read-only. '

__main__.py

@@ -242,9 +242,9 @@ def do_analysis(self, filepath):
         self.to_worker.put(['analyse', filepath])
         signal, data = self.from_worker.get()
         if signal == 'error':
-            return False
+            return False, data
         elif signal == 'done':
-            return True
+            return True, data
         else:
             raise ValueError('invalid signal %s'%str(signal))
 
@@ -684,6 +684,7 @@ def do_analysis(self, filepath):
             routine.set_status('clear')
         remaining = self.todo()
         error = False
+        updated_data = {}
         while remaining:
             self.logger.debug('%d routines left to do'%remaining)
             for routine in self.routines:
@@ -694,7 +695,7 @@ def do_analysis(self, filepath):
             if routine is not None:
                 self.logger.info('running analysis routine %s'%routine.shortname)
                 routine.set_status('working')
-                success = routine.do_analysis(filepath)
+                success, updated_data = routine.do_analysis(filepath)
                 if success:
                     routine.set_status('done')
                     self.logger.debug('success')
@@ -713,11 +714,11 @@ def do_analysis(self, filepath):
             except ZeroDivisionError:
                 # All routines got deleted mid-analysis, we're done here:
                 status_percent = 100.0
-            self.to_filebox.put(['progress', status_percent])
+            self.to_filebox.put(['progress', status_percent, updated_data])
         if error:
-            self.to_filebox.put(['error', None])
+            self.to_filebox.put(['error', None, updated_data])
         else:
-            self.to_filebox.put(['done', 100.0])
+            self.to_filebox.put(['done', 100.0, {}])
         self.logger.debug('completed analysis of %s'%filepath)
 
     def reorder(self, order):
@@ -1285,16 +1286,25 @@ def mark_as_deleted_off_disk(self, filepath):
         app.output_box.output('Warning: Shot deleted from disk or no longer readable %s\n' % filepath, red=True)
 
     @inmain_decorator()
-    def update_row(self, filepath, dataframe_already_updated=False, status_percent=None, new_row_data=None):
+    def update_row(self, filepath, dataframe_already_updated=False, status_percent=None, new_row_data=None, updated_row_data=None):
         """"Updates a row in the dataframe and Qt model
         to the data in the HDF5 file for that shot. Also sets the percent done, if specified"""
         # Update the row in the dataframe first:
+        if (new_row_data is None) == (updated_row_data is None) and not dataframe_already_updated:
+            raise ValueError('Exactly one of new_row_data or updated_row_data must be provided')
+
         df_row_index = np.where(self.dataframe['filepath'].values == filepath)
         try:
             df_row_index = df_row_index[0][0]
         except IndexError:
             # Row has been deleted, nothing to do here:
             return
+
+        if updated_row_data is not None and not dataframe_already_updated:
+            for group, name in updated_row_data:
+                self.dataframe.loc[df_row_index, (group, name) + ('',) * (self.nlevels - 2)] = updated_row_data[group, name]
+            dataframe_already_updated = True
+
         if not dataframe_already_updated:
             if new_row_data is None:
                 raise ValueError("If dataframe_already_updated is False, then new_row_data, as returned "
@@ -1357,6 +1367,8 @@ def update_row(self, filepath, dataframe_already_updated=False, status_percent=N
             if not isinstance(column_name, tuple):
                 # One of our special columns, does not correspond to a column in the dataframe:
                 continue
+            if updated_row_data is not None and column_name not in updated_row_data:
+                continue
             item = self._model.item(model_row_number, column_number)
             if item is None:
                 # This is the first time we've written a value to this part of the model:
@@ -1704,33 +1716,29 @@ def do_singleshot_analysis(self, filepath):
             return
         self.to_singleshot.put(filepath)
         while True:
-            signal, status_percent = self.from_singleshot.get()
+            signal, status_percent, updated_data = self.from_singleshot.get()
             if signal in ['error', 'progress']:
-                # Do the file reading here outside the GUI thread so as not to hang the GUI:
-                try:
-                    new_row_data = get_dataframe_from_shot(filepath)
-                except IOError:
-                    self.shots_model.mark_as_deleted_off_disk(filepath)
-                    new_row_data = None
-                else:
-                    self.shots_model.update_row(filepath, status_percent=status_percent, new_row_data=new_row_data)
+                for file in updated_data:
+                    self.shots_model.update_row(file, status_percent=status_percent, updated_row_data=updated_data[file])
             if signal == 'done':
                 # No need to update the dataframe again, that should have been done with the last 'progress' signal:
                 self.shots_model.update_row(filepath, status_percent=status_percent, dataframe_already_updated=True)
                 return
             if signal == 'error':
-                # If new_row_data is None, that indicates that we got an
-                # IOError error above. Do not pause analysis in this
-                # case, as an error is expected given the shot file doesn't
-                # exist.
-                if new_row_data is not None:
+                if not os.path.exists(filepath):
+                    # Do not pause if the file has been deleted. An error is
+                    # no surprise there:
+                    self.shots_model.mark_as_deleted_off_disk(filepath)
+                else:
                     self.pause_analysis()
                 return
 
     def do_multishot_analysis(self):
         self.to_multishot.put(None)
         while True:
-            signal, _ = self.from_multishot.get()
+            signal, _, updated_data = self.from_multishot.get()
+            for file in updated_data:
+                self.shots_model.update_row(file, updated_row_data=updated_data[file])
             if signal == 'done':
                 self.multishot_required = False
                 return

analysis_subprocess.py

@@ -253,9 +253,9 @@ def mainloop(self):
                     path = data
                     success = self.do_analysis(path)
                     if success:
-                        self.to_parent.put(['done', None])
+                        self.to_parent.put(['done', lyse._updated_data])
                     else:
-                        self.to_parent.put(['error', None])
+                        self.to_parent.put(['error', lyse._updated_data])
                 else:
                     self.to_parent.put(['error','invalid task %s'%str(task)])
 
@@ -280,7 +280,7 @@ def do_analysis(self, path):
         sandbox.deprecation_messages['path'] = deprecation_message
         # Use lyse.path instead:
         lyse.path = path
-
+        lyse._updated_data = {}
         # Do not let the modulewatcher unload any modules whilst we're working:
         try:
             with self.modulewatcher.lock: