Merged in monashspinor/labscript_devices/RFBlaster_kloned (pull request labscript-suite#30)

RFBlaster stability improvements

Author: rpanderson

RFBlaster.py

@@ -21,6 +21,8 @@
 
 from labscript_devices import BLACS_tab, runviewer_parser
 
+from labscript_utils.setup_logging import setup_logging
+
 # Define a RFBlasterPseudoclock that only accepts one child clockline
 class RFBlasterPseudoclock(Pseudoclock):    
     def add_device(self, device):
@@ -261,7 +263,7 @@ def initialise_GUI(self):
         self.address = "http://" + str(self.BLACS_connection) + ":8080"
 
         # Create and set the primary worker
-        self.create_worker("main_worker",RFBlasterWorker,{'address':self.address, 'num_DDS':self.num_DDS})
+        self.create_worker("main_worker", RFBlasterWorker, {'address': self.address, 'num_DDS': self.num_DDS})
         self.primary_worker = "main_worker"
 
         # Set the capabilities of this device
@@ -369,13 +371,39 @@ class RFBlasterWorker(Worker):
     def init(self):
         exec('from numpy import *', globals())
         global h5py; import labscript_utils.h5_lock, h5py
-        self.timeout = 30 #How long do we wait until we assume that the RFBlaster is dead? (in seconds)
+        global re; import re
+        self.timeout = 10   # How long do we wait until we assume that the RFBlaster is dead? (in seconds)
+        self.retries = 3    # Retry attempts before (a) giving up, or (b) attempting to restart kloned (uniform timeout)
+        p = re.compile('http://([0-9.]+):[0-9]+')
+        m = p.match(self.address)
+        self.ip = m.group(1)
+        # self.ip = self.BLACS_connection
+        self.netlogger = setup_logging('rfBlaster_%s' % self.ip)
+        self.netlogger.info('init: Started logging')
 
         # See if the RFBlaster answers
         self.http_request()
 
         self._last_program_manual_values = {}
-        
+
+    def restart_kloned(self, respawn_netcat=True):
+        import socket, time
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.settimeout(self.timeout)
+        self.netlogger.info('restart_kloned: Connecting to %s...' % self.ip)
+        s.connect((self.ip, 8009))
+        self.netlogger.info('restart_kloned: Connected!')
+        if respawn_netcat:
+            self.netlogger.info('restart_kloned: Respawning netcat...')
+            s.sendall(b'nohup nc -l -p 8009 -e /bin/sh &')
+            time.sleep(0.5)
+        self.netlogger.info('restart_kloned: Trying to start/restart kloned...')
+        s.sendall(b'./startup/klone_start.sh')
+        time.sleep(0.5)
+        s.shutdown(socket.SHUT_WR)
+        self.netlogger.info('restart_kloned: Finished. Closing socket.')
+        s.close()
+
     def program_manual(self,values):
         self._last_program_manual_values = values
 
@@ -395,7 +423,7 @@ def program_manual(self,values):
     def transition_to_buffered(self,device_name,h5file,initial_values,fresh):
         with h5py.File(h5file,'r') as hdf5_file:
             group = hdf5_file['devices'][device_name]
-            #Strip out the binary files and submit to the webserver
+            # Strip out the binary files and submit to the webserver
             form = MultiPartForm()
             self.final_values = {}
             finalfreq = zeros(self.num_DDS)
@@ -410,9 +438,9 @@ def transition_to_buffered(self,device_name,h5file,initial_values,fresh):
                                                  'gate':True
                                                 }
                 data = group['BINARY_CODE/DDS%d'%i].value
-                form.add_file_content("pulse_ch%d"%i,"output_ch%d.bin"%i,data)
+                form.add_file_content("pulse_ch%d"%i, "output_ch%d.bin"%i, data)
 
-        form.add_field("upload_and_run","Upload and start")
+        form.add_field("upload_and_run", "Upload and start")
         self.http_request(form)
         return self.final_values
 
@@ -426,8 +454,8 @@ def abort_transition_to_buffered(self):
 
     def abort_buffered(self):
         form = MultiPartForm()
-        #tell the rfblaster to stop
-        form.add_field("halt","Halt execution")
+        # Tell the rfblaster to stop
+        form.add_field("halt", "Halt execution")
         self.http_request(form)
         return True
 
@@ -438,9 +466,11 @@ def transition_to_manual(self):
     def http_request(self, form=None): 
         """Make a HTTP request to the RFBlaster, optionally submitting a form"""
         if PY2:
-            from urllib2 import urlopen, Request
+            from urllib2 import urlopen, Request, URLError, httplib
+            HTTPError = httplib.HTTPException
         else:
             from urllib.request import urlopen, Request
+            from urllib.error import URLError, HTTPError
 
         req = Request(self.address)
         if form is not None:
@@ -449,17 +479,38 @@ def http_request(self, form=None):
             req.add_header(b'Content-length', len(body))
             req.data = body
 
-        page = b''.join(urlopen(req, timeout=self.timeout).readlines())
-        return page
+        self._connection_attempt = 1
+        self._kloned_attempted = False
+        response = None
+        while not response:
+            try:
+                self.netlogger.info('Connection attempt %i.' % self._connection_attempt)
+                response = b''.join(urlopen(req, timeout=self.timeout).readlines())
+                self.netlogger.info('Connected!')
+                break
+            except (URLError, HTTPError) as e:
+                self.netlogger.warning(str(e))
+                if self._connection_attempt < self.retries:
+                    self.netlogger.info('Connection failed. Trying again (%i more attempts remain).' % (self.retries - self._connection_attempt))
+                    self._connection_attempt += 1
+                elif not self._kloned_attempted:
+                    self._kloned_attempted = True
+                    self.restart_kloned()
+                    self._connection_attempt = 1
+                else:
+                    self.netlogger.error(str(e))   
+                    raise e
+
+        return response
 
     def get_web_values(self, page): 
         page = page.decode('utf8')
         import re
-        #prepare regular expressions for finding the values:
+        # Prepare regular expressions for finding the values:
         search = re.compile(r'name="([fap])_ch(\d+?)_in"\s*?value="([0-9.]+?)"')
-        webvalues = re.findall(search,page)
+        webvalues = re.findall(search, page)
 
-        register_name_map = {'f':'freq','a':'amp','p':'phase'}
+        register_name_map = {'f': 'freq', 'a': 'amp', 'p': 'phase'}
         newvals = {}
         for i in range(self.num_DDS):
             newvals['dds %d'%i] = {}
@@ -480,7 +531,7 @@ def get_web_values(self, page):
         return newvals
 
     def check_remote_values(self):
-        #read the webserver page to see what values it puts in the form
+        # Read the webserver page to see what values it puts in the form
         return self.get_web_values(self.http_request())
 
     def shutdown(self):