Merge python3 into master (#679)

* python3 WIP (#674)

* python3: parens around single-line print statements

* python3: subprocess.check_output() => subprocess.getoutput()

* python3: 2to3

* python3: don't use relative protos, generate from root and include relative to root. Delete string.decode(), will fix with future unicode_literal later.

* Python3: PhaseExecutionOutcome has __new__ and not __init__ since named tuples are immutable

* Python3: Initialize TestRecord.start_time_millis to 0. In PlugsTest.test_initialize, only compare the knowable values of the plug_manager dictionary.

* Python3: self.join fails with float_info.max, replace with 5 seconds (it seems to be a placeholder value). Encode the test_attachment string as utf-8

* Python3: CodeInfo must derive from HashableRecord since it's held in a dict.

* Python3: Don't utf8-encode the unit suffixes

* Python3: explicitly encode value_binary fields as utf-8

* Python3: BytesIO instead of StringIO in tests.

* Python3: Store attachments as unicode strings containing base64-encoded payloads (used to be byte strings of b64 payloads). Use a StringIO for the test that needs it.

* Python3: Remove 2to3 unnecessary double-parentheses around print calls.

* Python3: Point Travis at Python 3.6

* Python3: Require protobuf 3.4.0, exploring Travis failures

* Python3: Require protobuf package >3, explicitly download + unpack protoc > 3. From Brandon Wallace (wallacbe)

* python3 updates to allow examples to run

* updates to allow examples to run in python27

* enable pyhon2.7 in travis

* add __next__method

* update test_json

* update long in data test

* Update raises regex

* travis updates

* remove python3 checks

* updates based on comments

* try to address feedback from fahhem

* update test_executor timeout

* address additional code review

* use hashable record

Author: wallacbe

.travis.yml

@@ -1,9 +1,15 @@
 language: python
-python:
-- '2.7'
+matrix:
+  include:
+    - python: 2.7
+      env: TOXENV=py27
+    - python: 3.6
+      env: TOXENV=py36
 addons:
   apt:
     packages:
+    - python3
+    - python3-pip
     - swig
     - libusb-1.0-0-dev
     - libprotobuf-dev
@@ -12,8 +18,6 @@ cache:
   apt: true
   directories:
   - $HOME/.cache/pip
-env:
-- TOXENV=py27
 install:
 - pip install tox coveralls
 - pip install -e .

CONTRIBUTING.md

@@ -213,8 +213,8 @@ accordingly.
 git clone https://github.com/google/openhtf.git
 
 # Install system-level third-party dependencies.
-sudo apt-get install python-pip swig libssl-dev python-dev libffi-dev \
-protobuf-compiler libprotobuf-dev
+sudo apt-get install python-pip swig libssl-dev python-dev python3-dev \
+libffi-dev protobuf-compiler libprotobuf-dev
 
 # Make sure pip is up-to-date.
 sudo pip install --upgrade pip

bin/units_from_xls.py

@@ -158,10 +158,10 @@ def __call__(self, name_or_suffix):
                          '15': 'FIFTEEN',
                          '30': 'THIRTY',
                          '\\': '_',
-                         unichr(160): '_',
-                         unichr(176): 'DEG_',
-                         unichr(186): 'DEG_',
-                         unichr(8211): '_',
+                         chr(160): '_',
+                         chr(176): 'DEG_',
+                         chr(186): 'DEG_',
+                         chr(8211): '_',
                         }
 
 
@@ -182,7 +182,7 @@ def main():
   args = parser.parse_args()
 
   if not os.path.exists(args.xlsfile):
-    print 'Unable to locate the file "%s".' % args.xlsfile
+    print('Unable to locate the file "%s".' % args.xlsfile)
     parser.print_help()
     sys.exit()
 
@@ -217,7 +217,7 @@ def unit_defs_from_sheet(sheet, column_names):
     rows = sheet.get_rows()
 
     # Find the indices for the columns we care about.
-    for idx, cell in enumerate(rows.next()):
+    for idx, cell in enumerate(next(rows)):
       if cell.value in column_names:
         col_indices[cell.value] = idx
 
@@ -245,7 +245,7 @@ def unit_key_from_name(name):
   """Return a legal python name for the given name for use as a unit key."""
   result = name
 
-  for old, new in UNIT_KEY_REPLACEMENTS.iteritems():
+  for old, new in UNIT_KEY_REPLACEMENTS.items():
     result = result.replace(old, new)
 
   # Collapse redundant underscores and convert to uppercase.

contrib/poll_stations.py

@@ -30,9 +30,10 @@
 
 """
 
+from __future__ import print_function
 import logging
 import os
-import Queue
+import queue
 import socket
 import sys
 import threading
@@ -111,7 +112,7 @@ class StationList(object):
   """
 
   def __init__(self):
-    self.update_queue = Queue.Queue()
+    self.update_queue = queue.Queue()
     self.stations = set()
     # Really, these threads should be tracked on a per-station basis, because
     # two stations *could* have RemoteTest instances that would compare equal
@@ -168,7 +169,7 @@ def watch_test(self, remote_test):
   def print_station(self, station):
     print(station)
     try:
-      for remote_test in station.tests.itervalues():
+      for remote_test in station.tests.values():
         # Trigger an update of the local history cache and state.
         remote_test.state
         remote_test.history
@@ -182,7 +183,7 @@ def print_station(self, station):
           self.update_threads[remote_test] = update_thread
     except socket.error as e:
       print(' |-- Connection Error: %s' % e)
-    print
+    print()
 
   def check_for_stations(self):
     """Discover for new stations, doesn't remove any stations."""
@@ -199,7 +200,7 @@ def mainloop(self):
       station_list.check_for_stations()
       try:
         self.update(self.update_queue.get(timeout=5))
-      except Queue.Empty:
+      except queue.Empty:
         pass
 
 

examples/all_the_things.py

@@ -28,7 +28,7 @@
 from openhtf.output import callbacks
 from openhtf.output.callbacks import json_factory
 
-import example_plugs
+from examples import example_plugs
 
 
 @htf.plug(example=example_plugs.ExamplePlug)
@@ -107,7 +107,7 @@ def measures_with_args(test, min, max):
 
 
 def attachments(test):
-  test.attach('test_attachment', 'This is test attachment data.')
+  test.attach('test_attachment', 'This is test attachment data.'.encode('utf-8'))
   test.attach_from_file(
       os.path.join(os.path.dirname(__file__), 'example_attachment.txt'))
 

examples/repeat.py

@@ -27,6 +27,7 @@
 phase can be limited specifying a PhaseOptions.repeat_limit.
 """
 
+from __future__ import print_function
 import openhtf
 import openhtf.plugs as plugs
 
@@ -40,7 +41,7 @@ def __init__(self):
   def run(self):
     """Increments counter and raises an exception for first two runs."""
     self.count += 1
-    print 'FailTwicePlug: Run number %s' % (self.count)
+    print('FailTwicePlug: Run number %s' % (self.count))
     if self.count < 3:
       raise RuntimeError('Fails a couple times')
 
@@ -56,7 +57,7 @@ def __init__(self):
   def run(self):
     """Increments counter and returns False indicating failure"""
     self.count += 1
-    print "FailAlwaysPlug: Run number %s" % (self.count)
+    print("FailAlwaysPlug: Run number %s" % (self.count))
 
     return False
 
@@ -70,10 +71,10 @@ def phase_repeat(test, test_plug):
     test_plug.run()
 
   except:
-    print "Error in phase_repeat, will retry"
+    print("Error in phase_repeat, will retry")
     return openhtf.PhaseResult.REPEAT
 
-  print "Completed phase_repeat"
+  print("Completed phase_repeat")
 
 
 # This phase demonstrates repeating a phase based upon a result returned from a
@@ -86,7 +87,7 @@ def phase_repeat_with_limit(test, test_plug):
   result = test_plug.run()
 
   if not result:
-    print "Invalid result in phase_repeat_with_limit, will retry"
+    print("Invalid result in phase_repeat_with_limit, will retry")
     return openhtf.PhaseResult.REPEAT
 
 if __name__ == '__main__':

examples/with_plugs.py

@@ -25,6 +25,7 @@
 end up with the 4 phases you want.
 """
 
+from __future__ import print_function
 import subprocess
 import time
 
@@ -52,7 +53,7 @@ def _get_command(self, count):
 
   def run(self, count):
     command = self._get_command(count)
-    print "running: %s" % ' '.join(command)
+    print("running: %s" % ' '.join(command))
     return subprocess.call(command)
 
 

openhtf/__init__.py

@@ -192,7 +192,7 @@ def configure(self, **kwargs):
     # side effects.
     create_arg_parser(add_help=True).parse_known_args()
     logs.setup_logger()
-    for key, value in kwargs.iteritems():
+    for key, value in kwargs.items():
       setattr(self._test_options, key, value)
 
   @classmethod
@@ -390,7 +390,7 @@ def format_strings(self, **kwargs):
         self, name=util.format_string(self.name, kwargs))
 
   def update(self, **kwargs):
-    for key, value in kwargs.iteritems():
+    for key, value in kwargs.items():
       if key not in self.__slots__:
         raise AttributeError('Type %s does not have attribute %s' % (
             type(self).__name__, key))
@@ -481,7 +481,7 @@ def with_plugs(self, **subplugs):
     plugs_by_name = {plug.name: plug for plug in self.plugs}
     new_plugs = dict(plugs_by_name)
 
-    for name, sub_class in subplugs.iteritems():
+    for name, sub_class in subplugs.items():
       original_plug = plugs_by_name.get(name)
       accept_substitute = True
       if original_plug is None:
@@ -503,7 +503,7 @@ def with_plugs(self, **subplugs):
 
     return mutablerecords.CopyRecord(
         self,
-        plugs=new_plugs.values(),
+        plugs=list(new_plugs.values()),
         options=self.options.format_strings(**subplugs),
         measurements=[m.with_args(**subplugs) for m in self.measurements])
 

openhtf/core/__init__.py

@@ -1 +1 @@
-from test_executor import TestExecutionError, TestStopError, TestExecutor
+from .test_executor import TestExecutionError, TestStopError, TestExecutor

openhtf/core/measurements.py

@@ -325,7 +325,7 @@ def is_value_set(self):
 
   def __iter__(self):  # pylint: disable=invalid-name
     """Iterate over items, allows easy conversion to a dict."""
-    return self.value_dict.iteritems()
+    return iter(self.value_dict.items())
 
   def __setitem__(self, coordinates, value):  # pylint: disable=invalid-name
     coordinates_len = len(coordinates) if hasattr(coordinates, '__len__') else 1
@@ -366,7 +366,7 @@ def value(self):
     if not self.is_value_set:
       raise MeasurementNotSetError('Measurement not yet set', self.name)
     return [dimensions + (value,) for dimensions, value in
-            self.value_dict.iteritems()]
+            self.value_dict.items()]
 
 
 class Collection(mutablerecords.Record('Collection', ['_measurements'])):
@@ -421,7 +421,7 @@ def _assert_valid_key(self, name):
   def __iter__(self):  # pylint: disable=invalid-name
     """Extract each MeasurementValue's value."""
     return ((key, meas.measured_value.value)
-            for key, meas in self._measurements.iteritems())
+            for key, meas in self._measurements.items())
 
   def __setattr__(self, name, value):  # pylint: disable=invalid-name
     self[name] = value
@@ -469,7 +469,7 @@ def _maybe_make(meas):
     """Turn strings into Measurement objects if necessary."""
     if isinstance(meas, Measurement):
       return meas
-    elif isinstance(meas, basestring):
+    elif isinstance(meas, str):
       return Measurement(meas, **kwargs)
     raise InvalidMeasurementType('Expected Measurement or string', meas)
 

openhtf/core/phase_executor.py

@@ -91,12 +91,13 @@ class PhaseExecutionOutcome(collections.namedtuple(
   other value will raise an InvalidPhaseResultError.
   """
 
-  def __init__(self, phase_result):
+  def __new__(cls, phase_result):
     if (phase_result is not None and
         not isinstance(phase_result, (openhtf.PhaseResult, ExceptionInfo)) and
         not isinstance(phase_result, threads.ThreadTerminationError)):
       raise InvalidPhaseResultError('Invalid phase result', phase_result)
-    super(PhaseExecutionOutcome, self).__init__(phase_result)
+    self = super(PhaseExecutionOutcome, cls).__new__(cls, phase_result)
+    return self
 
   @property
   def is_fail_and_continue(self):
@@ -208,7 +209,7 @@ def execute_phase(self, phase):
       hit its limit for repetitions.
     """
     repeat_count = 1
-    repeat_limit = phase.options.repeat_limit or sys.maxint
+    repeat_limit = phase.options.repeat_limit or sys.maxsize
     while not self._stopping.is_set():
       is_last_repeat = repeat_count >= repeat_limit
       phase_execution_outcome = self._execute_phase_once(phase, is_last_repeat)
@@ -260,7 +261,7 @@ def stop(self, timeout_s=None):
 
     if phase_thread.is_alive():
       phase_thread.kill()
-    
+
       _LOG.debug('Waiting for cancelled phase to exit: %s', phase_thread)
       timeout = timeouts.PolledTimeout.from_seconds(timeout_s)
       while phase_thread.is_alive() and not timeout.has_expired():

openhtf/core/station_api.py

@@ -69,7 +69,7 @@
 import socket
 import threading
 import time
-import xmlrpclib
+import xmlrpc.client
 
 import mutablerecords
 
@@ -83,10 +83,11 @@
 from openhtf.util import threads
 from openhtf.util import timeouts
 from openhtf.util import xmlrpcutil
+from past.builtins import long
 
 # Fix for xmlrpclib to use <i8> for longs and ints instead of <int>, because our
 # timestamps are in millis, which are too big for 4-byte ints.
-xmlrpclib.Marshaller.dispatch[long] = xmlrpclib.Marshaller.dispatch[int] = (
+xmlrpc.client.Marshaller.dispatch[long] = xmlrpc.client.Marshaller.dispatch[int] = (
     lambda _, v, w: w('<value><i8>%d</i8></value>' % v))
 
 _LOG = logging.getLogger(__name__)
@@ -408,7 +409,7 @@ def wait_for_update(self, timeout_s=1):
     try:
       remote_state_dict = self.proxy_factory(timeout_s + 1).wait_for_update(
           self.test_uid, summary_dict, timeout_s)
-    except xmlrpclib.Fault as fault:
+    except xmlrpc.client.Fault as fault:
       # TODO(madsci): This is a super kludge, eventually implement the
       # ReraisingMixin for ServerProxy, but that's hard, so do this for now.
       if 'openhtf.io.station_api.UpdateTimeout' in fault.faultString:
@@ -770,7 +771,7 @@ def wait_for_plug_update(self, test_uid, plug_name, current_state, timeout_s):
   def _summary_for_state_dict(state_dict):
     """Return a dict for state with counts swapped in for phase/log records."""
     state_dict_summary = {
-        k: v for k, v in state_dict.iteritems() if k != 'plugs'}
+        k: v for k, v in state_dict.items() if k != 'plugs'}
     state_dict_summary['test_record'] = data.convert_to_base_types(
         state_dict_summary['test_record'])
     state_dict_summary['test_record']['phases'] = len(

openhtf/core/test_executor.py

@@ -109,7 +109,8 @@ def finalize(self):
   def wait(self):
     """Waits until death."""
     try:
-      self.join(sys.float_info.max)  # Timeout needed for SIGINT handling.
+      # Timeout needed for SIGINT handling. Timeout is not actually used.
+      self.join(60)
     except KeyboardInterrupt:
       self.test_state.logger.info('KeyboardInterrupt caught, aborting test.')
       raise

openhtf/core/test_record.py

@@ -125,7 +125,7 @@ def _get_source_safely(obj):
     return ''
 
 
-class CodeInfo(mutablerecords.Record(
+class CodeInfo(mutablerecords.HashableRecord(
     'CodeInfo', ['name', 'docstring', 'sourcecode'])):
   """Information regarding the running tester code."""