[ot] scripts/opentitan/ot: move OpenTitan Python modules to python/qemu/ot

Signed-off-by: Emmanuel Blot <eblot@rivosinc.com>

Author: rivos-eblot

docs/opentitan/pymod.md

@@ -5,7 +5,7 @@ to access the devices from the local host (or also from a remote host when the C
 using a TCP socket).
 
 * `python/qemu/jtag`: JTAG / TAP controller client, using the _Remote BitBang Protocol_
-* `scripts/opentitan/ot`: OpenTitan tools
+* `python/qemu/ot`: OpenTitan tools
   * `dtm`: Debug Transport Module support,
   * `dm`: RISC-V Debug Module support,
   * `lc_ctrl`: [Life Cycle controller](lc_ctrl_dmi.md) over JTAG/DMI support,

python/qemu/ot/.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 80

python/qemu/ot/.pylintrc

@@ -0,0 +1,15 @@
+[MESSAGES CONTROL]
+
+disable=
+    import-error,
+    too-few-public-methods,
+    too-many-arguments,
+    too-many-branches,
+    too-many-function-args,
+    too-many-instance-attributes,
+    too-many-lines,
+    too-many-locals,
+    too-many-nested-blocks,
+    too-many-public-methods,
+    too-many-statements,
+    unspecified-encoding

scripts/opentitan/ot/__init__.py renamed to python/qemu/ot/__init__.py

@@ -11,9 +11,8 @@
 from sys import modules
 from typing import Optional
 
-# pylint: disable=import-error
-from jtag.bits import BitSequence  # noqa: E402
-from jtag.jtag import JtagEngine  # noqa: E402
+from jtag.bits import BitSequence
+from jtag.jtag import JtagEngine
 
 
 class DMIError(RuntimeError):

scripts/opentitan/ot/bitfield.py renamed to python/qemu/ot/bitfield.py

@@ -8,10 +8,6 @@
 
 from logging import getLogger
 
-# see scripts/jtag
-# pylint: disable=import-error
-# pylint: disable=unused-import
-from jtag.jtag import JtagError  # noqa: F401
 from .sysmbox import SysMbox
 from ..dtm import DebugTransportModule
 

scripts/opentitan/ot/devproxy.py renamed to python/qemu/ot/devproxy.py

@@ -7,8 +7,8 @@
 """
 
 from logging import getLogger
-from re import finditer
 from typing import TextIO
+import re
 
 from ot.util.misc import camel_to_snake_case
 
@@ -28,19 +28,19 @@ def load(self, svp: TextIO):
            :param svp: System Verilog stream with OTP definitions.
         """
         svdata = svp.read()
-        for smo in finditer(r"\stypedef\s+enum\s+logic\s+\[[^]]+\]\s"
-                            r"{((?:\s+\w+,?)+)\s*}\s(\w+)_sel_e;", svdata):
+        for smo in re.finditer(r"\stypedef\s+enum\s+logic\s+\[[^]]+\]\s"
+                               r"{((?:\s+\w+,?)+)\s*}\s(\w+)_sel_e;", svdata):
             values, name = smo.groups()
             if name in self._consts:
                 raise ValueError(f'Multiple definitions of enumeration {name}')
             enums = self._enums[name] = {}
-            for emo in finditer(r"\s+(\w+),?", values):
+            for emo in re.finditer(r"\s+(\w+),?", values):
                 vname = camel_to_snake_case(emo.group(1))
                 enums[vname] = len(enums)
 
-        for amo in finditer(r"\s+parameter\s+(\w+)_array_t\s+(\w+)\s+=\s+"
-                            r"{(\s+(?:(?:64|128)'h[0-9A-F]+,?\s+)+)};",
-                            svdata):
+        for amo in re.finditer(r"\s+parameter\s+(\w+)_array_t\s+(\w+)\s+=\s+"
+                               r"{(\s+(?:(?:64|128)'h[0-9A-F]+,?\s+)+)};",
+                               svdata):
             _type, name, values = amo.groups()
             sc_name = camel_to_snake_case(name)
             sc_parts = sc_name.split('_')
@@ -52,7 +52,7 @@ def load(self, svp: TextIO):
             if name in self._consts:
                 raise ValueError(f'Multiple definitions of constant {name}')
             consts = self._consts[name] = []
-            for cmo in finditer(r"(64|128)'h([0-9A-F]+),?", values):
+            for cmo in re.finditer(r"(64|128)'h([0-9A-F]+),?", values):
                 consts.append(cmo.group(2).lower())
             # RTL order in array is reversed
             consts.reverse()

scripts/opentitan/ot/dm/__init__.py renamed to python/qemu/ot/dm/__init__.py

@@ -10,9 +10,9 @@
 from configparser import ConfigParser, NoOptionError
 from io import BytesIO
 from logging import getLogger
-from re import match as re_match, sub as re_sub
 from struct import calcsize as scalc, pack as spack, unpack as sunpack
 from typing import Any, BinaryIO, Optional, Sequence, TextIO, Union
+import re
 
 from .map import OtpMap
 from .partition import OtpPartition, OtpLifecycleExtension
@@ -142,18 +142,18 @@ def load_vmem(self, vfp: TextIO, vmem_kind: Optional[str] = None,
             raise ValueError(f"Unknown VMEM file kind '{vmem_kind}'")
         for lno, line in enumerate(vfp, start=1):
             if vkind is None:
-                kmo = re_match(self.RE_VMEMDESC, line)
+                kmo = re.match(self.RE_VMEMDESC, line)
                 if kmo:
                     vkind = kmo.group(1)
                     row_count = int(kmo.group(2))
                     bits = int(kmo.group(3))
                     byte_count = bits // 8
                     continue
-            line = re_sub(r'//.*', '', line)
+            line = re.sub(r'//.*', '', line)
             line = line.strip()
             if not line:
                 continue
-            lmo = re_match(self.RE_VMEMLOC, line)
+            lmo = re.match(self.RE_VMEMLOC, line)
             if not lmo:
                 self._log.error('Unexpected line @ %d: %s', lno, line)
                 continue

scripts/opentitan/ot/dm/dm.py renamed to python/qemu/ot/dm/dm.py

@@ -10,9 +10,9 @@
 from io import StringIO
 from logging import getLogger
 from os.path import basename
-from re import finditer, match, sub
 from textwrap import fill
 from typing import TextIO
+import re
 
 from ot.util.misc import camel_to_snake_case, group
 
@@ -53,7 +53,7 @@ def load(self, svp: TextIO):
             if cmt >= 0:
                 line = line[:cmt]
             line = line.strip()
-            abmo = match(ab_re, line)
+            abmo = re.match(ab_re, line)
             if not sequences and abmo:
                 name = abmo.group(1)
                 sval = abmo.group(2)
@@ -64,7 +64,7 @@ def load(self, svp: TextIO):
                     continue
                 codes[name] = val
                 continue
-            smo = match(tbl_re, line)
+            smo = re.match(tbl_re, line)
             if smo:
                 kind = smo.group(1).lower()
                 name = smo.group(2)
@@ -79,7 +79,7 @@ def load(self, svp: TextIO):
                 continue
         self._sequences = sequences
         svp.seek(0)
-        for tmo in finditer(r"\s+parameter\s+lc_token_t\s+(\w+)\s+="
+        for tmo in re.finditer(r"\s+parameter\s+lc_token_t\s+(\w+)\s+="
                             r"\s+\{\s+128'h([0-9A-F]+)\s+\};",
                             svp.getvalue()):
             token, value = tmo.group(1), tmo.group(2)
@@ -150,7 +150,7 @@ def _save_template(self, cfp: TextIO) -> None:
         for stname, stwords in states.items():
             print(f'    [LC_STATE_{stname.upper()}] = {{', file=cfp)
             for wgrp in group(stwords, len(stwords)//2):
-                items = (sub(r'(\d+)', r'(\1)', wg) for wg in wgrp)
+                items = (re.sub(r'(\d+)', r'(\1)', wg) for wg in wgrp)
                 stws = ' '.join(f'{w:<6s}' for w in (f'{i},' for i in items))
                 print(f'        {stws.rstrip()}', file=cfp)
             print('    },', file=cfp)

scripts/opentitan/ot/dm/otp.py renamed to python/qemu/ot/dm/otp.py

@@ -9,8 +9,8 @@
 from io import BytesIO
 from logging import getLogger
 from os.path import basename
-from re import compile as re_compile
 from typing import BinaryIO, Iterator, NamedTuple, Optional
+import re
 
 try:
     _ELF_ERROR = None
@@ -52,7 +52,7 @@ class ElfBlob:
     RUST_ERROR = _RUST_ERROR
     """Report whether Rust tools have been loaded."""
 
-    RUST_TRAIL_CRE = re_compile(r'::h[0-9a-f]{16}$')
+    RUST_TRAIL_CRE = re.compile(r'::h[0-9a-f]{16}$')
     """Regex to get rid of Rust trailing symbol string."""
 
     def __init__(self):

scripts/opentitan/ot/dm/regs.py renamed to python/qemu/ot/dm/regs.py

@@ -7,9 +7,9 @@
 """
 
 from io import BytesIO
-from re import sub
 from sys import stdout
 from typing import Any, Iterable, Optional, TextIO
+import re
 
 try:
     # only available from Python 3.12+
@@ -109,4 +109,4 @@ def round_up(value: int, rnd: int) -> int:
 def camel_to_snake_case(camel: str) -> str:
     """Convert CamelString string into snake_case lower string."""
     pattern = r'(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])'
-    return sub(pattern, '_', camel).lower()
+    return re.sub(pattern, '_', camel).lower()

scripts/opentitan/ot/dtm/__init__.py renamed to python/qemu/ot/dtm/__init__.py

@@ -1,2 +1,3 @@
 [flake8]
 max-line-length = 80
+extend-ignore = E402

scripts/opentitan/ot/dtm/dtm.py renamed to python/qemu/ot/dtm/dtm.py

@@ -1,6 +1,7 @@
 [MESSAGES CONTROL]
 
 disable=
+    import-error,
     too-few-public-methods,
     too-many-arguments,
     too-many-branches,
@@ -11,4 +12,6 @@ disable=
     too-many-nested-blocks,
     too-many-public-methods,
     too-many-statements,
-    unspecified-encoding
+    unspecified-encoding,
+    wrong-import-order,
+    wrong-import-position,

scripts/opentitan/ot/lc_ctrl/__init__.py renamed to python/qemu/ot/lc_ctrl/__init__.py

@@ -11,11 +11,15 @@
 from argparse import ArgumentParser
 from configparser import ConfigParser
 from logging import getLogger
-from os.path import isdir, isfile, join as joinpath, normpath
-from re import match, search
-from sys import exit as sysexit, modules, stderr, stdout
+from os.path import dirname, isdir, isfile, join as joinpath, normpath
 from traceback import format_exc
 from typing import Optional
+import re
+import sys
+
+QEMU_PYPATH = joinpath(dirname(dirname(dirname(normpath(__file__)))),
+                       'python', 'qemu')
+sys.path.append(QEMU_PYPATH)
 
 try:
     _HJSON_ERROR = None
@@ -26,10 +30,10 @@ def hjload(*_, **__):  # noqa: E301
         """dummy func if HJSON module is not available"""
         return {}
 
-from ot.util.log import configure_loggers
-from ot.util.misc import camel_to_snake_case
 from ot.otp.const import OtpConstants
 from ot.otp.lifecycle import OtpLifecycle
+from ot.util.log import configure_loggers
+from ot.util.misc import camel_to_snake_case
 
 
 OtParamRegex = str
@@ -130,7 +134,7 @@ def save(self, socid: Optional[str], count: Optional[int],
             with open(outpath, 'wt') as ofp:
                 cfg.write(ofp)
         else:
-            cfg.write(stdout)
+            cfg.write(sys.stdout)
 
     @classmethod
     def add_pair(cls, data: dict[str, str], kname: str, value: str) -> None:
@@ -147,7 +151,7 @@ def _load_top_values(self, module: dict, odict: dict, multi: bool,
             if not isinstance(params, dict):
                 continue
             for regex in regexes:
-                pmo = match(regex, params['name'])
+                pmo = re.match(regex, params['name'])
                 if not pmo:
                     continue
                 value = params.get('default')
@@ -157,7 +161,7 @@ def _load_top_values(self, module: dict, odict: dict, multi: bool,
                     value = value[2:]
                 kname = camel_to_snake_case(pmo.group(1))
                 if multi:
-                    imo = search(r'(\d+)$', modname)
+                    imo = re.search(r'(\d+)$', modname)
                     idx = int(imo.group(1)) if imo else 'None'
                     if idx not in odict:
                         odict[idx] = {}
@@ -221,7 +225,7 @@ def main():
     debug = True
     default_top = 'darjeeling'
     try:
-        desc = modules[__name__].__doc__.split('.', 1)[0].strip()
+        desc = sys.modules[__name__].__doc__.split('.', 1)[0].strip()
         argparser = ArgumentParser(description=f'{desc}.')
         files = argparser.add_argument_group(title='Files')
         files.add_argument('opentitan', nargs=1, metavar='TOPDIR',
@@ -289,12 +293,12 @@ def main():
         cfg.save(args.socid, args.count, args.out)
 
     except (IOError, ValueError, ImportError) as exc:
-        print(f'\nError: {exc}', file=stderr)
+        print(f'\nError: {exc}', file=sys.stderr)
         if debug:
-            print(format_exc(chain=False), file=stderr)
-        sysexit(1)
+            print(format_exc(chain=False), file=sys.stderr)
+        sys.exit(1)
     except KeyboardInterrupt:
-        sysexit(2)
+        sys.exit(2)
 
 
 if __name__ == '__main__':