From b8e506a761465bb4b4ebc9c59165502e1200fc56 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 28 Jan 2025 08:58:02 -0600 Subject: [PATCH 1/6] Set columns explicitly --- jarvis_util/introspect/system_info.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jarvis_util/introspect/system_info.py b/jarvis_util/introspect/system_info.py index 0de5876..20500b1 100644 --- a/jarvis_util/introspect/system_info.py +++ b/jarvis_util/introspect/system_info.py @@ -205,7 +205,10 @@ class PyLsblk(Exec): rota: whether or not the device is rotational host: the host this record corresponds to """ - + columns = [ + 'parent', 'device', 'size', 'mount', 'model', 'tran', + 'rota', 'dev_type', 'host' + ] def __init__(self, exec_info): cmd = 'pylsblk' super().__init__(cmd, exec_info.mod(collect_output=True)) @@ -224,7 +227,7 @@ def wait(self): dev['tran'] = 'nvme' dev['host'] = host total.append(dev) - self.df = sdf.SmallDf(rows=total) + self.df = sdf.SmallDf(rows=total, columns=self.columns) class Blkid(Exec): From 8457aa0bacbdcc0e78dbfa082ca83590ee462ce5 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 28 Jan 2025 09:16:16 -0600 Subject: [PATCH 2/6] Use fs_mount over mount --- jarvis_util/introspect/system_info.py | 21 +++++++++++---------- jarvis_util/util/small_df.py | 3 +++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/jarvis_util/introspect/system_info.py b/jarvis_util/introspect/system_info.py index 20500b1..6ff86c8 100644 --- a/jarvis_util/introspect/system_info.py +++ b/jarvis_util/introspect/system_info.py @@ -220,13 +220,13 @@ def __init__(self, exec_info): def wait(self): super().wait() total = [] - for host, stdout in self.stdout.items(): - lsblk_data = yaml.load(stdout, Loader=yaml.FullLoader) - for dev in lsblk_data: - if dev['tran'] == 'pcie': - dev['tran'] = 'nvme' - dev['host'] = host - total.append(dev) + # for host, stdout in self.stdout.items(): + # lsblk_data = yaml.load(stdout, Loader=yaml.FullLoader) + # for dev in lsblk_data: + # if dev['tran'] == 'pcie': + # dev['tran'] = 'nvme' + # dev['host'] = host + # total.append(dev) self.df = sdf.SmallDf(rows=total, columns=self.columns) @@ -505,16 +505,17 @@ def modify(self, exec_info): """ def introspect_fs(self, exec_info, sudo=False): - lsblk = PyLsblk(exec_info.mod(hide_output=True)) + lsblk = PyLsblk(exec_info.mod(hide_output=True)) blkid = Blkid(exec_info.mod(hide_output=True)) list_fs = ListFses(exec_info.mod(hide_output=True)) fs = sdf.merge([lsblk.df, blkid.df], on=['device', 'host'], - how='outer') + how='outer') fs[:, 'shared'] = False fs = sdf.merge([fs, list_fs.df], on=['device', 'host'], how='outer') + self.fs['mount'] = self.fs['fs_mount'] fs = self._find_common_mounts(fs, exec_info) fs = self._label_user_mounts(fs) fs = fs.drop_columns([ @@ -526,7 +527,7 @@ def introspect_fs(self, exec_info, sudo=False): def _find_common_mounts(self, fs, exec_info): """ Finds mount point points common across all hosts - """ + """ io_groups = fs.groupby(['mount', 'device']) common = [] for name, group in io_groups.groups.items(): diff --git a/jarvis_util/util/small_df.py b/jarvis_util/util/small_df.py index ffc6185..53a95cc 100644 --- a/jarvis_util/util/small_df.py +++ b/jarvis_util/util/small_df.py @@ -165,6 +165,9 @@ def merge(self, other, on=None): for row in rows: if '$#matched' in row: del row['$#matched'] + for col in self.columns: + if col not in row: + row[col] = None return SmallDf(rows=rows) def _find_unmatched(self, orig_rows): From 14a1f992e477fe67ae4ba30e9217af833e2bd09e Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 28 Jan 2025 09:16:58 -0600 Subject: [PATCH 3/6] Use fs_mount over mount --- jarvis_util/introspect/system_info.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jarvis_util/introspect/system_info.py b/jarvis_util/introspect/system_info.py index 6ff86c8..23c9aa3 100644 --- a/jarvis_util/introspect/system_info.py +++ b/jarvis_util/introspect/system_info.py @@ -220,13 +220,13 @@ def __init__(self, exec_info): def wait(self): super().wait() total = [] - # for host, stdout in self.stdout.items(): - # lsblk_data = yaml.load(stdout, Loader=yaml.FullLoader) - # for dev in lsblk_data: - # if dev['tran'] == 'pcie': - # dev['tran'] = 'nvme' - # dev['host'] = host - # total.append(dev) + for host, stdout in self.stdout.items(): + lsblk_data = yaml.load(stdout, Loader=yaml.FullLoader) + for dev in lsblk_data: + if dev['tran'] == 'pcie': + dev['tran'] = 'nvme' + dev['host'] = host + total.append(dev) self.df = sdf.SmallDf(rows=total, columns=self.columns) From df44f21131fee98b6d4f98b219970619a994ef45 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 28 Jan 2025 09:23:58 -0600 Subject: [PATCH 4/6] Add dev type --- jarvis_util/introspect/system_info.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/jarvis_util/introspect/system_info.py b/jarvis_util/introspect/system_info.py index 23c9aa3..2f75f66 100644 --- a/jarvis_util/introspect/system_info.py +++ b/jarvis_util/introspect/system_info.py @@ -225,10 +225,22 @@ def wait(self): for dev in lsblk_data: if dev['tran'] == 'pcie': dev['tran'] = 'nvme' + dev['dev_type'] = self.GetDevType(dev) dev['host'] = host total.append(dev) self.df = sdf.SmallDf(rows=total, columns=self.columns) + def GetDevType(self, dev): + if dev['tran'] == 'sata': + if dev['rota']: + return str(StorageDeviceType.HDD) + else: + return str(StorageDeviceType.SSD) + elif dev['tran'] == 'nvme': + return str(StorageDeviceType.NVME) + elif dev['tran'] == 'dimm': + return str(StorageDeviceType.PMEM) + class Blkid(Exec): """ From d611e98b82e5d35ebe4c9010b992c94f25fd564f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 28 Jan 2025 12:24:10 -0600 Subject: [PATCH 5/6] Remove self.fs from mount setting and print --- jarvis_util/introspect/system_info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jarvis_util/introspect/system_info.py b/jarvis_util/introspect/system_info.py index 2f75f66..2acaef9 100644 --- a/jarvis_util/introspect/system_info.py +++ b/jarvis_util/introspect/system_info.py @@ -527,13 +527,14 @@ def introspect_fs(self, exec_info, sudo=False): fs = sdf.merge([fs, list_fs.df], on=['device', 'host'], how='outer') - self.fs['mount'] = self.fs['fs_mount'] + fs['mount'] = fs['fs_mount'] fs = self._find_common_mounts(fs, exec_info) fs = self._label_user_mounts(fs) fs = fs.drop_columns([ 'used', 'use%', 'fs_mount', 'partuuid', 'fs_size', 'partlabel', 'label', 'host']) self.fs = fs + print(self.fs) return self.fs def _find_common_mounts(self, fs, exec_info): From bbab838e0420e2c73079123b0d424e377768f903 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 28 Jan 2025 13:18:29 -0600 Subject: [PATCH 6/6] Remove print statement --- jarvis_util/introspect/system_info.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jarvis_util/introspect/system_info.py b/jarvis_util/introspect/system_info.py index 2acaef9..25a2093 100644 --- a/jarvis_util/introspect/system_info.py +++ b/jarvis_util/introspect/system_info.py @@ -528,6 +528,7 @@ def introspect_fs(self, exec_info, sudo=False): on=['device', 'host'], how='outer') fs['mount'] = fs['fs_mount'] + print(fs) fs = self._find_common_mounts(fs, exec_info) fs = self._label_user_mounts(fs) fs = fs.drop_columns([