From 67c4ecf5369c8e76ad3fafdf3e6c7a2849042015 Mon Sep 17 00:00:00 2001
From: Ved Shanbhogue <ved@rivosinc.com>
Date: Wed, 25 Dec 2024 15:50:52 -0600
Subject: [PATCH] MSI stay pending when masked and are released when unmasked

---
 .../libiommu/include/iommu_interrupt.h        |  1 +
 .../libiommu/src/iommu_interrupt.c            | 43 +++++++++++++++----
 iommu_ref_model/libiommu/src/iommu_reg.c      |  4 ++
 3 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/iommu_ref_model/libiommu/include/iommu_interrupt.h b/iommu_ref_model/libiommu/include/iommu_interrupt.h
index 43ef96ff..388dfb3f 100644
--- a/iommu_ref_model/libiommu/include/iommu_interrupt.h
+++ b/iommu_ref_model/libiommu/include/iommu_interrupt.h
@@ -13,4 +13,5 @@
 #define MSI_VEC_CTRL_MASK_BIT 1
 
 extern void generate_interrupt(uint8_t unit);
+extern void release_pending_interrupt(uint8_t vec);
 #endif // __IOMMU_INTERRUPT_H__
diff --git a/iommu_ref_model/libiommu/src/iommu_interrupt.c b/iommu_ref_model/libiommu/src/iommu_interrupt.c
index 2436de12..73115e50 100644
--- a/iommu_ref_model/libiommu/src/iommu_interrupt.c
+++ b/iommu_ref_model/libiommu/src/iommu_interrupt.c
@@ -4,6 +4,21 @@
 // Author: ved@rivosinc.com
 
 #include "iommu.h"
+// MSI pending bit array.
+uint8_t msi_pending[16] = {0};
+static void
+do_msi(
+    uint32_t msi_data, uint64_t msi_addr) {
+    uint8_t  status;
+    status = write_memory((char *)&msi_data, msi_addr, 4);
+    if ( status & ACCESS_FAULT ) {
+        // If an access fault is detected on a MSI write using msi_addr_x,
+        // then the IOMMU reports a "IOMMU MSI write access fault" (cause 273) fault,
+        // with TTYP set to 0 and iotval set to the value of msi_addr_x.
+        report_fault(273, msi_addr, 0, TTYPE_NONE, 0, 0, 0, 0, 0);
+    }
+    return;
+}
 void
 generate_interrupt(
     uint8_t unit) {
@@ -11,7 +26,7 @@ generate_interrupt(
     msi_addr_t msi_addr;
     uint32_t msi_data;
     msi_vec_ctrl_t msi_vec_ctrl;
-    uint8_t  vec, status;
+    uint8_t  vec;
 
     // Interrupt pending status register (ipsr)
     // This 32-bits register (RW1C) reports the pending interrupts
@@ -74,15 +89,27 @@ generate_interrupt(
         // When the mask bit M is 1, the corresponding interrupt vector is
         // masked and the IOMMU is prohibited from sending the associated
         // message.
-        if ( msi_vec_ctrl.m == 1 )
+        if ( msi_vec_ctrl.m == 1 ) {
+            // Pending messages for that vector are later generated if the
+            // corresponding mask bit is cleared to 0.
+            msi_pending[vec] = 1;
             return;
-        status = write_memory((char *)&msi_data, msi_addr.raw, 4);
-        if ( status & ACCESS_FAULT ) {
-            // If an access fault is detected on a MSI write using msi_addr_x,
-            // then the IOMMU reports a "IOMMU MSI write access fault" (cause 273) fault,
-            // with TTYP set to 0 and iotval set to the value of msi_addr_x.
-            report_fault(273, msi_addr.raw, 0, TTYPE_NONE, 0, 0, 0, 0, 0);
         }
+        do_msi(msi_data, msi_addr.raw);
+    }
+    return;
+}
+void
+release_pending_interrupt(
+    uint8_t vec) {
+    msi_addr_t msi_addr;
+    uint32_t msi_data;
+
+    if ( msi_pending[vec] == 1 ) {
+        msi_addr.raw = g_reg_file.msi_cfg_tbl[vec].msi_addr.raw;
+        msi_data = g_reg_file.msi_cfg_tbl[vec].msi_data;
+        do_msi(msi_data, msi_addr.raw);
+        msi_pending[vec] = 0;
     }
     return;
 }
diff --git a/iommu_ref_model/libiommu/src/iommu_reg.c b/iommu_ref_model/libiommu/src/iommu_reg.c
index adf4448b..43e2f5f5 100644
--- a/iommu_ref_model/libiommu/src/iommu_reg.c
+++ b/iommu_ref_model/libiommu/src/iommu_reg.c
@@ -842,6 +842,10 @@ write_register(
             x = (offset - MSI_ADDR_0_OFFSET) / 16;
             if ( x >= (1UL << g_num_vec_bits) )
                 break;
+            if ( msi_vec_ctrl_temp.m == 1 &&
+                 g_reg_file.msi_cfg_tbl[x].msi_vec_ctrl.m == 0 ) {
+                release_pending_interrupt(x);
+            }
             g_reg_file.msi_cfg_tbl[x].msi_vec_ctrl.m = msi_vec_ctrl_temp.m;
             break;
     }