Move data-driven aggregation logic tests to rust (#346)

* initial test works

* fix this

* fix json formatting

* move files

* get rid of c tests

* restore

* fix build ??

* gr

* format

Author: jayantk

CMakeLists.txt

@@ -108,15 +108,16 @@ target_link_libraries( test_net ${PC_DEP} )
 # add_executable( test_publish_websocket pctest/test_publish_websocket.cpp )
 # target_link_libraries( test_publish_websocket Qt5::Core Qt5::Network Qt5::WebSockets /usr/local/lib/libjcon.so)
 
-add_executable( test_qset pctest/test_qset.cpp )
-target_link_libraries( test_qset ${PC_DEP} )
+add_executable( test_pd pctest/test_pd.cpp )
+target_link_libraries( test_pd ${PC_DEP} )
 add_executable( test_twap pctest/test_twap.cpp )
 target_link_libraries( test_twap ${PC_DEP} )
 add_executable( leader_stats pctest/leader_stats.cpp )
 target_link_libraries( leader_stats ${PC_DEP} )
 
 add_test( test_unit test_unit )
 add_test( test_net test_net )
+add_test( test_pd test_pd )
 
 
 #

pctest/test_qset.cpp renamed to pctest/test_pd.cpp

@@ -99,91 +99,10 @@ void test_pd()
   PC_TEST_CHECK( pd_gt( n2, n1, dec_fact ) );
 }
 
-void test_qs()
+int main(int,char**)
 {
   PC_TEST_START
   test_pd();
   PC_TEST_END
-}
-
-int main( int argc,char** argv )
-{
-  if ( argc < 2 ) {
-    return usage();
-  }
-  // unit test piece first
-  test_qs();
-
-  // construct quotes
-  mem_map mf;
-  std::string file = argv[1];
-  mf.set_file(file );
-  if ( !mf.init() ) {
-    std::cerr << "test_qset: failed to read file[" << file << "]"
-              << std::endl;
-    return 1;
-  }
-  jtree pt;
-  pt.parse( mf.data(), mf.size() );
-  if ( !pt.is_valid() ) {
-    std::cerr << "test_qset: failed to parse file[" << file << "]"
-              << std::endl;
-    return 1;
-  }
-  pc_price_t px[1];
-  __builtin_memset( px, 0, sizeof( pc_price_t ) );
-  uint64_t slot = 1000UL;
-  px->last_slot_ = slot;
-  px->agg_.pub_slot_ = slot;
-  px->expo_      = pt.get_int( pt.find_val( 1, "exponent" ) );
-  px->num_       = 0;
-  uint32_t qt =pt.find_val( 1, "quotes" );
-  for( uint32_t it = pt.get_first( qt ); it; it = pt.get_next( it ) ) {
-    pc_price_comp_t *ptr = &px->comp_[px->num_++];
-    ptr->latest_.status_ = pt.get_int( pt.find_val( it, "status" ) );
-    ptr->latest_.price_ = pt.get_int( pt.find_val( it, "price" ) );
-    int64_t conf = pt.get_int( pt.find_val( it,  "conf" ) );
-    assert( conf >= 0 );
-    ptr->latest_.conf_  = static_cast< uint64_t >( conf );
-    int64_t pub_slot = pt.get_int( pt.find_val( it,  "slot_diff" ) );
-    ptr->latest_.pub_slot_ =
-      static_cast< uint64_t >( static_cast< int64_t >( slot ) + pub_slot );
-  }
-  upd_aggregate( px, slot+1, 1234 );
-
-  char const* status = "invalid value";
-  switch ( px->agg_.status_ ) {
-  case PC_STATUS_UNKNOWN:
-    status = "unknown";
-    break;
-  case PC_STATUS_TRADING:
-    status = "trading";
-    break;
-  case PC_STATUS_HALTED:
-    status = "halted";
-    break;
-  case PC_STATUS_AUCTION:
-    status = "auction";
-    break;
-  }
-
-  // generate result
-  json_wtr jw;
-  jw.add_val( json_wtr::e_obj );
-  jw.add_key( "exponent", (int64_t)px->expo_ );
-  jw.add_key( "price", px->agg_.price_ );
-  jw.add_key( "conf", px->agg_.conf_ );
-  jw.add_key( "status", status );
-  jw.pop();
-  net_buf *hd, *tl;
-  jw.detach( hd, tl );
-  while( hd ) {
-    net_buf *nxt = hd->next_;
-    std::cout.write( hd->buf_, hd->size_ );
-    hd->dealloc();
-    hd = nxt;
-  }
-  std::cout << std::endl;
-
   return 0;
 }

program/rust/Cargo.toml

@@ -23,6 +23,9 @@ hex = "0.3.1"
 quickcheck = "1"
 quickcheck_macros = "1"
 bincode = "1.3.3"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+test-generator = "0.3.1"
 
 [features]
 debug = []

program/rust/src/tests/mod.rs

@@ -3,6 +3,7 @@ mod test_add_mapping;
 mod test_add_price;
 mod test_add_product;
 mod test_add_publisher;
+mod test_aggregation;
 mod test_del_price;
 mod test_del_product;
 mod test_del_publisher;

program/rust/src/tests/test_aggregation.rs

@@ -0,0 +1,98 @@
+use {
+    crate::{
+        accounts::PriceAccount,
+        processor::c_upd_aggregate,
+    },
+    bytemuck::Zeroable,
+    serde::{
+        Deserialize,
+        Serialize,
+    },
+    std::fs::File,
+};
+extern crate test_generator;
+
+use test_generator::test_resources;
+
+#[test_resources("program/rust/test_data/aggregation/*.json")]
+fn test_quote_set(input_path_raw: &str) {
+    // For some reason these tests have a different working directory than the macro.
+    let input_path = input_path_raw.replace("program/rust/", "");
+    let result_path = input_path.replace(".json", ".result");
+
+    let file = File::open(input_path).expect("Test file not found");
+    let quote_set: QuoteSet = serde_json::from_reader(&file).expect("Unable to parse JSON");
+
+    let result_file = File::open(result_path).expect("Test file not found");
+    let expected_result: QuoteSetResult =
+        serde_json::from_reader(&result_file).expect("Unable to parse JSON");
+
+    let current_slot = 1000; // arbitrary
+    let current_timestamp = 1234; // also arbitrary
+    let mut price_account: PriceAccount = PriceAccount::zeroed();
+
+    price_account.last_slot_ = current_slot;
+    price_account.agg_.pub_slot_ = current_slot;
+    price_account.exponent = quote_set.exponent;
+    price_account.num_ = quote_set.quotes.len() as u32;
+    for quote_idx in 0..quote_set.quotes.len() {
+        let mut current_component = &mut price_account.comp_[quote_idx];
+        let quote = &quote_set.quotes[quote_idx];
+        current_component.latest_.status_ = quote.status;
+        current_component.latest_.price_ = quote.price;
+        current_component.latest_.conf_ = quote.conf;
+        let slot_diff = quote.slot_diff.unwrap_or(0);
+        assert!(slot_diff > -(current_slot as i64));
+        current_component.latest_.pub_slot_ = ((current_slot as i64) + slot_diff) as u64;
+    }
+
+    unsafe {
+        c_upd_aggregate(
+            (&mut price_account as *mut PriceAccount) as *mut u8,
+            current_slot + 1,
+            current_timestamp,
+        );
+    }
+
+    // For some idiotic reason the status in the input is a number and the output is a string.
+    let result_status: String = match price_account.agg_.status_ {
+        0 => "unknown",
+        1 => "trading",
+        2 => "halted",
+        3 => "auction",
+        4 => "ignored",
+        _ => "invalid status",
+    }
+    .into();
+
+    let actual_result = QuoteSetResult {
+        exponent: price_account.exponent,
+        price:    price_account.agg_.price_,
+        conf:     price_account.agg_.conf_,
+        status:   result_status,
+    };
+
+    assert_eq!(expected_result, actual_result);
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct Quote {
+    price:     i64,
+    conf:      u64,
+    status:    u32,
+    slot_diff: Option<i64>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct QuoteSet {
+    exponent: i32,
+    quotes:   Vec<Quote>,
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+struct QuoteSetResult {
+    exponent: i32,
+    price:    i64,
+    conf:     u64,
+    status:   String,
+}

pyth/tests/qset/1.json renamed to program/rust/test_data/aggregation/1.json

@@ -4,17 +4,17 @@
   {
    "price": 10000,
    "conf": 1000,
-   "status": 1,
+   "status": 1
   },
   {
    "price": 10000,
    "conf": 1000,
-   "status": 1,
+   "status": 1
   },
   {
    "price": 10000,
    "conf": 1000,
-   "status": 1,
+   "status": 1
   }
  ]
 }

pyth/tests/qset/1.result renamed to program/rust/test_data/aggregation/1.result

@@ -1,5 +1,5 @@
 {
-  "comment": "1 good quote, 1 off by factor of 10 - this is the low quote number, tightest outlier limit."
+  "comment": "1 good quote, 1 off by factor of 10 - this is the low quote number, tightest outlier limit.",
   "exponent": -8,
   "quotes": [
     {