improve test structure (#533)

* Refactors test list declarations to be tuples of an explicit test ID, a comment string, and the args.
* Moves most test list declarations to the single place they are used.
* Simplifies running tests.
* Deletes some unused regression test helper binaries.

Author: macklin-10x

enclone_exec/tests/enclone_test2.rs

@@ -170,18 +170,23 @@ fn test_site_examples() {
 #[test]
 fn test_enclone_examples() {
     PrettyTrace::new().on();
-    for t in 0..EXAMPLES.len() {
-        let testn = format!("{}", EXAMPLES[t]);
-        let out_file = format!("../enclone_help/src/example{}", t + 1);
+    let examples = [
+        (1, r###"BCR=123089 CDR3=CARRYFGVVADAFDIW"###),
+        (
+            2,
+            r###"BCR=123085 GEX=123217 LVARSP=gex,IGHV2-5_g_μ CDR3=CALMGTYCSGDNCYSWFDPW"###,
+        ),
+    ];
+    for (num, args) in examples {
+        let out_file = format!("../enclone_help/src/example{}", num);
         let old = read_to_string(&out_file).unwrap();
-        let args = testn.split(' ').collect::<Vec<&str>>();
         let mut new = Command::new(env!("CARGO_BIN_EXE_enclone"));
         let mut new = new.arg(format!(
             "PRE=../enclone-data/big_inputs/version{}",
             TEST_FILES_VERSION
         ));
-        for i in 0..args.len() {
-            new = new.arg(&args[i]);
+        for arg in args.split(' ') {
+            new = new.arg(arg);
         }
         let new = new
             .arg("FORCE_EXTERNAL")
@@ -192,7 +197,7 @@ fn test_enclone_examples() {
         if new.status.code() != Some(0) {
             eprint!(
                 "\nenclone_test_examples: example{} failed to execute, stderr =\n{}",
-                t + 1,
+                num,
                 strme(&new.stderr),
             );
             eprintln!("If it's not clear what is happening, make sure you've run ./build.\n");
@@ -201,7 +206,7 @@ fn test_enclone_examples() {
         if old != new2 {
             eprintln!(
                 "\nenclone_test_examples: the file enclone_help/src/example{} is not up to date\n",
-                t + 1
+                num
             );
             eprintln!("old output =\n{}", old);
             eprintln!("new output =\n{}\n", new2);
@@ -371,19 +376,17 @@ fn test_enclone_prebuild() {
         remove_file(&mb).unwrap();
     }
 
-    let test_id = 48;
-    let it = test_id - 1;
-    let testn = TESTS[it];
-    let out_file = format!("testx/inputs/outputs/enclone_test{}_output", test_id);
+    let (test_num, comment, args) = TESTS[47];
+    assert_eq!(48, test_num);
+    let out_file = format!("testx/inputs/outputs/enclone_test{}_output", test_num);
     let old = read_to_string(&out_file).unwrap();
-    let args = testn.split(' ').collect::<Vec<&str>>();
     let mut new = Command::new(env!("CARGO_BIN_EXE_enclone"));
     let mut new = new.arg(format!(
         "PRE=../enclone-data/big_inputs/version{}",
         TEST_FILES_VERSION
     ));
-    for i in 0..args.len() {
-        new = new.arg(&args[i]);
+    for arg in args.split(' ') {
+        new = new.arg(arg);
     }
     // dubious use of expect:
     let new = new
@@ -398,7 +401,7 @@ fn test_enclone_prebuild() {
             "\nenclone_test_prebuild: first pass output has changed.\n\
             If you are OK with the new output, it should work to type:\n\
             enclone {} > enclone_exec/{}\n",
-            testn, out_file
+            args, out_file
         );
         eprintln!("old output =\n{}\n", old);
         eprintln!("new output =\n{}\n", new2);
@@ -407,16 +410,13 @@ fn test_enclone_prebuild() {
     }
 
     // Second pass: run without PREBUILD
-
-    let testn = TESTS[it];
-    let args = testn.split(' ').collect::<Vec<&str>>();
     let mut new = Command::new(env!("CARGO_BIN_EXE_enclone"));
     let mut new = new.arg(format!(
         "PRE=../enclone-data/big_inputs/version{}",
         TEST_FILES_VERSION
     ));
-    for i in 0..args.len() {
-        new = new.arg(&args[i]);
+    for arg in args.split(' ') {
+        new = new.arg(arg);
     }
     // dubious use of expect:
     let new = new
@@ -562,42 +562,29 @@ fn test_proto_write() -> Result<(), Error> {
 
 #[cfg(not(feature = "cpu"))]
 #[test]
-fn test_annotated_example() {
+fn test_annotated_example() -> Result<(), String> {
     PrettyTrace::new().on();
-    let it = 0;
-    let test = "BCR=123085 CDR3=CTRDRDLRGATDAFDIW";
-    let mut out = String::new();
-    let mut log = String::new();
-    let mut ok = false;
     run_test(
         env!("CARGO_BIN_EXE_enclone"),
-        it,
-        "",
-        &test,
         "annotated_example_test",
-        &mut ok,
-        &mut log,
-        &mut out,
         0,
-    );
-    print!("{}", log);
-    if !ok {
-        let mut log = Vec::<u8>::new();
-        emit_red_escape(&mut log);
-        fwriteln!(
-            log,
-            "Oh no: the results for the annotated example on the landing page have \
-            changed.  Assuming that\nthe change is intentional, to fix this you \
-            need to do two things:\n\
-            1. Update the test output.\n\
-            2. Manually update the annotated example output.\n\
-            Because the second item is such a big pain, we stopped doing it, but you should\n\
-            check to make sure that it has not changed too much from what is shown."
-        );
-        emit_end_escape(&mut log);
-        eprintln!("{}", strme(&log));
-        panic!("failed");
-    }
+        1,
+        "",
+        "BCR=123085 CDR3=CTRDRDLRGATDAFDIW",
+    )
+    .map_err(|res| {
+        format!(
+            "{}\n\nOh no: the results for the annotated example on the landing page have \
+        changed. Assuming that the change is intentional, to fix this you \
+        need to do two things:\n\
+        1. Update the test output.\n\
+        2. Manually update the annotated example output.\n\
+        Because the second item is such a big pain, we stopped doing it, but you should \
+        check to make sure that it has not changed too much from what is shown.",
+            res.log
+        )
+    })?;
+    Ok(())
 }
 
 // ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓

enclone_exec/tests/enclone_test3.rs

@@ -3,220 +3,8 @@
 // Information about enclone tests.
 pub mod main_testlist;
 
-pub fn enclone_testdata() -> String {
-    include_str!["enclone.testdata"].to_string()
-}
-pub fn enclone_testdata_public_bcr_human() -> String {
-    include_str!["testdata.public.bcr.human"].to_string()
-}
-pub fn enclone_testdata_public_tcr_human() -> String {
-    include_str!["testdata.public.tcr.human"].to_string()
-}
-pub fn enclone_testdata_public_tcr_mouse() -> String {
-    include_str!["testdata.public.tcr.mouse"].to_string()
-}
-pub fn enclone_testdata_public_gex_human() -> String {
-    include_str!["testdata.public.gex.human"].to_string()
-}
-
 pub const TEST_FILES_VERSION: u8 = 15;
 
-// Unaccounted time test.  This is separated out so that we can avoid running it in parallel with
-// other tests, to reduce the sporadic failure rate.  Also this is off in GitHub Actions.
-
-pub const UNAC_TESTS: [&str; 1] = [
-    // 1. enforce no unaccounted time
-    r###"BCR=123085 COMPE UNACCOUNTED NOPRINT EXPECT_OK"###,
-];
-
-// Tests that are affected by the D region alignment algorithm.  All such tests should go here.
-
-pub const DTESTS: [&str; 15] = [
-    // 1. test ALIGN_2ND<n>
-    r###"BCR=123085 CDR3=CKVMLYDSRGSDYYYVMDVW ALIGN_2ND1 CVARS=d1_name"###,
-    // 2. test JALIGN_2ND<n>
-    r###"BCR=123085 CDR3=CKVMLYDSRGSDYYYVMDVW JALIGN_2ND1 CVARS=d2_name"###,
-    // 3. test ALIGN_JALIGN_CONSISTENCY
-    r###"BCR=123085 CELLS=1 CHAINS=2 ALIGN1 JALIGN1 ALIGN_JALIGN_CONSISTENCY AMINO=cdr3
-         PLAIN NOPAGER EXPECT_OK"###,
-    // 4. test D_INCONSISTENT, and lock number of inconsistencies
-    r###"BCR=123085 D_INCONSISTENT CVARS=d1_name COMPLETE NGROUP"###,
-    // 5. the JALIGN1 in this example had a boundary location that was off by one
-    r###"BCR=165807 JALIGN1 AMINO=cdr3 CVARS=d1_score,d2_score CDR3=CAKEYYDFWSGYSDVRGVIPNIDYW"###,
-    // 6. the JALIGN1 in this example had a boundary location that was off by one
-    r###"BCR=123085 CELLS=2 JALIGN1 AMINO=cdr3 CVARS=d1_name CDR3=CAKAGPTESGYYVWYFDLW"###,
-    // 7. test d_inconsistent_{%,n}
-    r###"BCR=123085 GVARS=d_inconsistent_%,d_inconsistent_n NOPRINT SUMMARY SUMMARY_CLEAN"###,
-    // 8. test ALIGN<n>
-    r###"BCR=123085 CDR3=CKVMLYDSRGSDYYYVMDVW ALIGN1 CVARS=d1_name"###,
-    // 9. test ALIGN<n> and JALIGN<n>, case where there's a D segment
-    r###"BCR=85333 ALIGN1 JALIGN1 CDR3=CARGYDFWSGYLVGNWAGDYYYYMDVW"###,
-    // 10. test ALIGN<n> and JALIGN<n>, case where there is no D segment
-    r###"BCR=85333 ALIGN1 JALIGN1 CDR3=CAKGKGFRNYYYYMDVW"###,
-    // 11. test d1 etc.
-    r###"BCR=123085 CVARS=d1_name,d2_name,d_Δ,d_delta AMINO=cdr3 CDR3=CARVRDILTGDYGMDVW"###,
-    // 12. test GROUP_VDJ_REFNAME_HEAVY (deprecated but supported)
-    r###"BCR=86237 GROUP_VDJ_REFNAME_HEAVY CDR3="CAKAVAGKAVAGGWDYW|CAKVSTGIAVAGPGDYW" COMPLETE"###,
-    // 13. test GROUP_VJ_REFNAME_HEAVY (deprecated but supported)
-    r###"BCR=86237 GROUP_VJ_REFNAME_HEAVY CDR3="CARGVLWFGELGAFDIW|CARAGLGVVLAARGAFDIW""###,
-    // 14. test placement of indel, needed shifting right
-    r###"BCR=123085 CELLS=1 CHAINS=2 AMINO=cdr3 JALIGN2 CDR3=CAKDKSRPPTHYYGSGSYYSRILDNW"###,
-    // 15. test placement of indel, needed shifting left
-    r###"BCR=123085 CELLS=1 CHAINS=2 AMINO=cdr3 JALIGN2 CDR3=CARMAQFYSGSGTYYIGPYYFEYW"###,
-];
-
-// Tests that are affected by the grouping algorithm.  All such tests should go here, if not
-// already in DTESTS.
-
-pub const GTESTS: [&str; 21] = [
-    // 1. test 5/8 for newline correctness (this grouping option deprecated but supported)
-    r###"BCR=85333 GROUP_VJ_REFNAME MIN_GROUP=2 AMINO= PLAIN SET_IN_STONE"###,
-    // 2. test 6/8 for newline correctness (this grouping option deprecated but supported)
-    r###"BCR=85333 GROUP_VJ_REFNAME MIN_GROUP=2 AMINO= PLAIN NGROUP SET_IN_STONE"###,
-    // 3. test 7/8 for newline correctness (this grouping option deprecated but supported)
-    r###"BCR=85333 GROUP_VJ_REFNAME MIN_GROUP=2 AMINO= PLAIN HTML SET_IN_STONE"###,
-    // 4. test 8/8 for newline correctness (this grouping option deprecated but supported)
-    r###"BCR=85333 GROUP_VJ_REFNAME MIN_GROUP=2 AMINO= PLAIN HTML NGROUP SET_IN_STONE"###,
-    // 5. test of GROUP
-    r###"BCR=123085 GROUP=vj_refname,cdr3_aa_heavy≥80%,cdr3_aa_light≥80% CVARS=cdr3_len
-         AMINO=cdr3 CDR3="CARHLQWELP.*W""###,
-    // 6. test of GROUP
-    r###"BCR=123085 GROUP=vj_refname,len,cdr3_len MIN_GROUP=2 MIN_CHAINS=2 CDR3="CQQSY.*TLATF"
-         CVARS=cdr3_len"###,
-    // 7. test of GROUP
-    r###"BCR=123085 GROUP=cdr3_aa_heavy≥100% MIN_GROUP=2 MIN_CHAINS=2 CVARS=cdr3_len
-         CDR3=CARPKSDYIIDAFDIW"###,
-    // 8. test of GROUP
-    r###"BCR=123085 GROUP=cdr3_aa_light≥100% MIN_GROUP=2 MIN_CHAINS=2 CVARS=cdr3_len
-         CDR3=CQTWGTGPWVF"###,
-    // 9. test of GROUP
-    r###"BCR=123085 GROUP=vj_refname,aa_heavy≥100% MIN_GROUP=2 MIN_CHAINS=2 CVARS=cdr3_len
-         CDR3=CARVPYYYDRSYYYYGMDVW"###,
-    // 10. test of AGROUP
-    r###"BCR=123085 AGROUP AG_CENTER=from_filters CDR3=CARHSYSSGWYDEWDYW
-         AG_DIST_FORMULA=cdr3_edit_distance AG_DIST_BOUND=top=2"###,
-    // 11. test of AGROUP
-    r###"BCR=123085 AGROUP AG_CENTER=from_filters CDR3=CAKDGGEHYYDSSGYYASYYFDYW 
-         AG_DIST_FORMULA=cdr3_edit_distance AG_DIST_BOUND=max=14"###,
-    // 12. test of AGROUP
-    r###"BCR=123085 AGROUP AG_CENTER=from_filters CDR3=CAKDGGEHYYDSSGYYASYYFDYW 
-         AG_DIST_FORMULA=cdr3_edit_distance AG_DIST_BOUND=max=13"###,
-    // 13. test of AGROUP
-    r###"BCR=123085 AGROUP AG_CENTER=copy_filters MIN_CELLS=2 MAX_CELLS=2
-         AG_DIST_FORMULA=cdr3_edit_distance AG_DIST_BOUND=max=3 MIN_GROUP=2"###,
-    // 14. test symmetric grouping stats
-    r###"BCR=123085 GROUP=vj_refname,cdr3_aa_heavy≥80%,cdr3_aa_light≥80% NOPRINT
-         SUMMARY SUMMARY_CLEAN"###,
-    // 15. test of GROUP
-    r###"BCR=123085 GROUP=cdr3_heavy≥100% MIN_GROUP=2 MIN_CHAINS=2 CVARS=cdr3_len
-         CDR3=CARPKSDYIIDAFDIW"###,
-    // 16. test of GROUP
-    r###"BCR=123085 GROUP="cdr3_light>=100%" MIN_GROUP=2 MIN_CHAINS=2 CVARS=cdr3_len
-         CDR3=CQTWGTGPWVF"###,
-    // 17. test of GROUP
-    r###"BCR=123085 GROUP=vj_refname,heavy≥96.6% MIN_GROUP=2 MIN_CHAINS=2 
-         CDR3="CARVIVGPKKLEGRLYSSSLHFDCW|CARVIVGPEKQEGRLYSSSLHFDYW" POUT=stdout PCOLS=vj_seq1"###,
-    // 18. test of GROUP
-    r###"BCR=123085 GROUP=vj_heavy_refname,cdr3_heavy_len,cdr3_heavy≥80% LVARS=n,donors,dref 
-         CVARS=const,cdr3_len AMINO=cdr3 CHAINS=2 MIN_GROUP=2
-         CDR3="CARDLHGYDPYGMDVW|CARELRHYDTYGMDVW""###,
-    // 19. test of GROUP
-    r###"BCR=123085 GROUP=vj_refname,cdr3_light_len LVARS=n,donors,dref CVARS=const,cdr3_len 
-         AMINO=cdr3 CHAINS=2 MIN_GROUP=2 CDR3="CARESAVAGDMDVW|CARDYGDYRWWVDGMDVW""###,
-    // 20. test of group
-    r###"BCR=123085 GROUP=vj_refname GROUP_CDR3=CACFGRIGVVVRAAHYW"###,
-    // 21. test of group, asserted at one time
-    r###"BCR=123085 GROUP=vj_refname,cdr3_len MIN_GROUP=3 HONEY=out=stdout,color=var,u1 
-         EXPECT_OK"###,
-];
-
-// Crash tests.  These are tests to make sure that certain options do not result in a crash, even
-// when run on a large and complex dataset.  The options are in groups because not all are
-// compatible with each other.  The datasets are defined by a single fixed list, to be enlarged
-// over time based on discovery of pathologies in particular PUBLIC datasets.
-// All run with certain shared options.
-
-pub const CRASH_DATA: &str = "BCR=\"45977;123085;testx/inputs/flaky\"";
-pub const CRASH_OPTS: &str = "NOPRINT BUILT_IN EXPECT_OK NO_PRE NFORCE";
-pub const CRASH_SETS: [&str; 6] = [
-    /* 1 */ "CONP SEQC SUM MEAN BARCODES DIFF_STYLE=C1 GROUP_VJ_REFNAME",
-    //
-    /* 2 */ "CONX FULL_SEQC DIFF_STYLE=C2 POUT=stdout PCOLS=count_CAR",
-    //
-    /* 3 */
-    "AMINO=fwr1,cdr1,fwr2,cdr2,fwr3,cdr3,fwr4 CVARS=d1_name,d2_name,d_delta,d_Δ,cigar",
-    //
-    /* 4 */
-    "PLOT_BY_ISOTYPE=stdout MIN_CELLS=3 GROUP_VJ_REFNAME_HEAVY ALIGN1 JALIGN1",
-    //
-    /* 5 */
-    "GROUP_VDJ_REFNAME_HEAVY GVARS=d_inconsistent_%,d_inconsistent_n",
-    //
-    /* 6 */
-    "GROUP=vj_refname,cdr3_aa_heavy≥90%,cdr3_aa_light≥90%",
-];
-
-// Test using datasets that are either in the extended public dataset collection,
-// or which require samtools.
-
-pub const EXTENDED_TESTS: [&str; 12] = [
-    // 1. Make sure that POUT works on full dataset.
-    // If we experience failures on other PUBLIC ids, we can add them to this list.
-    r###"BCR=86237 RE POUT=/dev/null NOPRINT EXPECT_OK NO_PRE NFORCE"###,
-    // 2. tests nd2
-    r###"BCR=47199,47200,47212 AMINO=cdr3 NCROSS LVARS=nd2 CDR3=CVKGKSGSFWYYFENW
-         NO_PRE NFORCE"###,
-    // 3. test sec and mem [requires samtools]
-    r###"BCR=123085 GEX=123217 LVARSP=sec,mem CDR3=CVKDRVTGTITELDYW"###,
-    // 4. crashed at one point
-    r###"BCR=128037,128040 GEX=127798,127801 LVARSP=pe1 NOPRINT EXPECT_OK NO_PRE NFORCE"###,
-    //
-    // 5. this added because it got better when a bug in bads detection was fixed
-    r###"TCR=163914 CDR3=CASRLGGEETQYF NO_PRE NFORCE"###,
-    // 6. Test PCHAINS=max.  For this we need a clonotype having at least five chains, and the
-    // question is whether the header line represents cvars for all the chains.  The output of
-    // this is expected to change whenever variables are added.
-    r###"BCR=123085,123089,124547 NWEAK_CHAINS NDOUBLET MIN_CHAINS=5 POUT=stdout PCHAINS=max
-         NOPRINT RE NO_PRE NFORCE"###,
-    // 7. test MIN_GROUP_DONORS
-    r###"BCR="40953;43899" MIX_DONORS MIN_GROUP=2 NO_PRE NFORCE
-         GROUP="cdr3_len,cdr3_aa_heavy>=85%,cdr3_aa_light>=85%,vj_refname" MIN_GROUP_DONORS=2"###,
-    // 8. this asserted at one point
-    r###"BUILT_IN GROUP=vj_refname,cdr3_aa_heavy≥90% MIN_CHAINS_EXACT=2 MIN_GROUP=2 
-         KEEP_CLONO_IF_CELL_MEAN="cdr3_len1>=18" BCR=1018096-1018098 JALIGN1 NO_PRE NFORCE
-         EXPECT_OK"###,
-    // 9. this clonotype included a junk chain before we made a change, and test "/outs"
-    r###"TCR=163911/outs CDR3=CAPSAGDKIIF AMINO=donor NO_PRE NFORCE"###,
-    // 10. test case where digit rows are just barely present
-    r###"TCR=163911 CDR3=CASSLVQPSTDTQYF AMINO=donor NO_PRE NFORCE"###,
-    // 11. this added because it got better when a noise filter was added, also tests u_max
-    r###"TCR=163914 CDR3=CASSLVQPSTDTQYF CVARSP=u_max NO_PRE NFORCE"###,
-    // 12. this added because it got better when a noise filter was added; also test FASTA
-    r###"TCR=163914 CDR3=CAFRGGSYIPTF FASTA=stdout NO_PRE NFORCE"###,
-];
-
-// Tests of internal features.
-
-pub const INTERNAL_TESTS: [&str; 3] = [
-    // 1. gave wrong result
-    r###"123085 CDR3=CARDRIAGRFGYGMDVW NFORCE"###,
-    // 2. test human + IMGT; note that specifying by number forces BCR+TCR reference checks
-    r###"123085 REQUIRE_UNBROKEN_OK IMGT ACCEPT_BROKEN EXPECT_NULL"###,
-    // 3. this crashed; it is not exactly an internal feature test but uses an internal feature
-    // (IMGT) to exhibit the phenomenon
-    r###"BCR=123085 IMGT RE ACCEPT_BROKEN POUT=stdout PCELL BARCODE=AGCAGCCCATTAGGCT-1
-         EXPECT_OK"###,
-];
-
-// List of examples in documentation.
-
-pub const EXAMPLES: [&str; 2] = [
-    // 1.
-    r###"BCR=123089 CDR3=CARRYFGVVADAFDIW"###,
-    // 2.
-    r###"BCR=123085 GEX=123217 LVARSP=gex,IGHV2-5_g_μ CDR3=CALMGTYCSGDNCYSWFDPW"###,
-];
-
 // List of examples on site.
 
 pub const SITE_EXAMPLES: [(&str, &str); 28] = [