Updates and improvements to the scripts and the pre-processor.

See the README on how to use things now.

preprocessor/README.md

@@ -28,7 +28,7 @@ Notation for Mutation Testing
 The pre-processor is line based.  For mutation testing we use a CPP-like
 if-block, as illustrated by the following example:
 ```
-#if !MUTATION(function_containing_the_mutant)
+#if !CN_MUTATE_function_containing_the_mutantion_block
 Normal
 code
 path
@@ -69,15 +69,62 @@ Some other variant
 ```
 
 
-Unit Tests
-==========
+Notation for Unit Tests
+=======================
 
 Unit tests are written as CPP conditionals where the condition is
-an identifier that starts with `CN_TEST`.  For example:
+an identifier that starts with `CN_TEST_` followed by the
+name of the function for the test.  For example:
 
 ```
-#if CN_TEST
-Lines only
-for test
+#if CN_TEST_function_name
+void function_name() {
+  write test here
+}
 #endif
 ```
+
+Sometimes we'd also like to indicate that a test is expected to fail. We can
+do this by adding `// fails` on the line declaring the test.
+
+```
+#if CN_TEST_function_name // fails
+void function_name() {
+  write test here
+}
+#endif
+```
+
+This indicates that we expect this test to fail.
+
+
+Scripts
+=======
+
+The directory also contains some scripts which use the preprocessor to run
+the test variations in a file:
+
+    * `run-all CFILE [LOG_FILE]`
+       Test all functions in a file, including unit tests, and mutants.
+       If a LOG_FILE is provided the output of the commands is stored
+       there.  This is useful if tests fails.
+       Mutants are considered to succeed if the corresponding test fails
+       (i.e., they found a bug)
+       Unit tests succeed if the result of testing matches the declarations
+       (see `// fails` above)
+
+    * `config` is a script fragment which defines the locations of external
+       tools
+
+    * `run-cn-test` is a script fragment which defines how we run CN tests
+
+    * `run-unit` can run a single unit test.
+       It just runs the unit test, without checking the expected outcome.
+
+    * `run-mutant` can run a single mutation.
+
+    * `run-prop-tests` test all "normal" functions (i.e., not unit tests or
+      mutants)
+
+
+

preprocessor/example.c

@@ -1,18 +1,18 @@
 // This file show how we envision using the preprocessor.
-#define S 11111111
 
-int puts(const char*);
+
+#define S 11111111
 
 int some_other_fun(int x)
-/*@ requires true; ensures return == 1i32; @*/
+/*@ requires true; ensures return == 0i32; @*/
 {
   return 0;
 }
 
 int inc(int x)
 /*@ requires 0i32 <= x && x < 10i32; ensures  return < 11i32; @*/
 {
-#if !MUTATION(inc)
+#if !CN_MUTATE_inc
   return x + 1;
 #elif X_PLUS_2
   return x + 2;
@@ -22,15 +22,15 @@ int inc(int x)
 }
 
 
-#if CN_TEST_A
-int main(int argc, char* argv[]) {
-  return inc(S);
+#if CN_TEST_A // fails
+int A() {
+  inc(S);
 }
 #endif
 
 #if CN_TEST_B
-int main(int argc, char* argv[]) {
-  return inc(5);
+int B() {
+  inc(5);
 }
 #endif
 

preprocessor/preproc_tut.ml

@@ -1,36 +1,38 @@
 
-let drop_prefix prefix str =
+let drop_token prefix str =
   if String.starts_with ~prefix str
     then
       let n = String.length prefix in
       let l = String.length str in
       Some (String.trim (String.sub str n (l - n)))
     else None
 
-let rec drop_prefixes prefixes str =
+let rec drop_tokens prefixes str =
   match prefixes with
   | [] -> Some str
   | p :: ps ->
-    match drop_prefix p str with
-    | Some rest -> drop_prefixes ps rest
+    match drop_token p str with
+    | Some rest -> drop_tokens ps rest
     | None -> None
 
 (* Should we start a new mutant block *)
-let start_mutant_block line =
-  match drop_prefixes ["#if"; "!"; "MUTATION"; "("] line with
-  | Some res when String.ends_with ~suffix:")" res ->
-    let fu = String.trim (String.sub res 0 (String.length res - 1)) in
-    Some fu
-  | _ -> None
-
+let start_mutant_block = drop_tokens ["#if"; "!"; "CN_MUTATE_"]
 
 (* Does this line start a mutant *)
-let start_mutant = drop_prefix "#elif"
+let start_mutant = drop_token "#elif"
 
+(* Does this line start a unit test *)
 let start_unit_test line =
-  match drop_prefix "#if" line with
-  | Some txt when String.starts_with ~prefix:"CN_TEST" txt -> Some txt
-  | _ -> None
+  match drop_tokens ["#if"; "CN_TEST_"] line with
+  | None -> None
+  | Some name_and_more ->
+    match String.split_on_char '/' name_and_more with
+    | [name; ""; rest] ->
+      let new_name = String.trim name in
+      let fails = String.equal "fails" (String.trim rest) in
+      Some (new_name, fails)
+    | _ -> Some (name_and_more, false)
+
 
 (* Ending for mutant blocks and units tests *)
 let end_named_block = String.starts_with ~prefix:"#endif"
@@ -48,6 +50,7 @@ type mode =
   | ExecuteMutant of string   (* Print only this specific mutant *)
   | CollectUnitTest           (* Print only names of unit tests *)
   | ExecuteUnitTest of string (* Print only this specific unit test *)
+  | ExecuteAllUnitTests       (* No mutants; all unit tests enabled *)
 
 
 (* The current state of the processor *)
@@ -90,9 +93,11 @@ let rec process_input mode start_line state =
            begin match start_unit_test line with
 
              (* start a unit test *)
-             | Some name ->
+             | Some (name,fails) ->
                begin match mode with
-               | CollectUnitTest -> print_endline name
+               | CollectUnitTest ->
+                 Printf.printf "%s/%s\n" name
+                   (if fails then "fails" else "succeeds")
                | _               -> ()
                end;
                InUnitTest (start_line, name) (* next state *)
@@ -168,6 +173,7 @@ let rec process_input mode start_line state =
       (* Line in a unit test *)  
       | InUnitTest (ln,name) ->
         begin match mode with
+        | ExecuteAllUnitTests                        -> print_endline line
         | ExecuteUnitTest t when String.equal name t -> print_endline line
         | _ -> ()
         end;
@@ -204,10 +210,14 @@ let options =
     "NAME\tShow mutant with the given name");
 
     ("--list-unit", Arg.Unit (set_command CollectUnitTest),
-    "\tShow the names of the unit tests in the input");
+    "\tShow unit test names and expected outcome");
 
     ("--show-unit", Arg.String (fun name -> set_command (ExecuteUnitTest name) ()),
-    "NAME\tExecute unit test with the given name")
+    "NAME\tExecute unit test with the given name");
+
+    ("--show-all-unit", Arg.Unit (set_command ExecuteAllUnitTests),
+    "\tExecute all unit tests without mutations");
+
   ]
 
 let () =

preprocessor/run-all

@@ -1,24 +1,75 @@
 #!/bin/bash
+# Run all tests and variations in a C file
 
-if [ $# -ne 1 ]; then
-  echo "USAGE: $0 CFILE"
-  echo "  Run all unit tests and mutants in a single C file."
+LOG_FILE=/dev/null
+
+function usage() {
+  echo "USAGE: $0 CFILE [LOG_FILE]"
+  echo "  Run all tests and variations in single C file."
   exit 1
+}
+
+if (($# > 2)) || (($# < 1)); then
+  usage
 fi
 
-FILE="$1"
+FILE=$1
+shift
+if (($# > 0)); then
+  LOG_FILE=$1
+fi
 
-SCRIPT_DIR="$(dirname $0)"
+echo "$FILE"
+SCRIPT_DIR=$(dirname "$0")
 source "$SCRIPT_DIR/config"
 
-for UNIT_TEST in $($PREPROC --list-unit < "$FILE"); do
-  echo $UNIT_TEST
-  $SCRIPT_DIR/run-unit-test "$FILE" "$UNIT_TEST"
+function log_section() {
+  echo "+------------------------------------------------------" >> "$LOG_FILE"
+  echo "| $1" >> "$LOG_FILE"
+  echo "+------------------------------------------------------" >> "$LOG_FILE"
+}
+
+function start_test() {
+  echo -n "  $1"
+  log_section "$1"
+}
+
+OK_MSG="[\033[1;92mOK\033[0m]"
+FAIL_MSG="[\033[1;91mFAIL\033[0m]"
+
+function end_test() {
+  if [ "$1" == "0" ]; then
+    echo -e $2
+  else
+    echo -e $3
+  fi
+}
+
+# Testing start from here
+
+echo -n "" > "$LOG_FILE"
+log_section "Test start $(date)"
+
+for UNIT in $("$PREPROC" --list-unit < "$FILE"); do
+  TEST=$(dirname "$UNIT")
+  EXPECT=$(basename "$UNIT")
+  start_test "Unit test $TEST: "
+  "$SCRIPT_DIR/run-unit" "$FILE" "$TEST" >> "$LOG_FILE"
+  RESULT=$?
+  if [[ "$EXPECT" == "succeeds" ]]
+    then end_test $RESULT $OK_MSG $FAIL_MSG
+    else end_test $RESULT $FAIL_MSG $OK_MSG
+  fi
 done
 
-for MUTANT in $($PREPROC --list-mutants < "$FILE"); do
-  echo $MUTANT
-  $SCRIPT_DIR/run-mutant "$FILE" "$MUTANT"
+start_test "Checking functions:"
+"$SCRIPT_DIR/run-prop-tests" "$FILE" >> "$LOG_FILE"
+end_test $? $OK_MSG $FAIL_MSG
+
+for MUTANT in $("$PREPROC" --list-mutants < "$FILE"); do
+  start_test "Mutant $MUTANT: "
+  "$SCRIPT_DIR/run-mutant" "$FILE" "$MUTANT" >> "$LOG_FILE"
+  end_test $? $FAIL_MSG $OK_MSG
 done
 
 

preprocessor/run-cn-test

@@ -0,0 +1,16 @@
+# Script fragment for runnin CN tests
+
+source "$SCRIPT_DIR/config"
+
+OUT_DIR=$(mktemp -d /tmp/cn-test-XXXXXX)
+trap 'rm -rf "$OUT_DIR"' EXIT
+
+"$PREPROC" $PREPROC_FLAGS <"$CFILE" >"$OUT_DIR/test-file.c"
+
+pushd "$OUT_DIR" > /dev/null || exit 2
+$CPP "test-file.c" > "test-file.i"
+mv "test-file.i" "test-file.c"
+"$CN" test "test-file.c" --progress-level=0 --output-dir=./run --seed=0 $CN_TEST_FLAGS
+RESULT=$?
+popd > /dev/null || exit 2
+exit $RESULT

preprocessor/run-mutant

@@ -1,30 +1,18 @@
 #!/bin/bash
+# Do property based testing for a specific mutant
+#################################################
 
 if [ $# -ne 2 ]; then
   echo "USAGE: $0 CFILE FUN_QUALIFIED_MUTANT_NAME"
-  echo "  Preprocess, and test a single mutant in a single C file."
+  echo "  Run a single mutant in the given C file."
   exit 1
 fi
 
-SCRIPT_DIR="$(dirname $0)"
-source "$SCRIPT_DIR/config"
-
-CFILE="$1"
+SCRIPT_DIR=$(dirname "$0")
+CFILE=$1
 FUN=$(dirname "$2")
 UNIT_TEST=$(basename "$2")
-
-OUT_DIR=$(mktemp -d /tmp/cn-run-mutant-XXXXXX)
-trap "rm -rf $OUT_DIR" EXIT
-
-cat "$CFILE" | $PREPROC --show-mutant "$UNIT_TEST" > "$OUT_DIR/$UNIT_TEST.c"
-
-pushd "$OUT_DIR"
-$CPP "$UNIT_TEST.c" > "$UNIT_TEST.i"
-mv "$UNIT_TEST.i" "$UNIT_TEST.c"
-$CN test "$UNIT_TEST.c" --output-dir=. --only="$FUN" --seed=0 > /dev/null
-RESULT=$?
-popd
-exit $RESULT
-
-
+PREPROC_FLAGS="--show-mutant $UNIT_TEST"
+CN_TEST_FLAGS="--only=$FUN"
+source "$SCRIPT_DIR/run-cn-test"
 

preprocessor/run-prop-tests

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Do property based testing for a non-mutated file
+##################################################
+
+if [ $# -ne 1 ]; then
+  echo "USAGE: $0 CFILE"
+  echo "  Run all property based tests in a given C file."
+  exit 1
+fi
+
+SCRIPT_DIR=$(dirname "$0")
+CFILE=$1
+PREPROC_FLAGS="--no-test"
+CN_TEST_FLAGS=""
+source "$SCRIPT_DIR/run-cn-test"
+
+