Adding example suite support (#89)

* Adding support for juint5 Suites and applying suite level JUnitPerf annotations

---------

Co-authored-by: noconnor <niall.oconnor@yahooinc.com>

.github/workflows/publish.yml

@@ -9,7 +9,6 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-          ref: ${{ github.ref_name }} # https://github.com/actions/checkout/issues/317
       - name: Set up Maven Central Repository
         uses: actions/setup-java@v3
         with:

README.md

@@ -80,12 +80,6 @@ This will override the `durationMs` set in the `@JUnitPerfTest` annotation.
 
 [CSV Reporting](#csv-reporting)
 
-[Custom Reporting](#custom-reporting)
-
-[Multiple Reports](#multiple-reports)
-
-[Grouping Reports](#grouping-reports)
-
 <br />
 
 #### HTML Reports

docs/junit5.md

@@ -13,6 +13,8 @@ Back to [index](../README.md) page.
 
 [Overriding Statistic Capturing](#overriding-statistic-capturing)
 
+[Test Suites](#test-suite-configuration)
+
 <br />
 
 ### Install Instructions
@@ -312,3 +314,38 @@ private final static JUnitPerfReportingConfig PERF_CONFIG = JUnitPerfReportingCo
 ``` 
 
 For each `@Test` instance, the `statisticsCalculatorSupplier` will be called to generate a new `StatisticsCalculator` instance
+
+<br />
+
+### Test Suite Configuration
+
+JUnitPerf annotations can also be applied at the suite level. <br />
+When running in a test suite, annotation resolution follows the following precedence order:
+
+* Test method level annotations will take precedence over all other annotation configurations
+* If no method level annotations are available, class level annotations will be considered next
+* If no class level annotations are available, an attempt will be made to identify suite level annotations (if the test is currently running as part of a suite)
+
+If order to apply the JUnitPerfTestInterceptor to all tests in a suite, jupiter global extensions must be enabled. <br />
+This can be done by:
+* Adding a run time VM arg: `-Djunit.jupiter.extensions.autodetection.enabled=true`
+* Specifying the extensions param as a ConfigurationParameter at the suite level: i.e. add this annotation to your suite `@ConfigurationParameter(key = "junit.jupiter.extensions.autodetection.enabled", value = "true")`
+
+The `junit-platform-suite-engine` dependency must also be added to your project:
+```
+<dependency>
+    <groupId>org.junit.platform</groupId>
+    <artifactId>junit-platform-suite-engine</artifactId>
+    <version>LATEST_VERSION</version>
+    <scope>test</scope>
+</dependency>
+```
+
+
+An example test case can be found in the [junit5-examples](../junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleTestSuiteUsage.java) module
+
+To run the example suite test you can use the following maven commands
+* from inside the junitperf project root directory:  `mvn clean install -Dgpg.skip`
+  * This will install all junitperf snapshot dependencies to your local maven repo 
+* from inside the `junit5-examples` folder: `mvn -Dtest=ExampleTestSuiteUsage -DskipTests=false test`
+  * Location of test reports will be printed to console (default location is the `junit5-examples/build/reports/` directory)

junit5-examples/pom.xml

@@ -25,6 +25,7 @@
             <groupId>org.junit.platform</groupId>
             <artifactId>junit-platform-suite-engine</artifactId>
             <version>1.9.3</version>
+            <scope>test</scope>
         </dependency>
     </dependencies>
 

junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleTestSuiteUsage.java

@@ -0,0 +1,39 @@
+package com.github.noconnor.junitperf.examples;
+
+import com.github.noconnor.junitperf.JUnitPerfReportingConfig;
+import com.github.noconnor.junitperf.JUnitPerfTest;
+import com.github.noconnor.junitperf.JUnitPerfTestActiveConfig;
+import com.github.noconnor.junitperf.JUnitPerfTestRequirement;
+import com.github.noconnor.junitperf.examples.existing.TestClassOne;
+import com.github.noconnor.junitperf.examples.existing.TestClassTwo;
+import org.junit.platform.suite.api.ConfigurationParameter;
+import org.junit.platform.suite.api.SelectClasses;
+import org.junit.platform.suite.api.Suite;
+
+import static com.github.noconnor.junitperf.examples.utils.ReportingUtils.newHtmlReporter;
+
+
+//
+// To run suite: mvn -Dtest=ExampleTestSuiteUsage -DskipTests=false test
+//
+
+@Suite
+//@SelectPackages({
+//        "com.github.noconnor.junitperf.examples.existing"
+//})
+@SelectClasses({
+        TestClassOne.class,
+        TestClassTwo.class
+})
+// ConfigurationParameter: Required to enable Test Suite Interceptor Reference: https://www.baeldung.com/junit-5-extensions#1-automatic-extension-registration
+@ConfigurationParameter(key = "junit.jupiter.extensions.autodetection.enabled", value = "true")
+@JUnitPerfTest(totalExecutions = 100)
+@JUnitPerfTestRequirement(allowedErrorPercentage = 0.01F)
+public class ExampleTestSuiteUsage {
+
+    @JUnitPerfTestActiveConfig
+    public static JUnitPerfReportingConfig config = JUnitPerfReportingConfig.builder()
+            .reportGenerator(newHtmlReporter("suite_reporter.html"))
+            .build();
+
+}

junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/existing/TestClassOne.java

@@ -0,0 +1,17 @@
+package com.github.noconnor.junitperf.examples.existing;
+
+import org.junit.jupiter.api.Test;
+
+public class TestClassOne {
+    @Test
+    public void sample_test1_class1() throws InterruptedException {
+        Thread.sleep(5);
+    }
+
+    @Test
+    public void sample_test2_class1() throws InterruptedException {
+        // Mock some processing logic
+        Thread.sleep(1);
+    }
+
+}

junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/existing/TestClassThree.java

@@ -0,0 +1,17 @@
+package com.github.noconnor.junitperf.examples.existing;
+
+import org.junit.jupiter.api.Test;
+
+public class TestClassThree {
+    @Test
+    public void sample_test1_class3() throws InterruptedException {
+        Thread.sleep(5);
+    }
+
+    @Test
+    public void sample_test2_class3() throws InterruptedException {
+        // Mock some processing logic
+        Thread.sleep(1);
+    }
+
+}

junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/existing/TestClassTwo.java

@@ -0,0 +1,12 @@
+package com.github.noconnor.junitperf.examples.existing;
+
+import org.junit.jupiter.api.Test;
+
+public class TestClassTwo {
+
+    @Test
+    public void sample_test1_class2() throws InterruptedException {
+        // Mock some processing logic
+        Thread.sleep(1);
+    }
+}

junitperf-core/src/main/java/com/github/noconnor/junitperf/JUnitPerfTest.java

@@ -6,7 +6,7 @@
 import java.lang.annotation.Target;
 
 @Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.METHOD})
+@Target({ElementType.METHOD, ElementType.TYPE})
 public @interface JUnitPerfTest {
 
   // Total number of threads to use during the test evaluations

junitperf-core/src/main/java/com/github/noconnor/junitperf/JUnitPerfTestRequirement.java

@@ -6,7 +6,7 @@
 import java.lang.annotation.Target;
 
 @Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.METHOD})
+@Target({ElementType.METHOD, ElementType.TYPE})
 public @interface JUnitPerfTestRequirement {
 
   // Expected target percentile distribution in the format "percentile1:expected_value_ms,percentile2:expected_value_ms,..."

junitperf-core/src/main/java/com/github/noconnor/junitperf/data/EvaluationContext.java

@@ -17,7 +17,6 @@
 import com.google.common.primitives.Floats;
 import com.google.common.primitives.Ints;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 import lombok.Getter;
 import lombok.Setter;
@@ -104,6 +103,9 @@ public class EvaluationContext {
   private final String testName;
   @Getter
   private final String startTime;
+  @Getter
+  @Setter
+  private String groupName;
 
   public EvaluationContext(String testName, long startTimeNs) {
     this(testName, startTimeNs, false);

junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/utils/ViewData.java

@@ -10,6 +10,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static java.util.Objects.nonNull;
+
 @Getter
 public class ViewData {
 
@@ -56,7 +58,7 @@ public static final class RequiredPercentilesData {
     private final List<RequiredPercentilesData> requiredPercentiles;
 
     public ViewData(EvaluationContext context) {
-        this.testName = context.getTestName();
+        this.testName = buildTestName(context);
         this.testNameColour = context.isSuccessful() ? SUCCESS_COLOUR : FAILED_COLOUR;
         this.chartData = buildChartData(context);
         this.csvData = buildCsvData(context);
@@ -85,6 +87,10 @@ public ViewData(EvaluationContext context) {
         this.requiredPercentiles = buildRequiredPercentileData(context);
     }
 
+    private static String buildTestName(EvaluationContext context) {
+        return nonNull(context.getGroupName()) ? context.getGroupName() + " : " + context.getTestName() : context.getTestName();
+    }
+
     private List<RequiredPercentilesData> buildRequiredPercentileData(EvaluationContext context) {
         return context.getRequiredPercentiles().entrySet()
                 .stream()

junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/EvaluationTask.java

@@ -96,9 +96,11 @@ private void evaluateStatement(long startMeasurements) {
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();
       } catch (Throwable throwable) {
-        stats.incrementEvaluationCount();
-        stats.incrementErrorCount();
-        stats.addLatencyMeasurement(nanoTime() - startTimeNs);
+        if (!(throwable.getCause() instanceof InterruptedException)) {
+          stats.incrementEvaluationCount();
+          stats.incrementErrorCount();
+          stats.addLatencyMeasurement(nanoTime() - startTimeNs);
+        }
       }
 
       try {

junitperf-core/src/test/java/com/github/noconnor/junitperf/data/EvaluationContextTest.java

@@ -4,6 +4,7 @@
 import static com.github.noconnor.junitperf.data.EvaluationContext.JUNITPERF_MAX_EXECUTIONS_PER_SECOND;
 import static com.github.noconnor.junitperf.data.EvaluationContext.JUNITPERF_RAMP_UP_PERIOD_MS;
 import static com.github.noconnor.junitperf.data.EvaluationContext.JUNITPERF_THREADS;
+import static com.github.noconnor.junitperf.data.EvaluationContext.JUNITPERF_TOTAL_EXECUTIONS;
 import static com.github.noconnor.junitperf.data.EvaluationContext.JUNITPERF_WARM_UP_MS;
 import static java.lang.System.nanoTime;
 import static java.util.Collections.emptyMap;
@@ -61,6 +62,7 @@ public void tearDown(){
     System.clearProperty(JUNITPERF_WARM_UP_MS);
     System.clearProperty(JUNITPERF_MAX_EXECUTIONS_PER_SECOND);
     System.clearProperty(JUNITPERF_RAMP_UP_PERIOD_MS);
+    System.clearProperty(JUNITPERF_TOTAL_EXECUTIONS);
   }
 
   @Test
@@ -71,6 +73,7 @@ public void whenLoadingJUnitPerfTestSettings_thenAppropriateContextSettingsShoul
     assertEquals(perfTestAnnotation.threads(), context.getConfiguredThreads());
     assertEquals(perfTestAnnotation.warmUpMs(), context.getConfiguredWarmUp());
     assertEquals(perfTestAnnotation.rampUpPeriodMs(), context.getConfiguredRampUpPeriodMs());
+    assertEquals(perfTestAnnotation.totalExecutions(), context.getConfiguredExecutionTarget());
   }
 
   @Test
@@ -80,12 +83,14 @@ public void whenLoadingJUnitPerfTestSettings_andEnvironmentHasOverrides_thenAppr
     System.setProperty(JUNITPERF_WARM_UP_MS, "2000");
     System.setProperty(JUNITPERF_MAX_EXECUTIONS_PER_SECOND, "55");
     System.setProperty(JUNITPERF_RAMP_UP_PERIOD_MS, "1000");
+    System.setProperty(JUNITPERF_TOTAL_EXECUTIONS, "400");
     context.loadConfiguration(perfTestAnnotation);
     assertEquals(60000, context.getConfiguredDuration());
     assertEquals(55, context.getConfiguredRateLimit());
     assertEquals(45, context.getConfiguredThreads());
     assertEquals(2000, context.getConfiguredWarmUp());
     assertEquals(1000, context.getConfiguredRampUpPeriodMs());
+    assertEquals(400, context.getConfiguredExecutionTarget());
   }
 
   @Test
@@ -401,6 +406,12 @@ public void whenSpecifyingAsyncFlag_thenIsAsyncShouldBeTrue() {
     assertTrue(context.isAsyncEvaluation());
   }
 
+  @Test
+  public void whenGroupNameIsSet_thenGroupNameShouldBeConfigured() {
+    context.setGroupName("unittest");
+    assertEquals("unittest", context.getGroupName());
+  }
+
   @Test
   public void whenDurationIsValid_thenDurationShouldBeFormatted() {
     context.setFinishTimeNs(startTimeNs + 100_000_000_000_000L);
@@ -423,6 +434,7 @@ private void initialisePerfTestAnnotation() {
     when(perfTestAnnotation.threads()).thenReturn(50);
     when(perfTestAnnotation.warmUpMs()).thenReturn(5);
     when(perfTestAnnotation.rampUpPeriodMs()).thenReturn(4);
+    when(perfTestAnnotation.totalExecutions()).thenReturn(345);
   }
 
   private void initialisePerfTestRequirementAnnotation() {

junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/utils/ViewDataTest.java

@@ -140,6 +140,14 @@ public void whenRequiredPercentilesIsNotSet_thenViewDataShouldBeMappedCorrectly(
         assertEquals(Collections.emptyList(), viewData.getRequiredPercentiles());
     }
 
+    @Test
+    public void whenGroupNameIsSet_thenTestNameShouldBeScopedByGroupName() {
+        EvaluationContext context = buildMockContext(1234F, true);
+        when(context.getGroupName()).thenReturn("ClassName");
+        ViewData viewData = new ViewData(context);
+        assertEquals("ClassName : Unittest", viewData.getTestName());
+    }
+
     private EvaluationContext buildMockContext(float dummyLatency, boolean isSuccessful) {
         EvaluationContext context = mock(EvaluationContext.class);
         when(context.getTestName()).thenReturn("Unittest");

junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/EvaluationTaskTest.java

@@ -83,6 +83,16 @@ public void whenRunning_andStatementEvaluationThrowsAnException_thenLatencyMeasu
     verify(statsMock, times(10)).addLatencyMeasurement(anyLong());
   }
 
+  @Test
+  public void whenRunning_andStatementEvaluationThrowsAnInterruptException_thenNoMeasurementsShouldBeTakenTaken() throws Throwable {
+    setExecutionCount(10);
+    mockNestedInterruptAfter(9);
+    task.run();
+    verify(statsMock, times(9)).addLatencyMeasurement(anyLong());
+    verify(statsMock, times(9)).incrementEvaluationCount();
+    verify(statsMock, never()).incrementErrorCount();
+  }
+
   @Test
   public void whenRunning_thenAnAttemptShouldBeMadeToRetrieveAPermit() {
     setExecutionCount(10);
@@ -200,7 +210,6 @@ private void mockEvaluationFailures(int desiredFailureCount) throws Throwable {
       }
       return null;
     }).when(statementMock).evaluate();
-
   }
 
   private void mockInterruptAfter(int desiredSuccessfulInvocations) throws Throwable {
@@ -213,6 +222,17 @@ private void mockInterruptAfter(int desiredSuccessfulInvocations) throws Throwab
     }).when(statementMock).evaluate();
   }
 
+  private void mockNestedInterruptAfter(int desiredSuccessfulInvocations) throws Throwable {
+    AtomicInteger executions = new AtomicInteger();
+    doAnswer(invocation -> {
+      if (executions.getAndIncrement() >= desiredSuccessfulInvocations) {
+        throw new RuntimeException("Nest interrupt", new InterruptedException("mock exception"));
+      }
+      return null;
+    }).when(statementMock).evaluate();
+  }
+
+
   private void mockTerminateAfter(int desiredSuccessfulInvocations) throws Throwable {
     AtomicInteger executions = new AtomicInteger();
     doAnswer(invocation -> {

junitperf-junit5/pom.xml

@@ -11,6 +11,7 @@
 
     <properties>
         <junit.jupiter.version>5.9.0</junit.jupiter.version>
+        <junit.jupiter.suite.api.version>1.9.3</junit.jupiter.suite.api.version>
         <mockito.junit.version>3.6.28</mockito.junit.version>
     </properties>
 
@@ -32,9 +33,9 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter-engine</artifactId>
-            <version>${junit.jupiter.version}</version>
+            <groupId>org.junit.platform</groupId>
+            <artifactId>junit-platform-suite-api</artifactId>
+            <version>${junit.jupiter.suite.api.version}</version>
             <scope>test</scope>
         </dependency>
     </dependencies>