Merge pull request #200 from The-Huginn/event-polling

Implement Simple Event polling and create profile based testing

.github/workflows/polling-tests.yml

@@ -0,0 +1,29 @@
+name: Java CI - Polling tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build-and-test:
+    name: JDK ${{matrix.java-version}} JVM build and tests for polling profile
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        java-version: [21]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      name: checkout
+
+    - uses: actions/setup-java@v1
+      name: Set up JDK ${{ matrix.java-version }}
+      with:
+        distribution: "temurin"
+        java-version: ${{ matrix.java-version }}
+
+    - name: Build with Maven
+      run: mvn clean test -Dquarkus.test.profile=test,polling

README.md

@@ -159,6 +159,14 @@ Run your application in dev mode that enables live coding using:
 
 Try to create a PR and update it a few times. The format check sends commit statuses that you will see in the PR.
 
+### Testing
+
+Our application currently runs in 2 different modes.
+* **SSE** - Events received by GitHub
+* **Event Polling** - Retrieve manually events in scheduled intervals
+
+For such use case, we have profile aware tests, where by default we test **SSE** events. To run **Event Polling** you should set `quarkus.test.profile=polling`.
+
 ## Deployment on OpenShift
 
 ### Requirements

pom.xml

@@ -1,13 +1,14 @@
 <?xml version="1.0"?>
 <project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.wildfly</groupId>
   <artifactId>wildfly-github-app</artifactId>
   <version>1.0.0-SNAPSHOT</version>
   <properties>
     <compiler-plugin.version>3.12.1</compiler-plugin.version>
     <failsafe.useModulePath>false</failsafe.useModulePath>
+    <format.skip>false</format.skip>
     <glob.version>0.9.0</glob.version>
     <maven.compiler.release>21</maven.compiler.release>
     <maven.compiler.source>21</maven.compiler.source>
@@ -56,6 +57,10 @@
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-smallrye-health</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.quarkus</groupId>
+      <artifactId>quarkus-scheduler</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-junit5</artifactId>
@@ -158,20 +163,18 @@
         </executions>
         <dependencies>
           <dependency>
-            <artifactId>quarkus-ide-config</artifactId>
             <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-ide-config</artifactId>
             <version>${quarkus.platform.version}</version>
           </dependency>
         </dependencies>
         <configuration>
-          <!-- store outside of target to speed up formatting when mvn clean is used -->
           <cachedir>.cache/formatter-maven-plugin-${formatter-maven-plugin.version}</cachedir>
           <configFile>eclipse-format.xml</configFile>
           <lineEnding>LF</lineEnding>
           <skip>${format.skip}</skip>
         </configuration>
       </plugin>
-
     </plugins>
   </build>
   <profiles>

src/main/java/org/wildfly/bot/polling/EventPollingProcessor.java

@@ -0,0 +1,139 @@
+package org.wildfly.bot.polling;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.quarkiverse.githubapp.GitHubEvent;
+import io.quarkiverse.githubapp.runtime.github.GitHubService;
+import io.quarkus.scheduler.Scheduled;
+import io.smallrye.mutiny.tuples.Tuple2;
+import io.vertx.core.json.Json;
+import io.vertx.core.json.JsonObject;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Event;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+import org.kohsuke.github.GHAppInstallation;
+import org.kohsuke.github.GHEvent;
+import org.kohsuke.github.GHEventInfo;
+import org.kohsuke.github.GHRepository;
+import org.kohsuke.github.GitHub;
+import org.wildfly.bot.polling.processors.PushEventPreprocessor;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Map;
+
+@ApplicationScoped
+public class EventPollingProcessor implements GitHubEventEmitter<Throwable> {
+
+    private static final String ACTION = "action";
+
+    /**
+     * @implNote Mapping between Events retrievable via GitHub API
+     *           https://docs.github.com/en/rest/using-the-rest-api/github-event-types?apiVersion=2022-11-28
+     *           and Events sent by GitHub SSE https://docs.github.com/en/webhooks/webhook-events-and-payloads
+     */
+    private static final Map<GHEvent, EventInfo> typeToEventMap = Map.of(
+            GHEvent.PULL_REQUEST, new EventInfo("pull_request", true),
+            GHEvent.PULL_REQUEST_REVIEW, new EventInfo("pull_request_review", true),
+            GHEvent.PUSH, new EventInfo("push", false));
+
+    private static final Map<GHEvent, GitHubEventPreprocessor> eventProcessorMap = Map.of(
+            GHEvent.PUSH, new PushEventPreprocessor());
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    private static final Logger LOG = Logger.getLogger(EventPollingProcessor.class);
+
+    @Inject
+    Event<GitHubEvent> gitHubEventEmitter;
+
+    @Inject
+    GitHubService gitHubService;
+
+    @Scheduled(every = "60s", delayed = "10s")
+    @Override
+    public void fire() throws IOException {
+        for (GHAppInstallation app : gitHubService.getApplicationClient().getApp().listInstallations()) {
+            GitHub gitHub = gitHubService.getInstallationClient(app.getId());
+            for (GHRepository repository : gitHub.getInstallation().listRepositories()) {
+                for (GHEventInfo eventInfo : repository.listEvents()) {
+                    try {
+                        String payload = payload(eventInfo);
+                        Tuple2<String, String> eventTuple = getEventTuple(payload, eventInfo.getType());
+                        String type = eventTuple.getItem1();
+                        if (type == null) {
+                            LOG.infof("Unable to determine the type of event with payload\n%s", payload);
+                            continue;
+                        }
+                        GitHubEvent gitHubEvent = new GitHubEvent(app.getId(),
+                                null, null, eventInfo.getRepository().getFullName(),
+                                type,
+                                eventTuple.getItem2(),
+                                payload,
+                                (JsonObject) Json.decodeValue(payload), true);
+
+                        try {
+                            GitHubEvent processedGitHubEvent = eventProcessorMap
+                                    .getOrDefault(eventInfo.getType(), GitHubEventPreprocessor.INSTANCE)
+                                    .process(gitHubEvent, repository);
+                            gitHubEventEmitter.fire(processedGitHubEvent);
+                        } catch (JsonProcessingException e) {
+                            LOG.warnf(e, "The preprocessors failed to process [%s] event with the payload [%s]",
+                                    eventInfo.getType(), payload);
+                        }
+
+                        // TODO logic for saving parsed events
+                        return;
+                    } catch (NoSuchFieldException | IllegalAccessException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Retrieves the event name and event action, if applicable
+     *
+     * @return Tuple2 of event name and event action, if event contains action
+     */
+    private static Tuple2<String, String> getEventTuple(String payload, GHEvent event) {
+        EventInfo eventInfo = typeToEventMap.getOrDefault(event, new EventInfo(null, false));
+        Tuple2<String, String> noActionTuple = Tuple2.of(eventInfo.name(), null);
+        if (!eventInfo.hasAction()) {
+            return noActionTuple;
+        }
+        try {
+            JsonNode jsonNode = objectMapper.readTree(payload);
+
+            if (!jsonNode.has(ACTION)) {
+                LOG.warnf(
+                        "For event [%s] there was an \"%s\" attribute in the json expected, but none found. Make sure, this attribute is defined",
+                        eventInfo.name(), ACTION);
+                return noActionTuple;
+            }
+
+            return Tuple2.of(eventInfo.name(), jsonNode.get(ACTION).asText());
+        } catch (JsonProcessingException e) {
+            return noActionTuple;
+        }
+    }
+
+    /**
+     * Retrieves payload from {@link GHEventInfo} using reflection. Unfortunately, there is no other way
+     * to retrieve this raw payload attribute
+     *
+     * @return full String representation of the payload in Json.
+     */
+    public static String payload(GHEventInfo ghEventInfo) throws NoSuchFieldException, IllegalAccessException {
+        Field payloadField = GHEventInfo.class.getDeclaredField("payload");
+        payloadField.setAccessible(true);
+
+        return payloadField.get(ghEventInfo).toString();
+    }
+
+    private record EventInfo(String name, boolean hasAction) {
+    }
+}

src/main/java/org/wildfly/bot/polling/GitHubEventEmitter.java

@@ -0,0 +1,7 @@
+package org.wildfly.bot.polling;
+
+@FunctionalInterface
+public interface GitHubEventEmitter<T extends Throwable> {
+
+    void fire() throws T;
+}

src/main/java/org/wildfly/bot/polling/GitHubEventPreprocessor.java

@@ -0,0 +1,64 @@
+package org.wildfly.bot.polling;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.quarkiverse.githubapp.GitHubEvent;
+import io.quarkus.arc.Arc;
+import io.quarkus.runtime.LaunchMode;
+import io.vertx.core.json.Json;
+import io.vertx.core.json.JsonObject;
+import org.kohsuke.github.GHRepository;
+
+/**
+ * Due to events retrieved from GitHub's API and events received as SSE,
+ * we do not have fully compatible payloads. Thus, if the payload received
+ * from SSE has some extra parameters, which are missing in the event retrieved
+ * from GitHub's API & you are using these parameters in your reactive CDI
+ * beans, you should manually update those fields.
+ * For such case you receive a raw value of created {@link GitHubEvent} and
+ * you can then create new object with updated values.
+ * You can expect authenticated {@link GHRepository} instance for retrieving
+ * the additional info.
+ * This would usually involve static info, where there are no further requests made.
+ * Opposite to most listSomething methods, which actually fetch up-to-date info.
+ */
+@FunctionalInterface
+public interface GitHubEventPreprocessor {
+    GitHubEventPreprocessor INSTANCE = new DefaultGitHubEventPreprocessor();
+    ObjectMapper objectMapper = new ObjectMapper();
+
+    GitHubEvent process(GitHubEvent gitHubEvent, GHRepository repository) throws JsonProcessingException;
+
+    static GitHubEvent updatePayload(GitHubEvent event, JsonNode payload) {
+        String payloadJson = payload.toPrettyString();
+        return new GitHubEvent(event.getInstallationId(),
+                event.getAppName().orElse(null), event.getDeliveryId(), event.getRepositoryOrThrow(),
+                event.getEvent(), event.getAction(), payloadJson, (JsonObject) Json.decodeValue(payloadJson),
+                event.isReplayed());
+    }
+
+    class DefaultGitHubEventPreprocessor implements GitHubEventPreprocessor {
+
+        private static final String REPOSITORY = "repository";
+
+        LaunchMode launchMode;
+
+        @Override
+        public GitHubEvent process(GitHubEvent gitHubEvent, GHRepository repository) throws JsonProcessingException {
+            // due to static initialization in INSTANCE = new ... we need to inject programmatically
+            if (launchMode == null) {
+                launchMode = Arc.container().instance(LaunchMode.class).get();
+            }
+            if (launchMode == LaunchMode.TEST) {
+                return gitHubEvent;
+            }
+            JsonNode payload = objectMapper.readTree(gitHubEvent.getPayload());
+            String repositoryValue = objectMapper.writeValueAsString(repository);
+            ((ObjectNode) payload).put(REPOSITORY, repositoryValue);
+
+            return updatePayload(gitHubEvent, payload);
+        }
+    }
+}

src/main/java/org/wildfly/bot/polling/processors/PushEventPreprocessor.java

@@ -0,0 +1,45 @@
+package org.wildfly.bot.polling.processors;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.quarkiverse.githubapp.GitHubEvent;
+import io.quarkus.logging.Log;
+import org.kohsuke.github.GHRepository;
+import org.wildfly.bot.polling.GitHubEventPreprocessor;
+
+public class PushEventPreprocessor implements GitHubEventPreprocessor {
+
+    private static final String HEAD_COMMIT = "head_commit";
+    private static final String COMMITS = "commits";
+    private static final String HEAD = "head";
+    private static final String SHA = "sha";
+
+    /**
+     * @implNote We simply copy the details of the latest head commit into a new field "head_commit". We do not
+     *           query the commit's details from GitHub, which would be done by repository.getCommit(sha). As we don't need
+     *           all the info and received info is sufficient.
+     */
+    @Override
+    public GitHubEvent process(GitHubEvent gitHubEvent, GHRepository repository) throws JsonProcessingException {
+        GitHubEvent preprocessed = GitHubEventPreprocessor.INSTANCE.process(gitHubEvent, repository);
+
+        JsonNode payload = objectMapper.readTree(preprocessed.getPayload());
+        JsonNode headCommit = null;
+        for (JsonNode commit : payload.get(COMMITS)) {
+            if (commit.has(SHA) && commit.get(SHA).asText().equals(payload.get(HEAD).asText())) {
+                headCommit = commit;
+            }
+        }
+
+        if (headCommit == null) {
+            Log.errorf("Unable to retrieve head commit from received payload [%s]", payload);
+            throw new RuntimeException("Unable to retrieve head commit from received payload [%s]".formatted(payload));
+        }
+
+        ((ObjectNode) payload).remove(HEAD_COMMIT);
+        ((ObjectNode) payload).putIfAbsent(HEAD_COMMIT, headCommit);
+
+        return GitHubEventPreprocessor.updatePayload(preprocessed, payload);
+    }
+}

src/main/resources/application.properties

@@ -7,6 +7,9 @@ quarkus.openshift.resources.limits.cpu=200m
 quarkus.openshift.resources.limits.memory=256Mi
 quarkus.openshift.jvm-arguments=-Xmx128M
 quarkus.openshift.native-arguments=-Xmx128M
+%dev.quarkus.scheduler.enabled=false
+%test.quarkus.scheduler.enabled=false
+%polling.quarkus.scheduler.enabled=false
 
 
 %test.quarkus.github-app.app-id=0
@@ -46,4 +49,4 @@ quarkus.mailer.auth-methods=DIGEST-MD5 CRAM-SHA256 CRAM-SHA1 CRAM-MD5 PLAIN LOGI
 quarkus.mailer.host=smtp.gmail.com
 quarkus.mailer.port=465
 quarkus.mailer.ssl=true
-%test.quarkus.mailer.username=foo@bar.baz
+%test.quarkus.mailer.username=foo@bar.baz

src/test/java/org/wildfly/bot/InstallationEventTest.java

@@ -27,7 +27,7 @@ record -> record.getLevel().intValue() >= Level.ALL.intValue());
 
     @Test
     public void testUnsuspendingApp() throws IOException {
-        given().when().payloadFromClasspath("/installation.json")
+        given().when().payloadFromClasspath("/webhooks/installation.json")
                 .event(GHEvent.INSTALLATION)
                 .then().github(mocks -> Assertions.assertTrue(inMemoryLogHandler.getRecords().stream()
                         .anyMatch(logRecord -> logRecord.getMessage().equals(