Merge pull request #280 from seart-group/enhancement/tokens

.run/springboot/GHSApplication [debug].run.xml

@@ -33,6 +33,10 @@
         <option name="name" value="logging.level.ch.usi.si.seart" />
         <option name="value" value="DEBUG" />
       </param>
+      <param>
+        <option name="name" value="logging.level.ch.usi.si.seart" />
+        <option name="value" value="TRACE" />
+      </param>
       <param>
         <option name="name" value="logging.level.org.flywaydb" />
         <option name="value" value="DEBUG" />

src/main/java/ch/usi/si/seart/config/GraphQlConfig.java

@@ -3,6 +3,7 @@
 import ch.usi.si.seart.config.properties.GitHubProperties;
 import ch.usi.si.seart.github.Endpoint;
 import ch.usi.si.seart.github.GitHubGraphQlConnector;
+import ch.usi.si.seart.github.GitHubHttpHeaders;
 import ch.usi.si.seart.github.GitHubTokenManager;
 import ch.usi.si.seart.reactive.LoggingFilterFunction;
 import io.netty.channel.ChannelOption;
@@ -42,7 +43,7 @@ WebClient webClient(
         return WebClient.builder()
                 .baseUrl(Endpoint.GRAPH_QL.toString())
                 .clientConnector(reactorClientHttpConnector)
-                .defaultHeader("X-GitHub-Api-Version", properties.getApiVersion())
+                .defaultHeader(GitHubHttpHeaders.X_GITHUB_API_VERSION, properties.getApiVersion())
                 .filter(loggingFilterFunction)
                 .filter(authorizationFilterFunction)
                 .build();

src/main/java/ch/usi/si/seart/config/HttpClientConfig.java → src/main/java/ch/usi/si/seart/config/OkHttpClientConfig.java

@@ -1,6 +1,8 @@
 package ch.usi.si.seart.config;
 
 import ch.usi.si.seart.config.properties.GitHubProperties;
+import ch.usi.si.seart.github.GitHubHttpHeaders;
+import ch.usi.si.seart.github.GitHubMediaTypes;
 import ch.usi.si.seart.github.GitHubRestConnector;
 import ch.usi.si.seart.http.interceptor.HeaderAttachmentInterceptor;
 import ch.usi.si.seart.http.interceptor.LoggingInterceptor;
@@ -15,13 +17,13 @@
 import java.util.concurrent.TimeUnit;
 
 @Configuration
-public class HttpClientConfig {
+public class OkHttpClientConfig {
 
     @Bean
     Headers headers(GitHubProperties properties) {
         return Headers.of(
-                HttpHeaders.ACCEPT, "application/vnd.github+json",
-                "X-GitHub-Api-Version", properties.getApiVersion()
+                HttpHeaders.ACCEPT, GitHubMediaTypes.APPLICATION_VND_GITHUB_V3_JSON_VALUE,
+                GitHubHttpHeaders.X_GITHUB_API_VERSION, properties.getApiVersion()
         );
     }
 

src/main/java/ch/usi/si/seart/github/GitHubHttpHeaders.java

@@ -0,0 +1,25 @@
+package ch.usi.si.seart.github;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public final class GitHubHttpHeaders {
+
+    public static final String X_GITHUB_REQUEST_ID = "X-GitHub-Request-Id";
+
+    public static final String X_GITHUB_API_VERSION = "X-GitHub-Api-Version";
+
+    public static final String X_OAUTH_SCOPES = "X-OAuth-Scopes";
+
+    public static final String X_ACCEPTED_OAUTH_SCOPES = "X-Accepted-OAuth-Scopes";
+
+    public static final String X_RATELIMIT_LIMIT = "X-RateLimit-Limit";
+
+    public static final String X_RATELIMIT_REMAINING = "X-RateLimit-Remaining";
+
+    public static final String X_RATELIMIT_RESET = "X-RateLimit-Reset";
+
+    public static final String X_RATELIMIT_USED = "X-RateLimit-Used";
+
+    public static final String X_RATELIMIT_RESOURCE = "X-RateLimit-Resource";
+}

src/main/java/ch/usi/si/seart/github/GitHubMediaTypes.java

@@ -0,0 +1,19 @@
+package ch.usi.si.seart.github;
+
+import lombok.experimental.UtilityClass;
+import org.springframework.http.MediaType;
+
+@UtilityClass
+public class GitHubMediaTypes {
+
+    public static final MediaType APPLICATION_VND_GITHUB_V3_JSON;
+
+    public static final String APPLICATION_VND_GITHUB_V3_JSON_VALUE;
+
+    static {
+        String type = "application";
+        String subtype = "vnd.github.v3+json";
+        APPLICATION_VND_GITHUB_V3_JSON = new MediaType(type, subtype);
+        APPLICATION_VND_GITHUB_V3_JSON_VALUE = type + "/" + subtype;
+    }
+}

src/main/java/ch/usi/si/seart/github/GitHubRestConnector.java

@@ -359,7 +359,7 @@ private RestResponse handleClientError(
                      * (3) The repository is taken down due to a TOS violation
                      * (e.g. https://api.github.com/repos/mlwrx1978/freenode/releases)
                      */
-                    String header = "X-RateLimit-Remaining";
+                    String header = GitHubHttpHeaders.X_RATELIMIT_REMAINING;
                     String value = headers.get(header);
                     int remaining = Optional.ofNullable(value)
                             .map(Integer::parseInt)

src/main/java/ch/usi/si/seart/github/GitHubTokenManager.java

@@ -12,19 +12,31 @@
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.Response;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.web.client.RestTemplateBuilder;
 import org.springframework.core.convert.ConversionService;
+import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
 import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
 import org.springframework.retry.RetryCallback;
 import org.springframework.retry.RetryContext;
 import org.springframework.retry.support.RetryTemplate;
 import org.springframework.stereotype.Component;
+import org.springframework.util.Assert;
+import org.springframework.web.client.DefaultResponseErrorHandler;
 import org.springframework.web.client.HttpClientErrorException;
 import org.springframework.web.client.HttpServerErrorException;
+import org.springframework.web.client.ResponseErrorHandler;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.DefaultUriBuilderFactory;
+import org.springframework.web.util.UriTemplateHandler;
 
-import javax.annotation.PostConstruct;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -36,7 +48,7 @@
 @Slf4j
 @Component
 @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
-public class GitHubTokenManager {
+public class GitHubTokenManager implements InitializingBean {
 
     OkHttpClient httpClient;
 
@@ -64,31 +76,6 @@ public GitHubTokenManager(
         this.tokens = new Cycle<>(properties.getTokens());
     }
 
-    @PostConstruct
-    void postConstruct() {
-        int size = tokens.size();
-        switch (size) {
-            case 0 -> {
-                log.warn("Access tokens not specified, can not mine the GitHub API!");
-                log.info(
-                        "Generate a new access token on https://github.com/settings/tokens " +
-                                "and add it to the `ghs.github.tokens` property in `ghs.properties`!"
-                );
-            }
-            case 1 -> {
-                log.info(
-                        "Single token specified for GitHub API mining, " +
-                                "consider adding more tokens to increase the crawler's efficiency."
-                );
-                currentToken = tokens.next();
-            }
-            default -> {
-                log.info("Loaded {} tokens for usage in mining!", size);
-                currentToken = tokens.next();
-            }
-        }
-    }
-
     public void replaceToken() {
         if (tokens.hasNext()) {
             currentToken = tokens.next();
@@ -118,6 +105,64 @@ public void replaceTokenIfExpired() {
         }
     }
 
+    @Override
+    public void afterPropertiesSet() {
+        GitHubTokenValidator validator = new GitHubTokenValidator();
+        tokens.toSet().forEach(validator::validate);
+        int size = tokens.size();
+        switch (size) {
+            case 0 -> {
+                log.warn("Access tokens not specified, can not mine the GitHub API!");
+                log.info(
+                        "Generate a new access token on https://github.com/settings/tokens " +
+                                "and add it to the `ghs.github.tokens` property in `ghs.properties`!"
+                );
+            }
+            case 1 -> {
+                log.info(
+                        "Single token specified for GitHub API mining, " +
+                                "consider adding more tokens to increase the crawler's efficiency."
+                );
+                currentToken = tokens.next();
+            }
+            default -> {
+                log.info("Loaded {} tokens for usage in mining!", size);
+                currentToken = tokens.next();
+            }
+        }
+    }
+
+    @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
+    private static final class GitHubTokenValidator {
+
+        RestTemplate template;
+
+        GitHubTokenValidator() {
+            UriTemplateHandler templateHandler = new DefaultUriBuilderFactory(Endpoint.RATE_LIMIT.toString());
+            ResponseErrorHandler errorHandler = new DefaultResponseErrorHandler();
+            this.template = new RestTemplateBuilder()
+                    .uriTemplateHandler(templateHandler)
+                    .errorHandler(errorHandler)
+                    .build();
+        }
+
+        public void validate(String token) {
+            try {
+                HttpHeaders headers = new HttpHeaders();
+                headers.setBearerAuth(token);
+                HttpEntity<?> entity = new HttpEntity<>(headers);
+                ResponseEntity<String> response = template.exchange("", HttpMethod.GET, entity, String.class);
+                headers = response.getHeaders();
+                String value = headers.getFirst(GitHubHttpHeaders.X_OAUTH_SCOPES);
+                Assert.notNull(value, "Token does not have any scopes!");
+                Set<String> scopes = Set.of(value.split(","));
+                Assert.isTrue(scopes.contains("repo"), "Token does not have the `repo` scope!");
+            } catch (RestClientException ex) {
+                throw new IllegalArgumentException(ex);
+            }
+        }
+    }
+
     private class RateLimitPollCallback implements RetryCallback<RateLimit, Exception> {
 
         @Override

src/main/java/ch/usi/si/seart/http/interceptor/LoggingInterceptor.java

@@ -1,8 +1,10 @@
 package ch.usi.si.seart.http.interceptor;
 
+import ch.usi.si.seart.github.GitHubHttpHeaders;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import okhttp3.Headers;
 import okhttp3.Interceptor;
 import okhttp3.Request;
 import okhttp3.Response;
@@ -35,6 +37,9 @@ public Response intercept(@NotNull Chain chain) throws IOException {
         long endNs = System.nanoTime();
         long ms = TimeUnit.NANOSECONDS.toMillis(endNs - startNs);
         log.debug("<<< {} ({}ms)", response.code(), ms);
+        Headers headers = response.headers();
+        String id = headers.get(GitHubHttpHeaders.X_GITHUB_REQUEST_ID);
+        log.trace("[[[ {} ]]]", id);
         return response;
     }
 }

src/main/java/ch/usi/si/seart/reactive/LoggingFilterFunction.java

@@ -1,10 +1,12 @@
 package ch.usi.si.seart.reactive;
 
+import ch.usi.si.seart.github.GitHubHttpHeaders;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
+import org.springframework.http.HttpHeaders;
 import org.springframework.web.reactive.function.client.ClientRequest;
 import org.springframework.web.reactive.function.client.ClientResponse;
 import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
@@ -29,6 +31,9 @@ public Mono<ClientResponse> filter(@NotNull ClientRequest request, @NotNull Exch
                     long endNs = System.nanoTime();
                     long ms = TimeUnit.NANOSECONDS.toMillis(endNs - startNs);
                     log.debug("<<< {} ({}ms)", response.rawStatusCode(), ms);
+                    HttpHeaders headers = response.headers().asHttpHeaders();
+                    String id = headers.getFirst(GitHubHttpHeaders.X_GITHUB_REQUEST_ID);
+                    log.trace("[[[ {} ]]]", id);
                 })
                 .doOnError(ex -> {
                     if (log.isDebugEnabled())