diff --git a/.run/springboot/GHSApplication [debug].run.xml b/.run/springboot/GHSApplication [debug].run.xml
index e016ae3a..44dd1315 100644
--- a/.run/springboot/GHSApplication [debug].run.xml
+++ b/.run/springboot/GHSApplication [debug].run.xml
@@ -33,6 +33,10 @@
+
+
+
+
diff --git a/src/main/java/ch/usi/si/seart/config/GraphQlConfig.java b/src/main/java/ch/usi/si/seart/config/GraphQlConfig.java
index 2f2924bc..e727d392 100644
--- a/src/main/java/ch/usi/si/seart/config/GraphQlConfig.java
+++ b/src/main/java/ch/usi/si/seart/config/GraphQlConfig.java
@@ -3,6 +3,7 @@
import ch.usi.si.seart.config.properties.GitHubProperties;
import ch.usi.si.seart.github.Endpoint;
import ch.usi.si.seart.github.GitHubGraphQlConnector;
+import ch.usi.si.seart.github.GitHubHttpHeaders;
import ch.usi.si.seart.github.GitHubTokenManager;
import ch.usi.si.seart.reactive.LoggingFilterFunction;
import io.netty.channel.ChannelOption;
@@ -42,7 +43,7 @@ WebClient webClient(
return WebClient.builder()
.baseUrl(Endpoint.GRAPH_QL.toString())
.clientConnector(reactorClientHttpConnector)
- .defaultHeader("X-GitHub-Api-Version", properties.getApiVersion())
+ .defaultHeader(GitHubHttpHeaders.X_GITHUB_API_VERSION, properties.getApiVersion())
.filter(loggingFilterFunction)
.filter(authorizationFilterFunction)
.build();
diff --git a/src/main/java/ch/usi/si/seart/config/HttpClientConfig.java b/src/main/java/ch/usi/si/seart/config/OkHttpClientConfig.java
similarity index 83%
rename from src/main/java/ch/usi/si/seart/config/HttpClientConfig.java
rename to src/main/java/ch/usi/si/seart/config/OkHttpClientConfig.java
index d7b6b7fe..6d0a17f4 100644
--- a/src/main/java/ch/usi/si/seart/config/HttpClientConfig.java
+++ b/src/main/java/ch/usi/si/seart/config/OkHttpClientConfig.java
@@ -1,6 +1,8 @@
package ch.usi.si.seart.config;
import ch.usi.si.seart.config.properties.GitHubProperties;
+import ch.usi.si.seart.github.GitHubHttpHeaders;
+import ch.usi.si.seart.github.GitHubMediaTypes;
import ch.usi.si.seart.github.GitHubRestConnector;
import ch.usi.si.seart.http.interceptor.HeaderAttachmentInterceptor;
import ch.usi.si.seart.http.interceptor.LoggingInterceptor;
@@ -15,13 +17,13 @@
import java.util.concurrent.TimeUnit;
@Configuration
-public class HttpClientConfig {
+public class OkHttpClientConfig {
@Bean
Headers headers(GitHubProperties properties) {
return Headers.of(
- HttpHeaders.ACCEPT, "application/vnd.github+json",
- "X-GitHub-Api-Version", properties.getApiVersion()
+ HttpHeaders.ACCEPT, GitHubMediaTypes.APPLICATION_VND_GITHUB_V3_JSON_VALUE,
+ GitHubHttpHeaders.X_GITHUB_API_VERSION, properties.getApiVersion()
);
}
diff --git a/src/main/java/ch/usi/si/seart/github/GitHubHttpHeaders.java b/src/main/java/ch/usi/si/seart/github/GitHubHttpHeaders.java
new file mode 100644
index 00000000..feba2d91
--- /dev/null
+++ b/src/main/java/ch/usi/si/seart/github/GitHubHttpHeaders.java
@@ -0,0 +1,25 @@
+package ch.usi.si.seart.github;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public final class GitHubHttpHeaders {
+
+ public static final String X_GITHUB_REQUEST_ID = "X-GitHub-Request-Id";
+
+ public static final String X_GITHUB_API_VERSION = "X-GitHub-Api-Version";
+
+ public static final String X_OAUTH_SCOPES = "X-OAuth-Scopes";
+
+ public static final String X_ACCEPTED_OAUTH_SCOPES = "X-Accepted-OAuth-Scopes";
+
+ public static final String X_RATELIMIT_LIMIT = "X-RateLimit-Limit";
+
+ public static final String X_RATELIMIT_REMAINING = "X-RateLimit-Remaining";
+
+ public static final String X_RATELIMIT_RESET = "X-RateLimit-Reset";
+
+ public static final String X_RATELIMIT_USED = "X-RateLimit-Used";
+
+ public static final String X_RATELIMIT_RESOURCE = "X-RateLimit-Resource";
+}
diff --git a/src/main/java/ch/usi/si/seart/github/GitHubMediaTypes.java b/src/main/java/ch/usi/si/seart/github/GitHubMediaTypes.java
new file mode 100644
index 00000000..6671baac
--- /dev/null
+++ b/src/main/java/ch/usi/si/seart/github/GitHubMediaTypes.java
@@ -0,0 +1,19 @@
+package ch.usi.si.seart.github;
+
+import lombok.experimental.UtilityClass;
+import org.springframework.http.MediaType;
+
+@UtilityClass
+public class GitHubMediaTypes {
+
+ public static final MediaType APPLICATION_VND_GITHUB_V3_JSON;
+
+ public static final String APPLICATION_VND_GITHUB_V3_JSON_VALUE;
+
+ static {
+ String type = "application";
+ String subtype = "vnd.github.v3+json";
+ APPLICATION_VND_GITHUB_V3_JSON = new MediaType(type, subtype);
+ APPLICATION_VND_GITHUB_V3_JSON_VALUE = type + "/" + subtype;
+ }
+}
diff --git a/src/main/java/ch/usi/si/seart/github/GitHubRestConnector.java b/src/main/java/ch/usi/si/seart/github/GitHubRestConnector.java
index ead2c69d..063b527e 100644
--- a/src/main/java/ch/usi/si/seart/github/GitHubRestConnector.java
+++ b/src/main/java/ch/usi/si/seart/github/GitHubRestConnector.java
@@ -359,7 +359,7 @@ private RestResponse handleClientError(
* (3) The repository is taken down due to a TOS violation
* (e.g. https://api.github.com/repos/mlwrx1978/freenode/releases)
*/
- String header = "X-RateLimit-Remaining";
+ String header = GitHubHttpHeaders.X_RATELIMIT_REMAINING;
String value = headers.get(header);
int remaining = Optional.ofNullable(value)
.map(Integer::parseInt)
diff --git a/src/main/java/ch/usi/si/seart/github/GitHubTokenManager.java b/src/main/java/ch/usi/si/seart/github/GitHubTokenManager.java
index f0eb8f07..41a09b17 100644
--- a/src/main/java/ch/usi/si/seart/github/GitHubTokenManager.java
+++ b/src/main/java/ch/usi/si/seart/github/GitHubTokenManager.java
@@ -12,19 +12,31 @@
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
+import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.core.convert.ConversionService;
+import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.stereotype.Component;
+import org.springframework.util.Assert;
+import org.springframework.web.client.DefaultResponseErrorHandler;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.HttpServerErrorException;
+import org.springframework.web.client.ResponseErrorHandler;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.DefaultUriBuilderFactory;
+import org.springframework.web.util.UriTemplateHandler;
-import javax.annotation.PostConstruct;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
@@ -36,7 +48,7 @@
@Slf4j
@Component
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
-public class GitHubTokenManager {
+public class GitHubTokenManager implements InitializingBean {
OkHttpClient httpClient;
@@ -64,31 +76,6 @@ public GitHubTokenManager(
this.tokens = new Cycle<>(properties.getTokens());
}
- @PostConstruct
- void postConstruct() {
- int size = tokens.size();
- switch (size) {
- case 0 -> {
- log.warn("Access tokens not specified, can not mine the GitHub API!");
- log.info(
- "Generate a new access token on https://github.com/settings/tokens " +
- "and add it to the `ghs.github.tokens` property in `ghs.properties`!"
- );
- }
- case 1 -> {
- log.info(
- "Single token specified for GitHub API mining, " +
- "consider adding more tokens to increase the crawler's efficiency."
- );
- currentToken = tokens.next();
- }
- default -> {
- log.info("Loaded {} tokens for usage in mining!", size);
- currentToken = tokens.next();
- }
- }
- }
-
public void replaceToken() {
if (tokens.hasNext()) {
currentToken = tokens.next();
@@ -118,6 +105,64 @@ public void replaceTokenIfExpired() {
}
}
+ @Override
+ public void afterPropertiesSet() {
+ GitHubTokenValidator validator = new GitHubTokenValidator();
+ tokens.toSet().forEach(validator::validate);
+ int size = tokens.size();
+ switch (size) {
+ case 0 -> {
+ log.warn("Access tokens not specified, can not mine the GitHub API!");
+ log.info(
+ "Generate a new access token on https://github.com/settings/tokens " +
+ "and add it to the `ghs.github.tokens` property in `ghs.properties`!"
+ );
+ }
+ case 1 -> {
+ log.info(
+ "Single token specified for GitHub API mining, " +
+ "consider adding more tokens to increase the crawler's efficiency."
+ );
+ currentToken = tokens.next();
+ }
+ default -> {
+ log.info("Loaded {} tokens for usage in mining!", size);
+ currentToken = tokens.next();
+ }
+ }
+ }
+
+ @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
+ private static final class GitHubTokenValidator {
+
+ RestTemplate template;
+
+ GitHubTokenValidator() {
+ UriTemplateHandler templateHandler = new DefaultUriBuilderFactory(Endpoint.RATE_LIMIT.toString());
+ ResponseErrorHandler errorHandler = new DefaultResponseErrorHandler();
+ this.template = new RestTemplateBuilder()
+ .uriTemplateHandler(templateHandler)
+ .errorHandler(errorHandler)
+ .build();
+ }
+
+ public void validate(String token) {
+ try {
+ HttpHeaders headers = new HttpHeaders();
+ headers.setBearerAuth(token);
+ HttpEntity> entity = new HttpEntity<>(headers);
+ ResponseEntity response = template.exchange("", HttpMethod.GET, entity, String.class);
+ headers = response.getHeaders();
+ String value = headers.getFirst(GitHubHttpHeaders.X_OAUTH_SCOPES);
+ Assert.notNull(value, "Token does not have any scopes!");
+ Set scopes = Set.of(value.split(","));
+ Assert.isTrue(scopes.contains("repo"), "Token does not have the `repo` scope!");
+ } catch (RestClientException ex) {
+ throw new IllegalArgumentException(ex);
+ }
+ }
+ }
+
private class RateLimitPollCallback implements RetryCallback {
@Override
diff --git a/src/main/java/ch/usi/si/seart/http/interceptor/LoggingInterceptor.java b/src/main/java/ch/usi/si/seart/http/interceptor/LoggingInterceptor.java
index 655d68c1..ac41f9ca 100644
--- a/src/main/java/ch/usi/si/seart/http/interceptor/LoggingInterceptor.java
+++ b/src/main/java/ch/usi/si/seart/http/interceptor/LoggingInterceptor.java
@@ -1,8 +1,10 @@
package ch.usi.si.seart.http.interceptor;
+import ch.usi.si.seart.github.GitHubHttpHeaders;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.experimental.FieldDefaults;
+import okhttp3.Headers;
import okhttp3.Interceptor;
import okhttp3.Request;
import okhttp3.Response;
@@ -35,6 +37,9 @@ public Response intercept(@NotNull Chain chain) throws IOException {
long endNs = System.nanoTime();
long ms = TimeUnit.NANOSECONDS.toMillis(endNs - startNs);
log.debug("<<< {} ({}ms)", response.code(), ms);
+ Headers headers = response.headers();
+ String id = headers.get(GitHubHttpHeaders.X_GITHUB_REQUEST_ID);
+ log.trace("[[[ {} ]]]", id);
return response;
}
}
diff --git a/src/main/java/ch/usi/si/seart/reactive/LoggingFilterFunction.java b/src/main/java/ch/usi/si/seart/reactive/LoggingFilterFunction.java
index a88c025f..fe6f4054 100644
--- a/src/main/java/ch/usi/si/seart/reactive/LoggingFilterFunction.java
+++ b/src/main/java/ch/usi/si/seart/reactive/LoggingFilterFunction.java
@@ -1,10 +1,12 @@
package ch.usi.si.seart.reactive;
+import ch.usi.si.seart.github.GitHubHttpHeaders;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.experimental.FieldDefaults;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
+import org.springframework.http.HttpHeaders;
import org.springframework.web.reactive.function.client.ClientRequest;
import org.springframework.web.reactive.function.client.ClientResponse;
import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
@@ -29,6 +31,9 @@ public Mono filter(@NotNull ClientRequest request, @NotNull Exch
long endNs = System.nanoTime();
long ms = TimeUnit.NANOSECONDS.toMillis(endNs - startNs);
log.debug("<<< {} ({}ms)", response.rawStatusCode(), ms);
+ HttpHeaders headers = response.headers().asHttpHeaders();
+ String id = headers.getFirst(GitHubHttpHeaders.X_GITHUB_REQUEST_ID);
+ log.trace("[[[ {} ]]]", id);
})
.doOnError(ex -> {
if (log.isDebugEnabled())