Merge pull request #264 from seart-group/feature/disabled

seart-group · Dec 28, 2023 · 30a27fe · 30a27fe
2 parents 0ff0d82 + 6f7428d
commit 30a27fe
Show file tree

Hide file tree

Showing 13 changed files with 2,324 additions and 208 deletions.
diff --git a/flyway/sql/V13__disable.sql b/flyway/sql/V13__disable.sql
@@ -0,0 +1,12 @@
+ALTER TABLE git_repo
+ADD COLUMN disabled BIT NULL
+AFTER archived;
+
+ALTER TABLE git_repo
+ADD COLUMN locked BIT NULL
+AFTER archived;
+
+UPDATE git_repo
+SET
+    disabled = false,
+    locked = false;
diff --git a/html/index.html b/html/index.html
@@ -1003,6 +1003,12 @@ <h2 class="mb-3">No Results! <i class="bi bi-emoji-frown-fill"></i></h2>
               {{#if archived}}
               <i class="octicon octicon-archive-24" title="Archived"></i>
               {{/if}}
+              {{#if disabled}}
+              <i class="octicon octicon-alert-24" title="Disabled"></i>
+              {{/if}}
+              {{#if locked}}
+              <i class="octicon octicon-lock-24" title="Locked"></i>
+              {{/if}}
               {{#if wiki}}
               <i class="octicon octicon-book-24" title="Has Wiki"></i>
               {{/if}}

diff --git a/html/js/search.js b/html/js/search.js
@@ -144,6 +144,8 @@
             mainLanguage,
             isFork,
             isArchived,
+            isDisabled,
+            isLocked,
             hasWiki,
             homepage,
             defaultBranch,
@@ -190,6 +192,8 @@
             properties: {
                 fork: isFork,
                 archived: isArchived,
+                disabled: isDisabled,
+                locked: isLocked,
                 wiki: hasWiki
             },
             statistics: {

diff --git a/src/main/java/ch/usi/si/seart/converter/GitRepoDtoToCsvConverter.java b/src/main/java/ch/usi/si/seart/converter/GitRepoDtoToCsvConverter.java
@@ -9,9 +9,6 @@
 import org.springframework.lang.NonNull;
 import org.springframework.stereotype.Component;
 
-/**
- * Convert a GitRepoDTO to it's CSV representation.
- */
 @Component
 @AllArgsConstructor(onConstructor_ = @Autowired)
 public class GitRepoDtoToCsvConverter implements Converter<GitRepoDto, GitRepoCsvDto> {
@@ -50,7 +47,10 @@ public GitRepoCsvDto convert(@NonNull GitRepoDto source) {
                 .metrics(source.getMetrics())
                 .lastCommit(source.getLastCommit())
                 .lastCommitSHA(source.getLastCommitSHA())
-                // Convert Collection types into stringified JSON
+                .hasWiki(source.getHasWiki())
+                .isArchived(source.getIsArchived())
+                .isDisabled(source.getIsDisabled())
+                .isLocked(source.getIsLocked())
                 .metricsString(csvMapper.valueToTree(source.getMetrics()).toString())
                 .labelsString(csvMapper.valueToTree(source.getLabels()).toString())
                 .languagesString(csvMapper.valueToTree(source.getLanguages()).toString())

diff --git a/src/main/java/ch/usi/si/seart/converter/GitRepoToDtoConverter.java b/src/main/java/ch/usi/si/seart/converter/GitRepoToDtoConverter.java
@@ -62,6 +62,8 @@ public GitRepoDto convert(@NonNull GitRepo source) {
                 )
                 .hasWiki(source.getHasWiki())
                 .isArchived(source.getIsArchived())
+                .isDisabled(source.getIsDisabled())
+                .isLocked(source.getIsLocked())
                 .languages(
                         source.getLanguages().stream()
                                 .map(l -> Map.entry(l.getLanguage().getName(), l.getSizeOfCode()))

diff --git a/src/main/java/ch/usi/si/seart/converter/StringToGitExceptionConverter.java b/src/main/java/ch/usi/si/seart/converter/StringToGitExceptionConverter.java
@@ -4,6 +4,7 @@
 import ch.usi.si.seart.exception.git.CompressionException;
 import ch.usi.si.seart.exception.git.GitException;
 import ch.usi.si.seart.exception.git.RepositoryDisabledException;
+import ch.usi.si.seart.exception.git.RepositoryLockedException;
 import ch.usi.si.seart.exception.git.RepositoryNotFoundException;
 import ch.usi.si.seart.exception.git.TerminalPromptsDisabledException;
 import ch.usi.si.seart.exception.git.config.InvalidProxyConfigurationException;
@@ -13,11 +14,12 @@
 
 public class StringToGitExceptionConverter implements Converter<String, GitException> {
 
-    private static final String EMPTY = "";
+    private static final String LOCKED = "is disabled";
     private static final String NOT_FOUND = "not found";
     private static final String EARLY_EOF = "early EOF";
     private static final String LONG_FILE_NAME = "Filename too long";
     private static final String FORBIDDEN = "The requested URL returned error: 403";
+    private static final String DISABLED = "Access to this repository has been disabled by GitHub staff";
     private static final String UNRESOLVABLE_HOST = "Could not resolve host: github.com";
     private static final String CHECKOUT_FAILED = "unable to checkout working tree";
     private static final String AUTHENTICATION_REQUIRED =
@@ -28,21 +30,28 @@ public class StringToGitExceptionConverter implements Converter<String, GitExcep
     @Override
     @NonNull
     public GitException convert(@NonNull String source) {
+        String remote = source.lines()
+                .filter(line -> line.startsWith("remote:"))
+                .findFirst()
+                .map(string -> string.substring(8))
+                .orElse("");
         String fatal = source.lines()
                 .filter(line -> line.startsWith("fatal:"))
-                .findAny()
+                .findFirst()
                 .map(string -> string.substring(7))
-                .orElse(EMPTY);
+                .orElse("");
         if (fatal.endsWith(NOT_FOUND))
             return new RepositoryNotFoundException(fatal);
-        if (fatal.endsWith(FORBIDDEN))
+        if (fatal.endsWith(FORBIDDEN) && remote.startsWith(DISABLED))
             return new RepositoryDisabledException(fatal);
+        if (fatal.endsWith(FORBIDDEN) && remote.contains(LOCKED))
+            return new RepositoryLockedException(fatal);
         if (fatal.endsWith(UNRESOLVABLE_HOST))
             return new InvalidProxyConfigurationException(fatal);
         if (fatal.endsWith(LONG_FILE_NAME))
             return new CheckoutException(fatal);
         return switch (fatal) {
-            case EMPTY -> new GitException();
+            case "" -> new GitException("\n" + source + "\n");
             case EARLY_EOF -> new CompressionException(fatal);
             case PROMPTS_DISABLED -> new TerminalPromptsDisabledException(fatal);
             case AUTHENTICATION_REQUIRED -> new InvalidUsernameException(fatal);

diff --git a/src/main/java/ch/usi/si/seart/dto/GitRepoDto.java b/src/main/java/ch/usi/si/seart/dto/GitRepoDto.java
@@ -23,46 +23,66 @@
 @NoArgsConstructor
 @AllArgsConstructor
 public class GitRepoDto {
+
     Long id;
+
     String name;
+
     Boolean isFork;
+
     Long commits;
     Long branches;
-    String defaultBranch;
     Long releases;
-    Long contributors;
+    Long forks;
+
+    String mainLanguage;
+    String defaultBranch;
+
     String license;
+    String homepage;
+
     Long watchers;
     Long stargazers;
-    Long forks;
+    Long contributors;
+
     Long size;
+
     Date createdAt;
     Date pushedAt;
     Date updatedAt;
-    String homepage;
-    String mainLanguage;
+
     Long totalIssues;
     Long openIssues;
+
     Long totalPullRequests;
     Long openPullRequests;
+
     Long blankLines;
     Long codeLines;
     Long commentLines;
+
     @JacksonXmlElementWrapper(localName = "metrics")
     @JacksonXmlProperty(localName = "metric")
     @Builder.Default
     List<Map<String, Object>> metrics = new ArrayList<>();
+
     Date lastCommit;
     String lastCommitSHA;
+
     Boolean hasWiki;
     Boolean isArchived;
+    Boolean isDisabled;
+    Boolean isLocked;
+
     @JacksonXmlProperty(localName = "languages", isAttribute = true)
     @Builder.Default
     Map<String, Long> languages = new LinkedHashMap<>();
+
     @JacksonXmlElementWrapper(localName = "labels")
     @JacksonXmlProperty(localName = "label")
     @Builder.Default
     Set<String> labels = new TreeSet<>();
+
     @JacksonXmlElementWrapper(localName = "topics")
     @JacksonXmlProperty(localName = "topic")
     @Builder.Default

diff --git a/src/main/java/ch/usi/si/seart/exception/git/RepositoryLockedException.java b/src/main/java/ch/usi/si/seart/exception/git/RepositoryLockedException.java
@@ -0,0 +1,7 @@
+package ch.usi.si.seart.exception.git;
+
+import lombok.experimental.StandardException;
+
+@StandardException
+public class RepositoryLockedException extends GitException {
+}
diff --git a/src/main/java/ch/usi/si/seart/job/CodeAnalysisJob.java b/src/main/java/ch/usi/si/seart/job/CodeAnalysisJob.java
@@ -3,6 +3,8 @@
 import ch.usi.si.seart.analysis.CLOCConnector;
 import ch.usi.si.seart.exception.StaticCodeAnalysisException;
 import ch.usi.si.seart.exception.git.GitException;
+import ch.usi.si.seart.exception.git.RepositoryDisabledException;
+import ch.usi.si.seart.exception.git.RepositoryLockedException;
 import ch.usi.si.seart.exception.git.RepositoryNotFoundException;
 import ch.usi.si.seart.git.GitConnector;
 import ch.usi.si.seart.git.LocalRepositoryClone;
@@ -117,11 +119,22 @@ private void gatherCodeMetricsFor(@NotNull GitRepo gitRepo) {
                 log.debug("No metrics were computed for: {}", name);
                 log.info("Skipping:  {} [{}]", name, id);
             }
+            gitRepo.setIsLocked(false);
+            gitRepo.setIsDisabled(false);
             gitRepo.setMetrics(metrics);
             gitRepo.setLastAnalyzed();
             gitRepoService.createOrUpdate(gitRepo);
+        } catch (RepositoryLockedException ignored) {
+            log.info("Locking:   {} [{}]", name, id);
+            gitRepo.setIsLocked(true);
+            gitRepo.setLastAnalyzed();
+            gitRepoService.createOrUpdate(gitRepo);
+        } catch (RepositoryDisabledException ignored) {
+            log.info("Disabling: {} [{}]", name, id);
+            gitRepo.setIsDisabled(true);
+            gitRepo.setLastAnalyzed();
+            gitRepoService.createOrUpdate(gitRepo);
         } catch (RepositoryNotFoundException ignored) {
-            log.debug("Remote not found {}, performing cleanup instead...", name);
             log.info("Deleting:  {} [{}]", name, id);
             gitRepoService.deleteRepoById(id);
         } catch (StaticCodeAnalysisException | GitException ex) {

diff --git a/src/main/java/ch/usi/si/seart/job/CrawlProjectsJob.java b/src/main/java/ch/usi/si/seart/job/CrawlProjectsJob.java
@@ -253,6 +253,10 @@ private void saveRetrievedRepo(JsonObject result, String language, int lowerInde
             gitRepo.setIsFork(isFork);
             Boolean isArchived = json.getAsJsonPrimitive("is_archived").getAsBoolean();
             gitRepo.setIsArchived(isArchived);
+            Boolean isDisabled = json.getAsJsonPrimitive("is_disabled").getAsBoolean();
+            gitRepo.setIsDisabled(isDisabled);
+            Boolean isLocked = json.getAsJsonPrimitive("is_locked").getAsBoolean();
+            gitRepo.setIsLocked(isLocked);
 
             Long stargazers = json.getAsJsonObject("stars")
                     .getAsJsonPrimitive("count")

diff --git a/src/main/java/ch/usi/si/seart/model/GitRepo.java b/src/main/java/ch/usi/si/seart/model/GitRepo.java
@@ -151,6 +151,12 @@ public class GitRepo {
     @Column(name = "archived")
     Boolean isArchived;
 
+    @Column(name = "disabled")
+    Boolean isDisabled;
+
+    @Column(name = "locked")
+    Boolean isLocked;
+
     @PastOrPresent
     @Column(name = "last_pinged")
     Date lastPinged;

diff --git a/src/main/resources/graphql-documents/repository.graphql b/src/main/resources/graphql-documents/repository.graphql
@@ -5,6 +5,8 @@ query($owner:String!, $name:String!) {
         homepage: homepageUrl
         is_fork: isFork
         is_archived: isArchived
+        is_disabled: isDisabled
+        is_locked: isLocked
         has_wiki: hasWikiEnabled
         forks: forkCount
         license: licenseInfo {