Merge pull request #5 from kravemir/optimization

Optimize resolution of configuration dependencies

Author: jwharm

plugin/src/main/java/io/github/jwharm/flatpakgradlegenerator/ArtifactResolver.java

@@ -19,12 +19,13 @@
 
 package io.github.jwharm.flatpakgradlegenerator;
 
-import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ModuleVersionIdentifier;
 import org.gradle.api.artifacts.ResolvedArtifact;
 
 import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
 import java.io.File;
 import java.io.IOException;
 import java.net.HttpURLConnection;
@@ -71,17 +72,16 @@ static ArtifactResolver getInstance(String dest) {
      *
      * @throws NoSuchAlgorithmException no provider for the SHA-512 algorithm
      */
-     Optional<byte[]> tryResolve(DependencyDetails dep,
-                                 String repository,
-                                 String filename)
-            throws NoSuchAlgorithmException {
+    Optional<byte[]> tryResolve(DependencyDetails dep,
+                                String repository,
+                                String filename) throws IOException, NoSuchAlgorithmException {
 
         // Build the url and try to download the file
         String url = repository + dep.path() + "/" + filename;
         var contents = getFileContentsFrom(url);
 
         if (contents.isPresent()) {
-            String sha512 = calculateSHA512(contents.get());
+            String sha512 = calculateSHA512(new ByteArrayInputStream(contents.get()));
             String dest = dep.path();
 
             // If the filename contains a path, cut it out and append it to the dest field
@@ -111,7 +111,7 @@ Optional<byte[]> tryResolve(DependencyDetails dep,
      * find the artifact in the local Gradle file, and use that file to calculate the SHA-512 hash
      * and add it to the JSON output.
      *
-     * @param  configuration the Gradle configuration that contains the artifact
+     * @param  artifacts     resolved artifacts for the gradle configuration containing the dependency
      * @param  id            the id of the dependency
      * @param  dep           a DependencyDetail instance with the Maven coordinates of the artifact
      * @param  repository    the repository to try to download from
@@ -120,7 +120,7 @@ Optional<byte[]> tryResolve(DependencyDetails dep,
      * @throws IOException              error while reading the jar file
      * @throws NoSuchAlgorithmException no provider for the SHA-512 algorithm
      */
-    void tryResolveCached(Configuration configuration,
+    void tryResolveCached(Set<ResolvedArtifact> artifacts,
                           ModuleVersionIdentifier id,
                           DependencyDetails dep,
                           String repository,
@@ -129,18 +129,13 @@ void tryResolveCached(Configuration configuration,
                           String altName)
             throws IOException, NoSuchAlgorithmException {
 
-        // Find the jar in the local Gradle cache
-        Set<ResolvedArtifact> artifacts = configuration
-                .getResolvedConfiguration()
-                .getResolvedArtifacts();
-
         for (var artifact : artifacts) {
             if (artifact.getModuleVersion().getId().equals(id)) {
                 File file = artifact.getFile();
 
                 // Check against filenames in the .module file
                 if (checkName)
-                    if (! (file.getName().equals(filename) ||
+                    if (!(file.getName().equals(filename) ||
                             file.getName().equals(altName)))
                         continue;
 
@@ -155,7 +150,7 @@ void tryResolveCached(Configuration configuration,
                         + file.getName();
                 var isValid = isValid(url);
 
-                if ((! isValid) && filename.contains("SNAPSHOT")) {
+                if ((!isValid) && filename.contains("SNAPSHOT")) {
                     // Try again, but this time, replace SNAPSHOT with snapshot details
                     url = repository
                             + dep.path()
@@ -165,9 +160,12 @@ void tryResolveCached(Configuration configuration,
                 }
 
                 if (isValid) {
+                    String sha512;
+
                     // Read the file from the local Gradle cache and calculate the SHA-512 hash
-                    byte[] bytes = Files.readAllBytes(file.toPath());
-                    String sha512 = calculateSHA512(bytes);
+                    try (var fileInputStream = Files.newInputStream(file.toPath())) {
+                        sha512 = calculateSHA512(fileInputStream);
+                    }
 
                     // Generate and append the json
                     generateJsonBlock(
@@ -280,16 +278,25 @@ private Optional<byte[]> getFileContents(String url) {
     /**
      * Generate an SHA-512 hash for a byte array using MessageDigest.
      *
-     * @param  contents the input byte array
+     * @param  contentsInputStream the input stream for contents to compute hash of
      * @return the SHA-512 hash
      * @throws NoSuchAlgorithmException no provider for the SHA-512 algorithm
+     * @throws IOException error when reading input
      */
-    private String calculateSHA512(byte[] contents) throws NoSuchAlgorithmException {
+    private String calculateSHA512(InputStream contentsInputStream) throws NoSuchAlgorithmException, IOException {
+        byte[] buffer = new byte[4096];
+
         // Create a MessageDigest object for SHA-512
         MessageDigest md = MessageDigest.getInstance("SHA-512");
 
-        // Update the MessageDigest with the bytes from the input String
-        md.update(contents);
+        int numRead;
+        do {
+            numRead = contentsInputStream.read(buffer);
+            if (numRead > 0) {
+                md.update(buffer, 0, numRead);
+            }
+        } while (numRead != -1);
+
         byte[] hashBytes = md.digest();
 
         // Convert the byte array to a hexadecimal string

plugin/src/main/java/io/github/jwharm/flatpakgradlegenerator/FlatpakGradleGeneratorTask.java

@@ -92,11 +92,8 @@ public void apply() throws NoSuchAlgorithmException, IOException {
                 "Resolving buildscript dependencies using repositories: {}",
                 buildScriptRepositories
         );
-        for (var configuration : project.getBuildscript().getConfigurations()) {
-            if (configuration.isCanBeResolved()) {
-                generateSourcesList(buildScriptRepositories, configuration);
-            }
-        }
+        generateSourcesListForConfigurations(buildScriptRepositories, project.getBuildscript().getConfigurations());
+        LOGGER.info("Buildscript dependencies resolved successfully");
 
         // List the declared repositories; include the buildscript classpath
         // repositories too
@@ -110,12 +107,10 @@ public void apply() throws NoSuchAlgorithmException, IOException {
                 "Resolving project dependencies using repositories: {}",
                 repositories
         );
-        for (var configuration : project.getConfigurations()) {
-            if (configuration.isCanBeResolved()) {
-                generateSourcesList(repositories, configuration);
-            }
-        }
+        generateSourcesListForConfigurations(repositories, project.getConfigurations());
+        LOGGER.info("Project dependencies resolved successfully");
 
+        LOGGER.info("Writing result to " + getOutputFile().getAsFile());
         // Merge the json blocks into a json list, and write the result to the
         // output file
         Files.writeString(
@@ -124,6 +119,21 @@ public void apply() throws NoSuchAlgorithmException, IOException {
         );
     }
 
+    private void generateSourcesListForConfigurations(
+            List<String> repositories,
+            Collection<Configuration> configurations
+    ) throws IOException, NoSuchAlgorithmException {
+        var resolvedConfigurations = configurations.stream()
+                .filter(Configuration::isCanBeResolved)
+                .toList();
+
+        LOGGER.info("Following configurations will be processed: {}", resolvedConfigurations);
+
+        for (Configuration configuration : resolvedConfigurations) {
+            generateSourcesList(repositories, configuration);
+        }
+    }
+
     /**
      * Get all Maven repositories that were declared in the build file.
      *
@@ -190,11 +200,21 @@ private void generateSourcesList(List<String> repositories,
         var dependencies = listDependencies(configuration);
 
         LOGGER.info(
-                "Resolving configuration {} with {} dependencies",
+                "Configuration processing of {}: starting with {} dependencies",
                 configuration.getName(),
                 dependencies.size()
         );
 
+        var resolvedArtifactsForConfiguration = configuration
+                .getResolvedConfiguration()
+                .getResolvedArtifacts();
+
+        LOGGER.info(
+                "Configuration processing of {}: artifacts resolved, total {}",
+                configuration.getName(),
+                resolvedArtifactsForConfiguration.size()
+        );
+
         for (var dependency : dependencies) {
 
             String id = dependency.getSelected().getId().getDisplayName();
@@ -229,7 +249,7 @@ private void generateSourcesList(List<String> repositories,
                         if (files != null) {
                             for (var file : files) {
                                 resolver.tryResolveCached(
-                                        configuration,
+                                        resolvedArtifactsForConfiguration,
                                         dependency.getSelected().getModuleVersion(),
                                         dep,
                                         repository,
@@ -243,7 +263,7 @@ private void generateSourcesList(List<String> repositories,
                         // No files declared for this variant in the Gradle module
                         // Get jar artifact from local Gradle cache
                         resolver.tryResolveCached(
-                                configuration,
+                                resolvedArtifactsForConfiguration,
                                 dependency.getSelected().getModuleVersion(),
                                 dep,
                                 repository,
@@ -257,7 +277,7 @@ private void generateSourcesList(List<String> repositories,
                 // Get jar artifact from local Gradle cache
                 else {
                     resolver.tryResolveCached(
-                            configuration,
+                            resolvedArtifactsForConfiguration,
                             dependency.getSelected().getModuleVersion(),
                             dep,
                             repository,
@@ -284,6 +304,12 @@ private void generateSourcesList(List<String> repositories,
                 if (module.isPresent() || pom.isPresent())
                     break;
             }
+
+            LOGGER.info(
+                    "Dependency {} in {} processed",
+                    id,
+                    configuration.getName()
+            );
         }
     }
 
@@ -320,7 +346,7 @@ private List<ResolvedDependencyResult> listDependencies(Configuration configurat
      *
      * @param dep details about the dependency
      */
-    private void addPluginMarker(DependencyDetails dep) throws NoSuchAlgorithmException {
+    private void addPluginMarker(DependencyDetails dep) throws IOException, NoSuchAlgorithmException {
 
         // This is the marker artifact we are looking for
         var marker = new DependencyDetails(

plugin/src/test/resources/dokka/build.gradle

@@ -3,6 +3,10 @@ plugins {
     id 'org.jetbrains.dokka' version '1.8.10'
 }
 
+repositories {
+    mavenCentral()
+}
+
 tasks.flatpakGradleGenerator {
     outputFile = file('%s')
 }