Retry API requests to SW360 and FOSSology Urls.

doc/UsageDoc/CA_UsageDocument.md

@@ -226,7 +226,9 @@ Currently LTA support is not provided for SBOM, hence until that is implemented
              ![image.png](../usagedocimg/output.PNG)
 
              Resulted `output.json` file will be having the list of installed packages  and the same file will be used as  an input to `Continuous clearing tool - Bom creator` as an argument(`--packagefilepath`). The remaining process is same as other project types.
+### **API Calls Retry Strategy**
 
+The retry strategy is implemented using the Polly library to handle transient errors such as HTTP request exceptions, task cancellations, and specific HTTP status codes (5xx server errors and 408 Request Timeout). The policy is configured to retry operations up to three times with increasing intervals between attempts (5, 10, and 30 seconds). This approach ensures that network communication is more resilient and reliable by automatically retrying failed requests due to transient issues.
 ### **Configuring the Continuous Clearing Tool**
 
    Arguments can be provided to the tool in two ways :

src/LCT.APICommunications.UTest/RetryHttpClientHandlerUTest.cs

@@ -0,0 +1,154 @@
+using log4net;
+using Moq.Protected;
+using Moq;
+using System.Net;
+
+namespace LCT.APICommunications.UTest
+{
+    public class RetryHttpClientHandlerUTest
+    {
+        private Mock<ILog> _mockLogger;
+
+        [SetUp]
+        public void SetUp()
+        {
+            // Mock the logger
+            _mockLogger = new Mock<ILog>();
+        }
+        [Test]
+        public async Task SendAsync_ShouldRetry_OnTransientErrors()
+        {
+            // Arrange
+            var handlerMock = new Mock<HttpMessageHandler>(MockBehavior.Strict);
+            handlerMock
+                .Protected()
+                .SetupSequence<Task<HttpResponseMessage>>(
+                    "SendAsync",
+                    ItExpr.IsAny<HttpRequestMessage>(),
+                    ItExpr.IsAny<CancellationToken>())
+                .ThrowsAsync(new HttpRequestException())
+                .ThrowsAsync(new TaskCanceledException())
+                .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK));
+
+            var retryHandler = new RetryHttpClientHandler
+            {
+                InnerHandler = handlerMock.Object
+            };
+
+            var httpClient = new HttpClient(retryHandler);
+
+            // Act
+            var response = await httpClient.GetAsync("http://test.com");
+
+            // Assert
+            Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+            handlerMock.Protected().Verify(
+                "SendAsync",
+                Times.Exactly(3), // 2 retries + 1 initial call
+                ItExpr.IsAny<HttpRequestMessage>(),
+                ItExpr.IsAny<CancellationToken>());
+        }
+
+        [Test]
+        public async Task SendAsync_ShouldNotRetry_OnNonTransientErrors()
+        {
+            // Arrange
+            var handlerMock = new Mock<HttpMessageHandler>(MockBehavior.Strict);
+            handlerMock
+                .Protected()
+                .Setup<Task<HttpResponseMessage>>(
+                    "SendAsync",
+                    ItExpr.IsAny<HttpRequestMessage>(),
+                    ItExpr.IsAny<CancellationToken>())
+                .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.BadRequest));
+
+            var retryHandler = new RetryHttpClientHandler
+            {
+                InnerHandler = handlerMock.Object
+            };
+
+            var httpClient = new HttpClient(retryHandler);
+
+            // Act
+            var response = await httpClient.GetAsync("http://test.com");
+
+            // Assert
+            Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
+            handlerMock.Protected().Verify(
+                "SendAsync",
+                Times.Once(), // No retries
+                ItExpr.IsAny<HttpRequestMessage>(),
+                ItExpr.IsAny<CancellationToken>());
+        }
+
+        [Test]
+        public async Task SendAsync_ShouldLogRetryAttempts()
+        {
+            // Arrange
+            var handlerMock = new Mock<HttpMessageHandler>(MockBehavior.Strict);
+            handlerMock
+                .Protected()
+                .SetupSequence<Task<HttpResponseMessage>>(
+                    "SendAsync",
+                    ItExpr.IsAny<HttpRequestMessage>(),
+                    ItExpr.IsAny<CancellationToken>())
+                .ThrowsAsync(new HttpRequestException())
+                .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK));
+
+
+            var retryHandler = new RetryHttpClientHandler()
+            {
+                InnerHandler = handlerMock.Object
+            };
+
+            var httpClient = new HttpClient(retryHandler);
+
+            // Act
+            var response = await httpClient.GetAsync("http://test.com");
+
+            // Assert
+            Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);            
+        }
+        [Test]
+        public async Task ExecuteWithRetryAsync_ShouldCompleteSuccessfully_WhenActionSucceedsAfterRetry()
+        {
+            // Arrange
+            var attempts = 0;
+            var action = new Func<Task>(() =>
+            {
+                attempts++;
+                if (attempts < ApiConstant.APIRetryCount)
+                {
+                    throw new WebException("Temporary error", WebExceptionStatus.Timeout);
+                }
+                return Task.CompletedTask; // Successfully completes after retries
+            });
+
+            // Act
+            await RetryHttpClientHandler.ExecuteWithRetryAsync(action);
+
+            // Assert
+            Assert.AreEqual(ApiConstant.APIRetryCount, attempts, "Action should have been attempted the expected number of times.");
+        }
+
+        [Test]
+        public async Task ExecuteWithRetryAsync_ShouldNotRetry_WhenNoWebExceptionIsThrown()
+        {
+            // Arrange
+            var actionExecuted = false;
+            var action = new Func<Task>(() =>
+            {
+                actionExecuted = true;
+                return Task.CompletedTask;
+            });
+
+            // Act
+            await RetryHttpClientHandler.ExecuteWithRetryAsync(action);
+
+            // Assert
+            Assert.IsTrue(actionExecuted, "Action should have been executed.");
+            _mockLogger.Verify(logger => logger.Debug(It.IsAny<string>()), Times.Never, "Retry should not occur if there is no exception.");
+        }
+
+    }
+}

src/LCT.APICommunications/ApiConstant.cs

@@ -4,6 +4,8 @@
 //  SPDX-License-Identifier: MIT
 // -------------------------------------------------------------------------------------------------------------------- 
 
+using System.Collections.Generic;
+
 namespace LCT.APICommunications
 {
     /// <summary>
@@ -73,5 +75,7 @@ public static class ApiConstant
         public const string PackageNotFound = "Package Not Found";
         public const string ArtifactoryRepoName = "ArtifactoryRepoName";
         public const string JfrogArtifactoryApiSearchAql = $"/api/search/aql";
+        public const int APIRetryCount = 3;
+        public static readonly List<int> APIRetryIntervals = [5, 10, 30]; // in seconds
     }
 }

src/LCT.APICommunications/DebianJfrogAPICommunication.cs

@@ -30,7 +30,11 @@ public DebianJfrogAPICommunication(string repoDomainName, string srcrepoName, Ar
 
         private static HttpClient GetHttpClient(ArtifactoryCredentials credentials)
         {
-            HttpClient httpClient = new HttpClient();
+            var handler = new RetryHttpClientHandler()
+            {
+                InnerHandler = new HttpClientHandler()
+            };
+            var httpClient = new HttpClient(handler);
             TimeSpan timeOutInSec = TimeSpan.FromSeconds(TimeoutInSec);
             httpClient.Timeout = timeOutInSec;
             httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", credentials.Token);

src/LCT.APICommunications/JfrogApicommunication.cs

@@ -32,7 +32,11 @@ protected JfrogApicommunication(string repoDomainName, string srcrepoName, Artif
 
         private static HttpClient GetHttpClient(ArtifactoryCredentials credentials)
         {
-            HttpClient httpClient = new HttpClient();
+            var handler = new RetryHttpClientHandler()
+            {
+                InnerHandler = new HttpClientHandler()
+            };
+            var httpClient = new HttpClient(handler);
             TimeSpan timeOutInSec = TimeSpan.FromSeconds(TimeoutInSec);
             httpClient.Timeout = timeOutInSec;
             httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", credentials.Token);

src/LCT.APICommunications/JfrogAqlApiCommunication.cs

@@ -116,7 +116,11 @@ public async Task<HttpResponseMessage> GetPackageInfo(ComponentsToArtifactory co
 
         private static HttpClient GetHttpClient(ArtifactoryCredentials credentials)
         {
-            HttpClient httpClient = new HttpClient();
+            var handler = new RetryHttpClientHandler()
+            {
+                InnerHandler = new HttpClientHandler()
+            };
+            var httpClient = new HttpClient(handler);
             httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", credentials.Token);
             return httpClient;
         }

src/LCT.APICommunications/LCT.APICommunications.csproj

@@ -23,6 +23,7 @@
     <PackageReference Include="Microsoft.PowerShell.Commands.Utility" Version="7.0.3" />
     <PackageReference Include="Microsoft.Web.Administration" Version="11.1.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="Polly" Version="8.5.2" />
     <PackageReference Include="System.Configuration.ConfigurationManager" Version="4.7.0" />
     <PackageReference Include="System.Drawing.Common" Version="8.0.6" />
     <PackageReference Include="System.Management.Automation" Version="7.0.3" />

src/LCT.APICommunications/MavenJfrogApiCommunication.cs

@@ -21,7 +21,11 @@ public MavenJfrogApiCommunication(string repoDomainName, string srcrepoName, Art
         }
         private static HttpClient GetHttpClient(ArtifactoryCredentials credentials)
         {
-            HttpClient httpClient = new HttpClient();
+            var handler = new RetryHttpClientHandler()
+            {
+                InnerHandler = new HttpClientHandler()
+            };
+            var httpClient = new HttpClient(handler);
             TimeSpan timeOutInSec = TimeSpan.FromSeconds(TimeoutInSec);
             httpClient.Timeout = timeOutInSec;
             httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", credentials.Token);

src/LCT.APICommunications/NpmJfrogAPICommunication.cs

@@ -28,7 +28,11 @@ public NpmJfrogApiCommunication(string repoDomainName, string srcrepoName, Artif
 
         private static HttpClient GetHttpClient(ArtifactoryCredentials credentials)
         {
-            HttpClient httpClient = new HttpClient();
+            var handler = new RetryHttpClientHandler()
+            {
+                InnerHandler = new HttpClientHandler() 
+            };
+            var httpClient = new HttpClient(handler);
             TimeSpan timeOutInSec = TimeSpan.FromSeconds(TimeoutInSec);
             httpClient.Timeout = timeOutInSec;
             httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", credentials.Token);

src/LCT.APICommunications/NugetJfrogAPICommunication.cs

@@ -28,7 +28,11 @@ public NugetJfrogApiCommunication(string repoDomainName, string srcrepoName, Art
 
         private static HttpClient GetHttpClient(ArtifactoryCredentials credentials)
         {
-            HttpClient httpClient = new HttpClient();
+            var handler = new RetryHttpClientHandler()
+            {
+                InnerHandler = new HttpClientHandler()
+            };
+            var httpClient = new HttpClient(handler);
             TimeSpan timeOutInSec = TimeSpan.FromSeconds(TimeoutInSec);
             httpClient.Timeout = timeOutInSec;
             httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", credentials.Token);

src/LCT.APICommunications/PythonJfrogAPICommunication.cs

@@ -24,8 +24,11 @@ public PythonJfrogApiCommunication(string repoDomainName, string srcrepoName, Ar
         }
         private static HttpClient GetHttpClient(ArtifactoryCredentials credentials)
         {
-            HttpClient httpClient = new HttpClient();
-
+            var handler = new RetryHttpClientHandler()
+            {
+                InnerHandler = new HttpClientHandler() 
+            };
+            var httpClient = new HttpClient(handler);
             TimeSpan timeOutInSec = TimeSpan.FromSeconds(TimeoutInSec);
             httpClient.Timeout = timeOutInSec;
             httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", credentials.Token);

src/LCT.APICommunications/RetryHttpClientHandler.cs

@@ -0,0 +1,84 @@
+using log4net;
+using Polly;
+using System;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Reflection;
+using System.Threading.Tasks;
+
+namespace LCT.APICommunications
+{
+    public class RetryHttpClientHandler : DelegatingHandler
+    {
+        private readonly AsyncPolicy<HttpResponseMessage> _retryPolicy;
+        private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+        private bool _initialRetryLogged = false;
+        public RetryHttpClientHandler()
+        {
+            // Define the retry policy (retry on 5xx, 408, and transient errors)
+            _retryPolicy = Policy
+                .Handle<HttpRequestException>() 
+                .Or<TaskCanceledException>()    
+                .OrResult<HttpResponseMessage>(r =>
+                    (r.StatusCode == HttpStatusCode.RequestTimeout
+                    || (int)r.StatusCode >= 500) 
+                    && r.StatusCode != HttpStatusCode.Unauthorized
+                    && r.StatusCode != HttpStatusCode.Forbidden)
+                .WaitAndRetryAsync(ApiConstant.APIRetryCount,
+                     GetRetryInterval,
+                    onRetry: (outcome, timespan, attempt, context) =>
+                    {
+                        Logger.Debug($"Retry attempt {attempt} due to: {(outcome.Exception != null ? outcome.Exception.Message : $"{outcome.Result.StatusCode}")}");
+                        if (!_initialRetryLogged && context["LogWarnings"] as bool? != false)
+                        {                           
+                            Logger.Warn($"Retry attempt triggered due to: {(outcome.Exception != null ? outcome.Exception.Message : $"{outcome.Result.StatusCode}")}");                            
+                        }
+                        context["RetryAttempt"] = attempt;
+                        _initialRetryLogged = true;
+
+                    });
+        }
+        protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, System.Threading.CancellationToken cancellationToken)
+        {
+            var context = new Context
+            {
+                ["LogWarnings"] = !request.Headers.TryGetValues("LogWarnings", out var logWarningsValues) || !bool.TryParse(logWarningsValues.FirstOrDefault(), out var logWarnings) || logWarnings
+            };
+
+            var response = await _retryPolicy.ExecuteAsync(async (ctx) =>
+            {
+                return await base.SendAsync(request, cancellationToken); // Pass the request to the next handler (HttpClient)
+            }, context);
+
+            if (_initialRetryLogged)
+            {
+                var attempt = context.ContainsKey("RetryAttempt") ? context["RetryAttempt"] : 0;
+                Logger.Debug($"Retry attempt successful after {attempt} attempts.");
+                _initialRetryLogged = false;
+            }
+
+            return response;
+        }
+        public static async Task ExecuteWithRetryAsync(Func<Task> action)
+        {
+            var retryPolicy = Policy
+                .Handle<WebException>()
+                .WaitAndRetryAsync(ApiConstant.APIRetryCount,
+                    GetRetryInterval,
+                    onRetry: (exception, timespan, attempt, context) =>
+                    {
+                        Logger.Debug($"Retry attempt {attempt} due to: {exception?.Message ?? "No exception"}");                        
+                    });
+
+            await retryPolicy.ExecuteAsync(action);
+        }
+        private static TimeSpan GetRetryInterval(int attempt)
+        {
+            if (attempt >= 1 && attempt <= ApiConstant.APIRetryIntervals.Count)
+                return TimeSpan.FromSeconds(ApiConstant.APIRetryIntervals[attempt - 1]);
+            return TimeSpan.Zero; // Default if out of range
+        }
+
+    }
+}