Escl: Improve reliability through network interruptions

- Use backported SocketsHttpHandler
- Set a ConnectTimeout of 5s
- Lock while processing NextDocument requests
- NextDocument responds with the same document on retries after error

#336

NAPS2.Escl.Server/EsclApiController.cs

@@ -260,10 +260,24 @@ public async Task NextDocument(string jobId)
             return;
         }
 
-        bool result;
+        await jobInfo.NextDocumentLock.Take();
         try
         {
-            result = await jobInfo.Job.WaitForNextDocument();
+            await WaitForAndWriteNextDocument(jobInfo);
+        }
+        finally
+        {
+            jobInfo.NextDocumentLock.Release();
+        }
+    }
+
+    private async Task WaitForAndWriteNextDocument(JobInfo jobInfo)
+    {
+        try
+        {
+            // If we already have a document (i.e. if a connection error occured during the previous NextDocument
+            // request), we stay at that same document and don't advance
+            jobInfo.NextDocumentReady = jobInfo.NextDocumentReady || await jobInfo.Job.WaitForNextDocument();
         }
         catch (Exception ex)
         {
@@ -272,15 +286,28 @@ public async Task NextDocument(string jobId)
             Response.StatusCode = 500;
             return;
         }
-        if (result)
+
+        // At this point either we have a document and can respond with it, or we have no documents left and should 404
+        if (jobInfo.NextDocumentReady)
         {
-            Response.Headers.Add("Content-Location", $"/eSCL/ScanJobs/{jobInfo.Id}/1");
-            SetChunkedResponse();
-            Response.ContentType = jobInfo.Job.ContentType;
-            Response.ContentEncoding = null;
-            using var stream = Response.OutputStream;
-            await jobInfo.Job.WriteDocumentTo(stream);
-            jobInfo.TransferredDocument();
+            try
+            {
+                Response.Headers.Add("Content-Location", $"/eSCL/ScanJobs/{jobInfo.Id}/1");
+                SetChunkedResponse();
+                Response.ContentType = jobInfo.Job.ContentType;
+                Response.ContentEncoding = null;
+                using var stream = Response.OutputStream;
+                await jobInfo.Job.WriteDocumentTo(stream);
+                jobInfo.NextDocumentReady = false;
+                jobInfo.TransferredDocument();
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "ESCL server error writing document");
+                // We don't transition state here, as the assumption is that the problem was network-related and the
+                // client will retry
+                Response.StatusCode = 500;
+            }
         }
         else
         {

NAPS2.Escl.Server/JobInfo.cs

@@ -48,6 +48,10 @@ public static JobInfo CreateNewJob(EsclServerState serverState, IEsclScanJob job
 
     public required IEsclScanJob Job { get; set; }
 
+    public SimpleAsyncLock NextDocumentLock { get; } = new();
+
+    public bool NextDocumentReady { get; set; }
+
     // This is different than EsclJobState.Completed; the ESCL state only transitions to completed once the client has
     // finished querying for documents. IsScanComplete is set to true immediately after the physical scan operation is
     // done.

NAPS2.Escl.Server/SimpleAsyncLock.cs

@@ -0,0 +1,37 @@
+namespace NAPS2.Escl.Server;
+
+internal class SimpleAsyncLock
+{
+    private readonly Queue<TaskCompletionSource<bool>> _listeners = new();
+    private bool _isTaken;
+
+    public Task Take()
+    {
+        lock (this)
+        {
+            if (!_isTaken)
+            {
+                _isTaken = true;
+                return Task.CompletedTask;
+            }
+            var tcs = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+            _listeners.Enqueue(tcs);
+            return tcs.Task;
+        }
+    }
+
+    public void Release()
+    {
+        lock (this)
+        {
+            if (_listeners.Count > 0)
+            {
+                _listeners.Dequeue().SetResult(true);
+            }
+            else
+            {
+                _isTaken = false;
+            }
+        }
+    }
+}

NAPS2.Escl/Client/EsclClient.cs

@@ -15,16 +15,24 @@ public class EsclClient
     private static readonly XNamespace PwgNs = EsclXmlHelper.PwgNs;
 
     // Client that verifies HTTPS certificates
-    private static readonly HttpClientHandler VerifiedHttpClientHandler = new() { MaxConnectionsPerServer = 256 };
-    private static readonly HttpClient VerifiedHttpClient = new();
+    private static readonly HttpMessageHandler VerifiedHttpClientHandler = new StandardSocketsHttpHandler
+    {
+        MaxConnectionsPerServer = 256,
+        ConnectTimeout = TimeSpan.FromSeconds(5)
+    };
+    private static readonly HttpClient VerifiedHttpClient = new(VerifiedHttpClientHandler);
 
     // Client that doesn't verify HTTPS certificates
-    private static readonly HttpClientHandler UnverifiedHttpClientHandler = new()
+    private static readonly HttpMessageHandler UnverifiedHttpClientHandler = new StandardSocketsHttpHandler
     {
         MaxConnectionsPerServer = 256,
-        // ESCL certificates are generally self-signed - we aren't trying to verify server authenticity, just ensure
-        // that the connection is encrypted and protect against passive interception.
-        ServerCertificateCustomValidationCallback = (_, _, _, _) => true
+        ConnectTimeout = TimeSpan.FromSeconds(5),
+        SslOptions =
+        {
+            // ESCL certificates are generally self-signed - we aren't trying to verify server authenticity, just ensure
+            // that the connection is encrypted and protect against passive interception.
+            RemoteCertificateValidationCallback = (_, _, _, _) => true
+        }
     };
     private static readonly HttpClient UnverifiedHttpClient = new(UnverifiedHttpClientHandler);
 
@@ -176,6 +184,10 @@ public async Task<EsclJob> CreateScanJob(EsclScanSettings settings)
                 ContentType = response.Content.Headers.ContentType?.MediaType,
                 ContentLocation = response.Content.Headers.ContentLocation?.ToString()
             };
+            if (doc.Data.Length == 0)
+            {
+                throw new Exception("ESCL response had no data, the connection may have been interrupted");
+            }
             Logger.LogDebug("GET OK: {Type} ({Bytes} bytes) {Location}", doc.ContentType, doc.Data.Length,
                 doc.ContentLocation);
             return doc;

NAPS2.Escl/NAPS2.Escl.csproj

@@ -18,6 +18,7 @@
     <ItemGroup>
         <PackageReference Include="NAPS2.Mdns" Version="1.0.1" />
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
+        <PackageReference Include="StandardSocketsHttpHandler" Version="2.2.0.8" />
         <PackageReference Include="System.Net.Http" Version="4.3.4" />
     </ItemGroup>