Fixes and enhancements for Azure Functions (isolated worker) (#2505)

This PR fixes several things in Azure Functions (isolated worker) which occur when using the latest templates for Azure Functions. - Newer versions of the Functions library prefer the gRPC-based implementation, which throws when `Url` is accessed. Instead, we read these from other data on the `FunctionContext`. - Address Functions library changes that break distributed tracing. We now parse the original headers from the `BindingContext` instead of the request, which may contain a `traceparent` with the sampling flag set to false when the user request does not include a specific `traceparent`. - We explicitly don't record activities from the Azure functions library as these are pretty broken (Azure/azure-functions-dotnet-worker#2733) and are redundant when using our middleware. - Downgrade several packages as the newer ones are now deprecated (thanks for the confusion, Microsoft!). - Update some outdated compiler pre-processor directives. - The final few commits focus on CI integration test hangs on Linux. We don't have a perfect solution for those, but after reviewing the hang dumps, I've avoided some of the potential causes of the hangs. We'll monitor subsequent PRs, and if they remain stable, we will readdress the original causes. A follow-up PR will update our documentation. Closes #2407 Closes #2311 Closes #2218
elastic · Nov 29, 2024 · 8edcb49 · 8edcb49
1 parent ba985d6
commit 8edcb49
Show file tree

Hide file tree

Showing 17 changed files with 171 additions and 56 deletions.
diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml
@@ -126,11 +126,14 @@ jobs:
 
       - name: Store crash dumps
         uses: actions/upload-artifact@v4
-        if: success() || failure()
+        if: failure()
         with:
-          name: results
-          retention-days: 1
-          path: build/output/**/*.dmp
+          name: hang-dumps
+          retention-days: 3
+          path: |
+            build/output/**/*.dmp
+            build/output/**/*.xml
+            build/output/**/*.pdb
     
   startup-hook-tests:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml
@@ -76,9 +76,11 @@ jobs:
         uses: actions/upload-artifact@v4
         if: success() || failure()
         with:
-          name: results
-          retention-days: 1
-          path: build/output/**/*.dmp
+          name: hang-dumps
+          retention-days: 3
+          path: |
+            build/output/**/*.dmp
+            build/output/**/*.xml
   
   startup-hook-tests:
     runs-on: windows-2022

diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -84,22 +84,23 @@
     <PackageVersion Include="Microsoft.AspNet.TelemetryCorrelation" Version="1.0.7" />
     <PackageVersion Include="Microsoft.AspNet.Web.Optimization" Version="1.1.3" />
     <PackageVersion Include="Microsoft.AspNet.WebApi" Version="5.2.4" />
-    <PackageVersion Include="Microsoft.AspNetCore.Hosting.Abstractions" Version="2.2.0" />
-    <PackageVersion Include="Microsoft.AspNetCore.Http" Version="2.2.2" />
-    <PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
-    <PackageVersion Include="Microsoft.AspNetCore.Http.Extensions" Version="2.2.0" />
+    <PackageVersion Include="Microsoft.AspNetCore.Hosting.Abstractions" Version="2.1.1" />
+    <PackageVersion Include="Microsoft.AspNetCore.Http" Version="2.1.34" />
+    <PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.1.1" />
+    <PackageVersion Include="Microsoft.AspNetCore.Http.Extensions" Version="2.1.21" />
     <PackageVersion Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="8.0.2" />
     <PackageVersion Include="Microsoft.AspNetCore.Identity.UI" Version="8.0.2" />
     <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.2" />
     <PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.2" />
     <PackageVersion Include="Microsoft.AspNetCore.Routing.Abstractions" Version="2.2.0" />
     <PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.0.0" />
     <PackageVersion Include="Microsoft.Azure.DocumentDB.Core" Version="2.22.0" />
-    <PackageVersion Include="Microsoft.Azure.Functions.Extensions" Version="1.0.0" />
-    <PackageVersion Include="Microsoft.Azure.Functions.Worker" Version="1.20.0" />
-    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Core" Version="1.6.0" />
-    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.1.0" />
-    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.16.1" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Extensions" Version="1.1.0" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Core" Version="2.0.0" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.2.0" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.0" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.0.0" />
     <PackageVersion Include="Microsoft.Azure.ServiceBus" Version="3.0.0" />
     <PackageVersion Include="Microsoft.Azure.Storage.Blob" Version="11.2.2" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="5.1.5" />

diff --git a/src/Elastic.Apm/AgentComponents.cs b/src/Elastic.Apm/AgentComponents.cs
@@ -17,7 +17,7 @@
 using Elastic.Apm.Metrics.MetricsProvider;
 using Elastic.Apm.Report;
 using Elastic.Apm.ServerInfo;
-#if NET5_0_OR_GREATER
+#if NET8_0_OR_GREATER
 using Elastic.Apm.OpenTelemetry;
 #endif
 
@@ -60,7 +60,7 @@ internal AgentComponents(
 				ApmServerInfo = apmServerInfo ?? new ApmServerInfo();
 				HttpTraceConfiguration = new HttpTraceConfiguration();
 
-#if NET5_0_OR_GREATER
+#if NET8_0_OR_GREATER
 				// Initialize early because ServerInfoCallback requires it and might execute
 				// before EnsureElasticActivityStarted runs
 				ElasticActivityListener = new ElasticActivityListener(this, HttpTraceConfiguration);
@@ -81,7 +81,7 @@ internal AgentComponents(
 					currentExecutionSegmentsContainer ?? new CurrentExecutionSegmentsContainer(), ApmServerInfo,
 					breakdownMetricsProvider);
 
-#if NET5_0_OR_GREATER
+#if NET8_0_OR_GREATER
 				EnsureElasticActivityStarted();
 #endif
 
@@ -118,7 +118,7 @@ internal AgentComponents(
 
 		private void EnsureElasticActivityStarted()
 		{
-#if !NET5_0_OR_GREATER
+#if !NET8_0_OR_GREATER
 			return;
 #else
 			if (!Configuration.OpenTelemetryBridgeEnabled) return;
@@ -145,7 +145,7 @@ private void EnsureElasticActivityStarted()
 
 		private void ServerInfoCallback(bool success, IApmServerInfo serverInfo)
 		{
-#if !NET5_0_OR_GREATER
+#if !NET8_0_OR_GREATER
 			return;
 #else
 			if (!Configuration.OpenTelemetryBridgeEnabled) return;
@@ -237,7 +237,7 @@ internal static IApmLogger GetGlobalLogger(IApmLogger fallbackLogger, LogLevel a
 			return fallbackLogger;
 		}
 
-#if NET5_0_OR_GREATER
+#if NET8_0_OR_GREATER
 		private ElasticActivityListener ElasticActivityListener { get; }
 #endif
 
@@ -281,7 +281,7 @@ public void Dispose()
 			if (PayloadSender is IDisposable disposablePayloadSender)
 				disposablePayloadSender.Dispose();
 			CentralConfigurationFetcher?.Dispose();
-#if NET5_0_OR_GREATER
+#if NET8_0_OR_GREATER
 			ElasticActivityListener?.Dispose();
 #endif
 		}

diff --git a/src/Elastic.Apm/Cloud/AzureFunctionsMetadataProvider.cs b/src/Elastic.Apm/Cloud/AzureFunctionsMetadataProvider.cs
@@ -59,7 +59,16 @@ internal static AzureFunctionsMetaData GetAzureFunctionsMetaData(IApmLogger logg
 					functionsExtensionVersion) ||
 				helper.NullOrEmptyVariable(AzureEnvironmentVariables.WebsiteOwnerName, websiteOwnerName) ||
 				helper.NullOrEmptyVariable(AzureEnvironmentVariables.WebsiteSiteName, websiteSiteName))
-				return new AzureFunctionsMetaData { IsValid = false };
+				return new AzureFunctionsMetaData
+				{
+					IsValid = false,
+					RegionName = regionName,
+					FunctionsExtensionVersion = functionsExtensionVersion,
+					FunctionsWorkerRuntime = functionsWorkerRuntime,
+					WebsiteSiteName = websiteSiteName,
+					WebsiteResourceGroup = websiteResourceGroup,
+					WebsiteInstanceId = websiteInstanceId,
+				};
 
 			var tokens = helper.TokenizeWebSiteOwnerName(websiteOwnerName);
 			if (!tokens.HasValue)

diff --git a/src/Elastic.Apm/OpenTelemetry/ElasticActivityListener.cs b/src/Elastic.Apm/OpenTelemetry/ElasticActivityListener.cs
@@ -71,6 +71,17 @@ internal void Start(Tracer tracerInternal)
 		private Action<Activity> ActivityStarted =>
 			activity =>
 			{
+				// Prevent recording of Azure Functions activities which are quite broken at the moment
+				// See https://github.com/Azure/azure-functions-dotnet-worker/issues/2733
+				// See https://github.com/Azure/azure-functions-dotnet-worker/issues/2875
+				// See https://github.com/Azure/azure-functions-host/issues/10641
+				// See https://github.com/Azure/azure-functions-dotnet-worker/issues/2810
+				if ((activity.Source.Name == "" && activity.DisplayName == "InvokeFunctionAsync")
+					|| (activity.Source.Name == "Microsoft.Azure.Functions.Worker"))
+				{
+					return;
+				}
+
 				// If the Elastic instrumentation for ServiceBus is present, we skip duplicating the instrumentation through the OTel bridge.
 				// Without this, we end up with some redundant spans in the trace with subtle differences.
 				if (HasServiceBusInstrumentation && activity.Tags.Any(kvp =>

diff --git a/src/Elastic.Apm/Report/PayloadSenderV2.cs b/src/Elastic.Apm/Report/PayloadSenderV2.cs
@@ -398,7 +398,7 @@ private void ProcessQueueItems(object[] queueItems)
 				{
 					content.Headers.ContentType = MediaTypeHeaderValue;
 
-#if NET5_0_OR_GREATER
+#if NET8_0_OR_GREATER
 					HttpResponseMessage response;
 					try
 					{

diff --git a/src/azure/Elastic.Apm.Azure.Functions/ApmMiddleware.cs b/src/azure/Elastic.Apm.Azure.Functions/ApmMiddleware.cs
@@ -10,6 +10,8 @@
 using Elastic.Apm.Extensions;
 using Elastic.Apm.Logging;
 using Elastic.Apm.Model;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Extensions;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Azure.Functions.Worker.Http;
 using Microsoft.Azure.Functions.Worker.Middleware;
@@ -56,24 +58,88 @@ await Agent.Tracer.CaptureTransaction(data.Name, ApiConstants.TypeRequest, async
 		}, data.TracingData);
 	}
 
+	private const string TraceParentHeader = "\"traceparent\":";
+	private const string TraceStateHeader = "\"tracestate\":";
+
 	private static TriggerSpecificData GetTriggerSpecificData(FunctionContext context)
 	{
-		var httpRequestData = GetHttpRequestData(context);
-		if (httpRequestData != null) // HTTP Trigger
+		try
 		{
-			Context.Logger.Trace()?.Log("HTTP Trigger type detected.");
+			string? traceparent = null;
+			string? tracestate = null;
+
+			// NOTE: We parse the original headers from BindingData as internally the Host sends the GrpcWorker a request. When no traceparent
+			// is present on the original request to the Functions host, one is added, with the sampled flag set as false. This is then present
+			// if we try to read it from DefaultHttpContext which means we don't record transactions. Currently this works around that issue to
+			// ensure we capture the trace based on the original request to the Functions host.
+			if (context.BindingContext.BindingData.TryGetValue("Headers", out var headers) && headers is string headersAsString)
+			{
+				var span = headersAsString.AsSpan();
+				var indexOfTraceparent = span.IndexOf(TraceParentHeader.AsSpan(), StringComparison.OrdinalIgnoreCase);
+
+				if (indexOfTraceparent > -1)
+				{
+					var traceparentSpan = span.Slice(indexOfTraceparent + TraceParentHeader.Length + 1);
+					var endOfTraceparentSpan = traceparentSpan.IndexOf('"');
+					traceparentSpan = traceparentSpan.Slice(0, endOfTraceparentSpan);
+
+					if (traceparentSpan.Length == 55)
+						traceparent = traceparentSpan.ToString();
+				}
 
-			httpRequestData.Headers.TryGetValues("traceparent", out var traceparent);
-			httpRequestData.Headers.TryGetValues("tracestate", out var tracestate);
+				var indexOfTracestate = span.IndexOf(TraceStateHeader.AsSpan(), StringComparison.OrdinalIgnoreCase);
 
-			return new($"{httpRequestData.Method} {httpRequestData.Url.AbsolutePath}", Trigger.TypeHttp,
-				TraceContext.TryExtractTracingData(traceparent?.FirstOrDefault(), tracestate?.FirstOrDefault()))
+				if (indexOfTracestate > -1)
+				{
+					var tracestateSpan = span.Slice(indexOfTracestate + TraceStateHeader.Length + 1);
+					var endOfTracestateSpan = tracestateSpan.IndexOf('"');
+					tracestate = tracestateSpan.Slice(0, endOfTracestateSpan).ToString();
+				}
+			}
+
+			var httpRequestContext = context.Items
+				.Where(i => i.Key is string s && s.Equals("HttpRequestContext", StringComparison.Ordinal))
+				.Select(i => i.Value)
+				.SingleOrDefault();
+
+			if (httpRequestContext is DefaultHttpContext requestContext)
 			{
-				Request = new Request(httpRequestData.Method, Url.FromUri(httpRequestData.Url))
+				Context.Logger.Trace()?.Log("HTTP Trigger type detected.");
+
+				var httpRequest = requestContext.Request;
+
+				if (Uri.TryCreate(httpRequest.GetDisplayUrl(), UriKind.Absolute, out var uri))
 				{
-					Headers = CreateHeadersDictionary(httpRequestData.Headers),
+					return new($"{httpRequest.Method} {uri.AbsolutePath}", Trigger.TypeHttp,
+						TraceContext.TryExtractTracingData(traceparent, tracestate))
+					{
+						Request = new Request(httpRequest.Method, Url.FromUri(uri))
+						{
+							Headers = CreateHeadersDictionary(httpRequest.Headers),
+						}
+					};
 				}
-			};
+			}
+
+			// This is the original code, left as a fallback for older versions of the Functions library
+			var httpRequestData = GetHttpRequestData(context);
+			if (httpRequestData != null) // HTTP Trigger
+			{
+				Context.Logger.Trace()?.Log("HTTP Trigger type detected.");
+
+				return new($"{httpRequestData.Method} {httpRequestData.Url.AbsolutePath}", Trigger.TypeHttp,
+					TraceContext.TryExtractTracingData(traceparent, tracestate))
+				{
+					Request = new Request(httpRequestData.Method, Url.FromUri(httpRequestData.Url))
+					{
+						Headers = CreateHeadersDictionary(httpRequestData.Headers),
+					}
+				};
+			}
+		}
+		catch
+		{
+			// ignored
 		}
 
 		// Generic
@@ -114,6 +180,9 @@ private static void SetTriggerSpecificResult(ITransaction transaction, bool succ
 	private static Dictionary<string, string> CreateHeadersDictionary(HttpHeadersCollection httpHeadersCollection) =>
 		httpHeadersCollection.ToDictionary(h => h.Key, h => string.Join(",", h.Value));
 
+	private static Dictionary<string, string> CreateHeadersDictionary(IHeaderDictionary headerDictionary) =>
+		headerDictionary.ToDictionary(h => h.Key, h => string.Join(",", h.Value.AsEnumerable()));
+
 	private static HttpRequestData? GetHttpRequestData(FunctionContext functionContext)
 	{
 		var inputData = GetProperty<IReadOnlyDictionary<string, object>>(functionContext, "InputData");

diff --git a/src/azure/Elastic.Apm.Azure.Functions/Elastic.Apm.Azure.Functions.csproj b/src/azure/Elastic.Apm.Azure.Functions/Elastic.Apm.Azure.Functions.csproj
@@ -1,9 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFrameworks>netstandard2.0;net8.0</TargetFrameworks>
     <Nullable>enable</Nullable>
-
     <AssemblyName>Elastic.Apm.Azure.Functions</AssemblyName>
     <RootNamespace>Elastic.Apm.Azure.Functions</RootNamespace>
     <PackageId>Elastic.Apm.Azure.Functions</PackageId>
@@ -18,9 +17,17 @@
     <ProjectReference Include="..\..\Elastic.Apm\Elastic.Apm.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(TargetFramework)' != 'netstandard2.0'">
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
     <PackageReference Include="Microsoft.AspNetCore.Http"/>
     <PackageReference Include="Microsoft.AspNetCore.Http.Extensions"/>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="Microsoft.Azure.Functions.Extensions"/>
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Core" />
   </ItemGroup>

diff --git a/src/integrations/Elastic.Apm.AspNetCore/Elastic.Apm.AspNetCore.csproj b/src/integrations/Elastic.Apm.AspNetCore/Elastic.Apm.AspNetCore.csproj
@@ -20,8 +20,8 @@
     <PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" />
     <PackageReference Include="Microsoft.AspNetCore.Http.Extensions" />
     <PackageReference Include="Microsoft.AspNetCore.Hosting.Abstractions" />
-    <PackageReference Include="Microsoft.AspNetCore.Routing.Abstractions" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" VersionOverride="2.2.0" />
+    <PackageReference Include="Microsoft.AspNetCore.Routing.Abstractions" VersionOverride="2.1.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" VersionOverride="2.1.1" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'net8.0' ">
     <FrameworkReference Include="Microsoft.AspNetCore.App" />

diff --git a/src/integrations/Elastic.Apm.Extensions.Hosting/Elastic.Apm.Extensions.Hosting.csproj b/src/integrations/Elastic.Apm.Extensions.Hosting/Elastic.Apm.Extensions.Hosting.csproj
@@ -21,10 +21,10 @@
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
-    <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" VersionOverride="2.2.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" VersionOverride="2.2.0" />
-    <PackageReference Include="Microsoft.Extensions.DependencyInjection" VersionOverride="2.2.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Configuration" VersionOverride="2.2.0" PrivateAssets="All" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" VersionOverride="2.1.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" VersionOverride="2.1.1" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" VersionOverride="2.1.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Configuration" VersionOverride="2.1.1" PrivateAssets="All" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/test/Elastic.Apm.Tests.Utilities/XUnit/DockerAttributes.cs b/test/Elastic.Apm.Tests.Utilities/XUnit/DockerAttributes.cs
@@ -67,7 +67,7 @@ static DockerUtils()
 	{
 		try
 		{
-			var result = Proc.Start(new StartArguments("docker", "--version"));
+			var result = Proc.Start(new StartArguments("docker", "--version"), TimeSpan.FromSeconds(30));
 			HasDockerInstalled = result.ExitCode == 0;
 		}
 		catch (Exception)

diff --git a/...e/applications/Elastic.AzureFunctionApp.Isolated/Elastic.AzureFunctionApp.Isolated.csproj b/...e/applications/Elastic.AzureFunctionApp.Isolated/Elastic.AzureFunctionApp.Isolated.csproj
@@ -8,13 +8,11 @@
     <IsPackable>false</IsPackable>
   </PropertyGroup>
   <ItemGroup>
-    <!-- 
-    The versions below are NOT the latest but anything newer breaks when attempting to run func
-    See: https://github.com/Azure/azure-functions-core-tools/issues/3594
-    -->
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker"/>
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk"/>
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http"/>
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" />
   </ItemGroup>
   <ItemGroup>
     <None Update="host.json">

diff --git a/test/azure/applications/Elastic.AzureFunctionApp.Isolated/HttpTrigger.cs b/test/azure/applications/Elastic.AzureFunctionApp.Isolated/HttpTrigger.cs
@@ -14,7 +14,7 @@ namespace Elastic.AzureFunctionApp.Isolated;
 public static class HttpTriggers
 {
 	[Function(FunctionName.SampleHttpTrigger)]
-	public static HttpResponseData Run([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req,
+	public static async Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req,
 		FunctionContext executionContext)
 	{
 		var logger = executionContext.GetLogger("SampleHttpTrigger");
@@ -23,11 +23,11 @@ public static HttpResponseData Run([HttpTrigger(AuthorizationLevel.Function, "ge
 		var response = req.CreateResponse(HttpStatusCode.OK);
 		response.Headers.Add("Content-Type", "text/plain; charset=utf-8");
 
-		response.WriteString("Hello Azure Functions!\n");
-		response.WriteString("======================\n");
+		await response.WriteStringAsync("Hello Azure Functions!\n");
+		await response.WriteStringAsync("======================\n");
 		foreach (DictionaryEntry e in Environment.GetEnvironmentVariables())
-			response.WriteString($"{e.Key} = {e.Value}\n");
-		response.WriteString("======================\n");
+			await response.WriteStringAsync($"{e.Key} = {e.Value}\n");
+		await response.WriteStringAsync("======================\n");
 
 		return response;
 	}