Open
Description
Engine.getSparseVectorValueCount might block 100% of the management threads in case we're facing frequent cache misses.
This causes the thread pool to queue and potentially starve important other work that's also depending on the management thread pool.
This was discovered on an internal serverless project.
Here's the corresponding hot_threads output:
0.3% [cpu=0.3%, other=0.0%] (1.3ms out of 500ms) cpu usage by thread 'elasticsearch[es-es-index-546895bb95-9l8qb][management][T#2]'
10/10 snapshots sharing following 98 elements
java.base@24/jdk.internal.misc.Unsafe.park(Native Method)
java.base@24/java.util.concurrent.locks.LockSupport.park(LockSupport.java:223)
java.base@24/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:789)
java.base@24/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1138)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.PlainActionFuture$Sync.get(PlainActionFuture.java:265)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.PlainActionFuture.get(PlainActionFuture.java:96)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.shared.SharedBlobCacheService$CacheFile.readSingleRegion(SharedBlobCacheService.java:1252)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.shared.SharedBlobCacheService$CacheFile.populateAndRead(SharedBlobCacheService.java:1229)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.cache.reader.CacheFileReader.doRead(CacheFileReader.java:124)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.cache.reader.CacheFileReader.read(CacheFileReader.java:94)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.readInternalSlow(BlobCacheIndexInput.java:175)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.doReadInternal(BlobCacheIndexInput.java:161)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.readInternal(BlobCacheIndexInput.java:131)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.refill(BlobCacheBufferedIndexInput.java:304)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.readByte(BlobCacheBufferedIndexInput.java:57)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.store.DataInput.readVInt(DataInput.java:121)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.readVInt(BlobCacheBufferedIndexInput.java:172)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:174)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.lambda$prepareSeekExact$1(SegmentTermsEnum.java:511)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum$$Lambda/0x0000000030b2a4a8.get(Unknown Source)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:599)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.engine.Engine.getSparseVectorValueCount(Engine.java:395)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.engine.Engine.sparseVectorStats(Engine.java:379)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.engine.Engine.sparseVectorStats(Engine.java:369)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.shard.IndexShard.sparseVectorStats(IndexShard.java:1482)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.admin.indices.stats.CommonStats.getShardLevelStats(CommonStats.java:192)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:564)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.stats(IndicesService.java:489)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.node.NodeService.stats(NodeService.java:188)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:161)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:47)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperationAsync(TransportNodesAction.java:245)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.lambda$messageReceived$0(TransportNodesAction.java:261)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler$$Lambda/0x00000000308c7978.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListener.run(ActionListener.java:465)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:579)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:632)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:621)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:470)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x0000000030802fa8.accept(Unknown Source)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:1062)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:1028)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$9(AuthorizationService.java:484)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x0000000030803438.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:211)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:474)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:450)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$3(AuthorizationService.java:337)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x00000000307f4fe8.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:258)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$0(RBACEngine.java:176)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda/0x00000000307f5478.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRoles$4(CompositeRolesStore.java:213)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x00000000307f56a8.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRole$6(CompositeRolesStore.java:232)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x000000003089b768.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.lambda$buildRole$0(RoleReferenceIntersection.java:49)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection$$Lambda/0x000000003087a7e8.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.GroupedActionListener.onResponse(GroupedActionListener.java:57)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.buildRoleFromRoleReference(CompositeRolesStore.java:330)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRole$5(CompositeRolesStore.java:231)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x000000003089b520.accept(Unknown Source)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.lambda$buildRole$1(RoleReferenceIntersection.java:53)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection$$Lambda/0x000000003087aa18.accept(Unknown Source)
java.base@24/java.lang.Iterable.forEach(Iterable.java:75)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.buildRole(RoleReferenceIntersection.java:53)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRole(CompositeRolesStore.java:230)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:206)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:172)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:353)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.ServerTransportFilter.lambda$inbound$1(ServerTransportFilter.java:114)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.ServerTransportFilter$$Lambda/0x0000000030639ab8.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$MappedActionListener.onResponse(ActionListenerImplementations.java:111)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticate(AuthenticatorChain.java:93)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:271)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:203)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.ServerTransportFilter.authenticate(ServerTransportFilter.java:127)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.ServerTransportFilter.inbound(ServerTransportFilter.java:105)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:643)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:90)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:1092)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:1044)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
0.2% [cpu=0.2%, other=0.0%] (933.6micros out of 500ms) cpu usage by thread 'elasticsearch[es-es-index-546895bb95-9l8qb][management][T#1]'
10/10 snapshots sharing following 98 elements
java.base@24/jdk.internal.misc.Unsafe.park(Native Method)
java.base@24/java.util.concurrent.locks.LockSupport.park(LockSupport.java:223)
java.base@24/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:789)
java.base@24/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1138)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.PlainActionFuture$Sync.get(PlainActionFuture.java:265)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.PlainActionFuture.get(PlainActionFuture.java:96)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.shared.SharedBlobCacheService$CacheFile.readSingleRegion(SharedBlobCacheService.java:1252)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.shared.SharedBlobCacheService$CacheFile.populateAndRead(SharedBlobCacheService.java:1229)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.cache.reader.CacheFileReader.doRead(CacheFileReader.java:124)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.cache.reader.CacheFileReader.read(CacheFileReader.java:94)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.readInternalSlow(BlobCacheIndexInput.java:175)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.doReadInternal(BlobCacheIndexInput.java:161)
org.elasticsearch.stateless@9.1.0/co.elastic.elasticsearch.stateless.lucene.BlobCacheIndexInput.readInternal(BlobCacheIndexInput.java:131)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.refill(BlobCacheBufferedIndexInput.java:304)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.readByte(BlobCacheBufferedIndexInput.java:57)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.store.DataInput.readVInt(DataInput.java:121)
org.elasticsearch.blobcache@9.1.0/org.elasticsearch.blobcache.common.BlobCacheBufferedIndexInput.readVInt(BlobCacheBufferedIndexInput.java:172)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:174)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.lambda$prepareSeekExact$1(SegmentTermsEnum.java:511)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum$$Lambda/0x0000000030b2a4a8.get(Unknown Source)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:599)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.engine.Engine.getSparseVectorValueCount(Engine.java:395)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.engine.Engine.sparseVectorStats(Engine.java:379)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.engine.Engine.sparseVectorStats(Engine.java:369)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.shard.IndexShard.sparseVectorStats(IndexShard.java:1482)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.admin.indices.stats.CommonStats.getShardLevelStats(CommonStats.java:192)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:564)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.stats(IndicesService.java:489)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.node.NodeService.stats(NodeService.java:188)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:161)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:47)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperationAsync(TransportNodesAction.java:245)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.lambda$messageReceived$0(TransportNodesAction.java:261)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler$$Lambda/0x00000000308c7978.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListener.run(ActionListener.java:465)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:579)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:632)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$3.onResponse(SecurityServerTransportInterceptor.java:621)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:470)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x0000000030802fa8.accept(Unknown Source)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:1062)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:1028)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$9(AuthorizationService.java:484)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x0000000030803438.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:211)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:474)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:450)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$3(AuthorizationService.java:337)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda/0x00000000307f4fe8.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:258)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:33)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$0(RBACEngine.java:176)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda/0x00000000307f5478.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRoles$4(CompositeRolesStore.java:213)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x00000000307f56a8.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRole$6(CompositeRolesStore.java:232)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x000000003089b768.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.lambda$buildRole$0(RoleReferenceIntersection.java:49)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection$$Lambda/0x000000003087a7e8.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.support.GroupedActionListener.onResponse(GroupedActionListener.java:57)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.buildRoleFromRoleReference(CompositeRolesStore.java:330)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRole$5(CompositeRolesStore.java:231)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore$$Lambda/0x000000003089b520.accept(Unknown Source)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.lambda$buildRole$1(RoleReferenceIntersection.java:53)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection$$Lambda/0x000000003087aa18.accept(Unknown Source)
java.base@24/java.lang.Iterable.forEach(Iterable.java:75)
org.elasticsearch.xcore@9.1.0/org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection.buildRole(RoleReferenceIntersection.java:53)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRole(CompositeRolesStore.java:230)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:206)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:172)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:353)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.ServerTransportFilter.lambda$inbound$1(ServerTransportFilter.java:114)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.ServerTransportFilter$$Lambda/0x0000000030639ab8.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$ResponseWrappingActionListener.onResponse(ActionListenerImplementations.java:261)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.action.ActionListenerImplementations$MappedActionListener.onResponse(ActionListenerImplementations.java:111)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authc.AuthenticatorChain.authenticate(AuthenticatorChain.java:93)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:271)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:203)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.ServerTransportFilter.authenticate(ServerTransportFilter.java:127)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.ServerTransportFilter.inbound(ServerTransportFilter.java:105)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:643)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:90)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:1092)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:1044)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)