diff --git a/docs/docs.jpa/src/main/asciidoc/persistenceproperties_ref.adoc b/docs/docs.jpa/src/main/asciidoc/persistenceproperties_ref.adoc index 08774eb1c5c..3a7a8aefef4 100644 --- a/docs/docs.jpa/src/main/asciidoc/persistenceproperties_ref.adoc +++ b/docs/docs.jpa/src/main/asciidoc/persistenceproperties_ref.adoc @@ -1,6 +1,6 @@ /////////////////////////////////////////////////////////////////////////////// - Copyright (c) 2022, 2023 Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2022, 2025 Oracle and/or its affiliates. All rights reserved. This program and the accompanying materials are made available under the terms of the Eclipse Public License v. 2.0, which is available at @@ -213,6 +213,7 @@ EclipseLink includes the following persistence property extensions for concurrency management: * link:#concurrency-manager-allow-concurrencyexception[`concurrency.manager.allow.concurrencyexception`] +* link:#concurrency-manager-allow-getcachekeyformerge-mode[`concurrency.manager.allow.getcachekeyformerge.mode`] * link:#concurrency-manager-allow-interruptedexception[`concurrency.manager.allow.interruptedexception`] * link:#concurrency-manager-allow-readlockstacktrace[`concurrency.manager.allow.readlockstacktrace`] * link:#concurrency-manager-build-object-complete-waittime[`concurrency.manager.build.object.complete.waittime`] @@ -286,6 +287,7 @@ The following lists the EclipseLink persistence property * link:#compositeunitmember[`composite-unit.member`] * link:#compositeunitproperties[`composite-unit.properties`] * link:#concurrency-manager-allow-concurrencyexception[`concurrency.manager.allow.concurrencyexception`] +* link:#concurrency-manager-allow-getcachekeyformerge-mode[`concurrency.manager.allow.getcachekeyformerge.mode`] * link:#concurrency-manager-allow-interruptedexception[`concurrency.manager.allow.interruptedexception`] * link:#concurrency-manager-allow-readlockstacktrace[`concurrency.manager.allow.readlockstacktrace`] * link:#concurrency-manager-build-object-complete-waittime[`concurrency.manager.build.object.complete.waittime`] @@ -3103,6 +3105,55 @@ persistence.xml_* ''''' +=== concurrency.manager.allow.getcachekeyformerge.mode + +This property control in `org.eclipse.persistence.internal.sessions.AbstractSession#getCacheKeyFromTargetSessionForMerge(java.lang.Object, org.eclipse.persistence.internal.descriptors.ObjectBuilder, org.eclipse.persistence.descriptors.ClassDescriptor, org.eclipse.persistence.internal.sessions.MergeManager)` +strategy how `org.eclipse.persistence.internal.identitymaps.CacheKey` will be fetched from shared cache. + + +*Values* + +link:#concurrency.manager.allow.getcachekeyformerge.mode[Table 5-30] +describes this persistence property's values. + +[[concurrency.manager.allow.getcachekeyformerge.mode]] + +*_Table 5-30 Valid Values for +concurrency.manager.allow.getcachekeyformerge.mode_* + +|======================================================================= +|*Value* |*Description* +|ORIGIN |(Default) There is infinite `java.lang.Object.wait()` call in case +of some conditions during time when object/entity referred from +`org.eclipse.persistence.internal.identitymaps.CacheKey` is locked +and modified by another thread. In some cases it should leads into deadlock. + +|WAITLOOP |Merge manager will try in the loop with timeout wait `cacheKey.wait(ConcurrencyUtil.SINGLETON.getAcquireWaitTime());` +fetch object/entity from `org.eclipse.persistence.internal.identitymaps.CacheKey`. +If fetch will be successful object/entity loop finish and continue with remaining code. +If not `java.lang.InterruptedException` is thrown and caught and used `org.eclipse.persistence.internal.identitymaps.CacheKey` +instance status is set into invalidation state. This strategy avoid deadlock issue, +but there should be impact to the performance. +|======================================================================= + +*Examples* + +link:#concurrency.manager.allow.getcachekeyformerge.mode[Example +5-24] shows how to use this property in the `persistence.xml` file. + +[[concurrency.manager.allow.getcachekeyformerge.mode]] + +*_Example 5-24 Using concurrency.manager.allow.getcachekeyformerge.mode in +persistence.xml_* + +[source,oac_no_warn] +---- + + +---- + +''''' + === concurrency.manager.allow.concurrencyexception See valid values table. diff --git a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/MergeManagerOperationMode.java b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/MergeManagerOperationMode.java new file mode 100644 index 00000000000..25572032c0d --- /dev/null +++ b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/MergeManagerOperationMode.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0, + * or the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause + */ + +// Contributors: +// Oracle - initial API and implementation +package org.eclipse.persistence.config; + +/** + * INTERNAL: + *

+ * Purpose: It is data model behind {@linkplain org.eclipse.persistence.config.SystemProperties#CONCURRENCY_MANAGER_ALLOW_GET_CACHE_KEY_FOR_MERGE_MODE} + * or {@linkplain org.eclipse.persistence.config.PersistenceUnitProperties#CONCURRENCY_MANAGER_ALLOW_GET_CACHE_KEY_FOR_MERGE_MODE}. + */ +public final class MergeManagerOperationMode { + + /** + * {@code ORIGIN} (DEFAULT) - There is infinite {@linkplain java.lang.Object#wait()} call in case of some conditions during time when object/entity referred from + * {@code org.eclipse.persistence.internal.identitymaps.CacheKey} is locked and modified by another thread. In some cases it should leads into deadlock. + */ + public static final String ORIGIN = "ORIGIN"; + + /** + * {@code WAITLOOP} - Merge manager will try in the loop with timeout wait {@code cacheKey.wait(ConcurrencyUtil.SINGLETON.getAcquireWaitTime());} + * fetch object/entity from {@linkplain org.eclipse.persistence.internal.identitymaps.CacheKey}. If fetch will be successful object/entity loop finish and continue + * with remaining code. If not @{code java.lang.InterruptedException} is thrown and caught and used {@linkplain org.eclipse.persistence.internal.identitymaps.CacheKey} instance + * status is set into invalidation state. This strategy avoid deadlock issue, but there should be impact to the performance. + */ + public static final String WAITLOOP = "WAITLOOP"; + + private MergeManagerOperationMode() { + // no instance please + } +} diff --git a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/PersistenceUnitProperties.java b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/PersistenceUnitProperties.java index 4e92eff95cc..bd7df40b512 100644 --- a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/PersistenceUnitProperties.java +++ b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/PersistenceUnitProperties.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2024 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2025 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 1998, 2024 IBM Corporation. All rights reserved. * * This program and the accompanying materials are made available under the @@ -4105,6 +4105,23 @@ public final class PersistenceUnitProperties { */ public static final String CONCURRENCY_MANAGER_ALLOW_INTERRUPTED_EXCEPTION = "eclipselink.concurrency.manager.allow.interruptedexception"; + /** + *

+ * This property control in {@link org.eclipse.persistence.internal.sessions.AbstractSession#getCacheKeyFromTargetSessionForMerge(java.lang.Object, org.eclipse.persistence.internal.descriptors.ObjectBuilder, org.eclipse.persistence.descriptors.ClassDescriptor, org.eclipse.persistence.internal.sessions.MergeManager)} + * strategy how {@code org.eclipse.persistence.internal.identitymaps.CacheKey} will be fetched from shared cache. + *

+ * Allowed Values (case-sensitive String): + *

+ */ + public static final String CONCURRENCY_MANAGER_ALLOW_GET_CACHE_KEY_FOR_MERGE_MODE = "eclipselink.concurrency.manager.allow.getcachekeyformerge.mode"; + /** *

* This property control (enable/disable) if {@code ConcurrencyException} fired when dead-lock diagnostic is enabled. diff --git a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/SystemProperties.java b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/SystemProperties.java index b6c5f2f3db1..368770aee1d 100644 --- a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/SystemProperties.java +++ b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/config/SystemProperties.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2024 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2025 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2024 Contributors to the Eclipse Foundation. All rights reserved. * * This program and the accompanying materials are made available under the @@ -158,6 +158,23 @@ public final class SystemProperties { */ public static final String CONCURRENCY_MANAGER_ALLOW_INTERRUPTED_EXCEPTION = "eclipselink.concurrency.manager.allow.interruptedexception"; + /** + *

+ * This property control in {@link org.eclipse.persistence.internal.sessions.AbstractSession#getCacheKeyFromTargetSessionForMerge(java.lang.Object, org.eclipse.persistence.internal.descriptors.ObjectBuilder, org.eclipse.persistence.descriptors.ClassDescriptor, org.eclipse.persistence.internal.sessions.MergeManager)} + * strategy how {@code org.eclipse.persistence.internal.identitymaps.CacheKey} will be fetched from shared cache. + *

+ * Allowed Values (case-sensitive String): + *

+ */ + public static final String CONCURRENCY_MANAGER_ALLOW_GET_CACHE_KEY_FOR_MERGE_MODE = "eclipselink.concurrency.manager.allow.getcachekeyformerge.mode"; + /** *

* This property control (enable/disable) if {@code ConcurrencyException} fired when dead-lock diagnostic is enabled. @@ -173,7 +190,7 @@ public final class SystemProperties { * locks and allow other threads to progress. * */ - public static final String CONCURRENCY_MANAGER_ALLOW_CONCURRENCY_EXCEPTION = "eclipselink.concurrency.manager.allow.concurrency.exception"; + public static final String CONCURRENCY_MANAGER_ALLOW_CONCURRENCY_EXCEPTION = "eclipselink.concurrency.manager.allow.concurrencyexception"; /** *

diff --git a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/ConcurrencyManager.java b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/ConcurrencyManager.java index ea4c8a0234d..0e2a558c346 100644 --- a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/ConcurrencyManager.java +++ b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/ConcurrencyManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2024 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2025 Oracle and/or its affiliates. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v. 2.0 which is available at @@ -863,7 +863,7 @@ public void releaseAllLocksAcquiredByThread(DeferredLockManager lockManager) { * @return Never null if the read lock manager does not yet exist for the current thread. otherwise its read log * manager is returned. */ - protected static ReadLockManager getReadLockManager(Thread thread) { + public static ReadLockManager getReadLockManager(Thread thread) { Map readLockManagers = getReadLockManagers(); return readLockManagers.get(thread); } @@ -1121,4 +1121,28 @@ public Lock getInstanceLock() { public Condition getInstanceLockCondition() { return this.instanceLockCondition; } + + /** + * Check if {@code org.eclipse.persistence.internal.helper.ConcurrencyManager} or child like {@code org.eclipse.persistence.internal.identitymaps.CacheKey} is currently being owned for writing + * and if that owning thread happens to be the current thread doing the check. + * + * @return {@code false} means either the thread is currently not owned by any other thread for writing purposes. Or otherwise if is owned by some thread + * but the thread is not the current thread. {@code false} is returned if and only if instance is being owned by some thread + * and that thread is not the current thread, it is some other competing thread. + */ + public boolean isAcquiredForWritingAndOwnedByDifferentThread() { + instanceLock.lock(); + try { + if (!this.isAcquired()) { + return false; + } + if (this.activeThread == null) { + return false; + } + Thread currentThread = Thread.currentThread(); + return this.activeThread != currentThread; + } finally { + instanceLock.unlock(); + } + } } diff --git a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/ConcurrencyUtil.java b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/ConcurrencyUtil.java index c46e469d14e..1d4440187dc 100644 --- a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/ConcurrencyUtil.java +++ b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/ConcurrencyUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2024 Oracle, IBM and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2025 Oracle, IBM and/or its affiliates. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v. 2.0 which is available at @@ -15,6 +15,7 @@ // IBM - ConcurrencyUtil call of ThreadMXBean.getThreadInfo() needs doPriv package org.eclipse.persistence.internal.helper; +import org.eclipse.persistence.config.MergeManagerOperationMode; import org.eclipse.persistence.config.SystemProperties; import org.eclipse.persistence.internal.helper.type.CacheKeyToThreadRelationships; import org.eclipse.persistence.internal.helper.type.ConcurrencyManagerState; @@ -23,6 +24,7 @@ import org.eclipse.persistence.internal.identitymaps.CacheKey; import org.eclipse.persistence.internal.localization.TraceLocalization; import org.eclipse.persistence.internal.security.PrivilegedAccessHelper; +import org.eclipse.persistence.internal.sessions.AbstractSession; import org.eclipse.persistence.logging.AbstractSessionLog; import org.eclipse.persistence.logging.SessionLog; @@ -73,6 +75,7 @@ public class ConcurrencyUtil { private boolean useSemaphoreToLimitConcurrencyOnWriteLockManagerAcquireRequiredLocks = getBooleanProperty(SystemProperties.CONCURRENCY_MANAGER_USE_SEMAPHORE_TO_SLOW_DOWN_WRITE_LOCK_MANAGER_ACQUIRE_REQUIRED_LOCKS, DEFAULT_USE_SEMAPHORE_TO_SLOW_DOWN_WRITE_LOCK_MANAGER_ACQUIRE_REQUIRED_LOCKS); private int noOfThreadsAllowedToObjectBuildInParallel = getIntProperty(SystemProperties.CONCURRENCY_MANAGER_OBJECT_BUILDING_NO_THREADS, DEFAULT_CONCURRENCY_MANAGER_OBJECT_BUILDING_NO_THREADS); private int noOfThreadsAllowedToDoWriteLockManagerAcquireRequiredLocksInParallel = getIntProperty(SystemProperties.CONCURRENCY_MANAGER_WRITE_LOCK_MANAGER_ACQUIRE_REQUIRED_LOCKS_NO_THREADS, DEFAULT_CONCURRENCY_MANAGER_WRITE_LOCK_MANAGER_ACQUIRE_REQUIRED_LOCKS_NO_THREADS); + private String concurrencyManagerAllowGetCacheKeyForMergeMode = getStringProperty(SystemProperties.CONCURRENCY_MANAGER_ALLOW_GET_CACHE_KEY_FOR_MERGE_MODE, MergeManagerOperationMode.ORIGIN); private long concurrencySemaphoreMaxTimePermit = getLongProperty(SystemProperties.CONCURRENCY_SEMAPHORE_MAX_TIME_PERMIT, DEFAULT_CONCURRENCY_SEMAPHORE_MAX_TIME_PERMIT); private long concurrencySemaphoreLogTimeout = getLongProperty(SystemProperties.CONCURRENCY_SEMAPHORE_LOG_TIMEOUT, DEFAULT_CONCURRENCY_SEMAPHORE_LOG_TIMEOUT); @@ -126,11 +129,15 @@ private ConcurrencyUtil() { * is most likely too dangerous and possibly the eclipselink code is not robust enough to code with such * scenarios We do not care so much about blowing up exception during object building but during * committing of transactions we are very afraid + * @return Returns a boolean value if the code believes that it is stuck. {@code true} means the code ended up either logging + * a tiny Dump message or the massive dump message. + * {@code false} means that the code did not do any logging and just quickly returned. This boolean flag is especially + * meaningful if we have the interrupt exceptions disabled and so the method is not being caused to blow up. + * In that case there is still a way to know if we believe to be stuck. * @throws InterruptedException - * we fire an interrupted exception to ensure that the code blows up and releases all of the locks it - * had. + * it fires an interrupted exception to ensure that the code blows up and releases all the locks it had. */ - public void determineIfReleaseDeferredLockAppearsToBeDeadLocked(ConcurrencyManager concurrencyManager, + public boolean determineIfReleaseDeferredLockAppearsToBeDeadLocked(ConcurrencyManager concurrencyManager, final long whileStartTimeMillis, DeferredLockManager lockManager, ReadLockManager readLockManager, boolean callerIsWillingToAllowInterruptedExceptionToBeFiredUpIfNecessary) throws InterruptedException { @@ -143,7 +150,7 @@ public void determineIfReleaseDeferredLockAppearsToBeDeadLocked(ConcurrencyManag if (!tooMuchTimeHasElapsed) { // this thread is not stuck for that long let us allow the code to continue waiting for the lock to be acquired // or for the deferred locks to be considered as finished - return; + return false; } // (b) We believe this is a dead lock @@ -160,7 +167,7 @@ public void determineIfReleaseDeferredLockAppearsToBeDeadLocked(ConcurrencyManag // this thread has recently logged a small message about the fact that it is stuck // no point in logging another message like that for some time // let us allow for this thread to silently continue stuck without logging anything - return ; + return true; } // (c) This thread it is dead lock since the whileStartDate indicates a dead lock and @@ -202,6 +209,7 @@ public void determineIfReleaseDeferredLockAppearsToBeDeadLocked(ConcurrencyManag } else { AbstractSessionLog.getLog().log(SessionLog.SEVERE, SessionLog.CACHE,"concurrency_manager_allow_concurrency_exception_fired_up"); } + return true; } /** @@ -332,6 +340,14 @@ public void setNoOfThreadsAllowedToDoWriteLockManagerAcquireRequiredLocksInParal this.noOfThreadsAllowedToDoWriteLockManagerAcquireRequiredLocksInParallel = noOfThreadsAllowedToDoWriteLockManagerAcquireRequiredLocksInParallel; } + public String getConcurrencyManagerAllowGetCacheKeyForMergeMode() { + return concurrencyManagerAllowGetCacheKeyForMergeMode; + } + + public void setConcurrencyManagerAllowGetCacheKeyForMergeMode(String concurrencyManagerAllowGetCacheKeyForMergeMode) { + this.concurrencyManagerAllowGetCacheKeyForMergeMode = concurrencyManagerAllowGetCacheKeyForMergeMode; + } + public long getConcurrencySemaphoreMaxTimePermit() { return concurrencySemaphoreMaxTimePermit; } @@ -437,7 +453,7 @@ public boolean tooMuchTimeHasElapsed(final long whileStartTimeMillis, final long } /** - * Invoke the {@link #dumpConcurrencyManagerInformationStep01(Map, Map, Map, Map, Map, Map, Map, Set, Map, Map)} if sufficient time has passed. + * Invoke the {@link #dumpConcurrencyManagerInformationStep01(Map, Map, Map, Map, Map, Map, Map, Set, Map, Map, Map)} if sufficient time has passed. * This log message will potentially create a massive dump in the server log file. So we need to check when was the * last time that the masive dump was produced and decide if we can log again the state of the concurrency manager. *

@@ -479,6 +495,7 @@ public void dumpConcurrencyManagerInformationIfAppropriate() { Set setThreadWaitingToReleaseDeferredLocksOriginal = ConcurrencyManager.getThreadsWaitingToReleaseDeferredLocksSnapshot(); Map mapThreadsThatAreCurrentlyWaitingToReleaseDeferredLocksJustificationClone = ConcurrencyManager.getThreadsWaitingToReleaseDeferredLocksJustificationSnapshot(); Map> mapThreadToObjectIdWithWriteLockManagerChangesOriginal = WriteLockManager.getMapWriteLockManagerThreadToObjectIdsWithChangeSetSnapshot(); + Map mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys = AbstractSession.getThreadsToWaitMergeManagerWaitingDeferredCacheKeysSnapshot(); dumpConcurrencyManagerInformationStep01( deferredLockManagers, readLockManagersOriginal, @@ -489,7 +506,8 @@ public void dumpConcurrencyManagerInformationIfAppropriate() { mapThreadToWaitOnAcquireReadLockMethodNameOriginal, setThreadWaitingToReleaseDeferredLocksOriginal, mapThreadsThatAreCurrentlyWaitingToReleaseDeferredLocksJustificationClone, - mapThreadToObjectIdWithWriteLockManagerChangesOriginal); + mapThreadToObjectIdWithWriteLockManagerChangesOriginal, + mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys); } /** @@ -543,7 +561,8 @@ protected void dumpConcurrencyManagerInformationStep01(Map mapThreadToWaitOnAcquireReadLockMethodNameOriginal, Set setThreadWaitingToReleaseDeferredLocksOriginal, Map mapThreadsThatAreCurrentlyWaitingToReleaseDeferredLocksJustificationClone, - Map> mapThreadToObjectIdWithWriteLockManagerChangesOriginal) { + Map> mapThreadToObjectIdWithWriteLockManagerChangesOriginal, + Map mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys) { // (a) create object to represent our cache state. ConcurrencyManagerState concurrencyManagerState = createConcurrencyManagerState( @@ -556,7 +575,8 @@ protected void dumpConcurrencyManagerInformationStep01(Mapissue 2094. + * @param mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys + * This parameter is related to the + * {@link AbstractSession#THREADS_TO_WAIT_MERGE_MANAGER_WAITING_DEFERRED_CACHE_KEYS} + * and to the issue 2094 it allows check + * threads that are at post-commit phase and are trying to merge their change set into the original + * objects in the cache. When this is taking place some of the cache keys that the merge manager is + * needing might be locked by other threads. This can lead to deadlocks, if our merge manager thread + * happens to be the owner of cache keys that matter to the owner of the cache keys the merge manager + * will need to acquire. + * @return A string describing all threads that are stuck in the + * {@code org.eclipse.persistence.internal.sessions.AbstractSession.getCacheKeyFromTargetSessionForMergeScenarioMergeManagerNotNullAndCacheKeyOriginalStillNull(CacheKey, Object, ObjectBuilder, ClassDescriptor, MergeManager) } + * logic. There is thread that want to return to the merge manage a cacheKey where the cacheKey object is no + * longer null. The threads are stuck because the cache key object is still null and the cache key is + * acquired most likely by some random thread doing object building. Deadlocks may be occurring between the + * merge manager and the object building threads. + */ + private String createInformationAboutThreadsHavingDifficultyGettingCacheKeysWithObjectDifferentThanNullDuringMergeClonesToCacheAfterTransactionCommit( + Map mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys) { + // (a) Create a header string of information + StringWriter writer = new StringWriter(); + writer.write(TraceLocalization.buildMessage("concurrency_util_threads_having_difficulty_getting_cache_keys_with_object_different_than_null_during_merge_clones_to_cache_after_transaction_commit_page_header" + , new Object[] {mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys.size()})); + int currentThreadNumber = 0; + for (Map.Entry currentEntry : mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys.entrySet()) { + currentThreadNumber++; + Thread thread = currentEntry.getKey(); + String justification = currentEntry.getValue(); + writer.write(TraceLocalization.buildMessage("concurrency_util_threads_having_difficulty_getting_cache_keys_with_object_different_than_null_during_merge_clones_to_cache_after_transaction_commit_body", + new Object[] {currentThreadNumber, thread.getName(), justification})); + } + writer.write(TraceLocalization.buildMessage("concurrency_util_threads_having_difficulty_getting_cache_keys_with_object_different_than_null_during_merge_clones_to_cache_after_transaction_commit_page_end")); + return writer.toString(); + } + /** * create a DTO that tries to represent the current snapshot of the concurrency manager and write lock manager cache * state @@ -689,7 +752,8 @@ public ConcurrencyManagerState createConcurrencyManagerState( Map mapThreadToWaitOnAcquireReadLockMethodNameOriginal, Set setThreadWaitingToReleaseDeferredLocksOriginal, Map mapThreadsThatAreCurrentlyWaitingToReleaseDeferredLocksJustificationClone, - Map> mapThreadToObjectIdWithWriteLockManagerChangesOriginal) { + Map> mapThreadToObjectIdWithWriteLockManagerChangesOriginal, + Map mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys) { // (a) As a first step we want to clone-copy the two maps // once we start working with the maps and using them to do dead lock detection // or simply print the state of the system we do not want the maps to continue changing as the threads referenced in the maps @@ -746,7 +810,8 @@ public ConcurrencyManagerState createConcurrencyManagerState( readLockManagerMapClone, deferredLockManagerMapClone, unifiedMapOfThreadsStuckTryingToAcquireWriteLock, mapThreadToWaitOnAcquireMethodNameClone, mapThreadToWaitOnAcquireReadLockClone, mapThreadToWaitOnAcquireReadLockMethodNameClone, setThreadWaitingToReleaseDeferredLocksClone, mapThreadsThatAreCurrentlyWaitingToReleaseDeferredLocksJustificationClone, - mapOfCacheKeyToDtosExplainingThreadExpectationsOnCacheKey, mapThreadToObjectIdWithWriteLockManagerChangesClone); + mapOfCacheKeyToDtosExplainingThreadExpectationsOnCacheKey, mapThreadToObjectIdWithWriteLockManagerChangesClone, + mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys); } /** @@ -1682,4 +1747,18 @@ private boolean getBooleanProperty(final String key, final boolean defaultValue) } return defaultValue; } + + private String getStringProperty(final String key, final String defaultValue) { + final String value = PrivilegedAccessHelper.callDoPrivileged( + () -> System.getProperty(key, String.valueOf(defaultValue)) + ); + if (value != null) { + try { + return value.trim(); + } catch (Exception ignoreE) { + return defaultValue; + } + } + return defaultValue; + } } diff --git a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/type/ConcurrencyManagerState.java b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/type/ConcurrencyManagerState.java index 854b21e8f9a..d260d313c36 100644 --- a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/type/ConcurrencyManagerState.java +++ b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/helper/type/ConcurrencyManagerState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2025 Oracle and/or its affiliates. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v. 2.0 which is available at @@ -82,6 +82,17 @@ public class ConcurrencyManagerState { */ private final Map> mapThreadToObjectIdWithWriteLockManagerChangesClone; + /** + * This field is related to the + * {@link org.eclipse.persistence.internal.sessions.AbstractSession#THREADS_TO_WAIT_MERGE_MANAGER_WAITING_DEFERRED_CACHE_KEYS} + * and to the bug https://github.com/eclipse-ee4j/eclipselink/issues/2094 it allows to monitor on threads + * that are at post-commit phase and are trying to merge their change set into the original objects in the cache. + * When this is taking place some of the cache keys that the merge manager is needing might be locked by other + * threads. This can lead to deadlocks, if our merge manager thread happens to be the owner of cache keys that + * matter to the owner of the cache keys the merge manager will need to acquire. + */ + private final Map mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys; + /** * Create a new ConcurrencyManagerState. * @@ -96,7 +107,8 @@ public ConcurrencyManagerState( Set setThreadWaitingToReleaseDeferredLocksClone, Map mapThreadsThatAreCurrentlyWaitingToReleaseDeferredLocksJustificationClone, Map mapOfCacheKeyToDtosExplainingThreadExpectationsOnCacheKey, - Map> mapThreadToObjectIdWithWriteLockManagerChangesClone) { + Map> mapThreadToObjectIdWithWriteLockManagerChangesClone, + Map mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys) { super(); this.readLockManagerMapClone = readLockManagerMapClone; this.deferredLockManagerMapClone = deferredLockManagerMapClone; @@ -108,6 +120,7 @@ public ConcurrencyManagerState( this.mapThreadsThatAreCurrentlyWaitingToReleaseDeferredLocksJustificationClone = mapThreadsThatAreCurrentlyWaitingToReleaseDeferredLocksJustificationClone; this.mapOfCacheKeyToDtosExplainingThreadExpectationsOnCacheKey = mapOfCacheKeyToDtosExplainingThreadExpectationsOnCacheKey; this.mapThreadToObjectIdWithWriteLockManagerChangesClone = mapThreadToObjectIdWithWriteLockManagerChangesClone; + this.mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys = mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys; } /** Getter for {@link #readLockManagerMapClone} */ @@ -159,4 +172,9 @@ public Map getUnifiedMapOfThreadsStuckTryingToAcquireWriteLockMe public Map getMapThreadToWaitOnAcquireReadLockCloneMethodName() { return unmodifiableMap(mapThreadToWaitOnAcquireReadLockCloneMethodName); } + + /** Getter for {@link #mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys} */ + public Map getMapThreadsToWaitMergeManagerWaitingDeferredCacheKeys() { + return unmodifiableMap(mapThreadsToWaitMergeManagerWaitingDeferredCacheKeys); + } } diff --git a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/localization/i18n/TraceLocalizationResource.java b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/localization/i18n/TraceLocalizationResource.java index 74f0ca4a107..76645c790c6 100644 --- a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/localization/i18n/TraceLocalizationResource.java +++ b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/localization/i18n/TraceLocalizationResource.java @@ -318,7 +318,24 @@ The thread is working in the context of (CacheKey) = ({2}) .\s + " stopping the candidate thread to make progress... We expect this code spot to never be invoked. " + " Either this thread made progress or if it continues to be stuck in the releaseDeferredLock " + " we most likely have an implementation bug somewhere. "}, - + { "concurrency_util_threads_having_difficulty_getting_cache_keys_with_object_different_than_null_during_merge_clones_to_cache_after_transaction_commit_justification", + " Merge manager logic is currently stuck in the process of trying to return the cache key: {0} \n" + + " this cache key is currently acquired by a competing thread: {1} . \n" + + " This cache key has the problem that its original object is still set to NULL. \n" + + " The operation of this current thread is that by waiting for some time the current owner of the cache key will finish object building and release the cache key. \n" + + " Note: There is real risk that we are in a deadlock. The daedlock exists if the current thread: {2} \n" + + " is owning other cache key resources as a writer. Any lock acquired by the current thread might be needed by competing threads. \n" + + " Much in the same manner that our current thread is stuck hoping to see this specific cache key being released. \n"}, + { "concurrency_util_threads_having_difficulty_getting_cache_keys_with_object_different_than_null_during_merge_clones_to_cache_after_transaction_commit_page_header" + , "Concurrency manager - Page 08 start - Threads in MergeManager Acquiring Cache Keys for Clones to be merged into eclipselink server session cache of originals" + + "\n This section provides information about threads within the MergeManager that require cache keys for merging clones with changes." + + "\n Specifically, it focuses on the threads working in the context of an ObjectChangeSet where the server session CacheKey is found to still have CacheKy.object null," + + "\n and the CacheKey is acquired by a competing thread (typically an ObjectBuilder thread)." + + "\nTotal number of threads waiting to see lock being released: {0}\n\n"}, + { "concurrency_util_threads_having_difficulty_getting_cache_keys_with_object_different_than_null_during_merge_clones_to_cache_after_transaction_commit_body" + , "[currentThreadNumber: {0}] [ThreadName: {1}]: Justification for being stuck: {2}\n"}, + { "concurrency_util_threads_having_difficulty_getting_cache_keys_with_object_different_than_null_during_merge_clones_to_cache_after_transaction_commit_page_end" + , "Concurrency manager - Page 08 end - Threads in MergeManager Acquiring Cache Keys for Clones to be merged into eclipselink server session cache of originals\n"}, { "XML_call", "XML call" }, { "XML_data_call", "XML data call" }, { "XML_data_delete", "XML data delete" }, diff --git a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/sessions/AbstractSession.java b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/sessions/AbstractSession.java index 870b0f4590b..2f173031116 100644 --- a/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/sessions/AbstractSession.java +++ b/foundation/org.eclipse.persistence.core/src/main/java/org/eclipse/persistence/internal/sessions/AbstractSession.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2024 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2025 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2024 IBM Corporation. All rights reserved. * * This program and the accompanying materials are made available under the @@ -35,6 +35,7 @@ // - 494610: Session Properties map should be Map package org.eclipse.persistence.internal.sessions; +import org.eclipse.persistence.config.MergeManagerOperationMode; import org.eclipse.persistence.config.PersistenceUnitProperties; import org.eclipse.persistence.config.QueryHints; import org.eclipse.persistence.config.ReferenceMode; @@ -61,8 +62,11 @@ import org.eclipse.persistence.internal.databaseaccess.Platform; import org.eclipse.persistence.internal.descriptors.ObjectBuilder; import org.eclipse.persistence.internal.helper.ConcurrencyManager; +import org.eclipse.persistence.internal.helper.ConcurrencyUtil; +import org.eclipse.persistence.internal.helper.DeferredLockManager; import org.eclipse.persistence.internal.helper.Helper; import org.eclipse.persistence.internal.helper.QueryCounter; +import org.eclipse.persistence.internal.helper.ReadLockManager; import org.eclipse.persistence.internal.helper.linkedlist.ExposedNodeLinkedList; import org.eclipse.persistence.internal.history.HistoricalSession; import org.eclipse.persistence.internal.identitymaps.CacheKey; @@ -71,6 +75,7 @@ import org.eclipse.persistence.internal.indirection.ProtectedValueHolder; import org.eclipse.persistence.internal.indirection.ProxyIndirectionPolicy; import org.eclipse.persistence.internal.localization.ExceptionLocalization; +import org.eclipse.persistence.internal.localization.TraceLocalization; import org.eclipse.persistence.internal.queries.JoinedAttributeManager; import org.eclipse.persistence.internal.security.PrivilegedAccessHelper; import org.eclipse.persistence.internal.security.PrivilegedClassForName; @@ -131,6 +136,7 @@ import java.util.Map; import java.util.Set; import java.util.Vector; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; /** @@ -163,6 +169,18 @@ * @see org.eclipse.persistence.sessions.Session */ public abstract class AbstractSession extends CoreAbstractSession implements org.eclipse.persistence.sessions.Session, CommandProcessor, Serializable, Cloneable { + + /** + * See issue 2094. + * These are threads involved in the "getCacheKeyFromTargetSessionForMerge" and that detect that the cache key + * somehow still has the Object of the cache key set to null. + * If the cache key is acquired by different thread, the thread will be waiting and hoping for that cache key + * to eventually stop being acquired. + * But this process is highly risk as the thread doing the wait might be the owner of resources of the thread + * that is currently the owner of the cache key it desires. + */ + private static final Map THREADS_TO_WAIT_MERGE_MANAGER_WAITING_DEFERRED_CACHE_KEYS = new ConcurrentHashMap<>(); + /** ExceptionHandler handles database exceptions. */ transient protected ExceptionHandler exceptionHandler; @@ -2809,20 +2827,71 @@ protected CacheKey getCacheKeyFromTargetSessionForMerge(Object implementation, O getIdentityMapAccessorInstance().getWriteLockManager().transitionToDeferredLocks(mergeManager); } cacheKey.acquireDeferredLock(); - original = cacheKey.getObject(); - if (original == null) { - cacheKey.getInstanceLock().lock(); - try { - if (cacheKey.isAcquired()) { + + switch (ConcurrencyUtil.SINGLETON.getConcurrencyManagerAllowGetCacheKeyForMergeMode()) { + case MergeManagerOperationMode.ORIGIN -> { + original = cacheKey.getObject(); + if (original == null) { + cacheKey.getInstanceLock().lock(); try { - cacheKey.wait(); - } catch (InterruptedException e) { - //ignore and return + if (cacheKey.isAcquired()) { + try { + cacheKey.getInstanceLockCondition().await(); + } catch (InterruptedException e) { + //ignore and return + } + } + original = cacheKey.getObject(); + } finally { + cacheKey.getInstanceLock().unlock(); } } + } + case MergeManagerOperationMode.WAITLOOP -> { + final Thread currentThread = Thread.currentThread(); + final String currentThreadName = currentThread.getName(); + final long whileStartTimeMillis = System.currentTimeMillis(); + final DeferredLockManager lockManager = ConcurrencyManager.getDeferredLockManager(currentThread); + final ReadLockManager readLockManager = ConcurrencyManager.getReadLockManager(currentThread); + original = cacheKey.getObject(); - } finally { - cacheKey.getInstanceLock().unlock(); + boolean originalIsStillNull = original == null; + boolean isToBeStuckIntoDeadlock = false; + if (!originalIsStillNull) { + return cacheKey; + } + cacheKey.getInstanceLock().lock(); + try { + boolean someOtherThreadCurrentlyOwningTheCacheKey = cacheKey.isAcquiredForWritingAndOwnedByDifferentThread(); + if (!someOtherThreadCurrentlyOwningTheCacheKey) { + return cacheKey; + } + final String cacheKeyToStringOwnedByADifferentThread = ConcurrencyUtil.SINGLETON.createToStringExplainingOwnedCacheKey(cacheKey); + String justification = TraceLocalization.buildMessage("concurrency_util_threads_having_difficulty_getting_cache_keys_with_object_different_than_null_during_merge_clones_to_cache_after_transaction_commit_justification", + new Object[] {cacheKeyToStringOwnedByADifferentThread, cacheKey.getActiveThread(), currentThreadName}); + try { + setThreadsToWaitMergeManagerWaitingDeferredCacheKeys(justification); + while (someOtherThreadCurrentlyOwningTheCacheKey && originalIsStillNull && !isToBeStuckIntoDeadlock) { + cacheKey.getInstanceLockCondition().await(ConcurrencyUtil.SINGLETON.getAcquireWaitTime(), TimeUnit.MILLISECONDS); + isToBeStuckIntoDeadlock = ConcurrencyUtil.SINGLETON.determineIfReleaseDeferredLockAppearsToBeDeadLocked(cacheKey, whileStartTimeMillis, + lockManager, readLockManager, true); + someOtherThreadCurrentlyOwningTheCacheKey = cacheKey.isAcquiredForWritingAndOwnedByDifferentThread(); + original = cacheKey.getObject(); + originalIsStillNull = original == null; + + } + } catch (InterruptedException e) { + cacheKey.setInvalidationState(CacheKey.CACHE_KEY_INVALID); + return cacheKey; + } finally { + if (isToBeStuckIntoDeadlock) { + cacheKey.setInvalidationState(CacheKey.CACHE_KEY_INVALID); + } + } + } finally { + clearThreadsToWaitMergeManagerWaitingDeferredCacheKeys(); + cacheKey.getInstanceLock().unlock(); + } } } cacheKey.releaseDeferredLock(); @@ -5294,4 +5363,16 @@ public void setTolerateInvalidJPQL(boolean b) { public boolean shouldTolerateInvalidJPQL() { return this.tolerateInvalidJPQL; } + + public static Map getThreadsToWaitMergeManagerWaitingDeferredCacheKeysSnapshot() { + return Map.copyOf(THREADS_TO_WAIT_MERGE_MANAGER_WAITING_DEFERRED_CACHE_KEYS); + } + + public static void clearThreadsToWaitMergeManagerWaitingDeferredCacheKeys() { + THREADS_TO_WAIT_MERGE_MANAGER_WAITING_DEFERRED_CACHE_KEYS.remove(Thread.currentThread()); + } + + public static void setThreadsToWaitMergeManagerWaitingDeferredCacheKeys(String justification) { + THREADS_TO_WAIT_MERGE_MANAGER_WAITING_DEFERRED_CACHE_KEYS.put(Thread.currentThread(), justification); + } } diff --git a/jpa/eclipselink.jpa.testapps/jpa.test.diagnostic.deadlock/src/main/resources/META-INF/persistence.xml b/jpa/eclipselink.jpa.testapps/jpa.test.diagnostic.deadlock/src/main/resources/META-INF/persistence.xml index 25c07b68f16..b5e854454bc 100644 --- a/jpa/eclipselink.jpa.testapps/jpa.test.diagnostic.deadlock/src/main/resources/META-INF/persistence.xml +++ b/jpa/eclipselink.jpa.testapps/jpa.test.diagnostic.deadlock/src/main/resources/META-INF/persistence.xml @@ -1,7 +1,7 @@