[HOPS-1581] Upgrade to NDB8

smkniazi · Gautier Berthou · commit 518df7dbdace · 2020-10-22T15:34:33.000+02:00
diff --git a/NDB/clusterj-fixes/clusterj-fix.patch b/NDB/clusterj-fixes/clusterj-fix.patch
@@ -1,11 +1,12 @@
 diff --git a/storage/ndb/clusterj/clusterj-api/src/main/java/com/mysql/clusterj/DynamicObject.java b/storage/ndb/clusterj/clusterj-api/src/main/java/com/mysql/clusterj/DynamicObject.java
-index 6f2abce3..c03898ca 100644
+index 70c2c4a8..ecb927b2 100644
 --- a/storage/ndb/clusterj/clusterj-api/src/main/java/com/mysql/clusterj/DynamicObject.java
 +++ b/storage/ndb/clusterj/clusterj-api/src/main/java/com/mysql/clusterj/DynamicObject.java
-@@ -49,13 +49,13 @@ public abstract class DynamicObject {
+@@ -56,14 +56,14 @@ public abstract class DynamicObject {
          return delegate.found();
      }
  
+-    @SuppressWarnings("deprecation")
 -    protected void finalize() throws Throwable {
 -        try {
 -            if (delegate != null) {
@@ -15,21 +16,22 @@ index 6f2abce3..c03898ca 100644
 -            super.finalize();
 -        }
 -    }
-+//   protected void finalize() throws Throwable {
-+//       try {
-+//           if (delegate != null) {
-+//               delegate.release();
-+//           }
-+//       } finally {
-+//           super.finalize();
-+//       }
-+//   }
++//    @SuppressWarnings("deprecation")
++//    protected void finalize() throws Throwable {
++//        try {
++//            if (delegate != null) {
++//                delegate.release();
++//            }
++//        } finally {
++//            super.finalize();
++//        }
++//    }
  }
 diff --git a/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/SessionFactoryImpl.java b/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/SessionFactoryImpl.java
-index ac4fcf73..13cb3e1a 100644
+index 58f36b4c..5660d882 100644
 --- a/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/SessionFactoryImpl.java
 +++ b/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/SessionFactoryImpl.java
-@@ -151,6 +151,7 @@ public class SessionFactoryImpl implements SessionFactory, Constants {
+@@ -158,6 +158,7 @@ public class SessionFactoryImpl implements SessionFactory, Constants {
              // if not using connection pooling, create a new session factory
              result = new SessionFactoryImpl(props);
          }
@@ -38,10 +40,10 @@ index ac4fcf73..13cb3e1a 100644
      }
  
 diff --git a/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/metadata/DomainTypeHandlerImpl.java b/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/metadata/DomainTypeHandlerImpl.java
-index ddc926a8..70c0698d 100644
+index 16fbbb0b..9cf6fde3 100644
 --- a/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/metadata/DomainTypeHandlerImpl.java
 +++ b/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/metadata/DomainTypeHandlerImpl.java
-@@ -60,9 +60,9 @@ import java.util.Map;
+@@ -64,9 +64,9 @@ import java.util.Map;
   */
  public class DomainTypeHandlerImpl<T> extends AbstractDomainTypeHandlerImpl<T> {
  
@@ -55,19 +57,19 @@ index ddc926a8..70c0698d 100644
      /** The domain class. */
      Class<T> cls;
 @@ -124,7 +124,7 @@ public class DomainTypeHandlerImpl<T> extends AbstractDomainTypeHandlerImpl<T> {
-             // Create a proxy class for the domain class
-             // Invoke the handler's finalizer method when the proxy is finalized
-             proxyClass = (Class<T>)Proxy.getProxyClass(
--                    cls.getClassLoader(), new Class[]{cls, Finalizable.class});
-+                    cls.getClassLoader(), new Class[]{cls});
-             ctor = getConstructorForInvocationHandler (proxyClass);
+                 throw new ClusterJUserException(local.message(
+                         "ERR_Not_Persistence_Capable_Type", name));
+             }
+-            proxyInterfaces = new Class<?>[] {cls, Finalizable.class};
++            proxyInterfaces = new Class<?>[] {cls};
+             // Get the table name from Persistence Capable annotation
              persistenceCapable = cls.getAnnotation(PersistenceCapable.class);
              if (persistenceCapable == null) {
 diff --git a/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/util/JDK14LoggerImpl.java b/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/util/JDK14LoggerImpl.java
-index 833acfac..a26e0a3d 100644
+index b762b229..0ebaabcd 100644
 --- a/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/util/JDK14LoggerImpl.java
 +++ b/storage/ndb/clusterj/clusterj-core/src/main/java/com/mysql/clusterj/core/util/JDK14LoggerImpl.java
-@@ -46,39 +46,15 @@ public class JDK14LoggerImpl implements Logger {
+@@ -53,66 +53,66 @@ public class JDK14LoggerImpl implements Logger {
      }
  
      public void detail(String message) {
@@ -77,6 +79,12 @@ index 833acfac..a26e0a3d 100644
 -        String className = element.getClassName();
 -        String methodName = element.getMethodName();
 -        delegate.logp(Level.FINEST, className, methodName, message);
++        // Throwable t = new Throwable();
++        // StackTraceElement[] stack = t.getStackTrace();
++        // StackTraceElement element = stack[1];
++        // String className = element.getClassName();
++        // String methodName = element.getMethodName();
++        // delegate.logp(Level.FINEST, className, methodName, message);
      }
  
      public void debug(String message) {
@@ -86,6 +94,12 @@ index 833acfac..a26e0a3d 100644
 -        String className = element.getClassName();
 -        String methodName = element.getMethodName();
 -        delegate.logp(Level.FINER, className, methodName, message);
++        // Throwable t = new Throwable();
++        // StackTraceElement[] stack = t.getStackTrace();
++        // StackTraceElement element = stack[1];
++        // String className = element.getClassName();
++        // String methodName = element.getMethodName();
++        // delegate.logp(Level.FINER, className, methodName, message);
      }
  
      public void trace(String message) {
@@ -95,6 +109,12 @@ index 833acfac..a26e0a3d 100644
 -        String className = element.getClassName();
 -        String methodName = element.getMethodName();
 -        delegate.logp(Level.FINE, className, methodName, message);
++        // Throwable t = new Throwable();
++        // StackTraceElement[] stack = t.getStackTrace();
++        // StackTraceElement element = stack[1];
++        // String className = element.getClassName();
++        // String methodName = element.getMethodName();
++        // delegate.logp(Level.FINE, className, methodName, message);
      }
  
      public void info(String message) {
@@ -104,64 +124,90 @@ index 833acfac..a26e0a3d 100644
 -        String className = element.getClassName();
 -        String methodName = element.getMethodName();
 -        delegate.logp(Level.INFO, className, methodName, message);
++        // Throwable t = new Throwable();
++        // StackTraceElement[] stack = t.getStackTrace();
++        // StackTraceElement element = stack[1];
++        // String className = element.getClassName();
++        // String methodName = element.getMethodName();
++        // delegate.logp(Level.INFO, className, methodName, message);
      }
  
      public void warn(String message) {
-diff --git a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionImpl.java b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionImpl.java
-index da573402..924d894a 100644
---- a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionImpl.java
-+++ b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionImpl.java
-@@ -17,6 +17,9 @@
+-        Throwable t = new Throwable();
+-        StackTraceElement[] stack = t.getStackTrace();
+-        StackTraceElement element = stack[1];
+-        String className = element.getClassName();
+-        String methodName = element.getMethodName();
+-        delegate.logp(Level.WARNING, className, methodName, message);
++        // Throwable t = new Throwable();
++        // StackTraceElement[] stack = t.getStackTrace();
++        // StackTraceElement element = stack[1];
++        // String className = element.getClassName();
++        // String methodName = element.getMethodName();
++        // delegate.logp(Level.WARNING, className, methodName, message);
+     }
  
- package com.mysql.clusterj.tie;
+     public void error(String message) {
+-        Throwable t = new Throwable();
+-        StackTraceElement[] stack = t.getStackTrace();
+-        StackTraceElement element = stack[1];
+-        String className = element.getClassName();
+-        String methodName = element.getMethodName();
+-        delegate.logp(Level.SEVERE, className, methodName, message);
++        // Throwable t = new Throwable();
++        // StackTraceElement[] stack = t.getStackTrace();
++        // StackTraceElement element = stack[1];
++        // String className = element.getClassName();
++        // String methodName = element.getMethodName();
++        // delegate.logp(Level.SEVERE, className, methodName, message);
+     }
  
-+import java.net.InetAddress;
-+import java.net.UnknownHostException; 
-+
- import java.util.IdentityHashMap;
- import java.util.Iterator;
- import java.util.Map;
-@@ -132,6 +135,11 @@ public class ClusterConnectionImpl
-         byteBufferPoolForPartitionKey =
-                 new FixedByteBufferPoolImpl(PARTITION_KEY_BUFFER_SIZE, "PartitionKeyBufferPool");
-         clusterConnection = Ndb_cluster_connection.create(connectString, nodeId);
-+        try {
-+            clusterConnection.set_name(InetAddress.getLocalHost().getHostName());
-+        } catch (UnknownHostException e) {
-+            e.printStackTrace();
-+        }
-         handleError(clusterConnection, connectString, nodeId);
-         int timeoutError = clusterConnection.set_timeout(connectTimeoutMgm);
-         handleError(timeoutError, connectString, nodeId, connectTimeoutMgm);
+     public void fatal(String message) {
+-        Throwable t = new Throwable();
+-        StackTraceElement[] stack = t.getStackTrace();
+-        StackTraceElement element = stack[1];
+-        String className = element.getClassName();
+-        String methodName = element.getMethodName();
+-        delegate.logp(Level.SEVERE, className, methodName, message);
++        // Throwable t = new Throwable();
++        // StackTraceElement[] stack = t.getStackTrace();
++        // StackTraceElement element = stack[1];
++        // String className = element.getClassName();
++        // String methodName = element.getMethodName();
++        // delegate.logp(Level.SEVERE, className, methodName, message);
+     }
+ 
+ }
 diff --git a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterTransactionImpl.java b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterTransactionImpl.java
-index ffb2b6a8..8f383525 100644
+index ccedf03e..bf5aaba1 100644
 --- a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterTransactionImpl.java
 +++ b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterTransactionImpl.java
-@@ -119,6 +119,9 @@ class ClusterTransactionImpl implements ClusterTransaction {
+@@ -127,6 +127,9 @@ class ClusterTransactionImpl implements ClusterTransaction {
  
      private List<Operation> operationsToCheck = new ArrayList<Operation>();
  
-+
 +    private boolean isPartitionKeySet  = false;
 +    private final boolean HOPS_PK_FIX = true;
++
      public ClusterTransactionImpl(ClusterConnectionImpl clusterConnectionImpl,
              DbImpl db, Dictionary ndbDictionary, String joinTransactionId) {
          this.db = db;
-@@ -136,6 +139,7 @@ class ClusterTransactionImpl implements ClusterTransaction {
+@@ -144,6 +147,7 @@ class ClusterTransactionImpl implements ClusterTransaction {
          if (ndbTransaction != null) {
              ndbTransaction.close();
              ndbTransaction = null;
 +            isPartitionKeySet = false;
          }
      }
  
-@@ -644,7 +648,14 @@ class ClusterTransactionImpl implements ClusterTransaction {
+@@ -656,7 +660,14 @@ class ClusterTransactionImpl implements ClusterTransaction {
              throw new ClusterJFatalInternalException(
                      local.message("ERR_Partition_Key_Null"));
          }
+-        this.partitionKey = (PartitionKeyImpl)partitionKey;
 +        if (HOPS_PK_FIX) {
 +            if (!isPartitionKeySet) {
-         this.partitionKey = (PartitionKeyImpl)partitionKey;
++                this.partitionKey = (PartitionKeyImpl)partitionKey;
 +                isPartitionKeySet = true;
 +            }
 +        } else {
@@ -171,28 +217,31 @@ index ffb2b6a8..8f383525 100644
  
      public String getCoordinatedTransactionId() {
 diff --git a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/DbImpl.java b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/DbImpl.java
-index f39cc103..162d449e 100644
+index 6b1b396f..ee835930 100644
 --- a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/DbImpl.java
 +++ b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/DbImpl.java
-@@ -264,7 +264,9 @@ class DbImpl implements com.mysql.clusterj.core.store.Db {
+@@ -271,7 +271,11 @@ class DbImpl implements com.mysql.clusterj.core.store.Db {
              handleError (ndbTransaction, ndb);
              return ndbTransaction;
          }
 -        key_part_ptrArray = Key_part_ptrArray.create(keyPartsSize + 1);
-+	while (key_part_ptrArray == null) {
-+	    key_part_ptrArray = Key_part_ptrArray.create(keyPartsSize + 1);
-+	}
++
++	      while (key_part_ptrArray == null) {
++	          key_part_ptrArray = Key_part_ptrArray.create(keyPartsSize + 1);
++	      }
++
          try {
              // the key part pointer array has one entry for each key part
              // plus one extra for "null-terminated array concept"
 diff --git a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/NdbRecordSmartValueHandlerImpl.java b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/NdbRecordSmartValueHandlerImpl.java
-index 84e8f5f7..38c520ca 100644
+index 44366a6e..c10d4ffa 100644
 --- a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/NdbRecordSmartValueHandlerImpl.java
 +++ b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/NdbRecordSmartValueHandlerImpl.java
-@@ -76,14 +76,14 @@ public class NdbRecordSmartValueHandlerImpl implements SmartValueHandler {
+@@ -83,15 +83,15 @@ public class NdbRecordSmartValueHandlerImpl implements SmartValueHandler {
      /** Finalize this object. This method is called by the garbage collector
       * when the proxy that delegates to this object is no longer reachable.
       */
+-    @SuppressWarnings("deprecation")
 -    protected void finalize() throws Throwable {
 -        if (logger.isDetailEnabled()) logger.detail("NdbRecordSmartValueHandler.finalize");
 -        try {
@@ -201,34 +250,15 @@ index 84e8f5f7..38c520ca 100644
 -            super.finalize();
 -        }
 -    }
-+//   protected void finalize() throws Throwable {
-+//       if (logger.isDetailEnabled()) logger.detail("NdbRecordSmartValueHandler.finalize");
-+//       try {
-+//           release();
-+//       } finally {
-+//           super.finalize();
-+//       }
-+//   }
++//    @SuppressWarnings("deprecation")
++//    protected void finalize() throws Throwable {
++//        if (logger.isDetailEnabled()) logger.detail("NdbRecordSmartValueHandler.finalize");
++//        try {
++//            release();
++//        } finally {
++//            super.finalize();
++//        }
++//    }
  
      /** Release any resources associated with this object.
       * This method is called by the owner of this object.
-diff --git a/storage/ndb/src/ndbjtie/NdbApiWrapper.hpp b/storage/ndb/src/ndbjtie/NdbApiWrapper.hpp
-index bc726f5a..88e296e2 100644
---- a/storage/ndb/src/ndbjtie/NdbApiWrapper.hpp
-+++ b/storage/ndb/src/ndbjtie/NdbApiWrapper.hpp
-@@ -3517,7 +3517,15 @@ struct NdbApiWrapper {
-     Ndb_cluster_connection__set_name
-     ( Ndb_cluster_connection & obj, const char * p0 )
-     {
-+        fprintf(stdout,"\n\nLogical Clocks AB. Hops custom libndbclient.so vVERSION\n");
-+        fprintf(stdout,"Set name called. Setting Name to: %s \n",p0);
-         obj.set_name(p0);
-+        //obj.set_recv_thread_activation_threshold(0);
-+        //int th = obj.get_recv_thread_activation_threshold();
-+        //fprintf(stderr,"Recv thread activation threshold is %d\n",th);
-+        Uint16 cpu_array[1] = {0};
-+        obj.set_recv_thread_cpu(cpu_array,1,0);
-+        fprintf(stdout,"Set the cpu affinity for recv thread to CPU 0\n\n");
-     }
- 
-     static void
diff --git a/NDB/upgrade-ndb.sh b/NDB/upgrade-ndb.sh
@@ -33,11 +33,11 @@ patch -p1 < clusterj-fix.patch
 BLD=$TMP/mysql-cluster-gpl-"$V"/bld
 mkdir $BLD
 cd $BLD
-cmake .. -DBUILD_CONFIG=mysql_release -DCPACK_MONOLITHIC_INSTALL=true -DDOWNLOAD_BOOST=1 -DWITH_BOOST=/tmp
+cmake .. -DBUILD_CONFIG=mysql_release -DCPACK_MONOLITHIC_INSTALL=true -DDOWNLOAD_BOOST=1 -DWITH_BOOST=/tmp -DWITH_NDB_JAVA=ON -DWITH_NDBCLUSTER=ON
 make -j$(expr $(nproc))
 
 #deploy clusterj to kompics repo
-mvn deploy:deploy-file -Dfile=storage/ndb/clusterj/clusterj-"$V".jar -DgroupId=com.mysql.ndb -DartifactId=clusterj-hops-fix -Dversion=$V -Dpackaging=jar -DrepositoryId=Hops -Durl=https://bbc1.sics.se/archiva/repository/Hops
+mvn deploy:deploy-file -Dfile=storage/ndb/clusterj/clusterj-"$V".jar -DgroupId=com.mysql.ndb -DartifactId=clusterj-hops-fix -Dversion=$V -Dpackaging=jar -DrepositoryId=Hops -Durl=https://archiva.hops.works/repository/Hops
 
 #deploy libndbclient to kompics
 cd $SRC/../../
diff --git a/pom.xml b/pom.xml
@@ -26,13 +26,13 @@
     <dependency>
       <groupId>com.mysql.ndb</groupId>
       <artifactId>clusterj-hops-fix</artifactId>
-      <version>7.6.12</version>
+      <version>8.0.21</version>
     </dependency>
 
     <dependency>
       <groupId>mysql</groupId>
       <artifactId>mysql-connector-java</artifactId>
-      <version>5.1.18</version>
+      <version>8.0.21</version>
     </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
diff --git a/src/main/java/io/hops/metadata/ndb/mysqlserver/HopsSQLExceptionHelper.java b/src/main/java/io/hops/metadata/ndb/mysqlserver/HopsSQLExceptionHelper.java
@@ -18,7 +18,7 @@
  */
 package io.hops.metadata.ndb.mysqlserver;
 
-import com.mysql.jdbc.MysqlErrorNumbers;
+import com.mysql.cj.exceptions.MysqlErrorNumbers;
 import io.hops.exception.StorageException;
 import io.hops.exception.TransientStorageException;
 
diff --git a/src/main/java/io/hops/metadata/ndb/mysqlserver/MySQLQueryHelper.java b/src/main/java/io/hops/metadata/ndb/mysqlserver/MySQLQueryHelper.java
@@ -18,7 +18,6 @@
  */
 package io.hops.metadata.ndb.mysqlserver;
 
-import com.mysql.jdbc.exceptions.MySQLIntegrityConstraintViolationException;
 import io.hops.exception.StorageException;
 
 import java.sql.Connection;
diff --git a/src/main/resources/ndb-config.properties.template b/src/main/resources/ndb-config.properties.template
@@ -7,7 +7,7 @@ com.mysql.clusterj.connection.pool.size=1
 com.mysql.clusterj.max.transactions=1024
 #com.mysql.clusterj.connection.pool.nodeids=
 
-io.hops.metadata.ndb.mysqlserver.data_source_class_name = com.mysql.jdbc.jdbc2.optional.MysqlDataSource
+io.hops.metadata.ndb.mysqlserver.data_source_class_name = com.mysql.cj.jdbc.MysqlDataSource
 
 io.hops.metadata.ndb.mysqlserver.host=
 io.hops.metadata.ndb.mysqlserver.port=3306
