Merge pull request #24 from sranka/feat/jsonb-support

feat: support postgresql types that map to JDBC OTHER type
sranka · Feb 12, 2025 · e969402 · e969402
2 parents 4b16de3 + d13d11f
commit e969402
Show file tree

Hide file tree

Showing 9 changed files with 51 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -16,13 +16,14 @@ The tool ignores missing tables and columns when importing the data.
    * *password* 
 3. Export to a zip file
    * jdbcimage export -url=jdbc:mariadb://localhost:3306/qa -user=root -password=root mysql.zip
-   * jdbcimage export -url=jdbc:postgresql://localhost:5432/inttests?currentSchema=qa -user=postgres -password=postres postgres.zip
+   * jdbcimage export -url=jdbc:postgresql://localhost:5432/inttests?stringtype=unspecified&currentSchema=qa -user=postgres -password=postres postgres.zip
    * jdbcimage export -url=jdbc:oracle:thin:@localhost:1521:XE -user=system -password=changeit oracle.zip
    * jdbcimage export -url=jdbc:sqlserver://localhost:1433;databaseName=XE -user=sa -password=changeit sqlserver.zip
 4. Import from a zip file
    * BEWARE: !!!import deletes data from all tables contained in the imported zip file!!!
    * jdbcimage import -url=jdbc:mariadb://localhost:3306/qa -user=root -password=root -ignored_tables=SCHEMAVERSION postgres.zip
-   * jdbcimage import -url=jdbc:postgresql://localhost:5432/inttests?currentSchema=qa -user=postgres -password=postres -ignored_tables=schemaversion mysql.zip
+   * jdbcimage import -url=jdbc:postgresql://localhost:5432/inttests?stringtype=unspecified&currentSchema=qa -user=postgres -password=postres -ignored_tables=schemaversion mysql.zip
+      * `stringtype=unspecified` is required in the connection string in order to import data that map to JDBC OTHER type, such as json or jsonb  
    * jdbcimage -Xmx1024m import -url=jdbc:oracle:thin:@localhost:1521:XE -user=system -password=changeit -ignored_tables=SCHEMAVERSION mysql.zip
    * jdbcimage import -url=jdbc:sqlserver://localhost:1433;databaseName=XE -user=sa -password=changeit -ignored_tables=SCHEMAVERSION mysql.zip
 5. Take a look at table data in a zip file

diff --git a/lib/postgresql-42.7.5.jar b/lib/postgresql-42.7.5.jar
diff --git a/lib/postgresql-9.4.jar b/lib/postgresql-9.4.jar
diff --git a/pom.xml b/pom.xml
@@ -34,7 +34,7 @@
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-dbcp2</artifactId>
-			<version>2.1.1</version>
+			<version>2.9.0</version>
 		</dependency>
 		<!-- logging -->
 		<dependency>

diff --git a/src/main/java/io/github/sranka/jdbcimage/db/DbImportResultConsumer.java b/src/main/java/io/github/sranka/jdbcimage/db/DbImportResultConsumer.java
@@ -115,7 +115,7 @@ public void accept(RowData t) {
                     if (value == null){
                         stmt.setNull(pos, type);
                     } else{
-                        // ENHANCEMENT: could be set directly from producer to avoid rowData
+                        value = db.toSupportedValue(type, value);
                         switch(type){
                             case Types.BIGINT:
                                 stmt.setLong(pos, (Long)value);
@@ -208,6 +208,13 @@ public void accept(RowData t) {
                             case Mssql.Types.SQL_VARIANT:
                                 stmt.setObject(pos, value);
                                 break;
+                            case Types.OTHER:
+                                if (value instanceof String){
+                                    // requires stringtype=unspecified in postgres connection string
+                                    stmt.setString(pos, (String)value);
+                                    break;
+                                }
+                                throw new IllegalStateException("Unable to set SQL type: "+type+" for value: "+value);
                             default:
                                 throw new IllegalStateException("Unable to set SQL type: "+type+" for value: "+value);
                         }

diff --git a/src/main/java/io/github/sranka/jdbcimage/kryo/KryoResultProducer.java b/src/main/java/io/github/sranka/jdbcimage/kryo/KryoResultProducer.java
@@ -58,7 +58,7 @@ public boolean fillData(RowData row) {
 		for(int i=0; i<types.length; i++){
 			Object val;
 			int dbType = types[i];
-			if (dbType == Mssql.Types.SQL_VARIANT) {
+			if (dbType == Mssql.Types.SQL_VARIANT || dbType == Types.OTHER) {
 				// read a specific type stored within sql_variant
 				dbType = in.readInt();
 			}

diff --git a/src/main/java/io/github/sranka/jdbcimage/kryo/KryoResultSetConsumer.java b/src/main/java/io/github/sranka/jdbcimage/kryo/KryoResultSetConsumer.java
@@ -160,6 +160,7 @@ public void accept(ResultSet rs) {
                         val = rs.getObject(i + 1);
                         if (val == null) {
                             clazz = String.class; // any class is good for serializing null
+                            out.writeInt(Types.VARCHAR);
                         } else {
                             // check that the class is supported
                             clazz = val.getClass();
@@ -172,6 +173,22 @@ public void accept(ResultSet rs) {
                             out.writeInt(type);
                         }
                         break;
+                    case Types.OTHER:
+                        val = rs.getObject(i + 1);
+                        if (val == null) {
+                            clazz = String.class; // any class is good for serializing null
+                            out.writeInt(Types.VARCHAR);
+                        } else if (val.getClass().getName().equals("org.postgresql.util.PGobject")) {
+                            // PGObject serialized as a string
+                            clazz = String.class;
+                            val = val.toString();
+                            out.writeInt(Types.VARCHAR);
+                        } else {
+                            throw new IllegalStateException("Unable to serialize SQL OTHER type: " + info.types[i]
+                                    + ", Class: " + val.getClass().getName()
+                                    + ", Object: " + val);
+                        }
+                        break;
                     default:
                         val = rs.getObject(i + 1);
                         throw new IllegalStateException("Unable to serialize SQL type: " + info.types[i]

diff --git a/src/main/java/io/github/sranka/jdbcimage/main/DBFacade.java b/src/main/java/io/github/sranka/jdbcimage/main/DBFacade.java
@@ -53,7 +53,7 @@ public boolean isTableIgnored(String tableName){
         if (ignoredTables == null){
             ignoredTables = Stream.of(
                     IGNORED_TABLES.split(","))
-                    .filter(x -> x!=null && x.trim().length()>0)
+                    .filter(x -> !x.trim().isEmpty())
                     .map(x -> {
                         Pattern pattern = Pattern.compile(x);
                         return (Predicate<String>) s -> pattern.matcher(s).matches();
@@ -150,6 +150,14 @@ public TableInfo getTableInfo(String tableName){
     public int toSupportedSqlType(int sqlType) {
         return sqlType;
     }
+    /**
+     * Converts the requested value to a DB-supported value.
+     * @param value value to process
+     * @return supported value
+     */
+    public Object toSupportedValue(int sqlType, Object value) {
+        return value;
+    }
     /**
      * Checks whether the database instance can create and use BLOB, CLOB and NCLOB instances.
      *
@@ -171,7 +179,7 @@ public Object convertCharacterStreamInput(Reader reader){
 
     @SuppressWarnings("WeakerAccess")
     public static class TableInfo{
-        private String tableName;
+        private final String tableName;
         private Map<String, Object> data;
         private Map<String, String> tableColumns;
 

diff --git a/src/main/java/io/github/sranka/jdbcimage/main/PostgreSQL.java b/src/main/java/io/github/sranka/jdbcimage/main/PostgreSQL.java
@@ -28,7 +28,7 @@ public class PostgreSQL extends DBFacade {
     public static final String STATE_TABLE_DDL = "CREATE TABLE "+STATE_TABLE_NAME+"( tableName varchar(64),constraintName varchar(64),sql varchar(512))";
     public static final String STATE_TABLE_SQL = "SELECT tableName,constraintName,sql FROM "+STATE_TABLE_NAME+ " order by tableName,constraintName";
 
-    private static Pattern identifyColumnPattern = Pattern.compile("^.*_([a-zA-z]*)_seq$");
+    private static final Pattern identifyColumnPattern = Pattern.compile("^.*_([a-zA-z]*)_seq$");
 
     @Override
     public void setupDataSource(BasicDataSource bds) {
@@ -286,6 +286,7 @@ public void modifyIndexes(boolean enable) {
         mainToolBase.out.println("Index " + (enable ? "enable" : "disable") + " not supported on PostgreSQL!");
     }
 
+    @SuppressWarnings("DuplicateBranchesInSwitch")
     @Override
     public int toSupportedSqlType(int sqlType) {
         switch (sqlType){
@@ -301,6 +302,15 @@ public int toSupportedSqlType(int sqlType) {
         return sqlType;
     }
 
+    @Override
+    public Object toSupportedValue(int sqlType, Object value){
+        // postgres doesn't support storing NULL (\0x00) characters in text fields
+        if (value instanceof String){
+            return ((String)value).replace("\u0000", "");
+        }
+        return value;
+    }
+
     @Override
     public boolean canCreateBlobs() {
         return false;
@@ -321,7 +331,6 @@ private String currentSchema() throws SQLException {
         if (cachedSchema == null) {
             // get current schema and create state table
             String schema;
-            boolean createStateTable = false;
             try (Connection con = mainToolBase.getReadOnlyConnection()) {
                 schema = con.getSchema(); // get current schema
             }