Merge pull request #391 from sisong/dev

hdiffz added "-c-ldef" ; used libdeflate compressor, compatible with zlib.

Author: sisong

.github/workflows/ci.yml

@@ -10,24 +10,29 @@ jobs:
       - uses: actions/checkout@v2
       - name: makeInit
         run: |
-          make LZMA=0 ZSTD=0 MD5=0 -j
+          make LDEF=0 LZMA=0 ZSTD=0 MD5=0 -j
           make clean
-      - name: makeUnDef
+      - name: makeNoAll
         run: |
           git clone --depth=1 https://github.com/sisong/libmd5.git ../libmd5
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
-          make DIR_DIFF=0 MT=0 BSD=0 VCD=0 ZLIB=0 BZIP2=0 -j
+          make DIR_DIFF=0 MT=0 BSD=0 VCD=0 ZLIB=0 LDEF=0 BZIP2=0 -j
+          make clean
+      - name: makeNoldef
+        run: |
+          make LDEF=0 -j
           make clean
       - name: makeAll
         run: |
+          git clone --depth=1 https://github.com/sisong/zlib.git ../zlib
+          git clone --depth=1 https://github.com/sisong/libdeflate.git ../libdeflate
           make -j
           make clean
-      - name: makeByCode
+      - name: makeByBz2Code
         run: |
-          git clone --depth=1 https://github.com/sisong/zlib.git ../zlib
           git clone --depth=1 https://github.com/sisong/bzip2.git ../bzip2
-          make ZLIB=1 BZIP2=1 -j
+          make BZIP2=1 -j
 
   clang-build:
     runs-on: ubuntu-latest
@@ -41,6 +46,8 @@ jobs:
           git clone --depth=1 https://github.com/sisong/libmd5.git ../libmd5
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
+          git clone --depth=1 https://github.com/sisong/zlib.git ../zlib
+          git clone --depth=1 https://github.com/sisong/libdeflate.git ../libdeflate
       - name: makeByClang
         run: |
           make CL=1 -j
@@ -54,6 +61,8 @@ jobs:
           git clone --depth=1 https://github.com/sisong/libmd5.git ../libmd5
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
+          git clone --depth=1 https://github.com/sisong/zlib.git ../zlib
+          git clone --depth=1 https://github.com/sisong/libdeflate.git ../libdeflate
       - name: buildByXcode
         run: |
           xcodebuild -workspace builds/xcode/HDiffPatch.xcworkspace -scheme hdiffz -configuration Release OBJROOT=$PWD/bin SYMROOT=$PWD/bin
@@ -72,7 +81,7 @@ jobs:
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
           cd ./builds/android_ndk_jni_mk
-          ndk-build NDK_PROJECT_PATH=. APP_BUILD_SCRIPT=./Android.mk NDK_APPLICATION_MK=./Application.mk APP_PLATFORM=android-16 APP_ABI=all
+          ndk-build NDK_PROJECT_PATH=. NDK_APPLICATION_MK=./Application.mk APP_PLATFORM=android-16 APP_ABI=all
 
   windows-ndk-build:
     runs-on: windows-latest
@@ -87,7 +96,7 @@ jobs:
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
           git clone --depth=1 https://github.com/sisong/bzip2.git ../bzip2
           cd ./builds/android_ndk_jni_mk
-          ndk-build NDK_PROJECT_PATH=. APP_BUILD_SCRIPT=./Android.mk NDK_APPLICATION_MK=./Application.mk BSD=1 BZIP2=1
+          ndk-build NDK_PROJECT_PATH=. NDK_APPLICATION_MK=./Application.mk BSD=1 BZIP2=1
 
   ubuntu-ndk-build:
     runs-on: ubuntu-latest
@@ -103,7 +112,7 @@ jobs:
           git clone --depth=1 https://github.com/sisong/bzip2.git ../bzip2
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
           cd ./builds/android_ndk_jni_mk
-          ndk-build NDK_PROJECT_PATH=. APP_BUILD_SCRIPT=./Android.mk NDK_APPLICATION_MK=./Application.mk BSD=1 BZIP2=1 VCD=1
+          ndk-build NDK_PROJECT_PATH=. NDK_APPLICATION_MK=./Application.mk BSD=1 BZIP2=1 VCD=1
 
   vc-build:
     runs-on: windows-latest
@@ -114,11 +123,12 @@ jobs:
           msbuild-architecture: x64
       - name: initAndClone
         run: |
-          git clone --depth=1 https://github.com/sisong/zlib.git ../zlib
-          git clone --depth=1 https://github.com/sisong/bzip2.git ../bzip2
           git clone --depth=1 https://github.com/sisong/libmd5.git ../libmd5
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
+          git clone --depth=1 https://github.com/sisong/bzip2.git ../bzip2
+          git clone --depth=1 https://github.com/sisong/zlib.git ../zlib
+          git clone --depth=1 https://github.com/sisong/libdeflate.git ../libdeflate
       - name: buildByVC
         run: |
           msbuild builds/vc/HDiffPatch.sln -t:rebuild -verbosity:diag -property:Configuration=Release

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 full changelog at: https://github.com/sisong/HDiffPatch/commits   
 
+## [v4.8.0](https://github.com/sisong/HDiffPatch/tree/v4.8.0) - 2024-07-26
+### Added
+* cmdline hdiffz support option "-c-ldef-{1..12}"; used libdeflate compressor, compatible with -c-zlib, faster or better than zlib;
+ (hpatchz now default closed libdeflate decompressor)
+* add plugin ldefCompressPlugin, pldefCompressPlugin, ldefDecompressPlugin;
+### Changed
+* released Android libhpatchz.so support Android 15 with 16KB page size;
+
 ## [v4.7.0](https://github.com/sisong/HDiffPatch/tree/v4.7.0) - 2024-07-12
 ### Added
 * cmdline hdiffz support option "-BSD -SD", to create diffFile compatible with another BSDIFF format "ENDSLEY/BSDIFF43", https://github.com/mendsley/bsdiff ; patch support this format from v4.6.7
@@ -54,7 +62,7 @@ if diffFile created by empty oldPath, then extract with default option `$selfExt
 
 ## [v4.3.0](https://github.com/sisong/HDiffPatch/tree/v4.3.0) - 2022-09-23
 ### Changed
-* recode some patch error code: decompresser errors, file error, disk space full error, jni error
+* recode some patch error code: decompressor errors, file error, disk space full error, jni error
 
 ## [v4.2.0](https://github.com/sisong/HDiffPatch/tree/v4.2.0) - 2022-05-15
 ### Added

Makefile

@@ -1,12 +1,22 @@
 # args
 DIR_DIFF := 1
 MT       := 1
+# used libdeflate?
+LDEF     := 1
 # 0: not need zlib;  1: compile zlib source code;  2: used -lz to link zlib lib;
-ZLIB     := 2
+ifeq ($(LDEF),0)
+  ZLIB     := 2
+else
+  ZLIB     := 1
+endif
 # 0: not need lzma;  1: compile lzma source code;  2: used -llzma to link lzma lib;
 LZMA     := 1
+# lzma decompressor used arm64 asm optimize? 
 ARM64ASM := 0
-RISCV32  := 0
+# lzma only can used software CRC? (no hardware CRC)
+USE_CRC_EMU := 0
+# supported atomic uint64?
+ATOMIC_U64 := 1
 # 0: not need zstd;  1: compile zstd source code;  2: used -lzstd to link zstd lib;
 ZSTD     := 1
 MD5      := 1
@@ -17,9 +27,9 @@ CL  	 := 0
 M32      := 0
 # build for out min size
 MINS     := 0
-# support VCDIFF? 
+# need support VCDIFF? 
 VCD      := 1
-# support bsdiff&bspatch?
+# need support bsdiff&bspatch?
 BSD      := 1
 ifeq ($(OS),Windows_NT) # mingw?
   CC    := gcc
@@ -28,10 +38,21 @@ else
   # 0: not need bzip2 (must BSD=0);  1: compile bzip2 source code;  2: used -lbz2 to link bzip2 lib;
   BZIP2 := 2
 endif
-ifeq ($(BZIP2),0)
-  ifeq ($(BSD),0)
+ifeq ($(BSD),0)
+else
+  ifeq ($(BZIP2),0)
+  $(error error: support bsdiff need BZIP2! set BSD=0 or BZIP2>0 continue)
+  endif
+endif
+
+ifeq ($(LDEF),0)
+else
+  ifeq ($(ZLIB),2)
+  $(error error: now libdeflate decompressor not support -lz! need zlib source code, set ZLIB=1 continue)
   else
-  $(error error: support bsdiff need BZIP2! set BSD=0 continue)
+    ifeq ($(ZLIB),0)
+    $(warning warning: libdeflate can't support all of the deflate code, when no zlib source code)
+    endif
   endif
 endif
 
@@ -159,7 +180,7 @@ ifeq ($(BZIP2),1) # http://www.bzip.org  https://github.com/sisong/bzip2
 endif
 
 ZLIB_PATH := ../zlib
-ifeq ($(ZLIB),1) # http://zlib.net  https://github.com/sisong/zlib  
+ifeq ($(ZLIB),1) # https://github.com/sisong/zlib/tree/bit_pos_padding
   HPATCH_OBJ += $(ZLIB_PATH)/adler32.o \
   				$(ZLIB_PATH)/crc32.o \
   				$(ZLIB_PATH)/inffast.o \
@@ -170,6 +191,14 @@ ifeq ($(ZLIB),1) # http://zlib.net  https://github.com/sisong/zlib
   HDIFF_OBJ +=  $(ZLIB_PATH)/deflate.o
 endif
 
+LDEF_PATH := ../libdeflate
+ifeq ($(LDEF),1) # https://github.com/sisong/libdeflate/tree/stream-mt
+  HPATCH_OBJ += $(LDEF_PATH)/lib/deflate_decompress.o\
+                $(LDEF_PATH)/lib/utils.o \
+                $(LDEF_PATH)/lib/x86/cpu_features.o
+  HDIFF_OBJ +=  $(LDEF_PATH)/lib/deflate_compress.o
+endif
+
 HDIFF_OBJ += \
     hdiffz_import_patch.o \
     libHDiffPatch/HPatchLite/hpatch_lite.o \
@@ -213,8 +242,7 @@ DEF_FLAGS := \
     -D_IS_NEED_DEFAULT_CompressPlugin=0 \
     -D_IS_NEED_ALL_ChecksumPlugin=0 \
     -D_IS_NEED_DEFAULT_ChecksumPlugin=0 
-ifeq ($(RISCV32),0)
-else
+ifeq ($(ATOMIC_U64),0)
   DEF_FLAGS += -D_IS_NO_ATOMIC_U64=1
 endif
 ifeq ($(M32),0)
@@ -243,7 +271,19 @@ ifeq ($(ZLIB),0)
 else
     DEF_FLAGS += -D_CompressPlugin_zlib
 	ifeq ($(ZLIB),1)
-        DEF_FLAGS += -I$(ZLIB_PATH)
+    DEF_FLAGS += -I$(ZLIB_PATH)
+	endif
+endif
+ifeq ($(LDEF),0)
+else
+    DEF_FLAGS += -D_CompressPlugin_ldef
+	ifeq ($(LDEF),1)
+    DEF_FLAGS += -I$(LDEF_PATH)
+	endif
+  ifeq ($(ZLIB),1)
+    DEF_FLAGS += -D_CompressPlugin_ldef_is_use_zlib=1
+  else
+    DEF_FLAGS += -D_CompressPlugin_ldef_is_use_zlib=0
 	endif
 endif
 ifeq ($(DIR_DIFF),0)
@@ -284,9 +324,10 @@ else
     else
       DEF_FLAGS += -DZ7_LZMA_DEC_OPT
     endif
-    ifeq ($(VCD),0)
-    else
-      DEF_FLAGS += -DUSE_CRC_EMU
+    ifneq ($(VCD),0)
+      ifneq ($(USE_CRC_EMU),0)
+        DEF_FLAGS += -DUSE_CRC_EMU
+      endif
     endif
   endif
 endif

README.md

@@ -1,5 +1,5 @@
 # [HDiffPatch]
-[![release](https://img.shields.io/badge/release-v4.7.0-blue.svg)](https://github.com/sisong/HDiffPatch/releases) 
+[![release](https://img.shields.io/badge/release-v4.8.0-blue.svg)](https://github.com/sisong/HDiffPatch/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/HDiffPatch/blob/master/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/HDiffPatch/pulls)
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/HDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/HDiffPatch/issues)   
@@ -50,17 +50,19 @@ apply the delta:
 `$ cd <dir>/HDiffPatch`   
 ### Linux or MacOS X ###
 Try:   
-`$ make LZMA=0 ZSTD=0 MD5=0`   
+`$ make LDEF=0 LZMA=0 ZSTD=0 MD5=0`   
 bzip2 : if the build fails with `fatal error: bzlib.h: No such file or directory`, use your system's package manager to install the libbz2 package and try again; or download & make with libbz2 source code:
 ```
 $ git clone https://github.com/sisong/bzip2.git ../bzip2
-$ make LZMA=0 ZSTD=0 MD5=0 BZIP2=1
+$ make LDEF=0 LZMA=0 ZSTD=0 MD5=0 BZIP2=1
 ```
-if need lzma zstd & md5 support, Try:    
+if need lzma zstd & md5 ... default support, Try:
 ```
 $ git clone https://github.com/sisong/libmd5.git ../libmd5
 $ git clone https://github.com/sisong/lzma.git ../lzma
 $ git clone https://github.com/sisong/zstd.git ../zstd
+$ git clone https://github.com/sisong/zlib.git ../zlib
+$ git clone https://github.com/sisong/libdeflate.git ../libdeflate
 $ make
 ```    
 Tip: You can use `$ make -j` to compile in parallel.
@@ -72,6 +74,7 @@ $ git clone https://github.com/sisong/libmd5.git ../libmd5
 $ git clone https://github.com/sisong/lzma.git ../lzma
 $ git clone https://github.com/sisong/zstd.git ../zstd
 $ git clone https://github.com/sisong/zlib.git   ../zlib
+$ git clone https://github.com/sisong/libdeflate.git ../libdeflate
 $ git clone https://github.com/sisong/bzip2.git  ../bzip2
 ```
 
@@ -149,11 +152,14 @@ options:
         -c-zlib[-{1..9}[-dictBits]]     DEFAULT level 9
             dictBits can 9--15, DEFAULT 15.
             support run by multi-thread parallel, fast!
+        -c-ldef[-{1..12}]               DEFAULT level 12
+            compatible with -c-zlib, faster or better compress than zlib;
+            used libdeflate compressor, & dictBits always 15.
+            support run by multi-thread parallel, fast!
         -c-bzip2[-{1..9}]               (or -bz2) DEFAULT level 9
         -c-pbzip2[-{1..9}]              (or -pbz2) DEFAULT level 8
             support run by multi-thread parallel, fast!
             NOTE: code not compatible with it compressed by -c-bzip2!
-               and code size may be larger than if it compressed by -c-bzip2.
         -c-lzma[-{0..9}[-dictSize]]     DEFAULT level 7
             dictSize can like 4096 or 4k or 4m or 128m etc..., DEFAULT 8m
             support run by 2-thread parallel.

README_cn.md

@@ -1,5 +1,5 @@
 # [HDiffPatch]
-[![release](https://img.shields.io/badge/release-v4.7.0-blue.svg)](https://github.com/sisong/HDiffPatch/releases) 
+[![release](https://img.shields.io/badge/release-v4.8.0-blue.svg)](https://github.com/sisong/HDiffPatch/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/HDiffPatch/blob/master/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/HDiffPatch/pulls)
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/HDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/HDiffPatch/issues)   
@@ -50,17 +50,19 @@
 `$ cd <dir>/HDiffPatch`   
 ### Linux or MacOS X ###
 试试:   
-`$ make LZMA=0 ZSTD=0 MD5=0`   
+`$ make LDEF=0 LZMA=0 ZSTD=0 MD5=0`   
 bzip2 : 如果编译失败，显示 `fatal error: bzlib.h: No such file or directory`，请使用系统的包管理器安装libbz2，然后再试一次；或者下载并使用libbz2源代码来编译:
 ```
 $ git clone https://github.com/sisong/bzip2.git ../bzip2
-$ make LZMA=0 ZSTD=0 MD5=0 BZIP2=1
+$ make LDEF=0 LZMA=0 ZSTD=0 MD5=0 BZIP2=1
 ```
-如果需要支持 lzma、zstd 和 md5，试试:    
+如果需要支持 lzma、zstd 和 md5 等 默认编译设置，试试:    
 ```
 $ git clone https://github.com/sisong/libmd5.git ../libmd5
 $ git clone https://github.com/sisong/lzma.git ../lzma
 $ git clone https://github.com/sisong/zstd.git ../zstd
+$ git clone https://github.com/sisong/zlib.git ../zlib
+$ git clone https://github.com/sisong/libdeflate.git ../libdeflate
 $ make
 ```    
 提示:你可以使用 `$ make -j` 来并行编译。
@@ -72,6 +74,7 @@ $ git clone https://github.com/sisong/libmd5.git ../libmd5
 $ git clone https://github.com/sisong/lzma.git ../lzma
 $ git clone https://github.com/sisong/zstd.git ../zstd
 $ git clone https://github.com/sisong/zlib.git   ../zlib
+$ git clone https://github.com/sisong/libdeflate.git ../libdeflate
 $ git clone https://github.com/sisong/bzip2.git  ../bzip2
 ```
 
@@ -149,9 +152,13 @@ $ git clone https://github.com/sisong/bzip2.git  ../bzip2
         -c-zlib[-{1..9}[-dictBits]]     默认级别 9
             压缩字典比特数dictBits可以为9到15, 默认为15。
             支持多线程并行压缩,很快！
+        -c-ldef[-{1..12}]               默认级别 12
+            输出压缩数据格式兼容于-c-zlib, 但比zlib压缩得更快或压缩得更小;
+            使用了libdeflate压缩算法，且压缩字典比特数dictBits始终为15。
+            支持多线程并行压缩,很快！
         -c-bzip2[-{1..9}]               (或 -bz2) 默认级别 9
         -c-pbzip2[-{1..9}]              (或 -pbz2) 默认级别 8
-            支持并行压缩,生成的补丁和-c-bzip2的输出格式不同,一般也可能稍大一点。
+            支持并行压缩,生成的补丁和-c-bzip2的输出格式稍有不同。
         -c-lzma[-{0..9}[-dictSize]]     默认级别 7
             压缩字典大小dictSize可以设置为 4096, 4k, 4m, 128m等, 默认为8m
             支持2个线程并行压缩。
@@ -161,7 +168,7 @@ $ git clone https://github.com/sisong/bzip2.git  ../bzip2
             警告: lzma和lzma2是不同的压缩编码格式。
         -c-zstd[-{0..22}[-dictBits]]    默认级别 20
             压缩字典比特数dictBits 可以为10到30, 默认为23。
-            支持多线程并行压缩,很快。
+            支持多线程并行压缩,较快。
   -C-checksumType
       为文件夹间diff设置数据校验算法, 默认为fadler64;
       支持的校验选项:

builds/android_ndk_jni_mk/Application.mk

@@ -3,5 +3,6 @@ APP_CFLAGS += -s -Wno-error=format-security
 APP_CFLAGS += -fvisibility=hidden -fvisibility-inlines-hidden
 APP_CFLAGS += -ffunction-sections -fdata-sections
 APP_LDFLAGS += -s -Wl,--gc-sections,--as-needed
+APP_LDFLAGS += -Wl,-z,max-page-size=16384
 APP_BUILD_SCRIPT := Android.mk
 APP_ABI := armeabi armeabi-v7a arm64-v8a x86 x86_64