Reorder arguments to vaeseq_u8 to improve AArch64 performance

georges-arm · georges-arm · commit e67e56a326e0 · 2025-05-21T10:39:57.000+01:00
The register arguments to the AESE instruction are commutative, however
the first input register is constrained in that it also names the output
register. The constraint on register allocation causes recent LLVM
versions to emit a lot of MOV instructions, significantly impacting
performance.

Swapping the register operands allows the compiler to emit significantly
fewer MOV instructions. This change improves performance on Arm
infrastructure micro-architectures by 14-36% depending on the
micro-architecture, except for the AEGIS-256x4 sub-test which regresses
slightly.
diff --git a/src/aegis128l/aegis128l_neon_aes.c b/src/aegis128l/aegis128l_neon_aes.c
@@ -32,7 +32,7 @@ typedef uint8x16_t aes_block_t;
 #    define AES_BLOCK_LOAD(A)         vld1q_u8(A)
 #    define AES_BLOCK_LOAD_64x2(A, B) vreinterpretq_u8_u64(vsetq_lane_u64((A), vmovq_n_u64(B), 1))
 #    define AES_BLOCK_STORE(A, B)     vst1q_u8((A), (B))
-#    define AES_ENC(A, B)             veorq_u8(vaesmcq_u8(vaeseq_u8((A), vmovq_n_u8(0))), (B))
+#    define AES_ENC(A, B)             veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), (A))), (B))
 
 static inline void
 aegis128l_update(aes_block_t *const state, const aes_block_t d1, const aes_block_t d2)
diff --git a/src/aegis128x2/aegis128x2_neon_aes.c b/src/aegis128x2/aegis128x2_neon_aes.c
@@ -64,8 +64,8 @@ AES_BLOCK_STORE(uint8_t *a, const aes_block_t b)
 static inline aes_block_t
 AES_ENC(const aes_block_t a, const aes_block_t b)
 {
-    return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8((a.b0), vmovq_n_u8(0))), (b.b0)),
-                           veorq_u8(vaesmcq_u8(vaeseq_u8((a.b1), vmovq_n_u8(0))), (b.b1)) };
+    return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b0)), b.b0),
+                           veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b1)), b.b1) };
 }
 
 static inline void
diff --git a/src/aegis128x4/aegis128x4_neon_aes.c b/src/aegis128x4/aegis128x4_neon_aes.c
@@ -70,10 +70,10 @@ AES_BLOCK_STORE(uint8_t *a, const aes_block_t b)
 static inline aes_block_t
 AES_ENC(const aes_block_t a, const aes_block_t b)
 {
-    return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8((a.b0), vmovq_n_u8(0))), (b.b0)),
-                           veorq_u8(vaesmcq_u8(vaeseq_u8((a.b1), vmovq_n_u8(0))), (b.b1)),
-                           veorq_u8(vaesmcq_u8(vaeseq_u8((a.b2), vmovq_n_u8(0))), (b.b2)),
-                           veorq_u8(vaesmcq_u8(vaeseq_u8((a.b3), vmovq_n_u8(0))), (b.b3)) };
+    return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b0)), b.b0),
+                           veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b1)), b.b1),
+                           veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b2)), b.b2),
+                           veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b3)), b.b3) };
 }
 
 static inline void
diff --git a/src/aegis256/aegis256_neon_aes.c b/src/aegis256/aegis256_neon_aes.c
@@ -32,7 +32,7 @@ typedef uint8x16_t aes_block_t;
 #    define AES_BLOCK_LOAD(A)         vld1q_u8(A)
 #    define AES_BLOCK_LOAD_64x2(A, B) vreinterpretq_u8_u64(vsetq_lane_u64((A), vmovq_n_u64(B), 1))
 #    define AES_BLOCK_STORE(A, B)     vst1q_u8((A), (B))
-#    define AES_ENC(A, B)             veorq_u8(vaesmcq_u8(vaeseq_u8((A), vmovq_n_u8(0))), (B))
+#    define AES_ENC(A, B)             veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), (A))), (B))
 
 static inline void
 aegis256_update(aes_block_t *const state, const aes_block_t d)
diff --git a/src/aegis256x2/aegis256x2_neon_aes.c b/src/aegis256x2/aegis256x2_neon_aes.c
@@ -64,8 +64,8 @@ AES_BLOCK_STORE(uint8_t *a, const aes_block_t b)
 static inline aes_block_t
 AES_ENC(const aes_block_t a, const aes_block_t b)
 {
-    return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8((a.b0), vmovq_n_u8(0))), (b.b0)),
-                           veorq_u8(vaesmcq_u8(vaeseq_u8((a.b1), vmovq_n_u8(0))), (b.b1)) };
+    return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b0)), b.b0),
+                           veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b1)), b.b1) };
 }
 
 static inline void
diff --git a/src/aegis256x4/aegis256x4_neon_aes.c b/src/aegis256x4/aegis256x4_neon_aes.c
@@ -70,10 +70,10 @@ AES_BLOCK_STORE(uint8_t *a, const aes_block_t b)
 static inline aes_block_t
 AES_ENC(const aes_block_t a, const aes_block_t b)
 {
-    return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8((a.b0), vmovq_n_u8(0))), (b.b0)),
-                           veorq_u8(vaesmcq_u8(vaeseq_u8((a.b1), vmovq_n_u8(0))), (b.b1)),
-                           veorq_u8(vaesmcq_u8(vaeseq_u8((a.b2), vmovq_n_u8(0))), (b.b2)),
-                           veorq_u8(vaesmcq_u8(vaeseq_u8((a.b3), vmovq_n_u8(0))), (b.b3)) };
+    return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b0)), b.b0),
+                           veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b1)), b.b1),
+                           veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b2)), b.b2),
+                           veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b3)), b.b3) };
 }
 
 static inline void

Original file line number	Diff line number	Diff line change
`@@ -64,8 +64,8 @@ AES_BLOCK_STORE(uint8_t *a, const aes_block_t b)`
`64`	`64`	`static inline aes_block_t`
`65`	`65`	`AES_ENC(const aes_block_t a, const aes_block_t b)`
`66`	`66`	`{`
`67`		`- return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8((a.b0), vmovq_n_u8(0))), (b.b0)),`
`68`		`- veorq_u8(vaesmcq_u8(vaeseq_u8((a.b1), vmovq_n_u8(0))), (b.b1)) };`
	`67`	`+ return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b0)), b.b0),`
	`68`	`+ veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b1)), b.b1) };`
`69`	`69`	`}`
`70`	`70`
`71`	`71`	`static inline void`
Original file line number	Diff line number	Diff line change
`@@ -70,10 +70,10 @@ AES_BLOCK_STORE(uint8_t *a, const aes_block_t b)`
`70`	`70`	`static inline aes_block_t`
`71`	`71`	`AES_ENC(const aes_block_t a, const aes_block_t b)`
`72`	`72`	`{`
`73`		`- return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8((a.b0), vmovq_n_u8(0))), (b.b0)),`
`74`		`- veorq_u8(vaesmcq_u8(vaeseq_u8((a.b1), vmovq_n_u8(0))), (b.b1)),`
`75`		`- veorq_u8(vaesmcq_u8(vaeseq_u8((a.b2), vmovq_n_u8(0))), (b.b2)),`
`76`		`- veorq_u8(vaesmcq_u8(vaeseq_u8((a.b3), vmovq_n_u8(0))), (b.b3)) };`
	`73`	`+ return (aes_block_t) { veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b0)), b.b0),`
	`74`	`+ veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b1)), b.b1),`
	`75`	`+ veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b2)), b.b2),`
	`76`	`+ veorq_u8(vaesmcq_u8(vaeseq_u8(vmovq_n_u8(0), a.b3)), b.b3) };`
`77`	`77`	`}`
`78`	`78`
`79`	`79`	`static inline void`