Fix swiglu backwards return type (Dao-AILab#1337)

ntenenz · web-flow · commit 7153673c1a3c · 2024-11-15T16:23:40.000-08:00
diff --git a/flash_attn/ops/activations.py b/flash_attn/ops/activations.py
@@ -110,7 +110,7 @@ def sqrelu_bwd(g, x):
 }
 """
 swiglu_bwd_codestring = """
-template <typename T> T swiglu_bwd(T x, T y, T g, T& dx, T& dy) {
+template <typename T> void swiglu_bwd(T x, T y, T g, T& dx, T& dy) {
     float x_sigmoid = 1.0f / (1.0f + ::exp(-float(x)));
     dx = x_sigmoid * (1 + float(x) * (1.0f - x_sigmoid)) * float(g) * float(y);
     dy = float(x) * x_sigmoid * float(g);

Original file line number	Diff line number	Diff line change
`@@ -110,7 +110,7 @@ def sqrelu_bwd(g, x):`
`110`	`110`	`}`
`111`	`111`	`"""`
`112`	`112`	`swiglu_bwd_codestring = """`
`113`		`-template <typename T> T swiglu_bwd(T x, T y, T g, T& dx, T& dy) {`
	`113`	`+template <typename T> void swiglu_bwd(T x, T y, T g, T& dx, T& dy) {`
`114`	`114`	`float x_sigmoid = 1.0f / (1.0f + ::exp(-float(x)));`
`115`	`115`	`dx = x_sigmoid * (1 + float(x) * (1.0f - x_sigmoid)) * float(g) * float(y);`
`116`	`116`	`dy = float(x) * x_sigmoid * float(g);`