Unbreak CFG

Author: turboderp

exllamav2/generator/sampler.py

@@ -188,11 +188,6 @@ def sample(
 
         logits = logits.squeeze(1)
 
-        # Prepare filter
-
-        logit_filter = torch.empty((batch_size, vocab_size), dtype = torch.bool)
-        ext_c.fast_fill_cpu_ones_bool(logit_filter)
-
         # Sync
 
         if sync:
@@ -206,6 +201,11 @@ def sample(
             logits = logits.unsqueeze(0)
             batch_size = 1
 
+        # Prepare filter
+
+        logit_filter = torch.empty((batch_size, vocab_size), dtype = torch.bool)
+        ext_c.fast_fill_cpu_ones_bool(logit_filter)
+
         # Repetition penalty
 
         if settings.token_repetition_penalty != 1.0 or \