We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 34038dd commit d293a50Copy full SHA for d293a50
src/datatrove/pipeline/filters/fineweb_quality_filter.py
@@ -8,7 +8,7 @@ class FineWebQualityFilter(BaseFilter):
8
9
def __init__(
10
self,
11
- exclusion_writer: DiskWriter = None,,
+ exclusion_writer: DiskWriter = None,
12
line_punct_thr: float = 0.12,
13
line_punct_exclude_zero: bool = False,
14
short_line_thr: float = 0.67,
0 commit comments