* fix logger fileno undefined error

* update downloading logics of models without file locks
modelscope · Feb 27, 2025 · da06b01 · da06b01
1 parent e283e8d
commit da06b01
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/data_juicer/utils/logger_utils.py b/data_juicer/utils/logger_utils.py
@@ -60,6 +60,9 @@ def __init__(self, level='INFO', caller_names=('datasets', 'logging')):
         self.buffer = StringIO()
         self.BUFFER_SIZE = 1024 * 1024
 
+    def fileno(self):
+        return self.buffer.fileno()
+
     def write(self, buf):
         full_name = get_caller_name(depth=1)
         module_name = full_name.rsplit('.', maxsplit=-1)[0]

diff --git a/data_juicer/utils/model_utils.py b/data_juicer/utils/model_utils.py
@@ -864,7 +864,8 @@ def update_sampling_params(sampling_params,
 }
 
 _MODELS_WITHOUT_FILE_LOCK = {
-    'kenlm', 'nltk', 'recognizeAnything', 'sentencepiece', 'spacy'
+    'fasttext', 'fastsam', 'kenlm', 'nltk', 'recognizeAnything',
+    'sentencepiece', 'spacy'
 }