Skip to content

Commit

Permalink
Warn on corrupted files.
Browse files Browse the repository at this point in the history
  • Loading branch information
atrabattoni committed Dec 26, 2024
1 parent 841a23a commit 425e65e
Showing 1 changed file with 16 additions and 7 deletions.
23 changes: 16 additions & 7 deletions xdas/core/routines.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import re
import warnings
from collections import defaultdict
from concurrent.futures import ProcessPoolExecutor, as_completed
from glob import glob
Expand Down Expand Up @@ -323,19 +324,27 @@ def open_mfdataarray(
)
max_workers = 1 if engine == "miniseed" else None # TODO: dirty fix
with ProcessPoolExecutor(max_workers=max_workers) as executor:
futures = [
executor.submit(open_dataarray, path, engine=engine, **kwargs)
futures_to_paths = {
executor.submit(open_dataarray, path, engine=engine, **kwargs): path
for path in paths
]
}
if verbose:
iterator = tqdm(
as_completed(futures),
total=len(futures),
as_completed(futures_to_paths),
total=len(futures_to_paths),
desc="Fetching metadata from files",
)
else:
iterator = as_completed(futures)
objs = [future.result() for future in iterator]
iterator = as_completed(futures_to_paths)
objs = []
for future in iterator:
try:
obj = future.result()
except Exception as e:
path = futures_to_paths[future]
warnings.warn(f"could not open {path}: {e}", RuntimeWarning)
else:
objs.append(obj)
return combine_by_coords(objs, dim, tolerance, squeeze, None, verbose)


Expand Down

0 comments on commit 425e65e

Please sign in to comment.