From 425e65e2d713d72b512fab63fd17d5fa0b13b840 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Thu, 26 Dec 2024 12:53:28 +0100 Subject: [PATCH] Warn on corrupted files. --- xdas/core/routines.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 11f1a9e..3b011ab 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -1,5 +1,6 @@ import os import re +import warnings from collections import defaultdict from concurrent.futures import ProcessPoolExecutor, as_completed from glob import glob @@ -323,19 +324,27 @@ def open_mfdataarray( ) max_workers = 1 if engine == "miniseed" else None # TODO: dirty fix with ProcessPoolExecutor(max_workers=max_workers) as executor: - futures = [ - executor.submit(open_dataarray, path, engine=engine, **kwargs) + futures_to_paths = { + executor.submit(open_dataarray, path, engine=engine, **kwargs): path for path in paths - ] + } if verbose: iterator = tqdm( - as_completed(futures), - total=len(futures), + as_completed(futures_to_paths), + total=len(futures_to_paths), desc="Fetching metadata from files", ) else: - iterator = as_completed(futures) - objs = [future.result() for future in iterator] + iterator = as_completed(futures_to_paths) + objs = [] + for future in iterator: + try: + obj = future.result() + except Exception as e: + path = futures_to_paths[future] + warnings.warn(f"could not open {path}: {e}", RuntimeWarning) + else: + objs.append(obj) return combine_by_coords(objs, dim, tolerance, squeeze, None, verbose)