update paths

dascore/io/ap_sensing/utils.py

@@ -4,8 +4,8 @@
 
 import dascore as dc
 from dascore.core import get_coord, get_coord_manager
-from dascore.utils.misc import _maybe_unpack, unbyte
 from dascore.utils.fs import get_uri
+from dascore.utils.misc import _maybe_unpack, unbyte
 
 
 def _get_version_string(resource):

dascore/io/asn/utils.py

@@ -5,10 +5,9 @@
 import dascore as dc
 import dascore.core
 from dascore.core.coords import get_coord
+from dascore.utils.fs import get_uri
 from dascore.utils.hdf5 import unpack_scalar_h5_dataset
 from dascore.utils.misc import unbyte
-from dascore.utils.fs import get_uri
-
 
 # --- Getting format/version
 

dascore/io/core.py

@@ -36,10 +36,10 @@
     MissingOptionalDependencyError,
     UnknownFiberFormatError,
 )
+from dascore.utils.fs import _iter_filesystem
 from dascore.utils.io import IOResourceManager, get_handle_from_resource
 from dascore.utils.mapping import FrozenDict
 from dascore.utils.misc import cached_method, iterate, warn_or_raise
-from dascore.utils.fs import _iter_filesystem
 from dascore.utils.models import (
     CommaSeparatedStr,
     DascoreBaseModel,

dascore/io/dasdae/core.py

@@ -10,9 +10,9 @@
     H5Writer,
 )
 from dascore.utils.misc import unbyte
-from ...utils.fs import get_uri
 from dascore.utils.patch import get_patch_names
 
+from ...utils.fs import get_uri
 from .utils import (
     _get_summary_from_patch_groups,
     _read_patch,

dascore/io/dasdae/utils.py

@@ -8,9 +8,9 @@
 import dascore as dc
 from dascore.core.coordmanager import get_coord_manager
 from dascore.core.coords import get_coord
+from dascore.utils.fs import get_uri
 from dascore.utils.hdf5 import Empty
 from dascore.utils.misc import suppress_warnings, unbyte
-from dascore.utils.fs import get_uri
 from dascore.utils.time import to_int
 
 # --- Functions for writing DASDAE format

dascore/io/dashdf5/core.py

@@ -6,8 +6,8 @@
 from dascore.constants import opt_timeable_types
 from dascore.io import FiberIO
 from dascore.utils.hdf5 import H5Reader
-from ...utils.fs import get_uri
 
+from ...utils.fs import get_uri
 from .utils import _get_cf_attrs, _get_cf_coords, _get_cf_version_str
 
 

dascore/io/febus/core.py

@@ -10,9 +10,9 @@
 from dascore.constants import opt_timeable_types
 from dascore.io import FiberIO
 from dascore.utils.hdf5 import H5Reader
-from ...utils.fs import get_uri
 from dascore.utils.models import UTF8Str
 
+from ...utils.fs import get_uri
 from .utils import (
     _get_febus_version_str,
     _read_febus,

dascore/io/gdr/core.py

@@ -16,8 +16,8 @@
     _get_version,
     _maybe_trim_data,
 )
-from dascore.utils.hdf5 import H5Reader
 from dascore.utils.fs import get_uri
+from dascore.utils.hdf5 import H5Reader
 
 
 class GDRPatchAttrs(dc.PatchAttrs):

dascore/io/neubrex/core.py

@@ -11,8 +11,8 @@
 import dascore.io.neubrex.utils_rfs as rfs_utils
 from dascore.constants import SpoolType
 from dascore.io import FiberIO
-from dascore.utils.hdf5 import H5Reader
 from dascore.utils.fs import get_uri
+from dascore.utils.hdf5 import H5Reader
 
 
 class NeubrexRFSPatchAttrs(dc.PatchAttrs):

dascore/io/prodml/utils.py

@@ -6,8 +6,8 @@
 from dascore.constants import VALID_DATA_TYPES
 from dascore.core.coordmanager import get_coord_manager
 from dascore.core.coords import get_coord
-from dascore.utils.misc import iterate, maybe_get_items, unbyte
 from dascore.utils.fs import get_uri
+from dascore.utils.misc import iterate, maybe_get_items, unbyte
 
 # --- Getting format/version
 

dascore/io/segy/core.py

@@ -6,8 +6,8 @@
 from dascore.io.core import FiberIO
 from dascore.utils.io import BinaryReader
 from dascore.utils.misc import optional_import
-from ...utils.fs import get_uri
 
+from ...utils.fs import get_uri
 from .utils import (
     _get_coords,
     _get_data_summary,

dascore/io/sentek/core.py

@@ -7,9 +7,9 @@
 import dascore as dc
 from dascore.io import BinaryReader
 from dascore.io.core import FiberIO
-from ...utils.fs import get_uri
 from dascore.utils.models import ArraySummary
 
+from ...utils.fs import get_uri
 from .utils import _get_patch_attrs, _get_version
 
 

dascore/io/silixah5/core.py

@@ -10,8 +10,8 @@
 import dascore.io.silixah5.utils as util
 from dascore.constants import opt_timeable_types
 from dascore.io import FiberIO
-from dascore.utils.hdf5 import H5Reader
 from dascore.utils.fs import get_uri
+from dascore.utils.hdf5 import H5Reader
 
 
 class SilixaPatchAttrs(dc.PatchAttrs):

dascore/io/tdms/core.py

@@ -6,8 +6,8 @@
 from dascore.constants import timeable_types
 from dascore.core import Patch
 from dascore.io import BinaryReader, FiberIO
-from ...utils.fs import get_uri
 
+from ...utils.fs import get_uri
 from .utils import _get_attrs_coords, _get_data, _get_version_str
 
 

dascore/io/terra15/core.py

@@ -8,8 +8,8 @@
 from dascore.constants import timeable_types
 from dascore.io import FiberIO
 from dascore.utils.hdf5 import H5Reader
-from ...utils.fs import get_uri
 
+from ...utils.fs import get_uri
 from .utils import (
     _get_default_attrs,
     _get_distance_coord,

dascore/io/xml_binary/core.py

@@ -11,8 +11,8 @@
 import dascore as dc
 from dascore.io import FiberIO
 from dascore.utils.models import UTF8Str
-from ...utils.fs import get_uri
 
+from ...utils.fs import get_uri
 from .utils import _load_patches, _paths_to_attrs, _read_xml_metadata
 
 

dascore/utils/fs.py

@@ -3,26 +3,23 @@
 
 These include actual file systems or virtual ones.
 """
+
 from __future__ import annotations
 
 import os
 import re
+from collections.abc import Generator, Iterable
 from pathlib import Path
-from typing import Iterable, Generator
-from typing_extensions import Self
 
 import fsspec
-
+from typing_extensions import Self
 
 # Detect if the string has an associated protocol.
 _PROTOCOL_DETECTION_REGEX = r"^([a-zA-Z][a-zA-Z0-9+.-]*):\/\/"
 
 
-
 def get_fspath(obj):
-    """
-
-    """
+    """ """
     uri = get_uri(obj)
     fs = fsspec.open(uri)
     return fs
@@ -34,18 +31,18 @@ class FSPath:
 
     This helps smooth out some of the edges of fsspec.
     """
-    fs: fsspec.AbstractFileSystem
 
     def __init__(self, obj):
-        """
-
-        """
+        """ """
         if isinstance(obj, FSPath):
             self.__dict__.update(obj.__dict__)
             return
-        if isinstance(obj, fsspec.core.OpenFile):
+        elif isinstance(obj, fsspec.core.OpenFile):
             self._fs = obj.fs
             self._path = Path(obj.path)
+        elif isinstance(obj, fsspec.spec.AbstractFileSystem):
+            self._fs = obj
+            self._path = Path("/")
         else:
             fs, path = fsspec.url_to_fs(obj)
             self._fs = fs
@@ -63,6 +60,10 @@ def path(self) -> Path:
         """Get the pathlib object representing this item."""
         return self._path
 
+    @property
+    def is_local(self):
+        return self._fs.protocol == ("file", "local")
+
     @property
     def parent(self) -> Path:
         """Get the pathlib object representing this item."""
@@ -88,12 +89,21 @@ def __truediv__(self, other: str) -> Self:
         """Enables division to add to string to Path."""
         return self.from_fs_path(fs=self._fs, path=self._path / other)
 
-
     def __repr__(self) -> str:
         return self._full_name()
 
-
-
+    def glob(self, arg: str) -> Generator[Self]:
+        """
+        Glob search the contents of the file system/directory.
+        """
+        # For local paths is probably better/faster to just use pathlibs glob
+        if self.is_local:
+            for item in self._path.glob(arg):
+                yield self.__class__(item)
+        # Otherwise, default to file system implementation.
+        glob_str = str(self._path / arg)
+        for obj in self._fs.glob(glob_str):
+            yield self.__class__.from_fs_path(self._fs, obj)
 
 
 def get_uri(obj) -> str:

dascore/utils/misc.py

@@ -685,5 +685,3 @@ def to_object_array(object_sequence):
     out = np.empty(len(object_sequence), dtype=object)
     out[:] = object_sequence
     return out
-
-

pyproject.toml

@@ -81,6 +81,8 @@ test = [
     "pytest-codeblocks",
     "pytest-cov",
     "twine",
+    "fsspec[github]",
+    "fsspec[http]"
 ]
 
 profile = [
@@ -133,11 +135,16 @@ GDR_DAS__V1 = "dascore.io.gdr.core:GDR_V1"
 
 # -- Pytest config
 # pytest marks
+
 [tool.pytest.ini_options]
 markers = [
     "network: marks tests as needing the internet",
 ]
 
+addopts = [
+    "-m not network"
+]
+
 # -- Linter configuration
 [tool.ruff]
 

tests/conftest.py

@@ -48,6 +48,14 @@ def pytest_addoption(parser):
         default=False,
         help="Run integration tests",
     )
+    # Note: we can't simply use marks as marks currently (jan 2025) be
+    # applied to fixtures.
+    parser.addoption(
+        "--network",
+        action="store_true",
+        default=False,
+        help="Run tests that require network connection. Can be slow.",
+    )
 
 
 def pytest_collection_modifyitems(config, items):
@@ -89,6 +97,7 @@ def swap_index_map_path(tmp_path_factory):
     tmp_map_path = tmp_path_factory.mktemp("cache_paths") / "cache_paths.json"
     setattr(DirectoryIndexer, "index_map_path", tmp_map_path)
 
+
 # --- Coordinate fixtures
 
 COORD_MANAGERS = []