Implement a pyfilesystem for the Gallery

pyproject.toml

@@ -24,6 +24,7 @@ exclude = ["examples/*", "rspace_client/tests/*"]
 python = "^3.7.11"
 requests = "^2.25.1"
 beautifulsoup4 = "^4.9.3"
+fs = "^2.4.16"
 
 [tool.poetry.dev-dependencies]
 black = "^21.6b0"

rspace_client/client_base.py

@@ -181,18 +181,25 @@ def get_link(self, response, link_rel):
             )
         )
 
-    def download_link_to_file(self, url, filename):
+    def download_link_to_file(self, url, filename, chunk_size=128):
         """
         Downloads a file from the API server.
         :param url: URL of the file to be downloaded
-        :param filename: file path to save the file to
+        :param filename: file path to save the file to or an already opened file object
+        :param chunk_size: size of the chunks to download at a time, default is 128
         """
         headers = {"apiKey": self.api_key, "Accept": "application/octet-stream"}
-        with open(filename, "wb") as fd:
+        if isinstance(filename, str):
+            with open(filename, "wb") as fd:
+                for chunk in requests.get(url, headers=headers).iter_content(
+                    chunk_size=chunk_size
+                ):
+                    fd.write(chunk)
+        else:
             for chunk in requests.get(url, headers=headers).iter_content(
-                chunk_size=128
+                chunk_size=chunk_size
             ):
-                fd.write(chunk)
+                filename.write(chunk)
 
     def link_exists(self, response, link_rel):
         """

rspace_client/eln/eln.py

@@ -334,17 +334,18 @@ def get_file_info(self, file_id):
         numeric_file_id = self._get_numeric_record_id(file_id)
         return self.retrieve_api_results("/files/{}".format(numeric_file_id))
 
-    def download_file(self, file_id, filename):
+    def download_file(self, file_id, filename, chunk_size=128):
         """
         Downloads file contents. More information on
         https://community.researchspace.com/public/apiDocs (or your own instance's /public/apiDocs).
         :param file_id: numeric document ID or global ID
-        :param filename: file path to save the file to
+        :param filename: file path to save the file to or a file object
+        :param chunk_size: size of chunks to download at a time (optional, default is 128 bytes)
         """
         numeric_file_id = self._get_numeric_record_id(file_id)
         url_base = self._get_api_url()
         return self.download_link_to_file(
-            f"{url_base}/files/{numeric_file_id}/file", filename
+            f"{url_base}/files/{numeric_file_id}/file", filename, chunk_size
         )
 
     def upload_file(self, file, folder_id=None, caption=None):

rspace_client/eln/fs.py

@@ -0,0 +1,132 @@
+from fs.base import FS
+from rspace_client.eln import eln
+from typing import Optional, List, Text, BinaryIO, Mapping, Any
+from fs.info import Info
+from fs.permissions import Permissions
+from fs.subfs import SubFS
+from fs import errors
+from fs.mode import Mode
+from io import BytesIO
+
+def path_to_id(path):
+    """
+        A path is a slash-delimited string of Global Ids. The last element is
+        the global id of the file or folder. This function extract just the id
+        of the file or folder, which will be a string-encoding of a number.
+    """
+    global_id = path
+    if '/' in path:
+        global_id = path.split('/')[-1]
+    return global_id[2:]
+
+
+def is_folder(path):
+    return path.split('/')[-1][:2] == "GF"
+
+
+class GalleryInfo(Info):
+    def __init__(self, obj, *args, **kwargs) -> None:
+        super().__init__(obj, *args, **kwargs)
+        self.globalId = obj['rspace']['globalId'];
+
+
+class GalleryFilesystem(FS):
+
+    def __init__(self, server: str, api_key: str) -> None:
+        super(GalleryFilesystem, self).__init__()
+        self.eln_client = eln.ELNClient(server, api_key)
+        self.gallery_id = next(file['id'] for file in self.eln_client.list_folder_tree()['records'] if file['name'] == 'Gallery')
+
+    def getinfo(self, path, namespaces=None) -> Info:
+        is_file = path.split('/')[-1][:2] == "GL"
+        info = None
+        if is_folder(path):
+            info = self.eln_client.get_folder(path_to_id(path))
+        if is_file:
+            info = self.eln_client.get_file_info(path_to_id(path))
+        if info is None:
+            raise errors.ResourceNotFound(path)
+        return GalleryInfo({
+            "basic": {
+                "name": (is_folder(path) and "GF" or "GL") + path_to_id(path),
+                "is_dir": is_folder(path),
+            },
+            "details": {
+                "size": info.get('size', 0),
+                "type": is_folder(path) and "1" or "2",
+            },
+            "rspace": info,
+        })
+
+    def listdir(self, path: Text) -> List[Text]:
+        id = path in [u'.', u'/', u'./'] and self.gallery_id or path_to_id(path)
+        return [file['globalId'] for file in self.eln_client.list_folder_tree(id)['records']]
+
+    def makedir(self, path: Text, permissions: Optional[Permissions] = None, recreate: bool = False) -> SubFS[FS]:
+        new_folder_name = path.split('/')[-1]
+        parent_id = path_to_id(path[:-(len(new_folder_name) + 1)])
+        new_id = self.eln_client.create_folder(new_folder_name, parent_id)['id']
+        return self.opendir("/GF" + str(new_id))
+
+    def openbin(self, path: Text, mode: Text = 'r', buffering: int = -1, **options) -> BinaryIO:
+        """
+        This method is added for conformance with the FS interface, but in
+        almost all circumstances you probably want to be using upload and
+        download directly as they have more information available e.g. uploaded
+        files will have the same name as the source file when called directly
+        """
+        _mode = Mode(mode)
+        if _mode.reading and _mode.writing:
+            raise errors.Unsupported("read/write mode")
+        if _mode.appending:
+            raise errors.Unsupported("appending mode")
+        if _mode.exclusive:
+            raise errors.Unsupported("exclusive mode")
+        if _mode.truncate:
+            raise errors.Unsupported("truncate mode")
+
+        if _mode.reading:
+            file = BytesIO()
+            self.download(path, file)
+            file.seek(0)
+            return file
+
+        if _mode.writing:
+            file = BytesIO()
+            def upload_callback():
+                file.seek(0)
+                self.upload(path, file)
+            file.close = upload_callback
+            return file
+
+        raise errors.Unsupported("mode {!r}".format(_mode))
+
+    def remove(self, path: Text) -> None:
+        raise NotImplementedError
+
+    def removedir(self, path: Text, recursive: bool = False, force: bool = False) -> None:
+        if path in [u'.', u'/', u'./']:
+            raise errors.RemoveRootError()
+        if (not is_folder(path)):
+            raise errors.DirectoryExpected(path)
+        if len(self.listdir(path)) > 0:
+            raise errors.DirectoryNotEmpty(path)
+        self.eln_client.delete_folder(path_to_id(path))
+
+    def setinfo(self, path: Text, info: Mapping[Text, Mapping[Text, object]]) -> None:
+        raise NotImplementedError
+
+    def download(self, path: Text, file: BinaryIO, chunk_size: Optional[int] = None, **options: Any) -> None:
+        if chunk_size is not None:
+            self.eln_client.download_file(path_to_id(path), file, chunk_size)
+        else:
+            self.eln_client.download_file(path_to_id(path), file)
+
+    def upload(self, path: Text, file: BinaryIO, chunk_size: Optional[int] = None, **options: Any) -> None:
+        """
+        :param path: Global Id of a folder in the appropriate gallery section or
+                     else if empty then the upload will be placed in the Api
+                     Imports folder of the relevant gallery section
+        :param file: a binary file object to be uploaded
+        """
+        self.eln_client.upload_file(file, path_to_id(path) if path else None)

rspace_client/tests/eln_fs_test.py

@@ -0,0 +1,184 @@
+from unittest.mock import patch, MagicMock, ANY
+import unittest
+from rspace_client.eln.fs import path_to_id, GalleryFilesystem
+from io import BytesIO
+
+def mock_requests_get(url, *args, **kwargs):
+    mock_response = MagicMock()
+    if url.endswith('/folders/tree'):
+        mock_response.json.return_value = {
+            'records': [
+                {'id': '123', 'name': 'Gallery'}
+            ]
+        }
+    elif url.endswith('/folders/tree/123'):
+        mock_response.json.return_value = {
+            'records': [
+                {'globalId': 'GF123', 'name': 'Folder1'},
+                {'globalId': 'GL456', 'name': 'File1'},
+                {'globalId': 'GF789', 'name': 'Folder2'},
+                {'globalId': 'GL012', 'name': 'File2'}
+            ]
+        }
+    elif url.endswith('/folders/tree/456'):
+            mock_response.json.return_value = {
+                'records': [
+                ]
+            }
+    elif url.endswith('/folders/123'):
+        mock_response.json.return_value = {
+            'id': '123',
+            'globalId': 'GF123',
+            'name': 'Test Folder',
+            'size': 0,
+            'created': '2025-02-11T16:00:16.392Z',
+            'lastModified': '2025-02-11T16:00:16.392Z',
+            'parentFolderId': 131,
+            'notebook': False,
+            'mediaType': 'Images',
+            'pathToRootFolder': None,
+            '_links': [{'link': 'http://localhost:8080/api/v1/folders/306', 'rel': 'self'}]
+        }
+    elif url.endswith('/folders/456'):
+        mock_response.json.return_value = {
+            'id': '456',
+            'globalId': 'GF456',
+            'name': 'newFolder',
+            'size': 0,
+            'created': '2025-02-11T16:00:16.392Z',
+            'lastModified': '2025-02-11T16:00:16.392Z',
+            'parentFolderId': 131,
+            'notebook': False,
+            'mediaType': 'Images',
+            'pathToRootFolder': None,
+            '_links': [{'link': 'http://localhost:8080/api/v1/folders/306', 'rel': 'self'}]
+        }
+    elif url.endswith('/files/123'):
+        mock_response.json.return_value = {
+            'id': '123',
+            'globalId': 'GF123',
+            'name': 'Test File',
+            'size': 1024,
+            'created': '2025-02-11T16:00:16.392Z',
+            'lastModified': '2025-02-11T16:00:16.392Z',
+            'parentFolderId': 131,
+            'notebook': False,
+            'mediaType': 'Images',
+            'pathToRootFolder': None,
+            '_links': [{'link': 'http://localhost:8080/api/v1/folders/306', 'rel': 'self'}]
+        }
+    else:
+        mock_response.json.return_value = {}
+    mock_response.headers = {'Content-Type': 'application/json'}
+    return mock_response
+
+def mock_requests_post(url, *args, **kwargs):
+    mock_response = MagicMock()
+    mock_response.json.return_value = {
+        'id': '456',
+        'globalId': 'GF456',
+        'name': 'newFolder',
+        'size': 0,
+        'created': '2025-02-11T16:00:16.392Z',
+        'lastModified': '2025-02-11T16:00:16.392Z',
+        'parentFolderId': 131,
+        'notebook': False,
+        'mediaType': 'Images',
+        'pathToRootFolder': None,
+        '_links': [{'link': 'http://localhost:8080/api/v1/folders/306', 'rel': 'self'}]
+        }
+    mock_response.headers = {'Content-Type': 'application/json'}
+    return mock_response
+
+
+class ElnFilesystemTest(unittest.TestCase):
+
+    @patch('requests.get', side_effect=mock_requests_get)
+    def setUp(self, mock_get) -> None:
+        super().setUp()
+        self.fs = GalleryFilesystem("https://example.com", "api_key")
+
+    def test_path_to_id(self):
+        self.assertEqual("123", path_to_id("GF123"))
+        self.assertEqual("123", path_to_id("/GF123"))
+        self.assertEqual("456", path_to_id("GF123/GF456"))
+        self.assertEqual("456", path_to_id("/GF123/GF456"))
+
+    @patch('requests.get', side_effect=mock_requests_get)
+    def test_get_info_folder(self, mock_get):
+        folder_info = self.fs.getinfo("GF123")
+        self.assertEqual("GF123", folder_info.raw["basic"]["name"])
+        self.assertTrue(folder_info.raw["basic"]["is_dir"])
+        self.assertEqual(0, folder_info.raw["details"]["size"])
+
+    @patch('requests.get', side_effect=mock_requests_get)
+    def test_get_info_file(self, mock_get):
+        file_info = self.fs.getinfo("GL123")
+        self.assertEqual("GL123", file_info.raw["basic"]["name"])
+        self.assertFalse(file_info.raw["basic"]["is_dir"])
+        self.assertEqual(1024, file_info.raw["details"]["size"])
+
+    @patch('requests.get', side_effect=mock_requests_get)
+    def test_listdir_root(self, mock_list_folder_tree):
+        result = self.fs.listdir('/')
+        expected = ['GF123', 'GL456', 'GF789', 'GL012']
+        self.assertEqual(result, expected)
+
+    @patch('requests.get', side_effect=mock_requests_get)
+    def test_listdir_root_specific_folder(self, mock_list_folder_tree):
+        expected = ['GF123', 'GL456', 'GF789', 'GL012']
+        result = self.fs.listdir('GF123')
+        self.assertEqual(result, expected)
+
+    @patch('requests.request', side_effect=mock_requests_post)
+    @patch('requests.get', side_effect=mock_requests_get)
+    def test_makedir(self, mock_get, mock_post):
+        self.fs.makedir('GF123/newFolder')
+        mock_post.assert_called_once_with(
+            'POST',
+            'https://example.com/api/v1/folders',
+            json={'name': 'newFolder', 'parentFolderId': 123, 'notebook': False},
+            headers=ANY
+        )
+
+    @patch('requests.request', side_effect=mock_requests_post)
+    @patch('requests.get', side_effect=mock_requests_get)
+    def test_removedir(self, mock_get, mock_post):
+        self.fs.removedir('GF456')
+        mock_post.assert_called_once_with(
+            'DELETE',
+            'https://example.com/api/v1/folders/456',
+            json=ANY,
+            headers=ANY
+        )
+
+    @patch('requests.get')
+    def test_download(self, mock_get):
+        mock_response = MagicMock()
+        mock_response.iter_content = MagicMock(return_value=[b'chunk1', b'chunk2', b'chunk3'])
+        mock_get.return_value = mock_response
+        file_obj = BytesIO()
+        self.fs.download('/GL123', file_obj)
+        file_obj.seek(0)
+        self.assertEqual(file_obj.read(), b'chunk1chunk2chunk3')
+        mock_get.assert_called_once_with(
+            'https://example.com/api/v1/files/123/file',
+            headers=ANY
+        )
+
+    @patch('requests.post')
+    def test_upload(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.json.return_value = {'id': '456'}
+        mock_post.return_value = mock_response
+        file_obj = BytesIO(b'test file content')
+        self.fs.upload('/GF123', file_obj)
+        mock_post.assert_called_once_with(
+            'https://example.com/api/v1/files',
+            files={'file': file_obj},
+            data={'folderId': 123},
+            headers=ANY
+        )
+
+if __name__ == '__main__':
+    unittest.main()