added the FtpDataProvider module, updated the FileDataProvider module

Author: davidnich

CMakeLists.txt

@@ -849,6 +849,7 @@ qore_user_module("qlib/TextWrap.qm" "Util")
 # base 3 modules
 qore_user_module("qlib/FilePoller.qm" "Util;DataProvider")
 qore_user_module("qlib/FileDataProvider" "FsUtil;DataProvider")
+qore_user_module("qlib/FtpDataProvider" "DataProvider")
 qore_user_module("qlib/SqlUtil" "Util;DataProvider")
 qore_user_module("qlib/ConnectionProvider" "Util;DataProvider")
 qore_user_module("qlib/CsvUtil" "Util;DataProvider")

Makefile.am

@@ -82,7 +82,8 @@ DOX_SPLIT_MODULES = \
     qlib/SalesforceRestDataProvider \
     qlib/CdsRestDataProvider \
     qlib/ServiceNowRestDataProvider \
-    qlib/FileDataProvider
+    qlib/FileDataProvider \
+    qlib/FtpDataProvider
 
 SqlUtil_modverdir = $(modverdir)/SqlUtil
 ConnectionProvider_modverdir = $(modverdir)/ConnectionProvider

bin/qdp

@@ -821,7 +821,7 @@ class InputHelper {
         Linenoise::set_callback(\completion());
 
         try {
-            Linenoise::history_load(HistoryFile);
+            Linenoise::history_load(ENV.HOME + "/" + HistoryFile);
         } catch (hash<ExceptionInfo> ex) {
             printf("cannot load history: %s: %s\n", ex.err, ex.desc);
         }

doxygen/lang/120_modules.dox.tmpl

@@ -86,6 +86,8 @@ ModuleName/
         management, programmatic DB access, schema and data synchronization, and more
     |user|<a href="../../modules/FsUtil/html/index.html">FsUtil</a>|Provides a high level API for working with the \
         filesystem
+    |user|<a href="../../modules/FtpDataProvider/html/index.html">FtpDataProvider</a>|Provides a \
+        <a href="../../modules/DataProvider/html/index.html">data provider</a> API for the FTP protocol
     |user|<a href="../../modules/FtpPoller/html/index.html">FtpPoller</a>|Provides an API for polling files from a \
         remote directory with the FTP protocol
     |user|<a href="../../modules/FtpPollerUtil/html/index.html">FtpPollerUtil</a>|Provides definitions for the \

doxygen/lang/900_release_notes.dox.tmpl

@@ -13,6 +13,8 @@
         (<a href="https://github.com/qorelanguage/qore/issues/4557">issue 4557</a>)
     - <a href="../../modules/FileDataProvider/html/index.html">FileDataProvider</a> module
       - new module
+    - <a href="../../modules/FtpDataProvider/html/index.html">FtpDataProvider</a> module
+      - new module
     - <a href="../../modules/FtpPoller/html/index.html">FtpPoller</a> module
       - added support for the event-based DataProvider API
         (<a href="https://github.com/qorelanguage/qore/issues/4557">issue 4557</a>)

examples/test/qlib/FileDataProvider/FileDataProvider.qtest

@@ -65,5 +65,27 @@ public class FileDataProviderTest inherits QUnit::Test {
         });
         assertEq(new_targ, h.path);
         assertFalse(is_file(new_targ));
+
+        prov = fdp.getChildProviderEx("create");
+        h = prov.doRequest({
+            "path": new_targ,
+            "data": str,
+        });
+        assertEq(new_targ, h.path);
+        assertTrue(is_file(new_targ));
+
+        prov = fdp.getChildProviderEx("get");
+        h = prov.doRequest({
+            "path": new_targ,
+            "text": True,
+        });
+        assertEq(new_targ, h.path);
+        assertEq(str, h."data");
+
+        prov = fdp.getChildProviderEx("list");
+        h = prov.doRequest({
+            "path": dirname(tmp.path),
+        })[0];
+        assertEq(basename(new_targ), h.name);
     }
 }

examples/test/qlib/FtpDataProvider/FtpDataProvider.qtest

@@ -0,0 +1,97 @@
+#!/usr/bin/env qore
+# -*- mode: qore; indent-tabs-mode: nil -*-
+
+%require-types
+%enable-all-warnings
+%new-style
+%strict-args
+%allow-injection
+
+%requires ../../../../qlib/Util.qm
+%requires ../../../../qlib/QUnit.qm
+%requires ../../../../qlib/Logger.qm
+%requires ../../../../qlib/DataProvider
+%requires ../../../../qlib/FsUtil.qm
+%requires ../../../../qlib/FtpDataProvider
+
+%include  ../../qore/classes/FtpServer.qc
+
+%exec-class FtpDataProviderTest
+
+public class FtpDataProviderTest inherits QUnit::Test {
+    constructor() : Test("FtpDataProvider Test", "1.0") {
+        addTestCase("test", \ftpDataProviderTest());
+
+        # Return for compatibility with test harness that checks return value.
+        set_return_value(main());
+    }
+
+    ftpDataProviderTest() {
+        int port = getPort();
+
+        # PORT only supports IPv4, so we can't use "localhost", which may resolve
+        # to an IPv6 address
+        FtpServer serv(port, m_options.verbose - 1, "127.0.0.1");
+        on_exit serv.shutdown();
+
+        serv.setFs();
+        serv.setDirect();
+
+        port = serv.getPort();
+        string url = sprintf("ftp://user:pass@127.0.0.1:%d", port);
+
+        FtpDataProvider root({"url": url});
+        AbstractDataProvider prov = root.getChildProviderEx("create-file");
+        string str = get_random_string();
+        string path = "/test.txt";
+        hash<auto> h = prov.doRequest({
+            "path": path,
+            "data": str,
+        });
+        assertEq(path, h.path);
+
+        prov = root.getChildProviderEx("get-file");
+        h = prov.doRequest({
+            "path": path,
+            "text": True,
+        });
+        assertEq(str, h."data");
+
+        prov = root.getChildProviderEx("stat");
+        h = prov.doRequest({
+            "path": path,
+        });
+        assertEq({
+            "name": "test.txt",
+            "path": "/test.txt",
+            "type": "REGULAR",
+            "size": 15,
+        }, h - "mtime");
+
+        /* test server doesn't support RNFR yet
+        string newpath = "/test1.txt";
+        prov = root.getChildProviderEx("move");
+        h = prov.doRequest({
+            "source": path,
+            "target": newpath,
+        });
+        assertEq(newpath, h.path);
+        path = newpath;
+        */
+
+        prov = root.getChildProviderEx("delete");
+        h = prov.doRequest({
+            "path": path,
+        });
+        assertEq(path, h.path);
+
+        prov = root.getChildProviderEx("stat");
+        assertThrows("FTP-STAT-ERROR", \prov.doRequest(), {
+            "path": path,
+        });
+    }
+
+    private softint getPort() {
+        return m_options.port ?? ENV.FTPCLIENT_TEST_PORT ?? 0;
+    }
+}

examples/test/qore/classes/FtpServer.qc

@@ -288,33 +288,65 @@ class FtpServer {
             }
 
             if (cmd =~ /^CWD/i) {
-                string path = (cmd =~ x/^CWD (.+)$/i)[0];
-                pwd = path;
-                ftpSend(ns, 250, "Okay.");
+                string dir = (cmd =~ x/^CWD (.+)$/i)[0];
+
+                *string path = t ? t.path : pwd;
+                path += DirSep + dir;
+
+                if (!is_dir(path)) {
+                    ftpSend(ns, 550, "Failed to change directory to " + dir);
+                } else {
+                    ftpSend(ns, 250, "Okay.");
+                    pwd = dir;
+                }
                 continue;
             }
 
             if (cmd =~ /^MDTM/i) {
                 string file_name = (cmd =~ x/^MDTM (.+)$/i)[0];
                 # printf("FILE_NAME: %y\n", file_name);
 
-                Dir dir();
-                dir.chdir(pwd);
-                auto file = dir.list(file_name, 0, True)[0];
+                *string path = t ? t.path : pwd;
+                path += DirSep + file_name;
 
-                ftpSend(ns, 250, file.mtime.toString());
+                *hash<StatInfo> info = hstat(path);
+                if (!info) {
+                    ftpSend(ns, 550, "Failed to retrieve modified date for " + file_name);
+                } else {
+                    ftpSend(ns, 250, info.mtime.toString());
+                }
                 continue;
             }
 
             if (cmd =~ /^SIZE/i) {
                 string file_name = (cmd =~ x/^SIZE (.+)$/i)[0];
                 # printf("FILE_NAME: %y\n", file_name);
 
-                Dir dir();
-                dir.chdir(pwd);
-                auto file = dir.list(file_name, 0, True)[0];
+                *string path = t ? t.path : pwd;
+                path += DirSep + file_name;
+
+                *hash<StatInfo> info = hstat(path);
+                if (!info) {
+                    ftpSend(ns, 550, "Failed to retrieve size for " + file_name);
+                } else {
+                    ftpSend(ns, 250, info.size.toString());
+                }
+                continue;
+            }
+
+            if (cmd =~ /^DELE/i) {
+                string file_name = (cmd =~ x/^DELE (.+)$/i)[0];
+                # printf("FILE_NAME: %y\n", file_name);
+
+                *string path = t ? t.path : pwd;
+                path += DirSep + file_name;
 
-                ftpSend(ns, 250, file.size.toString());
+                int rc = unlink(path);
+                if (rc) {
+                    ftpSend(ns, 550, "Failed to delete " + file_name);
+                } else {
+                    ftpSend(ns, 250, "Okay.");
+                }
                 continue;
             }
 

lib/QC_FtpClient.qpp

@@ -176,7 +176,7 @@ nothing FtpClient::connect() {
 
     @par Example:
     @code{.py}
-delete ftp;
+ftp.disconnect();
     @endcode
  */
 nothing FtpClient::disconnect() {

lib/ql_file.qpp

@@ -683,8 +683,9 @@ nothing unlink() [flags=RUNTIME_NOOP;dom=FILESYSTEM] {
 
     @par Example:
     @code{.py}
-if (unlink(path))
+if (umask(mask)) {
     printf("%s: %s\n", path, strerror());
+}
     @endcode
  */
 int umask(softint mask) [dom=PROCESS] {

qlib/DataProvider/DataProvider.qc

@@ -40,6 +40,7 @@ public class DataProvider {
             "filepoller": "FilePoller",
             "fixedlengthread": "FixedLengthUtil",
             "fixedlengthwrite": "FixedLengthUtil",
+            "ftp": "FtpDataProvider",
             "ftppoller": "FtpPoller",
             "salesforcerest": "SalesforceRestDataProvider",
             "servicenowrest": "ServiceNowRestDataProvider",

qlib/FileDataProvider/FileCopyDataProvider.qc

@@ -1,7 +1,7 @@
 # -*- mode: qore; indent-tabs-mode: nil -*-
-#! Qore FileCopyDataProvider module definition
+#! Qore FileDataProvider module definition
 
-/** FileCopyDataProvider.qc Copyright 2019 - 2022 Qore Technologies, s.r.o.
+/** FileCopyDataProvider.qc Copyright 2022 Qore Technologies, s.r.o.
 
     Permission is hereby granted, free of charge, to any person obtaining a
     copy of this software and associated documentation files (the "Software"),
@@ -22,9 +22,9 @@
     DEALINGS IN THE SOFTWARE.
 */
 
-#! contains all public definitions in the FileCopyDataProvider module
+#! contains all public definitions in the FileDataProvider module
 public namespace FileDataProvider {
-#! The File data provider class
+#! The File copy data provider class
 public class FileCopyDataProvider inherits AbstractDataProvider {
     public {
         #! Provider info

qlib/FileDataProvider/FileCopyRequestDataType.qc

@@ -21,7 +21,7 @@
     DEALINGS IN THE SOFTWARE.
 */
 
-#! contains all public definitions in the FileCopyDataProvider module
+#! contains all public definitions in the FileDataProvider module
 public namespace FileDataProvider {
 #! Data type for copy file request calls
 public class FileCopyRequestDataType inherits FileMoveRequestDataType {

qlib/FileDataProvider/FileCreateDataProvider.qc

@@ -0,0 +1,92 @@
+# -*- mode: qore; indent-tabs-mode: nil -*-
+#! Qore FileDataProvider module definition
+
+/** FileCreateDataProvider.qc Copyright 2022 Qore Technologies, s.r.o.
+
+    Permission is hereby granted, free of charge, to any person obtaining a
+    copy of this software and associated documentation files (the "Software"),
+    to deal in the Software without restriction, including without limitation
+    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+    and/or sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+*/
+
+#! contains all public definitions in the FileDataProvider module
+public namespace FileDataProvider {
+#! The create file data provider class
+public class FileCreateDataProvider inherits AbstractDataProvider {
+    public {
+        #! Provider info
+        const ProviderInfo = <DataProviderInfo>{
+            "type": "FileCreateDataProvider",
+            "supports_request": True,
+        };
+
+        #! Request type
+        const RequestType = new FileCreateRequestDataType();
+
+        #! Response type
+        const ResponseType = new FilePathDataType();
+    }
+
+    #! Creates the object from constructor options
+    constructor(*hash<auto> options) {
+        checkOptions("CONSTRUCTOR-ERROR", NOTHING, options);
+    }
+
+    #! Returns the data provider name
+    string getName() {
+        return "create";
+    }
+
+    #! Returns the data provider description
+    *string getDesc() {
+        return "Create file data provider; creates a file in the given location";
+    }
+
+    #! Makes a request and returns the response
+    /** @param req the request info
+        @param request_options the request options after processing by validateRequestOptions()
+
+        @return the response to the request
+    */
+    private auto doRequestImpl(auto req, *hash<auto> request_options) {
+        File f();
+        f.open2(req.path, O_CREAT | O_TRUNC | O_WRONLY, req.mode);
+        f.write(req."data");
+        return {
+            "path": req.path,
+        };
+    }
+
+    #! Returns the description of a successful request message, if any
+    /** @return the request type for this provider
+    */
+    private *AbstractDataProviderType getRequestTypeImpl() {
+        return RequestType;
+    }
+
+    #! Returns the description of a response message, if this object represents a response message
+    /** @return the response type for this response message
+    */
+    private *AbstractDataProviderType getResponseTypeImpl() {
+        return ResponseType;
+    }
+
+    #! Returns data provider static info
+    hash<DataProviderInfo> getStaticInfoImpl() {
+        return ProviderInfo;
+    }
+}
+}