diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 4f97cdde5ac4..1a891b6e959e 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -68,6 +68,10 @@ add_library( gdalalg_vector_select.cpp gdalalg_vector_sql.cpp gdalalg_vector_write.cpp + gdalalg_vfs.cpp + gdalalg_vfs_copy.cpp + gdalalg_vfs_delete.cpp + gdalalg_vfs_list.cpp gdalinfo_lib.cpp gdalbuildvrt_lib.cpp gdal_grid_lib.cpp diff --git a/apps/gdalalg_vfs.cpp b/apps/gdalalg_vfs.cpp new file mode 100644 index 000000000000..7f299280b22a --- /dev/null +++ b/apps/gdalalg_vfs.cpp @@ -0,0 +1,48 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: gdal "vfs" subcommand + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2025, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "gdalalgorithm.h" + +#include "gdalalg_vfs_copy.h" +#include "gdalalg_vfs_delete.h" +#include "gdalalg_vfs_list.h" + +/************************************************************************/ +/* GDALVFSAlgorithm */ +/************************************************************************/ + +class GDALVFSAlgorithm final : public GDALAlgorithm +{ + public: + static constexpr const char *NAME = "vfs"; + static constexpr const char *DESCRIPTION = + "GDAL Virtual file system (VSI) commands."; + static constexpr const char *HELP_URL = "/programs/gdal_vfs.html"; + + GDALVFSAlgorithm() : GDALAlgorithm(NAME, DESCRIPTION, HELP_URL) + { + RegisterSubAlgorithm(); + RegisterSubAlgorithm(); + RegisterSubAlgorithm(); + } + + private: + bool RunImpl(GDALProgressFunc, void *) override + { + CPLError(CE_Failure, CPLE_AppDefined, + "The Run() method should not be called directly on the \"gdal " + "vfs\" program."); + return false; + } +}; + +GDAL_STATIC_REGISTER_ALG(GDALVFSAlgorithm); diff --git a/apps/gdalalg_vfs_copy.cpp b/apps/gdalalg_vfs_copy.cpp new file mode 100644 index 000000000000..a15765bb2fd5 --- /dev/null +++ b/apps/gdalalg_vfs_copy.cpp @@ -0,0 +1,316 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: gdal "vfs copy" subcommand + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2025, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "gdalalg_vfs_copy.h" + +#include "cpl_conv.h" +#include "cpl_string.h" +#include "cpl_vsi.h" + +#include + +//! @cond Doxygen_Suppress + +#ifndef _ +#define _(x) (x) +#endif + +/************************************************************************/ +/* GDALVFSCopyAlgorithm::GDALVFSCopyAlgorithm() */ +/************************************************************************/ + +GDALVFSCopyAlgorithm::GDALVFSCopyAlgorithm() + : GDALAlgorithm(NAME, DESCRIPTION, HELP_URL) +{ + { + auto &arg = + AddArg("source", 0, _("Source file or directory name"), &m_source) + .SetPositional() + .SetRequired(); + SetAutoCompleteFunctionForFilename(arg, 0); + arg.AddValidationAction( + [this]() + { + if (m_source.empty()) + { + ReportError(CE_Failure, CPLE_IllegalArg, + "Source filename cannot be empty"); + return false; + } + return true; + }); + } + { + auto &arg = + AddArg("destination", 0, _("Destination file or directory name"), + &m_destination) + .SetPositional() + .SetRequired(); + SetAutoCompleteFunctionForFilename(arg, 0); + arg.AddValidationAction( + [this]() + { + if (m_destination.empty()) + { + ReportError(CE_Failure, CPLE_IllegalArg, + "Destination filename cannot be empty"); + return false; + } + return true; + }); + } + + AddArg("recursive", 'r', _("Copy subdirectories recursively"), + &m_recursive); + + AddArg("skip-errors", 0, _("Skip errors"), &m_skip); + AddProgressArg(); +} + +/************************************************************************/ +/* GDALVFSCopyAlgorithm::RunImpl() */ +/************************************************************************/ + +bool GDALVFSCopyAlgorithm::RunImpl(GDALProgressFunc pfnProgress, + void *pProgressData) +{ + if (m_recursive || cpl::ends_with(m_source, "/*") || + cpl::ends_with(m_source, "\\*")) + { + // Make sure that copy -r [srcdir/]lastsubdir targetdir' creates + // targetdir/lastsubdir if targetdir already exists (like cp -r does). + if (m_source.back() == '/') + m_source.pop_back(); + + if (!cpl::ends_with(m_source, "/*") && !cpl::ends_with(m_source, "\\*")) + { + VSIStatBufL statBufSrc; + bool srcExists = + VSIStatExL(m_source.c_str(), &statBufSrc, + VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0; + if (!srcExists) + { + srcExists = + VSIStatExL( + std::string(m_source).append("/").c_str(), &statBufSrc, + VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0; + } + VSIStatBufL statBufDst; + const bool dstExists = + VSIStatExL(m_destination.c_str(), &statBufDst, + VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0; + if (srcExists && VSI_ISDIR(statBufSrc.st_mode) && dstExists && + VSI_ISDIR(statBufDst.st_mode)) + { + if (m_destination.back() == '/') + m_destination.pop_back(); + const auto srcLastSlashPos = m_source.rfind('/'); + if (srcLastSlashPos != std::string::npos) + m_destination += m_source.substr(srcLastSlashPos); + else + m_destination = CPLFormFilenameSafe( + m_destination.c_str(), m_source.c_str(), nullptr); + } + } + else + { + m_source.resize(m_source.size() - 2); + } + + uint64_t curAmount = 0; + return CopyRecursive(m_source, m_destination, 0, m_recursive ? -1 : 0, + curAmount, 0, pfnProgress, pProgressData); + } + else + { + VSIStatBufL statBufSrc; + bool srcExists = + VSIStatExL(m_source.c_str(), &statBufSrc, + VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0; + if (!srcExists) + { + ReportError(CE_Failure, CPLE_FileIO, "%s does not exist", + m_source.c_str()); + return false; + } + if (VSI_ISDIR(statBufSrc.st_mode)) + { + ReportError(CE_Failure, CPLE_FileIO, + "%s is a directory. Use -r/--recursive option", + m_source.c_str()); + return false; + } + + return CopySingle(m_source, m_destination, ~(static_cast(0)), + pfnProgress, pProgressData); + } +} + +/************************************************************************/ +/* GDALVFSCopyAlgorithm::CopySingle() */ +/************************************************************************/ + +bool GDALVFSCopyAlgorithm::CopySingle(const std::string &src, + const std::string &dstIn, uint64_t size, + GDALProgressFunc pfnProgress, + void *pProgressData) const +{ + CPLDebug("gdal_vfs_copy", "Copying file %s...", src.c_str()); + VSIStatBufL sStat; + std::string dst = dstIn; + const bool bExists = + VSIStatExL(dst.back() == '/' ? dst.c_str() + : std::string(dst).append("/").c_str(), + &sStat, VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0; + if ((!bExists && dst.back() == '/') || + (bExists && VSI_ISDIR(sStat.st_mode))) + { + const std::string filename = CPLGetFilename(src.c_str()); + dst = CPLFormFilenameSafe(dst.c_str(), filename.c_str(), nullptr); + } + return VSICopyFile(src.c_str(), dst.c_str(), nullptr, size, nullptr, + pfnProgress, pProgressData) == 0 || + m_skip; +} + +/************************************************************************/ +/* GDALVFSCopyAlgorithm::CopyRecursive() */ +/************************************************************************/ + +bool GDALVFSCopyAlgorithm::CopyRecursive(const std::string &srcIn, + const std::string &dst, int depth, + int maxdepth, uint64_t &curAmount, + uint64_t totalAmount, + GDALProgressFunc pfnProgress, + void *pProgressData) const +{ + std::string src(srcIn); + if (src.back() == '/') + src.pop_back(); + + if (pfnProgress && depth == 0) + { + CPLDebug("gdal_vfs_copy", "Listing source files..."); + std::unique_ptr dir( + VSIOpenDir(src.c_str(), maxdepth, nullptr), VSICloseDir); + if (dir) + { + while (const auto entry = VSIGetNextDirEntry(dir.get())) + { + if (!(entry->pszName[0] == '.' && + (entry->pszName[1] == '.' || entry->pszName[1] == 0))) + { + totalAmount += entry->nSize + 1; + if (!pfnProgress(0.0, "", pProgressData)) + return false; + } + } + } + } + + CPLDebug("gdal_vfs_copy", "Copying directory %s...", src.c_str()); + std::unique_ptr dir( + VSIOpenDir(src.c_str(), 0, nullptr), VSICloseDir); + if (dir) + { + VSIStatBufL sStat; + if (VSIStatL(dst.c_str(), &sStat) != 0) + { + if (VSIMkdir(dst.c_str(), 0755) != 0) + { + ReportError(m_skip ? CE_Warning : CE_Failure, CPLE_FileIO, + "Cannot create directory %s", dst.c_str()); + return m_skip; + } + } + + while (const auto entry = VSIGetNextDirEntry(dir.get())) + { + if (!(entry->pszName[0] == '.' && + (entry->pszName[1] == '.' || entry->pszName[1] == 0))) + { + const std::string subsrc = + CPLFormFilenameSafe(src.c_str(), entry->pszName, nullptr); + if (VSI_ISDIR(entry->nMode)) + { + const std::string subdest = CPLFormFilenameSafe( + dst.c_str(), entry->pszName, nullptr); + if (maxdepth < 0 || depth < maxdepth) + { + if (!CopyRecursive(subsrc, subdest, depth + 1, maxdepth, + curAmount, totalAmount, pfnProgress, + pProgressData) && + !m_skip) + { + return false; + } + } + else + { + if (VSIStatL(subdest.c_str(), &sStat) != 0) + { + if (VSIMkdir(subdest.c_str(), 0755) != 0) + { + ReportError(m_skip ? CE_Warning : CE_Failure, + CPLE_FileIO, + "Cannot create directory %s", + subdest.c_str()); + if (!m_skip) + return false; + } + } + } + curAmount += 1; + + if (pfnProgress && + !pfnProgress( + std::min(1.0, static_cast(curAmount) / + static_cast(totalAmount)), + "", pProgressData)) + { + return false; + } + } + else + { + void *pScaledProgressData = GDALCreateScaledProgress( + static_cast(curAmount) / + static_cast(totalAmount), + std::min(1.0, static_cast(curAmount + + entry->nSize + 1) / + static_cast(totalAmount)), + pfnProgress, pProgressData); + const bool bRet = CopySingle( + subsrc, dst, entry->nSize, + pScaledProgressData ? GDALScaledProgress : nullptr, + pScaledProgressData); + GDALDestroyScaledProgress(pScaledProgressData); + + curAmount += entry->nSize + 1; + + if (!bRet) + return false; + } + } + } + } + else + { + ReportError(m_skip ? CE_Warning : CE_Failure, CPLE_AppDefined, + "%s is not a directory or cannot be opened", src.c_str()); + if (!m_skip) + return false; + } + return true; +} + +//! @endcond diff --git a/apps/gdalalg_vfs_copy.h b/apps/gdalalg_vfs_copy.h new file mode 100644 index 000000000000..0b68831129cc --- /dev/null +++ b/apps/gdalalg_vfs_copy.h @@ -0,0 +1,59 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: gdal "vfs copy" subcommand + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2025, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#ifndef GDALALG_VFS_COPY_INCLUDED +#define GDALALG_VFS_COPY_INCLUDED + +#include "gdalalgorithm.h" + +//! @cond Doxygen_Suppress + +/************************************************************************/ +/* GDALVFSCopyAlgorithm */ +/************************************************************************/ + +class GDALVFSCopyAlgorithm final : public GDALAlgorithm +{ + public: + static constexpr const char *NAME = "copy"; + static constexpr const char *DESCRIPTION = + "Copy files located on GDAL Virtual file systems (VSI)."; + static constexpr const char *HELP_URL = "/programs/gdal_vfs_copy.html"; + + static std::vector GetAliasesStatic() + { + return {"cp"}; + } + + GDALVFSCopyAlgorithm(); + + private: + std::string m_source{}; + std::string m_destination{}; + bool m_recursive = false; + bool m_skip = false; + + bool RunImpl(GDALProgressFunc, void *) override; + + bool CopySingle(const std::string &src, const std::string &dst, + uint64_t size, GDALProgressFunc pfnProgress, + void *pProgressData) const; + + bool CopyRecursive(const std::string &src, const std::string &dst, + int depth, int maxdepth, uint64_t &curAmount, + uint64_t totalAmount, GDALProgressFunc pfnProgress, + void *pProgressData) const; +}; + +//! @endcond + +#endif diff --git a/apps/gdalalg_vfs_delete.cpp b/apps/gdalalg_vfs_delete.cpp new file mode 100644 index 000000000000..85ccae4cada6 --- /dev/null +++ b/apps/gdalalg_vfs_delete.cpp @@ -0,0 +1,90 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: gdal "vfs delete" subcommand + * Author: Even Rouault + * + ****************************************************************************** + * Deleteright (c) 2025, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "gdalalg_vfs_delete.h" + +#include "cpl_conv.h" +#include "cpl_string.h" +#include "cpl_vsi.h" + +#include + +//! @cond Doxygen_Suppress + +#ifndef _ +#define _(x) (x) +#endif + +/************************************************************************/ +/* GDALVFSDeleteAlgorithm::GDALVFSDeleteAlgorithm() */ +/************************************************************************/ + +GDALVFSDeleteAlgorithm::GDALVFSDeleteAlgorithm() + : GDALAlgorithm(NAME, DESCRIPTION, HELP_URL) +{ + { + auto &arg = AddArg("filename", 0, _("File or directory name to delete"), + &m_filename) + .SetPositional() + .SetRequired(); + SetAutoCompleteFunctionForFilename(arg, 0); + arg.AddValidationAction( + [this]() + { + if (m_filename.empty()) + { + ReportError(CE_Failure, CPLE_IllegalArg, + "Filename cannot be empty"); + return false; + } + return true; + }); + } + + AddArg("recursive", 'r', _("Delete directories recursively"), &m_recursive) + .AddShortNameAlias('R'); +} + +/************************************************************************/ +/* GDALVFSDeleteAlgorithm::RunImpl() */ +/************************************************************************/ + +bool GDALVFSDeleteAlgorithm::RunImpl(GDALProgressFunc, void *) +{ + bool ret = false; + VSIStatBufL sStat; + if (VSIStatL(m_filename.c_str(), &sStat) != 0) + { + ReportError(CE_Failure, CPLE_FileIO, "%s does not exist", + m_filename.c_str()); + } + else + { + if (m_recursive) + { + ret = VSIRmdirRecursive(m_filename.c_str()) == 0; + } + else + { + ret = VSI_ISDIR(sStat.st_mode) ? VSIRmdir(m_filename.c_str()) == 0 + : VSIUnlink(m_filename.c_str()) == 0; + } + if (!ret) + { + ReportError(CE_Failure, CPLE_FileIO, "Cannot delete %s", + m_filename.c_str()); + } + } + return ret; +} + +//! @endcond diff --git a/apps/gdalalg_vfs_delete.h b/apps/gdalalg_vfs_delete.h new file mode 100644 index 000000000000..ec20766cb45d --- /dev/null +++ b/apps/gdalalg_vfs_delete.h @@ -0,0 +1,48 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: gdal "vfs delete" subcommand + * Author: Even Rouault + * + ****************************************************************************** + * Deleteright (c) 2025, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#ifndef GDALALG_VFS_DELETE_INCLUDED +#define GDALALG_VFS_DELETE_INCLUDED + +#include "gdalalgorithm.h" + +//! @cond Doxygen_Suppress + +/************************************************************************/ +/* GDALVFSDeleteAlgorithm */ +/************************************************************************/ + +class GDALVFSDeleteAlgorithm final : public GDALAlgorithm +{ + public: + static constexpr const char *NAME = "delete"; + static constexpr const char *DESCRIPTION = + "Delete files located on GDAL Virtual file systems (VSI)."; + static constexpr const char *HELP_URL = "/programs/gdal_vfs_delete.html"; + + static std::vector GetAliasesStatic() + { + return {"rm", "rmdir", "del"}; + } + + GDALVFSDeleteAlgorithm(); + + private: + std::string m_filename{}; + bool m_recursive = false; + + bool RunImpl(GDALProgressFunc, void *) override; +}; + +//! @endcond + +#endif diff --git a/apps/gdalalg_vfs_list.cpp b/apps/gdalalg_vfs_list.cpp new file mode 100644 index 000000000000..338717188929 --- /dev/null +++ b/apps/gdalalg_vfs_list.cpp @@ -0,0 +1,307 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: gdal "vfs list" subcommand + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2025, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "gdalalg_vfs_list.h" + +#include "cpl_string.h" +#include "cpl_time.h" +#include "cpl_vsi.h" + +#include + +//! @cond Doxygen_Suppress + +#ifndef _ +#define _(x) (x) +#endif + +/************************************************************************/ +/* GDALVFSListAlgorithm::GDALVFSListAlgorithm() */ +/************************************************************************/ + +GDALVFSListAlgorithm::GDALVFSListAlgorithm() + : GDALAlgorithm(NAME, DESCRIPTION, HELP_URL), m_oWriter(JSONPrint, this) +{ + auto &arg = AddArg("filename", 0, _("File or directory name"), &m_filename) + .SetPositional() + .SetRequired(); + SetAutoCompleteFunctionForFilename(arg, 0); + + AddOutputFormatArg(&m_format).SetDefault("json").SetChoices("json", "text"); + + AddArg("long-listing", 'l', _("Use a long listing format"), &m_longListing) + .AddAlias("long"); + AddArg("recursive", 'R', _("List subdirectories recursively"), + &m_recursive); + AddArg("depth", 0, _("Maximum depth in recursive mode"), &m_depth) + .SetMinValueIncluded(1); + AddArg("absolute-path", 0, _("Display absolute path"), &m_absolutePath) + .AddAlias("abs"); + AddArg("tree", 0, _("Use a hierarchical presentation for JSON output"), + &m_JSONAsTree); + + AddOutputStringArg(&m_output); + AddArg( + "stdout", 0, + _("Directly output on stdout. If enabled, output-string will be empty"), + &m_stdout) + .SetHiddenForCLI(); +} + +/************************************************************************/ +/* GDALVFSListAlgorithm::Print() */ +/************************************************************************/ + +void GDALVFSListAlgorithm::Print(const char *str) +{ + if (m_stdout) + fwrite(str, 1, strlen(str), stdout); + else + m_output += str; +} + +/************************************************************************/ +/* GDALVFSListAlgorithm::JSONPrint() */ +/************************************************************************/ + +/* static */ void GDALVFSListAlgorithm::JSONPrint(const char *pszTxt, + void *pUserData) +{ + static_cast(pUserData)->Print(pszTxt); +} + +/************************************************************************/ +/* GetDepth() */ +/************************************************************************/ + +static int GetDepth(const std::string &filename) +{ + int depth = 0; + const char sep = VSIGetDirectorySeparator(filename.c_str())[0]; + for (size_t i = 0; i < filename.size(); ++i) + { + if ((filename[i] == sep || filename[i] == '/') && + i != filename.size() - 1) + ++depth; + } + return depth; +} + +/************************************************************************/ +/* GDALVFSListAlgorithm::PrintEntry() */ +/************************************************************************/ + +void GDALVFSListAlgorithm::PrintEntry(const VSIDIREntry *entry) +{ + std::string filename; + if (m_format == "json" && m_JSONAsTree) + { + filename = CPLGetFilename(entry->pszName); + } + else if (m_absolutePath) + { + if (CPLIsFilenameRelative(m_filename.c_str())) + { + char *pszCurDir = CPLGetCurrentDir(); + if (!pszCurDir) + pszCurDir = CPLStrdup("."); + if (m_filename == ".") + filename = pszCurDir; + else + filename = + CPLFormFilenameSafe(pszCurDir, m_filename.c_str(), nullptr); + CPLFree(pszCurDir); + } + else + { + filename = m_filename; + } + filename = + CPLFormFilenameSafe(filename.c_str(), entry->pszName, nullptr); + } + else + { + filename = entry->pszName; + } + + char permissions[1 + 3 + 3 + 3 + 1] = "----------"; + struct tm bdt; + memset(&bdt, 0, sizeof(bdt)); + + if (m_longListing) + { + if (entry->bModeKnown) + { + if (VSI_ISDIR(entry->nMode)) + permissions[0] = 'd'; + for (int i = 0; i < 9; ++i) + { + if (entry->nMode & (1 << i)) + permissions[9 - i] = (i % 3) == 0 ? 'x' + : (i % 3) == 1 ? 'w' + : 'r'; + } + } + else if (VSI_ISDIR(entry->nMode)) + { + strcpy(permissions, "dr-xr-xr-x"); + } + else + { + strcpy(permissions, "-r--r--r--"); + } + + CPLUnixTimeToYMDHMS(entry->nMTime, &bdt); + } + + if (m_format == "json") + { + if (m_JSONAsTree) + { + while (!m_stackNames.empty() && + GetDepth(m_stackNames.back()) >= GetDepth(entry->pszName)) + { + m_oWriter.EndArray(); + m_oWriter.EndObj(); + m_stackNames.pop_back(); + } + } + + if (m_longListing) + { + m_oWriter.StartObj(); + m_oWriter.AddObjKey("name"); + m_oWriter.Add(filename); + m_oWriter.AddObjKey("type"); + m_oWriter.Add(VSI_ISDIR(entry->nMode) ? "directory" : "file"); + m_oWriter.AddObjKey("size"); + m_oWriter.Add(static_cast(entry->nSize)); + if (entry->bMTimeKnown) + { + m_oWriter.AddObjKey("last_modification_date"); + m_oWriter.Add(CPLSPrintf("%04d-%02d-%02d %02d:%02d:%02dZ", + bdt.tm_year + 1900, bdt.tm_mon + 1, + bdt.tm_mday, bdt.tm_hour, bdt.tm_min, + bdt.tm_sec)); + } + if (entry->bModeKnown) + { + m_oWriter.AddObjKey("permissions"); + m_oWriter.Add(permissions); + } + if (m_JSONAsTree && VSI_ISDIR(entry->nMode)) + { + m_stackNames.push_back(entry->pszName); + m_oWriter.AddObjKey("entries"); + m_oWriter.StartArray(); + } + else + { + m_oWriter.EndObj(); + } + } + else + { + if (m_JSONAsTree && VSI_ISDIR(entry->nMode)) + { + m_oWriter.StartObj(); + m_oWriter.AddObjKey("name"); + m_oWriter.Add(filename); + + m_stackNames.push_back(entry->pszName); + m_oWriter.AddObjKey("entries"); + m_oWriter.StartArray(); + } + else + { + m_oWriter.Add(filename); + } + } + } + else if (m_longListing) + { + Print(CPLSPrintf("%s 1 unknown unknown %12" PRIu64 + " %04d-%02d-%02d %02d:%02d %s\n", + permissions, static_cast(entry->nSize), + bdt.tm_year + 1900, bdt.tm_mon + 1, bdt.tm_mday, + bdt.tm_hour, bdt.tm_min, filename.c_str())); + } + else + { + Print(filename.c_str()); + Print("\n"); + } +} + +/************************************************************************/ +/* GDALVFSListAlgorithm::RunImpl() */ +/************************************************************************/ + +bool GDALVFSListAlgorithm::RunImpl(GDALProgressFunc, void *) +{ + VSIStatBufL sStat; + if (VSIStatL(m_filename.c_str(), &sStat) != 0) + { + ReportError(CE_Failure, CPLE_FileIO, "'%s' does not exist", + m_filename.c_str()); + return false; + } + + bool ret = false; + if (VSI_ISDIR(sStat.st_mode)) + { + std::unique_ptr dir( + VSIOpenDir(m_filename.c_str(), + m_recursive ? (m_depth == 0 ? 0 + : m_depth > 0 ? m_depth - 1 + : -1) + : 0, + nullptr), + VSICloseDir); + if (dir) + { + ret = true; + if (m_format == "json") + m_oWriter.StartArray(); + while (const auto entry = VSIGetNextDirEntry(dir.get())) + { + if (!(entry->pszName[0] == '.' && + (entry->pszName[1] == '.' || entry->pszName[1] == 0))) + { + PrintEntry(entry); + } + } + while (!m_stackNames.empty()) + { + m_stackNames.pop_back(); + m_oWriter.EndArray(); + m_oWriter.EndObj(); + } + if (m_format == "json") + m_oWriter.EndArray(); + } + } + else + { + ret = true; + VSIDIREntry sEntry; + sEntry.pszName = CPLStrdup(m_filename.c_str()); + sEntry.bModeKnown = true; + sEntry.nMode = sStat.st_mode; + sEntry.nSize = sStat.st_size; + PrintEntry(&sEntry); + } + + return ret; +} + +//! @endcond diff --git a/apps/gdalalg_vfs_list.h b/apps/gdalalg_vfs_list.h new file mode 100644 index 000000000000..08301e9705ca --- /dev/null +++ b/apps/gdalalg_vfs_list.h @@ -0,0 +1,65 @@ +/****************************************************************************** + * + * Project: GDAL + * Purpose: gdal "vfs list" subcommand + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2025, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#ifndef GDALALG_VFS_LIST_INCLUDED +#define GDALALG_VFS_LIST_INCLUDED + +#include "gdalalgorithm.h" + +#include "cpl_json_streaming_writer.h" + +//! @cond Doxygen_Suppress + +/************************************************************************/ +/* GDALVFSListAlgorithm */ +/************************************************************************/ + +struct VSIDIREntry; + +class GDALVFSListAlgorithm final : public GDALAlgorithm +{ + public: + static constexpr const char *NAME = "list"; + static constexpr const char *DESCRIPTION = + "List files of one of the GDAL Virtual file systems (VSI)."; + static constexpr const char *HELP_URL = "/programs/gdal_vfs_list.html"; + + static std::vector GetAliasesStatic() + { + return {"ls"}; + } + + GDALVFSListAlgorithm(); + + private: + CPLJSonStreamingWriter m_oWriter; + std::string m_filename{}; + std::string m_format{}; + std::string m_output{}; + int m_depth = -1; + bool m_stdout = false; + bool m_longListing = false; + bool m_recursive = false; + bool m_JSONAsTree = false; + bool m_absolutePath = false; + + std::vector m_stackNames{}; + + bool RunImpl(GDALProgressFunc, void *) override; + void Print(const char *str); + void PrintEntry(const VSIDIREntry *entry); + static void JSONPrint(const char *pszTxt, void *pUserData); +}; + +//! @endcond + +#endif diff --git a/autotest/gcore/vsiaz.py b/autotest/gcore/vsiaz.py index 9628d3a1e8cb..9e2b4f78b9ea 100755 --- a/autotest/gcore/vsiaz.py +++ b/autotest/gcore/vsiaz.py @@ -377,7 +377,13 @@ def test_vsiaz_fake_readdir(): dir_contents = gdal.ReadDir("/vsiaz/") assert dir_contents == ["mycontainer1", "mycontainer2"] - assert gdal.VSIStatL("/vsiaz/mycontainer1", gdal.VSI_STAT_CACHE_ONLY) is not None + stat = gdal.VSIStatL("/vsiaz/mycontainer1", gdal.VSI_STAT_CACHE_ONLY) + assert stat is not None + assert stat.mode == 16384 + + stat = gdal.VSIStatL("/vsiaz/") + assert stat is not None + assert stat.mode == 16384 ############################################################################### diff --git a/autotest/gcore/vsis3.py b/autotest/gcore/vsis3.py index 40021219baa0..5766bf1fd4db 100755 --- a/autotest/gcore/vsis3.py +++ b/autotest/gcore/vsis3.py @@ -1950,6 +1950,101 @@ def test_vsis3_opendir_synthetize_missing_directory(aws_test_config, webserver_p gdal.CloseDir(d) +############################################################################### +# Test OpenDir() with a fake AWS server on /vsis3/ root + + +def test_vsis3_opendir_from_prefix(aws_test_config, webserver_port): + + handler = webserver.SequentialHandler() + handler.add( + "GET", + "/", + 200, + {"Content-type": "application/xml"}, + """ + + + + bucket1 + + + bucket2 + + + + """, + ) + handler.add( + "GET", + "/bucket1/", + 200, + {"Content-type": "application/xml"}, + """ + + + + + test1.txt + 1970-01-01T00:00:01.000Z + 40 + + + test2.txt + 1970-01-01T00:00:01.000Z + 40 + + + """, + ) + handler.add( + "GET", + "/bucket2/", + 200, + {"Content-type": "application/xml"}, + """ + + + + + test3.txt + 1970-01-01T00:00:01.000Z + 40 + + + """, + ) + with webserver.install_http_handler(handler): + d = gdal.OpenDir("/vsis3/") + assert d is not None + try: + + entry = gdal.GetNextDirEntry(d) + assert entry.name == "bucket1" + assert entry.mode == 16384 + + entry = gdal.GetNextDirEntry(d) + assert entry.name == "bucket1/test1.txt" + assert entry.mode == 32768 + + entry = gdal.GetNextDirEntry(d) + assert entry.name == "bucket1/test2.txt" + assert entry.mode == 32768 + + entry = gdal.GetNextDirEntry(d) + assert entry.name == "bucket2" + assert entry.mode == 16384 + + entry = gdal.GetNextDirEntry(d) + assert entry.name == "bucket2/test3.txt" + assert entry.mode == 32768 + + assert gdal.GetNextDirEntry(d) is None + + finally: + gdal.CloseDir(d) + + ############################################################################### # Test simple PUT support with a fake AWS server diff --git a/autotest/utilities/test_gdal.py b/autotest/utilities/test_gdal.py index d55eadec2ae3..bd19f5e62a2c 100755 --- a/autotest/utilities/test_gdal.py +++ b/autotest/utilities/test_gdal.py @@ -173,6 +173,14 @@ def test_gdal_completion(gdal_path): == "** description:\\ Target\\ resolution\\ (in\\ destination\\ CRS\\ units)" ) + out = gdaltest.runexternal( + f"{gdal_path} completion gdal raster convert --input /vsi" + ).split(" ") + assert "/vsimem/" in out + + # Just run it. Result will depend on local configuration + gdaltest.runexternal(f"{gdal_path} completion gdal raster convert --input /vsis3/") + def test_gdal_completion_co(gdal_path): diff --git a/autotest/utilities/test_gdalalg_vfs.py b/autotest/utilities/test_gdalalg_vfs.py new file mode 100755 index 000000000000..d35e7bf49ea3 --- /dev/null +++ b/autotest/utilities/test_gdalalg_vfs.py @@ -0,0 +1,27 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# Project: GDAL/OGR Test Suite +# Purpose: 'gdal vfs' testing +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2025, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + +import pytest + +from osgeo import gdal + + +def get_alg(): + return gdal.GetGlobalAlgorithmRegistry()["vfs"] + + +def test_gdalalg_vfs(): + + alg = get_alg() + with pytest.raises(Exception): + alg.Run() diff --git a/autotest/utilities/test_gdalalg_vfs_copy.py b/autotest/utilities/test_gdalalg_vfs_copy.py new file mode 100755 index 000000000000..f239fd2eaadd --- /dev/null +++ b/autotest/utilities/test_gdalalg_vfs_copy.py @@ -0,0 +1,192 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# Project: GDAL/OGR Test Suite +# Purpose: 'gdal vfs copy' testing +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2025, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + +import pytest + +from osgeo import gdal + + +def get_alg(): + return gdal.GetGlobalAlgorithmRegistry()["vfs"]["copy"] + + +def test_gdalalg_vfs_copy_empty_source(): + + alg = get_alg() + with pytest.raises(Exception, match="Source filename cannot be empty"): + alg["source"] = "" + + +def test_gdalalg_vfs_copy_empty_destination(): + + alg = get_alg() + with pytest.raises(Exception, match="Destination filename cannot be empty"): + alg["destination"] = "" + + +def test_gdalalg_vfs_copy_single_dir_destination(tmp_vsimem): + + alg = get_alg() + alg["source"] = "../gcore/data/byte.tif" + alg["destination"] = tmp_vsimem + assert alg.Run() + assert gdal.VSIStatL(tmp_vsimem / "byte.tif").size == 736 + + +def test_gdalalg_vfs_copy_single_file_destination(tmp_vsimem): + + alg = get_alg() + alg["source"] = "../gcore/data/byte.tif" + alg["destination"] = tmp_vsimem / "out.tif" + assert alg.Run() + assert gdal.VSIStatL(tmp_vsimem / "out.tif").size == 736 + + +def test_gdalalg_vfs_copy_single_progress(tmp_vsimem): + + last_pct = [0] + + def my_progress(pct, msg, user_data): + last_pct[0] = pct + return True + + alg = get_alg() + alg["source"] = "../gcore/data/byte.tif" + alg["destination"] = tmp_vsimem + assert alg.Run(my_progress) + assert last_pct[0] == 1.0 + assert gdal.VSIStatL(tmp_vsimem / "byte.tif").size == 736 + + +def test_gdalalg_vfs_copy_single_source_does_not_exist(): + + alg = get_alg() + alg["source"] = "/i_do/not/exist.bin" + alg["destination"] = "/vsimem/" + with pytest.raises(Exception, match="does not exist"): + alg.Run() + + +def test_gdalalg_vfs_copy_single_source_is_directory(): + + alg = get_alg() + alg["source"] = "../gcore" + alg["destination"] = "/vsimem/" + with pytest.raises(Exception, match="is a directory"): + alg.Run() + + +def test_gdalalg_vfs_copy_recursive_destination_does_not_exist(tmp_vsimem): + + gdal.Mkdir(tmp_vsimem / "src", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "a", "foo") + gdal.Mkdir(tmp_vsimem / "src" / "subdir", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "subdir" / "b", "bar") + + last_pct = [0] + + def my_progress(pct, msg, user_data): + last_pct[0] = pct + return True + + alg = get_alg() + alg["source"] = tmp_vsimem / "src" + alg["destination"] = tmp_vsimem / "dst" + alg["recursive"] = True + assert alg.Run(my_progress) + assert last_pct[0] == 1.0 + res = set(gdal.ReadDirRecursive(tmp_vsimem / "dst")) + assert set(res) == set(gdal.ReadDirRecursive(tmp_vsimem / "src")) + + +def test_gdalalg_vfs_copy_recursive_destination_exists(tmp_vsimem): + + gdal.Mkdir(tmp_vsimem / "src", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "a", "foo") + gdal.Mkdir(tmp_vsimem / "src" / "subdir", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "subdir" / "b", "bar") + + gdal.Mkdir(tmp_vsimem / "dst", 0o755) + + alg = get_alg() + alg["source"] = tmp_vsimem / "src" + alg["destination"] = tmp_vsimem / "dst" + alg["recursive"] = True + assert alg.Run() + res = set(gdal.ReadDirRecursive(tmp_vsimem / "dst")) + res.remove("src/") + assert set([x[len("src/") :] for x in res]) == set( + gdal.ReadDirRecursive(tmp_vsimem / "src") + ) + + +def test_gdalalg_vfs_copy_recursive_source_ends_slash_star(tmp_vsimem): + + gdal.Mkdir(tmp_vsimem / "src", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "a", "foo") + gdal.Mkdir(tmp_vsimem / "src" / "subdir", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "subdir" / "b", "bar") + + alg = get_alg() + alg["source"] = tmp_vsimem / "src" / "*" + alg["destination"] = tmp_vsimem / "dst" + alg["recursive"] = True + assert alg.Run() + res = set(gdal.ReadDirRecursive(tmp_vsimem / "dst")) + assert set(res) == set(gdal.ReadDirRecursive(tmp_vsimem / "src")) + + +def test_gdalalg_vfs_copy_source_ends_slash_star(tmp_vsimem): + + gdal.Mkdir(tmp_vsimem / "src", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "a", "foo") + gdal.Mkdir(tmp_vsimem / "src" / "subdir", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "subdir" / "b", "bar") + + alg = get_alg() + alg["source"] = tmp_vsimem / "src" / "*" + alg["destination"] = tmp_vsimem / "dst" + assert alg.Run() + res = set(gdal.ReadDirRecursive(tmp_vsimem / "dst")) + assert set(res) == set(["a", "subdir/"]) + + +def test_gdalalg_vfs_copy_recursive_destination_cannot_be_created(tmp_vsimem): + + gdal.Mkdir(tmp_vsimem / "src", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "a", "foo") + gdal.Mkdir(tmp_vsimem / "src" / "subdir", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "subdir" / "b", "bar") + + alg = get_alg() + alg["source"] = tmp_vsimem / "src" + alg["destination"] = "/i_do/not/exist" + alg["recursive"] = True + with pytest.raises(Exception, match="Cannot create directory /i_do/not/exist"): + alg.Run() + + +def test_gdalalg_vfs_copy_recursive_destination_cannot_be_created_skip(tmp_vsimem): + + gdal.Mkdir(tmp_vsimem / "src", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "a", "foo") + gdal.Mkdir(tmp_vsimem / "src" / "subdir", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "src" / "subdir" / "b", "bar") + + alg = get_alg() + alg["source"] = tmp_vsimem / "src" + alg["destination"] = "/i_do/not/exist" + alg["recursive"] = True + alg["skip-errors"] = True + with gdal.quiet_errors(): + assert alg.Run() diff --git a/autotest/utilities/test_gdalalg_vfs_delete.py b/autotest/utilities/test_gdalalg_vfs_delete.py new file mode 100755 index 000000000000..f7f61b30d9e0 --- /dev/null +++ b/autotest/utilities/test_gdalalg_vfs_delete.py @@ -0,0 +1,81 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# Project: GDAL/OGR Test Suite +# Purpose: 'gdal vfs delete' testing +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2025, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + +import sys + +import pytest + +from osgeo import gdal + + +def get_alg(): + return gdal.GetGlobalAlgorithmRegistry()["vfs"]["delete"] + + +def test_gdalalg_vfs_delete_empty_filename(): + + alg = get_alg() + with pytest.raises(Exception, match="Filename cannot be empty"): + alg["filename"] = "" + + +def test_gdalalg_vfs_delete_file(tmp_vsimem): + + gdal.FileFromMemBuffer(tmp_vsimem / "test", "test") + + alg = get_alg() + alg["filename"] = tmp_vsimem / "test" + assert alg.Run() + + assert gdal.VSIStatL(tmp_vsimem / "test") is None + + +def test_gdalalg_vfs_delete_file_not_existing(): + + alg = get_alg() + alg["filename"] = "/i_do/not/exist" + with pytest.raises(Exception, match="does not exist"): + alg.Run() + + +def test_gdalalg_vfs_delete_dir(tmp_path): + + gdal.Mkdir(tmp_path / "subdir", 0o755) + + alg = get_alg() + alg["filename"] = tmp_path / "subdir" + assert alg.Run() + + assert gdal.VSIStatL(tmp_path / "subdir") is None + + +@pytest.mark.skipif(sys.platform == "win32", reason="incompatible platform") +def test_gdalalg_vfs_delete_file_failed(): + + alg = get_alg() + alg["filename"] = "/dev/null" + with pytest.raises(Exception, match="Cannot delete /dev/null"): + alg.Run() + + +def test_gdalalg_vfs_delete_dir_recursive(tmp_path): + + gdal.Mkdir(tmp_path / "subdir", 0o755) + open(tmp_path / "subdir" / "file", "wb").close() + + alg = get_alg() + alg["filename"] = tmp_path / "subdir" + alg["recursive"] = True + assert alg.Run() + + assert gdal.VSIStatL(tmp_path / "subdir") is None diff --git a/autotest/utilities/test_gdalalg_vfs_list.py b/autotest/utilities/test_gdalalg_vfs_list.py new file mode 100755 index 000000000000..70e0b97ebf61 --- /dev/null +++ b/autotest/utilities/test_gdalalg_vfs_list.py @@ -0,0 +1,295 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# Project: GDAL/OGR Test Suite +# Purpose: 'gdal vfs list' testing +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2025, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + +import json +import os + +import pytest + +from osgeo import gdal + + +def get_alg(): + return gdal.GetGlobalAlgorithmRegistry()["vfs"]["list"] + + +def del_last_modification_date(j): + if isinstance(j, list): + for subj in j: + del_last_modification_date(subj) + elif isinstance(j, dict): + if "last_modification_date" in j: + del j["last_modification_date"] + for k in j: + del_last_modification_date(j[k]) + + +def test_gdalalg_vfs_list(tmp_vsimem): + + alg = get_alg() + alg["filename"] = tmp_vsimem / "i_do_not_exist" + with pytest.raises(Exception): + alg.Run() + + alg = get_alg() + alg["filename"] = tmp_vsimem + assert alg.Run() + assert json.loads(alg["output-string"]) == [] + + gdal.FileFromMemBuffer(tmp_vsimem / "a", "a") + gdal.Mkdir(tmp_vsimem / "subdir", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "subdir" / "b", "b") + gdal.FileFromMemBuffer(tmp_vsimem / "subdir" / "c", "c") + gdal.Mkdir(tmp_vsimem / "subdir" / "subsubdir", 0o755) + gdal.FileFromMemBuffer(tmp_vsimem / "d", "d") + + alg = get_alg() + alg["filename"] = tmp_vsimem + assert alg.Run() + assert json.loads(alg["output-string"]) == ["a", "d", "subdir"] + + alg = get_alg() + alg["filename"] = tmp_vsimem / "a" + assert alg.Run() + assert json.loads(alg["output-string"]) == str(tmp_vsimem / "a") + + alg = get_alg() + alg["filename"] = tmp_vsimem + alg["recursive"] = True + assert alg.Run() + assert json.loads(alg["output-string"]) == [ + "a", + "d", + "subdir", + "subdir/b", + "subdir/c", + "subdir/subsubdir", + ] + + alg = get_alg() + alg["filename"] = tmp_vsimem + alg["recursive"] = True + alg["depth"] = 1 + assert alg.Run() + assert json.loads(alg["output-string"]) == [ + "a", + "d", + "subdir", + ] + + alg = get_alg() + alg["filename"] = tmp_vsimem + alg["recursive"] = True + alg["depth"] = 2 + assert alg.Run() + assert json.loads(alg["output-string"]) == [ + "a", + "d", + "subdir", + "subdir/b", + "subdir/c", + "subdir/subsubdir", + ] + + alg = get_alg() + alg["filename"] = tmp_vsimem + alg["long-listing"] = True + assert alg.Run() + j = json.loads(alg["output-string"]) + del_last_modification_date(j) + + assert j == [ + { + "name": "a", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "name": "d", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "name": "subdir", + "permissions": "d---------", + "size": 0, + "type": "directory", + }, + ] + + alg = get_alg() + alg["filename"] = tmp_vsimem + alg["recursive"] = True + alg["long-listing"] = True + assert alg.Run() + j = json.loads(alg["output-string"]) + del_last_modification_date(j) + + assert j == [ + { + "name": "a", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "name": "d", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "name": "subdir", + "permissions": "d---------", + "size": 0, + "type": "directory", + }, + { + "name": "subdir/b", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "name": "subdir/c", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "name": "subdir/subsubdir", + "permissions": "d---------", + "size": 0, + "type": "directory", + }, + ] + + alg = get_alg() + alg["filename"] = tmp_vsimem + alg["recursive"] = True + alg["long-listing"] = True + alg["tree"] = True + assert alg.Run() + j = json.loads(alg["output-string"]) + del_last_modification_date(j) + + assert j == [ + { + "name": "a", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "name": "d", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "name": "subdir", + "permissions": "d---------", + "size": 0, + "type": "directory", + "entries": [ + { + "name": "b", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "name": "c", + "permissions": "----------", + "size": 1, + "type": "file", + }, + { + "entries": [], + "name": "subsubdir", + "permissions": "d---------", + "size": 0, + "type": "directory", + }, + ], + }, + ] + + alg = get_alg() + alg["filename"] = tmp_vsimem + alg["recursive"] = True + alg["tree"] = True + assert alg.Run() + j = json.loads(alg["output-string"]) + del_last_modification_date(j) + + assert j == [ + "a", + "d", + { + "name": "subdir", + "entries": [ + "b", + "c", + { + "entries": [], + "name": "subsubdir", + }, + ], + }, + ] + + alg = get_alg() + alg["filename"] = tmp_vsimem + alg["recursive"] = True + alg["format"] = "text" + alg["absolute-path"] = True + assert alg.Run() + assert alg["output-string"][0:-1].split("\n") == [ + str(tmp_vsimem) + "/" + x + for x in [ + "a", + "d", + "subdir", + "subdir/b", + "subdir/c", + "subdir/subsubdir", + ] + ] + + alg = get_alg() + alg["filename"] = "data" + assert alg.Run() + assert "utmsmall.tif" in json.loads(alg["output-string"]) + + alg = get_alg() + alg["filename"] = "." + assert alg.Run() + + alg = get_alg() + alg["filename"] = "data" + alg["absolute-path"] = True + assert alg.Run() + assert os.path.join(os.getcwd(), "data", "utmsmall.tif").replace("\\", "/") in [ + x.replace("\\", "/") for x in json.loads(alg["output-string"]) + ] + + alg = get_alg() + alg["filename"] = "data" + alg["long-listing"] = True + alg["format"] = "text" + assert alg.Run() + assert "unknown unknown" in alg["output-string"] + assert "utmsmall.tif" in alg["output-string"] diff --git a/doc/source/conf.py b/doc/source/conf.py index bb76870a0747..5b867e952283 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -532,6 +532,34 @@ def check_python_bindings(): [author_evenr], 1, ), + ( + "programs/gdal_vfs", + "gdal-vfs", + "Entry point for GDAL Virtual file system (VSI) commands", + [author_evenr], + 1, + ), + ( + "programs/gdal_vfs_copy", + "gdal-vfs-copy", + "Copy files located on GDAL Virtual file systems (VSI)", + [author_evenr], + 1, + ), + ( + "programs/gdal_vfs_delete", + "gdal-vfs-delete", + "Delete files located on GDAL Virtual file systems (VSI)", + [author_evenr], + 1, + ), + ( + "programs/gdal_vfs_list", + "gdal-vfs-list", + "List files of one of the GDAL Virtual file systems (VSI)", + [author_evenr], + 1, + ), # Traditional utilities ( "programs/gdalinfo", diff --git a/doc/source/programs/gdal_vfs.rst b/doc/source/programs/gdal_vfs.rst new file mode 100644 index 000000000000..befd263557bd --- /dev/null +++ b/doc/source/programs/gdal_vfs.rst @@ -0,0 +1,38 @@ +.. _gdal_vfs_command: + +================================================================================ +"gdal vfs" command +================================================================================ + +.. versionadded:: 3.11 + +.. only:: html + + Entry point for GDAL Virtual file system (VSI) commands + +.. Index:: gdal vfs + +The subcommands of :program:`gdal vs` allow manipulation of files located +on the :ref:`virtual_file_systems`. + +Synopsis +-------- + +.. program-output:: gdal vfs --help-doc + +Available sub-commands +---------------------- + +- :ref:`gdal_vfs_copy_subcommand` +- :ref:`gdal_vfs_delete_subcommand` +- :ref:`gdal_vfs_list_subcommand` + +Examples +-------- + +.. example:: + :title: Listing recursively files in /vsis3/bucket with details + + .. code-block:: console + + $ gdal vfs list -lR --of=text /vsis3/bucket diff --git a/doc/source/programs/gdal_vfs_copy.rst b/doc/source/programs/gdal_vfs_copy.rst new file mode 100644 index 000000000000..d134b909cc54 --- /dev/null +++ b/doc/source/programs/gdal_vfs_copy.rst @@ -0,0 +1,56 @@ +.. _gdal_vfs_copy_subcommand: + +================================================================================ +"gdal vfs copy" sub-command +================================================================================ + +.. versionadded:: 3.11 + +.. only:: html + + Copy files located on GDAL Virtual file systems (VSI) + +.. Index:: gdal vfs copy + +Synopsis +-------- + +.. program-output:: gdal vfs copy --help-doc + +Description +----------- + +:program:`gdal vfs copy` copy files and directories located on :ref:`virtual_file_systems`. + +It can copy files and directories between different virtual file systems. + +This is the equivalent of the UNIX ``cp`` command, and ``gdal vfs cp`` is an +alias for ``gdal vfs copy``. + +Options ++++++++ + +.. option:: -r, --recursive + + Copy directories recursively. + +.. option:: --skip-errors + + Skip errors that occur while while copying. + +Examples +-------- + +.. example:: + :title: Copy recursively files from /vsis3/bucket/my_dir to local directory, creating a my_dir directory if it does not exist. + + .. code-block:: console + + $ gdal vfs copy -r /vsis3/bucket/my_dir . + +.. example:: + :title: Copy recursively files from /vsis3/bucket/my_dir to local directory, *without* creating a my_dir directory, and with progress bar + + .. code-block:: console + + $ gdal vfs copy --progress -r /vsis3/bucket/my_dir/* . diff --git a/doc/source/programs/gdal_vfs_delete.rst b/doc/source/programs/gdal_vfs_delete.rst new file mode 100644 index 000000000000..e4da232e329f --- /dev/null +++ b/doc/source/programs/gdal_vfs_delete.rst @@ -0,0 +1,48 @@ +.. _gdal_vfs_delete_subcommand: + +================================================================================ +"gdal vfs delete" sub-command +================================================================================ + +.. versionadded:: 3.11 + +.. only:: html + + Delete files located on GDAL Virtual file systems (VSI) + +.. Index:: gdal vfs delete + +Synopsis +-------- + +.. program-output:: gdal vfs delete --help-doc + +Description +----------- + +:program:`gdal vfs delete` delete files and directories located on :ref:`virtual_file_systems`. + +This is the equivalent of the UNIX ``rm`` command, and ``gdal vfs rm`` is an +alias for ``gdal vfs delete``. + +.. warning:: + + Be careful. This command cannot be undone. It can also act on the "real" + file system. + +Options ++++++++ + +.. option:: -r, -R, --recursive + + Delete directories recursively. + +Examples +-------- + +.. example:: + :title: Delete recursively files from /vsis3/bucket/my_dir + + .. code-block:: console + + $ gdal vfs delete -r /vsis3/bucket/my_dir diff --git a/doc/source/programs/gdal_vfs_list.rst b/doc/source/programs/gdal_vfs_list.rst new file mode 100644 index 000000000000..78896292e3fa --- /dev/null +++ b/doc/source/programs/gdal_vfs_list.rst @@ -0,0 +1,76 @@ +.. _gdal_vfs_list_subcommand: + +================================================================================ +"gdal vfs list" sub-command +================================================================================ + +.. versionadded:: 3.11 + +.. only:: html + + List files of one of the GDAL Virtual file systems (VSI) + +.. Index:: gdal vfs list + +Synopsis +-------- + +.. program-output:: gdal vfs list --help-doc + +Description +----------- + +:program:`gdal vfs list` list files of :ref:`virtual_file_systems`. + +This is the equivalent of the UNIX ``ls`` command, and ``gdal vfs ls`` is an +alias for ``gdal vfs list``. + +By default, it outputs file names, at the immediate level, without details, +and in JSON format. + +Options ++++++++ + +.. option:: --filename + + Any file name or directory name, of one of the GDAL Virtual file systems. + Required. + +.. option:: -f, --of, --format, --output-format json|text + + Which output format to use. Default is JSON. + +.. option:: -l, --long, --long-listing + + Use a long listing format, adding permissions, file size and last modification + date. + +.. option:: -R, --recursive + + List subdirectories recursively. By default the depth is unlimited, but + it can be reduced with :option:`--depth`. + +.. option:: --depth + + Maximum depth in recursive mode. 1 corresponds to no recursion, 2 to + the immediate subdirectories, etc. + +.. option:: --abs, --absolute-path + + Whether to report file names as absolute paths. By default, they are relative + to the input file name. + +.. option:: --tree + + Use a hierarchical presentation for JSON output, instead of a flat list. + Only valid when :option:`--output-format` is set to ``json`` (or let at its default value). + +Examples +-------- + +.. example:: + :title: Listing recursively files in /vsis3/bucket with details + + .. code-block:: console + + $ gdal vfs list -lR --of=text /vsis3/bucket diff --git a/doc/source/programs/index.rst b/doc/source/programs/index.rst index 1108fa2e72b0..84501d540a05 100644 --- a/doc/source/programs/index.rst +++ b/doc/source/programs/index.rst @@ -80,6 +80,10 @@ single :program:`gdal` program that accepts commands and subcommands. gdal_vector_reproject gdal_vector_select gdal_vector_sql + gdal_vfs + gdal_vfs_copy + gdal_vfs_delete + gdal_vfs_list .. only:: html @@ -138,6 +142,10 @@ single :program:`gdal` program that accepts commands and subcommands. - :ref:`gdal_vector_select_subcommand`: Select a subset of fields from a vector dataset. - :ref:`gdal_vector_rasterize_subcommand`: Burns vector geometries into a raster - :ref:`gdal_vector_sql_subcommand`: Apply SQL statement(s) to a dataset + - :ref:`gdal_vfs_command`: Entry point for GDAL Virtual file system (VSI) commands + - :ref:`gdal_vfs_copy_subcommand`: Copy files located on GDAL Virtual file systems (VSI) + - :ref:`gdal_vfs_delete_subcommand`: Delete files located on GDAL Virtual file systems (VSI) + - :ref:`gdal_vfs_list_subcommand`: List files of one of the GDAL Virtual file systems (VSI) "Traditional" applications diff --git a/doc/source/spelling_wordlist.txt b/doc/source/spelling_wordlist.txt index 2883ba30c7b7..34d24a3d6d9c 100644 --- a/doc/source/spelling_wordlist.txt +++ b/doc/source/spelling_wordlist.txt @@ -1298,8 +1298,6 @@ hibase hicover hidensity hiduff -hiearchical -Hiearchical hieast hielev hifuel @@ -3483,6 +3481,7 @@ vetoers vetos Vexcel vfk +vfs Viersen viewshed Viewshed diff --git a/gcore/gdalalgorithm.cpp b/gcore/gdalalgorithm.cpp index 3f76fa15ea3c..003389f9c29a 100644 --- a/gcore/gdalalgorithm.cpp +++ b/gcore/gdalalgorithm.cpp @@ -886,6 +886,19 @@ GDALInConstructionAlgorithmArg::AddHiddenAlias(const std::string &alias) return *this; } +/************************************************************************/ +/* GDALInConstructionAlgorithmArg::AddShortNameAlias() */ +/************************************************************************/ + +GDALInConstructionAlgorithmArg & +GDALInConstructionAlgorithmArg::AddShortNameAlias(char shortNameAlias) +{ + m_decl.AddShortNameAlias(shortNameAlias); + if (m_owner) + m_owner->AddShortNameAliasFor(this, shortNameAlias); + return *this; +} + /************************************************************************/ /* GDALInConstructionAlgorithmArg::SetPositional() */ /************************************************************************/ @@ -1467,24 +1480,39 @@ bool GDALAlgorithm::ParseCommandLineArguments( } else if (strArg.size() >= 2 && strArg[0] == '-') { - if (strArg.size() != 2) + for (size_t j = 1; j < strArg.size(); ++j) { - ReportError( - CE_Failure, CPLE_IllegalArg, - "Option '%s' not recognized. Should be either a long " - "option or a one-letter short option.", - strArg.c_str()); - return false; + name.clear(); + name += strArg[j]; + auto iterArg = m_mapShortNameToArg.find(name); + if (iterArg == m_mapShortNameToArg.end()) + { + ReportError(CE_Failure, CPLE_IllegalArg, + "Short name option '%s' is unknown.", + name.c_str()); + return false; + } + arg = iterArg->second; + if (strArg.size() > 2) + { + if (arg->GetType() != GAAT_BOOLEAN) + { + ReportError(CE_Failure, CPLE_IllegalArg, + "Invalid argument '%s'. Option '%s' is not " + "a boolean option.", + strArg.c_str(), name.c_str()); + return false; + } + + if (!ParseArgument(arg, name, "true", inConstructionValues)) + return false; + } } - name = strArg; - auto iterArg = m_mapShortNameToArg.find(name.substr(1)); - if (iterArg == m_mapShortNameToArg.end()) + if (strArg.size() > 2) { - ReportError(CE_Failure, CPLE_IllegalArg, - "Short name option '%s' is unknown.", name.c_str()); - return false; + lArgs.erase(lArgs.begin() + i); + continue; } - arg = iterArg->second; } else { @@ -2087,6 +2115,29 @@ void GDALAlgorithm::AddAliasFor(GDALInConstructionAlgorithmArg *arg, //! @endcond +/************************************************************************/ +/* GDALAlgorithm::AddShortNameAliasFor() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress +void GDALAlgorithm::AddShortNameAliasFor(GDALInConstructionAlgorithmArg *arg, + char shortNameAlias) +{ + std::string alias; + alias += shortNameAlias; + if (cpl::contains(m_mapShortNameToArg, alias)) + { + ReportError(CE_Failure, CPLE_AppDefined, + "Short name '%s' already declared.", alias.c_str()); + } + else + { + m_mapShortNameToArg[alias] = arg; + } +} + +//! @endcond + /************************************************************************/ /* GDALAlgorithm::SetPositional() */ /************************************************************************/ @@ -2270,52 +2321,80 @@ inline const char *MsgOrDefault(const char *helpMessage, } /************************************************************************/ -/* GDALAlgorithm::AddInputDatasetArg() */ +/* GDALAlgorithm::SetAutoCompleteFunctionForFilename() */ /************************************************************************/ -GDALInConstructionAlgorithmArg &GDALAlgorithm::AddInputDatasetArg( - GDALArgDatasetValue *pValue, GDALArgDatasetValueType type, - bool positionalAndRequired, const char *helpMessage) +/* static */ +void GDALAlgorithm::SetAutoCompleteFunctionForFilename( + GDALInConstructionAlgorithmArg &arg, GDALArgDatasetValueType type) { - auto &arg = AddArg( - GDAL_ARG_NAME_INPUT, 'i', - MsgOrDefault(helpMessage, - CPLSPrintf("Input %s dataset", - GDALArgDatasetValueTypeName(type).c_str())), - pValue, type); - if (positionalAndRequired) - arg.SetPositional().SetRequired(); - arg.SetAutoCompleteFunction( - [type](const std::string ¤tValue) + [type](const std::string ¤tValue) -> std::vector { std::vector oRet; + { + CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); + VSIStatBufL sStat; + if (!currentValue.empty() && currentValue.back() != '/' && + VSIStatL(currentValue.c_str(), &sStat) == 0) + { + return oRet; + } + } + auto poDM = GetGDALDriverManager(); std::set oExtensions; - for (int i = 0; i < poDM->GetDriverCount(); ++i) + if (type) + { + for (int i = 0; i < poDM->GetDriverCount(); ++i) + { + auto poDriver = poDM->GetDriver(i); + if (((type & GDAL_OF_RASTER) != 0 && + poDriver->GetMetadataItem(GDAL_DCAP_RASTER)) || + ((type & GDAL_OF_VECTOR) != 0 && + poDriver->GetMetadataItem(GDAL_DCAP_VECTOR)) || + ((type & GDAL_OF_MULTIDIM_RASTER) != 0 && + poDriver->GetMetadataItem(GDAL_DCAP_MULTIDIM_RASTER))) + { + const char *pszExtensions = + poDriver->GetMetadataItem(GDAL_DMD_EXTENSIONS); + if (pszExtensions) + { + const CPLStringList aosExts( + CSLTokenizeString2(pszExtensions, " ", 0)); + for (const char *pszExt : cpl::Iterate(aosExts)) + oExtensions.insert(CPLString(pszExt).tolower()); + } + } + } + } + + std::string osDir; + const CPLStringList aosVSIPrefixes(VSIGetFileSystemsPrefixes()); + std::string osPrefix; + if (STARTS_WITH(currentValue.c_str(), "/vsi")) { - auto poDriver = poDM->GetDriver(i); - if (((type & GDAL_OF_RASTER) != 0 && - poDriver->GetMetadataItem(GDAL_DCAP_RASTER)) || - ((type & GDAL_OF_VECTOR) != 0 && - poDriver->GetMetadataItem(GDAL_DCAP_VECTOR)) || - ((type & GDAL_OF_MULTIDIM_RASTER) != 0 && - poDriver->GetMetadataItem(GDAL_DCAP_MULTIDIM_RASTER))) + for (const char *pszPrefix : cpl::Iterate(aosVSIPrefixes)) { - const char *pszExtensions = - poDriver->GetMetadataItem(GDAL_DMD_EXTENSIONS); - if (pszExtensions) + if (STARTS_WITH(currentValue.c_str(), pszPrefix)) { - const CPLStringList aosExts( - CSLTokenizeString2(pszExtensions, " ", 0)); - for (const char *pszExt : cpl::Iterate(aosExts)) - oExtensions.insert(CPLString(pszExt).tolower()); + osPrefix = pszPrefix; + break; } } + if (osPrefix.empty()) + return aosVSIPrefixes; + if (currentValue == osPrefix) + osDir = osPrefix; + } + if (osDir.empty()) + { + osDir = CPLGetDirnameSafe(currentValue.c_str()); + if (!osPrefix.empty() && osDir.size() < osPrefix.size()) + osDir = osPrefix; } - std::string osDir = CPLGetDirnameSafe(currentValue.c_str()); auto psDir = VSIOpenDir(osDir.c_str(), 0, nullptr); const std::string osSep = VSIGetDirectorySeparator(osDir.c_str()); if (currentValue.empty()) @@ -2331,9 +2410,11 @@ GDALInConstructionAlgorithmArg &GDALAlgorithm::AddInputDatasetArg( currentFilename.c_str())) && strcmp(psEntry->pszName, ".") != 0 && strcmp(psEntry->pszName, "..") != 0 && - !strstr(psEntry->pszName, ".aux.xml")) + (oExtensions.empty() || + !strstr(psEntry->pszName, ".aux.xml"))) { - if (cpl::contains( + if (oExtensions.empty() || + cpl::contains( oExtensions, CPLString(CPLGetExtensionSafe(psEntry->pszName)) .tolower()) || @@ -2355,6 +2436,26 @@ GDALInConstructionAlgorithmArg &GDALAlgorithm::AddInputDatasetArg( } return oRet; }); +} + +/************************************************************************/ +/* GDALAlgorithm::AddInputDatasetArg() */ +/************************************************************************/ + +GDALInConstructionAlgorithmArg &GDALAlgorithm::AddInputDatasetArg( + GDALArgDatasetValue *pValue, GDALArgDatasetValueType type, + bool positionalAndRequired, const char *helpMessage) +{ + auto &arg = AddArg( + GDAL_ARG_NAME_INPUT, 'i', + MsgOrDefault(helpMessage, + CPLSPrintf("Input %s dataset", + GDALArgDatasetValueTypeName(type).c_str())), + pValue, type); + if (positionalAndRequired) + arg.SetPositional().SetRequired(); + + SetAutoCompleteFunctionForFilename(arg, type); return arg; } @@ -3587,6 +3688,14 @@ GDALAlgorithm::GetArgNamesForCLI() const opt += arg->GetShortName(); addComma = true; } + for (char alias : arg->GetShortNameAliases()) + { + if (addComma) + opt += ", "; + opt += "-"; + opt += alias; + addComma = true; + } for (const std::string &alias : arg->GetAliases()) { if (addComma) @@ -4297,6 +4406,14 @@ GDALAlgorithm::GetAutoComplete(std::vector &args, else if (!args.empty() && STARTS_WITH(args.back().c_str(), "/vsi")) { auto arg = GetArg(GDAL_ARG_NAME_INPUT); + for (const char *name : + {"dataset", "filename", "like", "source", "destination"}) + { + if (!arg) + { + arg = GetArg(name); + } + } if (arg) { ret = arg->GetAutoCompleteChoices(args.back()); diff --git a/gcore/gdalalgorithm.h b/gcore/gdalalgorithm.h index cfcefa154913..716302b561b4 100644 --- a/gcore/gdalalgorithm.h +++ b/gcore/gdalalgorithm.h @@ -559,6 +559,13 @@ class CPL_DLL GDALAlgorithmArgDecl final return *this; } + /** Declare a shortname alias.*/ + GDALAlgorithmArgDecl &AddShortNameAlias(char shortNameAlias) + { + m_shortNameAliases.push_back(shortNameAlias); + return *this; + } + /** Declare an hidden alias (i.e. not exposed in usage). * Must be 2 characters at least. */ GDALAlgorithmArgDecl &AddHiddenAlias(const std::string &alias) @@ -854,6 +861,12 @@ class CPL_DLL GDALAlgorithmArgDecl final return m_aliases; } + /** Return the shortname aliases (potentially none) */ + inline const std::vector &GetShortNameAliases() const + { + return m_shortNameAliases; + } + /** Return the description */ inline const std::string &GetDescription() const { @@ -1112,6 +1125,7 @@ class CPL_DLL GDALAlgorithmArgDecl final std::map> m_metadata{}; std::vector m_aliases{}; std::vector m_hiddenAliases{}; + std::vector m_shortNameAliases{}; std::vector m_choices{}; std::vector m_hiddenChoices{}; std::variant, @@ -1188,6 +1202,12 @@ class CPL_DLL GDALAlgorithmArg /* non-final */ return m_decl.GetAliases(); } + /** Alias for GDALAlgorithmArgDecl::GetShortNameAliases() */ + inline const std::vector &GetShortNameAliases() const + { + return m_decl.GetShortNameAliases(); + } + /** Alias for GDALAlgorithmArgDecl::GetDescription() */ inline const std::string &GetDescription() const { @@ -1617,6 +1637,9 @@ class CPL_DLL GDALInConstructionAlgorithmArg final : public GDALAlgorithmArg /** Add a non-documented alias for the argument */ GDALInConstructionAlgorithmArg &AddHiddenAlias(const std::string &alias); + /** Add a shortname alias for the argument */ + GDALInConstructionAlgorithmArg &AddShortNameAlias(char shortNameAlias); + /** Alias for GDALAlgorithmArgDecl::SetPositional() */ GDALInConstructionAlgorithmArg &SetPositional(); @@ -2301,6 +2324,11 @@ class CPL_DLL GDALAlgorithmRegistry const std::string &helpMessage, double *pValue); + /** Register an auto complete function for a filename argument */ + static void + SetAutoCompleteFunctionForFilename(GDALInConstructionAlgorithmArg &arg, + GDALArgDatasetValueType type); + /** Add dataset argument. */ GDALInConstructionAlgorithmArg & AddArg(const std::string &longName, char chShortName, @@ -2470,6 +2498,10 @@ class CPL_DLL GDALAlgorithmRegistry //! @cond Doxygen_Suppress void AddAliasFor(GDALInConstructionAlgorithmArg *arg, const std::string &alias); + + void AddShortNameAliasFor(GDALInConstructionAlgorithmArg *arg, + char shortNameAlias); + void SetPositional(GDALInConstructionAlgorithmArg *arg); //! @endcond diff --git a/port/cpl_string.h b/port/cpl_string.h index 42b8f56d80de..147adfe54c47 100644 --- a/port/cpl_string.h +++ b/port/cpl_string.h @@ -645,6 +645,7 @@ extern "C++" #include // For std::input_iterator_tag #include +#include #include // For std::pair /*! @cond Doxygen_Suppress */ @@ -669,6 +670,60 @@ extern "C++" { /*! @cond Doxygen_Suppress */ + + /** Equivalent of C++20 std::string::starts_with(const char*) */ + template + inline bool starts_with(const StringType &str, const char *prefix) + { + const size_t prefixLen = strlen(prefix); + return str.size() >= prefixLen && + str.compare(0, prefixLen, prefix, prefixLen) == 0; + } + + /** Equivalent of C++20 std::string::starts_with(const std::string &) */ + template + inline bool starts_with(const StringType &str, const std::string &prefix) + { + return str.size() >= prefix.size() && + str.compare(0, prefix.size(), prefix) == 0; + } + + /** Equivalent of C++20 std::string::starts_with(std::string_view) */ + template + inline bool starts_with(const StringType &str, std::string_view prefix) + { + return str.size() >= prefix.size() && + str.compare(0, prefix.size(), prefix) == 0; + } + + /** Equivalent of C++20 std::string::ends_with(const char*) */ + template + inline bool ends_with(const StringType &str, const char *suffix) + { + const size_t suffixLen = strlen(suffix); + return str.size() >= suffixLen && + str.compare(str.size() - suffixLen, suffixLen, suffix, + suffixLen) == 0; + } + + /** Equivalent of C++20 std::string::ends_with(const std::string &) */ + template + inline bool ends_with(const StringType &str, const std::string &suffix) + { + return str.size() >= suffix.size() && + str.compare(str.size() - suffix.size(), suffix.size(), suffix) == + 0; + } + + /** Equivalent of C++20 std::string::ends_with(std::string_view) */ + template + inline bool ends_with(const StringType &str, std::string_view suffix) + { + return str.size() >= suffix.size() && + str.compare(str.size() - suffix.size(), suffix.size(), suffix) == + 0; + } + /** Iterator for a CSLConstList */ struct CPL_DLL CSLIterator { diff --git a/port/cpl_vsil_az.cpp b/port/cpl_vsil_az.cpp index fdfaa2d69d85..ceae94bfd1f1 100644 --- a/port/cpl_vsil_az.cpp +++ b/port/cpl_vsil_az.cpp @@ -54,47 +54,16 @@ const char GDAL_MARKER_FOR_DIR[] = ".gdal_marker_for_dir"; /* VSIDIRAz */ /************************************************************************/ -struct VSIDIRAz : public VSIDIRWithMissingDirSynthesis +struct VSIDIRAz : public VSIDIRS3Like { - int nRecurseDepth = 0; - - std::string osNextMarker{}; - int nPos = 0; - - std::string osBucket{}; - std::string osObjectKey{}; - IVSIS3LikeFSHandler *poFS = nullptr; - std::unique_ptr poHandleHelper{}; - int nMaxFiles = 0; - bool bCacheEntries = true; - bool m_bSynthetizeMissingDirectories = false; - std::string m_osFilterPrefix{}; - - explicit VSIDIRAz(IVSIS3LikeFSHandler *poFSIn) : poFS(poFSIn) + explicit VSIDIRAz(IVSIS3LikeFSHandler *poFSIn) : VSIDIRS3Like(poFSIn) { } - VSIDIRAz(const VSIDIRAz &) = delete; - VSIDIRAz &operator=(const VSIDIRAz &) = delete; - - const VSIDIREntry *NextDirEntry() override; - - bool IssueListDir(); + bool IssueListDir() override; bool AnalyseAzureFileList(const std::string &osBaseURL, const char *pszXML); - void clear(); }; -/************************************************************************/ -/* clear() */ -/************************************************************************/ - -void VSIDIRAz::clear() -{ - osNextMarker.clear(); - nPos = 0; - aoEntries.clear(); -} - /************************************************************************/ /* AnalyseAzureFileList() */ /************************************************************************/ @@ -459,37 +428,6 @@ bool VSIDIRAz::IssueListDir() return ret; } -/************************************************************************/ -/* NextDirEntry() */ -/************************************************************************/ - -const VSIDIREntry *VSIDIRAz::NextDirEntry() -{ - constexpr int ARBITRARY_LIMIT = 10; - for (int i = 0; i < ARBITRARY_LIMIT; ++i) - { - if (nPos < static_cast(aoEntries.size())) - { - auto &entry = aoEntries[nPos]; - nPos++; - return entry.get(); - } - if (osNextMarker.empty()) - { - return nullptr; - } - if (!IssueListDir()) - { - return nullptr; - } - } - CPLError(CE_Failure, CPLE_AppDefined, - "More than %d consecutive List Blob " - "requests returning no blobs", - ARBITRARY_LIMIT); - return nullptr; -} - /************************************************************************/ /* VSIAzureFSHandler */ /************************************************************************/ @@ -828,33 +766,43 @@ int VSIAzureFSHandler::Stat(const char *pszFilename, VSIStatBufL *pStatBuf, return nRet; } - if (osFilename.find('/', GetFSPrefix().size()) == std::string::npos) + if (osFilename.size() > GetFSPrefix().size() && + osFilename.find('/', GetFSPrefix().size()) == std::string::npos) { osFilename += "/"; } - if (osFilename.size() > GetFSPrefix().size()) + // Special case for container + std::string osFilenameWithoutEndSlash(osFilename); + if (osFilename.size() > GetFSPrefix().size() && + osFilenameWithoutEndSlash.back() == '/') + osFilenameWithoutEndSlash.pop_back(); + if (osFilenameWithoutEndSlash.find('/', GetFSPrefix().size()) == + std::string::npos) { - // Special case for container - std::string osFilenameWithoutEndSlash(osFilename); - if (osFilenameWithoutEndSlash.back() == '/') - osFilenameWithoutEndSlash.resize(osFilenameWithoutEndSlash.size() - - 1); - if (osFilenameWithoutEndSlash.find('/', GetFSPrefix().size()) == - std::string::npos) + char **papszFileList = ReadDir(GetFSPrefix().c_str()); + if (osFilename.size() == GetFSPrefix().size()) { - char **papszFileList = ReadDir(GetFSPrefix().c_str()); - const int nIdx = CSLFindString( - papszFileList, - osFilenameWithoutEndSlash.substr(GetFSPrefix().size()).c_str()); CSLDestroy(papszFileList); - if (nIdx >= 0) + if (papszFileList) { pStatBuf->st_mtime = 0; pStatBuf->st_size = 0; pStatBuf->st_mode = S_IFDIR; return 0; } + return -1; + } + const int nIdx = CSLFindString( + papszFileList, + osFilenameWithoutEndSlash.substr(GetFSPrefix().size()).c_str()); + CSLDestroy(papszFileList); + if (nIdx >= 0) + { + pStatBuf->st_mtime = 0; + pStatBuf->st_size = 0; + pStatBuf->st_mode = S_IFDIR; + return 0; } } @@ -2529,7 +2477,6 @@ VSIDIR *VSIAzureFSHandler::OpenDir(const char *pszPath, int nRecurseDepth, VSIDIRAz *dir = new VSIDIRAz(this); dir->nRecurseDepth = nRecurseDepth; - dir->poFS = this; dir->poHandleHelper = std::move(poHandleHelper); dir->osBucket = std::move(osBucket); dir->osObjectKey = std::move(osObjectKey); diff --git a/port/cpl_vsil_curl_class.h b/port/cpl_vsil_curl_class.h index 7908de18bc9e..ba680bbd1add 100644 --- a/port/cpl_vsil_curl_class.h +++ b/port/cpl_vsil_curl_class.h @@ -1011,6 +1011,49 @@ struct VSIDIRWithMissingDirSynthesis : public VSIDIR bool bAddEntryForThisSubdir); }; +/************************************************************************/ +/* VSIDIRS3Like */ +/************************************************************************/ + +struct VSIDIRS3Like : public VSIDIRWithMissingDirSynthesis +{ + int nRecurseDepth = 0; + + std::string osNextMarker{}; + int nPos = 0; + + std::string osBucket{}; + std::string osObjectKey{}; + VSICurlFilesystemHandlerBase *poFS = nullptr; + IVSIS3LikeFSHandler *poS3FS = nullptr; + std::unique_ptr poHandleHelper{}; + int nMaxFiles = 0; + bool bCacheEntries = true; + bool m_bSynthetizeMissingDirectories = false; + std::string m_osFilterPrefix{}; + + // used when listing only the file system prefix + std::unique_ptr m_subdir{nullptr, + VSICloseDir}; + + explicit VSIDIRS3Like(IVSIS3LikeFSHandler *poFSIn) + : poFS(poFSIn), poS3FS(poFSIn) + { + } + + explicit VSIDIRS3Like(VSICurlFilesystemHandlerBase *poFSIn) : poFS(poFSIn) + { + } + + VSIDIRS3Like(const VSIDIRS3Like &) = delete; + VSIDIRS3Like &operator=(const VSIDIRS3Like &) = delete; + + const VSIDIREntry *NextDirEntry() override; + + virtual bool IssueListDir() = 0; + void clear(); +}; + /************************************************************************/ /* CurlRequestHelper */ /************************************************************************/ diff --git a/port/cpl_vsil_s3.cpp b/port/cpl_vsil_s3.cpp index f79de5ba203f..2396394bde64 100644 --- a/port/cpl_vsil_s3.cpp +++ b/port/cpl_vsil_s3.cpp @@ -62,55 +62,29 @@ namespace cpl /* VSIDIRS3 */ /************************************************************************/ -struct VSIDIRS3 : public VSIDIRWithMissingDirSynthesis +struct VSIDIRS3 : public VSIDIRS3Like { - int nRecurseDepth = 0; - - std::string osNextMarker{}; - int nPos = 0; - - std::string osBucket{}; - std::string osObjectKey{}; - VSICurlFilesystemHandlerBase *poFS = nullptr; - IVSIS3LikeFSHandler *poS3FS = nullptr; - IVSIS3LikeHandleHelper *poS3HandleHelper = nullptr; - int nMaxFiles = 0; - bool bCacheEntries = true; - bool m_bSynthetizeMissingDirectories = false; - std::string m_osFilterPrefix{}; - - explicit VSIDIRS3(IVSIS3LikeFSHandler *poFSIn) - : poFS(poFSIn), poS3FS(poFSIn) - { - } - - explicit VSIDIRS3(VSICurlFilesystemHandlerBase *poFSIn) : poFS(poFSIn) + explicit VSIDIRS3(IVSIS3LikeFSHandler *poFSIn) : VSIDIRS3Like(poFSIn) { } - ~VSIDIRS3() + explicit VSIDIRS3(VSICurlFilesystemHandlerBase *poFSIn) + : VSIDIRS3Like(poFSIn) { - delete poS3HandleHelper; } - VSIDIRS3(const VSIDIRS3 &) = delete; - VSIDIRS3 &operator=(const VSIDIRS3 &) = delete; - - const VSIDIREntry *NextDirEntry() override; - - bool IssueListDir(); + bool IssueListDir() override; bool AnalyseS3FileList(const std::string &osBaseURL, const char *pszXML, const std::set &oSetIgnoredStorageClasses, bool &bIsTruncated); - void clear(); }; /************************************************************************/ /* clear() */ /************************************************************************/ -void VSIDIRS3::clear() +void VSIDIRS3Like::clear() { osNextMarker.clear(); nPos = 0; @@ -498,39 +472,39 @@ bool VSIDIRS3::IssueListDir() while (true) { - poS3HandleHelper->ResetQueryParameters(); - const std::string osBaseURL(poS3HandleHelper->GetURL()); + poHandleHelper->ResetQueryParameters(); + const std::string osBaseURL(poHandleHelper->GetURL()); CURL *hCurlHandle = curl_easy_init(); if (!osBucket.empty()) { if (nRecurseDepth == 0) - poS3HandleHelper->AddQueryParameter("delimiter", "/"); + poHandleHelper->AddQueryParameter("delimiter", "/"); if (!l_osNextMarker.empty()) - poS3HandleHelper->AddQueryParameter("marker", l_osNextMarker); + poHandleHelper->AddQueryParameter("marker", l_osNextMarker); if (!osMaxKeys.empty()) - poS3HandleHelper->AddQueryParameter("max-keys", osMaxKeys); + poHandleHelper->AddQueryParameter("max-keys", osMaxKeys); if (!osObjectKey.empty()) - poS3HandleHelper->AddQueryParameter( + poHandleHelper->AddQueryParameter( "prefix", osObjectKey + "/" + m_osFilterPrefix); else if (!m_osFilterPrefix.empty()) - poS3HandleHelper->AddQueryParameter("prefix", m_osFilterPrefix); + poHandleHelper->AddQueryParameter("prefix", m_osFilterPrefix); } struct curl_slist *headers = VSICurlSetOptions( - hCurlHandle, poS3HandleHelper->GetURL().c_str(), nullptr); + hCurlHandle, poHandleHelper->GetURL().c_str(), nullptr); headers = VSICurlMergeHeaders( - headers, poS3HandleHelper->GetCurlHeaders("GET", headers)); + headers, poHandleHelper->GetCurlHeaders("GET", headers)); // Disable automatic redirection unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); CurlRequestHelper requestHelper; - const long response_code = - requestHelper.perform(hCurlHandle, headers, poFS, poS3HandleHelper); + const long response_code = requestHelper.perform( + hCurlHandle, headers, poFS, poHandleHelper.get()); NetworkStatisticsLogger::LogGET(requestHelper.sWriteFuncData.nSize); @@ -538,7 +512,7 @@ bool VSIDIRS3::IssueListDir() requestHelper.sWriteFuncData.pBuffer == nullptr) { if (requestHelper.sWriteFuncData.pBuffer != nullptr && - poS3HandleHelper->CanRestartOnError( + poHandleHelper->CanRestartOnError( requestHelper.sWriteFuncData.pBuffer, requestHelper.sWriteFuncHeaderData.pBuffer, false)) { @@ -574,13 +548,42 @@ bool VSIDIRS3::IssueListDir() /* NextDirEntry() */ /************************************************************************/ -const VSIDIREntry *VSIDIRS3::NextDirEntry() +const VSIDIREntry *VSIDIRS3Like::NextDirEntry() { - while (true) + constexpr int ARBITRARY_LIMIT = 10; + for (int i = 0; i < ARBITRARY_LIMIT; ++i) { if (nPos < static_cast(aoEntries.size())) { auto &entry = aoEntries[nPos]; + if (osBucket.empty()) + { + if (m_subdir) + { + if (auto subentry = m_subdir->NextDirEntry()) + { + const std::string name = std::string(entry->pszName) + .append("/") + .append(subentry->pszName); + CPLFree(const_cast(subentry)->pszName); + const_cast(subentry)->pszName = + CPLStrdup(name.c_str()); + return subentry; + } + m_subdir.reset(); + nPos++; + continue; + } + else if (nRecurseDepth != 0) + { + m_subdir.reset(VSIOpenDir(std::string(poFS->GetFSPrefix()) + .append(entry->pszName) + .c_str(), + nRecurseDepth - 1, nullptr)); + if (m_subdir) + return entry.get(); + } + } nPos++; return entry.get(); } @@ -593,6 +596,11 @@ const VSIDIREntry *VSIDIRS3::NextDirEntry() return nullptr; } } + CPLError(CE_Failure, CPLE_AppDefined, + "More than %d consecutive List Blob " + "requests returning no blobs", + ARBITRARY_LIMIT); + return nullptr; } /************************************************************************/ @@ -3275,8 +3283,8 @@ VSIDIR *IVSIS3LikeFSHandler::OpenDir(const char *pszPath, int nRecurseDepth, osObjectKey = osDirnameWithoutPrefix.substr(nSlashPos + 1); } - IVSIS3LikeHandleHelper *poS3HandleHelper = - CreateHandleHelper(osBucket.c_str(), true); + auto poS3HandleHelper = std::unique_ptr( + CreateHandleHelper(osBucket.c_str(), true)); if (poS3HandleHelper == nullptr) { return nullptr; @@ -3284,8 +3292,7 @@ VSIDIR *IVSIS3LikeFSHandler::OpenDir(const char *pszPath, int nRecurseDepth, VSIDIRS3 *dir = new VSIDIRS3(this); dir->nRecurseDepth = nRecurseDepth; - dir->poFS = this; - dir->poS3HandleHelper = poS3HandleHelper; + dir->poHandleHelper = std::move(poS3HandleHelper); dir->osBucket = std::move(osBucket); dir->osObjectKey = std::move(osObjectKey); dir->nMaxFiles = atoi(CSLFetchNameValueDef(papszOptions, "MAXFILES", "0"));