Skip to content

Commit c98cbbc

Browse files
rouaultrouault
authored
Merge pull request #12132 from rouault/gdal_fs
Add 'gdal vfs list/copy/delete'
2 parents 88b3b5b + 58b157d commit c98cbbc

28 files changed

+2274
-179
lines changed

‎apps/CMakeLists.txt

+4
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ add_library(
6868
gdalalg_vector_select.cpp
6969
gdalalg_vector_sql.cpp
7070
gdalalg_vector_write.cpp
71+
gdalalg_vfs.cpp
72+
gdalalg_vfs_copy.cpp
73+
gdalalg_vfs_delete.cpp
74+
gdalalg_vfs_list.cpp
7175
gdalinfo_lib.cpp
7276
gdalbuildvrt_lib.cpp
7377
gdal_grid_lib.cpp

‎apps/gdalalg_vfs.cpp

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/******************************************************************************
2+
*
3+
* Project: GDAL
4+
* Purpose: gdal "vfs" subcommand
5+
* Author: Even Rouault <even dot rouault at spatialys.com>
6+
*
7+
******************************************************************************
8+
* Copyright (c) 2025, Even Rouault <even dot rouault at spatialys.com>
9+
*
10+
* SPDX-License-Identifier: MIT
11+
****************************************************************************/
12+
13+
#include "gdalalgorithm.h"
14+
15+
#include "gdalalg_vfs_copy.h"
16+
#include "gdalalg_vfs_delete.h"
17+
#include "gdalalg_vfs_list.h"
18+
19+
/************************************************************************/
20+
/* GDALVFSAlgorithm */
21+
/************************************************************************/
22+
23+
class GDALVFSAlgorithm final : public GDALAlgorithm
24+
{
25+
public:
26+
static constexpr const char *NAME = "vfs";
27+
static constexpr const char *DESCRIPTION =
28+
"GDAL Virtual file system (VSI) commands.";
29+
static constexpr const char *HELP_URL = "/programs/gdal_vfs.html";
30+
31+
GDALVFSAlgorithm() : GDALAlgorithm(NAME, DESCRIPTION, HELP_URL)
32+
{
33+
RegisterSubAlgorithm<GDALVFSCopyAlgorithm>();
34+
RegisterSubAlgorithm<GDALVFSDeleteAlgorithm>();
35+
RegisterSubAlgorithm<GDALVFSListAlgorithm>();
36+
}
37+
38+
private:
39+
bool RunImpl(GDALProgressFunc, void *) override
40+
{
41+
CPLError(CE_Failure, CPLE_AppDefined,
42+
"The Run() method should not be called directly on the \"gdal "
43+
"vfs\" program.");
44+
return false;
45+
}
46+
};
47+
48+
GDAL_STATIC_REGISTER_ALG(GDALVFSAlgorithm);

‎apps/gdalalg_vfs_copy.cpp

+316
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,316 @@
1+
/******************************************************************************
2+
*
3+
* Project: GDAL
4+
* Purpose: gdal "vfs copy" subcommand
5+
* Author: Even Rouault <even dot rouault at spatialys.com>
6+
*
7+
******************************************************************************
8+
* Copyright (c) 2025, Even Rouault <even dot rouault at spatialys.com>
9+
*
10+
* SPDX-License-Identifier: MIT
11+
****************************************************************************/
12+
13+
#include "gdalalg_vfs_copy.h"
14+
15+
#include "cpl_conv.h"
16+
#include "cpl_string.h"
17+
#include "cpl_vsi.h"
18+
19+
#include <algorithm>
20+
21+
//! @cond Doxygen_Suppress
22+
23+
#ifndef _
24+
#define _(x) (x)
25+
#endif
26+
27+
/************************************************************************/
28+
/* GDALVFSCopyAlgorithm::GDALVFSCopyAlgorithm() */
29+
/************************************************************************/
30+
31+
GDALVFSCopyAlgorithm::GDALVFSCopyAlgorithm()
32+
: GDALAlgorithm(NAME, DESCRIPTION, HELP_URL)
33+
{
34+
{
35+
auto &arg =
36+
AddArg("source", 0, _("Source file or directory name"), &m_source)
37+
.SetPositional()
38+
.SetRequired();
39+
SetAutoCompleteFunctionForFilename(arg, 0);
40+
arg.AddValidationAction(
41+
[this]()
42+
{
43+
if (m_source.empty())
44+
{
45+
ReportError(CE_Failure, CPLE_IllegalArg,
46+
"Source filename cannot be empty");
47+
return false;
48+
}
49+
return true;
50+
});
51+
}
52+
{
53+
auto &arg =
54+
AddArg("destination", 0, _("Destination file or directory name"),
55+
&m_destination)
56+
.SetPositional()
57+
.SetRequired();
58+
SetAutoCompleteFunctionForFilename(arg, 0);
59+
arg.AddValidationAction(
60+
[this]()
61+
{
62+
if (m_destination.empty())
63+
{
64+
ReportError(CE_Failure, CPLE_IllegalArg,
65+
"Destination filename cannot be empty");
66+
return false;
67+
}
68+
return true;
69+
});
70+
}
71+
72+
AddArg("recursive", 'r', _("Copy subdirectories recursively"),
73+
&m_recursive);
74+
75+
AddArg("skip-errors", 0, _("Skip errors"), &m_skip);
76+
AddProgressArg();
77+
}
78+
79+
/************************************************************************/
80+
/* GDALVFSCopyAlgorithm::RunImpl() */
81+
/************************************************************************/
82+
83+
bool GDALVFSCopyAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
84+
void *pProgressData)
85+
{
86+
if (m_recursive || cpl::ends_with(m_source, "/*") ||
87+
cpl::ends_with(m_source, "\\*"))
88+
{
89+
// Make sure that copy -r [srcdir/]lastsubdir targetdir' creates
90+
// targetdir/lastsubdir if targetdir already exists (like cp -r does).
91+
if (m_source.back() == '/')
92+
m_source.pop_back();
93+
94+
if (!cpl::ends_with(m_source, "/*") && !cpl::ends_with(m_source, "\\*"))
95+
{
96+
VSIStatBufL statBufSrc;
97+
bool srcExists =
98+
VSIStatExL(m_source.c_str(), &statBufSrc,
99+
VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0;
100+
if (!srcExists)
101+
{
102+
srcExists =
103+
VSIStatExL(
104+
std::string(m_source).append("/").c_str(), &statBufSrc,
105+
VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0;
106+
}
107+
VSIStatBufL statBufDst;
108+
const bool dstExists =
109+
VSIStatExL(m_destination.c_str(), &statBufDst,
110+
VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0;
111+
if (srcExists && VSI_ISDIR(statBufSrc.st_mode) && dstExists &&
112+
VSI_ISDIR(statBufDst.st_mode))
113+
{
114+
if (m_destination.back() == '/')
115+
m_destination.pop_back();
116+
const auto srcLastSlashPos = m_source.rfind('/');
117+
if (srcLastSlashPos != std::string::npos)
118+
m_destination += m_source.substr(srcLastSlashPos);
119+
else
120+
m_destination = CPLFormFilenameSafe(
121+
m_destination.c_str(), m_source.c_str(), nullptr);
122+
}
123+
}
124+
else
125+
{
126+
m_source.resize(m_source.size() - 2);
127+
}
128+
129+
uint64_t curAmount = 0;
130+
return CopyRecursive(m_source, m_destination, 0, m_recursive ? -1 : 0,
131+
curAmount, 0, pfnProgress, pProgressData);
132+
}
133+
else
134+
{
135+
VSIStatBufL statBufSrc;
136+
bool srcExists =
137+
VSIStatExL(m_source.c_str(), &statBufSrc,
138+
VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0;
139+
if (!srcExists)
140+
{
141+
ReportError(CE_Failure, CPLE_FileIO, "%s does not exist",
142+
m_source.c_str());
143+
return false;
144+
}
145+
if (VSI_ISDIR(statBufSrc.st_mode))
146+
{
147+
ReportError(CE_Failure, CPLE_FileIO,
148+
"%s is a directory. Use -r/--recursive option",
149+
m_source.c_str());
150+
return false;
151+
}
152+
153+
return CopySingle(m_source, m_destination, ~(static_cast<uint64_t>(0)),
154+
pfnProgress, pProgressData);
155+
}
156+
}
157+
158+
/************************************************************************/
159+
/* GDALVFSCopyAlgorithm::CopySingle() */
160+
/************************************************************************/
161+
162+
bool GDALVFSCopyAlgorithm::CopySingle(const std::string &src,
163+
const std::string &dstIn, uint64_t size,
164+
GDALProgressFunc pfnProgress,
165+
void *pProgressData) const
166+
{
167+
CPLDebug("gdal_vfs_copy", "Copying file %s...", src.c_str());
168+
VSIStatBufL sStat;
169+
std::string dst = dstIn;
170+
const bool bExists =
171+
VSIStatExL(dst.back() == '/' ? dst.c_str()
172+
: std::string(dst).append("/").c_str(),
173+
&sStat, VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG) == 0;
174+
if ((!bExists && dst.back() == '/') ||
175+
(bExists && VSI_ISDIR(sStat.st_mode)))
176+
{
177+
const std::string filename = CPLGetFilename(src.c_str());
178+
dst = CPLFormFilenameSafe(dst.c_str(), filename.c_str(), nullptr);
179+
}
180+
return VSICopyFile(src.c_str(), dst.c_str(), nullptr, size, nullptr,
181+
pfnProgress, pProgressData) == 0 ||
182+
m_skip;
183+
}
184+
185+
/************************************************************************/
186+
/* GDALVFSCopyAlgorithm::CopyRecursive() */
187+
/************************************************************************/
188+
189+
bool GDALVFSCopyAlgorithm::CopyRecursive(const std::string &srcIn,
190+
const std::string &dst, int depth,
191+
int maxdepth, uint64_t &curAmount,
192+
uint64_t totalAmount,
193+
GDALProgressFunc pfnProgress,
194+
void *pProgressData) const
195+
{
196+
std::string src(srcIn);
197+
if (src.back() == '/')
198+
src.pop_back();
199+
200+
if (pfnProgress && depth == 0)
201+
{
202+
CPLDebug("gdal_vfs_copy", "Listing source files...");
203+
std::unique_ptr<VSIDIR, decltype(&VSICloseDir)> dir(
204+
VSIOpenDir(src.c_str(), maxdepth, nullptr), VSICloseDir);
205+
if (dir)
206+
{
207+
while (const auto entry = VSIGetNextDirEntry(dir.get()))
208+
{
209+
if (!(entry->pszName[0] == '.' &&
210+
(entry->pszName[1] == '.' || entry->pszName[1] == 0)))
211+
{
212+
totalAmount += entry->nSize + 1;
213+
if (!pfnProgress(0.0, "", pProgressData))
214+
return false;
215+
}
216+
}
217+
}
218+
}
219+
220+
CPLDebug("gdal_vfs_copy", "Copying directory %s...", src.c_str());
221+
std::unique_ptr<VSIDIR, decltype(&VSICloseDir)> dir(
222+
VSIOpenDir(src.c_str(), 0, nullptr), VSICloseDir);
223+
if (dir)
224+
{
225+
VSIStatBufL sStat;
226+
if (VSIStatL(dst.c_str(), &sStat) != 0)
227+
{
228+
if (VSIMkdir(dst.c_str(), 0755) != 0)
229+
{
230+
ReportError(m_skip ? CE_Warning : CE_Failure, CPLE_FileIO,
231+
"Cannot create directory %s", dst.c_str());
232+
return m_skip;
233+
}
234+
}
235+
236+
while (const auto entry = VSIGetNextDirEntry(dir.get()))
237+
{
238+
if (!(entry->pszName[0] == '.' &&
239+
(entry->pszName[1] == '.' || entry->pszName[1] == 0)))
240+
{
241+
const std::string subsrc =
242+
CPLFormFilenameSafe(src.c_str(), entry->pszName, nullptr);
243+
if (VSI_ISDIR(entry->nMode))
244+
{
245+
const std::string subdest = CPLFormFilenameSafe(
246+
dst.c_str(), entry->pszName, nullptr);
247+
if (maxdepth < 0 || depth < maxdepth)
248+
{
249+
if (!CopyRecursive(subsrc, subdest, depth + 1, maxdepth,
250+
curAmount, totalAmount, pfnProgress,
251+
pProgressData) &&
252+
!m_skip)
253+
{
254+
return false;
255+
}
256+
}
257+
else
258+
{
259+
if (VSIStatL(subdest.c_str(), &sStat) != 0)
260+
{
261+
if (VSIMkdir(subdest.c_str(), 0755) != 0)
262+
{
263+
ReportError(m_skip ? CE_Warning : CE_Failure,
264+
CPLE_FileIO,
265+
"Cannot create directory %s",
266+
subdest.c_str());
267+
if (!m_skip)
268+
return false;
269+
}
270+
}
271+
}
272+
curAmount += 1;
273+
274+
if (pfnProgress &&
275+
!pfnProgress(
276+
std::min(1.0, static_cast<double>(curAmount) /
277+
static_cast<double>(totalAmount)),
278+
"", pProgressData))
279+
{
280+
return false;
281+
}
282+
}
283+
else
284+
{
285+
void *pScaledProgressData = GDALCreateScaledProgress(
286+
static_cast<double>(curAmount) /
287+
static_cast<double>(totalAmount),
288+
std::min(1.0, static_cast<double>(curAmount +
289+
entry->nSize + 1) /
290+
static_cast<double>(totalAmount)),
291+
pfnProgress, pProgressData);
292+
const bool bRet = CopySingle(
293+
subsrc, dst, entry->nSize,
294+
pScaledProgressData ? GDALScaledProgress : nullptr,
295+
pScaledProgressData);
296+
GDALDestroyScaledProgress(pScaledProgressData);
297+
298+
curAmount += entry->nSize + 1;
299+
300+
if (!bRet)
301+
return false;
302+
}
303+
}
304+
}
305+
}
306+
else
307+
{
308+
ReportError(m_skip ? CE_Warning : CE_Failure, CPLE_AppDefined,
309+
"%s is not a directory or cannot be opened", src.c_str());
310+
if (!m_skip)
311+
return false;
312+
}
313+
return true;
314+
}
315+
316+
//! @endcond

0 commit comments

Comments
 (0)