Skip to content

Commit 625ba81

Browse files
SebastianStehleSebastianStehle
committed
Download assets.
1 parent 239f6f5 commit 625ba81

File tree

11 files changed

+229
-49
lines changed

11 files changed

+229
-49
lines changed

‎cli/Squidex.CLI/Squidex.CLI/Commands/App_Assets.cs

Lines changed: 72 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
// ==========================================================================
77

88
using System;
9+
using System.Collections.Concurrent;
10+
using System.Collections.Generic;
911
using System.IO;
1012
using System.Linq;
1113
using System.Threading.Tasks;
@@ -74,15 +76,15 @@ public async Task Import(ImportArguments arguments)
7476
{
7577
var existing = existings.Items.First();
7678

77-
log.WriteLine($"Updating: {file.FullName}");
79+
log.WriteLine($"Uploading: {file.FullName}");
7880

7981
await assets.PutAssetContentAsync(session.App, existing.Id, fileParameter);
8082

8183
log.StepSuccess();
8284
}
8385
else
8486
{
85-
log.WriteLine($"Uploading: {file.FullName}");
87+
log.WriteLine($"Uploading New: {file.FullName}");
8688

8789
var result = await assets.PostAssetAsync(session.App, parentId, duplicate: arguments.Duplicate, file: fileParameter);
8890

@@ -110,6 +112,56 @@ public async Task Import(ImportArguments arguments)
110112
}
111113
}
112114

115+
[Command(Name = "export", Description = "Export all files to the source folder.")]
116+
public async Task Export(ImportArguments arguments)
117+
{
118+
var session = configuration.StartSession();
119+
120+
var assets = session.Assets;
121+
122+
using (var fs = FileSystems.Create(arguments.Path))
123+
{
124+
var folderTree = new FolderTree(session);
125+
var folderNames = new HashSet<string>();
126+
127+
var parentId = await folderTree.GetIdAsync(arguments.TargetFolder);
128+
129+
var downloadPipeline = new DownloadPipeline(session, log, fs)
130+
{
131+
FilePathProviderAsync = async asset =>
132+
{
133+
var assetFolder = await folderTree.GetPathAsync(asset.ParentId);
134+
var assetPath = asset.FileName;
135+
136+
if (!string.IsNullOrWhiteSpace(assetFolder))
137+
{
138+
assetPath = Path.Combine(assetFolder, assetPath);
139+
}
140+
141+
if (!folderNames.Add(assetPath))
142+
{
143+
assetPath = Path.Combine(assetFolder, $"{asset.Id}_{asset.FileName}");
144+
}
145+
146+
return FilePath.Create(assetPath);
147+
}
148+
};
149+
150+
await assets.GetAllByQueryAsync(session.App, async asset =>
151+
{
152+
await downloadPipeline.DownloadAsync(asset);
153+
},
154+
new AssetQuery
155+
{
156+
ParentId = parentId
157+
});
158+
159+
await downloadPipeline.CompleteAsync();
160+
161+
log.WriteLine("> Export completed");
162+
}
163+
}
164+
113165
[Validator(typeof(Validator))]
114166
public sealed class ImportArguments : IArgumentModel
115167
{
@@ -130,6 +182,24 @@ public Validator()
130182
}
131183
}
132184
}
185+
186+
[Validator(typeof(Validator))]
187+
public sealed class ExportArguments : IArgumentModel
188+
{
189+
[Operand(Name = "folder", Description = "The source folder.")]
190+
public string Path { get; set; }
191+
192+
[Option(ShortName = "t", LongName = "target", Description = "Path to the target folder.")]
193+
public string SourceFolder { get; set; }
194+
195+
public sealed class Validator : AbstractValidator<ExportArguments>
196+
{
197+
public Validator()
198+
{
199+
RuleFor(x => x.Path).NotEmpty();
200+
}
201+
}
202+
}
133203
}
134204
}
135205
}

‎cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/FileSystem/FilePath.cs

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
// All rights reserved. Licensed under the MIT license.
66
// ==========================================================================
77

8-
using System.Collections.Generic;
98
using System.IO;
109
using System.Linq;
1110

@@ -15,21 +14,26 @@ public sealed class FilePath
1514
{
1615
public static readonly FilePath Root = new FilePath(string.Empty);
1716

18-
public IEnumerable<string> Elements { get; }
17+
public string[] Elements { get; }
1918

2019
public FilePath(params string[] elements)
2120
{
2221
Elements = elements;
2322
}
2423

24+
public static FilePath Create(string path)
25+
{
26+
return new FilePath(path.Split('/', '\\'));
27+
}
28+
2529
public FilePath Combine(FilePath path)
2630
{
2731
return new FilePath(Elements.Concat(path.Elements).ToArray());
2832
}
2933

3034
public override string ToString()
3135
{
32-
return Path.Combine(Elements.ToArray());
36+
return Path.Combine(Elements);
3337
}
3438
}
3539
}

‎cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/AssetsSynchronizer.cs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,10 @@ public Task CleanupAsync(IFileSystem fs)
4040

4141
public async Task ExportAsync(ISyncService sync, SyncOptions options, ISession session)
4242
{
43-
var downloadPipeline = new DownloadPipeline(session, log, sync.FileSystem);
43+
var downloadPipeline = new DownloadPipeline(session, log, sync.FileSystem)
44+
{
45+
FilePathProvider = asset => asset.Id.GetBlobPath()
46+
};
4447

4548
var assets = new List<AssetModel>();
4649
var assetBatch = 0;
@@ -95,7 +98,10 @@ public async Task ImportAsync(ISyncService sync, SyncOptions options, ISession s
9598
{
9699
if (model?.Assets?.Count > 0)
97100
{
98-
var uploader = new UploadPipeline(session, log, sync.FileSystem);
101+
var uploader = new UploadPipeline(session, log, sync.FileSystem)
102+
{
103+
FilePathProvider = asset => asset.Id.GetBlobPath()
104+
};
99105

100106
await uploader.UploadAsync(model.Assets);
101107
await uploader.CompleteAsync();

‎cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/DownloadPipeline.cs

Lines changed: 50 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,50 @@ namespace Squidex.CLI.Commands.Implementation.Sync.Assets
1717
{
1818
public sealed class DownloadPipeline
1919
{
20-
private readonly ActionBlock<AssetDto> pipeline;
20+
private readonly ITargetBlock<AssetDto> pipelineStart;
21+
private readonly IDataflowBlock pipelineEnd;
22+
23+
public Func<AssetDto, FilePath> FilePathProvider { get; set; }
24+
25+
public Func<AssetDto, Task<FilePath>> FilePathProviderAsync { get; set; }
2126

2227
public DownloadPipeline(ISession session, ILogger log, IFileSystem fs)
2328
{
24-
pipeline = new ActionBlock<AssetDto>(async asset =>
29+
var fileNameStep = new TransformBlock<AssetDto, (AssetDto, FilePath)>(async asset =>
30+
{
31+
FilePath path;
32+
33+
if (FilePathProvider != null)
34+
{
35+
path = FilePathProvider(asset);
36+
}
37+
else if (FilePathProviderAsync != null)
38+
{
39+
path = await FilePathProviderAsync(asset);
40+
}
41+
else
42+
{
43+
path = new FilePath(asset.Id);
44+
}
45+
46+
return (asset, path);
47+
},
48+
new ExecutionDataflowBlockOptions
49+
{
50+
MaxDegreeOfParallelism = 1,
51+
MaxMessagesPerTask = 1,
52+
BoundedCapacity = 1
53+
});
54+
55+
var downloadStep = new ActionBlock<(AssetDto, FilePath)>(async item =>
2556
{
26-
var process = $"Downloading {asset.Id}";
57+
var (asset, path) = item;
58+
59+
var process = $"Downloading {path}";
2760

2861
try
2962
{
30-
var assetFile = fs.GetBlobFile(asset.Id);
63+
var assetFile = fs.GetFile(path);
3164
var assetHash = GetFileHash(assetFile, asset);
3265

3366
if (assetHash == null || !string.Equals(asset.FileHash, assetHash))
@@ -36,9 +69,9 @@ public DownloadPipeline(ISession session, ILogger log, IFileSystem fs)
3669

3770
await using (response.Stream)
3871
{
39-
await using (var fileStream = assetFile.OpenWrite())
72+
await using (var stream = assetFile.OpenWrite())
4073
{
41-
await response.Stream.CopyToAsync(fileStream);
74+
await response.Stream.CopyToAsync(stream);
4275
}
4376
}
4477

@@ -59,6 +92,14 @@ public DownloadPipeline(ISession session, ILogger log, IFileSystem fs)
5992
MaxMessagesPerTask = 1,
6093
BoundedCapacity = 16
6194
});
95+
96+
fileNameStep.LinkTo(downloadStep, new DataflowLinkOptions
97+
{
98+
PropagateCompletion = true
99+
});
100+
101+
pipelineStart = fileNameStep;
102+
pipelineEnd = downloadStep;
62103
}
63104

64105
private static string GetFileHash(IFile file, AssetDto asset)
@@ -101,14 +142,14 @@ private static string GetFileHash(IFile file, AssetDto asset)
101142

102143
public Task DownloadAsync(AssetDto asset)
103144
{
104-
return pipeline.SendAsync(asset);
145+
return pipelineStart.SendAsync(asset);
105146
}
106147

107148
public Task CompleteAsync()
108149
{
109-
pipeline.Complete();
150+
pipelineEnd.Complete();
110151

111-
return pipeline.Completion;
152+
return pipelineEnd.Completion;
112153
}
113154
}
114155
}

‎cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/Extensions.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,12 @@ public static class Extensions
1515
{
1616
public static IFile GetBlobFile(this IFileSystem fs, string id)
1717
{
18-
return fs.GetFile(new FilePath("assets", "files", $"{id}.blob"));
18+
return fs.GetFile(GetBlobPath(id));
19+
}
20+
21+
public static FilePath GetBlobPath(this string id)
22+
{
23+
return new FilePath("assets", "files", $"{id}.blob");
1924
}
2025

2126
public static BulkUpdateAssetsJobDto ToMoveJob(this AssetModel model, string parentId)

‎cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Assets/UploadPipeline.cs

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,52 @@ namespace Squidex.CLI.Commands.Implementation.Sync.Assets
1616
{
1717
public sealed class UploadPipeline
1818
{
19-
private readonly ActionBlock<AssetModel> pipeline;
19+
private readonly ITargetBlock<AssetModel> pipelineStart;
20+
private readonly IDataflowBlock pipelineEnd;
21+
22+
public Func<AssetModel, FilePath> FilePathProvider { get; set; }
23+
24+
public Func<AssetModel, Task<FilePath>> FilePathProviderAsync { get; set; }
2025

2126
public UploadPipeline(ISession session, ILogger log, IFileSystem fs)
2227
{
2328
var tree = new FolderTree(session);
2429

25-
pipeline = new ActionBlock<AssetModel>(async asset =>
30+
var fileNameStep = new TransformBlock<AssetModel, (AssetModel, FilePath)>(async asset =>
31+
{
32+
FilePath path;
33+
34+
if (FilePathProvider != null)
35+
{
36+
path = FilePathProvider(asset);
37+
}
38+
else if (FilePathProviderAsync != null)
39+
{
40+
path = await FilePathProviderAsync(asset);
41+
}
42+
else
43+
{
44+
path = new FilePath(asset.Id);
45+
}
46+
47+
return (asset, path);
48+
},
49+
new ExecutionDataflowBlockOptions
50+
{
51+
MaxDegreeOfParallelism = 1,
52+
MaxMessagesPerTask = 1,
53+
BoundedCapacity = 1
54+
});
55+
56+
var uploadStep = new ActionBlock<(AssetModel, FilePath)>(async item =>
2657
{
27-
var process = $"Uploading {asset.Id}";
58+
var (asset, path) = item;
59+
60+
var process = $"Uploading {path}";
2861

2962
try
3063
{
31-
var assetFile = fs.GetBlobFile(asset.Id);
64+
var assetFile = fs.GetFile(path);
3265

3366
await using (var stream = assetFile.OpenRead())
3467
{
@@ -56,21 +89,29 @@ public UploadPipeline(ISession session, ILogger log, IFileSystem fs)
5689
MaxMessagesPerTask = 1,
5790
BoundedCapacity = 16
5891
});
92+
93+
fileNameStep.LinkTo(uploadStep, new DataflowLinkOptions
94+
{
95+
PropagateCompletion = true
96+
});
97+
98+
pipelineStart = fileNameStep;
99+
pipelineEnd = uploadStep;
59100
}
60101

61102
public async Task UploadAsync(IEnumerable<AssetModel> assets)
62103
{
63104
foreach (var asset in assets)
64105
{
65-
await pipeline.SendAsync(asset);
106+
await pipelineStart.SendAsync(asset);
66107
}
67108
}
68109

69110
public Task CompleteAsync()
70111
{
71-
pipeline.Complete();
112+
pipelineEnd.Complete();
72113

73-
return pipeline.Completion;
114+
return pipelineEnd.Completion;
74115
}
75116
}
76117
}

‎cli/Squidex.CLI/Squidex.CLI/Commands/Implementation/Sync/Contents/ContentsSynchronizer.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ public sealed class ContentsSynchronizer : ISynchronizer
2020
private const string Ref = "../__json/contents";
2121
private readonly ILogger log;
2222

23-
public string Name => "contents";
23+
public string Name => "Contents";
2424

2525
public ContentsSynchronizer(ILogger log)
2626
{

0 commit comments

Comments
 (0)