Skip to content

Commit 7c449d7

Browse files
SebastianStehleSebastianStehle
committed
New feature to generate contents.
1 parent 2b0a308 commit 7c449d7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+698
-188
lines changed

‎.editorconfig renamed to ‎cli/Squidex.CLI/.editorconfig

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ insert_final_newline = true
1313
indent_style = space
1414
indent_size = 4
1515

16+
csharp_style_namespace_declarations = file_scoped
17+
1618
# FAILING ANALYZERS
1719
dotnet_diagnostic.RECS0002.severity = none
1820
dotnet_diagnostic.RECS0117.severity = none
@@ -22,6 +24,9 @@ dotnet_diagnostic.SA1649.severity = none
2224
# CA1707: Identifiers should not contain underscores
2325
dotnet_diagnostic.CA1707.severity = none
2426

27+
# CA2016: Forward the 'CancellationToken' parameter to methods
28+
dotnet_diagnostic.CA2016.severity = none
29+
2530
# CS8618: Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
2631
dotnet_diagnostic.CS8618.severity = none
2732

@@ -81,9 +86,6 @@ dotnet_diagnostic.MA0038.severity = none
8186

8287
# MA0039: Do not write your own certificate validation method
8388
dotnet_diagnostic.MA0039.severity = none
84-
85-
# MA0048: File name must match type name
86-
dotnet_diagnostic.MA0048.severity = none
8789

8890
# MA0049: Type name should not match containing namespace
8991
dotnet_diagnostic.MA0049.severity = none
@@ -166,5 +168,8 @@ dotnet_diagnostic.SA1601.severity = none
166168
# SA1602: Enumeration items should be documented
167169
dotnet_diagnostic.SA1602.severity = none
168170

171+
# SA1615: Element return value should be documented
172+
dotnet_diagnostic.SA1615.severity = none
173+
169174
# SA1623: Property summary documentation should match accessors
170175
dotnet_diagnostic.SA1623.severity = none

‎cli/Squidex.CLI/Directory.Build.props

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,6 @@
1515
<PackageTags>Squidex HeadlessCMS</PackageTags>
1616
<PublishRepositoryUrl>true</PublishRepositoryUrl>
1717
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
18-
<Version>10.3</Version>
18+
<Version>11.0</Version>
1919
</PropertyGroup>
2020
</Project>

‎cli/Squidex.CLI/Squidex.CLI.Core/Commands/Implementation/AI/AIContentGenerator.cs

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
// ==========================================================================
2+
// Squidex Headless CMS
3+
// ==========================================================================
4+
// Copyright (c) Squidex UG (haftungsbeschraenkt)
5+
// All rights reserved. Licensed under the MIT license.
6+
// ==========================================================================
7+
8+
using System.Text;
9+
using Markdig;
10+
using Markdig.Syntax;
11+
using Newtonsoft.Json;
12+
using Newtonsoft.Json.Linq;
13+
using OpenAI;
14+
using OpenAI.Managers;
15+
using OpenAI.ObjectModels.RequestModels;
16+
using OpenAI.ObjectModels.ResponseModels;
17+
using Squidex.CLI.Commands.Implementation.Utils;
18+
using Squidex.CLI.Configuration;
19+
using Squidex.ClientLibrary;
20+
21+
namespace Squidex.CLI.Commands.Implementation.AI;
22+
23+
public sealed class AIContentGenerator
24+
{
25+
private readonly IConfigurationStore configurationStore;
26+
27+
public AIContentGenerator(IConfigurationStore configurationStore)
28+
{
29+
this.configurationStore = configurationStore;
30+
}
31+
32+
public async Task<GeneratedContent> GenerateAsync(string description, string apiKey, string? schemaName = null,
33+
CancellationToken ct = default)
34+
{
35+
var cachedResponse = await MakeRequestAsync(description, apiKey, ct);
36+
37+
return ParseResult(schemaName, cachedResponse);
38+
}
39+
40+
private async Task<ChatCompletionCreateResponse> MakeRequestAsync(string description, string apiKey,
41+
CancellationToken ct)
42+
{
43+
var client = new OpenAIService(new OpenAiOptions
44+
{
45+
ApiKey = apiKey,
46+
});
47+
48+
var cacheKey = $"openapi/query-cache/{description.ToSha256Base64()}";
49+
var (cachedResponse, _) = configurationStore.Get<ChatCompletionCreateResponse>(cacheKey);
50+
51+
if (cachedResponse == null)
52+
{
53+
cachedResponse = await client.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest
54+
{
55+
Messages = new List<ChatMessage>
56+
{
57+
ChatMessage.FromSystem("Create a list as json array. The list is described as followed:"),
58+
ChatMessage.FromUser(description),
59+
ChatMessage.FromSystem("Also create a JSON object with the field names of this list as keys and the json type as value (string)."),
60+
ChatMessage.FromSystem("Also create a JSON array that only contains the name of the list above as valid slug."),
61+
},
62+
Model = OpenAI.ObjectModels.Models.ChatGpt3_5Turbo
63+
}, cancellationToken: ct);
64+
65+
configurationStore.Set(cacheKey, cachedResponse);
66+
}
67+
68+
return cachedResponse;
69+
}
70+
71+
private static GeneratedContent ParseResult(string? schemaName, ChatCompletionCreateResponse cachedResponse)
72+
{
73+
var parsed = Markdown.Parse(cachedResponse.Choices[0].Message.Content);
74+
75+
var codeBlocks = parsed.OfType<FencedCodeBlock>().ToList();
76+
if (codeBlocks.Count != 3)
77+
{
78+
ThrowParsingException("3 code blocks expected");
79+
return default!;
80+
}
81+
82+
var schemaObject = ParseJson<JObject>(codeBlocks[1], "Schema");
83+
var schemaFields = new List<UpsertSchemaFieldDto>();
84+
85+
foreach (var (key, value) in schemaObject)
86+
{
87+
var fieldType = value?.ToString();
88+
89+
switch (fieldType?.ToLowerInvariant())
90+
{
91+
case "string":
92+
case "text":
93+
schemaFields.Add(new UpsertSchemaFieldDto
94+
{
95+
Name = key!,
96+
Properties = new StringFieldPropertiesDto()
97+
});
98+
break;
99+
case "double":
100+
case "float":
101+
case "int":
102+
case "integer":
103+
case "long":
104+
case "number":
105+
case "real":
106+
schemaFields.Add(new UpsertSchemaFieldDto
107+
{
108+
Name = key!,
109+
Properties = new NumberFieldPropertiesDto()
110+
});
111+
break;
112+
case "bit":
113+
case "bool":
114+
case "boolean":
115+
schemaFields.Add(new UpsertSchemaFieldDto
116+
{
117+
Name = key!,
118+
Properties = new BooleanFieldPropertiesDto()
119+
});
120+
break;
121+
default:
122+
ThrowParsingException($"Unexpected field type '{fieldType}' for field '{key}'");
123+
return default!;
124+
}
125+
}
126+
127+
var nameArray = ParseJson<JArray>(codeBlocks[2], "SchemaName");
128+
if (nameArray.Count != 1)
129+
{
130+
ThrowParsingException("'SchemaName' json has an unexpected structure.");
131+
return default!;
132+
}
133+
134+
if (string.IsNullOrWhiteSpace(schemaName))
135+
{
136+
schemaName = nameArray[0].ToString();
137+
}
138+
139+
var contentsBlock = ParseJson<JArray>(codeBlocks[0], "Contents");
140+
var contentsList = new List<Dictionary<string, object>>();
141+
142+
foreach (var obj in contentsBlock.OfType<JObject>())
143+
{
144+
contentsList.Add(obj.OfType<JProperty>().ToDictionary(x => x.Name, x => (object)x.Value));
145+
}
146+
147+
return new GeneratedContent
148+
{
149+
SchemaFields = schemaFields,
150+
SchemaName = schemaName,
151+
Contents = contentsList
152+
};
153+
}
154+
155+
private static void ThrowParsingException(string reason)
156+
{
157+
throw new InvalidOperationException($"OpenAPI does not return a parsable result: {reason}.");
158+
}
159+
160+
private static T ParseJson<T>(LeafBlock block, string name) where T : JToken
161+
{
162+
JToken jsonNode;
163+
try
164+
{
165+
var jsonText = GetText(block);
166+
167+
jsonNode = JToken.Parse(jsonText);
168+
}
169+
catch (JsonException)
170+
{
171+
ThrowParsingException($"'{name}' code is not valid json.");
172+
return default!;
173+
}
174+
175+
if (jsonNode is not T typed)
176+
{
177+
ThrowParsingException($"'{name}' json has an unexpected structure.");
178+
return default!;
179+
}
180+
181+
return typed;
182+
183+
static string GetText(LeafBlock block)
184+
{
185+
var sb = new StringBuilder();
186+
187+
var lines = block.Lines.Lines;
188+
189+
if (lines != null)
190+
{
191+
foreach (var line in lines)
192+
{
193+
sb.AppendLine(line.Slice.ToString());
194+
}
195+
}
196+
197+
return sb.ToString();
198+
}
199+
}
200+
}

‎cli/Squidex.CLI/Squidex.CLI.Core/Commands/Implementation/AI/GeneratedContent.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// ==========================================================================
2+
// Squidex Headless CMS
3+
// ==========================================================================
4+
// Copyright (c) Squidex UG (haftungsbeschraenkt)
5+
// All rights reserved. Licensed under the MIT license.
6+
// ==========================================================================
7+
8+
using Squidex.ClientLibrary;
9+
10+
namespace Squidex.CLI.Commands.Implementation.AI;
11+
12+
public sealed class GeneratedContent
13+
{
14+
public List<UpsertSchemaFieldDto> SchemaFields { get; set; }
15+
16+
public string SchemaName { get; set; }
17+
18+
public List<Dictionary<string, object>> Contents { get; set; }
19+
}

‎cli/Squidex.CLI/Squidex.CLI.Core/Commands/Implementation/ImExport/ExportHelper.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,6 @@ public static async Task ExportAsync(this ISession session, IExportSettings sett
6565
while (totalRead < total);
6666
}
6767

68-
log.WriteLine("> Exported: {0} of {1}. Completed.", totalRead, total);
68+
log.Completed($"Export of {totalRead}/{total} content items completed.");
6969
}
7070
}

‎cli/Squidex.CLI/Squidex.CLI.Core/Commands/Implementation/ImExport/ImportHelper.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ public static async Task ImportAsync(this ISession session, IImportSettings sett
9393
}
9494
}
9595

96-
log.WriteLine("> Imported: {0}. Completed.", totalWritten);
96+
log.Completed($"Import of {totalWritten} content items completed");
9797
}
9898

9999
public static IEnumerable<DynamicData> Read(this Csv2SquidexConverter converter, Stream stream, string delimiter)

‎cli/Squidex.CLI/Squidex.CLI.Core/Commands/Implementation/LogExtensions.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,11 @@ public static void StepFailed(this ILogger log, Exception ex)
123123
HandleException(ex, log.StepFailed);
124124
}
125125

126+
public static void Completed(this ILogger log, string message)
127+
{
128+
log.WriteLine($"> {message}");
129+
}
130+
126131
public static void HandleException(Exception ex, Action<string> error)
127132
{
128133
switch (ex)

‎cli/Squidex.CLI/Squidex.CLI.Core/Commands/Implementation/Sync/Assets/DownloadPipeline.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ public DownloadPipeline(ISession session, ILogger log, IFileSystem fs)
5656
var (asset, path) = item;
5757

5858
var process = $"Downloading {path}";
59-
6059
try
6160
{
6261
var assetFile = fs.GetFile(path);

‎cli/Squidex.CLI/Squidex.CLI.Core/Commands/Implementation/Utils/Extensions.cs

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
// All rights reserved. Licensed under the MIT license.
66
// ==========================================================================
77

8+
using System.Runtime.CompilerServices;
89
using System.Security.Cryptography;
910
using System.Text;
1011
using System.Text.RegularExpressions;
@@ -105,6 +106,34 @@ public static IEnumerable<IEnumerable<T>> Batch<T>(this IEnumerable<T> source, i
105106
}
106107
}
107108

109+
public static string WithoutPrefix(this string value, string prefix)
110+
{
111+
if (value.EndsWith(prefix, StringComparison.OrdinalIgnoreCase))
112+
{
113+
return value[..^prefix.Length];
114+
}
115+
116+
return value;
117+
}
118+
119+
public static string ToSha256Base64(this string value)
120+
{
121+
return ToSha256Base64(Encoding.UTF8.GetBytes(value));
122+
}
123+
124+
public static string ToSha256Base64(this byte[] bytes)
125+
{
126+
var hashBytes = SHA256.HashData(bytes);
127+
128+
var result =
129+
Convert.ToBase64String(hashBytes)
130+
.Replace("+", string.Empty, StringComparison.Ordinal)
131+
.Replace("=", string.Empty, StringComparison.Ordinal)
132+
.ToLowerInvariant();
133+
134+
return result;
135+
}
136+
108137
public static DirectoryInfo CreateDirectory(this DirectoryInfo directory, string name)
109138
{
110139
return Directory.CreateDirectory(Path.Combine(directory.FullName, name));

‎cli/Squidex.CLI/Squidex.CLI.Core/Configuration/Configuration.cs renamed to ‎cli/Squidex.CLI/Squidex.CLI.Core/Configuration/ConfigurationModel.cs

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,3 @@
66
// ==========================================================================
77

88
namespace Squidex.CLI.Configuration;
9-
10-
public sealed class Configuration
11-
{
12-
public Dictionary<string, ConfiguredApp> Apps { get; } = new Dictionary<string, ConfiguredApp>();
13-
14-
public string? CurrentApp { get; set; }
15-
}

0 commit comments

Comments
 (0)