TextGeneration Pipeline

OnnxStack.Console/Examples/TextGenerationExample.cs

@@ -0,0 +1,37 @@
+using OnnxStack.Core.Config;
+using OnnxStack.TextGeneration.Models;
+using OnnxStack.TextGeneration.Pipelines;
+
+namespace OnnxStack.Console.Runner
+{
+    public sealed class TextGenerationExample : IExampleRunner
+    {
+        public TextGenerationExample()
+        {
+        }
+
+        public int Index => 40;
+
+        public string Name => "Text Generation Demo";
+
+        public string Description => "Text Generation Example";
+
+        public async Task RunAsync()
+        {
+            var pipeline = TextGenerationPipeline.CreatePipeline("D:\\Repositories\\phi2_onnx", executionProvider: ExecutionProvider.Cuda);
+
+            await pipeline.LoadAsync();
+
+            while (true)
+            {
+                OutputHelpers.WriteConsole("Enter Prompt: ", ConsoleColor.Gray);
+                var promptOptions = new PromptOptionsModel(OutputHelpers.ReadConsole(ConsoleColor.Cyan));
+                var searchOptions = new SearchOptionsModel();
+                await foreach (var token in pipeline.RunAsync(promptOptions, searchOptions))
+                {
+                    OutputHelpers.WriteConsole(token.Content, ConsoleColor.Yellow, false);
+                }
+            }
+        }
+    }
+}

OnnxStack.Console/OnnxStack.Console.csproj

@@ -22,6 +22,7 @@
     <ProjectReference Include="..\OnnxStack.FeatureExtractor\OnnxStack.FeatureExtractor.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-Nvidia'" />
     <ProjectReference Include="..\OnnxStack.StableDiffusion\OnnxStack.StableDiffusion.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-Nvidia'" />
     <ProjectReference Include="..\OnnxStack.ImageUpscaler\OnnxStack.ImageUpscaler.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Debug-Nvidia'" />
+    <ProjectReference Include="..\OnnxStack.TextGeneration\OnnxStack.TextGeneration.csproj" />
   </ItemGroup>
 
   <ItemGroup>

OnnxStack.TextGeneration/Common/TextGenerationModel.cs

@@ -0,0 +1,70 @@
+using Microsoft.ML.OnnxRuntime;
+using Microsoft.ML.OnnxRuntimeGenAI;
+using OnnxStack.Core.Config;
+
+namespace OnnxStack.TextGeneration.Common
+{
+    public class TextGenerationModel : IDisposable //: OnnxModelSession
+    {
+        private Model _model;
+        private Tokenizer _tokenizer;
+        private readonly TextGenerationModelConfig _configuration;
+        public TextGenerationModel(TextGenerationModelConfig configuration)
+        {
+            _configuration = configuration;
+        }
+
+        public Model Model => _model;
+        public Tokenizer Tokenizer => _tokenizer;
+
+
+        /// <summary>
+        /// Loads the model session.
+        /// </summary>
+        public async Task LoadAsync()
+        {
+            if (_model is not null)
+                return; // Already Loaded
+
+            await Task.Run(() =>
+            {
+                _model = new Model(_configuration.OnnxModelPath);
+                _tokenizer = new Tokenizer(_model);
+            });
+        }
+
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            _tokenizer?.Dispose();
+            _model?.Dispose();
+            _model = null;
+            _tokenizer = null;
+        }
+
+
+        public static TextGenerationModel Create(TextGenerationModelConfig configuration)
+        {
+            return new TextGenerationModel(configuration);
+        }
+
+        public static TextGenerationModel Create(string modelPath, int deviceId = 0, ExecutionProvider executionProvider = ExecutionProvider.DirectML)
+        {
+            var configuration = new TextGenerationModelConfig
+            {
+                DeviceId = deviceId,
+                ExecutionProvider = executionProvider,
+                ExecutionMode = ExecutionMode.ORT_SEQUENTIAL,
+                InterOpNumThreads = 0,
+                IntraOpNumThreads = 0,
+                OnnxModelPath = modelPath
+            };
+            return new TextGenerationModel(configuration);
+        }
+
+
+    }
+}

OnnxStack.TextGeneration/Common/TextGenerationModelConfig.cs

@@ -0,0 +1,9 @@
+using OnnxStack.Core.Config;
+
+namespace OnnxStack.TextGeneration.Common
+{
+    public record TextGenerationModelConfig : OnnxModelConfig
+    {
+
+    }
+}

OnnxStack.TextGeneration/Common/TextGenerationModelSet.cs

@@ -0,0 +1,20 @@
+using Microsoft.ML.OnnxRuntime;
+using OnnxStack.Core.Config;
+using System.Text.Json.Serialization;
+
+namespace OnnxStack.TextGeneration.Common
+{
+    public record TextGenerationModelSet : IOnnxModelSetConfig
+    {
+        public string Name { get; set; }
+        public bool IsEnabled { get; set; }
+        public int DeviceId { get; set; }
+        public int InterOpNumThreads { get; set; }
+        public int IntraOpNumThreads { get; set; }
+        public ExecutionMode ExecutionMode { get; set; }
+        public ExecutionProvider ExecutionProvider { get; set; }
+
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public TextGenerationModelConfig TextGenerationConfig { get; set; }
+    }
+}

OnnxStack.TextGeneration/Models/PromptOptionsModel.cs

@@ -0,0 +1,4 @@
+namespace OnnxStack.TextGeneration.Models
+{
+    public record class PromptOptionsModel(string Prompt);
+}

OnnxStack.TextGeneration/Models/SearchOptionsModel.cs

@@ -0,0 +1,20 @@
+namespace OnnxStack.TextGeneration.Models
+{ 
+    public class SearchOptionsModel
+    {
+        public int TopK { get; set; } = 50;
+        public float TopP { get; set; } = 0.95f;
+        public float Temperature { get; set; } = 1;
+        public float RepetitionPenalty { get; set; } = 0.9f;
+        public bool PastPresentShareBuffer { get; set; } = false;
+        public int NumReturnSequences { get; set; } = 1;
+        public int NumBeams { get; set; } = 1;
+        public int NoRepeatNgramSize { get; set; } = 0;
+        public int MinLength { get; set; } = 0;
+        public int MaxLength { get; set; } = 512;
+        public float LengthPenalty { get; set; } = 1;
+        public float DiversityPenalty { get; set; } = 0;
+        public bool EarlyStopping { get; set; } = true;
+        public bool DoSample { get; set; } = false;
+    }
+}

OnnxStack.TextGeneration/Models/TokenModel.cs

@@ -0,0 +1,4 @@
+namespace OnnxStack.TextGeneration.Models
+{
+    public readonly record struct TokenModel(int Id, string Content);
+}

OnnxStack.TextGeneration/OnnxStack.TextGeneration.csproj

@@ -0,0 +1,42 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>disable</Nullable>
+    <PlatformTarget>x64</PlatformTarget>
+    <Platforms>x64</Platforms>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Include="..\Assets\OnnxStack - 128x128.png" Link="OnnxStack - 128x128.png">
+      <PackagePath>\</PackagePath>
+      <Pack>True</Pack>
+    </None>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="OnnxStack.Core" Version="0.25.0" Condition=" '$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release-Nvidia'" />
+    <ProjectReference Include="..\OnnxStack.Core\OnnxStack.Core.csproj" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)'=='Debug-Nvidia'" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Binaries\cpu\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Reference Include="Microsoft.ML.OnnxRuntimeGenAI">
+      <HintPath>Binaries\cuda\Microsoft.ML.OnnxRuntimeGenAI.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="Binaries\cuda\Microsoft.ML.OnnxRuntimeGenAI.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Binaries\cuda\onnxruntime-genai.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>