From 737e17965f7a8412a2f4fa29ddc421690a9233ba Mon Sep 17 00:00:00 2001 From: Ani <115020168+drawbyperpetual@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:07:12 +0200 Subject: [PATCH] [AdvancedPaste] Semantic Kernel support --- Directory.Packages.props | 3 +- .../AdvancedPaste/AdvancedPaste.csproj | 1 + .../AdvancedPasteXAML/App.xaml.cs | 6 +- .../Controls/PromptBox.xaml.cs | 4 +- .../Helpers/AICompletionsHelper.cs | 142 ------------ .../AdvancedPaste/Helpers/ClipboardHelper.cs | 60 ++++- .../AdvancedPaste/Helpers/ErrorHelpers.cs | 19 ++ .../AdvancedPaste/Helpers/IUserSettings.cs | 2 - .../AdvancedPaste/Helpers/TransformHelpers.cs | 148 ++++++++++++ .../AdvancedPaste/Helpers/UserSettings.cs | 4 - .../AdvancedPaste/Models/ClipboardFormat.cs | 2 +- .../Models/PasteActionException.cs | 2 +- .../AdvancedPaste/Models/PasteFormat.cs | 2 +- .../Models/PasteFormatMetadataAttribute.cs | 2 + .../AdvancedPaste/Models/PasteFormats.cs | 68 +++++- .../Services/IAICredentialsProvider.cs | 14 ++ .../Services/ICustomTextTransformService.cs | 12 + .../AdvancedPaste/Services/IKernelService.cs | 13 ++ .../Services/IPasteFormatExecutor.cs | 3 +- .../Services/OpenAI/CredentialsProvider.cs | 37 +++ .../OpenAI/CustomTextTransformService.cs | 87 ++++++++ .../Services/OpenAI/KernelExtensions.cs | 43 ++++ .../Services/OpenAI/KernelService.cs | 178 +++++++++++++++ .../Services/PasteFormatExecutor.cs | 211 +----------------- .../Strings/en-us/Resources.resw | 3 - .../ViewModels/OptionsViewModel.cs | 87 ++++---- .../AdvancedPasteProperties.cs | 5 - 27 files changed, 722 insertions(+), 436 deletions(-) delete mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Helpers/ErrorHelpers.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Helpers/TransformHelpers.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CredentialsProvider.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelExtensions.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 0967532dc32c..c5e1241e07b2 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,7 +4,7 @@ - + @@ -31,6 +31,7 @@ + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj index ca34cf03ff2a..46775779acc2 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj @@ -55,6 +55,7 @@ + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs index 77f85518037b..7094a823531e 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs @@ -72,9 +72,11 @@ public App() Host = Microsoft.Extensions.Hosting.Host.CreateDefaultBuilder().UseContentRoot(AppContext.BaseDirectory).ConfigureServices((context, services) => { services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); }).Build(); viewModel = GetService(); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs index b33f998cbf5f..0f5631d56ac6 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs @@ -105,9 +105,9 @@ private async void InputTxtBox_KeyDown(object sender, Microsoft.UI.Xaml.Input.Ke } } - private void PreviewPasteBtn_Click(object sender, RoutedEventArgs e) + private async void PreviewPasteBtn_Click(object sender, RoutedEventArgs e) { - ViewModel.PasteCustom(); + await ViewModel.PasteCustomAsync(); } private void ThumbUpDown_Click(object sender, RoutedEventArgs e) diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs deleted file mode 100644 index d9586638b890..000000000000 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Microsoft Corporation -// The Microsoft Corporation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Globalization; -using System.IO; -using System.Net; - -using Azure; -using Azure.AI.OpenAI; -using ManagedCommon; -using Microsoft.PowerToys.Settings.UI.Library; -using Microsoft.PowerToys.Telemetry; -using Windows.Security.Credentials; - -namespace AdvancedPaste.Helpers -{ - public class AICompletionsHelper - { - // Return Response and Status code from the request. - public struct AICompletionsResponse - { - public AICompletionsResponse(string response, int apiRequestStatus) - { - Response = response; - ApiRequestStatus = apiRequestStatus; - } - - public string Response { get; } - - public int ApiRequestStatus { get; } - } - - private string _openAIKey; - - private string _modelName = "gpt-3.5-turbo-instruct"; - - public bool IsAIEnabled => !string.IsNullOrEmpty(this._openAIKey); - - public AICompletionsHelper() - { - this._openAIKey = LoadOpenAIKey(); - } - - public void SetOpenAIKey(string openAIKey) - { - this._openAIKey = openAIKey; - } - - public string GetKey() - { - return _openAIKey; - } - - public static string LoadOpenAIKey() - { - PasswordVault vault = new PasswordVault(); - - try - { - PasswordCredential cred = vault.Retrieve("https://platform.openai.com/api-keys", "PowerToys_AdvancedPaste_OpenAIKey"); - if (cred is not null) - { - return cred.Password.ToString(); - } - } - catch (Exception) - { - } - - return string.Empty; - } - - private Response GetAICompletion(string systemInstructions, string userMessage) - { - OpenAIClient azureAIClient = new OpenAIClient(_openAIKey); - - var response = azureAIClient.GetCompletions( - new CompletionsOptions() - { - DeploymentName = _modelName, - Prompts = - { - systemInstructions + "\n\n" + userMessage, - }, - Temperature = 0.01F, - MaxTokens = 2000, - }); - - if (response.Value.Choices[0].FinishReason == "length") - { - Console.WriteLine("Cut off due to length constraints"); - } - - return response; - } - - public AICompletionsResponse AIFormatString(string inputInstructions, string inputString) - { - string systemInstructions = $@"You are tasked with reformatting user's clipboard data. Use the user's instructions, and the content of their clipboard below to edit their clipboard content as they have requested it. - -Do not output anything else besides the reformatted clipboard content."; - - string userMessage = $@"User instructions: -{inputInstructions} - -Clipboard Content: -{inputString} - -Output: -"; - - string aiResponse = null; - Response rawAIResponse = null; - int apiRequestStatus = (int)HttpStatusCode.OK; - try - { - rawAIResponse = this.GetAICompletion(systemInstructions, userMessage); - aiResponse = rawAIResponse.Value.Choices[0].Text; - - int promptTokens = rawAIResponse.Value.Usage.PromptTokens; - int completionTokens = rawAIResponse.Value.Usage.CompletionTokens; - PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent(promptTokens, completionTokens, _modelName)); - } - catch (Azure.RequestFailedException error) - { - Logger.LogError("GetAICompletion failed", error); - PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomErrorEvent(error.Message)); - apiRequestStatus = error.Status; - } - catch (Exception error) - { - Logger.LogError("GetAICompletion failed", error); - PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomErrorEvent(error.Message)); - apiRequestStatus = -1; - } - - return new AICompletionsResponse(aiResponse, apiRequestStatus); - } - } -} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardHelper.cs index 9ec1dd1618de..3edcf62b4f33 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardHelper.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardHelper.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using AdvancedPaste.Models; @@ -41,13 +42,55 @@ internal static async Task GetAvailableClipboardFormatsAsync(Da if (storageItems.Count == 1 && storageItems.Single() is StorageFile file && ImageFileTypes.Contains(file.FileType)) { - availableClipboardFormats |= ClipboardFormat.ImageFile; + availableClipboardFormats |= ClipboardFormat.Image; + } + + if (availableClipboardFormats == ClipboardFormat.None) + { + // Advertise the "generic" File format only if there is no other specific format available; confusing for AI otherwise. + availableClipboardFormats |= ClipboardFormat.File; } } return availableClipboardFormats; } + internal static async Task HasDataAsync(DataPackageView clipboardData) + { + var availableFormats = await GetAvailableClipboardFormatsAsync(clipboardData); + + return availableFormats == ClipboardFormat.Text ? !string.IsNullOrEmpty(await clipboardData.GetTextAsync()) : availableFormats != ClipboardFormat.None; + } + + internal static async Task TryCopyPasteDataPackageAsync(DataPackage dataPackage, Action onCopied) + { + Logger.LogTrace(); + + if (await HasDataAsync(dataPackage.GetView())) + { + Clipboard.SetContent(dataPackage); + await FlushAsync(); + onCopied(); + SendPasteKeyCombination(); + } + } + + internal static DataPackage CreateDataPackageFromText(string text) + { + DataPackage dataPackage = new(); + dataPackage.SetText(text); + return dataPackage; + } + + internal static async Task CreateDataPackageFromFileContentAsync(string fileName) + { + var storageFile = await StorageFile.GetFileFromPathAsync(fileName); + + DataPackage dataPackage = new(); + dataPackage.SetStorageItems([storageFile]); + return dataPackage; + } + internal static void SetClipboardTextContent(string text) { Logger.LogTrace(); @@ -71,7 +114,7 @@ private static bool Flush() { try { - Task.Run(Clipboard.Flush).Wait(); + Clipboard.Flush(); return true; } catch (Exception ex) @@ -86,17 +129,10 @@ private static bool Flush() return false; } - private static async Task FlushAsync() => await Task.Run(Flush); - - internal static async Task SetClipboardFileContentAsync(string fileName) + private static async Task FlushAsync() { - var storageFile = await StorageFile.GetFileFromPathAsync(fileName); - - DataPackage output = new(); - output.SetStorageItems([storageFile]); - Clipboard.SetContent(output); - - await FlushAsync(); + // This should run on the UI thread to avoid the "calling application is not the owner of the data on the clipboard" error. + return await Task.Factory.StartNew(Flush, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.FromCurrentSynchronizationContext()); } internal static void SetClipboardImageContent(RandomAccessStreamReference image) diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ErrorHelpers.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ErrorHelpers.cs new file mode 100644 index 000000000000..d7ff360c3d94 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ErrorHelpers.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Globalization; +using System.Net; + +namespace AdvancedPaste.Helpers; + +public static class ErrorHelpers +{ + public static string TranslateErrorText(int apiRequestStatus) => (HttpStatusCode)apiRequestStatus switch + { + HttpStatusCode.TooManyRequests => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyTooManyRequests"), + HttpStatusCode.Unauthorized => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyUnauthorized"), + HttpStatusCode.OK => string.Empty, + _ => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyError") + apiRequestStatus.ToString(CultureInfo.InvariantCulture), + }; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs index 49dbfda94596..d0fee7c15ef6 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs @@ -14,8 +14,6 @@ public interface IUserSettings { public bool ShowCustomPreview { get; } - public bool SendPasteKeyCombination { get; } - public bool CloseAfterLosingFocus { get; } public IReadOnlyList CustomActions { get; } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/TransformHelpers.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/TransformHelpers.cs new file mode 100644 index 000000000000..0d0529373137 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/TransformHelpers.cs @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Globalization; +using System.IO; +using System.Threading.Tasks; + +using AdvancedPaste.Models; +using ManagedCommon; +using Windows.ApplicationModel.DataTransfer; +using Windows.Graphics.Imaging; +using Windows.Storage.Streams; + +namespace AdvancedPaste.Helpers; + +public static class TransformHelpers +{ + public static async Task TransformAsync(PasteFormats format, DataPackageView clipboardData) + { + return format switch + { + PasteFormats.PlainText => ToPlainText(clipboardData), + PasteFormats.Markdown => ToMarkdown(clipboardData), + PasteFormats.Json => ToJson(clipboardData), + PasteFormats.ImageToText => await ImageToTextAsync(clipboardData), + PasteFormats.PasteAsTxtFile => await ToTxtFileAsync(clipboardData), + PasteFormats.PasteAsPngFile => await ToPngFileAsync(clipboardData), + PasteFormats.PasteAsHtmlFile => await ToHtmlFileAsync(clipboardData), + PasteFormats.KernelQuery => throw new ArgumentException($"Unsupported format {format}", nameof(format)), + _ => throw new ArgumentException($"Unknown value {format}", nameof(format)), + }; + } + + private static DataPackage ToPlainText(DataPackageView clipboardData) + { + Logger.LogTrace(); + return CreateDataPackageFromText(MarkdownHelper.PasteAsPlainTextFromClipboard(clipboardData)); + } + + private static DataPackage ToMarkdown(DataPackageView clipboardData) + { + Logger.LogTrace(); + return CreateDataPackageFromText(MarkdownHelper.ToMarkdown(clipboardData)); + } + + private static DataPackage ToJson(DataPackageView clipboardData) + { + Logger.LogTrace(); + return CreateDataPackageFromText(JsonHelper.ToJsonFromXmlOrCsv(clipboardData)); + } + + private static async Task ImageToTextAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + + var bitmap = await ClipboardHelper.GetClipboardImageContentAsync(clipboardData) ?? throw new ArgumentException("No image content found in clipboard", nameof(clipboardData)); + var text = await OcrHelpers.ExtractTextAsync(bitmap); + return CreateDataPackageFromText(text); + } + + private static async Task ToPngFileAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + + var clipboardBitmap = await ClipboardHelper.GetClipboardImageContentAsync(clipboardData); + + using var pngStream = new InMemoryRandomAccessStream(); + var encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, pngStream); + encoder.SetSoftwareBitmap(clipboardBitmap); + await encoder.FlushAsync(); + + return await CreateDataPackageFromFileContentAsync(pngStream.AsStreamForRead(), "png"); + } + + private static async Task ToTxtFileAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + + var text = await ClipboardHelper.GetClipboardTextOrHtmlTextAsync(clipboardData); + return await CreateDataPackageFromFileContentAsync(text, "txt"); + } + + private static async Task ToHtmlFileAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + + var cfHtml = await ClipboardHelper.GetClipboardHtmlContentAsync(clipboardData); + var html = RemoveHtmlMetadata(cfHtml); + + return await CreateDataPackageFromFileContentAsync(html, "html"); + } + + /// + /// Removes leading CF_HTML metadata from HTML clipboard data. + /// See: https://learn.microsoft.com/en-us/windows/win32/dataxchg/html-clipboard-format + /// + private static string RemoveHtmlMetadata(string cfHtml) + { + int? GetIntTagValue(string tagName) + { + var tagNameWithColon = tagName + ":"; + int tagStartPos = cfHtml.IndexOf(tagNameWithColon, StringComparison.InvariantCulture); + + const int tagValueLength = 10; + return tagStartPos != -1 && int.TryParse(cfHtml.AsSpan(tagStartPos + tagNameWithColon.Length, tagValueLength), CultureInfo.InvariantCulture, out int result) ? result : null; + } + + var startFragmentIndex = GetIntTagValue("StartFragment"); + var endFragmentIndex = GetIntTagValue("EndFragment"); + + return (startFragmentIndex == null || endFragmentIndex == null) ? cfHtml : cfHtml[startFragmentIndex.Value..endFragmentIndex.Value]; + } + + private static async Task CreateDataPackageFromFileContentAsync(string data, string fileExtension) + { + if (string.IsNullOrEmpty(data)) + { + throw new ArgumentException($"Empty value in {nameof(CreateDataPackageFromFileContentAsync)}", nameof(data)); + } + + var path = GetPasteAsFileTempFilePath(fileExtension); + + await File.WriteAllTextAsync(path, data); + return await ClipboardHelper.CreateDataPackageFromFileContentAsync(path); + } + + private static async Task CreateDataPackageFromFileContentAsync(Stream stream, string fileExtension) + { + var path = GetPasteAsFileTempFilePath(fileExtension); + + using var fileStream = File.Create(path); + await stream.CopyToAsync(fileStream); + + return await ClipboardHelper.CreateDataPackageFromFileContentAsync(path); + } + + private static string GetPasteAsFileTempFilePath(string fileExtension) + { + var prefix = ResourceLoaderInstance.ResourceLoader.GetString("PasteAsFile_FilePrefix"); + var timestamp = DateTime.Now.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture); + + return Path.Combine(Path.GetTempPath(), $"{prefix}{timestamp}.{fileExtension}"); + } + + private static DataPackage CreateDataPackageFromText(string content) => ClipboardHelper.CreateDataPackageFromText(content); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs index fe60dd7f53be..699b011f1c98 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs @@ -35,8 +35,6 @@ internal sealed class UserSettings : IUserSettings, IDisposable public bool ShowCustomPreview { get; private set; } - public bool SendPasteKeyCombination { get; private set; } - public bool CloseAfterLosingFocus { get; private set; } public IReadOnlyList AdditionalActions => _additionalActions; @@ -48,7 +46,6 @@ public UserSettings() _settingsUtils = new SettingsUtils(); ShowCustomPreview = true; - SendPasteKeyCombination = true; CloseAfterLosingFocus = false; _additionalActions = []; _customActions = []; @@ -99,7 +96,6 @@ void UpdateSettings() var properties = settings.Properties; ShowCustomPreview = properties.ShowCustomPreview; - SendPasteKeyCombination = properties.SendPasteKeyCombination; CloseAfterLosingFocus = properties.CloseAfterLosingFocus; var sourceAdditionalActions = properties.AdditionalActions; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardFormat.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardFormat.cs index a63e79735ed9..0b619769934c 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardFormat.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardFormat.cs @@ -14,5 +14,5 @@ public enum ClipboardFormat Html = 1 << 1, Audio = 1 << 2, Image = 1 << 3, - ImageFile = 1 << 4, + File = 1 << 4, } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionException.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionException.cs index fed4e24c501a..6abb9b060ac6 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionException.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionException.cs @@ -6,6 +6,6 @@ namespace AdvancedPaste.Models; -public sealed class PasteActionException(string message) : Exception(message) +public sealed class PasteActionException(string message, Exception innerException) : Exception(message, innerException) { } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs index c38a54b84319..ab51f1d629db 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs @@ -33,7 +33,7 @@ public PasteFormat(PasteFormats format, ClipboardFormat clipboardFormats, bool i } public PasteFormat(AdvancedPasteCustomAction customAction, ClipboardFormat clipboardFormats, bool isAIServiceEnabled) - : this(PasteFormats.Custom, clipboardFormats, isAIServiceEnabled) + : this(PasteFormats.KernelQuery, clipboardFormats, isAIServiceEnabled) { Name = customAction.Name; Prompt = customAction.Prompt; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormatMetadataAttribute.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormatMetadataAttribute.cs index cb3a8a954e37..bd0f329f3ac8 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormatMetadataAttribute.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormatMetadataAttribute.cs @@ -20,4 +20,6 @@ public sealed class PasteFormatMetadataAttribute : Attribute public ClipboardFormat SupportedClipboardFormats { get; init; } public string IPCKey { get; init; } + + public string KernelFunctionDescription { get; init; } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs index fe710d54101d..ab753b535a27 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs @@ -3,32 +3,82 @@ // See the LICENSE file in the project root for more information. using Microsoft.PowerToys.Settings.UI.Library; +using Windows.Devices.Radios; namespace AdvancedPaste.Models; public enum PasteFormats { - [PasteFormatMetadata(IsCoreAction = true, ResourceId = "PasteAsPlainText", IconGlyph = "\uE8E9", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Text)] + [PasteFormatMetadata( + IsCoreAction = true, + ResourceId = "PasteAsPlainText", + IconGlyph = "\uE8E9", + RequiresAIService = false, + SupportedClipboardFormats = ClipboardFormat.Text)] PlainText, - [PasteFormatMetadata(IsCoreAction = true, ResourceId = "PasteAsMarkdown", IconGlyph = "\ue8a5", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Text)] + [PasteFormatMetadata( + IsCoreAction = true, + ResourceId = "PasteAsMarkdown", + IconGlyph = "\ue8a5", + RequiresAIService = false, + SupportedClipboardFormats = ClipboardFormat.Text, + KernelFunctionDescription = "Takes clipboard text and formats it as markdown text.")] Markdown, - [PasteFormatMetadata(IsCoreAction = true, ResourceId = "PasteAsJson", IconGlyph = "\uE943", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Text)] + [PasteFormatMetadata( + IsCoreAction = true, + ResourceId = "PasteAsJson", + IconGlyph = "\uE943", + RequiresAIService = false, + SupportedClipboardFormats = ClipboardFormat.Text, + KernelFunctionDescription = "Takes clipboard text and formats it as JSON text.")] Json, - [PasteFormatMetadata(IsCoreAction = false, ResourceId = "ImageToText", IconGlyph = "\uE91B", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Image | ClipboardFormat.ImageFile, IPCKey = AdvancedPasteAdditionalActions.PropertyNames.ImageToText)] + [PasteFormatMetadata( + IsCoreAction = false, + ResourceId = "ImageToText", + IconGlyph = "\uE91B", + RequiresAIService = false, + SupportedClipboardFormats = ClipboardFormat.Image, + IPCKey = AdvancedPasteAdditionalActions.PropertyNames.ImageToText, + KernelFunctionDescription = "Takes an image in the clipboard and extracts all text from it using OCR.")] ImageToText, - [PasteFormatMetadata(IsCoreAction = false, ResourceId = "PasteAsTxtFile", IconGlyph = "\uE8D2", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Text | ClipboardFormat.Html, IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsTxtFile)] + [PasteFormatMetadata( + IsCoreAction = false, + ResourceId = "PasteAsTxtFile", + IconGlyph = "\uE8D2", + RequiresAIService = false, + SupportedClipboardFormats = ClipboardFormat.Text | ClipboardFormat.Html, + IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsTxtFile, + KernelFunctionDescription = "Takes text or HTML data in the clipboard and transforms it to a TXT file.")] PasteAsTxtFile, - [PasteFormatMetadata(IsCoreAction = false, ResourceId = "PasteAsPngFile", IconGlyph = "\uE8B9", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Image | ClipboardFormat.ImageFile, IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsPngFile)] + [PasteFormatMetadata( + IsCoreAction = false, + ResourceId = "PasteAsPngFile", + IconGlyph = "\uE8B9", + RequiresAIService = false, + SupportedClipboardFormats = ClipboardFormat.Image, + IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsPngFile, + KernelFunctionDescription = "Takes an image in the clipboard and transforms it to a PNG file.")] PasteAsPngFile, - [PasteFormatMetadata(IsCoreAction = false, ResourceId = "PasteAsHtmlFile", IconGlyph = "\uF6FA", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Html, IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsHtmlFile)] + [PasteFormatMetadata( + IsCoreAction = false, + ResourceId = "PasteAsHtmlFile", + IconGlyph = "\uF6FA", + RequiresAIService = false, + SupportedClipboardFormats = ClipboardFormat.Html, + IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsHtmlFile, + KernelFunctionDescription = "Takes HTML data in the clipboard and transforms it to an HTML file.")] PasteAsHtmlFile, - [PasteFormatMetadata(IsCoreAction = false, IconGlyph = "\uE945", RequiresAIService = true, SupportedClipboardFormats = ClipboardFormat.Text)] - Custom, + [PasteFormatMetadata( + IsCoreAction = false, + IconGlyph = "\uE945", + RequiresAIService = true, + SupportedClipboardFormats = ClipboardFormat.Text | ClipboardFormat.Html | ClipboardFormat.Audio | ClipboardFormat.Image)] + KernelQuery, } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs new file mode 100644 index 000000000000..54759b7dc87e --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace AdvancedPaste.Services; + +public interface IAICredentialsProvider +{ + bool IsConfigured { get; } + + string Key { get; } + + bool Refresh(); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs new file mode 100644 index 000000000000..82ffbffc8b1f --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; + +namespace AdvancedPaste.Services; + +public interface ICustomTextTransformService +{ + Task TransformStringAsync(string inputInstructions, string inputString); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs new file mode 100644 index 000000000000..339bf4e09be8 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; +using Windows.ApplicationModel.DataTransfer; + +namespace AdvancedPaste.Services; + +public interface IKernelService +{ + Task GetCompletionAsync(string inputInstructions, DataPackageView clipboardData); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IPasteFormatExecutor.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IPasteFormatExecutor.cs index e0bb39ab7c74..72f08396df77 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/IPasteFormatExecutor.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IPasteFormatExecutor.cs @@ -4,10 +4,11 @@ using System.Threading.Tasks; using AdvancedPaste.Models; +using Windows.ApplicationModel.DataTransfer; namespace AdvancedPaste.Services; public interface IPasteFormatExecutor { - Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source); + Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source); } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CredentialsProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CredentialsProvider.cs new file mode 100644 index 000000000000..859bea82fb5c --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CredentialsProvider.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Windows.Security.Credentials; + +namespace AdvancedPaste.Services.OpenAI; + +public sealed class CredentialsProvider : IAICredentialsProvider +{ + public CredentialsProvider() => Refresh(); + + public string Key { get; private set; } + + public bool IsConfigured => !string.IsNullOrEmpty(Key); + + public bool Refresh() + { + var oldKey = Key; + Key = LoadKey(); + return oldKey != Key; + } + + private static string LoadKey() + { + try + { + var cred = new PasswordVault().Retrieve("https://platform.openai.com/api-keys", "PowerToys_AdvancedPaste_OpenAIKey"); + return cred?.Password ?? string.Empty; + } + catch (Exception) + { + return string.Empty; + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs new file mode 100644 index 000000000000..efc55f42bbcd --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading.Tasks; +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using Azure; +using Azure.AI.OpenAI; +using ManagedCommon; +using Microsoft.PowerToys.Telemetry; + +namespace AdvancedPaste.Services.OpenAI; + +public sealed class CustomTextTransformService(IAICredentialsProvider aiCredentialsProvider) : ICustomTextTransformService +{ + private const string ModelName = "gpt-3.5-turbo-instruct"; + + private readonly IAICredentialsProvider _aiCredentialsProvider = aiCredentialsProvider; + + private async Task GetAICompletionAsync(string systemInstructions, string userMessage) + { + OpenAIClient azureAIClient = new(_aiCredentialsProvider.Key); + + var response = await azureAIClient.GetCompletionsAsync( + new() + { + DeploymentName = ModelName, + Prompts = + { + systemInstructions + "\n\n" + userMessage, + }, + Temperature = 0.01F, + MaxTokens = 2000, + }); + + if (response.Value.Choices[0].FinishReason == "length") + { + Logger.LogDebug("Cut off due to length constraints"); + } + + return response; + } + + public async Task TransformStringAsync(string inputInstructions, string inputString) + { + if (string.IsNullOrWhiteSpace(inputInstructions)) + { + return string.Empty; + } + + if (string.IsNullOrWhiteSpace(inputString)) + { + Logger.LogWarning("Clipboard has no usable text data"); + return string.Empty; + } + + string systemInstructions = +$@"You are tasked with reformatting user's clipboard data. Use the user's instructions, and the content of their clipboard below to edit their clipboard content as they have requested it. +Do not output anything else besides the reformatted clipboard content."; + + string userMessage = +$@"User instructions: +{inputInstructions} + +Clipboard Content: +{inputString} + +Output: +"; + + try + { + var reponse = await GetAICompletionAsync(systemInstructions, userMessage); + PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent(reponse.Usage.PromptTokens, reponse.Usage.CompletionTokens, ModelName)); + return reponse.Choices[0].Text; + } + catch (Exception ex) + { + Logger.LogError($"{nameof(GetAICompletionAsync)} failed", ex); + PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomErrorEvent(ex.Message)); + + throw new PasteActionException(ErrorHelpers.TranslateErrorText((ex as RequestFailedException)?.Status ?? -1), ex); + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelExtensions.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelExtensions.cs new file mode 100644 index 000000000000..109a51729462 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelExtensions.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading.Tasks; +using AdvancedPaste.Helpers; +using Microsoft.SemanticKernel; +using Windows.ApplicationModel.DataTransfer; + +namespace AdvancedPaste.Services.OpenAI; + +internal static class KernelExtensions +{ + private const string DataPackageKey = "DataPackage"; + private const string LastErrorKey = "LastError"; + + internal static DataPackageView GetDataPackageView(this Kernel kernel) + { + kernel.Data.TryGetValue(DataPackageKey, out object obj); + return obj as DataPackageView ?? (obj as DataPackage)?.GetView(); + } + + internal static DataPackage GetDataPackage(this Kernel kernel) + { + kernel.Data.TryGetValue(DataPackageKey, out object obj); + return obj as DataPackage ?? new(); + } + + internal static async Task GetDataFormatsAsync(this Kernel kernel) + { + var clipboardFormats = await ClipboardHelper.GetAvailableClipboardFormatsAsync(kernel.GetDataPackageView()); + return clipboardFormats.ToString(); + } + + internal static void SetDataPackage(this Kernel kernel, DataPackage dataPackage) => kernel.Data[DataPackageKey] = dataPackage; + + internal static void SetDataPackageView(this Kernel kernel, DataPackageView dataPackageView) => kernel.Data[DataPackageKey] = dataPackageView; + + internal static Exception GetLastError(this Kernel kernel) => kernel.Data.TryGetValue(LastErrorKey, out object obj) ? obj as Exception : null; + + internal static void SetLastError(this Kernel kernel, Exception error) => kernel.Data[LastErrorKey] = error; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs new file mode 100644 index 000000000000..553c24465025 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using ManagedCommon; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Windows.ApplicationModel.DataTransfer; + +namespace AdvancedPaste.Services.OpenAI; + +public sealed class KernelService(IAICredentialsProvider aiCredentialsProvider, ICustomTextTransformService customTextTransformService) : IKernelService +{ + private const string ModelName = "gpt-4o"; + private readonly IAICredentialsProvider _aiCredentialsProvider = aiCredentialsProvider; + private readonly ICustomTextTransformService _customTextTransformService = customTextTransformService; + + public async Task GetCompletionAsync(string inputInstructions, DataPackageView clipboardData) + { + Logger.LogTrace(); + + var kernel = CreateKernel(); + kernel.SetDataPackageView(clipboardData); + + OpenAIPromptExecutionSettings executionSettings = new() + { + ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions, + Temperature = 0.01, + }; + + ChatHistory chatHistory = []; + + chatHistory.AddSystemMessage(""" + You are an agent who is tasked with helping users paste their clipboard data. You have functions available to help you with this task. + You never need to ask permission, always try to do as the user asks. The user will only input one message and will not be available for further questions, so try your best. + The user will put in a request to format their clipboard data and you will fulfill it. + You will not directly see the output clipboard content, and do not need to provide it in the chat. You just need to do the transform operations as needed. + """); + chatHistory.AddSystemMessage($"Available clipboard formats: {await kernel.GetDataFormatsAsync()}"); + chatHistory.AddUserMessage(inputInstructions); + + try + { + var result = await kernel.GetRequiredService().GetChatMessageContentAsync(chatHistory, executionSettings, kernel); + chatHistory.AddAssistantMessage(result.Content); + + if (kernel.GetLastError() is Exception ex) + { + throw ex; + } + + var outputPackage = kernel.GetDataPackage(); + + if (result == null || !(await ClipboardHelper.HasDataAsync(outputPackage.GetView()))) + { + throw new InvalidOperationException("No data was returned from the completion operation"); + } + + Logger.LogDebug($"Completion done: \n{FormatChatHistory(chatHistory)}"); + return outputPackage; + } + catch (Exception ex) + { + Logger.LogError($"Error executing completion", ex); + Logger.LogError($"Completion Error: \n{FormatChatHistory(chatHistory)}"); + + if (ex is HttpOperationException error) + { + throw new PasteActionException(ErrorHelpers.TranslateErrorText((int?)error.StatusCode ?? -1), error); + } + else + { + throw; + } + } + } + + private Kernel CreateKernel() + { + var kernelBuilder = Kernel.CreateBuilder().AddOpenAIChatCompletion(ModelName, _aiCredentialsProvider.Key); + kernelBuilder.Plugins.AddFromFunctions("Actions", CreateKernelFunctions()); + return kernelBuilder.Build(); + } + + private IEnumerable CreateKernelFunctions() + { + KernelReturnParameterMetadata returnParameter = new() { Description = "Array of available clipboard formats after operation" }; + + var customTransformFunction = KernelFunctionFactory.CreateFromMethod( + method: ExecuteCustomTransformAsync, + functionName: "CustomTransform", + description: "Takes input instructions and transforms clipboard text (not TXT files) with these input instructions, putting the result back on the clipboard. This uses AI to accomplish the task.", + parameters: [new("inputInstructions") { Description = "Input instructions to AI", ParameterType = typeof(string) }], + returnParameter); + + var standardTransformFunctions = from format in Enum.GetValues() + let description = PasteFormat.MetadataDict[format].KernelFunctionDescription + where !string.IsNullOrEmpty(description) + select KernelFunctionFactory.CreateFromMethod( + method: async (Kernel kernel) => await ExecuteStandardTransformAsync(kernel, format), + functionName: format.ToString(), + description: $"{description} Puts the result back on the clipboard.", + parameters: null, + returnParameter); + + return standardTransformFunctions.Prepend(customTransformFunction); + } + + private Task ExecuteCustomTransformAsync(Kernel kernel, string inputInstructions) => + ExecuteTransformAsync( + kernel, + async dataPackageView => + { + var inputString = await dataPackageView.GetTextAsync(); + var aICompletionsResponse = await _customTextTransformService.TransformStringAsync(inputInstructions, inputString); + return ClipboardHelper.CreateDataPackageFromText(aICompletionsResponse); + }); + + private Task ExecuteStandardTransformAsync(Kernel kernel, PasteFormats format) => + ExecuteTransformAsync( + kernel, + async dataPackageView => await TransformHelpers.TransformAsync(format, dataPackageView)); + + private static async Task ExecuteTransformAsync(Kernel kernel, Func> transformFunc) + { + kernel.SetLastError(null); + + try + { + var input = kernel.GetDataPackageView(); + var output = await transformFunc(input); + kernel.SetDataPackage(output); + return await kernel.GetDataFormatsAsync(); + } + catch (Exception ex) + { + kernel.SetLastError(ex); + throw; + } + } + + private static string FormatChatHistory(ChatHistory chatHistory) => string.Join(Environment.NewLine, chatHistory.Select(FormatChatMessage)); + + private static string FormatChatMessage(ChatMessageContent chatMessage) + { + static string Redact(object data) => +#if DEBUG + data?.ToString(); +#else + "[Redacted]"; +#endif + + static string FormatKernelArguments(KernelArguments kernelArguments) => + string.Join(", ", kernelArguments?.Select(argument => $"{argument.Key}: {Redact(argument.Value)}") ?? []); + + static string FormatKernelContent(KernelContent kernelContent) => +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + kernelContent switch + { + FunctionCallContent functionCallContent => $"{functionCallContent.FunctionName}({FormatKernelArguments(functionCallContent.Arguments)})", + FunctionResultContent functionResultContent => functionResultContent.FunctionName, + _ => kernelContent.ToString(), + }; +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + var role = chatMessage.Role; + var content = string.Join(" / ", chatMessage.Items.Select(FormatKernelContent)); + var redactedContent = role == AuthorRole.System || role == AuthorRole.Tool ? content : Redact(content); + return $"-> {role}: {redactedContent}"; + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs index f77cf8ef997d..d1e6849b4122 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs @@ -3,25 +3,20 @@ // See the LICENSE file in the project root for more information. using System; -using System.Globalization; -using System.IO; -using System.Net; using System.Threading.Tasks; using AdvancedPaste.Helpers; using AdvancedPaste.Models; using ManagedCommon; using Microsoft.PowerToys.Telemetry; using Windows.ApplicationModel.DataTransfer; -using Windows.Graphics.Imaging; -using Windows.Storage.Streams; namespace AdvancedPaste.Services; -public sealed class PasteFormatExecutor(AICompletionsHelper aiHelper) : IPasteFormatExecutor +public sealed class PasteFormatExecutor(IKernelService kernelService) : IPasteFormatExecutor { - private readonly AICompletionsHelper _aiHelper = aiHelper; + private readonly IKernelService _kernelService = kernelService; - public async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source) + public async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source) { if (!pasteFormat.IsEnabled) { @@ -30,47 +25,7 @@ public async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, Paste WriteTelemetry(pasteFormat.Format, source); - return await ExecutePasteFormatCoreAsync(pasteFormat, Clipboard.GetContent()); - } - - private async Task ExecutePasteFormatCoreAsync(PasteFormat pasteFormat, DataPackageView clipboardData) - { - switch (pasteFormat.Format) - { - case PasteFormats.PlainText: - ToPlainText(clipboardData); - return null; - - case PasteFormats.Markdown: - ToMarkdown(clipboardData); - return null; - - case PasteFormats.Json: - ToJson(clipboardData); - return null; - - case PasteFormats.ImageToText: - await ImageToTextAsync(clipboardData); - return null; - - case PasteFormats.PasteAsTxtFile: - await ToTxtFileAsync(clipboardData); - return null; - - case PasteFormats.PasteAsPngFile: - await ToPngFileAsync(clipboardData); - return null; - - case PasteFormats.PasteAsHtmlFile: - await ToHtmlFileAsync(clipboardData); - return null; - - case PasteFormats.Custom: - return await ToCustomAsync(pasteFormat.Prompt, clipboardData); - - default: - throw new ArgumentException($"Unknown paste format {pasteFormat.Format}", nameof(pasteFormat)); - } + return await ExecutePasteFormatCoreAsync(pasteFormat.Format, pasteFormat.Prompt, Clipboard.GetContent()); } private static void WriteTelemetry(PasteFormats format, PasteActionSource source) @@ -94,160 +49,12 @@ private static void WriteTelemetry(PasteFormats format, PasteActionSource source } } - private void ToPlainText(DataPackageView clipboardData) - { - Logger.LogTrace(); - SetClipboardTextContent(MarkdownHelper.PasteAsPlainTextFromClipboard(clipboardData)); - } - - private void ToMarkdown(DataPackageView clipboardData) - { - Logger.LogTrace(); - SetClipboardTextContent(MarkdownHelper.ToMarkdown(clipboardData)); - } - - private void ToJson(DataPackageView clipboardData) - { - Logger.LogTrace(); - SetClipboardTextContent(JsonHelper.ToJsonFromXmlOrCsv(clipboardData)); - } - - private async Task ImageToTextAsync(DataPackageView clipboardData) - { - Logger.LogTrace(); - - var bitmap = await ClipboardHelper.GetClipboardImageContentAsync(clipboardData); - var text = await OcrHelpers.ExtractTextAsync(bitmap); - SetClipboardTextContent(text); - } - - private async Task ToPngFileAsync(DataPackageView clipboardData) - { - Logger.LogTrace(); - - var clipboardBitmap = await ClipboardHelper.GetClipboardImageContentAsync(clipboardData); - - using var pngStream = new InMemoryRandomAccessStream(); - var encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, pngStream); - encoder.SetSoftwareBitmap(clipboardBitmap); - await encoder.FlushAsync(); - - await SetClipboardFileContentAsync(pngStream.AsStreamForRead(), "png"); - } - - private async Task ToTxtFileAsync(DataPackageView clipboardData) - { - Logger.LogTrace(); - - var text = await ClipboardHelper.GetClipboardTextOrHtmlTextAsync(clipboardData); - await SetClipboardFileContentAsync(text, "txt"); - } - - private async Task ToHtmlFileAsync(DataPackageView clipboardData) - { - Logger.LogTrace(); - - var cfHtml = await ClipboardHelper.GetClipboardHtmlContentAsync(clipboardData); - var html = RemoveHtmlMetadata(cfHtml); - - await SetClipboardFileContentAsync(html, "html"); - } - - /// - /// Removes leading CF_HTML metadata from HTML clipboard data. - /// See: https://learn.microsoft.com/en-us/windows/win32/dataxchg/html-clipboard-format - /// - private static string RemoveHtmlMetadata(string cfHtml) - { - int? GetIntTagValue(string tagName) - { - var tagNameWithColon = tagName + ":"; - int tagStartPos = cfHtml.IndexOf(tagNameWithColon, StringComparison.InvariantCulture); - - const int tagValueLength = 10; - return tagStartPos != -1 && int.TryParse(cfHtml.AsSpan(tagStartPos + tagNameWithColon.Length, tagValueLength), CultureInfo.InvariantCulture, out int result) ? result : null; - } - - var startFragmentIndex = GetIntTagValue("StartFragment"); - var endFragmentIndex = GetIntTagValue("EndFragment"); - - return (startFragmentIndex == null || endFragmentIndex == null) ? cfHtml : cfHtml[startFragmentIndex.Value..endFragmentIndex.Value]; - } - - private static async Task SetClipboardFileContentAsync(string data, string fileExtension) - { - if (string.IsNullOrEmpty(data)) - { - throw new ArgumentException($"Empty value in {nameof(SetClipboardFileContentAsync)}", nameof(data)); - } - - var path = GetPasteAsFileTempFilePath(fileExtension); - - await File.WriteAllTextAsync(path, data); - await ClipboardHelper.SetClipboardFileContentAsync(path); - } - - private static async Task SetClipboardFileContentAsync(Stream stream, string fileExtension) - { - var path = GetPasteAsFileTempFilePath(fileExtension); - - using var fileStream = File.Create(path); - await stream.CopyToAsync(fileStream); - - await ClipboardHelper.SetClipboardFileContentAsync(path); - } - - private static string GetPasteAsFileTempFilePath(string fileExtension) - { - var prefix = ResourceLoaderInstance.ResourceLoader.GetString("PasteAsFile_FilePrefix"); - var timestamp = DateTime.Now.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture); - - return Path.Combine(Path.GetTempPath(), $"{prefix}{timestamp}.{fileExtension}"); - } - - private async Task ToCustomAsync(string prompt, DataPackageView clipboardData) - { - Logger.LogTrace(); - - if (string.IsNullOrWhiteSpace(prompt)) - { - return string.Empty; - } - - if (!clipboardData.Contains(StandardDataFormats.Text)) - { - Logger.LogWarning("Clipboard does not contain text data"); - return string.Empty; - } - - var currentClipboardText = await clipboardData.GetTextAsync(); - - if (string.IsNullOrWhiteSpace(currentClipboardText)) - { - Logger.LogWarning("Clipboard has no usable text data"); - return string.Empty; - } - - var aiResponse = await Task.Run(() => _aiHelper.AIFormatString(prompt, currentClipboardText)); - - return aiResponse.ApiRequestStatus == (int)HttpStatusCode.OK - ? aiResponse.Response - : throw new PasteActionException(TranslateErrorText(aiResponse.ApiRequestStatus)); - } - - private void SetClipboardTextContent(string content) + private async Task ExecutePasteFormatCoreAsync(PasteFormats format, string prompt, DataPackageView clipboardData) { - if (!string.IsNullOrEmpty(content)) + return format switch { - ClipboardHelper.SetClipboardTextContent(content); - } + PasteFormats.KernelQuery => await _kernelService.GetCompletionAsync(prompt, clipboardData), + _ => await TransformHelpers.TransformAsync(format, clipboardData), + }; } - - private static string TranslateErrorText(int apiRequestStatus) => (HttpStatusCode)apiRequestStatus switch - { - HttpStatusCode.TooManyRequests => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyTooManyRequests"), - HttpStatusCode.Unauthorized => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyUnauthorized"), - HttpStatusCode.OK => string.Empty, - _ => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyError") + apiRequestStatus.ToString(CultureInfo.InvariantCulture), - }; } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw b/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw index 0ec81547e20e..73a1cebb96bf 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw +++ b/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw @@ -123,9 +123,6 @@ Clipboard does not contain any usable formats - - Clipboard data is not text - To custom with AI is not enabled diff --git a/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs b/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs index f0726f5175c6..2ca2e294332d 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; @@ -33,7 +34,7 @@ public sealed partial class OptionsViewModel : ObservableObject, IDisposable private readonly DispatcherTimer _clipboardTimer; private readonly IUserSettings _userSettings; private readonly IPasteFormatExecutor _pasteFormatExecutor; - private readonly AICompletionsHelper _aiHelper; + private readonly IAICredentialsProvider _aiCredentialsProvider; public DataPackageView ClipboardData { get; set; } @@ -68,21 +69,19 @@ public sealed partial class OptionsViewModel : ObservableObject, IDisposable public ObservableCollection CustomActionPasteFormats { get; } = []; - public bool IsAIServiceEnabled => IsAllowedByGPO && _aiHelper.IsAIEnabled; + public bool IsAIServiceEnabled => IsAllowedByGPO && _aiCredentialsProvider.IsConfigured; - public bool IsCustomAIEnabled => IsAIServiceEnabled && ClipboardHasText; + public bool IsCustomAIEnabled => IsAIServiceEnabled && ClipboardHasData; public bool ClipboardHasData => AvailableClipboardFormats != ClipboardFormat.None; - private bool ClipboardHasText => AvailableClipboardFormats.HasFlag(ClipboardFormat.Text); - private bool Visible => GetMainWindow()?.Visible is true; public event EventHandler CustomActionActivated; - public OptionsViewModel(AICompletionsHelper aiHelper, IUserSettings userSettings, IPasteFormatExecutor pasteFormatExecutor) + public OptionsViewModel(IAICredentialsProvider aiCredentialsProvider, IUserSettings userSettings, IPasteFormatExecutor pasteFormatExecutor) { - _aiHelper = aiHelper; + _aiCredentialsProvider = aiCredentialsProvider; _userSettings = userSettings; _pasteFormatExecutor = pasteFormatExecutor; @@ -212,7 +211,7 @@ public async Task OnShowAsync() _dispatcherQueue.TryEnqueue(() => { - GetMainWindow()?.FinishLoading(_aiHelper.IsAIEnabled); + GetMainWindow()?.FinishLoading(_aiCredentialsProvider.IsConfigured); OnPropertyChanged(nameof(InputTxtBoxPlaceholderText)); OnPropertyChanged(nameof(AIDisabledErrorText)); OnPropertyChanged(nameof(IsAIServiceEnabled)); @@ -255,17 +254,12 @@ public string AIDisabledErrorText { get { - if (!ClipboardHasText) - { - return ResourceLoaderInstance.ResourceLoader.GetString("ClipboardDataNotTextWarning"); - } - if (!IsAllowedByGPO) { return ResourceLoaderInstance.ResourceLoader.GetString("OpenAIGpoDisabled"); } - if (!_aiHelper.IsAIEnabled) + if (!_aiCredentialsProvider.IsConfigured) { return ResourceLoaderInstance.ResourceLoader.GetString("OpenAINotConfigured"); } @@ -280,24 +274,22 @@ public string AIDisabledErrorText private string _customFormatResult; [RelayCommand] - public void PasteCustom() + public async Task PasteCustomAsync() { var text = GeneratedResponses.ElementAtOrDefault(CurrentResponseIndex); if (!string.IsNullOrEmpty(text)) { - ClipboardHelper.SetClipboardTextContent(text); - HideWindow(); - - if (_userSettings.SendPasteKeyCombination) - { - ClipboardHelper.SendPasteKeyCombination(); - } - - Query = string.Empty; + await CopyPasteAndHideAsync(ClipboardHelper.CreateDataPackageFromText(text)); } } + private async Task CopyPasteAndHideAsync(DataPackage package) + { + await ClipboardHelper.TryCopyPasteDataPackageAsync(package, HideWindow); + Query = string.Empty; + } + // Command to select the previous custom format [RelayCommand] public void PreviousCustomFormat() @@ -336,7 +328,7 @@ internal async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteAction { if (Busy) { - Logger.LogWarning($"Execution of {pasteFormat.Name} from {source} suppressed as busy"); + Logger.LogWarning($"Execution of {pasteFormat.Format} from {source} suppressed as busy"); return; } @@ -347,11 +339,14 @@ internal async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteAction return; } + var elapsedWatch = Stopwatch.StartNew(); + Logger.LogDebug($"Started executing {pasteFormat.Format} from source {source}"); + Busy = true; PasteOperationErrorText = string.Empty; Query = pasteFormat.Query; - if (pasteFormat.Format == PasteFormats.Custom) + if (pasteFormat.Format == PasteFormats.KernelQuery) { SaveQuery(Query); } @@ -361,30 +356,32 @@ internal async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteAction // Minimum time to show busy spinner for AI actions when triggered by global keyboard shortcut. var aiActionMinTaskTime = TimeSpan.FromSeconds(2); var delayTask = (Visible && source == PasteActionSource.GlobalKeyboardShortcut) ? Task.Delay(aiActionMinTaskTime) : Task.CompletedTask; - var aiOutput = await _pasteFormatExecutor.ExecutePasteFormatAsync(pasteFormat, source); + var dataPackage = await _pasteFormatExecutor.ExecutePasteFormatAsync(pasteFormat, source); await delayTask; - if (pasteFormat.Format != PasteFormats.Custom) + if (pasteFormat.Format != PasteFormats.KernelQuery) { - HideWindow(); - - if (source == PasteActionSource.GlobalKeyboardShortcut || _userSettings.SendPasteKeyCombination) - { - ClipboardHelper.SendPasteKeyCombination(); - } + await CopyPasteAndHideAsync(dataPackage); } else { - var pasteResult = source == PasteActionSource.GlobalKeyboardShortcut || !_userSettings.ShowCustomPreview; + var dataPackageView = dataPackage.GetView(); + var text = (await ClipboardHelper.GetAvailableClipboardFormatsAsync(dataPackageView)).HasFlag(ClipboardFormat.Text) ? await dataPackageView.GetTextAsync() : string.Empty; + bool hasText = !string.IsNullOrEmpty(text); - GeneratedResponses.Add(aiOutput); - CurrentResponseIndex = GeneratedResponses.Count - 1; + if (hasText) + { + GeneratedResponses.Add(text); + CurrentResponseIndex = GeneratedResponses.Count - 1; + } + + bool pasteResult = source == PasteActionSource.GlobalKeyboardShortcut || !_userSettings.ShowCustomPreview || !hasText; CustomActionActivated?.Invoke(this, new CustomActionActivatedEventArgs(pasteFormat.Prompt, pasteResult)); if (pasteResult) { - PasteCustom(); + await CopyPasteAndHideAsync(dataPackage); } } } @@ -395,6 +392,8 @@ internal async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteAction } Busy = false; + elapsedWatch.Stop(); + Logger.LogDebug($"Finished executing {pasteFormat.Format} from source {source}; timeTakenMs={elapsedWatch.ElapsedMilliseconds}"); } internal async Task ExecutePasteFormatAsync(VirtualKey key) @@ -453,7 +452,7 @@ internal void SaveQuery(string inputQuery) if (clipboardData == null || !clipboardData.Contains(StandardDataFormats.Text)) { - Logger.LogWarning("Clipboard does not contain text data"); + Logger.LogDebug("Clipboard does not contain text data; not saving query"); return; } @@ -499,15 +498,7 @@ private bool UpdateOpenAIKey() { UpdateAllowedByGPO(); - if (IsAllowedByGPO) - { - var oldKey = _aiHelper.GetKey(); - var newKey = AICompletionsHelper.LoadOpenAIKey(); - _aiHelper.SetOpenAIKey(newKey); - return newKey != oldKey; - } - - return false; + return IsAllowedByGPO && _aiCredentialsProvider.Refresh(); } } } diff --git a/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs b/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs index 8322302ddfce..bb0744078e36 100644 --- a/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs +++ b/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs @@ -24,17 +24,12 @@ public AdvancedPasteProperties() CustomActions = new(); AdditionalActions = new(); ShowCustomPreview = true; - SendPasteKeyCombination = true; CloseAfterLosingFocus = false; } [JsonConverter(typeof(BoolPropertyJsonConverter))] public bool ShowCustomPreview { get; set; } - [JsonConverter(typeof(BoolPropertyJsonConverter))] - [CmdConfigureIgnore] - public bool SendPasteKeyCombination { get; set; } - [JsonConverter(typeof(BoolPropertyJsonConverter))] public bool CloseAfterLosingFocus { get; set; }