Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

adds images in history, create file for pasted images in chat #241664

Merged
merged 7 commits into from
Feb 24, 2025
+150 −29
Merged

adds images in history, create file for pasted images in chat #241664

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a693738
references for pasted images and adding images back to history
justschen Feb 24, 2025
d637ea9
cleanup
justschen Feb 24, 2025
354278d
make sure to resize after re-reading the file
justschen Feb 24, 2025
d437990
change name to specify vscode chat images
justschen Feb 24, 2025
5043ffc
adds to workspaceStorage
justschen Feb 24, 2025
48cfa2e
delete temp storage after a whilegit add .
justschen Feb 24, 2025
a9d41cb
some cleanup
justschen Feb 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 48 additions & 10 deletions src/vs/workbench/contrib/chat/browser/chatInputPart.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -535,12 +535,13 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
return this.container;
}

showPreviousValue(): void {
async showPreviousValue(): Promise<void> {
const inputState = this.getInputState();
if (this.history.isAtEnd()) {
this.saveCurrentValue(inputState);
} else {
if (!this.history.has({ text: this._inputEditor.getValue(), state: inputState })) {
const currentEntry = this.getFilteredEntry(this._inputEditor.getValue(), inputState);
if (!this.history.has(currentEntry)) {
this.saveCurrentValue(inputState);
this.history.resetCursor();
}
Expand All @@ -549,12 +550,13 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
this.navigateHistory(true);
}

showNextValue(): void {
async showNextValue(): Promise<void> {
const inputState = this.getInputState();
if (this.history.isAtEnd()) {
return;
} else {
if (!this.history.has({ text: this._inputEditor.getValue(), state: inputState })) {
const currentEntry = this.getFilteredEntry(this._inputEditor.getValue(), inputState);
if (!this.history.has(currentEntry)) {
this.saveCurrentValue(inputState);
this.history.resetCursor();
}
Expand All @@ -563,11 +565,29 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
this.navigateHistory(false);
}

private navigateHistory(previous: boolean): void {
private async navigateHistory(previous: boolean): Promise<void> {
const historyEntry = previous ?
this.history.previous() : this.history.next();

const historyAttachments = historyEntry.state?.chatContextAttachments ?? [];
let historyAttachments = historyEntry.state?.chatContextAttachments ?? [];

// Check for images in history to restore the value.
if (historyAttachments.length > 0) {
historyAttachments = await Promise.all(historyAttachments.map(async (attachment) => {
if (attachment.isImage && attachment.references?.length && URI.isUri(attachment.references[0].reference)) {
try {
const buffer = await this.fileService.readFile(attachment.references[0].reference);
const newAttachment = { ...attachment };
newAttachment.value = buffer.value.buffer;
return newAttachment;
} catch (error) {
this.logService.error('Failed to restore image from history', error);
}
}
return attachment;
}));
}

this._attachmentModel.clearAndSetContext(...historyAttachments);

aria.status(historyEntry.text);
Expand Down Expand Up @@ -607,8 +627,7 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
}

private saveCurrentValue(inputState: IChatInputState): void {
inputState.chatContextAttachments = inputState.chatContextAttachments?.filter(attachment => !attachment.isImage);
const newEntry = { text: this._inputEditor.getValue(), state: inputState };
const newEntry = this.getFilteredEntry(this._inputEditor.getValue(), inputState);
this.history.replaceLast(newEntry);
}

Expand All @@ -628,8 +647,7 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
if (isUserQuery) {
const userQuery = this._inputEditor.getValue();
const inputState = this.getInputState();
inputState.chatContextAttachments = inputState.chatContextAttachments?.filter(attachment => !attachment.isImage);
const entry: IChatHistoryEntry = { text: userQuery, state: inputState };
const entry = this.getFilteredEntry(userQuery, inputState);
this.history.replaceLast(entry);
this.history.add({ text: '' });
}
Expand All @@ -645,6 +663,26 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
}
}

// A funtion that filters out specifically the `value` property of the attachment.
private getFilteredEntry(query: string, inputState: IChatInputState): IChatHistoryEntry {
const attachmentsWithoutImageValues = inputState.chatContextAttachments?.map(attachment => {
if (attachment.isImage && attachment.references?.length && attachment.value) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there some case where an image attachment doesn't have references, and then the image data will still be saved in history?

Copy link
Contributor Author

@justschen justschen Feb 24, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there shouldn't be case where images don't have references (besides if it comes from web and is an image, then it is okay because the value will be a URL)

const newAttachment = { ...attachment };
newAttachment.value = undefined;
return newAttachment;
}
return attachment;
});

inputState.chatContextAttachments = attachmentsWithoutImageValues;
const newEntry = {
text: query,
state: inputState,
};

return newEntry;
}

private _acceptInputForVoiceover(): void {
const domNode = this._inputEditor.getDomNode();
if (!domNode) {
Expand Down
66 changes: 52 additions & 14 deletions src/vs/workbench/contrib/chat/browser/chatPasteProviders.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,27 @@
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { VSBuffer } from '../../../../base/common/buffer.js';
import { CancellationToken } from '../../../../base/common/cancellation.js';
import { Codicon } from '../../../../base/common/codicons.js';
import { createStringDataTransferItem, IDataTransferItem, IReadonlyVSDataTransfer, VSDataTransfer } from '../../../../base/common/dataTransfer.js';
import { HierarchicalKind } from '../../../../base/common/hierarchicalKind.js';
import { Disposable } from '../../../../base/common/lifecycle.js';
import { Mimes } from '../../../../base/common/mime.js';
import { basename, joinPath } from '../../../../base/common/resources.js';
import { URI, UriComponents } from '../../../../base/common/uri.js';
import { IRange } from '../../../../editor/common/core/range.js';
import { DocumentPasteContext, DocumentPasteEdit, DocumentPasteEditProvider, DocumentPasteEditsSession } from '../../../../editor/common/languages.js';
import { ITextModel } from '../../../../editor/common/model.js';
import { ILanguageFeaturesService } from '../../../../editor/common/services/languageFeatures.js';
import { Disposable } from '../../../../base/common/lifecycle.js';
import { ChatInputPart } from './chatInputPart.js';
import { IChatWidgetService } from './chat.js';
import { Codicon } from '../../../../base/common/codicons.js';
import { IModelService } from '../../../../editor/common/services/model.js';
import { localize } from '../../../../nls.js';
import { IChatRequestPasteVariableEntry, IChatRequestVariableEntry } from '../common/chatModel.js';
import { IFileService } from '../../../../platform/files/common/files.js';
import { IWorkspaceContextService } from '../../../../platform/workspace/common/workspace.js';
import { IExtensionService, isProposedApiEnabled } from '../../../services/extensions/common/extensions.js';
import { Mimes } from '../../../../base/common/mime.js';
import { IModelService } from '../../../../editor/common/services/model.js';
import { URI, UriComponents } from '../../../../base/common/uri.js';
import { basename } from '../../../../base/common/resources.js';
import { IChatRequestPasteVariableEntry, IChatRequestVariableEntry } from '../common/chatModel.js';
import { IChatWidgetService } from './chat.js';
import { ChatInputPart } from './chatInputPart.js';
import { resizeImage } from './imageUtils.js';

const COPY_MIME_TYPES = 'application/vnd.code.additional-editor-data';
Expand All @@ -41,6 +43,8 @@ export class PasteImageProvider implements DocumentPasteEditProvider {
constructor(
private readonly chatWidgetService: IChatWidgetService,
private readonly extensionService: IExtensionService,
@IFileService private readonly fileService: IFileService,
@IWorkspaceContextService private readonly workspaceService: IWorkspaceContextService,
) { }

async provideDocumentPasteEdits(model: ITextModel, ranges: readonly IRange[], dataTransfer: IReadonlyVSDataTransfer, context: DocumentPasteContext, token: CancellationToken): Promise<DocumentPasteEditsSession | undefined> {
Expand Down Expand Up @@ -90,12 +94,17 @@ export class PasteImageProvider implements DocumentPasteEditProvider {
tempDisplayName = `${displayName} ${appendValue}`;
}

const fileReference = await this.createFileForMedia(currClipboard, mimeType);
if (token.isCancellationRequested || !fileReference) {
return;
}

const scaledImageData = await resizeImage(currClipboard);
if (token.isCancellationRequested || !scaledImageData) {
return;
}

const scaledImageContext = await getImageAttachContext(scaledImageData, mimeType, token, tempDisplayName);
const scaledImageContext = await getImageAttachContext(scaledImageData, mimeType, token, tempDisplayName, fileReference);
if (token.isCancellationRequested || !scaledImageContext) {
return;
}
Expand All @@ -111,9 +120,35 @@ export class PasteImageProvider implements DocumentPasteEditProvider {
const edit = createCustomPasteEdit(model, scaledImageContext, mimeType, this.kind, localize('pastedImageAttachment', 'Pasted Image Attachment'), this.chatWidgetService);
return createEditSession(edit);
}

private async createFileForMedia(
dataTransfer: Uint8Array,
mimeType: string,
): Promise<URI | undefined> {
const workspaceFolder = this.workspaceService.getWorkspace().folders[0];
if (!workspaceFolder) {
return;
}

const imagesFolder = joinPath(workspaceFolder.uri, '.chat-images');
const exists = await this.fileService.exists(imagesFolder);
if (!exists) {
await this.fileService.createFolder(imagesFolder);
}

const ext = mimeType.split('/')[1] || 'png';
const filename = `image-${Date.now()}.${ext}`;
const fileUri = joinPath(imagesFolder, filename);


const buffer = VSBuffer.wrap(dataTransfer);
await this.fileService.writeFile(fileUri, buffer);

return fileUri;
}
}

async function getImageAttachContext(data: Uint8Array, mimeType: string, token: CancellationToken, displayName: string): Promise<IChatRequestVariableEntry | undefined> {
async function getImageAttachContext(data: Uint8Array, mimeType: string, token: CancellationToken, displayName: string, resource: URI): Promise<IChatRequestVariableEntry | undefined> {
const imageHash = await imageToHash(data);
if (token.isCancellationRequested) {
return undefined;
Expand All @@ -125,7 +160,8 @@ async function getImageAttachContext(data: Uint8Array, mimeType: string, token:
name: displayName,
isImage: true,
icon: Codicon.fileMedia,
mimeType
mimeType,
references: [{ reference: resource, kind: 'reference' }]
};
}

Expand Down Expand Up @@ -308,10 +344,12 @@ export class ChatPasteProvidersFeature extends Disposable {
@ILanguageFeaturesService languageFeaturesService: ILanguageFeaturesService,
@IChatWidgetService chatWidgetService: IChatWidgetService,
@IExtensionService extensionService: IExtensionService,
@IWorkspaceContextService workspaceService: IWorkspaceContextService,
@IFileService fileService: IFileService,
@IModelService modelService: IModelService
) {
super();
this._register(languageFeaturesService.documentPasteEditProvider.register({ scheme: ChatInputPart.INPUT_SCHEME, pattern: '*', hasAccessToAllModels: true }, new PasteImageProvider(chatWidgetService, extensionService)));
this._register(languageFeaturesService.documentPasteEditProvider.register({ scheme: ChatInputPart.INPUT_SCHEME, pattern: '*', hasAccessToAllModels: true }, new PasteImageProvider(chatWidgetService, extensionService, fileService, workspaceService)));
this._register(languageFeaturesService.documentPasteEditProvider.register({ scheme: ChatInputPart.INPUT_SCHEME, pattern: '*', hasAccessToAllModels: true }, new PasteTextProvider(chatWidgetService, modelService)));
this._register(languageFeaturesService.documentPasteEditProvider.register('*', new CopyTextProvider()));
}
Expand Down
Loading