diff --git a/package.json b/package.json index b9c761a821..8ce9eb1a9e 100644 --- a/package.json +++ b/package.json @@ -3914,6 +3914,16 @@ "onExp" ] }, + "github.copilot.nextEditSuggestions.preferredModel": { + "type": "string", + "default": "none", + "markdownDescription": "%github.copilot.config.nextEditSuggestions.preferredModel%", + "tags": [ + "advanced", + "experimental", + "onExp" + ] + }, "github.copilot.chat.suggestRelatedFilesFromGitHistory.useEmbeddings": { "type": "boolean", "default": false, diff --git a/package.nls.json b/package.nls.json index fef8de49f3..db143ec896 100644 --- a/package.nls.json +++ b/package.nls.json @@ -411,6 +411,7 @@ "github.copilot.config.inlineEdits.nextCursorPrediction.displayLine": "Display predicted cursor line for next edit suggestions.", "github.copilot.config.inlineEdits.nextCursorPrediction.currentFileMaxTokens": "Maximum tokens for current file in next cursor prediction.", "github.copilot.config.inlineEdits.renameSymbolSuggestions": "Enable rename symbol suggestions in inline edits.", + "github.copilot.config.nextEditSuggestions.preferredModel": "Preferred model for next edit suggestions.", "github.copilot.command.refreshAgentSessions": "Refresh Agent Sessions", "github.copilot.command.deleteAgentSession": "Delete Agent Session", "github.copilot.command.cli.sessions.resumeInTerminal": "Resume Agent Session in Terminal", diff --git a/src/extension/extension/vscode-node/services.ts b/src/extension/extension/vscode-node/services.ts index 7a28a57851..4f97775f61 100644 --- a/src/extension/extension/vscode-node/services.ts +++ b/src/extension/extension/vscode-node/services.ts @@ -34,6 +34,8 @@ import { IIgnoreService, NullIgnoreService } from '../../../platform/ignore/comm import { VsCodeIgnoreService } from '../../../platform/ignore/vscode-node/ignoreService'; import { IImageService } from '../../../platform/image/common/imageService'; import { ImageServiceImpl } from '../../../platform/image/node/imageServiceImpl'; +import { IInlineEditsModelService } from '../../../platform/inlineEdits/common/inlineEditsModelService'; +import { InlineEditsModelService } from '../../../platform/inlineEdits/node/inlineEditsModelService'; import { ILanguageContextProviderService } from '../../../platform/languageContextProvider/common/languageContextProviderService'; import { ILanguageContextService } from '../../../platform/languageServer/common/languageContextService'; import { ICompletionsFetchService } from '../../../platform/nesFetch/common/completionsFetchService'; @@ -204,6 +206,7 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio builder.define(ITodoListContextProvider, new SyncDescriptor(TodoListContextProvider)); builder.define(IGithubAvailableEmbeddingTypesService, new SyncDescriptor(GithubAvailableEmbeddingTypesService)); builder.define(IRerankerService, new SyncDescriptor(RerankerService)); + builder.define(IInlineEditsModelService, new SyncDescriptor(InlineEditsModelService)); } function setupMSFTExperimentationService(builder: IInstantiationServiceBuilder, extensionContext: ExtensionContext) { diff --git a/src/extension/inlineEdits/vscode-node/inlineCompletionProvider.ts b/src/extension/inlineEdits/vscode-node/inlineCompletionProvider.ts index 53d1b0c0be..37c0ad7c5c 100644 --- a/src/extension/inlineEdits/vscode-node/inlineCompletionProvider.ts +++ b/src/extension/inlineEdits/vscode-node/inlineCompletionProvider.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as l10n from '@vscode/l10n'; -import { CancellationToken, Command, EndOfLine, InlineCompletionContext, InlineCompletionDisplayLocation, InlineCompletionDisplayLocationKind, InlineCompletionEndOfLifeReason, InlineCompletionEndOfLifeReasonKind, InlineCompletionItem, InlineCompletionItemProvider, InlineCompletionList, InlineCompletionsDisposeReason, InlineCompletionsDisposeReasonKind, NotebookCell, NotebookCellKind, Position, Range, TextDocument, TextDocumentShowOptions, Event as vscodeEvent, window, workspace } from 'vscode'; +import { CancellationToken, Command, EndOfLine, InlineCompletionContext, InlineCompletionDisplayLocation, InlineCompletionDisplayLocationKind, InlineCompletionEndOfLifeReason, InlineCompletionEndOfLifeReasonKind, InlineCompletionItem, InlineCompletionItemProvider, InlineCompletionList, InlineCompletionModelInfo, InlineCompletionsDisposeReason, InlineCompletionsDisposeReasonKind, NotebookCell, NotebookCellKind, Position, Range, TextDocument, TextDocumentShowOptions, Event as vscodeEvent, window, workspace } from 'vscode'; import { ConfigKey, IConfigurationService } from '../../../platform/configuration/common/configurationService'; import { IDiffService } from '../../../platform/diff/common/diffService'; import { stringEditFromDiff } from '../../../platform/editing/common/edit'; @@ -13,6 +13,7 @@ import { EditSurvivalReporter } from '../../../platform/editSurvivalTracking/com import { IGitExtensionService } from '../../../platform/git/common/gitExtensionService'; import { DocumentId } from '../../../platform/inlineEdits/common/dataTypes/documentId'; import { InlineEditRequestLogContext } from '../../../platform/inlineEdits/common/inlineEditLogContext'; +import { IInlineEditsModelService } from '../../../platform/inlineEdits/common/inlineEditsModelService'; import { ShowNextEditPreference } from '../../../platform/inlineEdits/common/statelessNextEditProvider'; import { shortenOpportunityId } from '../../../platform/inlineEdits/common/utils/utils'; import { ILogService } from '../../../platform/log/common/logService'; @@ -24,11 +25,12 @@ import { IExperimentationService } from '../../../platform/telemetry/common/null import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; import { IWorkspaceService } from '../../../platform/workspace/common/workspaceService'; import { findCell, findNotebook, isNotebookCell } from '../../../util/common/notebooks'; -import { ITracer, createTracer } from '../../../util/common/tracing'; +import { createTracer, ITracer } from '../../../util/common/tracing'; import { raceCancellation, timeout } from '../../../util/vs/base/common/async'; import { CancellationTokenSource } from '../../../util/vs/base/common/cancellation'; -import { Event } from '../../../util/vs/base/common/event'; -import { IObservable } from '../../../util/vs/base/common/observable'; +import { Emitter, Event } from '../../../util/vs/base/common/event'; +import { Disposable } from '../../../util/vs/base/common/lifecycle'; +import { autorun, IObservable, observableFromEvent } from '../../../util/vs/base/common/observable'; import { basename } from '../../../util/vs/base/common/path'; import { StringEdit } from '../../../util/vs/editor/common/core/edits/stringEdit'; import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation'; @@ -101,7 +103,7 @@ function isLlmCompletionInfo(item: NesCompletionInfo): item is LlmCompletionInfo const GoToNextEdit = l10n.t('Go To Inline Suggestion'); -export class InlineCompletionProviderImpl implements InlineCompletionItemProvider { +export class InlineCompletionProviderImpl extends Disposable implements InlineCompletionItemProvider { public readonly displayName = 'Inline Suggestion'; private readonly _tracer: ITracer; @@ -110,6 +112,17 @@ export class InlineCompletionProviderImpl implements InlineCompletionItemProvide public readonly handleDidPartiallyAcceptCompletionItem = undefined; public readonly handleDidRejectCompletionItem = undefined; + //#region Model picker + private _isModelPickerEnabled: IObservable = this._configurationService.getExperimentBasedConfigObservable(ConfigKey.TeamInternal.InlineEditsModelPickerEnabled, this._expService); + + public modelInfo: InlineCompletionModelInfo | undefined; + + private readonly _onDidChangeModelInfo = this._register(new Emitter()); + public onDidChangeModelInfo = this._onDidChangeModelInfo.event; + + public setCurrentModelId: ((modelId: string) => Thenable) | undefined; + //#endregion + private readonly _displayNextEditorNES: boolean; private readonly _renameSymbolSuggestions: IObservable; @@ -129,10 +142,22 @@ export class InlineCompletionProviderImpl implements InlineCompletionItemProvide @INotebookService private readonly _notebookService: INotebookService, @IWorkspaceService private readonly _workspaceService: IWorkspaceService, @IRequestLogger private readonly _requestLogger: IRequestLogger, + @IInlineEditsModelService private readonly _modelService: IInlineEditsModelService, ) { + super(); this._tracer = createTracer(['NES', 'Provider'], (s) => this._logService.trace(s)); this._displayNextEditorNES = this._configurationService.getExperimentBasedConfig(ConfigKey.Advanced.UseAlternativeNESNotebookFormat, this._expService); this._renameSymbolSuggestions = this._configurationService.getExperimentBasedConfigObservable(ConfigKey.Advanced.InlineEditsRenameSymbolSuggestions, this._expService); + + this.setCurrentModelId = (modelId: string) => this._modelService.setCurrentModelId(modelId); + + const modelListUpdatedObs = observableFromEvent(this, this._modelService.onModelListUpdated, () => this._modelService.modelInfo); + + this._register(autorun(reader => { + this.modelInfo = this._isModelPickerEnabled.read(reader) ? modelListUpdatedObs.read(reader) : undefined; + this._onDidChangeModelInfo.fire(); + })); + } // copied from `vscodeWorkspace.ts` `DocumentFilter#_enabledLanguages` diff --git a/src/extension/test/node/services.ts b/src/extension/test/node/services.ts index 0cb712be28..f08e31c7ed 100644 --- a/src/extension/test/node/services.ts +++ b/src/extension/test/node/services.ts @@ -19,6 +19,8 @@ import { IGitExtensionService } from '../../../platform/git/common/gitExtensionS import { IGitService } from '../../../platform/git/common/gitService'; import { NullGitDiffService } from '../../../platform/git/common/nullGitDiffService'; import { NullGitExtensionService } from '../../../platform/git/common/nullGitExtensionService'; +import { IInlineEditsModelService } from '../../../platform/inlineEdits/common/inlineEditsModelService'; +import { InlineEditsModelService } from '../../../platform/inlineEdits/node/inlineEditsModelService'; import { ILogService } from '../../../platform/log/common/logService'; import { EditLogService, IEditLogService } from '../../../platform/multiFileEdit/common/editLogService'; import { IMultiFileEditInternalTelemetryService, MultiFileEditInternalTelemetryService } from '../../../platform/multiFileEdit/common/multiFileEditQualityTelemetry'; @@ -94,6 +96,7 @@ export function createExtensionUnitTestingServices(disposables: Pick): ModelConfig | undefined { - const configString = this.configService.getExperimentBasedConfig(configKey, this.expService); - if (configString === undefined) { - return undefined; - } - - let parsedConfig: xtabPromptOptions.ModelConfiguration | undefined; - try { - parsedConfig = JSON.parse(configString); - } catch (e: unknown) { - /* __GDPR__ - "incorrectNesModelConfig" : { - "owner": "ulugbekna", - "comment": "Capture if model configuration string is invalid JSON.", - "configName": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Name of the configuration that failed to parse." }, - "errorMessage": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error message from JSON.parse." }, - "configValue": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The invalid JSON string." } - } - */ - this.telemetryService.sendMSFTTelemetryEvent('incorrectNesModelConfig', { configName: configKey.id, errorMessage: errors.toString(errors.fromUnknown(e)), configValue: configString }); - } - - if (parsedConfig) { - return XtabProvider.overrideModelConfig(originalModelConfig, parsedConfig); - } - - return undefined; + const modelConfig = this.modelService.selectedModelConfiguration(); + return XtabProvider.overrideModelConfig(sourcedModelConfig, modelConfig); } private static overrideModelConfig(modelConfig: ModelConfig, overridingConfig: xtabPromptOptions.ModelConfiguration): ModelConfig { diff --git a/src/platform/configuration/common/configurationService.ts b/src/platform/configuration/common/configurationService.ts index ffa9b0dd58..e27f40d5d7 100644 --- a/src/platform/configuration/common/configurationService.ts +++ b/src/platform/configuration/common/configurationService.ts @@ -681,6 +681,7 @@ export namespace ConfigKey { export const InlineEditsNextCursorPredictionDisplayLine = defineAndMigrateExpSetting('chat.advanced.inlineEdits.nextCursorPrediction.displayLine', 'chat.inlineEdits.nextCursorPrediction.displayLine', true); export const InlineEditsNextCursorPredictionCurrentFileMaxTokens = defineAndMigrateExpSetting('chat.advanced.inlineEdits.nextCursorPrediction.currentFileMaxTokens', 'chat.inlineEdits.nextCursorPrediction.currentFileMaxTokens', xtabPromptOptions.DEFAULT_OPTIONS.currentFile.maxTokens); export const InlineEditsRenameSymbolSuggestions = defineSetting('chat.inlineEdits.renameSymbolSuggestions', ConfigType.ExperimentBased, { defaultValue: false, teamDefaultValue: true }); + export const InlineEditsPreferredModel = defineSetting('nextEditSuggestions.preferredModel', ConfigType.ExperimentBased, "none"); export const DiagnosticsContextProvider = defineAndMigrateExpSetting('chat.advanced.inlineEdits.diagnosticsContextProvider.enabled', 'chat.inlineEdits.diagnosticsContextProvider.enabled', false); export const Gemini3ReplaceStringOnly = defineSetting('chat.edits.gemini3ReplaceStringOnly', ConfigType.ExperimentBased, false); export const Gemini3MultiReplaceString = defineSetting('chat.edits.gemini3MultiReplaceString', ConfigType.ExperimentBased, false); @@ -702,6 +703,8 @@ export namespace ConfigKey { */ export const DebugReportFeedback = defineTeamInternalSetting('chat.advanced.debug.reportFeedback', ConfigType.Simple, { defaultValue: false, teamDefaultValue: true }); export const InlineEditsIgnoreCompletionsDisablement = defineTeamInternalSetting('chat.advanced.inlineEdits.ignoreCompletionsDisablement', ConfigType.Simple, false, vBoolean()); + export const InlineEditsModelPickerEnabled = defineTeamInternalSetting('chat.advanced.inlineEdits.modelPicker.enabled', ConfigType.ExperimentBased, false, vBoolean()); + export const InlineEditsUseSlashModels = defineTeamInternalSetting('chat.advanced.inlineEdits.useSlashModels', ConfigType.ExperimentBased, false); export const InlineEditsLogContextRecorderEnabled = defineTeamInternalSetting('chat.advanced.inlineEdits.logContextRecorder.enabled', ConfigType.Simple, false); export const InlineEditsHideInternalInterface = defineTeamInternalSetting('chat.advanced.inlineEdits.hideInternalInterface', ConfigType.Simple, false, vBoolean()); export const InlineEditsLogCancelledRequests = defineTeamInternalSetting('chat.advanced.inlineEdits.logCancelledRequests', ConfigType.Simple, false, vBoolean()); diff --git a/src/platform/inlineEdits/common/dataTypes/inlineEditsModelsTypes.ts b/src/platform/inlineEdits/common/dataTypes/inlineEditsModelsTypes.ts new file mode 100644 index 0000000000..2541027446 --- /dev/null +++ b/src/platform/inlineEdits/common/dataTypes/inlineEditsModelsTypes.ts @@ -0,0 +1,54 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IValidator, vArray, vObj, vString } from '../../../configuration/common/validator'; + +export namespace WireTypes { + + export namespace Capabilities { + export type t = { + promptStrategy: string; + }; + export function is(obj: unknown): obj is t { + return !!obj && typeof obj === 'object' && + typeof (obj as t).promptStrategy === 'string'; + } + export const validator: IValidator = vObj({ + promptStrategy: vString(), + }); + } + + export namespace Model { + export type t = { + name: string; + api_provider: string; + capabilities: Capabilities.t; + }; + export const validator: IValidator = vObj({ + name: vString(), + api_provider: vString(), + capabilities: Capabilities.validator, + }); + export function is(obj: unknown): obj is t { + return !!obj && typeof obj === 'object' && + typeof (obj as t).name === 'string' && + typeof (obj as t).api_provider === 'string' && + Capabilities.is((obj as t).capabilities); + } + } + + export namespace ModelList { + export type t = { + models: Model.t[]; + }; + export const validator: IValidator = vObj({ + models: vArray(Model.validator), + }); + export function is(obj: unknown): obj is t { + return !!obj && typeof obj === 'object' && Array.isArray((obj as t).models) && (obj as t).models.every(Model.is); + } + } +} + diff --git a/src/platform/inlineEdits/common/dataTypes/xtabPromptOptions.ts b/src/platform/inlineEdits/common/dataTypes/xtabPromptOptions.ts index f4190a5b57..1cffa829c5 100644 --- a/src/platform/inlineEdits/common/dataTypes/xtabPromptOptions.ts +++ b/src/platform/inlineEdits/common/dataTypes/xtabPromptOptions.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { assertNever } from '../../../../util/vs/base/common/assert'; -import { vBoolean, vEnum, vObj, vRequired, vString, vUndefined, vUnion } from '../../../configuration/common/validator'; +import { IValidator, vBoolean, vEnum, vObj, vRequired, vString, vUndefined, vUnion } from '../../../configuration/common/validator'; export type RecentlyViewedDocumentsOptions = { readonly nDocuments: number; @@ -58,6 +58,10 @@ export enum PromptingStrategy { Xtab275 = 'xtab275', } +export function isPromptingStrategy(value: string): value is PromptingStrategy { + return (Object.values(PromptingStrategy) as string[]).includes(value); +} + export enum ResponseFormat { CodeBlock = 'codeBlock', UnifiedWithXml = 'unifiedWithXml', @@ -123,7 +127,7 @@ export interface ModelConfiguration { includeTagsInCurrentFile: boolean; } -export const MODEL_CONFIGURATION_VALIDATOR = vObj({ +export const MODEL_CONFIGURATION_VALIDATOR: IValidator = vObj({ 'modelName': vRequired(vString()), 'promptingStrategy': vUnion(vEnum(...Object.values(PromptingStrategy)), vUndefined()), 'includeTagsInCurrentFile': vRequired(vBoolean()), diff --git a/src/platform/inlineEdits/common/inlineEditsModelService.ts b/src/platform/inlineEdits/common/inlineEditsModelService.ts new file mode 100644 index 0000000000..9a9bd8547a --- /dev/null +++ b/src/platform/inlineEdits/common/inlineEditsModelService.ts @@ -0,0 +1,23 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type * as vscode from 'vscode'; +import { createServiceIdentifier } from '../../../util/common/services'; +import { Event } from '../../../util/vs/base/common/event'; +import { ModelConfiguration } from './dataTypes/xtabPromptOptions'; + +export interface IInlineEditsModelService { + readonly _serviceBrand: undefined; + + readonly modelInfo: vscode.InlineCompletionModelInfo | undefined; + + readonly onModelListUpdated: Event; + + setCurrentModelId(modelId: string): Promise; + + selectedModelConfiguration(): ModelConfiguration; +} + +export const IInlineEditsModelService = createServiceIdentifier('IInlineEditsModelService'); diff --git a/src/platform/inlineEdits/node/inlineEditsModelService.ts b/src/platform/inlineEdits/node/inlineEditsModelService.ts new file mode 100644 index 0000000000..dfc0d2c565 --- /dev/null +++ b/src/platform/inlineEdits/node/inlineEditsModelService.ts @@ -0,0 +1,366 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { isDeepStrictEqual } from 'util'; +import type * as vscode from 'vscode'; +import { filterMap } from '../../../util/common/arrays'; +import * as errors from '../../../util/common/errors'; +import { createTracer } from '../../../util/common/tracing'; +import { pushMany } from '../../../util/vs/base/common/arrays'; +import { softAssert } from '../../../util/vs/base/common/assert'; +import { Emitter } from '../../../util/vs/base/common/event'; +import { Disposable } from '../../../util/vs/base/common/lifecycle'; +import { autorun, observableFromEvent } from '../../../util/vs/base/common/observable'; +import { CopilotToken } from '../../authentication/common/copilotToken'; +import { ICopilotTokenStore } from '../../authentication/common/copilotTokenStore'; +import { ConfigKey, ExperimentBasedConfig, IConfigurationService } from '../../configuration/common/configurationService'; +import { ICAPIClientService } from '../../endpoint/common/capiClient'; +import { ILogService } from '../../log/common/logService'; +import { IFetcherService, Response } from '../../networking/common/fetcherService'; +import { IExperimentationService } from '../../telemetry/common/nullExperimentationService'; +import { ITelemetryService } from '../../telemetry/common/telemetry'; +import { WireTypes } from '../common/dataTypes/inlineEditsModelsTypes'; +import { isPromptingStrategy, ModelConfiguration, PromptingStrategy } from '../common/dataTypes/xtabPromptOptions'; +import { IInlineEditsModelService } from '../common/inlineEditsModelService'; + +type Model = { + modelName: string; + promptingStrategy: PromptingStrategy | undefined; + includeTagsInCurrentFile: boolean; +} + +const IN_TESTING = false; +const STUB_MODELS_FOR_TESTING: WireTypes.ModelList.t = { + models: [ + { + name: 'model-1', + api_provider: 'openai', + capabilities: { + promptStrategy: 'codexv21nesUnified', + }, + }, + { + name: 'model-2', + api_provider: 'openai', + capabilities: { + promptStrategy: 'xtabUnifiedModel', + }, + }, + { + name: 'model-3', + api_provider: 'openai', + capabilities: { + promptStrategy: 'xtab-v1', + }, + } + ], +}; + +export class InlineEditsModelService extends Disposable implements IInlineEditsModelService { + + _serviceBrand: undefined; + + private static readonly COPILOT_NES_XTAB_MODEL: Model = { + modelName: 'copilot-nes-xtab', + promptingStrategy: undefined, + includeTagsInCurrentFile: true, + }; + + private static readonly COPILOT_NES_OCT: Model = { + modelName: 'copilot-nes-oct', + promptingStrategy: PromptingStrategy.Xtab275, + includeTagsInCurrentFile: false, + }; + + private _copilotTokenObs = observableFromEvent(this, this._tokenStore.onDidStoreUpdate, () => this._tokenStore.copilotToken); + + private _preferredModelNameObs = this._configService.getExperimentBasedConfigObservable(ConfigKey.Advanced.InlineEditsPreferredModel, this._expService); + private _localModelConfigObs = this._configService.getConfigObservable(ConfigKey.TeamInternal.InlineEditsXtabProviderModelConfiguration); + private _expBasedModelConfigObs = this._configService.getExperimentBasedConfigObservable(ConfigKey.TeamInternal.InlineEditsXtabProviderModelConfigurationString, this._expService); + private _defaultModelConfigObs = this._configService.getExperimentBasedConfigObservable(ConfigKey.TeamInternal.InlineEditsXtabProviderDefaultModelConfigurationString, this._expService); + + private _modelInfo: { readonly modelList: readonly Model[]; readonly currentModelId: string }; + + private readonly _onModelListUpdated = this._register(new Emitter()); + public readonly onModelListUpdated = this._onModelListUpdated.event; + + private _tracer = createTracer(['NES', 'ModelsService'], (msg) => this._logService.trace(msg)); + + constructor( + @ICAPIClientService private readonly _capiClient: ICAPIClientService, + @IFetcherService private readonly _fetchService: IFetcherService, + @ICopilotTokenStore private readonly _tokenStore: ICopilotTokenStore, + @IConfigurationService private readonly _configService: IConfigurationService, + @IExperimentationService private readonly _expService: IExperimentationService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, + @ILogService private readonly _logService: ILogService, + ) { + super(); + + const tracer = this._tracer.sub('constructor'); + + const defaultModel = this.determineDefaultModel(this._copilotTokenObs.get(), this._defaultModelConfigObs.get()); + + this._modelInfo = { modelList: [defaultModel], currentModelId: defaultModel.modelName }; + + tracer.trace('initial modelInfo', this._modelInfo); + + this._register(autorun((reader) => { + this.refreshModelsInfo({ + copilotToken: this._copilotTokenObs.read(reader), + preferredModelName: this._preferredModelNameObs.read(reader), + localModelConfig: this._localModelConfigObs.read(reader), + modelConfigString: this._expBasedModelConfigObs.read(reader), + defaultModelConfigString: this._defaultModelConfigObs.read(reader), + }); + })); + + tracer.trace('updated model info', this._modelInfo); + } + + get modelInfo(): vscode.InlineCompletionModelInfo | undefined { + const tracer = this._tracer.sub('modelInfo.getter'); + + tracer.trace('model info', this._modelInfo); + + const models: vscode.InlineCompletionModel[] = this._modelInfo.modelList.map(m => ({ + id: m.modelName, + name: m.modelName, + })); + + return { + models, + currentModelId: this._modelInfo.currentModelId, + }; + } + + + async setCurrentModelId(modelId: string): Promise { + if (this._modelInfo.currentModelId === modelId) { + return; + } + if (!this._modelInfo.modelList.some(m => m.modelName === modelId)) { + this._logService.warn(`Trying to set unknown model id: ${modelId}`); + return; + } + this._modelInfo = { ...this._modelInfo, currentModelId: modelId }; + await this._configService.setConfig(ConfigKey.Advanced.InlineEditsPreferredModel, modelId); + this._onModelListUpdated.fire(); + } + + async refreshModelsInfo( + { + copilotToken, + preferredModelName, + localModelConfig, + modelConfigString, + defaultModelConfigString, + }: { + copilotToken: CopilotToken | undefined; + preferredModelName: string; + localModelConfig: ModelConfiguration | undefined; + modelConfigString: string | undefined; + defaultModelConfigString: string | undefined; + }, + ): Promise { + + const tracer = this._tracer.sub('refreshModelsInfo'); + + tracer.trace('Fetching latest models...'); + const useSlashModels = this._configService.getExperimentBasedConfig(ConfigKey.TeamInternal.InlineEditsUseSlashModels, this._expService); + const fetchModelsPromise = useSlashModels ? this._fetchLatestModels(copilotToken) : Promise.resolve(undefined); + + const models: Model[] = []; + + if (modelConfigString) { + tracer.trace('Parsing modelConfigurationString...'); + const parsedConfig = this.parseModelConfigStringSetting(ConfigKey.TeamInternal.InlineEditsXtabProviderModelConfigurationString); + if (parsedConfig && !models.some(m => m.modelName === parsedConfig.modelName)) { + tracer.trace(`Adding model from modelConfigurationString: ${parsedConfig.modelName}`); + models.push({ ...parsedConfig }); + } else { + tracer.trace('No valid model found in modelConfigurationString.'); + } + } + + if (copilotToken) { + tracer.trace('Awaiting fetched models...'); + const fetchedModels = await fetchModelsPromise; + if (fetchedModels === undefined) { + tracer.trace('No fetched models available.'); + } else { + tracer.trace(`Fetched ${fetchedModels.models.length} models.`); + const filteredFetchedModels = filterMap(fetchedModels.models, (m) => { + if (!isPromptingStrategy(m.capabilities.promptStrategy)) { + return undefined; + } + return { + modelName: m.name, + promptingStrategy: m.capabilities.promptStrategy, + includeTagsInCurrentFile: false, // FIXME@ulugbekna: determine this based on model capabilities and config + } satisfies Model; + }); + tracer.trace(`Adding ${filteredFetchedModels.length} fetched models after filtering.`); + pushMany(models, filteredFetchedModels); + } + } + + if (localModelConfig) { + if (models.some(m => m.modelName === localModelConfig.modelName)) { + tracer.trace('Local model configuration already exists in the model list, skipping.'); + } else { + tracer.trace(`Adding local model configuration: ${localModelConfig.modelName}`); + models.push({ ...localModelConfig }); + } + } + + const defaultModel = this.determineDefaultModel(copilotToken, defaultModelConfigString); + if (defaultModel) { + if (models.some(m => m.modelName === defaultModel.modelName)) { + tracer.trace('Default model configuration already exists in the model list, skipping.'); + } else { + tracer.trace(`Adding default model configuration: ${defaultModel.modelName}`); + models.push(defaultModel); + } + } + + const hasModelListChanged = !isDeepStrictEqual(this._modelInfo.modelList, models); + + if (!hasModelListChanged) { + tracer.trace('Model list unchanged, not updating.'); + } else { + this._modelInfo = { + modelList: models, + currentModelId: this._pickedModel({ preferredModelName, models }), + }; + tracer.trace('Model list updated, firing event.'); + this._onModelListUpdated.fire(); + } + } + + public selectedModelConfiguration(): ModelConfiguration { + const tracer = this._tracer.sub('selectedModelConfiguration'); + const model = this._modelInfo.modelList.find(m => m.modelName === this._modelInfo.currentModelId); + if (model) { + tracer.trace(`Selected model found: ${model.modelName}`); + return { + modelName: model.modelName, + promptingStrategy: model.promptingStrategy, + includeTagsInCurrentFile: model.includeTagsInCurrentFile, + }; + } + tracer.trace('No selected model found, using default model.'); + return this.determineDefaultModel(undefined, undefined); + } + + private determineDefaultModel(copilotToken: CopilotToken | undefined, defaultModelConfigString: string | undefined): Model { + // if a default model config string is specified, use that + if (defaultModelConfigString) { + const parsedConfig = this.parseModelConfigStringSetting(ConfigKey.TeamInternal.InlineEditsXtabProviderDefaultModelConfigurationString); + if (parsedConfig) { + return { ...parsedConfig }; + } + } + + // otherwise, use built-in defaults + if (copilotToken?.isFcv1()) { + return InlineEditsModelService.COPILOT_NES_XTAB_MODEL; + } else { + return InlineEditsModelService.COPILOT_NES_OCT; + } + } + + private _pickedModel({ + preferredModelName, + models + }: { + preferredModelName: string; + models: Model[]; + }): string { + const userHasPreferredModel = preferredModelName !== 'none'; + + // FIXME@ulugbekna: respect exp-set model name + + if (userHasPreferredModel && models.some(m => m.modelName === preferredModelName)) { + return preferredModelName; + } + + softAssert(models.length > 0, 'InlineEdits model list should have at least one model'); + + if (models.length > 0) { + return models[0].modelName; + } + + return this.determineDefaultModel(undefined, undefined).modelName; + } + + private async _fetchLatestModels(copilotToken: CopilotToken | undefined): Promise { + if (!copilotToken) { + return undefined; + } + + if (IN_TESTING) { + return STUB_MODELS_FOR_TESTING; + } + + const url = `${this._capiClient.proxyBaseURL}/models`; + + let r: Response; + try { + r = await this._fetchService.fetch(url, { + headers: { + 'Authorization': `Bearer ${copilotToken.token}`, + }, + method: 'GET', + timeout: 10_000, + }); + } catch (e) { + this._logService.error('Failed to fetch model list', e); + return; + } + + if (!r.ok) { + this._logService.error(`Failed to fetch model list: ${r.status} ${r.statusText}`); + return; + } + + try { + const jsonData: unknown = await r.json(); + if (!WireTypes.ModelList.is(jsonData)) { + throw new Error('Invalid model list response'); // TODO@ulugbekna: add telemetry + } + return jsonData; + } catch (e) { + this._logService.error(e, 'Failed to process /models response'); + return; + } + } + + private parseModelConfigStringSetting(configKey: ExperimentBasedConfig): ModelConfiguration | undefined { + const configString = this._configService.getExperimentBasedConfig(configKey, this._expService); + if (configString === undefined) { + return undefined; + } + + let parsedConfig: ModelConfiguration | undefined; + try { + parsedConfig = JSON.parse(configString); + // FIXME@ulugbekna: validate parsedConfig structure + } catch (e: unknown) { + /* __GDPR__ + "incorrectNesModelConfig" : { + "owner": "ulugbekna", + "comment": "Capture if model configuration string is invalid JSON.", + "configName": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Name of the configuration that failed to parse." }, + "errorMessage": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error message from JSON.parse." }, + "configValue": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The invalid JSON string." } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('incorrectNesModelConfig', { configName: configKey.id, errorMessage: errors.toString(errors.fromUnknown(e)), configValue: configString }); + } + + return parsedConfig; + } +} diff --git a/src/util/common/tracing.ts b/src/util/common/tracing.ts index 284083773b..5ab848f150 100644 --- a/src/util/common/tracing.ts +++ b/src/util/common/tracing.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ + type LogFn = (message: string) => void; export interface SubTracingOptions { @@ -82,22 +83,43 @@ export class Tracer implements ITracer { } private stringify(value: unknown): string { - if (!value) { - return JSON.stringify(value); - } - if (typeof value === 'string') { - return value; - } else if (typeof value === 'object') { - const toStringValue = value.toString(); + + function stringifyObj(obj: Object): string { + const toStringValue = obj.toString(); if (toStringValue && toStringValue !== '[object Object]') { return toStringValue; } - if (value instanceof Error) { - return value.stack || value.message; + if (obj instanceof Error) { + return obj.stack || obj.message; } - return JSON.stringify(value, null, '\t'); + return JSON.stringify(obj, null, 2); } - return value.toString(); + + if (!value) { + return JSON.stringify(value, null, 2); + } + if (typeof value === 'string') { + return value; + } + + if (typeof value === 'object') { + return stringifyObj(value); + } + + if (typeof value === 'function') { + return value.name ? `[Function: ${value.name}]` : '[Function]'; + } + + if (Array.isArray(value)) { + return `[${value.map(v => this.stringify(v)).join(', ')}]`; + } + + const valueToString = value.toString(); + if (valueToString && valueToString !== '[object Object]') { + return valueToString; + } + + return stringifyObj(value as Object); } } diff --git a/test/outcome/inlineedit-goldenscenario-xtab-external.json b/test/outcome/inlineedit-goldenscenario-xtab-external.json index 86adea570f..d5909685c2 100644 --- a/test/outcome/inlineedit-goldenscenario-xtab-external.json +++ b/test/outcome/inlineedit-goldenscenario-xtab-external.json @@ -2,61 +2,61 @@ { "name": "InlineEdit GoldenScenario ([xtab]) [external] [cpp] - [MustHave] 8-cppIndividual-1-point.cpp", "requests": [ - "18b0153202214f3eb6ea3ed73c0df7c81d3fccc9efbf763747fdcf8cc976521a" + "3e83718b244169a245116a734ad684e399d6bda07c4048a0710a11bdae682f30" ] }, { "name": "InlineEdit GoldenScenario ([xtab]) [external] [cpp] - [MustHave] 8-cppIndividual-2-collection-farewell", "requests": [ - "19a8e50993649f5fa89e3b0cd7c3e987ead4ccfec55a0870be86ce3910c0e387" + "b22e1e06d883a1abeb77ca57b76053eb5823cfaf55f9d2bc9a04e0f3f0df23e1" ] }, { "name": "InlineEdit GoldenScenario ([xtab]) [external] [cpp] - [MustHave] 9-cppProject-add-header-expect-implementation", "requests": [ - "fd833677e1d1f3868c0ac3cfb38669d394b74c2f81b11efbf225cd157a58e56c" + "70ce9cbde1ca9f94a5c4df5d469d2bd557df0f65cf519fad1fed7a842094ef4a" ] }, { "name": "InlineEdit GoldenScenario ([xtab]) [external] [cpp] - [MustHave] 9-cppProject-add-implementation-expect-header", "requests": [ - "044861dec72d6aeed8602537ee55316fc6ee65d809199c7705e81cbd00f6afaf" + "edb6fe334c633f2744dbad643a1737187804ac2f009064035c9c2d9f9922beaf" ] }, { "name": "InlineEdit GoldenScenario ([xtab]) [external] [java] - [MustHave] 6-vscode-remote-try-java-part-1", "requests": [ - "1e55e59ab0ea0bc59e20bbd92425fdcf917e0fff1442afeca4afc9c44656042b" + "4d325e9c41fac468cdd6d831fc5b236711ebea2a8ef65e622e2763011951dd0a" ] }, { "name": "InlineEdit GoldenScenario ([xtab]) [external] [java] - [MustHave] 6-vscode-remote-try-java-part-2", "requests": [ - "fc8f8cfc63e544cc0ff17a0dd9a50c7ea2c537ed0dc2ee45b78d7dbe8e4ee9ff" + "443c65f47005555a3515b66c7ee2ca40cf9d9af5b4b8e8901ce770aac8c60611" ] }, { "name": "InlineEdit GoldenScenario ([xtab]) [external] [python] - Notebook 10-update-name-in-same-cell-of-notebook", "requests": [ - "c6eadfad537deafedc3a0996374b219829a2332d6a5d03ec584c42cb7a464540" + "4e300be35be6cd4dcc6013e05e6f0b4b611d285155a08910281d65198b43402e" ] }, { "name": "InlineEdit GoldenScenario ([xtab]) [external] [python] - Notebook 11-update-name-in-next-cell-of-notebook", "requests": [ - "87025b6ed3d719e019ec1889aac8d9f23c0c80eb707935c374d42a58bbe72000" + "585d18aaf6a58b1cb606bb6acfb0a9806152357225f9c5eb8a2fa7e4e863aa2c" ] }, { "name": "InlineEdit GoldenScenario ([xtab]) [external] [typescript] - [MustHave] 1-point.ts", "requests": [ - "e616f37dad7d37f952fb75a984cd829198dab974b07d0667e435a652ef8b45b8" + "fb29a030475a31d669924b1005a3e83b2c2daaade2344d7c66bd945cc12c57e8" ] }, { "name": "InlineEdit GoldenScenario ([xtab]) [external] [typescript] - [NiceToHave] 2-helloworld-sample-remove-generic-parameter", "requests": [ - "b8f71b88715f36e3a5ae263ea3be4ac030bec689579e342ce1dc3deaad39d1b1" + "a096c2880bf860590cf636f149df783722967e4e558abeb7170b4f164becfac3" ] } ] \ No newline at end of file diff --git a/test/simulation/baseline.json b/test/simulation/baseline.json index d7753d9fad..c817392c3f 100644 --- a/test/simulation/baseline.json +++ b/test/simulation/baseline.json @@ -3734,7 +3734,7 @@ "contentFilterCount": 0, "passCount": 10, "failCount": 0, - "score": 1, + "score": 0.1, "attributes": { "CompScore1": 1, "CompScore2": 0.5, @@ -3820,7 +3820,7 @@ "contentFilterCount": 0, "passCount": 10, "failCount": 0, - "score": 0.7, + "score": 1, "attributes": { "CompScore1": 0 } diff --git a/test/simulation/cache/layers/6fbd9c7b-e371-4190-a689-532b4da5b28c.sqlite b/test/simulation/cache/layers/6fbd9c7b-e371-4190-a689-532b4da5b28c.sqlite new file mode 100644 index 0000000000..64f78d7911 --- /dev/null +++ b/test/simulation/cache/layers/6fbd9c7b-e371-4190-a689-532b4da5b28c.sqlite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e84e7656903a3a55837ae5a4305e61c2a82b410d97860cf50432a81388da11a4 +size 135168