Skip to content

fix: Improved semantic highlighting performance for huge files #828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 19, 2025
Merged

fix: Improved semantic highlighting performance for huge files #828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 52 additions & 10 deletions ...com/redhat/devtools/lsp4ij/features/semanticTokens/LSPSemanticTokensHighlightVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
/*******************************************************************************
* Copyright (c) 2024 Red Hat, Inc.
* Copyright (c) 2024-2025 Red Hat, Inc.
* Distributed under license by Red Hat, Inc. All rights reserved.
* This program is made available under the terms of the
* Eclipse Public License v2.0 which accompanies this distribution,
* and is available at http://www.eclipse.org/legal/epl-v20.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
* FalsePattern - Performance improvements for huge files
******************************************************************************/
package com.redhat.devtools.lsp4ij.features.semanticTokens;

Expand All @@ -15,14 +16,17 @@
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
import com.intellij.psi.impl.source.tree.LeafElement;
import com.redhat.devtools.lsp4ij.LSPFileSupport;
import com.redhat.devtools.lsp4ij.LSPIJUtils;
import com.redhat.devtools.lsp4ij.LanguageServersRegistry;
import com.redhat.devtools.lsp4ij.client.ExecuteLSPFeatureStatus;
import com.redhat.devtools.lsp4ij.client.indexing.ProjectIndexingManager;
import com.redhat.devtools.lsp4ij.internal.SimpleLanguageUtils;
import org.eclipse.lsp4j.SemanticTokensParams;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -45,6 +49,7 @@
@ApiStatus.Internal
public class LSPSemanticTokensHighlightVisitor implements HighlightVisitor {


private static final Logger LOGGER = LoggerFactory.getLogger(LSPSemanticTokensHighlightVisitor.class);
;

Expand All @@ -53,22 +58,49 @@ public boolean suitableForFile(@NotNull PsiFile file) {
return LanguageServersRegistry.getInstance().isFileSupported(file);
}

private HighlightInfoHolder holder;
private LazyHighlightInfo[] lazyInfos;

@Override
public boolean analyze(@NotNull PsiFile file, boolean updateWholeFile, @NotNull HighlightInfoHolder holder, @NotNull Runnable action) {
if (ProjectIndexingManager.canExecuteLSPFeature(file) != ExecuteLSPFeatureStatus.NOW) {
return true;
}
action.run();
// run unconditionally, because the LSP semanticTokens API sucks and is file-level only
highlightSemanticTokens(file, holder);
try {
if (SimpleLanguageUtils.isSupported(file.getLanguage())) {
highlightSemanticTokens(file, holder);
this.lazyInfos = null;
this.holder = null;
} else {
this.lazyInfos = highlightSemanticTokens(file, null);
this.holder = holder;
}
action.run();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any reason why action.run() is called at the end although before it was called at first?

} finally {
this.holder = null;
this.lazyInfos = null;
}
return true;
}

@Override
public void visit(@NotNull PsiElement element) {
if (lazyInfos == null || !(element instanceof LeafElement))
return;
int start = element.getTextOffset();
if (start < 0)
return;
int end = start + element.getTextLength();
for (int i = start; i < end && i < lazyInfos.length; i++) {
var info = lazyInfos[i];
if (info != null) {
holder.add(info.resolve(i));
lazyInfos[i] = null;
}
}
}

private void highlightSemanticTokens(@NotNull PsiFile file, @NotNull HighlightInfoHolder holder) {
private static LazyHighlightInfo[] highlightSemanticTokens(@NotNull PsiFile file, @Nullable HighlightInfoHolder holder) {
// Consume LSP 'textDocument/semanticTokens/full' request
LSPSemanticTokensSupport semanticTokensSupport = LSPFileSupport.getSupport(file).getSemanticTokensSupport();
var params = new SemanticTokensParams(LSPIJUtils.toTextDocumentIdentifier(file.getVirtualFile()));
Expand All @@ -79,14 +111,14 @@ private void highlightSemanticTokens(@NotNull PsiFile file, @NotNull HighlightIn
ProcessCanceledException e) {//Since 2024.2 ProcessCanceledException extends CancellationException so we can't use multicatch to keep backward compatibility
//TODO delete block when minimum required version is 2024.2
semanticTokensSupport.cancel();
return;
return null;
} catch (CancellationException e) {
// cancel the LSP requests textDocument/semanticTokens/full
semanticTokensSupport.cancel();
return;
return null;
} catch (ExecutionException e) {
LOGGER.error("Error while consuming LSP 'textDocument/semanticTokens/full' request", e);
return;
return null;
}

if (isDoneNormally(semanticTokensFuture)) {
Expand All @@ -95,11 +127,21 @@ private void highlightSemanticTokens(@NotNull PsiFile file, @NotNull HighlightIn
if (semanticTokens != null) {
var document = LSPIJUtils.getDocument(file.getVirtualFile());
if (document == null) {
return;
return null;
}
if (holder != null) {
semanticTokens.highlight(file, document, (start, end, colorKey) ->
holder.add(LazyHighlightInfo.resolve(start, end, colorKey)));
return null;
} else {
var infos = new LazyHighlightInfo[document.getTextLength()];
semanticTokens.highlight(file, document, (start, end, colorKey) ->
infos[start] = new LazyHighlightInfo(end, colorKey));
return infos;
}
semanticTokens.highlight(file, document, holder::add);
}
}
return null;
}

@Override
Expand Down
35 changes: 35 additions & 0 deletions src/main/java/com/redhat/devtools/lsp4ij/features/semanticTokens/LazyHighlightInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*******************************************************************************
* Copyright (c) 2025 Red Hat, Inc.
* Distributed under license by Red Hat, Inc. All rights reserved.
* This program is made available under the terms of the
* Eclipse Public License v2.0 which accompanies this distribution,
* and is available at http://www.eclipse.org/legal/epl-v20.html
*
* Contributors:
* FalsePattern - initial API and implementation
******************************************************************************/
package com.redhat.devtools.lsp4ij.features.semanticTokens;

import com.intellij.codeInsight.daemon.impl.HighlightInfo;
import com.intellij.openapi.editor.colors.TextAttributesKey;

import static com.intellij.codeHighlighting.RainbowHighlighter.RAINBOW_ELEMENT;

public record LazyHighlightInfo(int end, TextAttributesKey colorKey) {
@FunctionalInterface
public interface Consumer {
void accept(int start, int end, TextAttributesKey colorKey);
}

public HighlightInfo resolve(int start) {
return resolve(start, end, colorKey);
}

public static HighlightInfo resolve(int start, int end, TextAttributesKey colorKey) {
return HighlightInfo
.newHighlightInfo(RAINBOW_ELEMENT)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why RAINBOW_ELEMENT?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh sorry it was like this.

.range(start, end)
.textAttributes(colorKey)
.create();
}
}
22 changes: 10 additions & 12 deletions src/main/java/com/redhat/devtools/lsp4ij/features/semanticTokens/SemanticTokensData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
/*******************************************************************************
* Copyright (c) 2024 Red Hat, Inc.
* Copyright (c) 2024-2025 Red Hat, Inc.
* Distributed under license by Red Hat, Inc. All rights reserved.
* This program is made available under the terms of the
* Eclipse Public License v2.0 which accompanies this distribution,
* and is available at http://www.eclipse.org/legal/epl-v20.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
* FalsePattern - Performance improvements for huge files
******************************************************************************/
package com.redhat.devtools.lsp4ij.features.semanticTokens;

Expand All @@ -28,9 +29,6 @@
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;

import static com.intellij.codeHighlighting.RainbowHighlighter.RAINBOW_ELEMENT;

/**
* Semantic data.
Expand Down Expand Up @@ -63,7 +61,7 @@ public SemanticTokens getSemanticTokens() {
@Nullable
public void highlight(@NotNull PsiFile file,
@NotNull Document document,
@NotNull Consumer<HighlightInfo> addInfo) {
@NotNull LazyHighlightInfo.Consumer addInfo) {
var inspector = SemanticTokensInspectorManager.getInstance(file.getProject());
boolean notifyInspector = inspector.hasSemanticTokensInspectorListener();
List<SemanticTokensHighlightInfo> highlightInfos = notifyInspector ? new ArrayList<>() : null;
Expand All @@ -80,9 +78,14 @@ public void highlight(@NotNull PsiFile file,
int offset = 0;
int length = 0;
String tokenType = null;
int cancelCounter = 0;
for (Integer data : dataStream) {
// Cancel LSP semantic tokens support as soon as possible.
ProgressManager.checkCanceled();
cancelCounter++;
if (cancelCounter >= 100) {
cancelCounter = 0;
ProgressManager.checkCanceled();
}

switch (idx % 5) {
case 0: // line
Expand All @@ -109,12 +112,7 @@ public void highlight(@NotNull PsiFile file,
int end = offset + length;
TextAttributesKey colorKey = tokenType != null ? semanticTokensColorsProvider.getTextAttributesKey(tokenType, tokenModifiers, file) : null;
if (colorKey != null) {
HighlightInfo highlightInfo = HighlightInfo
.newHighlightInfo(RAINBOW_ELEMENT)
.range(start, end)
.textAttributes(colorKey)
.create();
addInfo.accept(highlightInfo);
addInfo.accept(start, end, colorKey);
}

if (notifyInspector) {
Expand Down
Loading