semantic_tokens: highlight python code in options

profiles/semantic_tokens/slapos_recipe_build.cfg

@@ -0,0 +1,43 @@
+[ok]
+recipe = slapos.recipe.build
+init =
+  import os
+  # comment
+  def f(param):
+    "docstring"
+    return g("string") + 1
+
+  multi_line_string = """
+  line 1
+  line 2
+  """
+
+  class Class:
+    @property
+    def p(self):
+      return 1
+
+install =
+  def f2(a:int) -> str:
+    f2(a + 1)
+  pass
+not-python =
+    nothing here
+
+
+[another]
+recipe = slapos.recipe.build
+init =
+  import another_init
+install =
+
+  import another_install
+
+[again-another]
+recipe = slapos.recipe.build
+init =
+
+
+  import again_another_init
+install =
+  import again_another_install

server/CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning][semver].
 
 ### Added
 
+ - semantic_tokens: highlight python code in options
  - completions: complete existing sections within `[` to override existing sections.
  - hover: improve hover of known recipes
  - Add support for plone.recipe.zope2instance

server/README.md

@@ -63,6 +63,10 @@ The automatic installation does not seem to work with theia and the python egg h
 - update a python package from `[versions]` to its latest version on pypi
 - compute the `md5sum` of an url
 
+## Semantic tokens
+
+- python code in options is highlighted.
+
 ## Template support
 
 - "current" buildout profile is guessed, then completions and diagnostics should work on any files.

server/buildoutls/recipes.py

@@ -2,6 +2,14 @@
 
 from typing import Dict, Optional, Sequence, Set
 
+import enum
+
+
+class RecipeOptionKind(enum.Enum):
+  Text = enum.auto()
+  ShellScript = enum.auto()
+  PythonScript = enum.auto()
+
 
 class RecipeOption:
   """A Recipe option."""
@@ -11,6 +19,7 @@ def __init__(
     documentation: str = "",
     valid_values: Sequence[str] = (),
     deprecated: Optional[str] = "",
+    kind: Optional[RecipeOptionKind] = RecipeOptionKind.Text,
   ):
     self.documentation = documentation
 
@@ -20,6 +29,9 @@ def __init__(
     self.deprecated = deprecated
     """Reason for the option to be deprected, if it is deprecated.
     """
+    self.kind = kind
+    """Type of the option.
+    """
 
 
 class Recipe:
@@ -205,9 +217,11 @@ def documentation(self) -> str:
   options={
     "command": RecipeOption(
       "Command to run when the buildout part is installed.",
+      kind=RecipeOptionKind.ShellScript,
     ),
     "update-command": RecipeOption(
       "Command to run when the buildout part is updated. This happens when buildout is run but the configuration for this buildout part has not changed.",
+      kind=RecipeOptionKind.ShellScript,
     ),
     "location": RecipeOption(
       """A list of filesystem paths that buildout should consider as being managed by this buildout part.
@@ -298,12 +312,15 @@ def documentation(self) -> str:
   options={
     "init": RecipeOption(
       "python code executed at initialization step",
+      kind=RecipeOptionKind.PythonScript,
     ),
     "install": RecipeOption(
       "python code executed at install step",
+      kind=RecipeOptionKind.PythonScript,
     ),
     "update": RecipeOption(
       "python code executed when updating",
+      kind=RecipeOptionKind.PythonScript,
     ),
   },
   generated_options={
@@ -359,7 +376,8 @@ def documentation(self) -> str:
       """Name of the configure command that will be run to generate the Makefile.
 This defaults to `./configure` which is fine for packages that come with a configure script.
 You may wish to change this when compiling packages with a different set up.
-See the *Compiling a Perl package* section for an example."""
+See the *Compiling a Perl package* section for an example.""",
+      kind=RecipeOptionKind.ShellScript,
     ),
     "configure-options": RecipeOption("""Extra options to be given to the configure script.
 By default only the `--prefix` option is passed which is set to the part directory.
@@ -415,18 +433,22 @@ def documentation(self) -> str:
     ),
     "pre-configure": RecipeOption(
       """Shell command that will be executed before running `configure` script.
-It takes the same effect as `pre-configure-hook` option except it's shell command."""
+It takes the same effect as `pre-configure-hook` option except it's shell command.""",
+      kind=RecipeOptionKind.ShellScript,
     ),
     "pre-build": RecipeOption(
       """Shell command that will be executed before running `make`.
-It takes the same effect as `pre-make-hook` option except it's shell command."""
+It takes the same effect as `pre-make-hook` option except it's shell command.""",
+      kind=RecipeOptionKind.ShellScript,
     ),
     "pre-install": RecipeOption(
-      """Shell command that will be executed before running `make` install."""
+      """Shell command that will be executed before running `make` install.""",
+      kind=RecipeOptionKind.ShellScript,
     ),
     "post-install": RecipeOption(
       """Shell command that will be executed after running `make` install.
-It takes the same effect as `post-make-hook` option except it's shell command."""
+It takes the same effect as `post-make-hook` option except it's shell command.""",
+      kind=RecipeOptionKind.ShellScript,
     ),
     "keep-compile-dir": RecipeOption(
       """Switch to optionally keep the temporary directory where the package was compiled.
@@ -589,10 +611,13 @@ def documentation(self) -> str:
         """The name of a script to generate that allows access to a Python interpreter that has the path set based on the eggs installed."""
       ),
       "extra-paths": RecipeOption("""Extra paths to include in a generated script."""),
-      "initialization": RecipeOption("""Specify some Python initialization code.
+      "initialization": RecipeOption(
+        """Specify some Python initialization code.
 This is very limited. 
 
-In particular, be aware that leading whitespace is stripped from the code given."""),
+In particular, be aware that leading whitespace is stripped from the code given.""",
+        kind=RecipeOptionKind.PythonScript,
+      ),
       "arguments": RecipeOption(
         """Specify some arguments to be passed to entry points as Python source."""
       ),
@@ -796,7 +821,8 @@ def documentation(self) -> str:
 ``sitecustomize.py`` script (Buildout >= 1.5) or within the instance script
 (Buildout < 1.5). This is very limited. In particular, be aware that leading
 whitespace is stripped from the code given. *added in version 4.2.14*
-"""
+""",
+      kind=RecipeOptionKind.PythonScript,
     ),
     "wsgi": RecipeOption(
       """

server/buildoutls/semantic_tokens.py

@@ -0,0 +1,127 @@
+from typing import List, Optional
+import logging
+import itertools
+
+from lsprotocol.types import SemanticTokens
+import pygments.lexers
+import pygments.token
+
+from .buildout import BuildoutProfile
+from .recipes import RecipeOptionKind
+from .server import LanguageServer
+from .types import SEMANTIC_TOKEN_TYPES
+
+
+logger = logging.getLogger(__name__)
+
+
+token_type_by_type = {t: SEMANTIC_TOKEN_TYPES.index(t) for t in SEMANTIC_TOKEN_TYPES}
+
+
+def get_token_type(token_pygment_type: pygments.token._TokenType) -> Optional[int]:
+  if token_pygment_type in pygments.token.Comment:
+    return token_type_by_type["comment"]
+  if token_pygment_type in pygments.token.String:
+    return token_type_by_type["string"]
+  if token_pygment_type in pygments.token.Number:
+    return token_type_by_type["number"]
+  if token_pygment_type in pygments.token.Name.Class:
+    return token_type_by_type["class"]
+  if token_pygment_type in pygments.token.Name.Function:
+    return token_type_by_type["function"]
+  if (
+    token_pygment_type in pygments.token.Name.Builtin
+    or token_pygment_type in pygments.token.Keyword.Constant
+  ):
+    return token_type_by_type["type"]
+  if token_pygment_type in pygments.token.Name:
+    return token_type_by_type["variable"]
+  if token_pygment_type in pygments.token.Keyword:
+    return token_type_by_type["keyword"]
+  return None
+
+
+def get_semantic_tokens(
+  ls: LanguageServer,
+  parsed: BuildoutProfile,
+) -> SemanticTokens:
+  data: List[int] = []
+
+  delta_line = delta_start = last_block_end = 0
+  for section_value in parsed.values():
+    if recipe := section_value.getRecipe():
+      for option_key, option_value in section_value.items():
+        if (
+          (option_definition := recipe.options.get(option_key))
+          and option_value.value
+          and option_definition.kind == RecipeOptionKind.PythonScript
+        ):
+          for option_value_location in option_value.locations:
+            if parsed.uri != option_value_location.uri:
+              continue
+            lexer = pygments.lexers.get_lexer_by_name("python")
+
+            doc = ls.workspace.get_text_document(option_value_location.uri)
+            source_code = "".join(
+              itertools.chain(
+                (
+                  doc.lines[option_value_location.range.start.line][
+                    option_value_location.range.start.character :
+                  ],
+                ),
+                doc.lines[
+                  option_value_location.range.start.line
+                  + 1 : option_value_location.range.end.line
+                ],
+                (doc.lines[option_value_location.range.end.line].rstrip(),),
+              )
+            )
+            this_block_start = option_value.location.range.start.line
+            delta_line += this_block_start - last_block_end
+            last_block_end = option_value.location.range.end.line
+
+            # skip empy lines at beginning
+            for line in source_code.splitlines():
+              if line.strip():
+                break
+              delta_line += 1
+
+            for token_pygment_type, token_text in lexer.get_tokens(source_code):
+              # A specific token i in the file consists of the following array indices:
+              #
+              # at index 5*i - deltaLine: token line number, relative to the previous token
+              # at index 5*i+1 - deltaStart: token start character, relative to the previous
+              #   token (relative to 0 or the previous token’s start if they are on the same
+              #   line)
+              # at index 5*i+2 - length: the length of the token.
+              # at index 5*i+3 - tokenType: will be looked up in
+              #   SemanticTokensLegend.tokenTypes. We currently ask that tokenType < 65536.
+              # at index 5*i+4 - tokenModifiers: each set bit will be looked up in
+              #   SemanticTokensLegend.tokenModifiers
+              token_type = get_token_type(token_pygment_type)
+              if token_type is not None:
+                # explode token spawning on multiple lines into multiple tokens
+                for token_text_line in token_text.splitlines(True):
+                  tok = [
+                    delta_line,
+                    delta_start,
+                    len(token_text_line),
+                    token_type,
+                    0,
+                  ]
+                  data.extend(tok)
+                  delta_line = 1 if "\n" in token_text_line else 0
+                  delta_start = len(token_text)
+              else:
+                if line_count := (token_text.replace("\r\n", "\n").count("\n")):
+                  delta_line += line_count
+                  delta_start = 0
+                else:
+                  delta_start += len(token_text)
+
+            if not source_code.endswith("\n"):
+              # pygments always output a final \n, but sometimes option does
+              # not include one, so we adjust for this case.
+              delta_line -= 1
+
+  return SemanticTokens(data=data)

server/buildoutls/server.py

@@ -12,6 +12,7 @@
   TEXT_DOCUMENT_DEFINITION,
   TEXT_DOCUMENT_DID_CHANGE,
   TEXT_DOCUMENT_DID_OPEN,
+  TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
   TEXT_DOCUMENT_DOCUMENT_LINK,
   TEXT_DOCUMENT_DOCUMENT_SYMBOL,
   TEXT_DOCUMENT_HOVER,
@@ -32,6 +33,9 @@
   DidOpenTextDocumentParams,
   DocumentLink,
   DocumentLinkParams,
+  SemanticTokens,
+  SemanticTokensLegend,
+  SemanticTokensParams,
   DocumentSymbol,
   DocumentSymbolParams,
   Hover,
@@ -59,6 +63,7 @@
   diagnostic,
   profiling,
   recipes,
+  semantic_tokens,
   types,
 )
 
@@ -760,3 +765,14 @@ async def lsp_document_link(
           target = urllib.parse.urljoin(base, extend)
         links.append(DocumentLink(range=extend_range, target=target))
   return links
+
+
+@server.feature(
+  TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
+  SemanticTokensLegend(token_types=types.SEMANTIC_TOKEN_TYPES, token_modifiers=[]),
+)
+async def lsp_semantic_tokens_full(
+  ls: LanguageServer, params: SemanticTokensParams
+) -> SemanticTokens:
+  parsed = await buildout.parse(ls, params.text_document.uri)
+  return semantic_tokens.get_semantic_tokens(ls, parsed)