-
Notifications
You must be signed in to change notification settings - Fork 221
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #278 from fwcd/semantic-tokens
Add support for semantic tokens/highlighting
- Loading branch information
Showing
8 changed files
with
302 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
200 changes: 200 additions & 0 deletions
200
server/src/main/kotlin/org/javacs/kt/semantictokens/SemanticTokens.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,200 @@ | ||
| package org.javacs.kt.semantictokens | ||
|
|
||
| import org.eclipse.lsp4j.SemanticTokenTypes | ||
| import org.eclipse.lsp4j.SemanticTokenModifiers | ||
| import org.eclipse.lsp4j.SemanticTokensLegend | ||
| import org.eclipse.lsp4j.Range | ||
| import org.javacs.kt.CompiledFile | ||
| import org.javacs.kt.position.range | ||
| import org.javacs.kt.position.offset | ||
| import org.javacs.kt.util.preOrderTraversal | ||
| import org.jetbrains.kotlin.descriptors.ClassDescriptor | ||
| import org.jetbrains.kotlin.descriptors.ClassKind | ||
| import org.jetbrains.kotlin.descriptors.ConstructorDescriptor | ||
| import org.jetbrains.kotlin.descriptors.FunctionDescriptor | ||
| import org.jetbrains.kotlin.descriptors.PropertyDescriptor | ||
| import org.jetbrains.kotlin.descriptors.VariableDescriptor | ||
| import org.jetbrains.kotlin.lexer.KtTokens | ||
| import org.jetbrains.kotlin.psi.KtClassOrObject | ||
| import org.jetbrains.kotlin.psi.KtFunction | ||
| import org.jetbrains.kotlin.psi.KtModifierListOwner | ||
| import org.jetbrains.kotlin.psi.KtNameReferenceExpression | ||
| import org.jetbrains.kotlin.psi.KtVariableDeclaration | ||
| import org.jetbrains.kotlin.psi.KtNamedDeclaration | ||
| import org.jetbrains.kotlin.psi.KtProperty | ||
| import org.jetbrains.kotlin.psi.KtParameter | ||
| import org.jetbrains.kotlin.psi.KtStringTemplateEntry | ||
| import org.jetbrains.kotlin.psi.KtStringTemplateExpression | ||
| import org.jetbrains.kotlin.psi.KtSimpleNameStringTemplateEntry | ||
| import org.jetbrains.kotlin.psi.KtBlockStringTemplateEntry | ||
| import org.jetbrains.kotlin.psi.KtEscapeStringTemplateEntry | ||
| import org.jetbrains.kotlin.resolve.BindingContext | ||
| import com.intellij.psi.PsiElement | ||
| import com.intellij.psi.PsiNameIdentifierOwner | ||
| import com.intellij.psi.PsiLiteralExpression | ||
| import com.intellij.psi.PsiType | ||
| import com.intellij.openapi.util.TextRange | ||
|
|
||
| enum class SemanticTokenType(val typeName: String) { | ||
| KEYWORD(SemanticTokenTypes.Keyword), | ||
| VARIABLE(SemanticTokenTypes.Variable), | ||
| FUNCTION(SemanticTokenTypes.Function), | ||
| PROPERTY(SemanticTokenTypes.Property), | ||
| PARAMETER(SemanticTokenTypes.Parameter), | ||
| ENUM_MEMBER(SemanticTokenTypes.EnumMember), | ||
| CLASS(SemanticTokenTypes.Class), | ||
| INTERFACE(SemanticTokenTypes.Interface), | ||
| ENUM(SemanticTokenTypes.Enum), | ||
| TYPE(SemanticTokenTypes.Type), | ||
| STRING(SemanticTokenTypes.String), | ||
| NUMBER(SemanticTokenTypes.Number), | ||
| // Since LSP does not provide a token type for string interpolation | ||
| // entries, we use Variable as a fallback here for now | ||
| INTERPOLATION_ENTRY(SemanticTokenTypes.Variable) | ||
| } | ||
|
|
||
| enum class SemanticTokenModifier(val modifierName: String) { | ||
| DECLARATION(SemanticTokenModifiers.Declaration), | ||
| DEFINITION(SemanticTokenModifiers.Definition), | ||
| ABSTRACT(SemanticTokenModifiers.Abstract), | ||
| READONLY(SemanticTokenModifiers.Readonly) | ||
| } | ||
|
|
||
| val semanticTokensLegend = SemanticTokensLegend( | ||
| SemanticTokenType.values().map { it.typeName }, | ||
| SemanticTokenModifier.values().map { it.modifierName } | ||
| ) | ||
|
|
||
| data class SemanticToken(val range: Range, val type: SemanticTokenType, val modifiers: Set<SemanticTokenModifier> = setOf()) | ||
|
|
||
| /** | ||
| * Computes LSP-encoded semantic tokens for the given range in the | ||
| * document. No range means the entire document. | ||
| */ | ||
| fun encodedSemanticTokens(file: CompiledFile, range: Range? = null): List<Int> = | ||
| encodeTokens(semanticTokens(file, range)) | ||
|
|
||
| /** | ||
| * Computes semantic tokens for the given range in the document. | ||
| * No range means the entire document. | ||
| */ | ||
| fun semanticTokens(file: CompiledFile, range: Range? = null): Sequence<SemanticToken> = | ||
| elementTokens(file.parse, file.compile, range) | ||
|
|
||
| fun encodeTokens(tokens: Sequence<SemanticToken>): List<Int> { | ||
| val encoded = mutableListOf<Int>() | ||
| var last: SemanticToken? = null | ||
|
|
||
| for (token in tokens) { | ||
| // Tokens must be on a single line | ||
| if (token.range.start.line == token.range.end.line) { | ||
| val length = token.range.end.character - token.range.start.character | ||
| val deltaLine = token.range.start.line - (last?.range?.start?.line ?: 0) | ||
| val deltaStart = token.range.start.character - (last?.takeIf { deltaLine == 0 }?.range?.start?.character ?: 0) | ||
|
|
||
| encoded.add(deltaLine) | ||
| encoded.add(deltaStart) | ||
| encoded.add(length) | ||
| encoded.add(encodeType(token.type)) | ||
| encoded.add(encodeModifiers(token.modifiers)) | ||
|
|
||
| last = token | ||
| } | ||
| } | ||
|
|
||
| return encoded | ||
| } | ||
|
|
||
| private fun encodeType(type: SemanticTokenType): Int = type.ordinal | ||
|
|
||
| private fun encodeModifiers(modifiers: Set<SemanticTokenModifier>): Int = modifiers | ||
| .map { 1 shl it.ordinal } | ||
| .fold(0, Int::or) | ||
|
|
||
| private fun elementTokens(element: PsiElement, bindingContext: BindingContext, range: Range? = null): Sequence<SemanticToken> { | ||
| val file = element.containingFile | ||
| val textRange = range?.let { TextRange(offset(file.text, it.start), offset(file.text, it.end)) } | ||
| return element | ||
| // TODO: Ideally we would like to cut-off subtrees outside our range, but this doesn't quite seem to work | ||
| // .preOrderTraversal { elem -> textRange?.let { it.contains(elem.textRange) } ?: true } | ||
| .preOrderTraversal() | ||
| .filter { elem -> textRange?.let { it.contains(elem.textRange) } ?: true } | ||
| .mapNotNull { elementToken(it, bindingContext) } | ||
| } | ||
|
|
||
| private fun elementToken(element: PsiElement, bindingContext: BindingContext): SemanticToken? { | ||
| val file = element.containingFile | ||
| val elementRange = range(file.text, element.textRange) | ||
|
|
||
| return when (element) { | ||
| // References (variables, types, functions, ...) | ||
|
|
||
| is KtNameReferenceExpression -> { | ||
| val target = bindingContext[BindingContext.REFERENCE_TARGET, element] | ||
| val tokenType = when (target) { | ||
| is PropertyDescriptor -> SemanticTokenType.PROPERTY | ||
| is VariableDescriptor -> SemanticTokenType.VARIABLE | ||
| is ConstructorDescriptor -> when (target.constructedClass.kind) { | ||
| ClassKind.ANNOTATION_CLASS -> SemanticTokenType.TYPE // annotations look nicer this way | ||
| else -> SemanticTokenType.FUNCTION | ||
| } | ||
| is FunctionDescriptor -> SemanticTokenType.FUNCTION | ||
| is ClassDescriptor -> when (target.kind) { | ||
| ClassKind.CLASS -> SemanticTokenType.CLASS | ||
| ClassKind.OBJECT -> SemanticTokenType.CLASS | ||
| ClassKind.INTERFACE -> SemanticTokenType.INTERFACE | ||
| ClassKind.ENUM_CLASS -> SemanticTokenType.ENUM | ||
| else -> SemanticTokenType.TYPE | ||
| } | ||
| else -> return null | ||
| } | ||
| val isConstant = (target as? VariableDescriptor)?.let { !it.isVar() || it.isConst() } ?: false | ||
| val modifiers = if (isConstant) setOf(SemanticTokenModifier.READONLY) else setOf() | ||
|
|
||
| SemanticToken(elementRange, tokenType, modifiers) | ||
| } | ||
|
|
||
| // Declarations (variables, types, functions, ...) | ||
|
|
||
| is PsiNameIdentifierOwner -> { | ||
| val tokenType = when (element) { | ||
| is KtParameter -> SemanticTokenType.PARAMETER | ||
| is KtProperty -> SemanticTokenType.PROPERTY | ||
| is KtVariableDeclaration -> SemanticTokenType.VARIABLE | ||
| is KtClassOrObject -> SemanticTokenType.CLASS | ||
| is KtFunction -> SemanticTokenType.FUNCTION | ||
| else -> return null | ||
| } | ||
| val identifierRange = element.nameIdentifier?.let { range(file.text, it.textRange) } ?: return null | ||
| val modifiers = mutableSetOf(SemanticTokenModifier.DECLARATION) | ||
|
|
||
| if (element is KtVariableDeclaration && (!element.isVar() || element.hasModifier(KtTokens.CONST_KEYWORD)) || element is KtParameter) { | ||
| modifiers.add(SemanticTokenModifier.READONLY) | ||
| } | ||
|
|
||
| if (element is KtModifierListOwner) { | ||
| if (element.hasModifier(KtTokens.ABSTRACT_KEYWORD)) { | ||
| modifiers.add(SemanticTokenModifier.ABSTRACT) | ||
| } | ||
| } | ||
|
|
||
| SemanticToken(identifierRange, tokenType, modifiers) | ||
| } | ||
|
|
||
| // Literals and string interpolations | ||
|
|
||
| is KtSimpleNameStringTemplateEntry, is KtBlockStringTemplateEntry -> | ||
| SemanticToken(elementRange, SemanticTokenType.INTERPOLATION_ENTRY) | ||
| is KtStringTemplateExpression -> SemanticToken(elementRange, SemanticTokenType.STRING) | ||
| is PsiLiteralExpression -> { | ||
| val tokenType = when (element.type) { | ||
| PsiType.INT, PsiType.LONG, PsiType.DOUBLE -> SemanticTokenType.NUMBER | ||
| PsiType.CHAR -> SemanticTokenType.STRING | ||
| PsiType.BOOLEAN, PsiType.NULL -> SemanticTokenType.KEYWORD | ||
| else -> return null | ||
| } | ||
| SemanticToken(elementRange, tokenType) | ||
| } | ||
| else -> null | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
50 changes: 50 additions & 0 deletions
50
server/src/test/kotlin/org/javacs/kt/SemanticTokensTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| package org.javacs.kt | ||
|
|
||
| import org.hamcrest.Matchers.* | ||
| import org.junit.Assert.assertThat | ||
| import org.junit.Test | ||
| import org.javacs.kt.semantictokens.encodeTokens | ||
| import org.javacs.kt.semantictokens.SemanticToken | ||
| import org.javacs.kt.semantictokens.SemanticTokenType | ||
| import org.javacs.kt.semantictokens.SemanticTokenModifier | ||
|
|
||
| class SemanticTokensTest : SingleFileTestFixture("semantictokens", "SemanticTokens.kt") { | ||
| @Test fun `tokenize file`() { | ||
| val varLine = 1 | ||
| val constLine = 2 | ||
| val classLine = 4 | ||
| val funLine = 6 | ||
|
|
||
| val expectedVar = sequenceOf( | ||
| SemanticToken(range(varLine, 5, varLine, 13), SemanticTokenType.PROPERTY, setOf(SemanticTokenModifier.DECLARATION)), // variable | ||
| ) | ||
| val expectedConst = sequenceOf( | ||
| SemanticToken(range(constLine, 5, constLine, 13), SemanticTokenType.PROPERTY, setOf(SemanticTokenModifier.DECLARATION, SemanticTokenModifier.READONLY)), // constant | ||
| SemanticToken(range(constLine, 15, constLine, 21), SemanticTokenType.CLASS), // String | ||
| SemanticToken(range(constLine, 24, constLine, 40), SemanticTokenType.STRING), // "test $variable" | ||
| SemanticToken(range(constLine, 30, constLine, 39), SemanticTokenType.INTERPOLATION_ENTRY), // $variable | ||
| SemanticToken(range(constLine, 31, constLine, 39), SemanticTokenType.PROPERTY), // variable | ||
| ) | ||
| val expectedClass = sequenceOf( | ||
| SemanticToken(range(classLine, 12, classLine, 16), SemanticTokenType.CLASS, setOf(SemanticTokenModifier.DECLARATION)), // Type | ||
| SemanticToken(range(classLine, 21, classLine, 29), SemanticTokenType.PARAMETER, setOf(SemanticTokenModifier.DECLARATION, SemanticTokenModifier.READONLY)), // property | ||
| SemanticToken(range(classLine, 31, classLine, 34), SemanticTokenType.CLASS), // Int | ||
| ) | ||
| val expectedFun = sequenceOf( | ||
| SemanticToken(range(funLine, 5, funLine, 6), SemanticTokenType.FUNCTION, setOf(SemanticTokenModifier.DECLARATION)), // f | ||
| SemanticToken(range(funLine, 7, funLine, 8), SemanticTokenType.PARAMETER, setOf(SemanticTokenModifier.DECLARATION, SemanticTokenModifier.READONLY)), // x | ||
| SemanticToken(range(funLine, 10, funLine, 13), SemanticTokenType.CLASS), // Int? | ||
| SemanticToken(range(funLine, 24, funLine, 27), SemanticTokenType.CLASS), // Int | ||
| SemanticToken(range(funLine, 30, funLine, 31), SemanticTokenType.FUNCTION), // f | ||
| SemanticToken(range(funLine, 32, funLine, 33), SemanticTokenType.VARIABLE, setOf(SemanticTokenModifier.READONLY)), // x | ||
| ) | ||
|
|
||
| val partialExpected = encodeTokens(expectedConst + expectedClass) | ||
| val partialResponse = languageServer.textDocumentService.semanticTokensRange(semanticTokensRangeParams(file, range(constLine, 0, classLine + 1, 0))).get()!! | ||
| assertThat(partialResponse.data, contains(*partialExpected.toTypedArray())) | ||
|
|
||
| val fullExpected = encodeTokens(expectedVar + expectedConst + expectedClass + expectedFun) | ||
| val fullResponse = languageServer.textDocumentService.semanticTokensFull(semanticTokensParams(file)).get()!! | ||
| assertThat(fullResponse.data, contains(*fullExpected.toTypedArray())) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| var variable = 3 | ||
| val constant: String = "test $variable" | ||
|
|
||
| data class Type(val property: Int) | ||
|
|
||
| fun f(x: Int? = null): Int = f(x) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters