Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tree-sitter rolling fixes, 1.125 (or 1.124.1) edition #1172

Merged
merged 7 commits into from
Jan 17, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[language-css] Update tree-sitter-css to latest…
…and improve highlighting of selectors while typing at end of file.
savetheclocktower committed Jan 12, 2025
commit 0bb26bd26f3e0026aaef8d722145b0ecc3309782
51 changes: 36 additions & 15 deletions packages/language-css/grammars/tree-sitter/queries/highlights.scm
Original file line number Diff line number Diff line change
@@ -1,23 +1,46 @@


; NOTE: `tree-sitter-css` recovers poorly from invalidity inside a block when
; you're adding a new property-value pair above others in a list. When the user
; is typing and the file is temporarily invalid, it will make incorrect guesses
; about tokens that occur between the cursor and the end of the block.
; WORKAROUNDS
; ===========

; Mark `ERROR` nodes that occur inside blocks. We are much more cautious about
; inferences inside blocks because we're often wrong.
(
(ERROR) @_IGNORE_
(#is? test.childOfType "block")
(#set! isErrorInsideBlock true)
)

; (stylesheet (ERROR)) can't be queried directly, but it's important that we be
; able to detect it.
(
(ERROR) @_IGNORE_
(#is? test.childOfType "stylesheet")
(#set! isErrorAtTopLevel true)
)

; This selector captures an empty file with (e.g.) the word `div` typed.
(ERROR
(identifier) @entity.name.tag.css
(#set! capture.final)
(#is? test.descendantOfNodeWithData isErrorAtTopLevel)
)

; When there's a parsing error inside a `block` node, too many things get
; incorrectly interpreted as `tag_name`s. Ignore all `tag_name` nodes until the
; error is resolved.
;
; The fix here is for `tree-sitter-css` to get better at recovering from its
; parsing error, but parser authors don't currently have much control over
; that. In the meantime, this query is a decent mitigation: it colors the
; affected tokens like plain text instead of assuming (nearly always
; incorrectly) them to be tag names.
; This should be fixed upstream because it has undesirable effects on nested
; selectors, but in the meantime this workaround is better than doing nothing.
;
; Ideally, this is temporary, and we can remove it soon. Until then, it makes
; syntax highlighting less obnoxious.
; Keep an eye on https://github.com/tree-sitter/tree-sitter-css/issues/65 to
; know when this workaround might no longer be necessary.

((tag_name) @_IGNORE_
(#is? test.descendantOfType "ERROR")
(#is? test.descendantOfNodeWithData isErrorInsideBlock)
(#set! capture.final))


(ERROR
(attribute_name) @_IGNORE_
(#set! capture.final))
@@ -26,8 +49,6 @@
(attribute_name) @invalid.illegal)
(#set! capture.final))

; WORKAROUND:
;
; In `::after`, the "after" has a node type of `tag_name`. Unclear whether this
; is a bug or intended behavior. We want to catch it here so that it doesn't
; get scoped like an HTML tag name in a selector.
@@ -38,7 +59,7 @@
(#set! adjust.startAt lastChild.previousSibling.startPosition)
(#set! adjust.endAt lastChild.endPosition))

; Claim this range and block it from being scoped as a tag name.
; Claim the `tag_name` range and block it from being scoped as a tag name.
(pseudo_element_selector
(tag_name) @_IGNORE_
(#is? test.last true)
Binary file modified packages/language-css/grammars/tree-sitter/tree-sitter-css.wasm
Binary file not shown.
14 changes: 14 additions & 0 deletions packages/language-css/spec/.eslintrc.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
module.exports = {
env: { jasmine: true },
globals: {
waitsForPromise: true,
runGrammarTests: true,
runFoldsTests: true
},
rules: {
"node/no-unpublished-require": "off",
"node/no-extraneous-require": "off",
"no-unused-vars": "off",
"no-empty": "off"
}
};
5 changes: 5 additions & 0 deletions packages/language-css/spec/fixtures/ends-in-tag-name.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@

/* Ensure that a file that ends in a bare tag name scopes the tag name properly. */

div
/* <- entity.name.tag.css */
10 changes: 10 additions & 0 deletions packages/language-css/spec/fixtures/sample.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
div {
/* <- entity.name.tag.css */
/* ^ punctuation.section.property-list.begin.bracket.curly.css */
}

/* Next test verifies that selectors are properly inferred at the end of a file before the `{` is typed. */

div.foo
/* <- entity.name.tag.css */
/* ^ entity.other.attribute-name.class.css */
21 changes: 21 additions & 0 deletions packages/language-css/spec/tree-sitter-grammar-spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
const path = require('path');

describe('WASM Tree-sitter CSS grammar', () => {

beforeEach(async () => {
await atom.packages.activatePackage('language-css');
});

it('passes grammar tests', async () => {
await runGrammarTests(
path.join(__dirname, 'fixtures', 'sample.css'),
/\/\*/,
/\*\//
);
await runGrammarTests(
path.join(__dirname, 'fixtures', 'ends-in-tag-name.css'),
/\/\*/,
/\*\//
);
});
});
45 changes: 27 additions & 18 deletions vendor/jasmine.js
Original file line number Diff line number Diff line change
@@ -46,7 +46,7 @@ jasmine.DEFAULT_UPDATE_INTERVAL = 250;
jasmine.MAX_PRETTY_PRINT_DEPTH = 40;

/**
* Default timeout interval in milliseconds for waitsFor() blocks.
* Default timeout interval in milliseconds for waitsfor () blocks.
*/
jasmine.DEFAULT_TIMEOUT_INTERVAL = 5000;

@@ -546,7 +546,7 @@ if (isCommonJS) exports.runs = runs;
/**
* Waits a fixed time period before moving to the next block.
*
* @deprecated Use waitsFor() instead
* @deprecated Use waitsfor () instead
* @param {Number} timeout milliseconds to wait
*/
var waits = function(timeout) {
@@ -2286,7 +2286,7 @@ jasmine.Spec.prototype.expect = function(actual) {
/**
* Waits a fixed time period before moving to the next block.
*
* @deprecated Use waitsFor() instead
* @deprecated Use waitsfor () instead
* @param {Number} timeout milliseconds to wait
*/
jasmine.Spec.prototype.waits = function(timeout) {
@@ -2716,17 +2716,25 @@ jasmine.Matchers.prototype.toSatisfy = function(fn) {
// position of the editor `expect` should be satisfied, and `testPosition`, that
// is where in file the test actually happened. This makes it easier for us
// to construct an error showing where EXACTLY was the assertion that failed
function normalizeTreeSitterTextData(editor, commentRegex) {
function normalizeTreeSitterTextData(editor, commentRegex, trailingCommentRegex) {
let allMatches = [], lastNonComment = 0
const checkAssert = new RegExp('^\\s*' + commentRegex.source + '\\s*[\\<\\-|\\^]')
console.log('editor:', editor.getText());
console.log('checkAssert', checkAssert);
editor.getBuffer().getLines().forEach((row, i) => {
const m = row.match(commentRegex)
if(m) {
console.log('does it match?', row, m);
if (m) {
if (trailingCommentRegex) {
row = row.replace(trailingCommentRegex, '')
}
row = row.trim()
// const scope = editor.scopeDescriptorForBufferPosition([i, m.index])
// FIXME: use editor.scopeDescriptorForBufferPosition when it works
const scope = editor.tokensForScreenRow(i)
const scopes = scope.flatMap(e => e.scopes)
if(scopes.find(s => s.match(/comment/)) && row.match(checkAssert)) {
console.log('scopes:', scopes);
if (scopes.find(s => s.match(/comment/)) && row.match(checkAssert)) {
allMatches.push({row: lastNonComment, text: row, col: m.index, testRow: i})
return
}
@@ -2735,7 +2743,7 @@ function normalizeTreeSitterTextData(editor, commentRegex) {
})
return allMatches.map(({text, row, col, testRow}) => {
const exactPos = text.match(/\^\s+(.*)/)
if(exactPos) {
if (exactPos) {
const expected = exactPos[1]
return {
expected,
@@ -2744,7 +2752,7 @@ function normalizeTreeSitterTextData(editor, commentRegex) {
}
} else {
const pos = text.match(/\<-\s+(.*)/)
if(!pos) throw new Error(`Can't match ${text}`)
if (!pos) throw new Error(`Can't match ${text}`)
return {
expected: pos[1],
editorPosition: {row, column: col},
@@ -2761,10 +2769,11 @@ async function openDocument(fullPath) {
return editor;
}

async function runGrammarTests(fullPath, commentRegex) {
async function runGrammarTests(fullPath, commentRegex, trailingCommentRegex = null) {
const editor = await openDocument(fullPath);

const normalized = normalizeTreeSitterTextData(editor, commentRegex)
const normalized = normalizeTreeSitterTextData(editor, commentRegex, trailingCommentRegex)
console.log('normalized', normalized);
expect(normalized.length).toSatisfy((n, reason) => {
reason("Tokenizer didn't run correctly - could not find any comment")
return n > 0
@@ -2773,7 +2782,7 @@ async function runGrammarTests(fullPath, commentRegex) {
expect(editor.scopeDescriptorForBufferPosition(editorPosition).scopes).toSatisfy((scopes, reason) => {
const dontFindScope = expected.startsWith("!");
expected = expected.replace(/^!/, "")
if(dontFindScope) {
if (dontFindScope) {
reason(`Expected to NOT find scope "${expected}" but found it\n` +
` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}`
);
@@ -2785,7 +2794,7 @@ async function runGrammarTests(fullPath, commentRegex) {
const normalized = expected.replace(/([\.\-])/g, '\\$1');
const scopeRegex = new RegExp('^' + normalized + '(\\..+)?$');
let result = scopes.find(e => e.match(scopeRegex)) !== undefined;
if(dontFindScope) result = !result;
if (dontFindScope) result = !result;
return result
})
})
@@ -2797,25 +2806,25 @@ async function runFoldsTests(fullPath, commentRegex) {
let grouped = {}
const normalized = normalizeTreeSitterTextData(editor, commentRegex).forEach(test => {
const [kind, id] = test.expected.split('.')
if(!kind || !id) {
if (!kind || !id) {
throw new Error(`Folds must be in the format fold_end.some-id\n` +
` at ${test.testPosition.row+1}:${test.testPosition.column+1}`)
}
grouped[id] ||= {}
grouped[id][kind] = test
})
for(const k in grouped) {
for (const k in grouped) {
const v = grouped[k]
const keys = Object.keys(v)
if(keys.indexOf('fold_begin') === -1)
if (keys.indexOf('fold_begin') === -1)
throw new Error(`Fold ${k} must contain fold_begin`)
if(keys.indexOf('fold_end') === -1)
if (keys.indexOf('fold_end') === -1)
throw new Error(`Fold ${k} must contain fold_end`)
if(keys.indexOf('fold_new_position') === -1)
if (keys.indexOf('fold_new_position') === -1)
throw new Error(`Fold ${k} must contain fold_new_position`)
}

for(const k in grouped) {
for (const k in grouped) {
const fold = grouped[k]
const begin = fold['fold_begin']
const end = fold['fold_end']