jenterkin/tree-sitter-jhcl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

misc: rework shell.nix, reformat, ditch .editorconfig

Browse Source
This commit is contained in:
Michael Hoffmann
2023-04-07 12:17:28 +02:00
parent 0ff887f2a6
commit b8fcca985c
5 changed files with 92 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.editorconfig
View File

@@ -1,10 +0,0 @@
root = true
[*.{cc,txt,js}]
indent_style = space
indent_size = 2
tab_width = 8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
charset = utf-8

3
CHANGELOG.md
View File

@@ -5,6 +5,9 @@
feature feature
* add dialects so we can have different queries in `nvim-treesitter` * add dialects so we can have different queries in `nvim-treesitter`
housekeeping:
* reformat using LSPs, ditch editorconfig
## 1.0.0 - 2022-12-02 ## 1.0.0 - 2022-12-02
breaking: breaking:

2
make_grammar.js
View File

@@ -287,7 +287,7 @@ module.exports = function make_grammar(dialect) {
$.template_interpolation, $.template_interpolation,
$.template_directive, $.template_directive,
$.template_literal, $.template_literal,
)), )),
template_literal: $ => prec.right(repeat1( template_literal: $ => prec.right(repeat1(
$._template_literal_chunk, $._template_literal_chunk,

25
shell.nix
View File

@@ -1,19 +1,12 @@
let let
nixpkgs = fetchTarball { pkgs = import <nixpkgs> { };
name = "nixpkgs";
url = "https://github.com/NixOS/nixpkgs/archive/refs/tags/21.05.tar.gz";
sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
};
pkgs = import nixpkgs {};
in in
pkgs.mkShell { pkgs.mkShell {
name = "env"; name = "env";
buildInputs = with pkgs; [ buildInputs = with pkgs; [
nodejs nodejs
]; tree-sitter
shellHook = '' emscripten
PATH=./node_modules/.bin:$PATH ];
command -v tree-sitter >/dev/null 2>&1 || npm install tree-sitter-cli@0.20.6 }
'';
}

162
src/scanner.cc
View File

@@ -1,15 +1,15 @@
#include <tree_sitter/parser.h> #include <tree_sitter/parser.h>
#include <climits>
#include <vector>
#include <string>
#include <wctype.h>
#include <assert.h> #include <assert.h>
#include <climits>
#include <string>
#include <vector>
#include <wctype.h>
namespace { namespace {
using std::vector;
using std::string; using std::string;
using std::vector;
enum TokenType { enum TokenType {
QUOTED_TEMPLATE_START, QUOTED_TEMPLATE_START,
@@ -39,8 +39,7 @@ struct Context {
struct Scanner { struct Scanner {
public: public:
unsigned serialize(char *buf) {
unsigned serialize(char* buf) {
unsigned size = 0; unsigned size = 0;
if (context_stack.size() > CHAR_MAX) { if (context_stack.size() > CHAR_MAX) {
@@ -48,8 +47,10 @@ public:
} }
buf[size++] = context_stack.size(); buf[size++] = context_stack.size();
for (vector<Context>::iterator it = context_stack.begin(); it != context_stack.end(); ++it) { for (vector<Context>::iterator it = context_stack.begin();
if (size + 2 + it->heredoc_identifier.size() >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) { it != context_stack.end(); ++it) {
if (size + 2 + it->heredoc_identifier.size() >=
TREE_SITTER_SERIALIZATION_BUFFER_SIZE) {
return 0; return 0;
} }
if (it->heredoc_identifier.size() > CHAR_MAX) { if (it->heredoc_identifier.size() > CHAR_MAX) {
@@ -63,7 +64,7 @@ public:
return size; return size;
} }
void deserialize(const char* buf, unsigned n) { void deserialize(const char *buf, unsigned n) {
context_stack.clear(); context_stack.clear();
if (n == 0) { if (n == 0) {
@@ -76,14 +77,15 @@ public:
Context ctx; Context ctx;
ctx.type = static_cast<ContextType>(buf[size++]); ctx.type = static_cast<ContextType>(buf[size++]);
uint8_t heredoc_identifier_size = buf[size++]; uint8_t heredoc_identifier_size = buf[size++];
ctx.heredoc_identifier.assign(buf + size, buf + size + heredoc_identifier_size); ctx.heredoc_identifier.assign(buf + size,
buf + size + heredoc_identifier_size);
size += heredoc_identifier_size; size += heredoc_identifier_size;
context_stack.push_back(ctx); context_stack.push_back(ctx);
} }
assert(size == n); assert(size == n);
} }
bool scan(TSLexer* lexer, const bool* valid_symbols) { bool scan(TSLexer *lexer, const bool *valid_symbols) {
bool has_leading_whitespace_with_newline = false; bool has_leading_whitespace_with_newline = false;
while (iswspace(lexer->lookahead)) { while (iswspace(lexer->lookahead)) {
if (lexer->lookahead == '\n') { if (lexer->lookahead == '\n') {
@@ -95,26 +97,25 @@ public:
return false; return false;
} }
// manage quoted context // manage quoted context
if (valid_symbols[QUOTED_TEMPLATE_START] && !in_quoted_context() && lexer->lookahead == '"') { if (valid_symbols[QUOTED_TEMPLATE_START] && !in_quoted_context() &&
Context ctx = { QUOTED_TEMPLATE, "" }; lexer->lookahead == '"') {
Context ctx = {QUOTED_TEMPLATE, ""};
context_stack.push_back(ctx); context_stack.push_back(ctx);
return accept_and_advance(lexer, QUOTED_TEMPLATE_START); return accept_and_advance(lexer, QUOTED_TEMPLATE_START);
} }
if (valid_symbols[QUOTED_TEMPLATE_END] && in_quoted_context() && lexer->lookahead == '"') { if (valid_symbols[QUOTED_TEMPLATE_END] && in_quoted_context() &&
lexer->lookahead == '"') {
context_stack.pop_back(); context_stack.pop_back();
return accept_and_advance(lexer, QUOTED_TEMPLATE_END); return accept_and_advance(lexer, QUOTED_TEMPLATE_END);
} }
// manage template interpolations // manage template interpolations
if ( if (valid_symbols[TEMPLATE_INTERPOLATION_START] &&
valid_symbols[TEMPLATE_INTERPOLATION_START] && valid_symbols[TEMPLATE_LITERAL_CHUNK] && !in_interpolation_context() &&
valid_symbols[TEMPLATE_LITERAL_CHUNK] && lexer->lookahead == '$') {
!in_interpolation_context() &&
lexer->lookahead == '$'
) {
advance(lexer); advance(lexer);
if (lexer->lookahead == '{') { if (lexer->lookahead == '{') {
Context ctx = { TEMPLATE_INTERPOLATION, "" }; Context ctx = {TEMPLATE_INTERPOLATION, ""};
context_stack.push_back(ctx); context_stack.push_back(ctx);
return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_START); return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_START);
} }
@@ -128,21 +129,19 @@ public:
} }
return accept_inplace(lexer, TEMPLATE_LITERAL_CHUNK); return accept_inplace(lexer, TEMPLATE_LITERAL_CHUNK);
} }
if (valid_symbols[TEMPLATE_INTERPOLATION_END] && in_interpolation_context() && lexer->lookahead == '}') { if (valid_symbols[TEMPLATE_INTERPOLATION_END] &&
in_interpolation_context() && lexer->lookahead == '}') {
context_stack.pop_back(); context_stack.pop_back();
return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_END); return accept_and_advance(lexer, TEMPLATE_INTERPOLATION_END);
} }
// manage template directives // manage template directives
if ( if (valid_symbols[TEMPLATE_DIRECTIVE_START] &&
valid_symbols[TEMPLATE_DIRECTIVE_START] && valid_symbols[TEMPLATE_LITERAL_CHUNK] && !in_directive_context() &&
valid_symbols[TEMPLATE_LITERAL_CHUNK] && lexer->lookahead == '%') {
!in_directive_context() &&
lexer->lookahead == '%'
) {
advance(lexer); advance(lexer);
if (lexer->lookahead == '{') { if (lexer->lookahead == '{') {
Context ctx = { TEMPLATE_DIRECTIVE, "" }; Context ctx = {TEMPLATE_DIRECTIVE, ""};
context_stack.push_back(ctx); context_stack.push_back(ctx);
return accept_and_advance(lexer, TEMPLATE_DIRECTIVE_START); return accept_and_advance(lexer, TEMPLATE_DIRECTIVE_START);
} }
@@ -156,7 +155,8 @@ public:
} }
return accept_inplace(lexer, TEMPLATE_LITERAL_CHUNK); return accept_inplace(lexer, TEMPLATE_LITERAL_CHUNK);
} }
if (valid_symbols[TEMPLATE_DIRECTIVE_END] && in_directive_context() && lexer->lookahead == '}') { if (valid_symbols[TEMPLATE_DIRECTIVE_END] && in_directive_context() &&
lexer->lookahead == '}') {
context_stack.pop_back(); context_stack.pop_back();
return accept_and_advance(lexer, TEMPLATE_DIRECTIVE_END); return accept_and_advance(lexer, TEMPLATE_DIRECTIVE_END);
} }
@@ -165,18 +165,21 @@ public:
if (valid_symbols[HEREDOC_IDENTIFIER] && !in_heredoc_context()) { if (valid_symbols[HEREDOC_IDENTIFIER] && !in_heredoc_context()) {
string identifier; string identifier;
// TODO: check that this is a valid identifier // TODO: check that this is a valid identifier
while (iswalnum(lexer->lookahead) || lexer->lookahead == '_' || lexer->lookahead == '-') { while (iswalnum(lexer->lookahead) || lexer->lookahead == '_' ||
lexer->lookahead == '-') {
identifier.push_back(lexer->lookahead); identifier.push_back(lexer->lookahead);
advance(lexer); advance(lexer);
} }
Context ctx = { HEREDOC_TEMPLATE, identifier }; Context ctx = {HEREDOC_TEMPLATE, identifier};
context_stack.push_back(ctx); context_stack.push_back(ctx);
return accept_inplace(lexer, HEREDOC_IDENTIFIER); return accept_inplace(lexer, HEREDOC_IDENTIFIER);
} }
if (valid_symbols[HEREDOC_IDENTIFIER] && in_heredoc_context() && has_leading_whitespace_with_newline) { if (valid_symbols[HEREDOC_IDENTIFIER] && in_heredoc_context() &&
has_leading_whitespace_with_newline) {
string expected_identifier = context_stack.back().heredoc_identifier; string expected_identifier = context_stack.back().heredoc_identifier;
for (string::iterator it = expected_identifier.begin(); it != expected_identifier.end(); ++it) { for (string::iterator it = expected_identifier.begin();
it != expected_identifier.end(); ++it) {
if (lexer->lookahead == *it) { if (lexer->lookahead == *it) {
advance(lexer); advance(lexer);
} else { } else {
@@ -186,7 +189,7 @@ public:
// check if the identifier is on a line of its own // check if the identifier is on a line of its own
lexer->mark_end(lexer); lexer->mark_end(lexer);
while (iswspace(lexer->lookahead) && lexer->lookahead != '\n') { while (iswspace(lexer->lookahead) && lexer->lookahead != '\n') {
advance(lexer); advance(lexer);
} }
if (lexer->lookahead == '\n') { if (lexer->lookahead == '\n') {
context_stack.pop_back(); context_stack.pop_back();
@@ -202,31 +205,32 @@ public:
// handle template literal chunks in quoted contexts // handle template literal chunks in quoted contexts
// //
// they may not contain newlines and may contain escape sequences // they may not contain newlines and may contain escape sequences
if (valid_symbols[TEMPLATE_LITERAL_CHUNK] && in_quoted_context()) { if (valid_symbols[TEMPLATE_LITERAL_CHUNK] && in_quoted_context()) {
switch (lexer->lookahead) { switch (lexer->lookahead) {
case '\\':
advance(lexer);
switch (lexer->lookahead) {
case '"':
case 'n':
case 'r':
case 't':
case '\\': case '\\':
advance(lexer); return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
switch (lexer->lookahead) { case 'u':
case '"': for (int i = 0; i < 4; i++) {
case 'n': if (!consume_wxdigit(lexer))
case 'r':
case 't':
case '\\':
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
case 'u':
for (int i = 0; i < 4; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
case 'U':
for (int i = 0; i < 8; i++) {
if (!consume_wxdigit(lexer)) return false;
}
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
default:
return false; return false;
} }
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
case 'U':
for (int i = 0; i < 8; i++) {
if (!consume_wxdigit(lexer))
return false;
}
return accept_and_advance(lexer, TEMPLATE_LITERAL_CHUNK);
default:
return false;
}
} }
} }
@@ -242,23 +246,21 @@ public:
private: private:
vector<Context> context_stack; vector<Context> context_stack;
void advance(TSLexer* lexer) { void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
lexer->advance(lexer, false);
}
void skip(TSLexer* lexer) { lexer->advance(lexer, true); } void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
bool accept_inplace(TSLexer* lexer, TokenType token) { bool accept_inplace(TSLexer *lexer, TokenType token) {
lexer->result_symbol = token; lexer->result_symbol = token;
return true; return true;
} }
bool accept_and_advance(TSLexer* lexer, TokenType token) { bool accept_and_advance(TSLexer *lexer, TokenType token) {
advance(lexer); advance(lexer);
return accept_inplace(lexer, token); return accept_inplace(lexer, token);
} }
bool consume_wxdigit(TSLexer* lexer) { bool consume_wxdigit(TSLexer *lexer) {
advance(lexer); advance(lexer);
return iswxdigit(lexer->lookahead); return iswxdigit(lexer->lookahead);
} }
@@ -270,13 +272,9 @@ private:
return context_stack.back().type == type; return context_stack.back().type == type;
} }
bool in_quoted_context() { bool in_quoted_context() { return in_context_type(QUOTED_TEMPLATE); }
return in_context_type(QUOTED_TEMPLATE);
}
bool in_heredoc_context() { bool in_heredoc_context() { return in_context_type(HEREDOC_TEMPLATE); }
return in_context_type(HEREDOC_TEMPLATE);
}
bool in_template_context() { bool in_template_context() {
return in_quoted_context() || in_heredoc_context(); return in_quoted_context() || in_heredoc_context();
@@ -286,9 +284,7 @@ private:
return in_context_type(TEMPLATE_INTERPOLATION); return in_context_type(TEMPLATE_INTERPOLATION);
} }
bool in_directive_context() { bool in_directive_context() { return in_context_type(TEMPLATE_DIRECTIVE); }
return in_context_type(TEMPLATE_DIRECTIVE);
}
}; };
} // namespace } // namespace
@@ -296,27 +292,27 @@ private:
extern "C" { extern "C" {
// tree sitter callbacks // tree sitter callbacks
void* tree_sitter_hcl_external_scanner_create() { void *tree_sitter_hcl_external_scanner_create() { return new Scanner(); }
return new Scanner();
}
void tree_sitter_hcl_external_scanner_destroy(void* p) { void tree_sitter_hcl_external_scanner_destroy(void *p) {
Scanner* scanner = static_cast<Scanner*>(p); Scanner *scanner = static_cast<Scanner *>(p);
delete scanner; delete scanner;
} }
unsigned tree_sitter_hcl_external_scanner_serialize(void* p, char* b) { unsigned tree_sitter_hcl_external_scanner_serialize(void *p, char *b) {
Scanner* scanner = static_cast<Scanner*>(p); Scanner *scanner = static_cast<Scanner *>(p);
return scanner->serialize(b); return scanner->serialize(b);
} }
void tree_sitter_hcl_external_scanner_deserialize(void* p, const char* b, unsigned n) { void tree_sitter_hcl_external_scanner_deserialize(void *p, const char *b,
Scanner* scanner = static_cast<Scanner*>(p); unsigned n) {
Scanner *scanner = static_cast<Scanner *>(p);
return scanner->deserialize(b, n); return scanner->deserialize(b, n);
} }
bool tree_sitter_hcl_external_scanner_scan(void* p, TSLexer* lexer, const bool* valid_symbols) { bool tree_sitter_hcl_external_scanner_scan(void *p, TSLexer *lexer,
Scanner* scanner = static_cast<Scanner*>(p); const bool *valid_symbols) {
Scanner *scanner = static_cast<Scanner *>(p);
return scanner->scan(lexer, valid_symbols); return scanner->scan(lexer, valid_symbols);
} }